fix(scheduler): release the macOS slot on the provisioning-deadline path

api/v1/ephemerd.proto

@@ -76,6 +76,42 @@ message StatusResponse {
   // The upgrade flow polls this after a restart to confirm the new binary
   // came up on the target version.
   string version = 6;
+  // held_slots is how many concurrency slots are currently HELD across every
+  // dispatch pool, which is not the same thing as active_jobs.
+  //
+  // active_jobs counts jobs the scheduler is tracking. A slot is taken before
+  // the job is tracked and given back after it is untracked, so a job that is
+  // still provisioning — or one whose slot leaked — is charged here and not
+  // there. Issue #196: a mac held its only slot for 28 hours after a wedged
+  // provision and reported status ok with active_jobs 0 the whole time.
+  // held_slots == slot_capacity with active_jobs 0 is that failure, visible.
+  int32 held_slots = 7;
+  // slot_capacity is the total number of slots across all pools.
+  int32 slot_capacity = 8;
+  // slot_pools breaks the totals down per dispatch pool (local, linux,
+  // macos), so an operator can tell which pool is saturated.
+  repeated SlotPool slot_pools = 9;
+  // abandoned_macos_vms counts macOS VMs whose teardown this daemon gave up
+  // on and has not seen finish.
+  //
+  // This is the COST of the #196 fix, made visible. When a Virtualization.
+  // framework stop wedges, the daemon walks away from it so the concurrency
+  // slot comes back — but the guest is still running and still holding its
+  // full configured RAM, and only a restart reclaims it. Non-zero means the
+  // node is losing memory it cannot get back; at abandoned_macos_vm_cap the
+  // macOS pool refuses further provisioning rather than growing without
+  // bound (Vz permits only a couple of concurrent macOS guests anyway).
+  int32 abandoned_macos_vms = 10;
+  // abandoned_macos_vm_cap is the value of abandoned_macos_vms at which the
+  // macOS pool stops taking work.
+  int32 abandoned_macos_vm_cap = 11;
+}
+
+// SlotPool is one dispatch pool's concurrency-slot occupancy.
+message SlotPool {
+  string pool = 1;
+  int32 held = 2;
+  int32 capacity = 3;
 }
 
 message ListJobsRequest {}

cmd/ephemerd/commands.go

@@ -69,13 +69,34 @@ func statusCmd() *cli.Command {
 				return fmt.Errorf("status: %w", err)
 			}
 
+			// held_slots is deliberately next to active_jobs: they differ
+			// exactly when a job is provisioning or a slot leaked, and a
+			// node whose slots are all held while active_jobs reads 0 is
+			// the shape of the 28-hour macOS stall in issue #196.
+			slots := make([]map[string]any, 0, len(resp.SlotPools))
+			for _, p := range resp.SlotPools {
+				slots = append(slots, map[string]any{
+					"pool":     p.Pool,
+					"held":     p.Held,
+					"capacity": p.Capacity,
+				})
+			}
+
 			data := map[string]any{
 				"status":         resp.Status,
 				"active_jobs":    resp.ActiveJobs,
 				"max_concurrent": resp.MaxConcurrent,
-				"draining":       resp.Draining,
-				"uptime":         resp.Uptime,
-				"version":        resp.Version,
+				"held_slots":     resp.HeldSlots,
+				"slot_capacity":  resp.SlotCapacity,
+				"slots":          slots,
+				// Non-zero means macOS guests are stranded on this node,
+				// holding RAM nothing but a restart reclaims; at the cap the
+				// macOS pool has stopped taking work. See issue #196.
+				"abandoned_macos_vms":    resp.AbandonedMacosVms,
+				"abandoned_macos_vm_cap": resp.AbandonedMacosVmCap,
+				"draining":               resp.Draining,
+				"uptime":                 resp.Uptime,
+				"version":                resp.Version,
 			}
 
 			pretty, _ := json.MarshalIndent(data, "", "  ")

docs/cli/status.md

@@ -16,8 +16,13 @@ Returns a JSON object with the following fields:
 | Field | Description |
 |-------|-------------|
 | `status` | Current daemon status |
-| `active_jobs` | Number of jobs currently running |
+| `active_jobs` | Number of jobs currently **tracked** as running |
 | `max_concurrent` | Maximum concurrent jobs allowed |
+| `held_slots` | Concurrency slots currently held across all dispatch pools |
+| `slot_capacity` | Total slots across all dispatch pools |
+| `slots` | Per-pool breakdown (`local`, `linux`, `macos`) of `held` and `capacity` |
+| `abandoned_macos_vms` | macOS VMs whose teardown the daemon gave up on and has not seen die |
+| `abandoned_macos_vm_cap` | Value of `abandoned_macos_vms` at which the macOS pool stops taking work |
 | `draining` | Whether the daemon is draining (shutting down gracefully) |
 | `uptime` | How long the daemon has been running |
 
@@ -28,11 +33,53 @@ Returns a JSON object with the following fields:
   "status": "running",
   "active_jobs": 2,
   "max_concurrent": 4,
+  "held_slots": 2,
+  "slot_capacity": 9,
+  "slots": [
+    { "pool": "local", "held": 2, "capacity": 4 },
+    { "pool": "linux", "held": 0, "capacity": 4 },
+    { "pool": "macos", "held": 0, "capacity": 1 }
+  ],
+  "abandoned_macos_vms": 0,
+  "abandoned_macos_vm_cap": 2,
   "draining": false,
   "uptime": "3h42m15s"
 }
 ```
 
+### held_slots vs active_jobs
+
+They are not the same number, and the difference is diagnostic.
+
+A dispatch takes a concurrency slot *before* it registers the runner and gives
+it back *after* the job is untracked, so a job that is still provisioning is
+counted in `held_slots` and not in `active_jobs`. Briefly, that is normal.
+
+Sustained, it is not. A pool sitting at `held == capacity` with `active_jobs`
+at 0 means the capacity is charged to something the scheduler is not tracking —
+a provision that is stuck, or a slot that leaked. The daemon also logs
+`waiting for a free concurrency slot` (and escalates to a suspected-leak error)
+when a job blocks on such a pool, so the log and this output agree.
+
+### abandoned_macos_vms
+
+When a macOS VM's `Stop()` does not return — Virtualization.framework can wedge
+a teardown indefinitely — the daemon walks away from it after a grace period so
+the concurrency slot comes back. The slot is the thing that matters: a held one
+costs the node its whole macOS capacity.
+
+The guest, however, is still running, and still holding its full configured RAM.
+Nothing in the daemon can reclaim it; only a restart can. So each abandonment is
+counted here, and once `abandoned_macos_vms` reaches `abandoned_macos_vm_cap` the
+macOS pool refuses to provision anything further and says so at `ERROR` — a
+bounded, legible failure instead of unbounded memory growth on a host that
+Virtualization.framework will soon refuse anyway.
+
+Non-zero is worth investigating. At the cap, restart the daemon.
+
+`/healthz` carries the same fields, plus `abandoned_macos_vm_details` (job id,
+VM instance id, reason and age per stranded guest) when there are any.
+
 ## Connection
 
 The command connects to the daemon's gRPC unix socket at `<data-dir>/ephemerd.sock`. If the daemon is not running or the socket does not exist, the command prints an error.

pkg/scheduler/grpc.go

@@ -79,13 +79,34 @@ func (c *controlServer) Status(ctx context.Context, req *apiv1.StatusRequest) (*
 	draining := c.sched.draining
 	c.sched.mu.Unlock()
 
+	// Slot occupancy is reported alongside the tracked-job count, never in
+	// place of it. ActiveJobs is len(s.running) and a slot is held from
+	// before trackRunning to after untrackRunning, so a stuck provision — or
+	// a leaked slot — shows up here and nowhere else. See issue #196.
+	var pools []*apiv1.SlotPool
+	for _, p := range c.sched.SlotUsage() {
+		pools = append(pools, &apiv1.SlotPool{
+			Pool:     p.Pool,
+			Held:     int32(p.Held),
+			Capacity: int32(p.Capacity),
+		})
+	}
+
 	return &apiv1.StatusResponse{
 		Status:        "ok",
 		ActiveJobs:    int32(activeJobs),
 		MaxConcurrent: int32(c.sched.cfg.MaxConcurrent),
-		Draining:      draining,
-		Uptime:        time.Since(c.sched.startTime).Truncate(time.Second).String(),
-		Version:       c.sched.cfg.Version,
+		HeldSlots:     int32(c.sched.HeldSlots()),
+		SlotCapacity:  int32(c.sched.SlotCapacity()),
+		SlotPools:     pools,
+		// The other half of the slot story: slots that came back only because
+		// the daemon abandoned a teardown. A guest per count, still holding
+		// its RAM, reclaimable only by restart. See issue #196 and slots.go.
+		AbandonedMacosVms:   int32(c.sched.AbandonedMacOSVMs()),
+		AbandonedMacosVmCap: int32(c.sched.abandonedMacVMCap()),
+		Draining:            draining,
+		Uptime:              time.Since(c.sched.startTime).Truncate(time.Second).String(),
+		Version:             c.sched.cfg.Version,
 	}, nil
 }
 

pkg/scheduler/macos_provision_test.go

@@ -132,13 +132,16 @@ func TestWaitForMacRunnerBounded_ReturnsWhenWaitSucceeds(t *testing.T) {
 	var stops atomic.Int32
 	quick := &fastMacVM{ip: "192.168.64.5", stops: &stops}
 
-	ip, err := s.waitForMacRunnerBounded(context.Background(), quick, quietLogger())
+	ip, alreadyStopped, err := s.waitForMacRunnerBounded(context.Background(), quick, macVMRef{JobID: 1, VMID: "1-happy"}, quietLogger())
 	if err != nil {
 		t.Fatalf("unexpected error on healthy provision: %v", err)
 	}
 	if ip != "192.168.64.5" {
 		t.Errorf("ip = %q, want 192.168.64.5", ip)
 	}
+	if alreadyStopped {
+		t.Error("healthy provision reported the VM as already stopped; the caller would skip a teardown it still owes")
+	}
 	if stops.Load() != 0 {
 		t.Errorf("VM was stopped %d times on the happy path, want 0", stops.Load())
 	}

pkg/scheduler/multiprovider_test.go

@@ -18,10 +18,21 @@ type mockProvider struct {
 	defaultImage string
 	events       chan providers.JobEvent
 
+	// releaseHook runs at the top of ReleaseJob, before the mutex is taken.
+	// Set once before the test starts and never mutated afterwards.
+	//
+	// It exists to model what ReleaseJob actually is in production — a GitHub
+	// API call, on context.Background(), with no deadline — so a test can
+	// assert that a concurrency slot is FREE while that call is still in
+	// flight. Without a slow ReleaseJob the release-before-teardown ordering
+	// this fix depends on is unobservable: the mock returns instantly, so
+	// moving the release after the teardown still passes.
+	releaseHook func(*providers.Claim)
+
 	mu       sync.Mutex
-	claims   []*providers.Claim  // jobs claimed via ClaimJob
-	releases []*providers.Claim  // jobs released via ReleaseJob
-	images   map[int64]string    // jobID → image for FetchJobImage
+	claims   []*providers.Claim // jobs claimed via ClaimJob
+	releases []*providers.Claim // jobs released via ReleaseJob
+	images   map[int64]string   // jobID → image for FetchJobImage
 }
 
 var _ providers.Poll = (*mockProvider)(nil)
@@ -62,6 +73,9 @@ func (m *mockProvider) ClaimJob(_ context.Context, event *providers.JobEvent, ru
 }
 
 func (m *mockProvider) ReleaseJob(_ context.Context, claim *providers.Claim) error {
+	if m.releaseHook != nil {
+		m.releaseHook(claim)
+	}
 	m.mu.Lock()
 	m.releases = append(m.releases, claim)
 	m.mu.Unlock()