From c2e4078e8edd336dc779a621c020727652bb682f Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Fri, 4 Sep 2026 20:04:55 -0700 Subject: [PATCH 1/5] fix(scheduler): release the macOS slot on the provisioning-deadline path (#196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fleet's mac served ZERO macOS jobs for ~28 hours while reporting `status: ok`, `active_jobs: 0` and answering /healthz with 200. Five macOS jobs queued in that window; none were provisioned, and the first of them aged out at GitHub's 24-hour queue timeout without ever running. An ephemerd restart fixed it instantly — the same code path then provisioned in 54 seconds, so the VM boot failure that started it was a one-off transient. The outage was entirely what happened afterwards. The mechanism. A macOS VM wedged before its runner became reachable. PR #178's provisioning watchdog fired on schedule and logged "macOS VM stuck in provisioning past deadline; force-stopping VM to reclaim the slot". It force-stopped the VM (the guest ignored the graceful request: "macOS VM did not stop gracefully, forcing") and then did a bare `<-resCh` to wait for the reachability goroutine to unwind — on the assumption that killing the VM always drops the guest connection and unblocks it. It did not. The wait never returned, so waitForMacRunnerBounded never returned, so handleMacOSJob never reached the `<-s.macSem` that the log line had just promised. The slot the watchdog existed to reclaim was reclaimed for the VM and never for the semaphore. Nothing showed it. The job had never reached trackRunning, so len(s.running) was 0 and every health signal — active_jobs, /healthz, `ephemerd jobs`, the orphan sweep — reported an idle, healthy node. And a job blocking on a full semaphore logged nothing at all, so the log went silent for 28 hours. This is the shape PR #190 already fixed on the local and Linux paths: release the slot as soon as the job stops being demand, never coupled to teardown completing. Same treatment here. - waitForMacRunnerBounded now stops the VM off-thread and bounds the unwind wait (awaitUnwind, 30s), then returns regardless. A wedged guest wait is a leaked goroutine nothing in this process can kill; blocking on it turned that into a leaked concurrency slot, which costs the node its entire macOS capacity. - Every macOS teardown that runs while a slot or a wait-goroutine could be held goes through stopVMBounded. This also closes the second instance of the same bug: the caller's error path called macVM.Stop() again, and darwinMacOSVM.Stop is a sync.Once — a second caller blocks until the first returns, so the "cleanup" Stop hung on the wedged one while still holding the slot. - The macOS wait-goroutine releases the moment the job is untracked instead of at the end, behind macVM.Stop() (up to 15s, unbounded when the force-stop wedges) and a ReleaseJob API call on context.Background() with no deadline. - All three dispatch paths now acquire through slotToken: an idempotent release with a deferred backstop, so no future error return can skip it and a panic cannot either. handleMacOSJob had six hand-written `<-s.macSem` returns; one of them was unreachable. Observability, both gaps filed with the issue: - status/healthz gain held_slots, slot_capacity and a per-pool breakdown, read straight off the semaphores so they count capacity charged to UNTRACKED work. active_jobs and max_concurrent are unchanged. held == capacity with active_jobs 0 is the outage, visible. - A dispatch that blocks on a full pool now logs after 30s and every 5 minutes thereafter, and escalates to Error ("suspected slot leak") once it has waited 15 minutes for a pool whose held slots no tracked job accounts for. That doubles as the leak watchdog: a held slot only matters when something is waiting for it, so the check lives on the waiting path and costs nothing on an idle node. #178's deadline behaviour is unchanged — the VM is still force-stopped — and the #151 busy-veto is untouched. --- api/v1/ephemerd.pb.go | 838 +++++++++++++++++++++--------------- api/v1/ephemerd.proto | 22 + cmd/ephemerd/commands.go | 16 + docs/cli/status.md | 28 +- pkg/scheduler/grpc.go | 16 + pkg/scheduler/scheduler.go | 274 +++++++++--- pkg/scheduler/slots.go | 256 +++++++++++ pkg/scheduler/slots_test.go | 598 +++++++++++++++++++++++++ 8 files changed, 1630 insertions(+), 418 deletions(-) create mode 100644 pkg/scheduler/slots.go create mode 100644 pkg/scheduler/slots_test.go diff --git a/api/v1/ephemerd.pb.go b/api/v1/ephemerd.pb.go index 4f3e689a..c9643ade 100644 --- a/api/v1/ephemerd.pb.go +++ b/api/v1/ephemerd.pb.go @@ -332,6 +332,21 @@ type StatusResponse struct { // The upgrade flow polls this after a restart to confirm the new binary // came up on the target version. Version string `protobuf:"bytes,6,opt,name=version,proto3" json:"version,omitempty"` + // held_slots is how many concurrency slots are currently HELD across every + // dispatch pool, which is not the same thing as active_jobs. + // + // active_jobs counts jobs the scheduler is tracking. A slot is taken before + // the job is tracked and given back after it is untracked, so a job that is + // still provisioning — or one whose slot leaked — is charged here and not + // there. Issue #196: a mac held its only slot for 28 hours after a wedged + // provision and reported status ok with active_jobs 0 the whole time. + // held_slots == slot_capacity with active_jobs 0 is that failure, visible. + HeldSlots int32 `protobuf:"varint,7,opt,name=held_slots,json=heldSlots,proto3" json:"held_slots,omitempty"` + // slot_capacity is the total number of slots across all pools. + SlotCapacity int32 `protobuf:"varint,8,opt,name=slot_capacity,json=slotCapacity,proto3" json:"slot_capacity,omitempty"` + // slot_pools breaks the totals down per dispatch pool (local, linux, + // macos), so an operator can tell which pool is saturated. + SlotPools []*SlotPool `protobuf:"bytes,9,rep,name=slot_pools,json=slotPools,proto3" json:"slot_pools,omitempty"` } func (x *StatusResponse) Reset() { @@ -408,6 +423,91 @@ func (x *StatusResponse) GetVersion() string { return "" } +func (x *StatusResponse) GetHeldSlots() int32 { + if x != nil { + return x.HeldSlots + } + return 0 +} + +func (x *StatusResponse) GetSlotCapacity() int32 { + if x != nil { + return x.SlotCapacity + } + return 0 +} + +func (x *StatusResponse) GetSlotPools() []*SlotPool { + if x != nil { + return x.SlotPools + } + return nil +} + +// SlotPool is one dispatch pool's concurrency-slot occupancy. +type SlotPool struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pool string `protobuf:"bytes,1,opt,name=pool,proto3" json:"pool,omitempty"` + Held int32 `protobuf:"varint,2,opt,name=held,proto3" json:"held,omitempty"` + Capacity int32 `protobuf:"varint,3,opt,name=capacity,proto3" json:"capacity,omitempty"` +} + +func (x *SlotPool) Reset() { + *x = SlotPool{} + if protoimpl.UnsafeEnabled { + mi := &file_api_v1_ephemerd_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SlotPool) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SlotPool) ProtoMessage() {} + +func (x *SlotPool) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_ephemerd_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SlotPool.ProtoReflect.Descriptor instead. +func (*SlotPool) Descriptor() ([]byte, []int) { + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{5} +} + +func (x *SlotPool) GetPool() string { + if x != nil { + return x.Pool + } + return "" +} + +func (x *SlotPool) GetHeld() int32 { + if x != nil { + return x.Held + } + return 0 +} + +func (x *SlotPool) GetCapacity() int32 { + if x != nil { + return x.Capacity + } + return 0 +} + type ListJobsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -417,7 +517,7 @@ type ListJobsRequest struct { func (x *ListJobsRequest) Reset() { *x = ListJobsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[5] + mi := &file_api_v1_ephemerd_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -430,7 +530,7 @@ func (x *ListJobsRequest) String() string { func (*ListJobsRequest) ProtoMessage() {} func (x *ListJobsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[5] + mi := &file_api_v1_ephemerd_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -443,7 +543,7 @@ func (x *ListJobsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListJobsRequest.ProtoReflect.Descriptor instead. func (*ListJobsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{5} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{6} } type ListJobsResponse struct { @@ -457,7 +557,7 @@ type ListJobsResponse struct { func (x *ListJobsResponse) Reset() { *x = ListJobsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[6] + mi := &file_api_v1_ephemerd_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -470,7 +570,7 @@ func (x *ListJobsResponse) String() string { func (*ListJobsResponse) ProtoMessage() {} func (x *ListJobsResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[6] + mi := &file_api_v1_ephemerd_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -483,7 +583,7 @@ func (x *ListJobsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListJobsResponse.ProtoReflect.Descriptor instead. func (*ListJobsResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{6} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{7} } func (x *ListJobsResponse) GetJobs() []*Job { @@ -504,7 +604,7 @@ type GetJobRequest struct { func (x *GetJobRequest) Reset() { *x = GetJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[7] + mi := &file_api_v1_ephemerd_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -517,7 +617,7 @@ func (x *GetJobRequest) String() string { func (*GetJobRequest) ProtoMessage() {} func (x *GetJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[7] + mi := &file_api_v1_ephemerd_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -530,7 +630,7 @@ func (x *GetJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetJobRequest.ProtoReflect.Descriptor instead. func (*GetJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{7} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{8} } func (x *GetJobRequest) GetId() int64 { @@ -559,7 +659,7 @@ type Job struct { func (x *Job) Reset() { *x = Job{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[8] + mi := &file_api_v1_ephemerd_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -572,7 +672,7 @@ func (x *Job) String() string { func (*Job) ProtoMessage() {} func (x *Job) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[8] + mi := &file_api_v1_ephemerd_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -585,7 +685,7 @@ func (x *Job) ProtoReflect() protoreflect.Message { // Deprecated: Use Job.ProtoReflect.Descriptor instead. func (*Job) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{8} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{9} } func (x *Job) GetId() int64 { @@ -662,7 +762,7 @@ type KillJobRequest struct { func (x *KillJobRequest) Reset() { *x = KillJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[9] + mi := &file_api_v1_ephemerd_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -675,7 +775,7 @@ func (x *KillJobRequest) String() string { func (*KillJobRequest) ProtoMessage() {} func (x *KillJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[9] + mi := &file_api_v1_ephemerd_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -688,7 +788,7 @@ func (x *KillJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KillJobRequest.ProtoReflect.Descriptor instead. func (*KillJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{9} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{10} } func (x *KillJobRequest) GetId() int64 { @@ -707,7 +807,7 @@ type KillJobResponse struct { func (x *KillJobResponse) Reset() { *x = KillJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[10] + mi := &file_api_v1_ephemerd_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -720,7 +820,7 @@ func (x *KillJobResponse) String() string { func (*KillJobResponse) ProtoMessage() {} func (x *KillJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[10] + mi := &file_api_v1_ephemerd_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -733,7 +833,7 @@ func (x *KillJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KillJobResponse.ProtoReflect.Descriptor instead. func (*KillJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{10} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{11} } type CordonRequest struct { @@ -745,7 +845,7 @@ type CordonRequest struct { func (x *CordonRequest) Reset() { *x = CordonRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[11] + mi := &file_api_v1_ephemerd_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -758,7 +858,7 @@ func (x *CordonRequest) String() string { func (*CordonRequest) ProtoMessage() {} func (x *CordonRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[11] + mi := &file_api_v1_ephemerd_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -771,7 +871,7 @@ func (x *CordonRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CordonRequest.ProtoReflect.Descriptor instead. func (*CordonRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{11} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{12} } type CordonResponse struct { @@ -785,7 +885,7 @@ type CordonResponse struct { func (x *CordonResponse) Reset() { *x = CordonResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[12] + mi := &file_api_v1_ephemerd_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -798,7 +898,7 @@ func (x *CordonResponse) String() string { func (*CordonResponse) ProtoMessage() {} func (x *CordonResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[12] + mi := &file_api_v1_ephemerd_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -811,7 +911,7 @@ func (x *CordonResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CordonResponse.ProtoReflect.Descriptor instead. func (*CordonResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{12} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{13} } func (x *CordonResponse) GetActiveJobs() int32 { @@ -830,7 +930,7 @@ type UncordonRequest struct { func (x *UncordonRequest) Reset() { *x = UncordonRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[13] + mi := &file_api_v1_ephemerd_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -843,7 +943,7 @@ func (x *UncordonRequest) String() string { func (*UncordonRequest) ProtoMessage() {} func (x *UncordonRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[13] + mi := &file_api_v1_ephemerd_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -856,7 +956,7 @@ func (x *UncordonRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UncordonRequest.ProtoReflect.Descriptor instead. func (*UncordonRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{13} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{14} } type UncordonResponse struct { @@ -870,7 +970,7 @@ type UncordonResponse struct { func (x *UncordonResponse) Reset() { *x = UncordonResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[14] + mi := &file_api_v1_ephemerd_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -883,7 +983,7 @@ func (x *UncordonResponse) String() string { func (*UncordonResponse) ProtoMessage() {} func (x *UncordonResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[14] + mi := &file_api_v1_ephemerd_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -896,7 +996,7 @@ func (x *UncordonResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UncordonResponse.ProtoReflect.Descriptor instead. func (*UncordonResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{14} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{15} } func (x *UncordonResponse) GetActiveJobs() int32 { @@ -918,7 +1018,7 @@ type GetJobLogsRequest struct { func (x *GetJobLogsRequest) Reset() { *x = GetJobLogsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[15] + mi := &file_api_v1_ephemerd_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -931,7 +1031,7 @@ func (x *GetJobLogsRequest) String() string { func (*GetJobLogsRequest) ProtoMessage() {} func (x *GetJobLogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[15] + mi := &file_api_v1_ephemerd_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -944,7 +1044,7 @@ func (x *GetJobLogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetJobLogsRequest.ProtoReflect.Descriptor instead. func (*GetJobLogsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{15} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{16} } func (x *GetJobLogsRequest) GetId() int64 { @@ -972,7 +1072,7 @@ type LogChunk struct { func (x *LogChunk) Reset() { *x = LogChunk{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[16] + mi := &file_api_v1_ephemerd_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -985,7 +1085,7 @@ func (x *LogChunk) String() string { func (*LogChunk) ProtoMessage() {} func (x *LogChunk) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[16] + mi := &file_api_v1_ephemerd_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -998,7 +1098,7 @@ func (x *LogChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use LogChunk.ProtoReflect.Descriptor instead. func (*LogChunk) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{16} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{17} } func (x *LogChunk) GetData() []byte { @@ -1036,7 +1136,7 @@ type UpgradeRequest struct { func (x *UpgradeRequest) Reset() { *x = UpgradeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[17] + mi := &file_api_v1_ephemerd_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1049,7 +1149,7 @@ func (x *UpgradeRequest) String() string { func (*UpgradeRequest) ProtoMessage() {} func (x *UpgradeRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[17] + mi := &file_api_v1_ephemerd_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1062,7 +1162,7 @@ func (x *UpgradeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeRequest.ProtoReflect.Descriptor instead. func (*UpgradeRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{17} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{18} } func (x *UpgradeRequest) GetTargetVersion() string { @@ -1117,7 +1217,7 @@ type UpgradeProgress struct { func (x *UpgradeProgress) Reset() { *x = UpgradeProgress{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[18] + mi := &file_api_v1_ephemerd_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1130,7 +1230,7 @@ func (x *UpgradeProgress) String() string { func (*UpgradeProgress) ProtoMessage() {} func (x *UpgradeProgress) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[18] + mi := &file_api_v1_ephemerd_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1143,7 +1243,7 @@ func (x *UpgradeProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeProgress.ProtoReflect.Descriptor instead. func (*UpgradeProgress) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{18} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{19} } func (x *UpgradeProgress) GetState() UpgradeState { @@ -1213,7 +1313,7 @@ type CreateJobRequest struct { func (x *CreateJobRequest) Reset() { *x = CreateJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[19] + mi := &file_api_v1_ephemerd_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1226,7 +1326,7 @@ func (x *CreateJobRequest) String() string { func (*CreateJobRequest) ProtoMessage() {} func (x *CreateJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[19] + mi := &file_api_v1_ephemerd_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1239,7 +1339,7 @@ func (x *CreateJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateJobRequest.ProtoReflect.Descriptor instead. func (*CreateJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{19} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{20} } func (x *CreateJobRequest) GetId() string { @@ -1286,7 +1386,7 @@ type CreateJobResponse struct { func (x *CreateJobResponse) Reset() { *x = CreateJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[20] + mi := &file_api_v1_ephemerd_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1299,7 +1399,7 @@ func (x *CreateJobResponse) String() string { func (*CreateJobResponse) ProtoMessage() {} func (x *CreateJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[20] + mi := &file_api_v1_ephemerd_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +1412,7 @@ func (x *CreateJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateJobResponse.ProtoReflect.Descriptor instead. func (*CreateJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{20} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{21} } type WaitJobRequest struct { @@ -1326,7 +1426,7 @@ type WaitJobRequest struct { func (x *WaitJobRequest) Reset() { *x = WaitJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[21] + mi := &file_api_v1_ephemerd_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1339,7 +1439,7 @@ func (x *WaitJobRequest) String() string { func (*WaitJobRequest) ProtoMessage() {} func (x *WaitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[21] + mi := &file_api_v1_ephemerd_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1352,7 +1452,7 @@ func (x *WaitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WaitJobRequest.ProtoReflect.Descriptor instead. func (*WaitJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{21} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{22} } func (x *WaitJobRequest) GetId() string { @@ -1373,7 +1473,7 @@ type WaitJobResponse struct { func (x *WaitJobResponse) Reset() { *x = WaitJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[22] + mi := &file_api_v1_ephemerd_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1386,7 +1486,7 @@ func (x *WaitJobResponse) String() string { func (*WaitJobResponse) ProtoMessage() {} func (x *WaitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[22] + mi := &file_api_v1_ephemerd_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1399,7 +1499,7 @@ func (x *WaitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WaitJobResponse.ProtoReflect.Descriptor instead. func (*WaitJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{22} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{23} } func (x *WaitJobResponse) GetExitCode() uint32 { @@ -1420,7 +1520,7 @@ type DestroyJobRequest struct { func (x *DestroyJobRequest) Reset() { *x = DestroyJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[23] + mi := &file_api_v1_ephemerd_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1433,7 +1533,7 @@ func (x *DestroyJobRequest) String() string { func (*DestroyJobRequest) ProtoMessage() {} func (x *DestroyJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[23] + mi := &file_api_v1_ephemerd_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1446,7 +1546,7 @@ func (x *DestroyJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyJobRequest.ProtoReflect.Descriptor instead. func (*DestroyJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{23} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{24} } func (x *DestroyJobRequest) GetId() string { @@ -1465,7 +1565,7 @@ type DestroyJobResponse struct { func (x *DestroyJobResponse) Reset() { *x = DestroyJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[24] + mi := &file_api_v1_ephemerd_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1478,7 +1578,7 @@ func (x *DestroyJobResponse) String() string { func (*DestroyJobResponse) ProtoMessage() {} func (x *DestroyJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[24] + mi := &file_api_v1_ephemerd_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1491,7 +1591,7 @@ func (x *DestroyJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyJobResponse.ProtoReflect.Descriptor instead. func (*DestroyJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{24} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{25} } // StreamContainerStatsRequest opens a long-lived sampling stream. The host @@ -1508,7 +1608,7 @@ type StreamContainerStatsRequest struct { func (x *StreamContainerStatsRequest) Reset() { *x = StreamContainerStatsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[25] + mi := &file_api_v1_ephemerd_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1521,7 +1621,7 @@ func (x *StreamContainerStatsRequest) String() string { func (*StreamContainerStatsRequest) ProtoMessage() {} func (x *StreamContainerStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[25] + mi := &file_api_v1_ephemerd_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1534,7 +1634,7 @@ func (x *StreamContainerStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamContainerStatsRequest.ProtoReflect.Descriptor instead. func (*StreamContainerStatsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{25} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{26} } func (x *StreamContainerStatsRequest) GetIntervalSeconds() uint32 { @@ -1556,7 +1656,7 @@ type ContainerStatsBatch struct { func (x *ContainerStatsBatch) Reset() { *x = ContainerStatsBatch{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[26] + mi := &file_api_v1_ephemerd_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +1669,7 @@ func (x *ContainerStatsBatch) String() string { func (*ContainerStatsBatch) ProtoMessage() {} func (x *ContainerStatsBatch) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[26] + mi := &file_api_v1_ephemerd_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,7 +1682,7 @@ func (x *ContainerStatsBatch) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerStatsBatch.ProtoReflect.Descriptor instead. func (*ContainerStatsBatch) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{26} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{27} } func (x *ContainerStatsBatch) GetTimestampUnixNano() int64 { @@ -1622,7 +1722,7 @@ type ContainerStats struct { func (x *ContainerStats) Reset() { *x = ContainerStats{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[27] + mi := &file_api_v1_ephemerd_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1635,7 +1735,7 @@ func (x *ContainerStats) String() string { func (*ContainerStats) ProtoMessage() {} func (x *ContainerStats) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[27] + mi := &file_api_v1_ephemerd_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1648,7 +1748,7 @@ func (x *ContainerStats) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerStats.ProtoReflect.Descriptor instead. func (*ContainerStats) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{27} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{28} } func (x *ContainerStats) GetId() string { @@ -1737,8 +1837,8 @@ var file_api_v1_ephemerd_proto_rawDesc = []byte{ 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x0f, 0x0a, - 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbe, - 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb8, + 0x02, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, @@ -1749,211 +1849,223 @@ var file_api_v1_ephemerd_proto_rawDesc = []byte{ 0x01, 0x28, 0x08, 0x52, 0x08, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, - 0x11, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x38, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x1f, 0x0a, 0x0d, - 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xd1, 0x01, - 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, - 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, - 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, - 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x02, 0x69, 0x64, 0x22, 0x11, 0x0a, 0x0f, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x0e, 0x43, 0x6f, 0x72, 0x64, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, - 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, - 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x55, 0x6e, - 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, - 0x10, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, - 0x62, 0x73, 0x22, 0x3b, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1d, 0x0a, 0x0a, 0x68, 0x65, 0x6c, 0x64, 0x5f, 0x73, 0x6c, 0x6f, 0x74, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x68, 0x65, 0x6c, 0x64, 0x53, 0x6c, 0x6f, 0x74, 0x73, 0x12, 0x23, + 0x0a, 0x0d, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x73, 0x6c, 0x6f, 0x74, 0x43, 0x61, 0x70, 0x61, 0x63, + 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x0a, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x70, 0x6f, 0x6f, 0x6c, + 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x09, + 0x73, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x22, 0x4e, 0x0a, 0x08, 0x53, 0x6c, 0x6f, + 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x68, 0x65, 0x6c, 0x64, 0x12, 0x1a, 0x0a, + 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x4c, 0x69, 0x73, + 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x38, 0x0a, 0x10, + 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x24, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, + 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x1f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, - 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x22, - 0x1e, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, - 0xbf, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x72, 0x6c, - 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x75, 0x72, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x15, - 0x64, 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x64, 0x72, 0x61, - 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, - 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x5f, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x6e, 0x6f, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, - 0x6f, 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, - 0x65, 0x22, 0x99, 0x02, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, - 0x73, 0x12, 0x29, 0x0a, 0x10, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x87, 0x01, - 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x69, 0x74, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x69, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x0a, 0x0e, - 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, - 0x0a, 0x0f, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x23, - 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, - 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x48, 0x0a, 0x1b, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, - 0x6e, 0x64, 0x73, 0x22, 0x78, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, - 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xca, 0x02, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x72, 0x65, 0x70, 0x6f, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, - 0x65, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, - 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x12, 0x21, 0x0a, 0x0c, - 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, - 0x2a, 0x0a, 0x11, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x6e, 0x6f, 0x6e, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x41, 0x6e, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, - 0x70, 0x75, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4c, 0x69, 0x6d, 0x69, - 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x5f, 0x72, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x74, 0x78, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x54, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0xaf, 0x02, 0x0a, 0x0c, 0x55, - 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x55, - 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, - 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, - 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x46, - 0x4c, 0x49, 0x47, 0x48, 0x54, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, - 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x50, 0x5f, 0x54, 0x4f, 0x5f, 0x44, - 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x49, 0x4e, 0x47, 0x10, - 0x03, 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x04, - 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x45, 0x5f, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x19, 0x0a, - 0x15, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, - 0x54, 0x41, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, - 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x57, 0x41, 0x50, 0x50, 0x49, - 0x4e, 0x47, 0x10, 0x07, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, - 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x09, 0x32, 0xfd, 0x04, 0x0a, - 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x4c, - 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, - 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1a, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x65, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x12, 0x44, 0x0a, 0x07, - 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, - 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x06, 0x43, 0x6f, 0x72, - 0x64, 0x6f, 0x6e, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, - 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, - 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, - 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, - 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, - 0x0a, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x65, 0x70, - 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, - 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd1, 0x02, 0x0a, - 0x08, 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4a, 0x0a, 0x09, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, - 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, - 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, - 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x44, - 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x14, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x12, 0x28, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x65, - 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, - 0x42, 0x28, 0x5a, 0x26, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, - 0x70, 0x68, 0x70, 0x6d, 0x2f, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xd1, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x03, 0x70, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, + 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, + 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x4b, + 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0x11, 0x0a, + 0x0f, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x31, 0x0a, 0x0e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, + 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, + 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, 0x10, 0x55, 0x6e, 0x63, 0x6f, 0x72, + 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x3b, 0x0a, 0x11, + 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x22, 0x1e, 0x0a, 0x08, 0x4c, 0x6f, 0x67, + 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, 0x01, 0x0a, 0x0e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x72, 0x6c, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, + 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x72, 0x6c, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, + 0x5f, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x6e, 0x6f, + 0x44, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x22, 0x99, 0x02, 0x0a, 0x0f, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, + 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, + 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, + 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x79, 0x74, + 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x87, 0x01, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x69, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, + 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, + 0x6f, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x0a, 0x0f, 0x57, 0x61, 0x69, 0x74, + 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, + 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, + 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x23, 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, + 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x14, 0x0a, + 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x48, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x78, 0x0a, + 0x13, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x6e, 0x69, 0x78, + 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xca, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, + 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x26, + 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x65, 0x6d, + 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x6e, 0x6f, 0x6e, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x41, 0x6e, 0x6f, 0x6e, + 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, 0x75, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, + 0x69, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, + 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x72, 0x78, 0x5f, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x52, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x74, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x78, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x2a, 0xaf, 0x02, 0x0a, 0x0c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x4c, 0x49, 0x47, 0x48, 0x54, 0x10, + 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x55, 0x50, 0x5f, 0x54, 0x4f, 0x5f, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, + 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x1d, 0x0a, 0x19, 0x55, + 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x57, + 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, + 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x56, 0x45, 0x52, 0x49, + 0x46, 0x59, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x55, 0x50, 0x47, 0x52, 0x41, + 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x47, 0x49, 0x4e, 0x47, + 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x53, 0x57, 0x41, 0x50, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, 0x1c, + 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, + 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, + 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, + 0x49, 0x4c, 0x45, 0x44, 0x10, 0x09, 0x32, 0xfd, 0x04, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1a, 0x2e, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, + 0x73, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, + 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, + 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x12, 0x44, 0x0a, 0x07, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, + 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, + 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, + 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, + 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, + 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, + 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x06, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x12, 0x1a, 0x2e, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x72, 0x64, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, + 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, + 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, + 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, + 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x46, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x0a, 0x50, 0x72, 0x75, 0x6e, 0x65, + 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, + 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd1, 0x02, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x70, 0x61, + 0x74, 0x63, 0x68, 0x12, 0x4a, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, + 0x12, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x44, 0x0a, 0x07, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, + 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x28, 0x2e, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x70, 0x68, 0x70, 0x6d, 0x2f, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, + 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1969,7 +2081,7 @@ func file_api_v1_ephemerd_proto_rawDescGZIP() []byte { } var file_api_v1_ephemerd_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_api_v1_ephemerd_proto_msgTypes = make([]protoimpl.MessageInfo, 28) +var file_api_v1_ephemerd_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_api_v1_ephemerd_proto_goTypes = []interface{}{ (UpgradeState)(0), // 0: ephemerd.v1.UpgradeState (*PruneCacheRequest)(nil), // 1: ephemerd.v1.PruneCacheRequest @@ -1977,66 +2089,68 @@ var file_api_v1_ephemerd_proto_goTypes = []interface{}{ (*PruneCacheResult)(nil), // 3: ephemerd.v1.PruneCacheResult (*StatusRequest)(nil), // 4: ephemerd.v1.StatusRequest (*StatusResponse)(nil), // 5: ephemerd.v1.StatusResponse - (*ListJobsRequest)(nil), // 6: ephemerd.v1.ListJobsRequest - (*ListJobsResponse)(nil), // 7: ephemerd.v1.ListJobsResponse - (*GetJobRequest)(nil), // 8: ephemerd.v1.GetJobRequest - (*Job)(nil), // 9: ephemerd.v1.Job - (*KillJobRequest)(nil), // 10: ephemerd.v1.KillJobRequest - (*KillJobResponse)(nil), // 11: ephemerd.v1.KillJobResponse - (*CordonRequest)(nil), // 12: ephemerd.v1.CordonRequest - (*CordonResponse)(nil), // 13: ephemerd.v1.CordonResponse - (*UncordonRequest)(nil), // 14: ephemerd.v1.UncordonRequest - (*UncordonResponse)(nil), // 15: ephemerd.v1.UncordonResponse - (*GetJobLogsRequest)(nil), // 16: ephemerd.v1.GetJobLogsRequest - (*LogChunk)(nil), // 17: ephemerd.v1.LogChunk - (*UpgradeRequest)(nil), // 18: ephemerd.v1.UpgradeRequest - (*UpgradeProgress)(nil), // 19: ephemerd.v1.UpgradeProgress - (*CreateJobRequest)(nil), // 20: ephemerd.v1.CreateJobRequest - (*CreateJobResponse)(nil), // 21: ephemerd.v1.CreateJobResponse - (*WaitJobRequest)(nil), // 22: ephemerd.v1.WaitJobRequest - (*WaitJobResponse)(nil), // 23: ephemerd.v1.WaitJobResponse - (*DestroyJobRequest)(nil), // 24: ephemerd.v1.DestroyJobRequest - (*DestroyJobResponse)(nil), // 25: ephemerd.v1.DestroyJobResponse - (*StreamContainerStatsRequest)(nil), // 26: ephemerd.v1.StreamContainerStatsRequest - (*ContainerStatsBatch)(nil), // 27: ephemerd.v1.ContainerStatsBatch - (*ContainerStats)(nil), // 28: ephemerd.v1.ContainerStats + (*SlotPool)(nil), // 6: ephemerd.v1.SlotPool + (*ListJobsRequest)(nil), // 7: ephemerd.v1.ListJobsRequest + (*ListJobsResponse)(nil), // 8: ephemerd.v1.ListJobsResponse + (*GetJobRequest)(nil), // 9: ephemerd.v1.GetJobRequest + (*Job)(nil), // 10: ephemerd.v1.Job + (*KillJobRequest)(nil), // 11: ephemerd.v1.KillJobRequest + (*KillJobResponse)(nil), // 12: ephemerd.v1.KillJobResponse + (*CordonRequest)(nil), // 13: ephemerd.v1.CordonRequest + (*CordonResponse)(nil), // 14: ephemerd.v1.CordonResponse + (*UncordonRequest)(nil), // 15: ephemerd.v1.UncordonRequest + (*UncordonResponse)(nil), // 16: ephemerd.v1.UncordonResponse + (*GetJobLogsRequest)(nil), // 17: ephemerd.v1.GetJobLogsRequest + (*LogChunk)(nil), // 18: ephemerd.v1.LogChunk + (*UpgradeRequest)(nil), // 19: ephemerd.v1.UpgradeRequest + (*UpgradeProgress)(nil), // 20: ephemerd.v1.UpgradeProgress + (*CreateJobRequest)(nil), // 21: ephemerd.v1.CreateJobRequest + (*CreateJobResponse)(nil), // 22: ephemerd.v1.CreateJobResponse + (*WaitJobRequest)(nil), // 23: ephemerd.v1.WaitJobRequest + (*WaitJobResponse)(nil), // 24: ephemerd.v1.WaitJobResponse + (*DestroyJobRequest)(nil), // 25: ephemerd.v1.DestroyJobRequest + (*DestroyJobResponse)(nil), // 26: ephemerd.v1.DestroyJobResponse + (*StreamContainerStatsRequest)(nil), // 27: ephemerd.v1.StreamContainerStatsRequest + (*ContainerStatsBatch)(nil), // 28: ephemerd.v1.ContainerStatsBatch + (*ContainerStats)(nil), // 29: ephemerd.v1.ContainerStats } var file_api_v1_ephemerd_proto_depIdxs = []int32{ 3, // 0: ephemerd.v1.PruneCacheResponse.results:type_name -> ephemerd.v1.PruneCacheResult - 9, // 1: ephemerd.v1.ListJobsResponse.jobs:type_name -> ephemerd.v1.Job - 0, // 2: ephemerd.v1.UpgradeProgress.state:type_name -> ephemerd.v1.UpgradeState - 28, // 3: ephemerd.v1.ContainerStatsBatch.stats:type_name -> ephemerd.v1.ContainerStats - 4, // 4: ephemerd.v1.Control.Status:input_type -> ephemerd.v1.StatusRequest - 6, // 5: ephemerd.v1.Control.ListJobs:input_type -> ephemerd.v1.ListJobsRequest - 8, // 6: ephemerd.v1.Control.GetJob:input_type -> ephemerd.v1.GetJobRequest - 10, // 7: ephemerd.v1.Control.KillJob:input_type -> ephemerd.v1.KillJobRequest - 16, // 8: ephemerd.v1.Control.GetJobLogs:input_type -> ephemerd.v1.GetJobLogsRequest - 12, // 9: ephemerd.v1.Control.Cordon:input_type -> ephemerd.v1.CordonRequest - 14, // 10: ephemerd.v1.Control.Uncordon:input_type -> ephemerd.v1.UncordonRequest - 18, // 11: ephemerd.v1.Control.Upgrade:input_type -> ephemerd.v1.UpgradeRequest - 1, // 12: ephemerd.v1.Control.PruneCache:input_type -> ephemerd.v1.PruneCacheRequest - 20, // 13: ephemerd.v1.Dispatch.CreateJob:input_type -> ephemerd.v1.CreateJobRequest - 22, // 14: ephemerd.v1.Dispatch.WaitJob:input_type -> ephemerd.v1.WaitJobRequest - 24, // 15: ephemerd.v1.Dispatch.DestroyJob:input_type -> ephemerd.v1.DestroyJobRequest - 26, // 16: ephemerd.v1.Dispatch.StreamContainerStats:input_type -> ephemerd.v1.StreamContainerStatsRequest - 5, // 17: ephemerd.v1.Control.Status:output_type -> ephemerd.v1.StatusResponse - 7, // 18: ephemerd.v1.Control.ListJobs:output_type -> ephemerd.v1.ListJobsResponse - 9, // 19: ephemerd.v1.Control.GetJob:output_type -> ephemerd.v1.Job - 11, // 20: ephemerd.v1.Control.KillJob:output_type -> ephemerd.v1.KillJobResponse - 17, // 21: ephemerd.v1.Control.GetJobLogs:output_type -> ephemerd.v1.LogChunk - 13, // 22: ephemerd.v1.Control.Cordon:output_type -> ephemerd.v1.CordonResponse - 15, // 23: ephemerd.v1.Control.Uncordon:output_type -> ephemerd.v1.UncordonResponse - 19, // 24: ephemerd.v1.Control.Upgrade:output_type -> ephemerd.v1.UpgradeProgress - 2, // 25: ephemerd.v1.Control.PruneCache:output_type -> ephemerd.v1.PruneCacheResponse - 21, // 26: ephemerd.v1.Dispatch.CreateJob:output_type -> ephemerd.v1.CreateJobResponse - 23, // 27: ephemerd.v1.Dispatch.WaitJob:output_type -> ephemerd.v1.WaitJobResponse - 25, // 28: ephemerd.v1.Dispatch.DestroyJob:output_type -> ephemerd.v1.DestroyJobResponse - 27, // 29: ephemerd.v1.Dispatch.StreamContainerStats:output_type -> ephemerd.v1.ContainerStatsBatch - 17, // [17:30] is the sub-list for method output_type - 4, // [4:17] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 6, // 1: ephemerd.v1.StatusResponse.slot_pools:type_name -> ephemerd.v1.SlotPool + 10, // 2: ephemerd.v1.ListJobsResponse.jobs:type_name -> ephemerd.v1.Job + 0, // 3: ephemerd.v1.UpgradeProgress.state:type_name -> ephemerd.v1.UpgradeState + 29, // 4: ephemerd.v1.ContainerStatsBatch.stats:type_name -> ephemerd.v1.ContainerStats + 4, // 5: ephemerd.v1.Control.Status:input_type -> ephemerd.v1.StatusRequest + 7, // 6: ephemerd.v1.Control.ListJobs:input_type -> ephemerd.v1.ListJobsRequest + 9, // 7: ephemerd.v1.Control.GetJob:input_type -> ephemerd.v1.GetJobRequest + 11, // 8: ephemerd.v1.Control.KillJob:input_type -> ephemerd.v1.KillJobRequest + 17, // 9: ephemerd.v1.Control.GetJobLogs:input_type -> ephemerd.v1.GetJobLogsRequest + 13, // 10: ephemerd.v1.Control.Cordon:input_type -> ephemerd.v1.CordonRequest + 15, // 11: ephemerd.v1.Control.Uncordon:input_type -> ephemerd.v1.UncordonRequest + 19, // 12: ephemerd.v1.Control.Upgrade:input_type -> ephemerd.v1.UpgradeRequest + 1, // 13: ephemerd.v1.Control.PruneCache:input_type -> ephemerd.v1.PruneCacheRequest + 21, // 14: ephemerd.v1.Dispatch.CreateJob:input_type -> ephemerd.v1.CreateJobRequest + 23, // 15: ephemerd.v1.Dispatch.WaitJob:input_type -> ephemerd.v1.WaitJobRequest + 25, // 16: ephemerd.v1.Dispatch.DestroyJob:input_type -> ephemerd.v1.DestroyJobRequest + 27, // 17: ephemerd.v1.Dispatch.StreamContainerStats:input_type -> ephemerd.v1.StreamContainerStatsRequest + 5, // 18: ephemerd.v1.Control.Status:output_type -> ephemerd.v1.StatusResponse + 8, // 19: ephemerd.v1.Control.ListJobs:output_type -> ephemerd.v1.ListJobsResponse + 10, // 20: ephemerd.v1.Control.GetJob:output_type -> ephemerd.v1.Job + 12, // 21: ephemerd.v1.Control.KillJob:output_type -> ephemerd.v1.KillJobResponse + 18, // 22: ephemerd.v1.Control.GetJobLogs:output_type -> ephemerd.v1.LogChunk + 14, // 23: ephemerd.v1.Control.Cordon:output_type -> ephemerd.v1.CordonResponse + 16, // 24: ephemerd.v1.Control.Uncordon:output_type -> ephemerd.v1.UncordonResponse + 20, // 25: ephemerd.v1.Control.Upgrade:output_type -> ephemerd.v1.UpgradeProgress + 2, // 26: ephemerd.v1.Control.PruneCache:output_type -> ephemerd.v1.PruneCacheResponse + 22, // 27: ephemerd.v1.Dispatch.CreateJob:output_type -> ephemerd.v1.CreateJobResponse + 24, // 28: ephemerd.v1.Dispatch.WaitJob:output_type -> ephemerd.v1.WaitJobResponse + 26, // 29: ephemerd.v1.Dispatch.DestroyJob:output_type -> ephemerd.v1.DestroyJobResponse + 28, // 30: ephemerd.v1.Dispatch.StreamContainerStats:output_type -> ephemerd.v1.ContainerStatsBatch + 18, // [18:31] is the sub-list for method output_type + 5, // [5:18] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_api_v1_ephemerd_proto_init() } @@ -2106,7 +2220,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListJobsRequest); i { + switch v := v.(*SlotPool); i { case 0: return &v.state case 1: @@ -2118,7 +2232,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListJobsResponse); i { + switch v := v.(*ListJobsRequest); i { case 0: return &v.state case 1: @@ -2130,7 +2244,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetJobRequest); i { + switch v := v.(*ListJobsResponse); i { case 0: return &v.state case 1: @@ -2142,7 +2256,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Job); i { + switch v := v.(*GetJobRequest); i { case 0: return &v.state case 1: @@ -2154,7 +2268,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KillJobRequest); i { + switch v := v.(*Job); i { case 0: return &v.state case 1: @@ -2166,7 +2280,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KillJobResponse); i { + switch v := v.(*KillJobRequest); i { case 0: return &v.state case 1: @@ -2178,7 +2292,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CordonRequest); i { + switch v := v.(*KillJobResponse); i { case 0: return &v.state case 1: @@ -2190,7 +2304,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CordonResponse); i { + switch v := v.(*CordonRequest); i { case 0: return &v.state case 1: @@ -2202,7 +2316,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UncordonRequest); i { + switch v := v.(*CordonResponse); i { case 0: return &v.state case 1: @@ -2214,7 +2328,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UncordonResponse); i { + switch v := v.(*UncordonRequest); i { case 0: return &v.state case 1: @@ -2226,7 +2340,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetJobLogsRequest); i { + switch v := v.(*UncordonResponse); i { case 0: return &v.state case 1: @@ -2238,7 +2352,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogChunk); i { + switch v := v.(*GetJobLogsRequest); i { case 0: return &v.state case 1: @@ -2250,7 +2364,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpgradeRequest); i { + switch v := v.(*LogChunk); i { case 0: return &v.state case 1: @@ -2262,7 +2376,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpgradeProgress); i { + switch v := v.(*UpgradeRequest); i { case 0: return &v.state case 1: @@ -2274,7 +2388,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateJobRequest); i { + switch v := v.(*UpgradeProgress); i { case 0: return &v.state case 1: @@ -2286,7 +2400,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateJobResponse); i { + switch v := v.(*CreateJobRequest); i { case 0: return &v.state case 1: @@ -2298,7 +2412,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WaitJobRequest); i { + switch v := v.(*CreateJobResponse); i { case 0: return &v.state case 1: @@ -2310,7 +2424,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WaitJobResponse); i { + switch v := v.(*WaitJobRequest); i { case 0: return &v.state case 1: @@ -2322,7 +2436,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DestroyJobRequest); i { + switch v := v.(*WaitJobResponse); i { case 0: return &v.state case 1: @@ -2334,7 +2448,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DestroyJobResponse); i { + switch v := v.(*DestroyJobRequest); i { case 0: return &v.state case 1: @@ -2346,7 +2460,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StreamContainerStatsRequest); i { + switch v := v.(*DestroyJobResponse); i { case 0: return &v.state case 1: @@ -2358,7 +2472,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ContainerStatsBatch); i { + switch v := v.(*StreamContainerStatsRequest); i { case 0: return &v.state case 1: @@ -2370,6 +2484,18 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerStatsBatch); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_v1_ephemerd_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ContainerStats); i { case 0: return &v.state @@ -2388,7 +2514,7 @@ func file_api_v1_ephemerd_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_api_v1_ephemerd_proto_rawDesc, NumEnums: 1, - NumMessages: 28, + NumMessages: 29, NumExtensions: 0, NumServices: 2, }, diff --git a/api/v1/ephemerd.proto b/api/v1/ephemerd.proto index ebad313c..d1b9299e 100644 --- a/api/v1/ephemerd.proto +++ b/api/v1/ephemerd.proto @@ -76,6 +76,28 @@ message StatusResponse { // The upgrade flow polls this after a restart to confirm the new binary // came up on the target version. string version = 6; + // held_slots is how many concurrency slots are currently HELD across every + // dispatch pool, which is not the same thing as active_jobs. + // + // active_jobs counts jobs the scheduler is tracking. A slot is taken before + // the job is tracked and given back after it is untracked, so a job that is + // still provisioning — or one whose slot leaked — is charged here and not + // there. Issue #196: a mac held its only slot for 28 hours after a wedged + // provision and reported status ok with active_jobs 0 the whole time. + // held_slots == slot_capacity with active_jobs 0 is that failure, visible. + int32 held_slots = 7; + // slot_capacity is the total number of slots across all pools. + int32 slot_capacity = 8; + // slot_pools breaks the totals down per dispatch pool (local, linux, + // macos), so an operator can tell which pool is saturated. + repeated SlotPool slot_pools = 9; +} + +// SlotPool is one dispatch pool's concurrency-slot occupancy. +message SlotPool { + string pool = 1; + int32 held = 2; + int32 capacity = 3; } message ListJobsRequest {} diff --git a/cmd/ephemerd/commands.go b/cmd/ephemerd/commands.go index 8788bf15..b8a5fe85 100644 --- a/cmd/ephemerd/commands.go +++ b/cmd/ephemerd/commands.go @@ -69,10 +69,26 @@ func statusCmd() *cli.Command { return fmt.Errorf("status: %w", err) } + // held_slots is deliberately next to active_jobs: they differ + // exactly when a job is provisioning or a slot leaked, and a + // node whose slots are all held while active_jobs reads 0 is + // the shape of the 28-hour macOS stall in issue #196. + slots := make([]map[string]any, 0, len(resp.SlotPools)) + for _, p := range resp.SlotPools { + slots = append(slots, map[string]any{ + "pool": p.Pool, + "held": p.Held, + "capacity": p.Capacity, + }) + } + data := map[string]any{ "status": resp.Status, "active_jobs": resp.ActiveJobs, "max_concurrent": resp.MaxConcurrent, + "held_slots": resp.HeldSlots, + "slot_capacity": resp.SlotCapacity, + "slots": slots, "draining": resp.Draining, "uptime": resp.Uptime, "version": resp.Version, diff --git a/docs/cli/status.md b/docs/cli/status.md index 1c3dd04f..cd915b95 100644 --- a/docs/cli/status.md +++ b/docs/cli/status.md @@ -16,8 +16,11 @@ Returns a JSON object with the following fields: | Field | Description | |-------|-------------| | `status` | Current daemon status | -| `active_jobs` | Number of jobs currently running | +| `active_jobs` | Number of jobs currently **tracked** as running | | `max_concurrent` | Maximum concurrent jobs allowed | +| `held_slots` | Concurrency slots currently held across all dispatch pools | +| `slot_capacity` | Total slots across all dispatch pools | +| `slots` | Per-pool breakdown (`local`, `linux`, `macos`) of `held` and `capacity` | | `draining` | Whether the daemon is draining (shutting down gracefully) | | `uptime` | How long the daemon has been running | @@ -28,11 +31,34 @@ Returns a JSON object with the following fields: "status": "running", "active_jobs": 2, "max_concurrent": 4, + "held_slots": 2, + "slot_capacity": 9, + "slots": [ + { "pool": "local", "held": 2, "capacity": 4 }, + { "pool": "linux", "held": 0, "capacity": 4 }, + { "pool": "macos", "held": 0, "capacity": 1 } + ], "draining": false, "uptime": "3h42m15s" } ``` +### held_slots vs active_jobs + +They are not the same number, and the difference is diagnostic. + +A dispatch takes a concurrency slot *before* it registers the runner and gives +it back *after* the job is untracked, so a job that is still provisioning is +counted in `held_slots` and not in `active_jobs`. Briefly, that is normal. + +Sustained, it is not. A pool sitting at `held == capacity` with `active_jobs` +at 0 means the capacity is charged to something the scheduler is not tracking — +a provision that is stuck, or a slot that leaked. The daemon also logs +`waiting for a free concurrency slot` (and escalates to a suspected-leak error) +when a job blocks on such a pool, so the log and this output agree. + +The same fields are on `/healthz`. + ## Connection The command connects to the daemon's gRPC unix socket at `/ephemerd.sock`. If the daemon is not running or the socket does not exist, the command prints an error. diff --git a/pkg/scheduler/grpc.go b/pkg/scheduler/grpc.go index 40f38694..e70ee379 100644 --- a/pkg/scheduler/grpc.go +++ b/pkg/scheduler/grpc.go @@ -79,10 +79,26 @@ func (c *controlServer) Status(ctx context.Context, req *apiv1.StatusRequest) (* draining := c.sched.draining c.sched.mu.Unlock() + // Slot occupancy is reported alongside the tracked-job count, never in + // place of it. ActiveJobs is len(s.running) and a slot is held from + // before trackRunning to after untrackRunning, so a stuck provision — or + // a leaked slot — shows up here and nowhere else. See issue #196. + var pools []*apiv1.SlotPool + for _, p := range c.sched.SlotUsage() { + pools = append(pools, &apiv1.SlotPool{ + Pool: p.Pool, + Held: int32(p.Held), + Capacity: int32(p.Capacity), + }) + } + return &apiv1.StatusResponse{ Status: "ok", ActiveJobs: int32(activeJobs), MaxConcurrent: int32(c.sched.cfg.MaxConcurrent), + HeldSlots: int32(c.sched.HeldSlots()), + SlotCapacity: int32(c.sched.SlotCapacity()), + SlotPools: pools, Draining: draining, Uptime: time.Since(c.sched.startTime).Truncate(time.Second).String(), Version: c.sched.cfg.Version, diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c3f72d6a..6d63e3ec 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -181,6 +181,22 @@ const defaultOrphanGrace = 10 * time.Minute // macOS slot in minutes instead of hours. const defaultMacOSProvisionTimeout = 5 * time.Minute +// macProvisionUnwindGrace bounds how long the provisioning watchdog waits for +// a force-stopped macOS VM to tear down and its reachability wait to unwind +// before giving up on both and returning anyway. +// +// Comfortably more than darwinMacOSVM.stop's own 15s graceful-shutdown window +// plus the force-stop that follows it, so a VM that CAN be killed is always +// reaped tidily. Past that we stop caring: the whole point of the deadline was +// to get the concurrency slot back, and waiting on an unkillable VM to say so +// is what cost the fleet 28 hours of macOS CI (#196). +const macProvisionUnwindGrace = 30 * time.Second + +// macTeardownGrace bounds a routine macOS VM Stop() on any path that is +// holding a slot or a wait-goroutine. Same reasoning as +// macProvisionUnwindGrace: teardown is best-effort, capacity is not. +const macTeardownGrace = 30 * time.Second + // jobKey uniquely identifies a job across providers. Different providers // can return the same int64 job ID, so we include the provider name. type jobKey struct { @@ -260,6 +276,13 @@ type Scheduler struct { // watchdog without a real Virtualization.framework VM. newMacOSVM func(cfg vm.MacOSVMConfig, jobID string) (vm.MacOSVM, error) + // macUnwindGrace / macStopGrace bound macOS VM teardown on the paths that + // must not be delayed by it (see slots.go). Zero uses the package + // defaults; only tests set them, to compress a 30s grace into + // milliseconds. + macUnwindGrace time.Duration + macStopGrace time.Duration + // busyProbe overrides the ground-truth "is this runner executing a // job" check that vetoes orphan teardown. Nil (the default) uses // probeRunnerBusy: local container/VM/process introspection, falling @@ -1164,15 +1187,25 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent } // Acquire Linux dispatch concurrency slot (separate from local/macOS) - select { - case s.linuxSem <- struct{}{}: - case <-ctx.Done(): + slot := s.acquireSlot(ctx, s.linuxSem, "linux", log) + if slot == nil { unsee() return } + // Backstop until the wait-goroutine below takes ownership: every exit + // from this function returns the slot, including one a future edit + // forgets to release explicitly. See slots.go — a single missed release + // on the macOS path cost 28 hours of CI (#196). + handedOff := false + defer func() { + if !handedOff { + slot.release() + } + }() + if v := s.admitDispatch(key); v != dispatchAdmit { v.log(log) - <-s.linuxSem + slot.release() return } @@ -1190,7 +1223,7 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent if err != nil { log.Error("failed to claim job", "error", err, "error_class", classifyErr(err)) unsee() - <-s.linuxSem + slot.release() // Replaces the old blind time.Sleep(backoffDuration): the // sem is released FIRST so we do not hold a slot idle across // the wait, then a rate-aware jittered retry is enqueued @@ -1211,7 +1244,7 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent } unsee() cancel() - <-s.linuxSem + slot.release() // Mirror the claim-failure branch above (and the local create // path): a failed dispatch create used to drop the job forever — // webhooks are never re-delivered. Slot released first; no-op for @@ -1234,18 +1267,12 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent log.Info("Linux runner dispatched", "name", claim.RunnerName) // Wait for the job to finish in the background + handedOff = true go func() { // Same slot-release ordering as the local wait-goroutine: release // once the job is untracked, never behind the destroy. The // deferred call is only the panic/early-return backstop. - released := false - release := func() { - if !released { - released = true - <-s.linuxSem - } - } - defer release() + defer slot.release() exitCode, err := s.cfg.LinuxDispatcher.Wait(jobCtx, claim.RunnerName) if err != nil { @@ -1275,7 +1302,7 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent // Release BEFORE the destroy RPC, mirroring the local path: the // dispatched env is keyed by its unique claim.RunnerName, so a // newly admitted job cannot collide with this teardown. - release() + slot.release() if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), claim.RunnerName); err != nil { log.Warn("dispatch destroy failed", "error", err) @@ -1313,15 +1340,27 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent } // Acquire macOS VM concurrency slot (separate from Linux/local sem). - select { - case s.macSem <- struct{}{}: - case <-ctx.Done(): + slot := s.acquireSlot(ctx, s.macSem, "macos", log) + if slot == nil { unsee() return } + // Backstop until the wait-goroutine below takes ownership. This function + // is issue #196's crime scene: it had six hand-written `<-s.macSem` + // returns and one of them was unreachable, because the provisioning + // watchdog it depended on could block forever before returning. The + // deferred release makes "did every path give the slot back?" a property + // of the function rather than of the reader's attention span. + handedOff := false + defer func() { + if !handedOff { + slot.release() + } + }() + if v := s.admitDispatch(key); v != dispatchAdmit { v.log(log) - <-s.macSem + slot.release() return } @@ -1350,7 +1389,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent artifacts.Cleanup(artifactsDir, s.cfg.Log) } unsee() - <-s.macSem + slot.release() s.enqueueRetryIfEligible(ctx, event, err) return } @@ -1366,22 +1405,25 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent artifacts.Cleanup(artifactsDir, s.cfg.Log) } unsee() - <-s.macSem + slot.release() return } // Write JIT config to the shared directory before booting if err := macVM.WriteJITConfig(claim.RunnerConfig); err != nil { log.Error("failed to write JIT config", "error", err) - macVM.Stop() + // Slot first, teardown second — for the rest of this function every + // failure path releases before it touches the VM, because Stop() is + // the call that hangs (#196). + unsee() + slot.release() + stopVMBounded(macVM, s.teardownGrace(), log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } if artifactsDir != "" { artifacts.Cleanup(artifactsDir, s.cfg.Log) } - unsee() - <-s.macSem return } @@ -1392,16 +1434,16 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // Boot the VM if err := macVM.Start(jobCtx); err != nil { log.Error("failed to start macOS VM", "error", err) - macVM.Stop() + unsee() + cancel() + slot.release() + stopVMBounded(macVM, s.teardownGrace(), log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } if artifactsDir != "" { artifacts.Cleanup(artifactsDir, s.cfg.Log) } - unsee() - cancel() - <-s.macSem return } @@ -1413,16 +1455,24 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent ip, err := s.waitForMacRunnerBounded(jobCtx, macVM, log) if err != nil { log.Error("macOS VM runner not reachable", "error", err) - macVM.Stop() + // THE 28-HOUR LINE (#196). The slot goes back here, first, before + // anything that talks to the VM or to GitHub. It used to go back + // last, after macVM.Stop() — and waitForMacRunnerBounded had already + // force-stopped this VM, so this Stop() re-entered a sync.Once whose + // first call was still wedged inside Virtualization.framework and + // blocked forever. The node logged "force-stopping VM to reclaim the + // slot", never reclaimed it, and starved every subsequent macOS job + // while reporting status: ok / active_jobs: 0. + unsee() + cancel() + slot.release() + stopVMBounded(macVM, s.teardownGrace(), log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } if artifactsDir != "" { artifacts.Cleanup(artifactsDir, s.cfg.Log) } - unsee() - cancel() - <-s.macSem return } @@ -1441,8 +1491,11 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent log.Info("macOS VM runner ready", "name", claim.RunnerName, "ip", ip) // Wait for the job to finish in the background + handedOff = true go func() { - defer func() { <-s.macSem }() + // Backstop only. The release that matters happens the moment the job + // is untracked, below. + defer slot.release() exitCode, err := macVM.Wait(jobCtx) if err != nil { @@ -1457,14 +1510,32 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent log.Info("macOS VM exited", "exit_code", exitCode) } - // Clean up + // Clean up. The entry may already be gone if a completed event tore + // this VM down by name (see handleCompleted). s.mu.Lock() rj, exists := s.running[key] if exists { s.untrackRunningLocked(key, rj) - s.mu.Unlock() + } + s.mu.Unlock() + + // Release BEFORE teardown, exactly as the local and Linux + // wait-goroutines do (PR #190). The job is untracked, so its demand + // is gone; everything below can block for a long time and none of it + // may delay the next macOS job. macVM.Stop() waits up to 15s for a + // graceful guest shutdown before forcing (and on the #196 node the + // force did not return either), and ReleaseJob is a GitHub API call + // on context.Background() with no deadline at all. This is safe + // against a newly admitted job: every VM, its clone, and its shared + // directory are keyed by job ID, so a fresh dispatch cannot touch + // anything this teardown still owns. + slot.release() + + if exists { metrics.JobsActive.Dec() - rj.macosVM.Stop() + if rj.macosVM != nil { + stopVMBounded(rj.macosVM, s.teardownGrace(), log) + } if rj.artifactsDir != "" { artifacts.Cleanup(rj.artifactsDir, s.cfg.Log) } @@ -1473,8 +1544,6 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent log.Debug("deregister runner after macOS VM cleanup", "error", err) } } - } else { - s.mu.Unlock() } // Self-heal: re-provision if this VM's dispatched job never ran. @@ -1496,10 +1565,26 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // // The only reliable way to interrupt a blocked SSH session call is to drop the // connection: force-stopping the VM tears down the guest, the SSH read errors -// out, and WaitForRunner unwinds. On timeout we do exactly that, then wait for -// the wait goroutine to return so there is no leak, and report a failure. The -// caller's normal error path then releases the slot and deregisters the runner -// — cleanup runs even though the wait had been stuck. +// out, and WaitForRunner unwinds. On timeout we do exactly that and report a +// failure; the caller's error path releases the slot and deregisters the +// runner, so cleanup runs even though the wait had been stuck. +// +// ISSUE #196 — why the teardown is now BOUNDED. The first version of this +// function force-stopped the VM and then did a bare `<-resCh`, waiting for the +// wait goroutine to unwind "so there is no leak". That assumed killing the VM +// always unblocks the guest wait. On the fleet's mac it did not: the VM's own +// stop path timed out ("macOS VM did not stop gracefully, forcing") and the +// wait never returned, so this function — which had just logged that it was +// force-stopping the VM TO RECLAIM THE SLOT — parked on that receive for 28 +// hours, still holding the slot. The job had never been tracked, so +// active_jobs stayed 0, /healthz stayed 200, and five macOS jobs queued behind +// it aged out at GitHub's 24-hour limit without ever running. +// +// A wedged guest wait is already a leaked goroutine that nothing in this +// process can kill. resCh is buffered so that goroutine can always complete +// its send and exit if it ever comes back. What must NOT happen is turning +// that leaked goroutine into a leaked concurrency slot: teardown gets a +// bounded grace period to be tidy, and then we return regardless. func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSVM, log *slog.Logger) (string, error) { timeout := s.cfg.MacOSProvisionTimeout if timeout <= 0 { @@ -1510,8 +1595,13 @@ func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSV ip string err error } + // Buffered: the wait goroutine must always be able to finish its send and + // exit, even long after we have stopped listening. Nothing below may + // depend on it ever getting that far. resCh := make(chan result, 1) + unwound := make(chan struct{}) go func() { + defer close(unwound) ip, err := macVM.WaitForRunner(ctx) resCh <- result{ip: ip, err: err} }() @@ -1524,16 +1614,62 @@ func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSV return r.ip, r.err case <-timer.C: log.Error("macOS VM stuck in provisioning past deadline; force-stopping VM to reclaim the slot", "timeout", timeout) - macVM.Stop() // drops the guest connection so WaitForRunner unblocks - <-resCh // wait goroutine unwinds now that the VM is gone + s.abandonMacProvision(macVM, unwound, log) return "", fmt.Errorf("timed out after %s waiting for macOS VM runner to become reachable", timeout) case <-ctx.Done(): - macVM.Stop() - <-resCh + s.abandonMacProvision(macVM, unwound, log) return "", ctx.Err() } } +// abandonMacProvision force-stops a macOS VM whose provisioning was given up +// on and gives the teardown a bounded chance to finish before returning. +// +// Stop() runs on its own goroutine for two reasons. The obvious one is that +// Virtualization.framework's stop can itself hang. The subtle one is +// sync.Once: darwinMacOSVM.Stop is `stopOnce.Do(m.stop)`, so a SECOND caller +// does not return early — it blocks until the first invocation returns. The +// caller's error path stops the VM again as part of its normal cleanup, and if +// this teardown were still wedged in the Once, that "cleanup" call would hang +// on a path that is holding a concurrency slot. Off-thread here plus +// stopVMBounded there means neither can pin the slot. +func (s *Scheduler) abandonMacProvision(macVM vm.MacOSVM, unwound <-chan struct{}, log *slog.Logger) { + // Dropping the guest is what unblocks a wedged SSH session call. + stopped := stopVMAsync(macVM) + if !awaitUnwind(stopped, unwound, s.provisionUnwindGrace()) { + log.Error("macOS VM teardown did not finish after the force-stop; abandoning it and reclaiming the concurrency slot anyway", + "grace", s.provisionUnwindGrace(), + "detail", "a wedged Vz stop or a wedged guest wait must never pin the macOS slot — see issue #196") + } +} + +// stopVMAsync stops macVM on its own goroutine and returns a channel closed +// when Stop returns. If Stop never returns, that goroutine is leaked — which +// is the correct trade: the alternative is leaking the concurrency slot, and +// a leaked slot takes the node's whole macOS capacity with it. +func stopVMAsync(macVM vm.MacOSVM) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + macVM.Stop() + }() + return done +} + +// stopVMBounded force-stops a VM without letting the teardown pin the caller. +// Used by every macOS cleanup path that runs while a slot could still be held +// or a job could still be waiting on this goroutine. +func stopVMBounded(macVM vm.MacOSVM, grace time.Duration, log *slog.Logger) { + timer := time.NewTimer(grace) + defer timer.Stop() + select { + case <-stopVMAsync(macVM): + case <-timer.C: + log.Warn("macOS VM stop did not return within the grace period; continuing without it", + "grace", grace) + } +} + // handleLocalJob provisions a runner using the local containerd Runtime. func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent) { jobID := event.JobID @@ -1549,15 +1685,23 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent } // Acquire concurrency slot - select { - case s.sem <- struct{}{}: - case <-ctx.Done(): + slot := s.acquireSlot(ctx, s.sem, "local", log) + if slot == nil { unsee() return } + // Backstop until the wait-goroutine below takes ownership; see the + // matching comment in handleLinuxJob and slots.go. + handedOff := false + defer func() { + if !handedOff { + slot.release() + } + }() + if v := s.admitDispatch(key); v != dispatchAdmit { v.log(log) - <-s.sem + slot.release() return } @@ -1622,7 +1766,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent artifacts.Cleanup(artifactsDir, s.cfg.Log) } unsee() - <-s.sem + slot.release() // Replaces the old blind 5s sleep with a rate-aware jittered // retry. No-op when Retry is disabled or the error is not // retryable. @@ -1666,7 +1810,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent } unsee() cancel() - <-s.sem + slot.release() // Mirror the claim-failure branch above: without this a job whose // env create failed was dropped forever (GitHub never re-delivers // the webhook) — on metal that was 1h43m of a job sitting queued @@ -1692,6 +1836,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent log.Info("runner environment ready", "name", claim.RunnerName) // Wait for the job to finish in the background + handedOff = true go func() { // Slot release is decoupled from teardown. It used to hang off a // bare defer, which meant the slot came back only after Destroy @@ -1699,14 +1844,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent // MaxConcurrent slot for its whole hang. The slot is released as // soon as the job is untracked below; the deferred call is only // the panic/early-return backstop. - released := false - release := func() { - if !released { - released = true - <-s.sem - } - } - defer release() + defer slot.release() exitCode, err := s.cfg.Runtime.Wait(jobCtx, env) if err != nil { @@ -1740,7 +1878,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent // runner dir, netns/HCN endpoint, dind socket all derive from it — // so a freshly admitted job cannot touch anything this teardown // still owns. - release() + slot.release() if exists { metrics.JobsActive.Dec() @@ -2025,10 +2163,24 @@ func (s *Scheduler) handleHealthz(w http.ResponseWriter, r *http.Request) { draining := s.draining s.mu.Unlock() + // held_slots/slot_capacity/slots are ADDITIVE — active_jobs and + // max_concurrent keep their exact meaning for anything already scraping + // this endpoint. + // + // active_jobs counts TRACKED jobs (len(s.running)). A job that is + // provisioning, or one whose slot leaked, is not tracked, so a node with + // every slot held reported active_jobs: 0 and looked idle — that is how + // the #196 macOS outage stayed invisible for 28 hours while /healthz + // returned 200. held_slots is read straight off the semaphores and so + // counts the capacity that is actually spoken for. held == capacity with + // active_jobs 0, sustained, means a stuck provision or a leak. status := map[string]any{ "status": "ok", "active_jobs": activeJobs, "max_concurrent": s.cfg.MaxConcurrent, + "held_slots": s.HeldSlots(), + "slot_capacity": s.SlotCapacity(), + "slots": s.SlotUsage(), "draining": draining, "uptime": time.Since(s.startTime).String(), } diff --git a/pkg/scheduler/slots.go b/pkg/scheduler/slots.go new file mode 100644 index 00000000..805435d1 --- /dev/null +++ b/pkg/scheduler/slots.go @@ -0,0 +1,256 @@ +package scheduler + +import ( + "context" + "log/slog" + "sync" + "time" +) + +// Concurrency slots. +// +// Every dispatch path holds one slot for the whole life of the job it +// provisions: s.sem (local runners), s.linuxSem (Linux dispatch to the VM +// worker), s.macSem (macOS VMs). The buffered channel IS the accounting — +// len() is how many slots are held, cap() is the pool size — so nothing here +// keeps a parallel counter that could drift from the semaphore it describes. +// +// Three rules, all of them written after an outage: +// +// 1. A slot is released the moment the job it represents stops being demand, +// i.e. as soon as the job is untracked. NEVER "once teardown finishes": +// teardown is precisely the thing that hangs. (PR #190, local + Linux +// paths; issue #196 extended it to the macOS wait-goroutine, which held +// the slot across macVM.Stop() and a ReleaseJob call on +// context.Background().) +// +// 2. Release is idempotent and has a deferred backstop, so an error return +// added later cannot forget it and a panic cannot skip it. +// +// 3. Nothing that can block forever may sit between the acquire and the +// release on a failure path. This is the whole of issue #196: the macOS +// provisioning watchdog force-stopped the wedged VM "to reclaim the slot" +// and then blocked FOREVER waiting for the wedged wait goroutine to +// unwind, so the release it was on its way to never ran. The node held +// the sole macOS slot for 28 hours while reporting status: ok and +// active_jobs: 0 (the job had never been tracked), and every macOS job +// queued in that window aged out at GitHub's 24h timeout without running. + +// These are vars rather than consts only so tests can compress the timeline; +// nothing outside tests reassigns them. +var ( + // slotWaitLogAfter is how long a dispatch must sit blocked on a full pool + // before it says so. Short enough that a genuinely stuck pool is visible + // within a minute of the next job arriving, long enough that ordinary + // queueing behind a running job on a max_concurrent = 1 node does not + // narrate itself. + slotWaitLogAfter = 30 * time.Second + + // slotWaitLogEvery re-states a still-blocked wait on this interval. A job + // can legitimately wait hours behind a long build, so this is quiet; the + // point is that a pool which is stuck forever keeps saying so instead of + // going silent after one line. Issue #196's log had NOTHING in it for 28 + // hours — five macOS jobs blocked on the leaked slot and not one of them + // left a trace. + slotWaitLogEvery = 5 * time.Minute + + // slotLeakSuspectAfter is when a blocked wait escalates from "busy" to + // "broken". A slot that is held while the scheduler tracks NO running job + // is either a provision in flight (bounded: minutes) or a leak (forever), + // so waiting this long against a pool that nothing is tracked against is + // reported at Error. + slotLeakSuspectAfter = 15 * time.Minute +) + +// slotToken is a one-shot handle to an acquired concurrency slot. +// +// It replaces the bare `<-s.macSem` that each of handleMacOSJob's six error +// returns had to remember on its own. The type exists so that "did this path +// release?" is answered by construction (a deferred backstop) rather than by +// reading every return statement. +type slotToken struct { + sem chan struct{} + once sync.Once +} + +// release returns the slot to its pool. Safe to call any number of times from +// any goroutine: only the first call drains the semaphore, so an explicit +// early release (the one that matters — it is what lets the next job in) can +// coexist with a deferred backstop without under-draining the channel and +// handing out a slot somebody else still holds. +func (t *slotToken) release() { + if t == nil { + return + } + t.once.Do(func() { <-t.sem }) +} + +// acquireSlot takes a slot from sem, blocking until one is free or ctx is +// done. It returns nil when ctx ended first; the caller must not provision. +// +// The only behavioural difference from the raw `select { case sem <- ... }` +// it replaces is that a wait which actually BLOCKS becomes visible in the log. +// A saturated pool and a leaked pool look identical from outside the process, +// and until issue #196 both looked identical from inside it too: the daemon +// logged nothing at all while five macOS jobs piled up behind a slot that +// would never come back. +func (s *Scheduler) acquireSlot(ctx context.Context, sem chan struct{}, pool string, log *slog.Logger) *slotToken { + // Fast path: a free slot logs nothing, which is the overwhelmingly + // common case and must stay allocation- and timer-free. + select { + case sem <- struct{}{}: + return &slotToken{sem: sem} + case <-ctx.Done(): + return nil + default: + } + + start := time.Now() + timer := time.NewTimer(slotWaitLogAfter) + defer timer.Stop() + for { + select { + case sem <- struct{}{}: + log.Info("acquired concurrency slot after waiting", + "pool", pool, "waited", time.Since(start).Truncate(time.Second), + "held", len(sem), "capacity", cap(sem)) + return &slotToken{sem: sem} + case <-ctx.Done(): + return nil + case <-timer.C: + held, tracked, waited := len(sem), s.ActiveJobs(), time.Since(start) + level, msg := slotWaitSeverity(waited, held, tracked) + log.Log(ctx, level, msg, + "pool", pool, + "held", held, + "capacity", cap(sem), + "tracked_jobs", tracked, + "waited", waited.Truncate(time.Second)) + timer.Reset(slotWaitLogEvery) + } + } +} + +// slotWaitSeverity decides how loudly a blocked dispatch complains. +// +// This is the leak watchdog, and it lives here rather than in a periodic +// sweeper on purpose: a held slot is only a problem when something is waiting +// for it, and a waiting dispatch is exactly this code path. It therefore +// costs nothing when the node is idle and cannot produce a false alarm on a +// node that simply has no macOS work. +// +// held > 0 with trackedJobs == 0 means the slot is charged to a job the +// scheduler is not tracking. That is legitimate for the length of a +// provision (bounded by MacOSProvisionTimeout, minutes) and illegitimate +// forever after — it is the exact signature of the 28-hour macOS outage, +// where `ephemerd status` reported active_jobs: 0 on a node whose only slot +// had been held since the previous afternoon. Pure so the escalation rule is +// testable without a real pool or a real wait. +func slotWaitSeverity(waited time.Duration, held, trackedJobs int) (slog.Level, string) { + if held > 0 && trackedJobs == 0 && waited >= slotLeakSuspectAfter { + return slog.LevelError, "blocked on a concurrency slot that no tracked job accounts for — suspected slot leak" + } + return slog.LevelWarn, "waiting for a free concurrency slot" +} + +// SlotUsage reports concurrency-slot occupancy for every dispatch pool. +// +// Held is read from the semaphore itself, so it counts slots that are charged +// to an UNTRACKED job — a provision still in flight, or a leak. That is the +// whole point: ActiveJobs() counts len(s.running) and so reported 0 on the +// node that had been wedged for 28 hours. A pool at held == capacity with +// nothing tracked against it is the signal that was missing. +func (s *Scheduler) SlotUsage() []SlotStats { + return []SlotStats{ + {Pool: "local", Held: len(s.sem), Capacity: cap(s.sem)}, + {Pool: "linux", Held: len(s.linuxSem), Capacity: cap(s.linuxSem)}, + {Pool: "macos", Held: len(s.macSem), Capacity: cap(s.macSem)}, + } +} + +// SlotStats is one dispatch pool's occupancy. +type SlotStats struct { + Pool string `json:"pool"` + Held int `json:"held"` + Capacity int `json:"capacity"` +} + +// HeldSlots is the total number of concurrency slots currently held across +// all pools, tracked or not. +func (s *Scheduler) HeldSlots() int { + total := 0 + for _, p := range s.SlotUsage() { + total += p.Held + } + return total +} + +// SlotCapacity is the total number of concurrency slots across all pools. +func (s *Scheduler) SlotCapacity() int { + total := 0 + for _, p := range s.SlotUsage() { + total += p.Capacity + } + return total +} + +// provisionUnwindGrace and teardownGrace are the effective bounds on macOS VM +// teardown. They read from the scheduler so tests can compress them, and fall +// back to the package defaults for a zero-valued Scheduler. +func (s *Scheduler) provisionUnwindGrace() time.Duration { + if s.macUnwindGrace > 0 { + return s.macUnwindGrace + } + return macProvisionUnwindGrace +} + +func (s *Scheduler) teardownGrace() time.Duration { + if s.macStopGrace > 0 { + return s.macStopGrace + } + return macTeardownGrace +} + +// awaitUnwind waits for an abandoned provisioning attempt's teardown +// (stopped) and its wait goroutine (unwound) to finish, but only for grace. +// It reports whether both arrived in time. +// +// This bound is the issue #196 fix in one function. The previous code did the +// moral equivalent of `<-resCh` with no deadline, on the reasoning that +// force-stopping the VM drops the guest connection and therefore unblocks the +// reachability wait. On metal that reasoning failed: the VM's own stop path +// timed out ("macOS VM did not stop gracefully, forcing") and the guest wait +// never returned, so the dispatch goroutine parked there permanently — still +// holding the macOS slot it had just logged that it was reclaiming. +// +// A wedged guest wait is already a leaked goroutine and nothing in this +// process can kill it. Blocking on it converts that leaked goroutine into a +// leaked concurrency slot, which is the difference between a wasted job and a +// dead node. So: give teardown a bounded chance to be tidy, then walk away. +func awaitUnwind(stopped, unwound <-chan struct{}, grace time.Duration) bool { + timer := time.NewTimer(grace) + defer timer.Stop() + for stopped != nil || unwound != nil { + // Prefer an already-finished signal over the deadline: with both + // ready, a plain select picks at random and would report a clean + // unwind as a timeout. + select { + case <-stopped: + stopped = nil + continue + case <-unwound: + unwound = nil + continue + default: + } + select { + case <-stopped: + stopped = nil + case <-unwound: + unwound = nil + case <-timer.C: + return false + } + } + return true +} diff --git a/pkg/scheduler/slots_test.go b/pkg/scheduler/slots_test.go new file mode 100644 index 00000000..59299b4f --- /dev/null +++ b/pkg/scheduler/slots_test.go @@ -0,0 +1,598 @@ +package scheduler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + apiv1 "github.com/ephpm/ephemerd/api/v1" + "github.com/ephpm/ephemerd/pkg/providers" + "github.com/ephpm/ephemerd/pkg/vm" +) + +// wedgedMacVM is the macOS VM from issue #196 and the reason +// blockingMacVM (macos_provision_test.go) was not enough: its WaitForRunner +// never returns AT ALL, not even when the VM is stopped, and its Stop() never +// returns either. +// +// That is what the fleet's mac actually did. The daemon logged +// +// ERROR macOS VM stuck in provisioning past deadline; force-stopping VM to reclaim the slot +// WARN macOS VM did not stop gracefully, forcing +// INFO macOS VM destroyed +// +// and then nothing for 28 hours: the guest wait never unwound, so the +// watchdog's `<-resCh` never returned, so the "reclaim the slot" it had just +// announced never happened. Every assertion below is about the dispatch path +// surviving a VM it cannot kill. +type wedgedMacVM struct { + stopCalls atomic.Int32 + // stopReturns, when closed, lets Stop() return. Left open to model a + // teardown wedged inside Virtualization.framework. + stopReturns chan struct{} +} + +// newWedgedMacVM registers cleanup that lets the wedged Stop() goroutines +// finish, so the test binary does not accumulate them across subtests. +func newWedgedMacVM(t *testing.T) *wedgedMacVM { + t.Helper() + m := &wedgedMacVM{stopReturns: make(chan struct{})} + t.Cleanup(func() { close(m.stopReturns) }) + return m +} + +func (m *wedgedMacVM) WriteJITConfig(string) error { return nil } +func (m *wedgedMacVM) Start(_ context.Context) error { return nil } +func (m *wedgedMacVM) RunnerAddress() string { return "" } +func (m *wedgedMacVM) Wait(context.Context) (int, error) { + select {} //nolint:staticcheck // models a VM whose wait never returns +} + +// WaitForRunner never returns — it ignores ctx exactly as a blocked +// golang.org/x/crypto/ssh session call does. +func (m *wedgedMacVM) WaitForRunner(context.Context) (string, error) { + select {} //nolint:staticcheck // models the wedged guest wait +} + +func (m *wedgedMacVM) Stop() { + m.stopCalls.Add(1) + <-m.stopReturns +} + +// newSlotTestScheduler builds a scheduler with a single macOS slot and +// millisecond teardown bounds, so the 30-second production graces do not turn +// every assertion into a 30-second test. +func newSlotTestScheduler(t *testing.T, prov providers.Provider, mac vm.MacOSVM) *Scheduler { + t.Helper() + s := New(Config{ + Providers: []providers.Provider{prov}, + MacOSVMConfig: &vm.MacOSVMConfig{}, + MaxMacOSVMs: 1, // the incident's max_concurrent = 1 + MacOSProvisionTimeout: 50 * time.Millisecond, + Log: quietLogger(), + }) + s.macUnwindGrace = 50 * time.Millisecond + s.macStopGrace = 50 * time.Millisecond + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { return mac, nil } + return s +} + +func macEvent(prov providers.Provider, jobID int64) providers.JobEvent { + return providers.JobEvent{ + Provider: prov, + Action: "queued", + Repo: "repo", + JobID: jobID, + Labels: []string{"self-hosted", "macos"}, + } +} + +// runMacDispatch runs handleMacOSJob and fails (rather than hanging) if it has +// not returned by deadline. A regression here must be a FAILURE, not a test +// that sits there until the package timeout. +func runMacDispatch(t *testing.T, s *Scheduler, event providers.JobEvent, deadline time.Duration) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + s.handleMacOSJob(context.Background(), event) + }() + select { + case <-done: + case <-time.After(deadline): + t.Fatalf("handleMacOSJob did not return within %s: the dispatch path is blocked on VM teardown while holding a slot", deadline) + } +} + +// assertSlotFree asserts the pool has a slot available right now. +func assertSlotFree(t *testing.T, sem chan struct{}, what string) { + t.Helper() + select { + case sem <- struct{}{}: + <-sem + default: + t.Fatalf("%s: no free slot (held=%d capacity=%d)", what, len(sem), cap(sem)) + } +} + +// TestHandleMacOSJob_UnkillableVMStillFreesSlot is the issue #196 regression +// test. The VM cannot be killed and its reachability wait never unwinds; the +// dispatch must still return and give the macOS slot back, because otherwise +// the node silently serves zero macOS jobs forever while reporting healthy. +// +// Without the fix this test FAILS on the deadline in runMacDispatch: +// waitForMacRunnerBounded parks on `<-resCh` after force-stopping the VM and +// handleMacOSJob never reaches its release. +func TestHandleMacOSJob_UnkillableVMStillFreesSlot(t *testing.T) { + prov := newMockProvider("mac-wedged") + defer func() { _ = prov.Stop(context.Background()) }() + + macVM := newWedgedMacVM(t) + s := newSlotTestScheduler(t, prov, macVM) + + runMacDispatch(t, s, macEvent(prov, 100817345293), 5*time.Second) + + if macVM.stopCalls.Load() == 0 { + t.Error("expected the wedged macOS VM to be force-stopped on the provisioning deadline (#178 behaviour must not regress)") + } + assertSlotFree(t, s.macSem, "after a provisioning deadline on an unkillable VM") + if held := len(s.macSem); held != 0 { + t.Errorf("macSem holds %d slots after the dispatch returned, want 0", held) + } + + // The next macOS job must be able to run. This is the assertion the + // production node failed for 28 hours. + second := newWedgedMacVM(t) + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { return second, nil } + runMacDispatch(t, s, macEvent(prov, 100817345294), 5*time.Second) + if second.stopCalls.Load() == 0 { + t.Error("the job queued behind the wedged one never reached provisioning: the slot was never actually reclaimed") + } + + // Nothing may be left tracked, and the ghost JIT runners must be gone. + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d after two abandoned provisions, want 0", n) + } + prov.mu.Lock() + releases := len(prov.releases) + prov.mu.Unlock() + if releases != 2 { + t.Errorf("ReleaseJob called %d times, want 2 (one ghost runner per abandoned provision)", releases) + } +} + +// TestHandleMacOSJob_ReleasesExactlyOncePerOutcome walks the distinct exits +// from the macOS dispatch and asserts each one leaves the pool exactly as it +// found it. "Exactly once" matters in both directions: a missed release +// starves the node, a double release hands out a slot somebody still holds. +func TestHandleMacOSJob_ReleasesExactlyOncePerOutcome(t *testing.T) { + prov := newMockProvider("mac-outcomes") + defer func() { _ = prov.Stop(context.Background()) }() + + t.Run("provisioning deadline", func(t *testing.T) { + s := newSlotTestScheduler(t, prov, newWedgedMacVM(t)) + runMacDispatch(t, s, macEvent(prov, 1), 5*time.Second) + if held := len(s.macSem); held != 0 { + t.Fatalf("held=%d after the deadline path, want 0", held) + } + // Capacity must be intact, i.e. the slot came back once and not twice. + assertCapacityIntact(t, s.macSem) + }) + + t.Run("vm create failure", func(t *testing.T) { + s := newSlotTestScheduler(t, prov, nil) + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { + return nil, errFakeCreate + } + runMacDispatch(t, s, macEvent(prov, 2), 5*time.Second) + if held := len(s.macSem); held != 0 { + t.Fatalf("held=%d after a VM create failure, want 0", held) + } + assertCapacityIntact(t, s.macSem) + }) + + t.Run("context cancelled before a slot is free", func(t *testing.T) { + s := newSlotTestScheduler(t, prov, newWedgedMacVM(t)) + // Fill the sole slot so the dispatch has to wait for it. + s.macSem <- struct{}{} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + s.handleMacOSJob(ctx, macEvent(prov, 3)) + }() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("handleMacOSJob did not return after its context was cancelled") + } + // It must not have taken a second slot, and must not have released + // the one it never acquired. + if held := len(s.macSem); held != 1 { + t.Fatalf("held=%d after a cancelled dispatch, want 1 (the pre-existing holder)", held) + } + <-s.macSem + assertCapacityIntact(t, s.macSem) + }) + + t.Run("successful job", func(t *testing.T) { + var stops atomic.Int32 + s := newSlotTestScheduler(t, prov, &fastMacVM{ip: "192.168.64.5", stops: &stops}) + runMacDispatch(t, s, macEvent(prov, 4), 5*time.Second) + // The wait-goroutine owns the slot now; fastMacVM.Wait returns + // immediately, so it comes back promptly. + waitForSlotFree(t, s.macSem, 5*time.Second) + assertCapacityIntact(t, s.macSem) + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d after the job finished, want 0", n) + } + }) +} + +var errFakeCreate = errors.New("vz: no macOS image installed") + +// assertCapacityIntact proves the pool can still be filled to capacity and no +// further — i.e. releases and acquires balanced exactly. +func assertCapacityIntact(t *testing.T, sem chan struct{}) { + t.Helper() + n := 0 + for i := 0; i < cap(sem); i++ { + select { + case sem <- struct{}{}: + n++ + default: + } + } + if n != cap(sem) { + t.Errorf("pool accepted %d of %d slots; a release was missed", n, cap(sem)) + } + select { + case sem <- struct{}{}: + t.Error("pool accepted more than its capacity; a slot was released twice") + default: + } + for i := 0; i < n; i++ { + <-sem + } +} + +func waitForSlotFree(t *testing.T, sem chan struct{}, within time.Duration) { + t.Helper() + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if len(sem) == 0 { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatalf("slot was not released within %s (held=%d)", within, len(sem)) +} + +// TestSlotTokenReleaseIsIdempotent covers the property the whole fix leans on: +// the explicit release that lets the next job in, and the deferred backstop +// that catches a forgotten path, are the same release. +func TestSlotTokenReleaseIsIdempotent(t *testing.T) { + sem := make(chan struct{}, 2) + sem <- struct{}{} + tok := &slotToken{sem: sem} + + tok.release() + if len(sem) != 0 { + t.Fatalf("held=%d after release, want 0", len(sem)) + } + + // A second holder takes the freed slot; further releases of the spent + // token must not steal it. + sem <- struct{}{} + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + tok.release() + }() + } + wg.Wait() + if len(sem) != 1 { + t.Errorf("held=%d after repeated releases of a spent token, want 1", len(sem)) + } + + // A nil token is a no-op, so a path that never acquired can still defer. + var nilTok *slotToken + nilTok.release() +} + +// TestAcquireSlotBlocksAndLogs checks the observability gap that let #196 hide: +// a dispatch parked on a full pool used to log nothing whatsoever. +func TestAcquireSlotBlocksAndLogs(t *testing.T) { + after, every, leak := slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter + defer func() { slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter = after, every, leak }() + slotWaitLogAfter = 10 * time.Millisecond + slotWaitLogEvery = 10 * time.Millisecond + slotLeakSuspectAfter = time.Hour // stay at Warn for this test + + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + s.macSem <- struct{}{} // pool full + + got := make(chan *slotToken, 1) + go func() { got <- s.acquireSlot(context.Background(), s.macSem, "macos", log) }() + + // Give the waiter time to block and log at least once, then free the slot. + time.Sleep(60 * time.Millisecond) + <-s.macSem + + select { + case tok := <-got: + if tok == nil { + t.Fatal("acquireSlot returned nil despite a slot becoming free") + } + tok.release() + case <-time.After(5 * time.Second): + t.Fatal("acquireSlot never returned after the slot was freed") + } + + out := buf.String() + if !strings.Contains(out, "waiting for a free concurrency slot") { + t.Errorf("blocked acquire logged nothing about waiting; got:\n%s", out) + } + if !strings.Contains(out, "pool=macos") { + t.Errorf("wait log did not name the pool; got:\n%s", out) + } + if !strings.Contains(out, "capacity=1") { + t.Errorf("wait log did not report the pool capacity; got:\n%s", out) + } +} + +// TestAcquireSlotFastPathIsSilent: the common case must not narrate itself. +func TestAcquireSlotFastPathIsSilent(t *testing.T) { + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + tok := s.acquireSlot(context.Background(), s.macSem, "macos", log) + if tok == nil { + t.Fatal("acquireSlot returned nil on an empty pool") + } + tok.release() + if buf.Len() != 0 { + t.Errorf("uncontended acquire logged %q, want silence", buf.String()) + } +} + +// TestAcquireSlotHonoursContext: a cancelled dispatch takes no slot. +func TestAcquireSlotHonoursContext(t *testing.T) { + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + s.macSem <- struct{}{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if tok := s.acquireSlot(ctx, s.macSem, "macos", quietLogger()); tok != nil { + t.Error("acquireSlot handed out a slot under a cancelled context") + } + if len(s.macSem) != 1 { + t.Errorf("held=%d, want 1 (the cancelled waiter must not have taken one)", len(s.macSem)) + } +} + +// TestSlotWaitSeverity pins the leak watchdog's escalation rule: held slots +// with nothing tracked, waited on long enough, is an Error — that combination +// is exactly what `ephemerd status` showed on the wedged mac. +func TestSlotWaitSeverity(t *testing.T) { + tests := []struct { + name string + waited time.Duration + held int + tracked int + wantLevel slog.Level + wantSuspect bool + }{ + {"busy node, short wait", time.Minute, 1, 1, slog.LevelWarn, false}, + {"busy node, long wait", 2 * time.Hour, 1, 1, slog.LevelWarn, false}, + {"untracked holder, short wait", time.Minute, 1, 0, slog.LevelWarn, false}, + {"untracked holder, long wait", slotLeakSuspectAfter, 1, 0, slog.LevelError, true}, + {"nothing held, long wait", 2 * time.Hour, 0, 0, slog.LevelWarn, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + level, msg := slotWaitSeverity(tc.waited, tc.held, tc.tracked) + if level != tc.wantLevel { + t.Errorf("level = %v, want %v", level, tc.wantLevel) + } + if suspect := strings.Contains(msg, "suspected slot leak"); suspect != tc.wantSuspect { + t.Errorf("message %q: leak-suspected = %v, want %v", msg, suspect, tc.wantSuspect) + } + }) + } +} + +// TestAwaitUnwind covers the bound that replaced the unbounded `<-resCh`. +func TestAwaitUnwind(t *testing.T) { + closed := func() chan struct{} { + c := make(chan struct{}) + close(c) + return c + } + + t.Run("both already finished", func(t *testing.T) { + if !awaitUnwind(closed(), closed(), time.Millisecond) { + t.Error("awaitUnwind reported a timeout when both signals were already done") + } + }) + + t.Run("both finish in time", func(t *testing.T) { + stopped, unwound := make(chan struct{}), make(chan struct{}) + go func() { time.Sleep(5 * time.Millisecond); close(stopped) }() + go func() { time.Sleep(10 * time.Millisecond); close(unwound) }() + if !awaitUnwind(stopped, unwound, 5*time.Second) { + t.Error("awaitUnwind reported a timeout for a teardown that completed") + } + }) + + t.Run("wait never unwinds", func(t *testing.T) { + start := time.Now() + if awaitUnwind(closed(), make(chan struct{}), 20*time.Millisecond) { + t.Error("awaitUnwind claimed a clean unwind for a wait that never returned") + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("awaitUnwind took %s to give up; it must be bounded by grace", elapsed) + } + }) + + t.Run("stop never returns", func(t *testing.T) { + if awaitUnwind(make(chan struct{}), closed(), 20*time.Millisecond) { + t.Error("awaitUnwind claimed a clean unwind while Stop was still wedged") + } + }) +} + +// TestStopVMBoundedDoesNotHangOnAWedgedStop: teardown is best-effort, +// capacity is not. +func TestStopVMBoundedDoesNotHangOnAWedgedStop(t *testing.T) { + macVM := newWedgedMacVM(t) + done := make(chan struct{}) + go func() { + defer close(done) + stopVMBounded(macVM, 20*time.Millisecond, quietLogger()) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("stopVMBounded blocked on a Stop() that never returns") + } + if macVM.stopCalls.Load() != 1 { + t.Errorf("Stop called %d times, want 1", macVM.stopCalls.Load()) + } +} + +// TestSlotUsageReportsHeldSlots is the reporting gap from the issue: a node +// with every slot held must not look idle. +func TestSlotUsageReportsHeldSlots(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + + if got := s.HeldSlots(); got != 0 { + t.Errorf("HeldSlots() = %d on a fresh scheduler, want 0", got) + } + if got := s.SlotCapacity(); got != 5 { // local 2 + linux 2 + macos 1 + t.Errorf("SlotCapacity() = %d, want 5", got) + } + + // Hold the macOS slot without tracking a job — the #196 state exactly. + s.macSem <- struct{}{} + defer func() { <-s.macSem }() + + if got := s.ActiveJobs(); got != 0 { + t.Fatalf("ActiveJobs() = %d, want 0 (an untracked holder is the whole point)", got) + } + if got := s.HeldSlots(); got != 1 { + t.Errorf("HeldSlots() = %d while the macOS slot is held, want 1", got) + } + + byPool := map[string]SlotStats{} + for _, p := range s.SlotUsage() { + byPool[p.Pool] = p + } + if mac := byPool["macos"]; mac.Held != 1 || mac.Capacity != 1 { + t.Errorf("macos pool = %+v, want held 1 capacity 1", mac) + } + if local := byPool["local"]; local.Held != 0 || local.Capacity != 2 { + t.Errorf("local pool = %+v, want held 0 capacity 2", local) + } +} + +// TestStatusRPCExposesHeldSlots is the same check for the control socket, +// which is what `ephemerd status` and mayfly actually read. +func TestStatusRPCExposesHeldSlots(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + cs := &controlServer{sched: s, log: quietLogger()} + + s.macSem <- struct{}{} + defer func() { <-s.macSem }() + + resp, err := cs.Status(context.Background(), &apiv1.StatusRequest{}) + if err != nil { + t.Fatalf("Status() error: %v", err) + } + // The pre-existing contract is unchanged. + if resp.Status != "ok" || resp.ActiveJobs != 0 || resp.MaxConcurrent != 2 { + t.Errorf("existing Status fields changed: %+v", resp) + } + if resp.HeldSlots != 1 { + t.Errorf("HeldSlots = %d, want 1 (the untracked macOS holder)", resp.HeldSlots) + } + if resp.SlotCapacity != 5 { + t.Errorf("SlotCapacity = %d, want 5", resp.SlotCapacity) + } + var sawMac bool + for _, p := range resp.SlotPools { + if p.Pool == "macos" { + sawMac = true + if p.Held != 1 || p.Capacity != 1 { + t.Errorf("macos pool = %+v, want held 1 capacity 1", p) + } + } + } + if !sawMac { + t.Error("Status did not break the macos pool out in SlotPools") + } +} + +// TestHealthzExposesHeldSlots checks the endpoint an operator (and the fleet +// monitor) actually reads. active_jobs keeps its old meaning; held_slots is +// what would have shown the wedged mac. +func TestHealthzExposesHeldSlots(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + s.macSem <- struct{}{} + defer func() { <-s.macSem }() + + rec := httptest.NewRecorder() + s.handleHealthz(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("healthz status = %d, want 200", rec.Code) + } + + var body struct { + Status string `json:"status"` + ActiveJobs int `json:"active_jobs"` + MaxConcurrent int `json:"max_concurrent"` + HeldSlots int `json:"held_slots"` + SlotCapacity int `json:"slot_capacity"` + Slots []SlotStats `json:"slots"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decoding healthz body %q: %v", rec.Body.String(), err) + } + // Backwards compatibility: the pre-existing fields must be untouched. + if body.Status != "ok" || body.ActiveJobs != 0 || body.MaxConcurrent != 2 { + t.Errorf("existing healthz fields changed: %+v", body) + } + if body.HeldSlots != 1 { + t.Errorf("held_slots = %d, want 1", body.HeldSlots) + } + if body.SlotCapacity != 5 { + t.Errorf("slot_capacity = %d, want 5", body.SlotCapacity) + } + var sawMac bool + for _, p := range body.Slots { + if p.Pool == "macos" { + sawMac = true + if p.Held != 1 || p.Capacity != 1 { + t.Errorf("macos pool in healthz = %+v, want held 1 capacity 1", p) + } + } + } + if !sawMac { + t.Error("healthz slots did not break out the macos pool") + } +} From b20564c9abf924a288d2b73538f46db1b17d0fa9 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Sat, 5 Sep 2026 01:01:15 -0700 Subject: [PATCH 2/5] fix(scheduler): scope the slot-leak escalation to the pool being waited on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The leak watchdog added with the #196 fix compares a PER-POOL held count against a GLOBAL tracked-job count: `held := len(sem)` is one pool, but `tracked := s.ActiveJobs()` is `len(s.running)` across every pool. Mixed units, and the mistake fails in the one direction that matters — it SUPPRESSES the alarm. On the node this was written for that is not hypothetical. The fleet's mac runs three pools: local runners, Linux jobs dispatched into its embedded VM, and macOS VMs. It serves Linux work more or less continuously. So for the whole 28 hours its macOS slot was leaked there was almost always some tracked job somewhere, and one is enough: `trackedJobs == 0` never held, so the escalation stayed at Warn and the single log line that would have named the outage — "blocked on a concurrency slot that no tracked job accounts for — suspected slot leak" — was never going to be emitted no matter how long the queued macOS jobs waited. The denominator now matches the numerator. trackedInPool(pool) counts only the running jobs whose slot came from that pool, classified by poolOf: macosVM != nil is macos, a non-empty dispatched name is linux, everything else is local — the same routing handleQueued uses to pick the semaphore in the first place. A leaked macOS slot is now reported at Error regardless of what the Linux pool is doing. The log key is renamed tracked_jobs -> pool_tracked_jobs so the number cannot be misread as the node-wide count; active_jobs, HeldSlots and SlotUsage are untouched. Also corrects the acquireSlot fast-path comment, which claimed the path was "allocation-free" while returning &slotToken{}. It is timer- and log-free, which is the property that actually matters; one small struct per job that is about to boot a VM is not worth claiming otherwise. Tests: - TestSlotLeakEscalationIsScopedToThePool: a tracked LOCAL job plus a macOS slot held by nothing tracked must still escalate to Error. Fails against the old rule (observed: level=WARN, "waiting for a free concurrency slot", pool_tracked_jobs=1). - TestTrackedInPool: one job per pool, each counted against its own pool, with ActiveJobs() still 3. - TestSlotWaitSeverity's table is renamed to say the third column is per-pool, so it stops reading as an endorsement of the global count. --- pkg/scheduler/slots.go | 62 ++++++++++++++++++++++--- pkg/scheduler/slots_test.go | 90 ++++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 9 deletions(-) diff --git a/pkg/scheduler/slots.go b/pkg/scheduler/slots.go index 805435d1..54db26ca 100644 --- a/pkg/scheduler/slots.go +++ b/pkg/scheduler/slots.go @@ -95,8 +95,9 @@ func (t *slotToken) release() { // logged nothing at all while five macOS jobs piled up behind a slot that // would never come back. func (s *Scheduler) acquireSlot(ctx context.Context, sem chan struct{}, pool string, log *slog.Logger) *slotToken { - // Fast path: a free slot logs nothing, which is the overwhelmingly - // common case and must stay allocation- and timer-free. + // Fast path: a free slot logs nothing and arms no timer, which is the + // overwhelmingly common case. (It still allocates the token — one tiny + // struct per job, against a job that is about to boot a VM.) select { case sem <- struct{}{}: return &slotToken{sem: sem} @@ -118,13 +119,22 @@ func (s *Scheduler) acquireSlot(ctx context.Context, sem chan struct{}, pool str case <-ctx.Done(): return nil case <-timer.C: - held, tracked, waited := len(sem), s.ActiveJobs(), time.Since(start) + // PER-POOL, not global. The first version of this compared the + // pool's held count against s.ActiveJobs(), which is + // len(s.running) across EVERY pool — so a single tracked Linux + // job running on the mac was enough to make a genuinely leaked + // macOS slot look accounted for and keep the escalation at Warn. + // On the #196 node that is not a hypothetical: the mac serves + // Linux jobs in its embedded VM continuously, so the one signal + // that would have named the outage was suppressed by unrelated + // work. + held, tracked, waited := len(sem), s.trackedInPool(pool), time.Since(start) level, msg := slotWaitSeverity(waited, held, tracked) log.Log(ctx, level, msg, "pool", pool, "held", held, "capacity", cap(sem), - "tracked_jobs", tracked, + "pool_tracked_jobs", tracked, "waited", waited.Truncate(time.Second)) timer.Reset(slotWaitLogEvery) } @@ -139,20 +149,58 @@ func (s *Scheduler) acquireSlot(ctx context.Context, sem chan struct{}, pool str // costs nothing when the node is idle and cannot produce a false alarm on a // node that simply has no macOS work. // -// held > 0 with trackedJobs == 0 means the slot is charged to a job the +// held > 0 with poolTrackedJobs == 0 means the slot is charged to a job the // scheduler is not tracking. That is legitimate for the length of a // provision (bounded by MacOSProvisionTimeout, minutes) and illegitimate // forever after — it is the exact signature of the 28-hour macOS outage, // where `ephemerd status` reported active_jobs: 0 on a node whose only slot // had been held since the previous afternoon. Pure so the escalation rule is // testable without a real pool or a real wait. -func slotWaitSeverity(waited time.Duration, held, trackedJobs int) (slog.Level, string) { - if held > 0 && trackedJobs == 0 && waited >= slotLeakSuspectAfter { +// +// poolTrackedJobs is scoped to THE POOL BEING WAITED ON (see trackedInPool), +// never to the whole scheduler. held is a per-pool number, so comparing it to +// a global job count mixes units: on a mac, which serves Linux jobs from its +// embedded VM alongside macOS VMs, one tracked Linux job made a leaked macOS +// slot look explained and downgraded this to Warn forever. +func slotWaitSeverity(waited time.Duration, held, poolTrackedJobs int) (slog.Level, string) { + if held > 0 && poolTrackedJobs == 0 && waited >= slotLeakSuspectAfter { return slog.LevelError, "blocked on a concurrency slot that no tracked job accounts for — suspected slot leak" } return slog.LevelWarn, "waiting for a free concurrency slot" } +// poolOf reports which dispatch pool a tracked job's slot came from. It +// mirrors the routing in handleQueued: macOS jobs hold macSem, Linux jobs +// dispatched into the VM worker hold linuxSem, everything else holds sem. +func poolOf(rj *runningJob) string { + switch { + case rj.macosVM != nil: + return "macos" + case rj.dispatched != "": + return "linux" + default: + return "local" + } +} + +// trackedInPool counts tracked running jobs whose slot came from pool. +// +// This is the denominator the leak watchdog needs: "is anything the scheduler +// knows about accounting for the slots THIS pool is holding". ActiveJobs() — +// the global count — cannot answer that on a node that runs more than one +// kind of job, and a mac runs three. +func (s *Scheduler) trackedInPool(pool string) int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, rj := range s.running { + if poolOf(rj) == pool { + n++ + } + } + return n +} + // SlotUsage reports concurrency-slot occupancy for every dispatch pool. // // Held is read from the semaphore itself, so it counts slots that are charged diff --git a/pkg/scheduler/slots_test.go b/pkg/scheduler/slots_test.go index 59299b4f..14bde2ce 100644 --- a/pkg/scheduler/slots_test.go +++ b/pkg/scheduler/slots_test.go @@ -398,8 +398,12 @@ func TestSlotWaitSeverity(t *testing.T) { wantLevel slog.Level wantSuspect bool }{ - {"busy node, short wait", time.Minute, 1, 1, slog.LevelWarn, false}, - {"busy node, long wait", 2 * time.Hour, 1, 1, slog.LevelWarn, false}, + {"this pool has a tracked job, short wait", time.Minute, 1, 1, slog.LevelWarn, false}, + // The third column is jobs tracked IN THIS POOL, not on the node. + // It used to be s.ActiveJobs() (every pool), which meant a Linux job + // on the mac explained away a leaked macOS slot; see + // TestSlotLeakEscalationIsScopedToThePool. + {"this pool has a tracked job, long wait", 2 * time.Hour, 1, 1, slog.LevelWarn, false}, {"untracked holder, short wait", time.Minute, 1, 0, slog.LevelWarn, false}, {"untracked holder, long wait", slotLeakSuspectAfter, 1, 0, slog.LevelError, true}, {"nothing held, long wait", 2 * time.Hour, 0, 0, slog.LevelWarn, false}, @@ -596,3 +600,85 @@ func TestHealthzExposesHeldSlots(t *testing.T) { t.Error("healthz slots did not break out the macos pool") } } + +// --------------------------------------------------------------------------- +// MINOR 4: the leak escalation is per-pool, not per-node. +// --------------------------------------------------------------------------- + +// TestTrackedInPool: jobs are counted against the pool whose slot they hold. +func TestTrackedInPool(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 2, Log: quietLogger()}) + s.mu.Lock() + s.running[jobKey{Provider: "p", JobID: 1}] = &runningJob{} // local + s.running[jobKey{Provider: "p", JobID: 2}] = &runningJob{dispatched: "r-2"} // linux + s.running[jobKey{Provider: "p", JobID: 3}] = &runningJob{ + macosVM: &fastMacVM{stops: new(atomic.Int32)}, + } // macos + s.mu.Unlock() + + for pool, want := range map[string]int{"local": 1, "linux": 1, "macos": 1} { + if got := s.trackedInPool(pool); got != want { + t.Errorf("trackedInPool(%q) = %d, want %d", pool, got, want) + } + } + if got := s.ActiveJobs(); got != 3 { + t.Errorf("ActiveJobs() = %d, want 3 (the global count keeps its meaning)", got) + } +} + +// TestSlotLeakEscalationIsScopedToThePool is the reason trackedInPool exists. +// +// The escalation compared a PER-POOL held count against s.ActiveJobs(), which +// is len(s.running) across every pool. The #196 node is a mac: it serves Linux +// jobs from its embedded VM continuously, so there was almost always a tracked +// job somewhere — and one was enough to make a leaked macOS slot look +// accounted for and hold the report at Warn forever. The one signal that would +// have named the outage was suppressed by unrelated work. +// +// Without the fix this test FAILS: the log says "waiting for a free +// concurrency slot" at Warn instead of naming the suspected leak. +func TestSlotLeakEscalationIsScopedToThePool(t *testing.T) { + after, every, leak := slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter + defer func() { slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter = after, every, leak }() + slotWaitLogAfter = 10 * time.Millisecond + slotWaitLogEvery = 10 * time.Millisecond + slotLeakSuspectAfter = 10 * time.Millisecond + + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + + // A perfectly healthy LOCAL job, tracked, doing real work. It has nothing + // to do with the macOS pool. + s.mu.Lock() + s.running[jobKey{Provider: "github", JobID: 1}] = &runningJob{repo: "myrepo"} + s.mu.Unlock() + + // The macOS slot is held by nothing tracked: the #196 state exactly. + s.macSem <- struct{}{} + + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + got := make(chan *slotToken, 1) + go func() { got <- s.acquireSlot(context.Background(), s.macSem, "macos", log) }() + + time.Sleep(60 * time.Millisecond) + <-s.macSem // let the waiter through so the goroutine (and the log) settle + select { + case tok := <-got: + tok.release() + case <-time.After(5 * time.Second): + t.Fatal("acquireSlot never returned after the slot was freed") + } + + out := buf.String() + if !strings.Contains(out, "suspected slot leak") { + t.Errorf("a leaked macOS slot was not escalated because an unrelated LOCAL job was tracked; got:\n%s", out) + } + if !strings.Contains(out, "level=ERROR") { + t.Errorf("leak report was not logged at ERROR; got:\n%s", out) + } + if !strings.Contains(out, "pool_tracked_jobs=0") { + t.Errorf("wait log did not report the POOL's tracked-job count; got:\n%s", out) + } +} + From 5df443b62f10cf7052b84fe537f695ea15cf3543 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Sat, 5 Sep 2026 01:05:53 -0700 Subject: [PATCH 3/5] fix(scheduler): bound every macOS teardown and cap the VMs we abandon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three problems, one mechanism: the #196 fix decided that a macOS VM stop which does not return must never pin the caller. It then applied that decision to exactly one caller. 1. THREE UNBOUNDED Stop() CALLS REMAINED, and the worst of them is on the event loop. sweepOrphanRunners is invoked SYNCHRONOUSLY from the `case <-cleanupTicker.C:` arm of Run's select — the scheduler's only event loop. A tracked macOS job nominated as an orphan whose Vz stop wedges is the exact failure #196 documented on metal, and it did not cost a slot there, it cost the daemon: no queued/in_progress/completed handling for ANY platform, and no drain on SIGTERM, while /healthz kept answering 200. Strictly worse than the leak the PR set out to fix, one call away. handleCompleted's bare Stop() parks the completed-event handler forever and with it the deferred sweepOrphanRunners that reconciles the runner ledger after every completion. destroyAll's bare Stop() is the force-kill arm of drain(), reached only after ShutdownTimeout has ALREADY expired — a wedge there means drain never returns, Run never returns, and the operator has to escalate SIGTERM to SIGKILL by hand, which defeats the entire point of having a shutdown timeout. All three now go through stopVMBounded. There is no longer any macOS stop in this package that may block its caller; stopVMAsync is the single place macVM.Stop() is called at all. 2. THE REDUNDANT SECOND STOP on the provisioning-deadline path. waitForMacRunnerBounded force-stops the VM when the deadline fires (#178, unchanged), and the caller's error path then stopped the same VM again. darwinMacOSVM.Stop is `stopOnce.Do(m.stop)`, so a second caller does not return early — it BLOCKS until the first invocation returns. Against the wedged guest that is the whole subject of #196 that is a full teardown grace (30s) of dead time on a path that still has a ghost JIT runner to deregister, plus a third goroutine parked inside the Once behind the other two. waitForMacRunnerBounded now reports whether it already stopped the VM and the caller skips its own stop when it did. 3. THE ABANDONED-VM TRADE HAD NO BOUND AND NO REPORTING. Walking away from a wedged stop is right — a leaked goroutine is far cheaper than a leaked slot, which costs the node its entire macOS capacity. But it is not free. Each abandonment leaves goroutines retaining a live *darwinMacOSVM and, expensively, a STILL-RUNNING macOS guest holding its full configured RAM and its disk clone. Only a daemon restart reclaims it. What used to bound this was the semaphore, by accident: leak the only slot and the node provisions zero further macOS VMs, so at most one guest could ever be stranded. Releasing the slot is precisely what removed that accidental cap — an abandoned VM is untracked and holds no slot, so MaxMacOSVMs does not charge for it either. A repeatably-wedging guest would therefore keep provisioning until Virtualization.framework refused (it allows only about two concurrent macOS guests on Apple Silicon) or the host OOMed. So the bound is explicit now. abandonedMacVMs counts VMs whose teardown was walked away from, keyed per attempt, charged in abandonVM and discharged if the stop ever does return (a Vz stop can be merely slow; a node that recovers must not need a restart to serve macOS again). At defaultMacAbandonedVMCap (2) the macOS pool refuses to provision, logging at Error with the count, the cap and what to do about it. The gate is checked before the semaphore wait and again after it, for the same reason the #154 cordon is re-checked in admitDispatch: a dispatch can be parked on a full pool for the entire life of the job ahead of it, which is exactly the window in which a teardown gets abandoned. Rejection follows the #154 cordon convention exactly: clear the pending stamp, leave the seen stamp to age out via seenTTL, leave the job queued on the platform so a healthy node can take it immediately. Observability: status and /healthz gain abandoned_macos_vms and abandoned_macos_vm_cap next to the held_slots fields, with abandoned_macos_vm_details on /healthz (job id, VM instance id, reason, age). held_slots showed the outage; this shows what the fix for it costs. proto fields are new numbers (10, 11); nothing existing moved. #178's deadline and force-stop are unchanged and #151's busy-veto is untouched. Tests, each verified to fail with its fix reverted: - TestSweepOrphanRunners_WedgedMacOSVMCannotBlockTheEventLoop — reverted: "sweepOrphanRunners did not return within 5s". - TestHandleCompleted_WedgedMacOSVMCannotBlockTheHandler — reverted: "handleCompleted did not return within 5s". - TestDrain_WedgedMacOSVMCannotBlockShutdown — reverted: "drain did not return within 10s". - TestMacOSPoolCordonsPastTheAbandonedVMCap — with the cordon disabled it provisioned a second VM past the cap and stranded two guests; also proves the cordon lifts when an abandoned stop finally returns. - TestHandleMacOSJob_DeadlinePathStopsTheVMExactlyOnce — with the skip removed: "Stop called 2 times", and AbandonedMacOSVMs()=2 for ONE wedged VM, i.e. a single wedge consuming the node's whole abandoned budget. - TestStatusAndHealthzReportAbandonedMacOSVMs. All deadline-guarded, so a regression is a failure rather than a hang. --- api/v1/ephemerd.pb.go | 457 ++++++++++++++------------ api/v1/ephemerd.proto | 14 + cmd/ephemerd/commands.go | 11 +- docs/cli/status.md | 23 +- pkg/scheduler/grpc.go | 11 +- pkg/scheduler/macos_provision_test.go | 5 +- pkg/scheduler/scheduler.go | 182 ++++++++-- pkg/scheduler/slots.go | 195 +++++++++++ pkg/scheduler/slots_test.go | 323 +++++++++++++++++- 9 files changed, 970 insertions(+), 251 deletions(-) diff --git a/api/v1/ephemerd.pb.go b/api/v1/ephemerd.pb.go index c9643ade..5f65245d 100644 --- a/api/v1/ephemerd.pb.go +++ b/api/v1/ephemerd.pb.go @@ -347,6 +347,20 @@ type StatusResponse struct { // slot_pools breaks the totals down per dispatch pool (local, linux, // macos), so an operator can tell which pool is saturated. SlotPools []*SlotPool `protobuf:"bytes,9,rep,name=slot_pools,json=slotPools,proto3" json:"slot_pools,omitempty"` + // abandoned_macos_vms counts macOS VMs whose teardown this daemon gave up + // on and has not seen finish. + // + // This is the COST of the #196 fix, made visible. When a Virtualization. + // framework stop wedges, the daemon walks away from it so the concurrency + // slot comes back — but the guest is still running and still holding its + // full configured RAM, and only a restart reclaims it. Non-zero means the + // node is losing memory it cannot get back; at abandoned_macos_vm_cap the + // macOS pool refuses further provisioning rather than growing without + // bound (Vz permits only a couple of concurrent macOS guests anyway). + AbandonedMacosVms int32 `protobuf:"varint,10,opt,name=abandoned_macos_vms,json=abandonedMacosVms,proto3" json:"abandoned_macos_vms,omitempty"` + // abandoned_macos_vm_cap is the value of abandoned_macos_vms at which the + // macOS pool stops taking work. + AbandonedMacosVmCap int32 `protobuf:"varint,11,opt,name=abandoned_macos_vm_cap,json=abandonedMacosVmCap,proto3" json:"abandoned_macos_vm_cap,omitempty"` } func (x *StatusResponse) Reset() { @@ -444,6 +458,20 @@ func (x *StatusResponse) GetSlotPools() []*SlotPool { return nil } +func (x *StatusResponse) GetAbandonedMacosVms() int32 { + if x != nil { + return x.AbandonedMacosVms + } + return 0 +} + +func (x *StatusResponse) GetAbandonedMacosVmCap() int32 { + if x != nil { + return x.AbandonedMacosVmCap + } + return 0 +} + // SlotPool is one dispatch pool's concurrency-slot occupancy. type SlotPool struct { state protoimpl.MessageState @@ -1837,8 +1865,8 @@ var file_api_v1_ephemerd_proto_rawDesc = []byte{ 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x0f, 0x0a, - 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb8, - 0x02, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x9d, + 0x03, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, @@ -1857,215 +1885,222 @@ var file_api_v1_ephemerd_proto_rawDesc = []byte{ 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x0a, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x09, - 0x73, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x22, 0x4e, 0x0a, 0x08, 0x53, 0x6c, 0x6f, - 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x68, 0x65, 0x6c, 0x64, 0x12, 0x1a, 0x0a, - 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x22, 0x11, 0x0a, 0x0f, 0x4c, 0x69, 0x73, - 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x38, 0x0a, 0x10, - 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x24, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, - 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x1f, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xd1, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1b, 0x0a, - 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x03, 0x70, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, - 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, - 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x4b, - 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0x11, 0x0a, - 0x0f, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x31, 0x0a, 0x0e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, - 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, - 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, 0x10, 0x55, 0x6e, 0x63, 0x6f, 0x72, - 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x3b, 0x0a, 0x11, - 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x22, 0x1e, 0x0a, 0x08, 0x4c, 0x6f, 0x67, - 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, 0x01, 0x0a, 0x0e, 0x55, 0x70, - 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x72, 0x6c, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, - 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x75, 0x72, 0x6c, 0x4f, 0x76, - 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, - 0x5f, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x6e, 0x6f, - 0x44, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x22, 0x99, 0x02, 0x0a, 0x0f, - 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, - 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, - 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x87, 0x01, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, - 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, - 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x69, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, - 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, - 0x6f, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, - 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x0a, 0x0f, 0x57, 0x61, 0x69, 0x74, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, - 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, - 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x23, 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, - 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x14, 0x0a, - 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x48, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, - 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x78, 0x0a, - 0x13, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x6e, 0x69, 0x78, - 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xca, 0x02, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, - 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x26, - 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, - 0x65, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, - 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x65, - 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x65, 0x6d, - 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x6e, 0x6f, 0x6e, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x41, 0x6e, 0x6f, 0x6e, - 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, 0x75, 0x5f, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, - 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x72, 0x78, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x52, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, - 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x74, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x78, 0x42, - 0x79, 0x74, 0x65, 0x73, 0x2a, 0xaf, 0x02, 0x0a, 0x0c, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x4c, 0x49, 0x47, 0x48, 0x54, 0x10, - 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x45, 0x5f, 0x55, 0x50, 0x5f, 0x54, 0x4f, 0x5f, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, - 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, - 0x5f, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x1d, 0x0a, 0x19, 0x55, - 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x57, - 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, - 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x56, 0x45, 0x52, 0x49, - 0x46, 0x59, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, 0x55, 0x50, 0x47, 0x52, 0x41, - 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x47, 0x49, 0x4e, 0x47, - 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x53, 0x57, 0x41, 0x50, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x12, 0x1c, - 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, - 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, - 0x49, 0x4c, 0x45, 0x44, 0x10, 0x09, 0x32, 0xfd, 0x04, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1a, 0x2e, 0x65, - 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, - 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, - 0x73, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, - 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, - 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x12, 0x44, 0x0a, 0x07, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, - 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, - 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, - 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, - 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, - 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x65, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, - 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x06, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x12, 0x1a, 0x2e, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x72, 0x64, - 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, - 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, - 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, - 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x46, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x0a, 0x50, 0x72, 0x75, 0x6e, 0x65, - 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, - 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, - 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd1, 0x02, 0x0a, 0x08, 0x44, 0x69, 0x73, 0x70, 0x61, - 0x74, 0x63, 0x68, 0x12, 0x4a, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, - 0x12, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x44, 0x0a, 0x07, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, - 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x28, 0x2e, 0x65, - 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, - 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, - 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x70, 0x68, 0x70, 0x6d, 0x2f, 0x65, - 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, - 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x62, 0x61, + 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x63, 0x6f, 0x73, 0x5f, 0x76, 0x6d, 0x73, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x61, 0x62, 0x61, 0x6e, 0x64, 0x6f, 0x6e, 0x65, + 0x64, 0x4d, 0x61, 0x63, 0x6f, 0x73, 0x56, 0x6d, 0x73, 0x12, 0x33, 0x0a, 0x16, 0x61, 0x62, 0x61, + 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x63, 0x6f, 0x73, 0x5f, 0x76, 0x6d, 0x5f, + 0x63, 0x61, 0x70, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x61, 0x62, 0x61, 0x6e, 0x64, + 0x6f, 0x6e, 0x65, 0x64, 0x4d, 0x61, 0x63, 0x6f, 0x73, 0x56, 0x6d, 0x43, 0x61, 0x70, 0x22, 0x4e, + 0x0a, 0x08, 0x53, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, + 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x12, 0x12, + 0x0a, 0x04, 0x68, 0x65, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x68, 0x65, + 0x6c, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x22, 0x11, + 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x38, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x1f, 0x0a, 0x0d, 0x47, + 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xd1, 0x01, 0x0a, + 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, + 0x22, 0x20, 0x0a, 0x0e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, + 0x69, 0x64, 0x22, 0x11, 0x0a, 0x0f, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x0e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x55, 0x6e, 0x63, + 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, 0x10, + 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, + 0x73, 0x22, 0x3b, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x22, 0x1e, + 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, + 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x72, 0x6c, 0x5f, + 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x75, 0x72, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x64, + 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, + 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x64, 0x72, 0x61, 0x69, + 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, + 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x5f, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x6e, 0x6f, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, + 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, + 0x22, 0x99, 0x02, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x27, 0x0a, 0x0f, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, + 0x12, 0x29, 0x0a, 0x10, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x87, 0x01, 0x0a, + 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x69, 0x74, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x69, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x57, + 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x0a, + 0x0f, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x23, 0x0a, + 0x11, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x48, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x73, 0x22, 0x78, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xca, 0x02, 0x0a, + 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, + 0x65, 0x70, 0x6f, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, + 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, 0x70, + 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d, + 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2a, + 0x0a, 0x11, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x6e, 0x6f, 0x6e, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x41, 0x6e, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, + 0x75, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x63, + 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x10, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x5f, 0x72, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, + 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x74, 0x78, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x54, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0xaf, 0x02, 0x0a, 0x0c, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, + 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, + 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x4c, + 0x49, 0x47, 0x48, 0x54, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, + 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x50, 0x5f, 0x54, 0x4f, 0x5f, 0x44, 0x41, + 0x54, 0x45, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x03, + 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, + 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x12, + 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, + 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, + 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x57, 0x41, 0x50, 0x50, 0x49, 0x4e, + 0x47, 0x10, 0x07, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, + 0x08, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x09, 0x32, 0xfd, 0x04, 0x0a, 0x07, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x4c, 0x69, + 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, + 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4a, + 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, + 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x12, 0x44, 0x0a, 0x07, 0x4b, + 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x12, + 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, + 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x06, 0x43, 0x6f, 0x72, 0x64, + 0x6f, 0x6e, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, + 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x72, + 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x55, + 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, + 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, + 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x0a, + 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd1, 0x02, 0x0a, 0x08, + 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4a, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x12, + 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, + 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x4a, + 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x44, 0x65, + 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x14, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x12, 0x28, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x65, 0x70, + 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, 0x42, + 0x28, 0x5a, 0x26, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x70, + 0x68, 0x70, 0x6d, 0x2f, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/api/v1/ephemerd.proto b/api/v1/ephemerd.proto index d1b9299e..c7d9a134 100644 --- a/api/v1/ephemerd.proto +++ b/api/v1/ephemerd.proto @@ -91,6 +91,20 @@ message StatusResponse { // slot_pools breaks the totals down per dispatch pool (local, linux, // macos), so an operator can tell which pool is saturated. repeated SlotPool slot_pools = 9; + // abandoned_macos_vms counts macOS VMs whose teardown this daemon gave up + // on and has not seen finish. + // + // This is the COST of the #196 fix, made visible. When a Virtualization. + // framework stop wedges, the daemon walks away from it so the concurrency + // slot comes back — but the guest is still running and still holding its + // full configured RAM, and only a restart reclaims it. Non-zero means the + // node is losing memory it cannot get back; at abandoned_macos_vm_cap the + // macOS pool refuses further provisioning rather than growing without + // bound (Vz permits only a couple of concurrent macOS guests anyway). + int32 abandoned_macos_vms = 10; + // abandoned_macos_vm_cap is the value of abandoned_macos_vms at which the + // macOS pool stops taking work. + int32 abandoned_macos_vm_cap = 11; } // SlotPool is one dispatch pool's concurrency-slot occupancy. diff --git a/cmd/ephemerd/commands.go b/cmd/ephemerd/commands.go index b8a5fe85..2b1992b8 100644 --- a/cmd/ephemerd/commands.go +++ b/cmd/ephemerd/commands.go @@ -89,9 +89,14 @@ func statusCmd() *cli.Command { "held_slots": resp.HeldSlots, "slot_capacity": resp.SlotCapacity, "slots": slots, - "draining": resp.Draining, - "uptime": resp.Uptime, - "version": resp.Version, + // Non-zero means macOS guests are stranded on this node, + // holding RAM nothing but a restart reclaims; at the cap the + // macOS pool has stopped taking work. See issue #196. + "abandoned_macos_vms": resp.AbandonedMacosVms, + "abandoned_macos_vm_cap": resp.AbandonedMacosVmCap, + "draining": resp.Draining, + "uptime": resp.Uptime, + "version": resp.Version, } pretty, _ := json.MarshalIndent(data, "", " ") diff --git a/docs/cli/status.md b/docs/cli/status.md index cd915b95..ab675dc0 100644 --- a/docs/cli/status.md +++ b/docs/cli/status.md @@ -21,6 +21,8 @@ Returns a JSON object with the following fields: | `held_slots` | Concurrency slots currently held across all dispatch pools | | `slot_capacity` | Total slots across all dispatch pools | | `slots` | Per-pool breakdown (`local`, `linux`, `macos`) of `held` and `capacity` | +| `abandoned_macos_vms` | macOS VMs whose teardown the daemon gave up on and has not seen die | +| `abandoned_macos_vm_cap` | Value of `abandoned_macos_vms` at which the macOS pool stops taking work | | `draining` | Whether the daemon is draining (shutting down gracefully) | | `uptime` | How long the daemon has been running | @@ -38,6 +40,8 @@ Returns a JSON object with the following fields: { "pool": "linux", "held": 0, "capacity": 4 }, { "pool": "macos", "held": 0, "capacity": 1 } ], + "abandoned_macos_vms": 0, + "abandoned_macos_vm_cap": 2, "draining": false, "uptime": "3h42m15s" } @@ -57,7 +61,24 @@ a provision that is stuck, or a slot that leaked. The daemon also logs `waiting for a free concurrency slot` (and escalates to a suspected-leak error) when a job blocks on such a pool, so the log and this output agree. -The same fields are on `/healthz`. +### abandoned_macos_vms + +When a macOS VM's `Stop()` does not return — Virtualization.framework can wedge +a teardown indefinitely — the daemon walks away from it after a grace period so +the concurrency slot comes back. The slot is the thing that matters: a held one +costs the node its whole macOS capacity. + +The guest, however, is still running, and still holding its full configured RAM. +Nothing in the daemon can reclaim it; only a restart can. So each abandonment is +counted here, and once `abandoned_macos_vms` reaches `abandoned_macos_vm_cap` the +macOS pool refuses to provision anything further and says so at `ERROR` — a +bounded, legible failure instead of unbounded memory growth on a host that +Virtualization.framework will soon refuse anyway. + +Non-zero is worth investigating. At the cap, restart the daemon. + +`/healthz` carries the same fields, plus `abandoned_macos_vm_details` (job id, +VM instance id, reason and age per stranded guest) when there are any. ## Connection diff --git a/pkg/scheduler/grpc.go b/pkg/scheduler/grpc.go index e70ee379..6c343e52 100644 --- a/pkg/scheduler/grpc.go +++ b/pkg/scheduler/grpc.go @@ -99,9 +99,14 @@ func (c *controlServer) Status(ctx context.Context, req *apiv1.StatusRequest) (* HeldSlots: int32(c.sched.HeldSlots()), SlotCapacity: int32(c.sched.SlotCapacity()), SlotPools: pools, - Draining: draining, - Uptime: time.Since(c.sched.startTime).Truncate(time.Second).String(), - Version: c.sched.cfg.Version, + // The other half of the slot story: slots that came back only because + // the daemon abandoned a teardown. A guest per count, still holding + // its RAM, reclaimable only by restart. See issue #196 and slots.go. + AbandonedMacosVms: int32(c.sched.AbandonedMacOSVMs()), + AbandonedMacosVmCap: int32(c.sched.abandonedMacVMCap()), + Draining: draining, + Uptime: time.Since(c.sched.startTime).Truncate(time.Second).String(), + Version: c.sched.cfg.Version, }, nil } diff --git a/pkg/scheduler/macos_provision_test.go b/pkg/scheduler/macos_provision_test.go index da57effa..52710880 100644 --- a/pkg/scheduler/macos_provision_test.go +++ b/pkg/scheduler/macos_provision_test.go @@ -132,13 +132,16 @@ func TestWaitForMacRunnerBounded_ReturnsWhenWaitSucceeds(t *testing.T) { var stops atomic.Int32 quick := &fastMacVM{ip: "192.168.64.5", stops: &stops} - ip, err := s.waitForMacRunnerBounded(context.Background(), quick, quietLogger()) + ip, alreadyStopped, err := s.waitForMacRunnerBounded(context.Background(), quick, macVMRef{JobID: 1, VMID: "1-happy"}, quietLogger()) if err != nil { t.Fatalf("unexpected error on healthy provision: %v", err) } if ip != "192.168.64.5" { t.Errorf("ip = %q, want 192.168.64.5", ip) } + if alreadyStopped { + t.Error("healthy provision reported the VM as already stopped; the caller would skip a teardown it still owes") + } if stops.Load() != 0 { t.Errorf("VM was stopped %d times on the happy path, want 0", stops.Load()) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 6d63e3ec..6bb26741 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -283,6 +283,16 @@ type Scheduler struct { macUnwindGrace time.Duration macStopGrace time.Duration + // macAbandoned counts macOS VMs whose teardown this daemon walked away + // from and has not seen die. Releasing the slot on a wedged teardown + // (#196) removed the semaphore's accidental cap on how many guests could + // be stranded; this is the explicit one. See slots.go. + macAbandoned abandonedMacVMs + + // macAbandonedCap overrides defaultMacAbandonedVMCap. Zero uses the + // default; only tests set it. + macAbandonedCap int + // busyProbe overrides the ground-truth "is this runner executing a // job" check that vetoes orphan teardown. Nil (the default) uses // probeRunnerBusy: local container/VM/process introspection, falling @@ -354,9 +364,16 @@ type runningJob struct { artifactsDir string // non-empty if OCI artifacts were extracted for this job dispatched string // non-empty if dispatched to Linux VM worker (stores container name) macosVM vm.MacOSVM // non-nil if running as a macOS VM job + macVMID string // macOS VM instance id; names this job's on-disk paths startedAt time.Time } +// macRef is the identity every macOS teardown site reports to the +// abandoned-VM registry. Safe on a non-macOS job (returns a zero VMID). +func (rj *runningJob) macRef(key jobKey) macVMRef { + return macVMRef{JobID: key.JobID, VMID: rj.macVMID} +} + // runnerName returns the name the runner was registered under with the // provider. Every provisioning path stores it on the claim; the Linux // dispatch path mirrors it in dispatched. @@ -1339,6 +1356,34 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent s.mu.Unlock() } + // refuseCordoned drops this dispatch because the macOS pool has given up + // on too many VMs to safely start another (see slots.go). + // + // It leaves the seen stamp and clears only pending, exactly like the #154 + // cordon rejection: seen ages out via seenTTL, so the node retries in ~10 + // minutes rather than re-logging this every poll, and the job stays queued + // on the platform so a healthy node can take it immediately. + refuseCordoned := func(n, limit int) { + s.mu.Lock() + delete(s.pending, key) + s.mu.Unlock() + log.Error("refusing to provision a macOS VM: this node has abandoned too many macOS VMs without confirming they died", + "abandoned_macos_vms", n, + "cap", limit, + "detail", "each one is a live guest holding its full configured RAM; starting more risks OOMing the host and Vz will refuse anyway. Restart ephemerd to reclaim them.") + } + + // vmID names the VM instance. Still the bare job id at this point; the + // per-attempt discriminator lands in the follow-up commit. + vmID := fmt.Sprintf("%d", jobID) + ref := macVMRef{JobID: jobID, VMID: vmID} + log = log.With("vm", vmID) + + if n, limit, cordoned := s.macPoolCordoned(); cordoned { + refuseCordoned(n, limit) + return + } + // Acquire macOS VM concurrency slot (separate from Linux/local sem). slot := s.acquireSlot(ctx, s.macSem, "macos", log) if slot == nil { @@ -1364,13 +1409,24 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent return } + // Re-check the abandoned-VM cordon now that the wait for the slot is over. + // Same reasoning as the #154 cordon re-check in admitDispatch: this + // dispatch may have been parked on the semaphore for the entire life of + // the job ahead of it, and that is exactly the window in which a wedged + // teardown gets abandoned. + if n, limit, cordoned := s.macPoolCordoned(); cordoned { + slot.release() + refuseCordoned(n, limit) + return + } + log.Info("provisioning macOS VM runner for job") // Extract OCI artifacts if an image is specified image := s.resolveImage(ctx, &event, "darwin") var artifactsDir string if image != "" && s.cfg.Artifacts != nil { - artifactsDir = artifacts.ArtifactsDir(s.cfg.DataDir, fmt.Sprintf("%d", jobID)) + artifactsDir = artifacts.ArtifactsDir(s.cfg.DataDir, vmID) log.Info("extracting OCI artifacts for macOS VM job", "image", image, "dest", artifactsDir) if err := s.cfg.Artifacts.Extract(ctx, image, artifactsDir); err != nil { log.Error("failed to extract OCI artifacts", "image", image, "error", err) @@ -1395,7 +1451,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent } // Create the macOS VM - macVM, err := s.newMacOSVM(*s.cfg.MacOSVMConfig, fmt.Sprintf("%d", jobID)) + macVM, err := s.newMacOSVM(*s.cfg.MacOSVMConfig, vmID) if err != nil { log.Error("failed to create macOS VM", "error", err) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { @@ -1417,7 +1473,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // the call that hangs (#196). unsee() slot.release() - stopVMBounded(macVM, s.teardownGrace(), log) + s.stopVMBounded(macVM, ref, log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } @@ -1437,7 +1493,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent unsee() cancel() slot.release() - stopVMBounded(macVM, s.teardownGrace(), log) + s.stopVMBounded(macVM, ref, log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } @@ -1452,7 +1508,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // can wedge WaitForRunner indefinitely: the job never gets tracked (so it // is invisible to `ephemerd jobs` and the orphan sweep) yet still holds // the sole macOS slot, stalling all macOS CI until a human intervenes. - ip, err := s.waitForMacRunnerBounded(jobCtx, macVM, log) + ip, alreadyStopped, err := s.waitForMacRunnerBounded(jobCtx, macVM, ref, log) if err != nil { log.Error("macOS VM runner not reachable", "error", err) // THE 28-HOUR LINE (#196). The slot goes back here, first, before @@ -1466,7 +1522,18 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent unsee() cancel() slot.release() - stopVMBounded(macVM, s.teardownGrace(), log) + // Do NOT stop it again when the deadline path already did. Stopping a + // second time is never a cheap no-op: darwinMacOSVM.Stop is a + // sync.Once, so a second caller BLOCKS until the first invocation + // returns. Against the wedged guest this whole PR is about, that is a + // full teardownGrace (30s) of dead time on a path that still has a + // ghost runner to deregister — and it spawns a third goroutine parked + // inside the Once behind the first two. Against a healthy VM it is a + // pointless goroutine. Either way the VM is already stopped-or- + // abandoned and accounted for, so skip it. + if !alreadyStopped { + s.stopVMBounded(macVM, ref, log) + } if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } @@ -1485,6 +1552,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent cancel: cancel, artifactsDir: artifactsDir, macosVM: macVM, + macVMID: vmID, startedAt: time.Now(), }, event.Provider, labelSetKey(event.Labels)) @@ -1534,7 +1602,7 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent if exists { metrics.JobsActive.Dec() if rj.macosVM != nil { - stopVMBounded(rj.macosVM, s.teardownGrace(), log) + s.stopVMBounded(rj.macosVM, rj.macRef(key), log) } if rj.artifactsDir != "" { artifacts.Cleanup(rj.artifactsDir, s.cfg.Log) @@ -1585,7 +1653,13 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // its send and exit if it ever comes back. What must NOT happen is turning // that leaked goroutine into a leaked concurrency slot: teardown gets a // bounded grace period to be tidy, and then we return regardless. -func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSVM, log *slog.Logger) (string, error) { +// +// The second return value reports whether this function already STOPPED the +// VM. The caller's error path must not stop it again: darwinMacOSVM.Stop is a +// sync.Once, so a redundant second Stop blocks until the first returns — +// against a wedged guest that is a full teardown grace of dead time and a +// third goroutine parked in the Once. See handleMacOSJob. +func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSVM, ref macVMRef, log *slog.Logger) (string, bool, error) { timeout := s.cfg.MacOSProvisionTimeout if timeout <= 0 { timeout = defaultMacOSProvisionTimeout @@ -1611,14 +1685,14 @@ func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSV select { case r := <-resCh: - return r.ip, r.err + return r.ip, false, r.err case <-timer.C: log.Error("macOS VM stuck in provisioning past deadline; force-stopping VM to reclaim the slot", "timeout", timeout) - s.abandonMacProvision(macVM, unwound, log) - return "", fmt.Errorf("timed out after %s waiting for macOS VM runner to become reachable", timeout) + s.abandonMacProvision(macVM, unwound, ref, log) + return "", true, fmt.Errorf("timed out after %s waiting for macOS VM runner to become reachable", timeout) case <-ctx.Done(): - s.abandonMacProvision(macVM, unwound, log) - return "", ctx.Err() + s.abandonMacProvision(macVM, unwound, ref, log) + return "", true, ctx.Err() } } @@ -1633,13 +1707,19 @@ func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSV // this teardown were still wedged in the Once, that "cleanup" call would hang // on a path that is holding a concurrency slot. Off-thread here plus // stopVMBounded there means neither can pin the slot. -func (s *Scheduler) abandonMacProvision(macVM vm.MacOSVM, unwound <-chan struct{}, log *slog.Logger) { +func (s *Scheduler) abandonMacProvision(macVM vm.MacOSVM, unwound <-chan struct{}, ref macVMRef, log *slog.Logger) { // Dropping the guest is what unblocks a wedged SSH session call. stopped := stopVMAsync(macVM) if !awaitUnwind(stopped, unwound, s.provisionUnwindGrace()) { log.Error("macOS VM teardown did not finish after the force-stop; abandoning it and reclaiming the concurrency slot anyway", "grace", s.provisionUnwindGrace(), "detail", "a wedged Vz stop or a wedged guest wait must never pin the macOS slot — see issue #196") + // Charge the guest we just walked away from. The slot comes back + // either way — that is the #196 fix and it is not negotiable — but the + // VM is still running and still holding its RAM, so it has to be + // counted against the cap that decides whether this node may start + // another one. See slots.go. + s.abandonVM(stopped, ref, "force-stop after the provisioning deadline did not return", log) } } @@ -1656,20 +1736,30 @@ func stopVMAsync(macVM vm.MacOSVM) <-chan struct{} { return done } -// stopVMBounded force-stops a VM without letting the teardown pin the caller. -// Used by every macOS cleanup path that runs while a slot could still be held -// or a job could still be waiting on this goroutine. -func stopVMBounded(macVM vm.MacOSVM, grace time.Duration, log *slog.Logger) { +// stopVMBounded force-stops a VM without letting the teardown pin the caller, +// and charges the VM to the abandoned registry if the grace expires. +// +// Used by EVERY macOS cleanup path, not just the ones holding a slot. Three of +// them used to call macosVM.Stop() directly — handleCompleted, destroyAll and +// sweepOrphanRunners — and the sweep is called synchronously from the +// cleanupTicker arm of Run's select, i.e. on the scheduler's only event loop. +// A wedged Vz stop there does not cost one job's slot, it costs the whole +// daemon: no queued, in_progress or completed handling for any platform, and +// no drain on SIGTERM. That is strictly worse than the leak this PR fixes, so +// there is no such thing as a macOS stop that may block its caller. +func (s *Scheduler) stopVMBounded(macVM vm.MacOSVM, ref macVMRef, log *slog.Logger) { + grace := s.teardownGrace() timer := time.NewTimer(grace) defer timer.Stop() + done := stopVMAsync(macVM) select { - case <-stopVMAsync(macVM): + case <-done: case <-timer.C: log.Warn("macOS VM stop did not return within the grace period; continuing without it", "grace", grace) + s.abandonVM(done, ref, "teardown did not return within the stop grace", log) } } - // handleLocalJob provisions a runner using the local containerd Runtime. func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent) { jobID := event.JobID @@ -2049,7 +2139,11 @@ func (s *Scheduler) handleCompleted(ctx context.Context, event providers.JobEven resetBackoff(event.Repo) job.cancel() if job.macosVM != nil { - job.macosVM.Stop() + // Bounded (#196). This ran as a bare Stop(), so a wedged Vz teardown + // parked the completed-event handler forever — and with it the + // deferred sweepOrphanRunners above, which is how the node reconciles + // runners after every completion. + s.stopVMBounded(job.macosVM, job.macRef(ownerKey), log) } else if job.dispatched != "" && s.cfg.LinuxDispatcher != nil { if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), job.dispatched); err != nil { log.Warn("failed to destroy dispatched runner", "error", err) @@ -2133,7 +2227,13 @@ func (s *Scheduler) destroyAll() { s.cfg.Log.Info("destroying runner on shutdown", "job_id", key.JobID, "provider", key.Provider) job.cancel() if job.macosVM != nil { - job.macosVM.Stop() + // Bounded (#196). destroyAll is the force-kill arm of drain(), so + // a bare Stop() here meant a wedged Vz teardown stopped the daemon + // from ever exiting: drain() never returns, Run() never returns, + // and SIGTERM has to be escalated to SIGKILL by hand. The + // shutdown timeout exists precisely so shutdown is bounded; a + // teardown that ignores it defeats it. + s.stopVMBounded(job.macosVM, job.macRef(key), s.cfg.Log.With("job_id", key.JobID)) } else if job.dispatched != "" && s.cfg.LinuxDispatcher != nil { if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), job.dispatched); err != nil { s.cfg.Log.Warn("failed to destroy dispatched runner", "job_id", key.JobID, "error", err) @@ -2174,15 +2274,26 @@ func (s *Scheduler) handleHealthz(w http.ResponseWriter, r *http.Request) { // returned 200. held_slots is read straight off the semaphores and so // counts the capacity that is actually spoken for. held == capacity with // active_jobs 0, sustained, means a stuck provision or a leak. + // + // abandoned_macos_vms is the companion number: slots that were given BACK + // by walking away from a teardown. It is the cost side of the #196 trade — + // each one is a macOS guest still holding its RAM that only a daemon + // restart reclaims — and at the cap the macOS pool stops taking work, so + // an operator has to be able to see it without reading the log. status := map[string]any{ - "status": "ok", - "active_jobs": activeJobs, - "max_concurrent": s.cfg.MaxConcurrent, - "held_slots": s.HeldSlots(), - "slot_capacity": s.SlotCapacity(), - "slots": s.SlotUsage(), - "draining": draining, - "uptime": time.Since(s.startTime).String(), + "status": "ok", + "active_jobs": activeJobs, + "max_concurrent": s.cfg.MaxConcurrent, + "held_slots": s.HeldSlots(), + "slot_capacity": s.SlotCapacity(), + "slots": s.SlotUsage(), + "abandoned_macos_vms": s.AbandonedMacOSVMs(), + "abandoned_macos_vm_cap": s.abandonedMacVMCap(), + "draining": draining, + "uptime": time.Since(s.startTime).String(), + } + if n := s.AbandonedMacOSVMDetails(); len(n) > 0 { + status["abandoned_macos_vm_details"] = n } w.Header().Set("Content-Type", "application/json") @@ -2465,7 +2576,16 @@ func (s *Scheduler) sweepOrphanRunners() { metrics.JobsActive.Dec() v.rj.cancel() if v.rj.macosVM != nil { - v.rj.macosVM.Stop() + // THE WORST ONE (#196). sweepOrphanRunners is called + // SYNCHRONOUSLY from the cleanupTicker arm of Run's select — the + // scheduler's only event loop. A tracked macOS job nominated as an + // orphan whose Vz stop wedges (the exact failure this PR exists + // for) used to block that loop forever: no queued, in_progress or + // completed handling for ANY platform, no drain on SIGTERM, the + // daemon alive and answering /healthz with 200 and doing nothing. + // That is strictly worse than the single leaked slot the rest of + // this PR is about, and it is one call away. + s.stopVMBounded(v.rj.macosVM, v.rj.macRef(v.key), s.cfg.Log.With("runner", v.name, "job_id", v.key.JobID)) } else if v.rj.dispatched != "" && s.cfg.LinuxDispatcher != nil { if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), v.rj.dispatched); err != nil { s.cfg.Log.Warn("failed to destroy orphaned dispatched runner", "runner", v.name, "error", err) diff --git a/pkg/scheduler/slots.go b/pkg/scheduler/slots.go index 54db26ca..23a507ba 100644 --- a/pkg/scheduler/slots.go +++ b/pkg/scheduler/slots.go @@ -3,6 +3,7 @@ package scheduler import ( "context" "log/slog" + "sort" "sync" "time" ) @@ -242,6 +243,200 @@ func (s *Scheduler) SlotCapacity() int { return total } +// --------------------------------------------------------------------------- +// Abandoned macOS VMs. +// +// The #196 fix trades a leaked concurrency slot for a leaked goroutine: when a +// Vz stop does not return, we walk away from it and give the slot back. That +// trade is right, but it is not free, and until this registry existed it was +// also UNBOUNDED and INVISIBLE. +// +// What one abandonment actually costs. Three goroutines (the WaitForRunner +// wait, abandonMacProvision's stopVMAsync, and — before the fix below — +// the caller's redundant stopVMBounded blocking inside darwinMacOSVM's +// stopOnce), each retaining a live *darwinMacOSVM, and, far more expensively, +// a STILL-RUNNING macOS guest holding its whole configured RAM +// (MacOSVMConfig.MemoryMB — gigabytes) plus its disk clone. +// +// What used to bound it. The semaphore, by accident: leak the only slot and +// the node provisions zero further macOS VMs, so exactly one guest could ever +// be stranded. Releasing the slot on the failure path is precisely what +// removed that accidental cap, and MaxMacOSVMs no longer charges for abandoned +// VMs because they are not tracked and no longer hold a slot. Left alone, a +// mac with a repeatably wedging guest would keep provisioning until +// Virtualization.framework refused (its concurrent macOS-guest limit is low — +// about two on Apple Silicon) or the host ran out of memory. +// +// So the bound is explicit now: count them, cap them, say so loudly, and +// report the count next to held_slots. A node that has given up on more than a +// couple of guests is broken in a way a restart fixes and no further +// provisioning will; refusing new macOS work there turns unbounded RAM growth +// into a legible, bounded failure that `ephemerd status` names. +// --------------------------------------------------------------------------- + +// defaultMacAbandonedVMCap is how many macOS VMs may be abandoned-but-not- +// confirmed-dead before the macOS pool stops taking work. +// +// Deliberately tiny. Each entry is a live guest holding multiple gigabytes, +// and Virtualization.framework itself only permits about two concurrent macOS +// guests on Apple Silicon — so the useful budget is "one transient hiccup", +// not "keep trying". +const defaultMacAbandonedVMCap = 2 + +// macVMRef identifies one macOS VM instance for the abandoned registry and for +// the log lines about it. VMID is the PER-ATTEMPT instance id (see +// macVMInstanceID), not the job id: two provisioning attempts for the same job +// are two different VMs and must be countable separately. +type macVMRef struct { + JobID int64 + VMID string +} + +// AbandonedMacVM is one macOS VM whose teardown was given up on, as reported +// by status/healthz. +type AbandonedMacVM struct { + JobID int64 `json:"job_id"` + VMID string `json:"vm_id"` + Reason string `json:"reason"` + HeldFor string `json:"held_for"` +} + +type abandonedEntry struct { + ref macVMRef + reason string + since time.Time +} + +// abandonedMacVMs is the registry. Entries are added when a stop is walked +// away from and removed if that stop ever returns — which does happen: a Vz +// stop can be slow rather than wedged, and a node that recovers must be +// allowed to serve macOS jobs again without a restart. +type abandonedMacVMs struct { + mu sync.Mutex + next uint64 + live map[uint64]abandonedEntry +} + +func (a *abandonedMacVMs) charge(ref macVMRef, reason string) (uint64, int) { + a.mu.Lock() + defer a.mu.Unlock() + if a.live == nil { + a.live = make(map[uint64]abandonedEntry) + } + a.next++ + id := a.next + a.live[id] = abandonedEntry{ref: ref, reason: reason, since: time.Now()} + return id, len(a.live) +} + +// discharge removes an entry and reports the remaining count and how long the +// VM was outstanding. +func (a *abandonedMacVMs) discharge(id uint64) (time.Duration, int) { + a.mu.Lock() + defer a.mu.Unlock() + e, ok := a.live[id] + delete(a.live, id) + remaining := len(a.live) + if !ok { + return 0, remaining + } + return time.Since(e.since), remaining +} + +func (a *abandonedMacVMs) count() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.live) +} + +func (a *abandonedMacVMs) snapshot() []AbandonedMacVM { + a.mu.Lock() + defer a.mu.Unlock() + out := make([]AbandonedMacVM, 0, len(a.live)) + for _, e := range a.live { + out = append(out, AbandonedMacVM{ + JobID: e.ref.JobID, + VMID: e.ref.VMID, + Reason: e.reason, + HeldFor: time.Since(e.since).Truncate(time.Second).String(), + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].VMID < out[j].VMID }) + return out +} + +// AbandonedMacOSVMs is how many macOS VMs this daemon has given up on and not +// yet seen die. Non-zero means guests are holding host RAM that nothing in +// this process can reclaim; at or above the cap the macOS pool is cordoned. +func (s *Scheduler) AbandonedMacOSVMs() int { return s.macAbandoned.count() } + +// AbandonedMacOSVMDetails lists them, newest first by instance id. +func (s *Scheduler) AbandonedMacOSVMDetails() []AbandonedMacVM { return s.macAbandoned.snapshot() } + +// abandonedMacVMCap is the effective cap; the field exists so tests can set it +// to 1 instead of staging two wedged guests. +func (s *Scheduler) abandonedMacVMCap() int { + if s.macAbandonedCap > 0 { + return s.macAbandonedCap + } + return defaultMacAbandonedVMCap +} + +// macPoolCordoned reports whether the macOS pool has given up on so many VMs +// that it must stop provisioning, and how many. +// +// Checked twice on the dispatch path for the same reason the #154 cordon is: +// once before blocking on the semaphore, and once after — a dispatch parked on +// a full pool can sit there for the entire life of the job ahead of it, and +// the node can break underneath it in that window. +func (s *Scheduler) macPoolCordoned() (int, int, bool) { + n, limit := s.macAbandoned.count(), s.abandonedMacVMCap() + return n, limit, n >= limit +} + +// abandonVM charges a walked-away-from teardown to the registry and arranges +// for it to be discharged if the stop ever returns. +// +// done is the channel stopVMAsync returns. If it is already closed the stop +// landed in the race between the caller's deadline and this call, and nothing +// is abandoned — that distinction matters, because charging a VM that is in +// fact dead would cordon a healthy node. +func (s *Scheduler) abandonVM(done <-chan struct{}, ref macVMRef, reason string, log *slog.Logger) { + select { + case <-done: + return // teardown finished after all; nothing was abandoned + default: + } + + id, n := s.macAbandoned.charge(ref, reason) + limit := s.abandonedMacVMCap() + log.Error("abandoned a macOS VM whose teardown never returned; its guest is still holding host RAM", + "vm", ref.VMID, + "job_id", ref.JobID, + "reason", reason, + "abandoned_macos_vms", n, + "cap", limit, + "detail", "walking away is what keeps the concurrency slot from leaking (#196), but the guest is not gone — only a daemon restart reclaims it") + if n >= limit { + log.Error("macOS pool cordoned: too many macOS VMs abandoned without confirmed death — refusing further macOS provisioning on this node", + "abandoned_macos_vms", n, + "cap", limit, + "detail", "each abandoned guest holds its full configured RAM and Virtualization.framework allows only a couple of concurrent macOS guests; provisioning past this point buys nothing and can OOM the host. Restart ephemerd to clear them.") + } + + // Discharge if it ever finishes. A Vz stop can be merely slow, and a node + // that recovers on its own must not need a restart to serve macOS again. + go func() { + <-done + held, remaining := s.macAbandoned.discharge(id) + log.Warn("a previously abandoned macOS VM finally finished stopping", + "vm", ref.VMID, + "job_id", ref.JobID, + "held_for", held.Truncate(time.Second), + "abandoned_macos_vms", remaining) + }() +} + // provisionUnwindGrace and teardownGrace are the effective bounds on macOS VM // teardown. They read from the scheduler so tests can compress them, and fall // back to the package defaults for a zero-valued Scheduler. diff --git a/pkg/scheduler/slots_test.go b/pkg/scheduler/slots_test.go index 14bde2ce..195d104e 100644 --- a/pkg/scheduler/slots_test.go +++ b/pkg/scheduler/slots_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "log/slog" "net/http" "net/http/httptest" @@ -465,10 +466,12 @@ func TestAwaitUnwind(t *testing.T) { // capacity is not. func TestStopVMBoundedDoesNotHangOnAWedgedStop(t *testing.T) { macVM := newWedgedMacVM(t) + s := New(Config{MaxMacOSVMs: 1, Log: quietLogger()}) + s.macStopGrace = 20 * time.Millisecond done := make(chan struct{}) go func() { defer close(done) - stopVMBounded(macVM, 20*time.Millisecond, quietLogger()) + s.stopVMBounded(macVM, macVMRef{JobID: 1, VMID: "1-wedged"}, quietLogger()) }() select { case <-done: @@ -478,6 +481,11 @@ func TestStopVMBoundedDoesNotHangOnAWedgedStop(t *testing.T) { if macVM.stopCalls.Load() != 1 { t.Errorf("Stop called %d times, want 1", macVM.stopCalls.Load()) } + // The VM it walked away from must be charged, or nothing bounds how many + // of them a node can strand. + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d after walking away from a wedged stop, want 1", n) + } } // TestSlotUsageReportsHeldSlots is the reporting gap from the issue: a node @@ -601,6 +609,319 @@ func TestHealthzExposesHeldSlots(t *testing.T) { } } +// --------------------------------------------------------------------------- +// MAJOR 1: no macOS Stop() may block its caller — least of all the event loop. +// --------------------------------------------------------------------------- + +// seedMacOSRunner files a tracked macOS job plus its runner-ledger entry, the +// state the orphan sweep, handleCompleted and destroyAll all operate on. +func seedMacOSRunner(s *Scheduler, prov providers.Provider, jobID int64, name string, dispatchedAt time.Time, macVM vm.MacOSVM) jobKey { + key := jobKey{Provider: prov.Name(), JobID: jobID} + _, cancel := context.WithCancel(context.Background()) + s.mu.Lock() + s.running[key] = &runningJob{ + provider: prov, + claim: &providers.Claim{RunnerID: jobID * 10, RunnerName: name, Repo: "myrepo"}, + repo: "myrepo", + cancel: cancel, + macosVM: macVM, + macVMID: fmt.Sprintf("%d-seeded", jobID), + startedAt: dispatchedAt, + } + s.runners[name] = &runnerBinding{ + intentKey: key, + dispatchedAt: dispatchedAt, + observable: true, + labelSet: labelSetKey([]string{"self-hosted", "macos"}), + } + s.mu.Unlock() + return key +} + +// newTeardownTestScheduler builds a scheduler whose macOS teardown bounds are +// milliseconds, for the paths that tear a TRACKED macOS job down. +func newTeardownTestScheduler(t *testing.T, prov providers.Provider, sweep bool) *Scheduler { + t.Helper() + s := New(Config{ + Providers: []providers.Provider{prov}, + MacOSVMConfig: &vm.MacOSVMConfig{}, + MaxMacOSVMs: 1, + ShutdownTimeout: 50 * time.Millisecond, + OrphanSweep: OrphanSweepConfig{Enabled: sweep, Grace: 10 * time.Minute}, + Log: quietLogger(), + }) + s.webhookMode = true + s.busyProbe = idleProbe + s.macStopGrace = 50 * time.Millisecond + s.macUnwindGrace = 50 * time.Millisecond + return s +} + +// mustReturnWithin fails (rather than hangs) if fn has not returned by d. +func mustReturnWithin(t *testing.T, d time.Duration, what string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + fn() + }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("%s did not return within %s: a macOS VM Stop() is blocking its caller", what, d) + } +} + +// TestSweepOrphanRunners_WedgedMacOSVMCannotBlockTheEventLoop is the worst of +// the three unbounded Stop() sites the #196 fix left behind. +// +// sweepOrphanRunners is called SYNCHRONOUSLY from the cleanupTicker arm of +// Run's select — the scheduler's one and only event loop. A tracked macOS job +// nominated as an orphan whose Vz stop wedges (exactly the failure that +// produced the 28-hour outage) parked that loop forever: no queued, +// in_progress or completed handling for any platform, and no drain on SIGTERM. +// That is strictly worse than the leaked slot the rest of the PR fixes. +// +// Without the fix this test FAILS on mustReturnWithin: the sweep sits inside +// wedgedMacVM.Stop() until the test binary's own timeout. +func TestSweepOrphanRunners_WedgedMacOSVMCannotBlockTheEventLoop(t *testing.T) { + prov := &reportingProvider{claimCountingProvider: newClaimCountingProvider("mac-sweep")} + s := newTeardownTestScheduler(t, prov, true) + + macVM := newWedgedMacVM(t) + key := seedMacOSRunner(s, prov, 4242, "mac-r-1", time.Now().Add(-30*time.Minute), macVM) + + mustReturnWithin(t, 5*time.Second, "sweepOrphanRunners", s.sweepOrphanRunners) + + if macVM.stopCalls.Load() == 0 { + t.Error("the orphaned macOS VM was never stopped; the sweep must still tear it down") + } + s.mu.Lock() + _, stillRunning := s.running[key] + s.mu.Unlock() + if stillRunning { + t.Error("orphaned macOS job is still tracked after the sweep") + } + // A stop it walked away from has to be counted, or nothing bounds them. + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d after the sweep abandoned a wedged stop, want 1", n) + } +} + +// TestHandleCompleted_WedgedMacOSVMCannotBlockTheHandler: the completed-event +// handler also ran a bare Stop(). A wedge there parks the handler forever, and +// with it the deferred sweepOrphanRunners that reconciles the ledger after +// every completion. +// +// Without the fix this FAILS on mustReturnWithin. +func TestHandleCompleted_WedgedMacOSVMCannotBlockTheHandler(t *testing.T) { + prov := &reportingProvider{claimCountingProvider: newClaimCountingProvider("mac-completed")} + s := newTeardownTestScheduler(t, prov, false) + + macVM := newWedgedMacVM(t) + key := seedMacOSRunner(s, prov, 5150, "mac-r-2", time.Now(), macVM) + + event := providers.JobEvent{ + Provider: prov, + Action: "completed", + Repo: "myrepo", + JobID: 5150, + RunnerName: "mac-r-2", + Conclusion: "success", + } + mustReturnWithin(t, 5*time.Second, "handleCompleted", func() { + s.handleCompleted(context.Background(), event) + }) + + if macVM.stopCalls.Load() == 0 { + t.Error("the completed job's macOS VM was never stopped") + } + s.mu.Lock() + _, stillRunning := s.running[key] + s.mu.Unlock() + if stillRunning { + t.Error("completed macOS job is still tracked") + } +} + +// TestDrain_WedgedMacOSVMCannotBlockShutdown: destroyAll is the force-kill arm +// of drain(), reached only after ShutdownTimeout has already expired. A bare +// Stop() there meant a wedged guest kept the process alive indefinitely — +// drain never returns, Run never returns, and SIGTERM has to be escalated to +// SIGKILL by hand. The shutdown timeout exists to bound shutdown; a teardown +// that ignores it defeats the entire mechanism. +// +// Without the fix this FAILS on mustReturnWithin. +func TestDrain_WedgedMacOSVMCannotBlockShutdown(t *testing.T) { + prov := &reportingProvider{claimCountingProvider: newClaimCountingProvider("mac-drain")} + s := newTeardownTestScheduler(t, prov, false) + + macVM := newWedgedMacVM(t) + seedMacOSRunner(s, prov, 6006, "mac-r-3", time.Now(), macVM) + + mustReturnWithin(t, 10*time.Second, "drain", s.drain) + + if macVM.stopCalls.Load() == 0 { + t.Error("the running macOS VM was never stopped on shutdown") + } + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d after drain force-killed everything, want 0", n) + } +} + +// --------------------------------------------------------------------------- +// MAJOR 3: the abandoned-VM trade is bounded and legible. +// --------------------------------------------------------------------------- + +// TestMacOSPoolCordonsPastTheAbandonedVMCap. +// +// The semaphore used to be the accidental limiter on stranded guests: leak the +// only slot and the node provisions zero more, so at most one guest could ever +// be abandoned. Releasing the slot on the failure path is exactly what removed +// that, and an abandoned VM is not tracked and no longer charged against +// MaxMacOSVMs — so without an explicit cap a repeatably-wedging guest lets the +// node keep provisioning until Virtualization.framework refuses (about two +// concurrent macOS guests on Apple Silicon) or the host runs out of RAM. +// +// Without the cap this test FAILS: the second dispatch constructs and boots +// another VM instead of being refused. +func TestMacOSPoolCordonsPastTheAbandonedVMCap(t *testing.T) { + prov := newMockProvider("mac-cap") + defer func() { _ = prov.Stop(context.Background()) }() + + // Constructed by hand rather than via newWedgedMacVM: this test needs to + // release the wedged Stop() mid-test to prove the cordon lifts. + first := &wedgedMacVM{stopReturns: make(chan struct{})} + s := newSlotTestScheduler(t, prov, first) + s.macAbandonedCap = 1 + + runMacDispatch(t, s, macEvent(prov, 1), 5*time.Second) + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Fatalf("AbandonedMacOSVMs() = %d after one unkillable VM, want 1", n) + } + + // Second job: must be refused before anything is constructed or booted. + var created atomic.Int32 + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { + created.Add(1) + return &wedgedMacVM{stopReturns: make(chan struct{})}, nil + } + runMacDispatch(t, s, macEvent(prov, 2), 5*time.Second) + if got := created.Load(); got != 0 { + t.Errorf("provisioned %d macOS VMs past the abandoned cap; the point of the cap is that it provisions none", got) + } + assertCapacityIntact(t, s.macSem) + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d, want 0", n) + } + + // The count is reportable, not just internal. + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d, want 1", n) + } + if d := s.AbandonedMacOSVMDetails(); len(d) != 1 || d[0].JobID != 1 { + t.Errorf("AbandonedMacOSVMDetails() = %+v, want one entry for job 1", d) + } + + // The cordon is not permanent: a merely-slow Vz stop that eventually + // returns discharges its entry and the pool takes work again. A node that + // recovers on its own must not need a restart. + close(first.stopReturns) + deadline := time.Now().Add(5 * time.Second) + for s.AbandonedMacOSVMs() != 0 && time.Now().Before(deadline) { + time.Sleep(2 * time.Millisecond) + } + if n := s.AbandonedMacOSVMs(); n != 0 { + t.Fatalf("AbandonedMacOSVMs() = %d after the abandoned stop finally returned, want 0", n) + } + + runMacDispatch(t, s, macEvent(prov, 3), 5*time.Second) + if created.Load() == 0 { + t.Error("the macOS pool stayed cordoned after its abandoned VMs were confirmed dead") + } +} + +// TestStatusAndHealthzReportAbandonedMacOSVMs: held_slots showed the outage; +// abandoned_macos_vms shows what the fix for it costs. An operator has to be +// able to see stranded guests without reading the log. +func TestStatusAndHealthzReportAbandonedMacOSVMs(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + s.macAbandoned.charge(macVMRef{JobID: 77, VMID: "77-deadbeef"}, "test") + + cs := &controlServer{sched: s, log: quietLogger()} + resp, err := cs.Status(context.Background(), &apiv1.StatusRequest{}) + if err != nil { + t.Fatalf("Status() error: %v", err) + } + if resp.AbandonedMacosVms != 1 { + t.Errorf("Status.AbandonedMacosVms = %d, want 1", resp.AbandonedMacosVms) + } + if resp.AbandonedMacosVmCap != defaultMacAbandonedVMCap { + t.Errorf("Status.AbandonedMacosVmCap = %d, want %d", resp.AbandonedMacosVmCap, defaultMacAbandonedVMCap) + } + // The pre-existing contract is untouched. + if resp.Status != "ok" || resp.ActiveJobs != 0 || resp.MaxConcurrent != 2 { + t.Errorf("existing Status fields changed: %+v", resp) + } + + rec := httptest.NewRecorder() + s.handleHealthz(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + var body struct { + ActiveJobs int `json:"active_jobs"` + HeldSlots int `json:"held_slots"` + AbandonedMacOSVMs int `json:"abandoned_macos_vms"` + AbandonedCap int `json:"abandoned_macos_vm_cap"` + Details []AbandonedMacVM `json:"abandoned_macos_vm_details"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decoding healthz body %q: %v", rec.Body.String(), err) + } + if body.AbandonedMacOSVMs != 1 { + t.Errorf("healthz abandoned_macos_vms = %d, want 1", body.AbandonedMacOSVMs) + } + if body.AbandonedCap != defaultMacAbandonedVMCap { + t.Errorf("healthz abandoned_macos_vm_cap = %d, want %d", body.AbandonedCap, defaultMacAbandonedVMCap) + } + if len(body.Details) != 1 || body.Details[0].VMID != "77-deadbeef" { + t.Errorf("healthz abandoned_macos_vm_details = %+v, want the stranded VM named", body.Details) + } +} + +// --------------------------------------------------------------------------- +// MINOR 5: the deadline path already stopped the VM; do not stop it again. +// --------------------------------------------------------------------------- + +// TestHandleMacOSJob_DeadlinePathStopsTheVMExactlyOnce. +// +// waitForMacRunnerBounded force-stops the VM when the deadline fires (#178, +// unchanged). The caller's error path then called stopVMBounded on the very +// same VM. Against darwinMacOSVM's stopOnce that is never a cheap no-op: the +// second caller BLOCKS until the first invocation returns, so on the wedged +// guest this whole PR is about it burned a full teardown grace and parked a +// third goroutine inside the Once — while a ghost JIT runner still needed +// deregistering. +// +// Without the fix this test FAILS: Stop is called twice and the second +// abandonment is charged against the cap as well, so a single wedged VM +// consumes the node's entire abandoned-VM budget. +func TestHandleMacOSJob_DeadlinePathStopsTheVMExactlyOnce(t *testing.T) { + prov := newMockProvider("mac-double-stop") + defer func() { _ = prov.Stop(context.Background()) }() + + macVM := newWedgedMacVM(t) + s := newSlotTestScheduler(t, prov, macVM) + s.macAbandonedCap = 99 // not the subject here + + runMacDispatch(t, s, macEvent(prov, 999), 5*time.Second) + + if got := macVM.stopCalls.Load(); got != 1 { + t.Errorf("Stop called %d times on the deadline path, want 1 — the second call re-enters a sync.Once that is still wedged", got) + } + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d for ONE wedged VM, want 1 — a redundant stop must not double-charge the cap", n) + } +} + // --------------------------------------------------------------------------- // MINOR 4: the leak escalation is per-pool, not per-node. // --------------------------------------------------------------------------- From 01ab77f161f380a71c50ac6b83f29b0281e37a3f Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Sat, 5 Sep 2026 01:07:28 -0700 Subject: [PATCH 4/5] fix(scheduler): give each macOS provisioning attempt its own VM instance id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releasing the macOS slot before teardown (#196) opened a same-job re-dispatch race, and the comment that said it was safe had the invariant exactly backwards. The comment claimed: "every VM, its clone, and its shared directory are keyed by job ID, so a fresh dispatch cannot touch anything this teardown still owns." Everything being keyed by job ID is not what makes a fresh dispatch safe — it is precisely what makes it collide. Nothing carried a per-attempt discriminator: jobDir /vm/macos/jobs/ (macosvm_darwin.go:49) clonePath /vm/macos/jobs/.img (macosvm_darwin.go:93) auxPath /vm/macos/jobs/.aux (macosvm_darwin.go:94) artifacts artifacts.ArtifactsDir(dataDir, jobID) and the failure paths now call unsee() — which clears BOTH seen and pending — before teardown, so the same job ID can immediately re-enter handleQueued. The poll re-emits a still-queued job about every 30s (webhook mode gets there via the catch-up poll or a redelivery), admitDispatch admits it, and maxProvisionAttempts allows several passes. The sequence: attempt 1 hits the provisioning deadline, releases the slot, and walks away from a Stop() still wedged in Virtualization.framework — its guest is alive with the .img and .aux files open. Attempt 2 takes the freed slot and apfsCopies over those same files, then writes its JIT config into the same share. Minutes later attempt 1's abandoned stop returns and runs os.Remove(clonePath) + os.Remove(auxPath) + os.RemoveAll(jobDir) (macosvm_darwin.go:851-865), deleting attempt 2's running disk and its .jit_config. Attempt 2's runner can never register, so it times out on its own deadline and strands another guest doing it. Self-sustaining, and the same applies to artifacts.Cleanup(artifactsDir) on every error branch. Approach. Two were available: refuse to re-provision a job ID that has an abandoned-but-unconfirmed-dead VM, or give each attempt disjoint paths. The second is smaller and provably safe — disjointness is a property of the names, not of a check somebody has to remember to keep correct — and it is strictly better for availability, since attempt 2 actually runs instead of waiting on a guest that may never die. It also needs no change to pkg/vm/macosvm_darwin.go: darwinMacOSVM derives every path it owns from the id string the scheduler hands NewMacOSVM, and that id is used for nothing but paths and log fields. So macVMInstanceID(jobID) returns "-<4 random hex bytes>". The job id stays in the name because it is what an operator greps for; the suffix is what makes attempts disjoint. Random rather than a counter so a daemon restart cannot re-issue an id whose files are still on disk from before it. The same id keys the artifacts dir, is stored on runningJob.macVMID, and is what every teardown site reports to the abandoned-VM registry, so the log and status name the specific VM rather than just the job. The new invariant, and the corrected comment: a fresh dispatch of the same job ID is safe not because paths are shared-but-unreachable, but because every path a teardown touches belongs to ONE attempt — the per-attempt VM instance id (share, clones, artifacts) or the per-claim runner name (ReleaseJob) — so a late teardown can only ever delete its own files. Tests, verified by reverting macVMInstanceID to fmt.Sprintf("%d", jobID): - TestHandleMacOSJob_RedispatchOfTheSameJobGetsDisjointResources drives two attempts at one job ID through handleMacOSJob against a fake VM that creates real files under the darwinMacOSVM layout and deletes them in Stop(). Reverted, it reports the shared id and then: "attempt 2's JIT config after attempt 1's late teardown is gone (.../vm/macos/jobs/100817345293/.jit_config)" "attempt 2's disk clone after attempt 1's late teardown is gone (.../vm/macos/jobs/100817345293.img)" which is the production failure, reproduced. - TestMacVMInstanceIDDiscriminatesAttempts — reverted: "instance id \"100817345293\" does not lead with the job id" (no suffix at all). NOTE FOR MAC-SIDE VERIFICATION: pkg/vm/macosvm_darwin.go is darwin-only cgo and does not build here, so it is deliberately untouched. The change relies on its documented behaviour that jobDir/clonePath/auxPath derive from the constructor's id argument and nothing else parses those names. --- pkg/scheduler/scheduler.go | 79 ++++++++++++-- pkg/scheduler/slots_test.go | 211 ++++++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 9 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 6bb26741..0bac88e2 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -2,6 +2,8 @@ package scheduler import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -364,7 +366,7 @@ type runningJob struct { artifactsDir string // non-empty if OCI artifacts were extracted for this job dispatched string // non-empty if dispatched to Linux VM worker (stores container name) macosVM vm.MacOSVM // non-nil if running as a macOS VM job - macVMID string // macOS VM instance id; names this job's on-disk paths + macVMID string // per-attempt macOS VM instance id (macVMInstanceID); owns this job's on-disk paths startedAt time.Time } @@ -1373,9 +1375,25 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent "detail", "each one is a live guest holding its full configured RAM; starting more risks OOMing the host and Vz will refuse anyway. Restart ephemerd to reclaim them.") } - // vmID names the VM instance. Still the bare job id at this point; the - // per-attempt discriminator lands in the follow-up commit. - vmID := fmt.Sprintf("%d", jobID) + // PER-ATTEMPT VM IDENTITY. Every macOS resource used to be keyed by job id + // alone: the VM's shared directory (/vm/macos/jobs/), its disk + // and aux clones (.img / .aux) and its extracted OCI artifacts. On + // the failure paths below this function now releases the slot BEFORE + // teardown, which means the same job id can be re-dispatched (poll re-emits + // it, or the webhook catch-up poll does) while the previous attempt's VM is + // still alive with a stop wedged inside Virtualization.framework. + // + // With shared paths that is mutual destruction: attempt 2 apfsCopies over + // the .img/.aux files attempt 1's guest still has open and writes its JIT + // config into the same share, and then attempt 1's abandoned stop finally + // returns minutes later and runs os.Remove(clonePath) + os.RemoveAll(jobDir) + // — deleting attempt 2's disk and its .jit_config, so attempt 2's runner can + // never register and it times out too. Self-sustaining, and each round + // strands another guest. + // + // A per-attempt suffix makes the attempts disjoint by construction, so a + // late teardown can only ever delete its own files. + vmID := macVMInstanceID(jobID) ref := macVMRef{JobID: jobID, VMID: vmID} log = log.With("vm", vmID) @@ -1426,6 +1444,10 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent image := s.resolveImage(ctx, &event, "darwin") var artifactsDir string if image != "" && s.cfg.Artifacts != nil { + // Keyed by the VM instance, not the job: this directory is deleted by + // artifacts.Cleanup on every failure path below, all of which now run + // after the slot is released and therefore possibly concurrently with + // a fresh attempt at the same job id. artifactsDir = artifacts.ArtifactsDir(s.cfg.DataDir, vmID) log.Info("extracting OCI artifacts for macOS VM job", "image", image, "dest", artifactsDir) if err := s.cfg.Artifacts.Extract(ctx, image, artifactsDir); err != nil { @@ -1450,7 +1472,8 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent return } - // Create the macOS VM + // Create the macOS VM. vmID (not the bare job id) is what every path this + // VM owns on disk is named after — see the per-attempt identity note above. macVM, err := s.newMacOSVM(*s.cfg.MacOSVMConfig, vmID) if err != nil { log.Error("failed to create macOS VM", "error", err) @@ -1593,10 +1616,20 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // may delay the next macOS job. macVM.Stop() waits up to 15s for a // graceful guest shutdown before forcing (and on the #196 node the // force did not return either), and ReleaseJob is a GitHub API call - // on context.Background() with no deadline at all. This is safe - // against a newly admitted job: every VM, its clone, and its shared - // directory are keyed by job ID, so a fresh dispatch cannot touch - // anything this teardown still owns. + // on context.Background() with no deadline at all. + // + // WHAT MAKES THAT SAFE. Releasing here means the SAME job id can be + // dispatched again while this teardown is still running — unsee() has + // cleared seen/pending on the failure paths, the poll re-emits a still- + // queued job about every 30s, and admitDispatch will admit it. So the + // invariant is NOT "a fresh dispatch cannot collide because everything + // is keyed by job id" — everything being keyed by job id is precisely + // what WOULD collide. The invariant is that every path this teardown + // touches is keyed by the PER-ATTEMPT VM instance id (vmID: the VM's + // shared dir, its .img/.aux clones, its artifacts dir) or by the + // per-claim runner name (ReleaseJob), so a later attempt at the same + // job never shares a byte of state with this one and this teardown can + // only ever delete its own files. slot.release() if exists { @@ -1760,6 +1793,34 @@ func (s *Scheduler) stopVMBounded(macVM vm.MacOSVM, ref macVMRef, log *slog.Logg s.abandonVM(done, ref, "teardown did not return within the stop grace", log) } } + +// macVMInstanceID names one PROVISIONING ATTEMPT, not one job. +// +// Everything a macOS VM owns on disk is named after this: its virtio-fs share +// (/vm/macos/jobs/), its disk and aux clones (.img, .aux) +// and its extracted OCI artifacts. Since the #196 fix releases the concurrency +// slot before teardown, a second attempt at the same job can start while the +// first attempt's VM is still alive with an unkillable stop pending — and that +// first attempt's teardown ends in os.Remove(clonePath) and +// os.RemoveAll(jobDir). Sharing those paths between attempts means attempt 1's +// late cleanup deletes attempt 2's running disk and its .jit_config; attempt 2 +// then waits out its own deadline for a runner that can never register, and +// strands another guest doing it. +// +// The job id stays in the name because it is what an operator greps for; the +// random suffix is what makes the attempts disjoint. It is random rather than +// a counter so that a daemon restart cannot re-issue an id whose files are +// still on disk from before the restart. +func macVMInstanceID(jobID int64) string { + var b [4]byte + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand does not fail in practice; if it ever does, a + // timestamp still discriminates attempts (they are minutes apart). + return fmt.Sprintf("%d-%d", jobID, time.Now().UnixNano()) + } + return fmt.Sprintf("%d-%s", jobID, hex.EncodeToString(b[:])) +} + // handleLocalJob provisions a runner using the local containerd Runtime. func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent) { jobID := event.JobID diff --git a/pkg/scheduler/slots_test.go b/pkg/scheduler/slots_test.go index 195d104e..8f5a1f0a 100644 --- a/pkg/scheduler/slots_test.go +++ b/pkg/scheduler/slots_test.go @@ -9,6 +9,8 @@ import ( "log/slog" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -769,6 +771,215 @@ func TestDrain_WedgedMacOSVMCannotBlockShutdown(t *testing.T) { } } +// --------------------------------------------------------------------------- +// MAJOR 2: a re-dispatch of the same job must not share state with an +// abandoned attempt at that job. +// --------------------------------------------------------------------------- + +// pathedMacVM owns real files, named the way darwinMacOSVM names them: +// +// /vm/macos/jobs/ the virtio-fs share (holds .jit_config) +// /vm/macos/jobs/.img the APFS disk clone +// +// and its Stop() deletes both, exactly as darwinMacOSVM.stop does +// (os.Remove(clonePath) + os.RemoveAll(jobDir)). The point of the fake is that +// the paths come from the id string the SCHEDULER chooses, so the test can +// observe whether two attempts at the same job collide on disk. +type pathedMacVM struct { + id string + jobDir string + clone string + + wedge bool // WaitForRunner never returns (the #196 guest) + gate chan struct{} // Stop blocks here before deleting; nil to not block + held chan struct{} // closed once Stop has finished its deletions + done chan struct{} // Wait blocks here; nil returns immediately + + stops atomic.Int32 + stopOnce sync.Once +} + +func newPathedMacVM(t *testing.T, dataDir, id string, wedge bool) *pathedMacVM { + t.Helper() + jobsDir := filepath.Join(dataDir, "vm", "macos", "jobs") + return &pathedMacVM{ + id: id, + jobDir: filepath.Join(jobsDir, id), + clone: filepath.Join(jobsDir, id+".img"), + wedge: wedge, + held: make(chan struct{}), + } +} + +func (m *pathedMacVM) jitPath() string { return filepath.Join(m.jobDir, ".jit_config") } + +func (m *pathedMacVM) WriteJITConfig(cfg string) error { + if err := os.MkdirAll(m.jobDir, 0o755); err != nil { + return err + } + return os.WriteFile(m.jitPath(), []byte(cfg), 0o600) +} + +func (m *pathedMacVM) Start(context.Context) error { + if err := os.MkdirAll(filepath.Dir(m.clone), 0o755); err != nil { + return err + } + return os.WriteFile(m.clone, []byte("disk-"+m.id), 0o600) +} + +func (m *pathedMacVM) RunnerAddress() string { return "192.168.64.9" } + +func (m *pathedMacVM) WaitForRunner(context.Context) (string, error) { + if m.wedge { + select {} //nolint:staticcheck // models the wedged guest wait + } + return "192.168.64.9", nil +} + +func (m *pathedMacVM) Wait(context.Context) (int, error) { + if m.done != nil { + <-m.done + } + return 0, nil +} + +// Stop mirrors darwinMacOSVM.Stop: a sync.Once wrapping a teardown that ends +// by deleting the VM's clone and its shared directory. +func (m *pathedMacVM) Stop() { + m.stops.Add(1) + m.stopOnce.Do(func() { + if m.gate != nil { + <-m.gate + } + _ = os.RemoveAll(m.jobDir) + _ = os.Remove(m.clone) + close(m.held) + }) +} + +func assertFileExists(t *testing.T, path, what string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Errorf("%s is gone (%s): %v", what, path, err) + } +} + +// TestHandleMacOSJob_RedispatchOfTheSameJobGetsDisjointResources is the race +// that release-before-teardown opened. +// +// Every macOS resource used to be keyed by job ID alone and nothing else: the +// VM's shared directory, its .img/.aux clones, its artifacts dir. The failure +// paths now release the slot and clear seen/pending BEFORE teardown, so the +// same job ID can be dispatched again — the poll re-emits a still-queued job +// about every 30s, and maxProvisionAttempts allows several passes — while the +// previous attempt's VM is still alive with a stop wedged inside +// Virtualization.framework. +// +// Attempt 2 then writes its JIT config and clones its disk over attempt 1's +// live files, attempt 1's abandoned stop returns minutes later and runs +// os.Remove(clone) + os.RemoveAll(jobDir), and attempt 2 loses its disk and its +// .jit_config to a teardown for a VM it has nothing to do with. Its runner can +// never register, so it times out too, and strands another guest doing it — +// self-sustaining. +// +// Without the per-attempt VM instance id this test FAILS: both attempts get +// the same id, and attempt 2's files are gone after attempt 1's late teardown. +func TestHandleMacOSJob_RedispatchOfTheSameJobGetsDisjointResources(t *testing.T) { + prov := newMockProvider("mac-redispatch") + defer func() { _ = prov.Stop(context.Background()) }() + + dataDir := t.TempDir() + s := newSlotTestScheduler(t, prov, nil) + s.cfg.DataDir = dataDir + s.macAbandonedCap = 5 // the cap is tested elsewhere; don't trip it here + + var mu sync.Mutex + var made []*pathedMacVM + s.newMacOSVM = func(_ vm.MacOSVMConfig, id string) (vm.MacOSVM, error) { + mu.Lock() + defer mu.Unlock() + // The first attempt is the #196 guest: it never becomes reachable and + // its Stop() does not return until the test lets it. + first := len(made) == 0 + m := newPathedMacVM(t, dataDir, id, first) + if first { + m.gate = make(chan struct{}) + } else { + m.done = make(chan struct{}) // stays "running" until released + } + made = append(made, m) + return m, nil + } + + // The same job id both times: this is a RE-dispatch, not a second job. + const jobID = int64(100817345293) + event := macEvent(prov, jobID) + + // Attempt 1 hits the provisioning deadline and is abandoned with its Stop + // still pending. + runMacDispatch(t, s, event, 5*time.Second) + + // Attempt 2 takes the freed slot. + runMacDispatch(t, s, event, 5*time.Second) + + mu.Lock() + n := len(made) + mu.Unlock() + if n != 2 { + t.Fatalf("expected 2 provisioning attempts, got %d", n) + } + mu.Lock() + one, two := made[0], made[1] + mu.Unlock() + t.Cleanup(func() { + if two.done != nil { + close(two.done) + } + }) + + if one.id == two.id { + t.Fatalf("both attempts at job %d were given the same VM instance id %q — "+ + "their shared dir and disk clone are the same files, so attempt 1's "+ + "late teardown will delete attempt 2's", jobID, one.id) + } + + assertFileExists(t, two.jitPath(), "attempt 2's JIT config") + assertFileExists(t, two.clone, "attempt 2's disk clone") + + // Attempt 1's abandoned teardown finally lands, minutes late in + // production. It must only ever delete its own files. + close(one.gate) + select { + case <-one.held: + case <-time.After(5 * time.Second): + t.Fatal("the abandoned teardown never completed") + } + + assertFileExists(t, two.jitPath(), "attempt 2's JIT config after attempt 1's late teardown") + assertFileExists(t, two.clone, "attempt 2's disk clone after attempt 1's late teardown") + + if got := s.ActiveJobs(); got != 1 { + t.Errorf("ActiveJobs() = %d, want 1 (attempt 2 must still be running)", got) + } +} + +// TestMacVMInstanceIDDiscriminatesAttempts: the id keeps the job number (it is +// what an operator greps for) and adds a per-attempt suffix. +func TestMacVMInstanceIDDiscriminatesAttempts(t *testing.T) { + const jobID = int64(100817345293) + seen := map[string]bool{} + for i := 0; i < 64; i++ { + id := macVMInstanceID(jobID) + if !strings.HasPrefix(id, "100817345293-") { + t.Fatalf("instance id %q does not lead with the job id", id) + } + if seen[id] { + t.Fatalf("macVMInstanceID repeated %q; two attempts would share a disk clone", id) + } + seen[id] = true + } +} + // --------------------------------------------------------------------------- // MAJOR 3: the abandoned-VM trade is bounded and legible. // --------------------------------------------------------------------------- From c38e041adb0d6ebf7585d6d1b13e138916679472 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Sat, 5 Sep 2026 01:10:08 -0700 Subject: [PATCH 5/5] test(scheduler): pin release-before-teardown ordering with a slow ReleaseJob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the suite tested the ORDER of the release and the teardown, only that the slot came back eventually. Delete the explicit slot.release() on the macOS provisioning-failure path or in the job wait-goroutine, lean on the deferred backstop instead, and everything still passed — because every teardown bound is compressed to 50ms in tests and mockProvider.ReleaseJob returns instantly, so "released now" and "released at function exit" look identical. In production they are not remotely identical. What stands between the job ending and the next macOS job starting on the backstop-only version is a VM stop that may never return plus a GitHub API call issued on context.Background() with no deadline at all. That gap is the whole subject of issue #196; a test suite that cannot see it will not stop it coming back. mockProvider gains a releaseHook that runs inside ReleaseJob, so a test can hold the provider call open and assert on the semaphore WHILE the teardown is still in flight. TestSlotIsFreeWhileTeardownIsStillInFlight uses it on both paths that matter. Verified by reverting each release in turn: - drop slot.release() on the provisioning-deadline path: "macSem held=1 while the ghost-runner deregistration was still in flight, want 0" - drop slot.release() in the wait-goroutine after untracking: "macSem held=1 while the finished job's runner was still being deregistered, want 0" Both pass with the release restored, and both are deadline-guarded so a regression fails rather than hangs. --- pkg/scheduler/multiprovider_test.go | 20 ++++++- pkg/scheduler/slots_test.go | 87 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/multiprovider_test.go b/pkg/scheduler/multiprovider_test.go index 158b58ea..bc002c60 100644 --- a/pkg/scheduler/multiprovider_test.go +++ b/pkg/scheduler/multiprovider_test.go @@ -18,10 +18,21 @@ type mockProvider struct { defaultImage string events chan providers.JobEvent + // releaseHook runs at the top of ReleaseJob, before the mutex is taken. + // Set once before the test starts and never mutated afterwards. + // + // It exists to model what ReleaseJob actually is in production — a GitHub + // API call, on context.Background(), with no deadline — so a test can + // assert that a concurrency slot is FREE while that call is still in + // flight. Without a slow ReleaseJob the release-before-teardown ordering + // this fix depends on is unobservable: the mock returns instantly, so + // moving the release after the teardown still passes. + releaseHook func(*providers.Claim) + mu sync.Mutex - claims []*providers.Claim // jobs claimed via ClaimJob - releases []*providers.Claim // jobs released via ReleaseJob - images map[int64]string // jobID → image for FetchJobImage + claims []*providers.Claim // jobs claimed via ClaimJob + releases []*providers.Claim // jobs released via ReleaseJob + images map[int64]string // jobID → image for FetchJobImage } var _ providers.Poll = (*mockProvider)(nil) @@ -62,6 +73,9 @@ func (m *mockProvider) ClaimJob(_ context.Context, event *providers.JobEvent, ru } func (m *mockProvider) ReleaseJob(_ context.Context, claim *providers.Claim) error { + if m.releaseHook != nil { + m.releaseHook(claim) + } m.mu.Lock() m.releases = append(m.releases, claim) m.mu.Unlock() diff --git a/pkg/scheduler/slots_test.go b/pkg/scheduler/slots_test.go index 8f5a1f0a..3c21f468 100644 --- a/pkg/scheduler/slots_test.go +++ b/pkg/scheduler/slots_test.go @@ -1214,3 +1214,90 @@ func TestSlotLeakEscalationIsScopedToThePool(t *testing.T) { } } +// --------------------------------------------------------------------------- +// MINOR 6: the slot must be free WHILE teardown is still in flight. +// --------------------------------------------------------------------------- + +// TestSlotIsFreeWhileTeardownIsStillInFlight pins the ORDERING, which nothing +// else in the suite did. +// +// Every teardown bound in this package is compressed to milliseconds in tests +// and mockProvider.ReleaseJob returns instantly, so deleting the explicit +// slot.release() and leaning on the deferred backstop still passed everything +// — even though in production the difference is a GitHub API call on +// context.Background() with no deadline, plus a VM stop that may never return, +// standing between the job ending and the next macOS job starting. This test +// makes ReleaseJob slow on purpose and asserts the slot is already back. +func TestSlotIsFreeWhileTeardownIsStillInFlight(t *testing.T) { + t.Run("provisioning failure path", func(t *testing.T) { + prov := newMockProvider("mac-order-fail") + defer func() { _ = prov.Stop(context.Background()) }() + + s := newSlotTestScheduler(t, prov, newWedgedMacVM(t)) + + observed := make(chan int, 4) + proceed := make(chan struct{}) + var once sync.Once + prov.releaseHook = func(*providers.Claim) { + once.Do(func() { + observed <- len(s.macSem) + <-proceed + }) + } + + done := make(chan struct{}) + go func() { + defer close(done) + s.handleMacOSJob(context.Background(), macEvent(prov, 11)) + }() + + select { + case held := <-observed: + if held != 0 { + t.Errorf("macSem held=%d while the ghost-runner deregistration was still in flight, want 0: "+ + "the slot is being held across a GitHub API call that has no deadline", held) + } + case <-time.After(5 * time.Second): + t.Fatal("ReleaseJob was never reached on the provisioning failure path") + } + close(proceed) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("handleMacOSJob did not return") + } + }) + + t.Run("job wait-goroutine", func(t *testing.T) { + prov := newMockProvider("mac-order-wait") + defer func() { _ = prov.Stop(context.Background()) }() + + var stops atomic.Int32 + s := newSlotTestScheduler(t, prov, &fastMacVM{ip: "192.168.64.5", stops: &stops}) + + observed := make(chan int, 4) + proceed := make(chan struct{}) + var once sync.Once + prov.releaseHook = func(*providers.Claim) { + once.Do(func() { + observed <- len(s.macSem) + <-proceed + }) + } + + runMacDispatch(t, s, macEvent(prov, 12), 5*time.Second) + + select { + case held := <-observed: + if held != 0 { + t.Errorf("macSem held=%d while the finished job's runner was still being deregistered, want 0: "+ + "the next macOS job is waiting on a teardown that cannot delay it", held) + } + case <-time.After(5 * time.Second): + t.Fatal("the wait-goroutine never reached ReleaseJob") + } + close(proceed) + waitForSlotFree(t, s.macSem, 5*time.Second) + assertCapacityIntact(t, s.macSem) + }) +}