diff --git a/api/v1/ephemerd.pb.go b/api/v1/ephemerd.pb.go index 4f3e689..5f65245 100644 --- a/api/v1/ephemerd.pb.go +++ b/api/v1/ephemerd.pb.go @@ -332,6 +332,35 @@ type StatusResponse struct { // The upgrade flow polls this after a restart to confirm the new binary // came up on the target version. Version string `protobuf:"bytes,6,opt,name=version,proto3" json:"version,omitempty"` + // held_slots is how many concurrency slots are currently HELD across every + // dispatch pool, which is not the same thing as active_jobs. + // + // active_jobs counts jobs the scheduler is tracking. A slot is taken before + // the job is tracked and given back after it is untracked, so a job that is + // still provisioning — or one whose slot leaked — is charged here and not + // there. Issue #196: a mac held its only slot for 28 hours after a wedged + // provision and reported status ok with active_jobs 0 the whole time. + // held_slots == slot_capacity with active_jobs 0 is that failure, visible. + HeldSlots int32 `protobuf:"varint,7,opt,name=held_slots,json=heldSlots,proto3" json:"held_slots,omitempty"` + // slot_capacity is the total number of slots across all pools. + SlotCapacity int32 `protobuf:"varint,8,opt,name=slot_capacity,json=slotCapacity,proto3" json:"slot_capacity,omitempty"` + // slot_pools breaks the totals down per dispatch pool (local, linux, + // macos), so an operator can tell which pool is saturated. + SlotPools []*SlotPool `protobuf:"bytes,9,rep,name=slot_pools,json=slotPools,proto3" json:"slot_pools,omitempty"` + // abandoned_macos_vms counts macOS VMs whose teardown this daemon gave up + // on and has not seen finish. + // + // This is the COST of the #196 fix, made visible. When a Virtualization. + // framework stop wedges, the daemon walks away from it so the concurrency + // slot comes back — but the guest is still running and still holding its + // full configured RAM, and only a restart reclaims it. Non-zero means the + // node is losing memory it cannot get back; at abandoned_macos_vm_cap the + // macOS pool refuses further provisioning rather than growing without + // bound (Vz permits only a couple of concurrent macOS guests anyway). + AbandonedMacosVms int32 `protobuf:"varint,10,opt,name=abandoned_macos_vms,json=abandonedMacosVms,proto3" json:"abandoned_macos_vms,omitempty"` + // abandoned_macos_vm_cap is the value of abandoned_macos_vms at which the + // macOS pool stops taking work. + AbandonedMacosVmCap int32 `protobuf:"varint,11,opt,name=abandoned_macos_vm_cap,json=abandonedMacosVmCap,proto3" json:"abandoned_macos_vm_cap,omitempty"` } func (x *StatusResponse) Reset() { @@ -408,6 +437,105 @@ func (x *StatusResponse) GetVersion() string { return "" } +func (x *StatusResponse) GetHeldSlots() int32 { + if x != nil { + return x.HeldSlots + } + return 0 +} + +func (x *StatusResponse) GetSlotCapacity() int32 { + if x != nil { + return x.SlotCapacity + } + return 0 +} + +func (x *StatusResponse) GetSlotPools() []*SlotPool { + if x != nil { + return x.SlotPools + } + return nil +} + +func (x *StatusResponse) GetAbandonedMacosVms() int32 { + if x != nil { + return x.AbandonedMacosVms + } + return 0 +} + +func (x *StatusResponse) GetAbandonedMacosVmCap() int32 { + if x != nil { + return x.AbandonedMacosVmCap + } + return 0 +} + +// SlotPool is one dispatch pool's concurrency-slot occupancy. +type SlotPool struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pool string `protobuf:"bytes,1,opt,name=pool,proto3" json:"pool,omitempty"` + Held int32 `protobuf:"varint,2,opt,name=held,proto3" json:"held,omitempty"` + Capacity int32 `protobuf:"varint,3,opt,name=capacity,proto3" json:"capacity,omitempty"` +} + +func (x *SlotPool) Reset() { + *x = SlotPool{} + if protoimpl.UnsafeEnabled { + mi := &file_api_v1_ephemerd_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SlotPool) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SlotPool) ProtoMessage() {} + +func (x *SlotPool) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_ephemerd_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SlotPool.ProtoReflect.Descriptor instead. +func (*SlotPool) Descriptor() ([]byte, []int) { + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{5} +} + +func (x *SlotPool) GetPool() string { + if x != nil { + return x.Pool + } + return "" +} + +func (x *SlotPool) GetHeld() int32 { + if x != nil { + return x.Held + } + return 0 +} + +func (x *SlotPool) GetCapacity() int32 { + if x != nil { + return x.Capacity + } + return 0 +} + type ListJobsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -417,7 +545,7 @@ type ListJobsRequest struct { func (x *ListJobsRequest) Reset() { *x = ListJobsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[5] + mi := &file_api_v1_ephemerd_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -430,7 +558,7 @@ func (x *ListJobsRequest) String() string { func (*ListJobsRequest) ProtoMessage() {} func (x *ListJobsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[5] + mi := &file_api_v1_ephemerd_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -443,7 +571,7 @@ func (x *ListJobsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListJobsRequest.ProtoReflect.Descriptor instead. func (*ListJobsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{5} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{6} } type ListJobsResponse struct { @@ -457,7 +585,7 @@ type ListJobsResponse struct { func (x *ListJobsResponse) Reset() { *x = ListJobsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[6] + mi := &file_api_v1_ephemerd_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -470,7 +598,7 @@ func (x *ListJobsResponse) String() string { func (*ListJobsResponse) ProtoMessage() {} func (x *ListJobsResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[6] + mi := &file_api_v1_ephemerd_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -483,7 +611,7 @@ func (x *ListJobsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListJobsResponse.ProtoReflect.Descriptor instead. func (*ListJobsResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{6} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{7} } func (x *ListJobsResponse) GetJobs() []*Job { @@ -504,7 +632,7 @@ type GetJobRequest struct { func (x *GetJobRequest) Reset() { *x = GetJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[7] + mi := &file_api_v1_ephemerd_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -517,7 +645,7 @@ func (x *GetJobRequest) String() string { func (*GetJobRequest) ProtoMessage() {} func (x *GetJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[7] + mi := &file_api_v1_ephemerd_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -530,7 +658,7 @@ func (x *GetJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetJobRequest.ProtoReflect.Descriptor instead. func (*GetJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{7} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{8} } func (x *GetJobRequest) GetId() int64 { @@ -559,7 +687,7 @@ type Job struct { func (x *Job) Reset() { *x = Job{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[8] + mi := &file_api_v1_ephemerd_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -572,7 +700,7 @@ func (x *Job) String() string { func (*Job) ProtoMessage() {} func (x *Job) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[8] + mi := &file_api_v1_ephemerd_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -585,7 +713,7 @@ func (x *Job) ProtoReflect() protoreflect.Message { // Deprecated: Use Job.ProtoReflect.Descriptor instead. func (*Job) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{8} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{9} } func (x *Job) GetId() int64 { @@ -662,7 +790,7 @@ type KillJobRequest struct { func (x *KillJobRequest) Reset() { *x = KillJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[9] + mi := &file_api_v1_ephemerd_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -675,7 +803,7 @@ func (x *KillJobRequest) String() string { func (*KillJobRequest) ProtoMessage() {} func (x *KillJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[9] + mi := &file_api_v1_ephemerd_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -688,7 +816,7 @@ func (x *KillJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KillJobRequest.ProtoReflect.Descriptor instead. func (*KillJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{9} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{10} } func (x *KillJobRequest) GetId() int64 { @@ -707,7 +835,7 @@ type KillJobResponse struct { func (x *KillJobResponse) Reset() { *x = KillJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[10] + mi := &file_api_v1_ephemerd_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -720,7 +848,7 @@ func (x *KillJobResponse) String() string { func (*KillJobResponse) ProtoMessage() {} func (x *KillJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[10] + mi := &file_api_v1_ephemerd_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -733,7 +861,7 @@ func (x *KillJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KillJobResponse.ProtoReflect.Descriptor instead. func (*KillJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{10} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{11} } type CordonRequest struct { @@ -745,7 +873,7 @@ type CordonRequest struct { func (x *CordonRequest) Reset() { *x = CordonRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[11] + mi := &file_api_v1_ephemerd_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -758,7 +886,7 @@ func (x *CordonRequest) String() string { func (*CordonRequest) ProtoMessage() {} func (x *CordonRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[11] + mi := &file_api_v1_ephemerd_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -771,7 +899,7 @@ func (x *CordonRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CordonRequest.ProtoReflect.Descriptor instead. func (*CordonRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{11} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{12} } type CordonResponse struct { @@ -785,7 +913,7 @@ type CordonResponse struct { func (x *CordonResponse) Reset() { *x = CordonResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[12] + mi := &file_api_v1_ephemerd_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -798,7 +926,7 @@ func (x *CordonResponse) String() string { func (*CordonResponse) ProtoMessage() {} func (x *CordonResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[12] + mi := &file_api_v1_ephemerd_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -811,7 +939,7 @@ func (x *CordonResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CordonResponse.ProtoReflect.Descriptor instead. func (*CordonResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{12} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{13} } func (x *CordonResponse) GetActiveJobs() int32 { @@ -830,7 +958,7 @@ type UncordonRequest struct { func (x *UncordonRequest) Reset() { *x = UncordonRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[13] + mi := &file_api_v1_ephemerd_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -843,7 +971,7 @@ func (x *UncordonRequest) String() string { func (*UncordonRequest) ProtoMessage() {} func (x *UncordonRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[13] + mi := &file_api_v1_ephemerd_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -856,7 +984,7 @@ func (x *UncordonRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UncordonRequest.ProtoReflect.Descriptor instead. func (*UncordonRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{13} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{14} } type UncordonResponse struct { @@ -870,7 +998,7 @@ type UncordonResponse struct { func (x *UncordonResponse) Reset() { *x = UncordonResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[14] + mi := &file_api_v1_ephemerd_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -883,7 +1011,7 @@ func (x *UncordonResponse) String() string { func (*UncordonResponse) ProtoMessage() {} func (x *UncordonResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[14] + mi := &file_api_v1_ephemerd_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -896,7 +1024,7 @@ func (x *UncordonResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UncordonResponse.ProtoReflect.Descriptor instead. func (*UncordonResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{14} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{15} } func (x *UncordonResponse) GetActiveJobs() int32 { @@ -918,7 +1046,7 @@ type GetJobLogsRequest struct { func (x *GetJobLogsRequest) Reset() { *x = GetJobLogsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[15] + mi := &file_api_v1_ephemerd_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -931,7 +1059,7 @@ func (x *GetJobLogsRequest) String() string { func (*GetJobLogsRequest) ProtoMessage() {} func (x *GetJobLogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[15] + mi := &file_api_v1_ephemerd_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -944,7 +1072,7 @@ func (x *GetJobLogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetJobLogsRequest.ProtoReflect.Descriptor instead. func (*GetJobLogsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{15} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{16} } func (x *GetJobLogsRequest) GetId() int64 { @@ -972,7 +1100,7 @@ type LogChunk struct { func (x *LogChunk) Reset() { *x = LogChunk{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[16] + mi := &file_api_v1_ephemerd_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -985,7 +1113,7 @@ func (x *LogChunk) String() string { func (*LogChunk) ProtoMessage() {} func (x *LogChunk) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[16] + mi := &file_api_v1_ephemerd_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -998,7 +1126,7 @@ func (x *LogChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use LogChunk.ProtoReflect.Descriptor instead. func (*LogChunk) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{16} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{17} } func (x *LogChunk) GetData() []byte { @@ -1036,7 +1164,7 @@ type UpgradeRequest struct { func (x *UpgradeRequest) Reset() { *x = UpgradeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[17] + mi := &file_api_v1_ephemerd_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1049,7 +1177,7 @@ func (x *UpgradeRequest) String() string { func (*UpgradeRequest) ProtoMessage() {} func (x *UpgradeRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[17] + mi := &file_api_v1_ephemerd_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1062,7 +1190,7 @@ func (x *UpgradeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeRequest.ProtoReflect.Descriptor instead. func (*UpgradeRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{17} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{18} } func (x *UpgradeRequest) GetTargetVersion() string { @@ -1117,7 +1245,7 @@ type UpgradeProgress struct { func (x *UpgradeProgress) Reset() { *x = UpgradeProgress{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[18] + mi := &file_api_v1_ephemerd_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1130,7 +1258,7 @@ func (x *UpgradeProgress) String() string { func (*UpgradeProgress) ProtoMessage() {} func (x *UpgradeProgress) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[18] + mi := &file_api_v1_ephemerd_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1143,7 +1271,7 @@ func (x *UpgradeProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use UpgradeProgress.ProtoReflect.Descriptor instead. func (*UpgradeProgress) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{18} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{19} } func (x *UpgradeProgress) GetState() UpgradeState { @@ -1213,7 +1341,7 @@ type CreateJobRequest struct { func (x *CreateJobRequest) Reset() { *x = CreateJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[19] + mi := &file_api_v1_ephemerd_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1226,7 +1354,7 @@ func (x *CreateJobRequest) String() string { func (*CreateJobRequest) ProtoMessage() {} func (x *CreateJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[19] + mi := &file_api_v1_ephemerd_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1239,7 +1367,7 @@ func (x *CreateJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateJobRequest.ProtoReflect.Descriptor instead. func (*CreateJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{19} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{20} } func (x *CreateJobRequest) GetId() string { @@ -1286,7 +1414,7 @@ type CreateJobResponse struct { func (x *CreateJobResponse) Reset() { *x = CreateJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[20] + mi := &file_api_v1_ephemerd_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1299,7 +1427,7 @@ func (x *CreateJobResponse) String() string { func (*CreateJobResponse) ProtoMessage() {} func (x *CreateJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[20] + mi := &file_api_v1_ephemerd_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +1440,7 @@ func (x *CreateJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateJobResponse.ProtoReflect.Descriptor instead. func (*CreateJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{20} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{21} } type WaitJobRequest struct { @@ -1326,7 +1454,7 @@ type WaitJobRequest struct { func (x *WaitJobRequest) Reset() { *x = WaitJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[21] + mi := &file_api_v1_ephemerd_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1339,7 +1467,7 @@ func (x *WaitJobRequest) String() string { func (*WaitJobRequest) ProtoMessage() {} func (x *WaitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[21] + mi := &file_api_v1_ephemerd_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1352,7 +1480,7 @@ func (x *WaitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use WaitJobRequest.ProtoReflect.Descriptor instead. func (*WaitJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{21} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{22} } func (x *WaitJobRequest) GetId() string { @@ -1373,7 +1501,7 @@ type WaitJobResponse struct { func (x *WaitJobResponse) Reset() { *x = WaitJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[22] + mi := &file_api_v1_ephemerd_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1386,7 +1514,7 @@ func (x *WaitJobResponse) String() string { func (*WaitJobResponse) ProtoMessage() {} func (x *WaitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[22] + mi := &file_api_v1_ephemerd_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1399,7 +1527,7 @@ func (x *WaitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use WaitJobResponse.ProtoReflect.Descriptor instead. func (*WaitJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{22} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{23} } func (x *WaitJobResponse) GetExitCode() uint32 { @@ -1420,7 +1548,7 @@ type DestroyJobRequest struct { func (x *DestroyJobRequest) Reset() { *x = DestroyJobRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[23] + mi := &file_api_v1_ephemerd_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1433,7 +1561,7 @@ func (x *DestroyJobRequest) String() string { func (*DestroyJobRequest) ProtoMessage() {} func (x *DestroyJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[23] + mi := &file_api_v1_ephemerd_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1446,7 +1574,7 @@ func (x *DestroyJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyJobRequest.ProtoReflect.Descriptor instead. func (*DestroyJobRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{23} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{24} } func (x *DestroyJobRequest) GetId() string { @@ -1465,7 +1593,7 @@ type DestroyJobResponse struct { func (x *DestroyJobResponse) Reset() { *x = DestroyJobResponse{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[24] + mi := &file_api_v1_ephemerd_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1478,7 +1606,7 @@ func (x *DestroyJobResponse) String() string { func (*DestroyJobResponse) ProtoMessage() {} func (x *DestroyJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[24] + mi := &file_api_v1_ephemerd_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1491,7 +1619,7 @@ func (x *DestroyJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DestroyJobResponse.ProtoReflect.Descriptor instead. func (*DestroyJobResponse) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{24} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{25} } // StreamContainerStatsRequest opens a long-lived sampling stream. The host @@ -1508,7 +1636,7 @@ type StreamContainerStatsRequest struct { func (x *StreamContainerStatsRequest) Reset() { *x = StreamContainerStatsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[25] + mi := &file_api_v1_ephemerd_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1521,7 +1649,7 @@ func (x *StreamContainerStatsRequest) String() string { func (*StreamContainerStatsRequest) ProtoMessage() {} func (x *StreamContainerStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[25] + mi := &file_api_v1_ephemerd_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1534,7 +1662,7 @@ func (x *StreamContainerStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamContainerStatsRequest.ProtoReflect.Descriptor instead. func (*StreamContainerStatsRequest) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{25} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{26} } func (x *StreamContainerStatsRequest) GetIntervalSeconds() uint32 { @@ -1556,7 +1684,7 @@ type ContainerStatsBatch struct { func (x *ContainerStatsBatch) Reset() { *x = ContainerStatsBatch{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[26] + mi := &file_api_v1_ephemerd_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +1697,7 @@ func (x *ContainerStatsBatch) String() string { func (*ContainerStatsBatch) ProtoMessage() {} func (x *ContainerStatsBatch) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[26] + mi := &file_api_v1_ephemerd_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,7 +1710,7 @@ func (x *ContainerStatsBatch) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerStatsBatch.ProtoReflect.Descriptor instead. func (*ContainerStatsBatch) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{26} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{27} } func (x *ContainerStatsBatch) GetTimestampUnixNano() int64 { @@ -1622,7 +1750,7 @@ type ContainerStats struct { func (x *ContainerStats) Reset() { *x = ContainerStats{} if protoimpl.UnsafeEnabled { - mi := &file_api_v1_ephemerd_proto_msgTypes[27] + mi := &file_api_v1_ephemerd_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1635,7 +1763,7 @@ func (x *ContainerStats) String() string { func (*ContainerStats) ProtoMessage() {} func (x *ContainerStats) ProtoReflect() protoreflect.Message { - mi := &file_api_v1_ephemerd_proto_msgTypes[27] + mi := &file_api_v1_ephemerd_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1648,7 +1776,7 @@ func (x *ContainerStats) ProtoReflect() protoreflect.Message { // Deprecated: Use ContainerStats.ProtoReflect.Descriptor instead. func (*ContainerStats) Descriptor() ([]byte, []int) { - return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{27} + return file_api_v1_ephemerd_proto_rawDescGZIP(), []int{28} } func (x *ContainerStats) GetId() string { @@ -1737,8 +1865,8 @@ var file_api_v1_ephemerd_proto_rawDesc = []byte{ 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x0f, 0x0a, - 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbe, - 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x9d, + 0x03, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, @@ -1749,211 +1877,230 @@ var file_api_v1_ephemerd_proto_rawDesc = []byte{ 0x01, 0x28, 0x08, 0x52, 0x08, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, - 0x11, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x38, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x1f, 0x0a, 0x0d, - 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xd1, 0x01, - 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, - 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, - 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, - 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x02, 0x69, 0x64, 0x22, 0x11, 0x0a, 0x0f, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x0e, 0x43, 0x6f, 0x72, 0x64, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, - 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, - 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x55, 0x6e, - 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, - 0x10, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, - 0x62, 0x73, 0x22, 0x3b, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, - 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x22, - 0x1e, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, - 0xbf, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x72, 0x6c, - 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x75, 0x72, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x15, - 0x64, 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x64, 0x72, 0x61, - 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, - 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x5f, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x6e, 0x6f, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, - 0x6f, 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, - 0x65, 0x22, 0x99, 0x02, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1d, 0x0a, 0x0a, 0x68, 0x65, 0x6c, 0x64, 0x5f, 0x73, 0x6c, 0x6f, 0x74, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x68, 0x65, 0x6c, 0x64, 0x53, 0x6c, 0x6f, 0x74, 0x73, 0x12, 0x23, + 0x0a, 0x0d, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x73, 0x6c, 0x6f, 0x74, 0x43, 0x61, 0x70, 0x61, 0x63, + 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x0a, 0x73, 0x6c, 0x6f, 0x74, 0x5f, 0x70, 0x6f, 0x6f, 0x6c, + 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x09, + 0x73, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x62, 0x61, + 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x63, 0x6f, 0x73, 0x5f, 0x76, 0x6d, 0x73, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x61, 0x62, 0x61, 0x6e, 0x64, 0x6f, 0x6e, 0x65, + 0x64, 0x4d, 0x61, 0x63, 0x6f, 0x73, 0x56, 0x6d, 0x73, 0x12, 0x33, 0x0a, 0x16, 0x61, 0x62, 0x61, + 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x63, 0x6f, 0x73, 0x5f, 0x76, 0x6d, 0x5f, + 0x63, 0x61, 0x70, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x61, 0x62, 0x61, 0x6e, 0x64, + 0x6f, 0x6e, 0x65, 0x64, 0x4d, 0x61, 0x63, 0x6f, 0x73, 0x56, 0x6d, 0x43, 0x61, 0x70, 0x22, 0x4e, + 0x0a, 0x08, 0x53, 0x6c, 0x6f, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, + 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x12, 0x12, + 0x0a, 0x04, 0x68, 0x65, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x68, 0x65, + 0x6c, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x22, 0x11, + 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x38, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x1f, 0x0a, 0x0d, 0x47, + 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x22, 0xd1, 0x01, 0x0a, + 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, + 0x67, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, + 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x69, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, + 0x22, 0x20, 0x0a, 0x0e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, + 0x69, 0x64, 0x22, 0x11, 0x0a, 0x0f, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x0e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, 0x22, 0x11, 0x0a, 0x0f, 0x55, 0x6e, 0x63, + 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, 0x10, + 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, - 0x73, 0x12, 0x29, 0x0a, 0x10, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x87, 0x01, - 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, + 0x73, 0x22, 0x3b, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x22, 0x1e, + 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xbf, + 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x72, 0x6c, 0x5f, + 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x75, 0x72, 0x6c, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x64, + 0x72, 0x61, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, + 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x64, 0x72, 0x61, 0x69, + 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, + 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x5f, 0x64, 0x72, 0x61, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x6e, 0x6f, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, + 0x72, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, + 0x22, 0x99, 0x02, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x27, 0x0a, 0x0f, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4a, 0x6f, 0x62, 0x73, + 0x12, 0x29, 0x0a, 0x10, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x65, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x22, 0x87, 0x01, 0x0a, + 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x69, 0x74, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x69, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x0a, 0x0e, 0x57, + 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, 0x0a, + 0x0f, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x23, 0x0a, + 0x11, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6a, 0x69, 0x74, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6a, 0x69, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x0a, 0x0e, - 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x2e, - 0x0a, 0x0f, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x23, - 0x0a, 0x11, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x02, 0x69, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, - 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x48, 0x0a, 0x1b, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, - 0x6e, 0x64, 0x73, 0x22, 0x78, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, - 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xca, 0x02, - 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x72, 0x65, 0x70, 0x6f, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, - 0x65, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, - 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x12, 0x21, 0x0a, 0x0c, - 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, - 0x2a, 0x0a, 0x11, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x6e, 0x6f, 0x6e, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x41, 0x6e, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, - 0x70, 0x75, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4c, 0x69, 0x6d, 0x69, - 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x5f, 0x72, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x74, 0x78, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x54, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0xaf, 0x02, 0x0a, 0x0c, 0x55, - 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x55, - 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, - 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, - 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x46, - 0x4c, 0x49, 0x47, 0x48, 0x54, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, - 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x50, 0x5f, 0x54, 0x4f, 0x5f, 0x44, - 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x49, 0x4e, 0x47, 0x10, - 0x03, 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x04, - 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x45, 0x5f, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x19, 0x0a, - 0x15, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, - 0x54, 0x41, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, - 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x57, 0x41, 0x50, 0x50, 0x49, - 0x4e, 0x47, 0x10, 0x07, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, - 0x10, 0x08, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x09, 0x32, 0xfd, 0x04, 0x0a, - 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x4c, - 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, + 0x69, 0x64, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x48, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x73, 0x22, 0x78, 0x0a, 0x13, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x6e, 0x61, 0x6e, 0x6f, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x55, 0x6e, 0x69, 0x78, 0x4e, 0x61, 0x6e, 0x6f, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x22, 0xca, 0x02, 0x0a, + 0x0e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x72, 0x65, 0x70, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, + 0x65, 0x70, 0x6f, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, + 0x5f, 0x6e, 0x61, 0x6e, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, 0x70, + 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6e, 0x6f, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d, + 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x2a, + 0x0a, 0x11, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x6e, 0x6f, 0x6e, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x41, 0x6e, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, + 0x75, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x63, + 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x10, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x4c, 0x69, 0x6d, 0x69, 0x74, + 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x5f, 0x72, 0x78, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, + 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x74, 0x78, 0x5f, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x54, 0x78, 0x42, 0x79, 0x74, 0x65, 0x73, 0x2a, 0xaf, 0x02, 0x0a, 0x0c, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, + 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, + 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x4c, + 0x49, 0x47, 0x48, 0x54, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, + 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x50, 0x5f, 0x54, 0x4f, 0x5f, 0x44, 0x41, + 0x54, 0x45, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x52, 0x41, 0x49, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x03, + 0x12, 0x1d, 0x0a, 0x19, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, + 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x12, + 0x1b, 0x0a, 0x17, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x19, 0x0a, 0x15, + 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x55, 0x50, 0x47, 0x52, 0x41, + 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, 0x57, 0x41, 0x50, 0x50, 0x49, 0x4e, + 0x47, 0x10, 0x07, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, + 0x08, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x50, 0x47, 0x52, 0x41, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x09, 0x32, 0xfd, 0x04, 0x0a, 0x07, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x4c, 0x69, + 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, + 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, + 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4a, + 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x65, 0x70, 0x68, 0x65, + 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x12, 0x44, 0x0a, 0x07, 0x4b, + 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x12, + 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x6f, + 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x06, 0x43, 0x6f, 0x72, 0x64, + 0x6f, 0x6e, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, + 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x72, + 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x55, + 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, + 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, - 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1a, - 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x65, 0x70, 0x68, - 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4a, 0x6f, 0x62, 0x12, 0x44, 0x0a, 0x07, - 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, - 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x4b, 0x69, 0x6c, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, - 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x15, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4c, - 0x6f, 0x67, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x06, 0x43, 0x6f, 0x72, - 0x64, 0x6f, 0x6e, 0x12, 0x1a, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x08, - 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x12, 0x1c, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, - 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, - 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, - 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, - 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, - 0x0a, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x65, 0x70, - 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, - 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd1, 0x02, 0x0a, - 0x08, 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4a, 0x0a, 0x09, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, - 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, - 0x12, 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, - 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, - 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x44, - 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, - 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x14, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, - 0x74, 0x73, 0x12, 0x28, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x65, - 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, - 0x42, 0x28, 0x5a, 0x26, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, - 0x70, 0x68, 0x70, 0x6d, 0x2f, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, + 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x0a, + 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, + 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd1, 0x02, 0x0a, 0x08, + 0x44, 0x69, 0x73, 0x70, 0x61, 0x74, 0x63, 0x68, 0x12, 0x4a, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1d, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x57, 0x61, 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x12, + 0x1b, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, + 0x69, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x65, + 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x4a, + 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x44, 0x65, + 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, + 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x14, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x12, 0x28, 0x2e, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x65, 0x70, + 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, 0x42, + 0x28, 0x5a, 0x26, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x70, + 0x68, 0x70, 0x6d, 0x2f, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -1969,7 +2116,7 @@ func file_api_v1_ephemerd_proto_rawDescGZIP() []byte { } var file_api_v1_ephemerd_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_api_v1_ephemerd_proto_msgTypes = make([]protoimpl.MessageInfo, 28) +var file_api_v1_ephemerd_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_api_v1_ephemerd_proto_goTypes = []interface{}{ (UpgradeState)(0), // 0: ephemerd.v1.UpgradeState (*PruneCacheRequest)(nil), // 1: ephemerd.v1.PruneCacheRequest @@ -1977,66 +2124,68 @@ var file_api_v1_ephemerd_proto_goTypes = []interface{}{ (*PruneCacheResult)(nil), // 3: ephemerd.v1.PruneCacheResult (*StatusRequest)(nil), // 4: ephemerd.v1.StatusRequest (*StatusResponse)(nil), // 5: ephemerd.v1.StatusResponse - (*ListJobsRequest)(nil), // 6: ephemerd.v1.ListJobsRequest - (*ListJobsResponse)(nil), // 7: ephemerd.v1.ListJobsResponse - (*GetJobRequest)(nil), // 8: ephemerd.v1.GetJobRequest - (*Job)(nil), // 9: ephemerd.v1.Job - (*KillJobRequest)(nil), // 10: ephemerd.v1.KillJobRequest - (*KillJobResponse)(nil), // 11: ephemerd.v1.KillJobResponse - (*CordonRequest)(nil), // 12: ephemerd.v1.CordonRequest - (*CordonResponse)(nil), // 13: ephemerd.v1.CordonResponse - (*UncordonRequest)(nil), // 14: ephemerd.v1.UncordonRequest - (*UncordonResponse)(nil), // 15: ephemerd.v1.UncordonResponse - (*GetJobLogsRequest)(nil), // 16: ephemerd.v1.GetJobLogsRequest - (*LogChunk)(nil), // 17: ephemerd.v1.LogChunk - (*UpgradeRequest)(nil), // 18: ephemerd.v1.UpgradeRequest - (*UpgradeProgress)(nil), // 19: ephemerd.v1.UpgradeProgress - (*CreateJobRequest)(nil), // 20: ephemerd.v1.CreateJobRequest - (*CreateJobResponse)(nil), // 21: ephemerd.v1.CreateJobResponse - (*WaitJobRequest)(nil), // 22: ephemerd.v1.WaitJobRequest - (*WaitJobResponse)(nil), // 23: ephemerd.v1.WaitJobResponse - (*DestroyJobRequest)(nil), // 24: ephemerd.v1.DestroyJobRequest - (*DestroyJobResponse)(nil), // 25: ephemerd.v1.DestroyJobResponse - (*StreamContainerStatsRequest)(nil), // 26: ephemerd.v1.StreamContainerStatsRequest - (*ContainerStatsBatch)(nil), // 27: ephemerd.v1.ContainerStatsBatch - (*ContainerStats)(nil), // 28: ephemerd.v1.ContainerStats + (*SlotPool)(nil), // 6: ephemerd.v1.SlotPool + (*ListJobsRequest)(nil), // 7: ephemerd.v1.ListJobsRequest + (*ListJobsResponse)(nil), // 8: ephemerd.v1.ListJobsResponse + (*GetJobRequest)(nil), // 9: ephemerd.v1.GetJobRequest + (*Job)(nil), // 10: ephemerd.v1.Job + (*KillJobRequest)(nil), // 11: ephemerd.v1.KillJobRequest + (*KillJobResponse)(nil), // 12: ephemerd.v1.KillJobResponse + (*CordonRequest)(nil), // 13: ephemerd.v1.CordonRequest + (*CordonResponse)(nil), // 14: ephemerd.v1.CordonResponse + (*UncordonRequest)(nil), // 15: ephemerd.v1.UncordonRequest + (*UncordonResponse)(nil), // 16: ephemerd.v1.UncordonResponse + (*GetJobLogsRequest)(nil), // 17: ephemerd.v1.GetJobLogsRequest + (*LogChunk)(nil), // 18: ephemerd.v1.LogChunk + (*UpgradeRequest)(nil), // 19: ephemerd.v1.UpgradeRequest + (*UpgradeProgress)(nil), // 20: ephemerd.v1.UpgradeProgress + (*CreateJobRequest)(nil), // 21: ephemerd.v1.CreateJobRequest + (*CreateJobResponse)(nil), // 22: ephemerd.v1.CreateJobResponse + (*WaitJobRequest)(nil), // 23: ephemerd.v1.WaitJobRequest + (*WaitJobResponse)(nil), // 24: ephemerd.v1.WaitJobResponse + (*DestroyJobRequest)(nil), // 25: ephemerd.v1.DestroyJobRequest + (*DestroyJobResponse)(nil), // 26: ephemerd.v1.DestroyJobResponse + (*StreamContainerStatsRequest)(nil), // 27: ephemerd.v1.StreamContainerStatsRequest + (*ContainerStatsBatch)(nil), // 28: ephemerd.v1.ContainerStatsBatch + (*ContainerStats)(nil), // 29: ephemerd.v1.ContainerStats } var file_api_v1_ephemerd_proto_depIdxs = []int32{ 3, // 0: ephemerd.v1.PruneCacheResponse.results:type_name -> ephemerd.v1.PruneCacheResult - 9, // 1: ephemerd.v1.ListJobsResponse.jobs:type_name -> ephemerd.v1.Job - 0, // 2: ephemerd.v1.UpgradeProgress.state:type_name -> ephemerd.v1.UpgradeState - 28, // 3: ephemerd.v1.ContainerStatsBatch.stats:type_name -> ephemerd.v1.ContainerStats - 4, // 4: ephemerd.v1.Control.Status:input_type -> ephemerd.v1.StatusRequest - 6, // 5: ephemerd.v1.Control.ListJobs:input_type -> ephemerd.v1.ListJobsRequest - 8, // 6: ephemerd.v1.Control.GetJob:input_type -> ephemerd.v1.GetJobRequest - 10, // 7: ephemerd.v1.Control.KillJob:input_type -> ephemerd.v1.KillJobRequest - 16, // 8: ephemerd.v1.Control.GetJobLogs:input_type -> ephemerd.v1.GetJobLogsRequest - 12, // 9: ephemerd.v1.Control.Cordon:input_type -> ephemerd.v1.CordonRequest - 14, // 10: ephemerd.v1.Control.Uncordon:input_type -> ephemerd.v1.UncordonRequest - 18, // 11: ephemerd.v1.Control.Upgrade:input_type -> ephemerd.v1.UpgradeRequest - 1, // 12: ephemerd.v1.Control.PruneCache:input_type -> ephemerd.v1.PruneCacheRequest - 20, // 13: ephemerd.v1.Dispatch.CreateJob:input_type -> ephemerd.v1.CreateJobRequest - 22, // 14: ephemerd.v1.Dispatch.WaitJob:input_type -> ephemerd.v1.WaitJobRequest - 24, // 15: ephemerd.v1.Dispatch.DestroyJob:input_type -> ephemerd.v1.DestroyJobRequest - 26, // 16: ephemerd.v1.Dispatch.StreamContainerStats:input_type -> ephemerd.v1.StreamContainerStatsRequest - 5, // 17: ephemerd.v1.Control.Status:output_type -> ephemerd.v1.StatusResponse - 7, // 18: ephemerd.v1.Control.ListJobs:output_type -> ephemerd.v1.ListJobsResponse - 9, // 19: ephemerd.v1.Control.GetJob:output_type -> ephemerd.v1.Job - 11, // 20: ephemerd.v1.Control.KillJob:output_type -> ephemerd.v1.KillJobResponse - 17, // 21: ephemerd.v1.Control.GetJobLogs:output_type -> ephemerd.v1.LogChunk - 13, // 22: ephemerd.v1.Control.Cordon:output_type -> ephemerd.v1.CordonResponse - 15, // 23: ephemerd.v1.Control.Uncordon:output_type -> ephemerd.v1.UncordonResponse - 19, // 24: ephemerd.v1.Control.Upgrade:output_type -> ephemerd.v1.UpgradeProgress - 2, // 25: ephemerd.v1.Control.PruneCache:output_type -> ephemerd.v1.PruneCacheResponse - 21, // 26: ephemerd.v1.Dispatch.CreateJob:output_type -> ephemerd.v1.CreateJobResponse - 23, // 27: ephemerd.v1.Dispatch.WaitJob:output_type -> ephemerd.v1.WaitJobResponse - 25, // 28: ephemerd.v1.Dispatch.DestroyJob:output_type -> ephemerd.v1.DestroyJobResponse - 27, // 29: ephemerd.v1.Dispatch.StreamContainerStats:output_type -> ephemerd.v1.ContainerStatsBatch - 17, // [17:30] is the sub-list for method output_type - 4, // [4:17] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 6, // 1: ephemerd.v1.StatusResponse.slot_pools:type_name -> ephemerd.v1.SlotPool + 10, // 2: ephemerd.v1.ListJobsResponse.jobs:type_name -> ephemerd.v1.Job + 0, // 3: ephemerd.v1.UpgradeProgress.state:type_name -> ephemerd.v1.UpgradeState + 29, // 4: ephemerd.v1.ContainerStatsBatch.stats:type_name -> ephemerd.v1.ContainerStats + 4, // 5: ephemerd.v1.Control.Status:input_type -> ephemerd.v1.StatusRequest + 7, // 6: ephemerd.v1.Control.ListJobs:input_type -> ephemerd.v1.ListJobsRequest + 9, // 7: ephemerd.v1.Control.GetJob:input_type -> ephemerd.v1.GetJobRequest + 11, // 8: ephemerd.v1.Control.KillJob:input_type -> ephemerd.v1.KillJobRequest + 17, // 9: ephemerd.v1.Control.GetJobLogs:input_type -> ephemerd.v1.GetJobLogsRequest + 13, // 10: ephemerd.v1.Control.Cordon:input_type -> ephemerd.v1.CordonRequest + 15, // 11: ephemerd.v1.Control.Uncordon:input_type -> ephemerd.v1.UncordonRequest + 19, // 12: ephemerd.v1.Control.Upgrade:input_type -> ephemerd.v1.UpgradeRequest + 1, // 13: ephemerd.v1.Control.PruneCache:input_type -> ephemerd.v1.PruneCacheRequest + 21, // 14: ephemerd.v1.Dispatch.CreateJob:input_type -> ephemerd.v1.CreateJobRequest + 23, // 15: ephemerd.v1.Dispatch.WaitJob:input_type -> ephemerd.v1.WaitJobRequest + 25, // 16: ephemerd.v1.Dispatch.DestroyJob:input_type -> ephemerd.v1.DestroyJobRequest + 27, // 17: ephemerd.v1.Dispatch.StreamContainerStats:input_type -> ephemerd.v1.StreamContainerStatsRequest + 5, // 18: ephemerd.v1.Control.Status:output_type -> ephemerd.v1.StatusResponse + 8, // 19: ephemerd.v1.Control.ListJobs:output_type -> ephemerd.v1.ListJobsResponse + 10, // 20: ephemerd.v1.Control.GetJob:output_type -> ephemerd.v1.Job + 12, // 21: ephemerd.v1.Control.KillJob:output_type -> ephemerd.v1.KillJobResponse + 18, // 22: ephemerd.v1.Control.GetJobLogs:output_type -> ephemerd.v1.LogChunk + 14, // 23: ephemerd.v1.Control.Cordon:output_type -> ephemerd.v1.CordonResponse + 16, // 24: ephemerd.v1.Control.Uncordon:output_type -> ephemerd.v1.UncordonResponse + 20, // 25: ephemerd.v1.Control.Upgrade:output_type -> ephemerd.v1.UpgradeProgress + 2, // 26: ephemerd.v1.Control.PruneCache:output_type -> ephemerd.v1.PruneCacheResponse + 22, // 27: ephemerd.v1.Dispatch.CreateJob:output_type -> ephemerd.v1.CreateJobResponse + 24, // 28: ephemerd.v1.Dispatch.WaitJob:output_type -> ephemerd.v1.WaitJobResponse + 26, // 29: ephemerd.v1.Dispatch.DestroyJob:output_type -> ephemerd.v1.DestroyJobResponse + 28, // 30: ephemerd.v1.Dispatch.StreamContainerStats:output_type -> ephemerd.v1.ContainerStatsBatch + 18, // [18:31] is the sub-list for method output_type + 5, // [5:18] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_api_v1_ephemerd_proto_init() } @@ -2106,7 +2255,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListJobsRequest); i { + switch v := v.(*SlotPool); i { case 0: return &v.state case 1: @@ -2118,7 +2267,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListJobsResponse); i { + switch v := v.(*ListJobsRequest); i { case 0: return &v.state case 1: @@ -2130,7 +2279,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetJobRequest); i { + switch v := v.(*ListJobsResponse); i { case 0: return &v.state case 1: @@ -2142,7 +2291,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Job); i { + switch v := v.(*GetJobRequest); i { case 0: return &v.state case 1: @@ -2154,7 +2303,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KillJobRequest); i { + switch v := v.(*Job); i { case 0: return &v.state case 1: @@ -2166,7 +2315,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KillJobResponse); i { + switch v := v.(*KillJobRequest); i { case 0: return &v.state case 1: @@ -2178,7 +2327,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CordonRequest); i { + switch v := v.(*KillJobResponse); i { case 0: return &v.state case 1: @@ -2190,7 +2339,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CordonResponse); i { + switch v := v.(*CordonRequest); i { case 0: return &v.state case 1: @@ -2202,7 +2351,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UncordonRequest); i { + switch v := v.(*CordonResponse); i { case 0: return &v.state case 1: @@ -2214,7 +2363,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UncordonResponse); i { + switch v := v.(*UncordonRequest); i { case 0: return &v.state case 1: @@ -2226,7 +2375,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetJobLogsRequest); i { + switch v := v.(*UncordonResponse); i { case 0: return &v.state case 1: @@ -2238,7 +2387,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogChunk); i { + switch v := v.(*GetJobLogsRequest); i { case 0: return &v.state case 1: @@ -2250,7 +2399,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpgradeRequest); i { + switch v := v.(*LogChunk); i { case 0: return &v.state case 1: @@ -2262,7 +2411,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpgradeProgress); i { + switch v := v.(*UpgradeRequest); i { case 0: return &v.state case 1: @@ -2274,7 +2423,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateJobRequest); i { + switch v := v.(*UpgradeProgress); i { case 0: return &v.state case 1: @@ -2286,7 +2435,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateJobResponse); i { + switch v := v.(*CreateJobRequest); i { case 0: return &v.state case 1: @@ -2298,7 +2447,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WaitJobRequest); i { + switch v := v.(*CreateJobResponse); i { case 0: return &v.state case 1: @@ -2310,7 +2459,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WaitJobResponse); i { + switch v := v.(*WaitJobRequest); i { case 0: return &v.state case 1: @@ -2322,7 +2471,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DestroyJobRequest); i { + switch v := v.(*WaitJobResponse); i { case 0: return &v.state case 1: @@ -2334,7 +2483,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DestroyJobResponse); i { + switch v := v.(*DestroyJobRequest); i { case 0: return &v.state case 1: @@ -2346,7 +2495,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StreamContainerStatsRequest); i { + switch v := v.(*DestroyJobResponse); i { case 0: return &v.state case 1: @@ -2358,7 +2507,7 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ContainerStatsBatch); i { + switch v := v.(*StreamContainerStatsRequest); i { case 0: return &v.state case 1: @@ -2370,6 +2519,18 @@ func file_api_v1_ephemerd_proto_init() { } } file_api_v1_ephemerd_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ContainerStatsBatch); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_v1_ephemerd_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ContainerStats); i { case 0: return &v.state @@ -2388,7 +2549,7 @@ func file_api_v1_ephemerd_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_api_v1_ephemerd_proto_rawDesc, NumEnums: 1, - NumMessages: 28, + NumMessages: 29, NumExtensions: 0, NumServices: 2, }, diff --git a/api/v1/ephemerd.proto b/api/v1/ephemerd.proto index ebad313..c7d9a13 100644 --- a/api/v1/ephemerd.proto +++ b/api/v1/ephemerd.proto @@ -76,6 +76,42 @@ message StatusResponse { // The upgrade flow polls this after a restart to confirm the new binary // came up on the target version. string version = 6; + // held_slots is how many concurrency slots are currently HELD across every + // dispatch pool, which is not the same thing as active_jobs. + // + // active_jobs counts jobs the scheduler is tracking. A slot is taken before + // the job is tracked and given back after it is untracked, so a job that is + // still provisioning — or one whose slot leaked — is charged here and not + // there. Issue #196: a mac held its only slot for 28 hours after a wedged + // provision and reported status ok with active_jobs 0 the whole time. + // held_slots == slot_capacity with active_jobs 0 is that failure, visible. + int32 held_slots = 7; + // slot_capacity is the total number of slots across all pools. + int32 slot_capacity = 8; + // slot_pools breaks the totals down per dispatch pool (local, linux, + // macos), so an operator can tell which pool is saturated. + repeated SlotPool slot_pools = 9; + // abandoned_macos_vms counts macOS VMs whose teardown this daemon gave up + // on and has not seen finish. + // + // This is the COST of the #196 fix, made visible. When a Virtualization. + // framework stop wedges, the daemon walks away from it so the concurrency + // slot comes back — but the guest is still running and still holding its + // full configured RAM, and only a restart reclaims it. Non-zero means the + // node is losing memory it cannot get back; at abandoned_macos_vm_cap the + // macOS pool refuses further provisioning rather than growing without + // bound (Vz permits only a couple of concurrent macOS guests anyway). + int32 abandoned_macos_vms = 10; + // abandoned_macos_vm_cap is the value of abandoned_macos_vms at which the + // macOS pool stops taking work. + int32 abandoned_macos_vm_cap = 11; +} + +// SlotPool is one dispatch pool's concurrency-slot occupancy. +message SlotPool { + string pool = 1; + int32 held = 2; + int32 capacity = 3; } message ListJobsRequest {} diff --git a/cmd/ephemerd/commands.go b/cmd/ephemerd/commands.go index 8788bf1..2b1992b 100644 --- a/cmd/ephemerd/commands.go +++ b/cmd/ephemerd/commands.go @@ -69,13 +69,34 @@ func statusCmd() *cli.Command { return fmt.Errorf("status: %w", err) } + // held_slots is deliberately next to active_jobs: they differ + // exactly when a job is provisioning or a slot leaked, and a + // node whose slots are all held while active_jobs reads 0 is + // the shape of the 28-hour macOS stall in issue #196. + slots := make([]map[string]any, 0, len(resp.SlotPools)) + for _, p := range resp.SlotPools { + slots = append(slots, map[string]any{ + "pool": p.Pool, + "held": p.Held, + "capacity": p.Capacity, + }) + } + data := map[string]any{ "status": resp.Status, "active_jobs": resp.ActiveJobs, "max_concurrent": resp.MaxConcurrent, - "draining": resp.Draining, - "uptime": resp.Uptime, - "version": resp.Version, + "held_slots": resp.HeldSlots, + "slot_capacity": resp.SlotCapacity, + "slots": slots, + // Non-zero means macOS guests are stranded on this node, + // holding RAM nothing but a restart reclaims; at the cap the + // macOS pool has stopped taking work. See issue #196. + "abandoned_macos_vms": resp.AbandonedMacosVms, + "abandoned_macos_vm_cap": resp.AbandonedMacosVmCap, + "draining": resp.Draining, + "uptime": resp.Uptime, + "version": resp.Version, } pretty, _ := json.MarshalIndent(data, "", " ") diff --git a/docs/cli/status.md b/docs/cli/status.md index 1c3dd04..ab675dc 100644 --- a/docs/cli/status.md +++ b/docs/cli/status.md @@ -16,8 +16,13 @@ Returns a JSON object with the following fields: | Field | Description | |-------|-------------| | `status` | Current daemon status | -| `active_jobs` | Number of jobs currently running | +| `active_jobs` | Number of jobs currently **tracked** as running | | `max_concurrent` | Maximum concurrent jobs allowed | +| `held_slots` | Concurrency slots currently held across all dispatch pools | +| `slot_capacity` | Total slots across all dispatch pools | +| `slots` | Per-pool breakdown (`local`, `linux`, `macos`) of `held` and `capacity` | +| `abandoned_macos_vms` | macOS VMs whose teardown the daemon gave up on and has not seen die | +| `abandoned_macos_vm_cap` | Value of `abandoned_macos_vms` at which the macOS pool stops taking work | | `draining` | Whether the daemon is draining (shutting down gracefully) | | `uptime` | How long the daemon has been running | @@ -28,11 +33,53 @@ Returns a JSON object with the following fields: "status": "running", "active_jobs": 2, "max_concurrent": 4, + "held_slots": 2, + "slot_capacity": 9, + "slots": [ + { "pool": "local", "held": 2, "capacity": 4 }, + { "pool": "linux", "held": 0, "capacity": 4 }, + { "pool": "macos", "held": 0, "capacity": 1 } + ], + "abandoned_macos_vms": 0, + "abandoned_macos_vm_cap": 2, "draining": false, "uptime": "3h42m15s" } ``` +### held_slots vs active_jobs + +They are not the same number, and the difference is diagnostic. + +A dispatch takes a concurrency slot *before* it registers the runner and gives +it back *after* the job is untracked, so a job that is still provisioning is +counted in `held_slots` and not in `active_jobs`. Briefly, that is normal. + +Sustained, it is not. A pool sitting at `held == capacity` with `active_jobs` +at 0 means the capacity is charged to something the scheduler is not tracking — +a provision that is stuck, or a slot that leaked. The daemon also logs +`waiting for a free concurrency slot` (and escalates to a suspected-leak error) +when a job blocks on such a pool, so the log and this output agree. + +### abandoned_macos_vms + +When a macOS VM's `Stop()` does not return — Virtualization.framework can wedge +a teardown indefinitely — the daemon walks away from it after a grace period so +the concurrency slot comes back. The slot is the thing that matters: a held one +costs the node its whole macOS capacity. + +The guest, however, is still running, and still holding its full configured RAM. +Nothing in the daemon can reclaim it; only a restart can. So each abandonment is +counted here, and once `abandoned_macos_vms` reaches `abandoned_macos_vm_cap` the +macOS pool refuses to provision anything further and says so at `ERROR` — a +bounded, legible failure instead of unbounded memory growth on a host that +Virtualization.framework will soon refuse anyway. + +Non-zero is worth investigating. At the cap, restart the daemon. + +`/healthz` carries the same fields, plus `abandoned_macos_vm_details` (job id, +VM instance id, reason and age per stranded guest) when there are any. + ## Connection The command connects to the daemon's gRPC unix socket at `/ephemerd.sock`. If the daemon is not running or the socket does not exist, the command prints an error. diff --git a/pkg/scheduler/grpc.go b/pkg/scheduler/grpc.go index 40f3869..6c343e5 100644 --- a/pkg/scheduler/grpc.go +++ b/pkg/scheduler/grpc.go @@ -79,13 +79,34 @@ func (c *controlServer) Status(ctx context.Context, req *apiv1.StatusRequest) (* draining := c.sched.draining c.sched.mu.Unlock() + // Slot occupancy is reported alongside the tracked-job count, never in + // place of it. ActiveJobs is len(s.running) and a slot is held from + // before trackRunning to after untrackRunning, so a stuck provision — or + // a leaked slot — shows up here and nowhere else. See issue #196. + var pools []*apiv1.SlotPool + for _, p := range c.sched.SlotUsage() { + pools = append(pools, &apiv1.SlotPool{ + Pool: p.Pool, + Held: int32(p.Held), + Capacity: int32(p.Capacity), + }) + } + return &apiv1.StatusResponse{ Status: "ok", ActiveJobs: int32(activeJobs), MaxConcurrent: int32(c.sched.cfg.MaxConcurrent), - Draining: draining, - Uptime: time.Since(c.sched.startTime).Truncate(time.Second).String(), - Version: c.sched.cfg.Version, + HeldSlots: int32(c.sched.HeldSlots()), + SlotCapacity: int32(c.sched.SlotCapacity()), + SlotPools: pools, + // The other half of the slot story: slots that came back only because + // the daemon abandoned a teardown. A guest per count, still holding + // its RAM, reclaimable only by restart. See issue #196 and slots.go. + AbandonedMacosVms: int32(c.sched.AbandonedMacOSVMs()), + AbandonedMacosVmCap: int32(c.sched.abandonedMacVMCap()), + Draining: draining, + Uptime: time.Since(c.sched.startTime).Truncate(time.Second).String(), + Version: c.sched.cfg.Version, }, nil } diff --git a/pkg/scheduler/macos_provision_test.go b/pkg/scheduler/macos_provision_test.go index da57eff..5271088 100644 --- a/pkg/scheduler/macos_provision_test.go +++ b/pkg/scheduler/macos_provision_test.go @@ -132,13 +132,16 @@ func TestWaitForMacRunnerBounded_ReturnsWhenWaitSucceeds(t *testing.T) { var stops atomic.Int32 quick := &fastMacVM{ip: "192.168.64.5", stops: &stops} - ip, err := s.waitForMacRunnerBounded(context.Background(), quick, quietLogger()) + ip, alreadyStopped, err := s.waitForMacRunnerBounded(context.Background(), quick, macVMRef{JobID: 1, VMID: "1-happy"}, quietLogger()) if err != nil { t.Fatalf("unexpected error on healthy provision: %v", err) } if ip != "192.168.64.5" { t.Errorf("ip = %q, want 192.168.64.5", ip) } + if alreadyStopped { + t.Error("healthy provision reported the VM as already stopped; the caller would skip a teardown it still owes") + } if stops.Load() != 0 { t.Errorf("VM was stopped %d times on the happy path, want 0", stops.Load()) } diff --git a/pkg/scheduler/multiprovider_test.go b/pkg/scheduler/multiprovider_test.go index 158b58e..bc002c6 100644 --- a/pkg/scheduler/multiprovider_test.go +++ b/pkg/scheduler/multiprovider_test.go @@ -18,10 +18,21 @@ type mockProvider struct { defaultImage string events chan providers.JobEvent + // releaseHook runs at the top of ReleaseJob, before the mutex is taken. + // Set once before the test starts and never mutated afterwards. + // + // It exists to model what ReleaseJob actually is in production — a GitHub + // API call, on context.Background(), with no deadline — so a test can + // assert that a concurrency slot is FREE while that call is still in + // flight. Without a slow ReleaseJob the release-before-teardown ordering + // this fix depends on is unobservable: the mock returns instantly, so + // moving the release after the teardown still passes. + releaseHook func(*providers.Claim) + mu sync.Mutex - claims []*providers.Claim // jobs claimed via ClaimJob - releases []*providers.Claim // jobs released via ReleaseJob - images map[int64]string // jobID → image for FetchJobImage + claims []*providers.Claim // jobs claimed via ClaimJob + releases []*providers.Claim // jobs released via ReleaseJob + images map[int64]string // jobID → image for FetchJobImage } var _ providers.Poll = (*mockProvider)(nil) @@ -62,6 +73,9 @@ func (m *mockProvider) ClaimJob(_ context.Context, event *providers.JobEvent, ru } func (m *mockProvider) ReleaseJob(_ context.Context, claim *providers.Claim) error { + if m.releaseHook != nil { + m.releaseHook(claim) + } m.mu.Lock() m.releases = append(m.releases, claim) m.mu.Unlock() diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c3f72d6..0bac88e 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -2,6 +2,8 @@ package scheduler import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -181,6 +183,22 @@ const defaultOrphanGrace = 10 * time.Minute // macOS slot in minutes instead of hours. const defaultMacOSProvisionTimeout = 5 * time.Minute +// macProvisionUnwindGrace bounds how long the provisioning watchdog waits for +// a force-stopped macOS VM to tear down and its reachability wait to unwind +// before giving up on both and returning anyway. +// +// Comfortably more than darwinMacOSVM.stop's own 15s graceful-shutdown window +// plus the force-stop that follows it, so a VM that CAN be killed is always +// reaped tidily. Past that we stop caring: the whole point of the deadline was +// to get the concurrency slot back, and waiting on an unkillable VM to say so +// is what cost the fleet 28 hours of macOS CI (#196). +const macProvisionUnwindGrace = 30 * time.Second + +// macTeardownGrace bounds a routine macOS VM Stop() on any path that is +// holding a slot or a wait-goroutine. Same reasoning as +// macProvisionUnwindGrace: teardown is best-effort, capacity is not. +const macTeardownGrace = 30 * time.Second + // jobKey uniquely identifies a job across providers. Different providers // can return the same int64 job ID, so we include the provider name. type jobKey struct { @@ -260,6 +278,23 @@ type Scheduler struct { // watchdog without a real Virtualization.framework VM. newMacOSVM func(cfg vm.MacOSVMConfig, jobID string) (vm.MacOSVM, error) + // macUnwindGrace / macStopGrace bound macOS VM teardown on the paths that + // must not be delayed by it (see slots.go). Zero uses the package + // defaults; only tests set them, to compress a 30s grace into + // milliseconds. + macUnwindGrace time.Duration + macStopGrace time.Duration + + // macAbandoned counts macOS VMs whose teardown this daemon walked away + // from and has not seen die. Releasing the slot on a wedged teardown + // (#196) removed the semaphore's accidental cap on how many guests could + // be stranded; this is the explicit one. See slots.go. + macAbandoned abandonedMacVMs + + // macAbandonedCap overrides defaultMacAbandonedVMCap. Zero uses the + // default; only tests set it. + macAbandonedCap int + // busyProbe overrides the ground-truth "is this runner executing a // job" check that vetoes orphan teardown. Nil (the default) uses // probeRunnerBusy: local container/VM/process introspection, falling @@ -331,9 +366,16 @@ type runningJob struct { artifactsDir string // non-empty if OCI artifacts were extracted for this job dispatched string // non-empty if dispatched to Linux VM worker (stores container name) macosVM vm.MacOSVM // non-nil if running as a macOS VM job + macVMID string // per-attempt macOS VM instance id (macVMInstanceID); owns this job's on-disk paths startedAt time.Time } +// macRef is the identity every macOS teardown site reports to the +// abandoned-VM registry. Safe on a non-macOS job (returns a zero VMID). +func (rj *runningJob) macRef(key jobKey) macVMRef { + return macVMRef{JobID: key.JobID, VMID: rj.macVMID} +} + // runnerName returns the name the runner was registered under with the // provider. Every provisioning path stores it on the claim; the Linux // dispatch path mirrors it in dispatched. @@ -1164,15 +1206,25 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent } // Acquire Linux dispatch concurrency slot (separate from local/macOS) - select { - case s.linuxSem <- struct{}{}: - case <-ctx.Done(): + slot := s.acquireSlot(ctx, s.linuxSem, "linux", log) + if slot == nil { unsee() return } + // Backstop until the wait-goroutine below takes ownership: every exit + // from this function returns the slot, including one a future edit + // forgets to release explicitly. See slots.go — a single missed release + // on the macOS path cost 28 hours of CI (#196). + handedOff := false + defer func() { + if !handedOff { + slot.release() + } + }() + if v := s.admitDispatch(key); v != dispatchAdmit { v.log(log) - <-s.linuxSem + slot.release() return } @@ -1190,7 +1242,7 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent if err != nil { log.Error("failed to claim job", "error", err, "error_class", classifyErr(err)) unsee() - <-s.linuxSem + slot.release() // Replaces the old blind time.Sleep(backoffDuration): the // sem is released FIRST so we do not hold a slot idle across // the wait, then a rate-aware jittered retry is enqueued @@ -1211,7 +1263,7 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent } unsee() cancel() - <-s.linuxSem + slot.release() // Mirror the claim-failure branch above (and the local create // path): a failed dispatch create used to drop the job forever — // webhooks are never re-delivered. Slot released first; no-op for @@ -1234,18 +1286,12 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent log.Info("Linux runner dispatched", "name", claim.RunnerName) // Wait for the job to finish in the background + handedOff = true go func() { // Same slot-release ordering as the local wait-goroutine: release // once the job is untracked, never behind the destroy. The // deferred call is only the panic/early-return backstop. - released := false - release := func() { - if !released { - released = true - <-s.linuxSem - } - } - defer release() + defer slot.release() exitCode, err := s.cfg.LinuxDispatcher.Wait(jobCtx, claim.RunnerName) if err != nil { @@ -1275,7 +1321,7 @@ func (s *Scheduler) handleLinuxJob(ctx context.Context, event providers.JobEvent // Release BEFORE the destroy RPC, mirroring the local path: the // dispatched env is keyed by its unique claim.RunnerName, so a // newly admitted job cannot collide with this teardown. - release() + slot.release() if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), claim.RunnerName); err != nil { log.Warn("dispatch destroy failed", "error", err) @@ -1312,16 +1358,83 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent s.mu.Unlock() } + // refuseCordoned drops this dispatch because the macOS pool has given up + // on too many VMs to safely start another (see slots.go). + // + // It leaves the seen stamp and clears only pending, exactly like the #154 + // cordon rejection: seen ages out via seenTTL, so the node retries in ~10 + // minutes rather than re-logging this every poll, and the job stays queued + // on the platform so a healthy node can take it immediately. + refuseCordoned := func(n, limit int) { + s.mu.Lock() + delete(s.pending, key) + s.mu.Unlock() + log.Error("refusing to provision a macOS VM: this node has abandoned too many macOS VMs without confirming they died", + "abandoned_macos_vms", n, + "cap", limit, + "detail", "each one is a live guest holding its full configured RAM; starting more risks OOMing the host and Vz will refuse anyway. Restart ephemerd to reclaim them.") + } + + // PER-ATTEMPT VM IDENTITY. Every macOS resource used to be keyed by job id + // alone: the VM's shared directory (/vm/macos/jobs/), its disk + // and aux clones (.img / .aux) and its extracted OCI artifacts. On + // the failure paths below this function now releases the slot BEFORE + // teardown, which means the same job id can be re-dispatched (poll re-emits + // it, or the webhook catch-up poll does) while the previous attempt's VM is + // still alive with a stop wedged inside Virtualization.framework. + // + // With shared paths that is mutual destruction: attempt 2 apfsCopies over + // the .img/.aux files attempt 1's guest still has open and writes its JIT + // config into the same share, and then attempt 1's abandoned stop finally + // returns minutes later and runs os.Remove(clonePath) + os.RemoveAll(jobDir) + // — deleting attempt 2's disk and its .jit_config, so attempt 2's runner can + // never register and it times out too. Self-sustaining, and each round + // strands another guest. + // + // A per-attempt suffix makes the attempts disjoint by construction, so a + // late teardown can only ever delete its own files. + vmID := macVMInstanceID(jobID) + ref := macVMRef{JobID: jobID, VMID: vmID} + log = log.With("vm", vmID) + + if n, limit, cordoned := s.macPoolCordoned(); cordoned { + refuseCordoned(n, limit) + return + } + // Acquire macOS VM concurrency slot (separate from Linux/local sem). - select { - case s.macSem <- struct{}{}: - case <-ctx.Done(): + slot := s.acquireSlot(ctx, s.macSem, "macos", log) + if slot == nil { unsee() return } + // Backstop until the wait-goroutine below takes ownership. This function + // is issue #196's crime scene: it had six hand-written `<-s.macSem` + // returns and one of them was unreachable, because the provisioning + // watchdog it depended on could block forever before returning. The + // deferred release makes "did every path give the slot back?" a property + // of the function rather than of the reader's attention span. + handedOff := false + defer func() { + if !handedOff { + slot.release() + } + }() + if v := s.admitDispatch(key); v != dispatchAdmit { v.log(log) - <-s.macSem + slot.release() + return + } + + // Re-check the abandoned-VM cordon now that the wait for the slot is over. + // Same reasoning as the #154 cordon re-check in admitDispatch: this + // dispatch may have been parked on the semaphore for the entire life of + // the job ahead of it, and that is exactly the window in which a wedged + // teardown gets abandoned. + if n, limit, cordoned := s.macPoolCordoned(); cordoned { + slot.release() + refuseCordoned(n, limit) return } @@ -1331,7 +1444,11 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent image := s.resolveImage(ctx, &event, "darwin") var artifactsDir string if image != "" && s.cfg.Artifacts != nil { - artifactsDir = artifacts.ArtifactsDir(s.cfg.DataDir, fmt.Sprintf("%d", jobID)) + // Keyed by the VM instance, not the job: this directory is deleted by + // artifacts.Cleanup on every failure path below, all of which now run + // after the slot is released and therefore possibly concurrently with + // a fresh attempt at the same job id. + artifactsDir = artifacts.ArtifactsDir(s.cfg.DataDir, vmID) log.Info("extracting OCI artifacts for macOS VM job", "image", image, "dest", artifactsDir) if err := s.cfg.Artifacts.Extract(ctx, image, artifactsDir); err != nil { log.Error("failed to extract OCI artifacts", "image", image, "error", err) @@ -1350,13 +1467,14 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent artifacts.Cleanup(artifactsDir, s.cfg.Log) } unsee() - <-s.macSem + slot.release() s.enqueueRetryIfEligible(ctx, event, err) return } - // Create the macOS VM - macVM, err := s.newMacOSVM(*s.cfg.MacOSVMConfig, fmt.Sprintf("%d", jobID)) + // Create the macOS VM. vmID (not the bare job id) is what every path this + // VM owns on disk is named after — see the per-attempt identity note above. + macVM, err := s.newMacOSVM(*s.cfg.MacOSVMConfig, vmID) if err != nil { log.Error("failed to create macOS VM", "error", err) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { @@ -1366,22 +1484,25 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent artifacts.Cleanup(artifactsDir, s.cfg.Log) } unsee() - <-s.macSem + slot.release() return } // Write JIT config to the shared directory before booting if err := macVM.WriteJITConfig(claim.RunnerConfig); err != nil { log.Error("failed to write JIT config", "error", err) - macVM.Stop() + // Slot first, teardown second — for the rest of this function every + // failure path releases before it touches the VM, because Stop() is + // the call that hangs (#196). + unsee() + slot.release() + s.stopVMBounded(macVM, ref, log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } if artifactsDir != "" { artifacts.Cleanup(artifactsDir, s.cfg.Log) } - unsee() - <-s.macSem return } @@ -1392,16 +1513,16 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // Boot the VM if err := macVM.Start(jobCtx); err != nil { log.Error("failed to start macOS VM", "error", err) - macVM.Stop() + unsee() + cancel() + slot.release() + s.stopVMBounded(macVM, ref, log) if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } if artifactsDir != "" { artifacts.Cleanup(artifactsDir, s.cfg.Log) } - unsee() - cancel() - <-s.macSem return } @@ -1410,19 +1531,38 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // can wedge WaitForRunner indefinitely: the job never gets tracked (so it // is invisible to `ephemerd jobs` and the orphan sweep) yet still holds // the sole macOS slot, stalling all macOS CI until a human intervenes. - ip, err := s.waitForMacRunnerBounded(jobCtx, macVM, log) + ip, alreadyStopped, err := s.waitForMacRunnerBounded(jobCtx, macVM, ref, log) if err != nil { log.Error("macOS VM runner not reachable", "error", err) - macVM.Stop() + // THE 28-HOUR LINE (#196). The slot goes back here, first, before + // anything that talks to the VM or to GitHub. It used to go back + // last, after macVM.Stop() — and waitForMacRunnerBounded had already + // force-stopped this VM, so this Stop() re-entered a sync.Once whose + // first call was still wedged inside Virtualization.framework and + // blocked forever. The node logged "force-stopping VM to reclaim the + // slot", never reclaimed it, and starved every subsequent macOS job + // while reporting status: ok / active_jobs: 0. + unsee() + cancel() + slot.release() + // Do NOT stop it again when the deadline path already did. Stopping a + // second time is never a cheap no-op: darwinMacOSVM.Stop is a + // sync.Once, so a second caller BLOCKS until the first invocation + // returns. Against the wedged guest this whole PR is about, that is a + // full teardownGrace (30s) of dead time on a path that still has a + // ghost runner to deregister — and it spawns a third goroutine parked + // inside the Once behind the first two. Against a healthy VM it is a + // pointless goroutine. Either way the VM is already stopped-or- + // abandoned and accounted for, so skip it. + if !alreadyStopped { + s.stopVMBounded(macVM, ref, log) + } if rmErr := event.Provider.ReleaseJob(ctx, claim); rmErr != nil { log.Warn("failed to remove ghost runner", "runner_id", claim.RunnerID, "error", rmErr) } if artifactsDir != "" { artifacts.Cleanup(artifactsDir, s.cfg.Log) } - unsee() - cancel() - <-s.macSem return } @@ -1435,14 +1575,18 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent cancel: cancel, artifactsDir: artifactsDir, macosVM: macVM, + macVMID: vmID, startedAt: time.Now(), }, event.Provider, labelSetKey(event.Labels)) log.Info("macOS VM runner ready", "name", claim.RunnerName, "ip", ip) // Wait for the job to finish in the background + handedOff = true go func() { - defer func() { <-s.macSem }() + // Backstop only. The release that matters happens the moment the job + // is untracked, below. + defer slot.release() exitCode, err := macVM.Wait(jobCtx) if err != nil { @@ -1457,14 +1601,42 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent log.Info("macOS VM exited", "exit_code", exitCode) } - // Clean up + // Clean up. The entry may already be gone if a completed event tore + // this VM down by name (see handleCompleted). s.mu.Lock() rj, exists := s.running[key] if exists { s.untrackRunningLocked(key, rj) - s.mu.Unlock() + } + s.mu.Unlock() + + // Release BEFORE teardown, exactly as the local and Linux + // wait-goroutines do (PR #190). The job is untracked, so its demand + // is gone; everything below can block for a long time and none of it + // may delay the next macOS job. macVM.Stop() waits up to 15s for a + // graceful guest shutdown before forcing (and on the #196 node the + // force did not return either), and ReleaseJob is a GitHub API call + // on context.Background() with no deadline at all. + // + // WHAT MAKES THAT SAFE. Releasing here means the SAME job id can be + // dispatched again while this teardown is still running — unsee() has + // cleared seen/pending on the failure paths, the poll re-emits a still- + // queued job about every 30s, and admitDispatch will admit it. So the + // invariant is NOT "a fresh dispatch cannot collide because everything + // is keyed by job id" — everything being keyed by job id is precisely + // what WOULD collide. The invariant is that every path this teardown + // touches is keyed by the PER-ATTEMPT VM instance id (vmID: the VM's + // shared dir, its .img/.aux clones, its artifacts dir) or by the + // per-claim runner name (ReleaseJob), so a later attempt at the same + // job never shares a byte of state with this one and this teardown can + // only ever delete its own files. + slot.release() + + if exists { metrics.JobsActive.Dec() - rj.macosVM.Stop() + if rj.macosVM != nil { + s.stopVMBounded(rj.macosVM, rj.macRef(key), log) + } if rj.artifactsDir != "" { artifacts.Cleanup(rj.artifactsDir, s.cfg.Log) } @@ -1473,8 +1645,6 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent log.Debug("deregister runner after macOS VM cleanup", "error", err) } } - } else { - s.mu.Unlock() } // Self-heal: re-provision if this VM's dispatched job never ran. @@ -1496,11 +1666,33 @@ func (s *Scheduler) handleMacOSJob(ctx context.Context, event providers.JobEvent // // The only reliable way to interrupt a blocked SSH session call is to drop the // connection: force-stopping the VM tears down the guest, the SSH read errors -// out, and WaitForRunner unwinds. On timeout we do exactly that, then wait for -// the wait goroutine to return so there is no leak, and report a failure. The -// caller's normal error path then releases the slot and deregisters the runner -// — cleanup runs even though the wait had been stuck. -func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSVM, log *slog.Logger) (string, error) { +// out, and WaitForRunner unwinds. On timeout we do exactly that and report a +// failure; the caller's error path releases the slot and deregisters the +// runner, so cleanup runs even though the wait had been stuck. +// +// ISSUE #196 — why the teardown is now BOUNDED. The first version of this +// function force-stopped the VM and then did a bare `<-resCh`, waiting for the +// wait goroutine to unwind "so there is no leak". That assumed killing the VM +// always unblocks the guest wait. On the fleet's mac it did not: the VM's own +// stop path timed out ("macOS VM did not stop gracefully, forcing") and the +// wait never returned, so this function — which had just logged that it was +// force-stopping the VM TO RECLAIM THE SLOT — parked on that receive for 28 +// hours, still holding the slot. The job had never been tracked, so +// active_jobs stayed 0, /healthz stayed 200, and five macOS jobs queued behind +// it aged out at GitHub's 24-hour limit without ever running. +// +// A wedged guest wait is already a leaked goroutine that nothing in this +// process can kill. resCh is buffered so that goroutine can always complete +// its send and exit if it ever comes back. What must NOT happen is turning +// that leaked goroutine into a leaked concurrency slot: teardown gets a +// bounded grace period to be tidy, and then we return regardless. +// +// The second return value reports whether this function already STOPPED the +// VM. The caller's error path must not stop it again: darwinMacOSVM.Stop is a +// sync.Once, so a redundant second Stop blocks until the first returns — +// against a wedged guest that is a full teardown grace of dead time and a +// third goroutine parked in the Once. See handleMacOSJob. +func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSVM, ref macVMRef, log *slog.Logger) (string, bool, error) { timeout := s.cfg.MacOSProvisionTimeout if timeout <= 0 { timeout = defaultMacOSProvisionTimeout @@ -1510,8 +1702,13 @@ func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSV ip string err error } + // Buffered: the wait goroutine must always be able to finish its send and + // exit, even long after we have stopped listening. Nothing below may + // depend on it ever getting that far. resCh := make(chan result, 1) + unwound := make(chan struct{}) go func() { + defer close(unwound) ip, err := macVM.WaitForRunner(ctx) resCh <- result{ip: ip, err: err} }() @@ -1521,19 +1718,109 @@ func (s *Scheduler) waitForMacRunnerBounded(ctx context.Context, macVM vm.MacOSV select { case r := <-resCh: - return r.ip, r.err + return r.ip, false, r.err case <-timer.C: log.Error("macOS VM stuck in provisioning past deadline; force-stopping VM to reclaim the slot", "timeout", timeout) - macVM.Stop() // drops the guest connection so WaitForRunner unblocks - <-resCh // wait goroutine unwinds now that the VM is gone - return "", fmt.Errorf("timed out after %s waiting for macOS VM runner to become reachable", timeout) + s.abandonMacProvision(macVM, unwound, ref, log) + return "", true, fmt.Errorf("timed out after %s waiting for macOS VM runner to become reachable", timeout) case <-ctx.Done(): + s.abandonMacProvision(macVM, unwound, ref, log) + return "", true, ctx.Err() + } +} + +// abandonMacProvision force-stops a macOS VM whose provisioning was given up +// on and gives the teardown a bounded chance to finish before returning. +// +// Stop() runs on its own goroutine for two reasons. The obvious one is that +// Virtualization.framework's stop can itself hang. The subtle one is +// sync.Once: darwinMacOSVM.Stop is `stopOnce.Do(m.stop)`, so a SECOND caller +// does not return early — it blocks until the first invocation returns. The +// caller's error path stops the VM again as part of its normal cleanup, and if +// this teardown were still wedged in the Once, that "cleanup" call would hang +// on a path that is holding a concurrency slot. Off-thread here plus +// stopVMBounded there means neither can pin the slot. +func (s *Scheduler) abandonMacProvision(macVM vm.MacOSVM, unwound <-chan struct{}, ref macVMRef, log *slog.Logger) { + // Dropping the guest is what unblocks a wedged SSH session call. + stopped := stopVMAsync(macVM) + if !awaitUnwind(stopped, unwound, s.provisionUnwindGrace()) { + log.Error("macOS VM teardown did not finish after the force-stop; abandoning it and reclaiming the concurrency slot anyway", + "grace", s.provisionUnwindGrace(), + "detail", "a wedged Vz stop or a wedged guest wait must never pin the macOS slot — see issue #196") + // Charge the guest we just walked away from. The slot comes back + // either way — that is the #196 fix and it is not negotiable — but the + // VM is still running and still holding its RAM, so it has to be + // counted against the cap that decides whether this node may start + // another one. See slots.go. + s.abandonVM(stopped, ref, "force-stop after the provisioning deadline did not return", log) + } +} + +// stopVMAsync stops macVM on its own goroutine and returns a channel closed +// when Stop returns. If Stop never returns, that goroutine is leaked — which +// is the correct trade: the alternative is leaking the concurrency slot, and +// a leaked slot takes the node's whole macOS capacity with it. +func stopVMAsync(macVM vm.MacOSVM) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) macVM.Stop() - <-resCh - return "", ctx.Err() + }() + return done +} + +// stopVMBounded force-stops a VM without letting the teardown pin the caller, +// and charges the VM to the abandoned registry if the grace expires. +// +// Used by EVERY macOS cleanup path, not just the ones holding a slot. Three of +// them used to call macosVM.Stop() directly — handleCompleted, destroyAll and +// sweepOrphanRunners — and the sweep is called synchronously from the +// cleanupTicker arm of Run's select, i.e. on the scheduler's only event loop. +// A wedged Vz stop there does not cost one job's slot, it costs the whole +// daemon: no queued, in_progress or completed handling for any platform, and +// no drain on SIGTERM. That is strictly worse than the leak this PR fixes, so +// there is no such thing as a macOS stop that may block its caller. +func (s *Scheduler) stopVMBounded(macVM vm.MacOSVM, ref macVMRef, log *slog.Logger) { + grace := s.teardownGrace() + timer := time.NewTimer(grace) + defer timer.Stop() + done := stopVMAsync(macVM) + select { + case <-done: + case <-timer.C: + log.Warn("macOS VM stop did not return within the grace period; continuing without it", + "grace", grace) + s.abandonVM(done, ref, "teardown did not return within the stop grace", log) } } +// macVMInstanceID names one PROVISIONING ATTEMPT, not one job. +// +// Everything a macOS VM owns on disk is named after this: its virtio-fs share +// (/vm/macos/jobs/), its disk and aux clones (.img, .aux) +// and its extracted OCI artifacts. Since the #196 fix releases the concurrency +// slot before teardown, a second attempt at the same job can start while the +// first attempt's VM is still alive with an unkillable stop pending — and that +// first attempt's teardown ends in os.Remove(clonePath) and +// os.RemoveAll(jobDir). Sharing those paths between attempts means attempt 1's +// late cleanup deletes attempt 2's running disk and its .jit_config; attempt 2 +// then waits out its own deadline for a runner that can never register, and +// strands another guest doing it. +// +// The job id stays in the name because it is what an operator greps for; the +// random suffix is what makes the attempts disjoint. It is random rather than +// a counter so that a daemon restart cannot re-issue an id whose files are +// still on disk from before the restart. +func macVMInstanceID(jobID int64) string { + var b [4]byte + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand does not fail in practice; if it ever does, a + // timestamp still discriminates attempts (they are minutes apart). + return fmt.Sprintf("%d-%d", jobID, time.Now().UnixNano()) + } + return fmt.Sprintf("%d-%s", jobID, hex.EncodeToString(b[:])) +} + // handleLocalJob provisions a runner using the local containerd Runtime. func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent) { jobID := event.JobID @@ -1549,15 +1836,23 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent } // Acquire concurrency slot - select { - case s.sem <- struct{}{}: - case <-ctx.Done(): + slot := s.acquireSlot(ctx, s.sem, "local", log) + if slot == nil { unsee() return } + // Backstop until the wait-goroutine below takes ownership; see the + // matching comment in handleLinuxJob and slots.go. + handedOff := false + defer func() { + if !handedOff { + slot.release() + } + }() + if v := s.admitDispatch(key); v != dispatchAdmit { v.log(log) - <-s.sem + slot.release() return } @@ -1622,7 +1917,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent artifacts.Cleanup(artifactsDir, s.cfg.Log) } unsee() - <-s.sem + slot.release() // Replaces the old blind 5s sleep with a rate-aware jittered // retry. No-op when Retry is disabled or the error is not // retryable. @@ -1666,7 +1961,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent } unsee() cancel() - <-s.sem + slot.release() // Mirror the claim-failure branch above: without this a job whose // env create failed was dropped forever (GitHub never re-delivers // the webhook) — on metal that was 1h43m of a job sitting queued @@ -1692,6 +1987,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent log.Info("runner environment ready", "name", claim.RunnerName) // Wait for the job to finish in the background + handedOff = true go func() { // Slot release is decoupled from teardown. It used to hang off a // bare defer, which meant the slot came back only after Destroy @@ -1699,14 +1995,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent // MaxConcurrent slot for its whole hang. The slot is released as // soon as the job is untracked below; the deferred call is only // the panic/early-return backstop. - released := false - release := func() { - if !released { - released = true - <-s.sem - } - } - defer release() + defer slot.release() exitCode, err := s.cfg.Runtime.Wait(jobCtx, env) if err != nil { @@ -1740,7 +2029,7 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent // runner dir, netns/HCN endpoint, dind socket all derive from it — // so a freshly admitted job cannot touch anything this teardown // still owns. - release() + slot.release() if exists { metrics.JobsActive.Dec() @@ -1911,7 +2200,11 @@ func (s *Scheduler) handleCompleted(ctx context.Context, event providers.JobEven resetBackoff(event.Repo) job.cancel() if job.macosVM != nil { - job.macosVM.Stop() + // Bounded (#196). This ran as a bare Stop(), so a wedged Vz teardown + // parked the completed-event handler forever — and with it the + // deferred sweepOrphanRunners above, which is how the node reconciles + // runners after every completion. + s.stopVMBounded(job.macosVM, job.macRef(ownerKey), log) } else if job.dispatched != "" && s.cfg.LinuxDispatcher != nil { if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), job.dispatched); err != nil { log.Warn("failed to destroy dispatched runner", "error", err) @@ -1995,7 +2288,13 @@ func (s *Scheduler) destroyAll() { s.cfg.Log.Info("destroying runner on shutdown", "job_id", key.JobID, "provider", key.Provider) job.cancel() if job.macosVM != nil { - job.macosVM.Stop() + // Bounded (#196). destroyAll is the force-kill arm of drain(), so + // a bare Stop() here meant a wedged Vz teardown stopped the daemon + // from ever exiting: drain() never returns, Run() never returns, + // and SIGTERM has to be escalated to SIGKILL by hand. The + // shutdown timeout exists precisely so shutdown is bounded; a + // teardown that ignores it defeats it. + s.stopVMBounded(job.macosVM, job.macRef(key), s.cfg.Log.With("job_id", key.JobID)) } else if job.dispatched != "" && s.cfg.LinuxDispatcher != nil { if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), job.dispatched); err != nil { s.cfg.Log.Warn("failed to destroy dispatched runner", "job_id", key.JobID, "error", err) @@ -2025,12 +2324,37 @@ func (s *Scheduler) handleHealthz(w http.ResponseWriter, r *http.Request) { draining := s.draining s.mu.Unlock() + // held_slots/slot_capacity/slots are ADDITIVE — active_jobs and + // max_concurrent keep their exact meaning for anything already scraping + // this endpoint. + // + // active_jobs counts TRACKED jobs (len(s.running)). A job that is + // provisioning, or one whose slot leaked, is not tracked, so a node with + // every slot held reported active_jobs: 0 and looked idle — that is how + // the #196 macOS outage stayed invisible for 28 hours while /healthz + // returned 200. held_slots is read straight off the semaphores and so + // counts the capacity that is actually spoken for. held == capacity with + // active_jobs 0, sustained, means a stuck provision or a leak. + // + // abandoned_macos_vms is the companion number: slots that were given BACK + // by walking away from a teardown. It is the cost side of the #196 trade — + // each one is a macOS guest still holding its RAM that only a daemon + // restart reclaims — and at the cap the macOS pool stops taking work, so + // an operator has to be able to see it without reading the log. status := map[string]any{ - "status": "ok", - "active_jobs": activeJobs, - "max_concurrent": s.cfg.MaxConcurrent, - "draining": draining, - "uptime": time.Since(s.startTime).String(), + "status": "ok", + "active_jobs": activeJobs, + "max_concurrent": s.cfg.MaxConcurrent, + "held_slots": s.HeldSlots(), + "slot_capacity": s.SlotCapacity(), + "slots": s.SlotUsage(), + "abandoned_macos_vms": s.AbandonedMacOSVMs(), + "abandoned_macos_vm_cap": s.abandonedMacVMCap(), + "draining": draining, + "uptime": time.Since(s.startTime).String(), + } + if n := s.AbandonedMacOSVMDetails(); len(n) > 0 { + status["abandoned_macos_vm_details"] = n } w.Header().Set("Content-Type", "application/json") @@ -2313,7 +2637,16 @@ func (s *Scheduler) sweepOrphanRunners() { metrics.JobsActive.Dec() v.rj.cancel() if v.rj.macosVM != nil { - v.rj.macosVM.Stop() + // THE WORST ONE (#196). sweepOrphanRunners is called + // SYNCHRONOUSLY from the cleanupTicker arm of Run's select — the + // scheduler's only event loop. A tracked macOS job nominated as an + // orphan whose Vz stop wedges (the exact failure this PR exists + // for) used to block that loop forever: no queued, in_progress or + // completed handling for ANY platform, no drain on SIGTERM, the + // daemon alive and answering /healthz with 200 and doing nothing. + // That is strictly worse than the single leaked slot the rest of + // this PR is about, and it is one call away. + s.stopVMBounded(v.rj.macosVM, v.rj.macRef(v.key), s.cfg.Log.With("runner", v.name, "job_id", v.key.JobID)) } else if v.rj.dispatched != "" && s.cfg.LinuxDispatcher != nil { if err := s.cfg.LinuxDispatcher.Destroy(context.Background(), v.rj.dispatched); err != nil { s.cfg.Log.Warn("failed to destroy orphaned dispatched runner", "runner", v.name, "error", err) diff --git a/pkg/scheduler/slots.go b/pkg/scheduler/slots.go new file mode 100644 index 0000000..23a507b --- /dev/null +++ b/pkg/scheduler/slots.go @@ -0,0 +1,499 @@ +package scheduler + +import ( + "context" + "log/slog" + "sort" + "sync" + "time" +) + +// Concurrency slots. +// +// Every dispatch path holds one slot for the whole life of the job it +// provisions: s.sem (local runners), s.linuxSem (Linux dispatch to the VM +// worker), s.macSem (macOS VMs). The buffered channel IS the accounting — +// len() is how many slots are held, cap() is the pool size — so nothing here +// keeps a parallel counter that could drift from the semaphore it describes. +// +// Three rules, all of them written after an outage: +// +// 1. A slot is released the moment the job it represents stops being demand, +// i.e. as soon as the job is untracked. NEVER "once teardown finishes": +// teardown is precisely the thing that hangs. (PR #190, local + Linux +// paths; issue #196 extended it to the macOS wait-goroutine, which held +// the slot across macVM.Stop() and a ReleaseJob call on +// context.Background().) +// +// 2. Release is idempotent and has a deferred backstop, so an error return +// added later cannot forget it and a panic cannot skip it. +// +// 3. Nothing that can block forever may sit between the acquire and the +// release on a failure path. This is the whole of issue #196: the macOS +// provisioning watchdog force-stopped the wedged VM "to reclaim the slot" +// and then blocked FOREVER waiting for the wedged wait goroutine to +// unwind, so the release it was on its way to never ran. The node held +// the sole macOS slot for 28 hours while reporting status: ok and +// active_jobs: 0 (the job had never been tracked), and every macOS job +// queued in that window aged out at GitHub's 24h timeout without running. + +// These are vars rather than consts only so tests can compress the timeline; +// nothing outside tests reassigns them. +var ( + // slotWaitLogAfter is how long a dispatch must sit blocked on a full pool + // before it says so. Short enough that a genuinely stuck pool is visible + // within a minute of the next job arriving, long enough that ordinary + // queueing behind a running job on a max_concurrent = 1 node does not + // narrate itself. + slotWaitLogAfter = 30 * time.Second + + // slotWaitLogEvery re-states a still-blocked wait on this interval. A job + // can legitimately wait hours behind a long build, so this is quiet; the + // point is that a pool which is stuck forever keeps saying so instead of + // going silent after one line. Issue #196's log had NOTHING in it for 28 + // hours — five macOS jobs blocked on the leaked slot and not one of them + // left a trace. + slotWaitLogEvery = 5 * time.Minute + + // slotLeakSuspectAfter is when a blocked wait escalates from "busy" to + // "broken". A slot that is held while the scheduler tracks NO running job + // is either a provision in flight (bounded: minutes) or a leak (forever), + // so waiting this long against a pool that nothing is tracked against is + // reported at Error. + slotLeakSuspectAfter = 15 * time.Minute +) + +// slotToken is a one-shot handle to an acquired concurrency slot. +// +// It replaces the bare `<-s.macSem` that each of handleMacOSJob's six error +// returns had to remember on its own. The type exists so that "did this path +// release?" is answered by construction (a deferred backstop) rather than by +// reading every return statement. +type slotToken struct { + sem chan struct{} + once sync.Once +} + +// release returns the slot to its pool. Safe to call any number of times from +// any goroutine: only the first call drains the semaphore, so an explicit +// early release (the one that matters — it is what lets the next job in) can +// coexist with a deferred backstop without under-draining the channel and +// handing out a slot somebody else still holds. +func (t *slotToken) release() { + if t == nil { + return + } + t.once.Do(func() { <-t.sem }) +} + +// acquireSlot takes a slot from sem, blocking until one is free or ctx is +// done. It returns nil when ctx ended first; the caller must not provision. +// +// The only behavioural difference from the raw `select { case sem <- ... }` +// it replaces is that a wait which actually BLOCKS becomes visible in the log. +// A saturated pool and a leaked pool look identical from outside the process, +// and until issue #196 both looked identical from inside it too: the daemon +// logged nothing at all while five macOS jobs piled up behind a slot that +// would never come back. +func (s *Scheduler) acquireSlot(ctx context.Context, sem chan struct{}, pool string, log *slog.Logger) *slotToken { + // Fast path: a free slot logs nothing and arms no timer, which is the + // overwhelmingly common case. (It still allocates the token — one tiny + // struct per job, against a job that is about to boot a VM.) + select { + case sem <- struct{}{}: + return &slotToken{sem: sem} + case <-ctx.Done(): + return nil + default: + } + + start := time.Now() + timer := time.NewTimer(slotWaitLogAfter) + defer timer.Stop() + for { + select { + case sem <- struct{}{}: + log.Info("acquired concurrency slot after waiting", + "pool", pool, "waited", time.Since(start).Truncate(time.Second), + "held", len(sem), "capacity", cap(sem)) + return &slotToken{sem: sem} + case <-ctx.Done(): + return nil + case <-timer.C: + // PER-POOL, not global. The first version of this compared the + // pool's held count against s.ActiveJobs(), which is + // len(s.running) across EVERY pool — so a single tracked Linux + // job running on the mac was enough to make a genuinely leaked + // macOS slot look accounted for and keep the escalation at Warn. + // On the #196 node that is not a hypothetical: the mac serves + // Linux jobs in its embedded VM continuously, so the one signal + // that would have named the outage was suppressed by unrelated + // work. + held, tracked, waited := len(sem), s.trackedInPool(pool), time.Since(start) + level, msg := slotWaitSeverity(waited, held, tracked) + log.Log(ctx, level, msg, + "pool", pool, + "held", held, + "capacity", cap(sem), + "pool_tracked_jobs", tracked, + "waited", waited.Truncate(time.Second)) + timer.Reset(slotWaitLogEvery) + } + } +} + +// slotWaitSeverity decides how loudly a blocked dispatch complains. +// +// This is the leak watchdog, and it lives here rather than in a periodic +// sweeper on purpose: a held slot is only a problem when something is waiting +// for it, and a waiting dispatch is exactly this code path. It therefore +// costs nothing when the node is idle and cannot produce a false alarm on a +// node that simply has no macOS work. +// +// held > 0 with poolTrackedJobs == 0 means the slot is charged to a job the +// scheduler is not tracking. That is legitimate for the length of a +// provision (bounded by MacOSProvisionTimeout, minutes) and illegitimate +// forever after — it is the exact signature of the 28-hour macOS outage, +// where `ephemerd status` reported active_jobs: 0 on a node whose only slot +// had been held since the previous afternoon. Pure so the escalation rule is +// testable without a real pool or a real wait. +// +// poolTrackedJobs is scoped to THE POOL BEING WAITED ON (see trackedInPool), +// never to the whole scheduler. held is a per-pool number, so comparing it to +// a global job count mixes units: on a mac, which serves Linux jobs from its +// embedded VM alongside macOS VMs, one tracked Linux job made a leaked macOS +// slot look explained and downgraded this to Warn forever. +func slotWaitSeverity(waited time.Duration, held, poolTrackedJobs int) (slog.Level, string) { + if held > 0 && poolTrackedJobs == 0 && waited >= slotLeakSuspectAfter { + return slog.LevelError, "blocked on a concurrency slot that no tracked job accounts for — suspected slot leak" + } + return slog.LevelWarn, "waiting for a free concurrency slot" +} + +// poolOf reports which dispatch pool a tracked job's slot came from. It +// mirrors the routing in handleQueued: macOS jobs hold macSem, Linux jobs +// dispatched into the VM worker hold linuxSem, everything else holds sem. +func poolOf(rj *runningJob) string { + switch { + case rj.macosVM != nil: + return "macos" + case rj.dispatched != "": + return "linux" + default: + return "local" + } +} + +// trackedInPool counts tracked running jobs whose slot came from pool. +// +// This is the denominator the leak watchdog needs: "is anything the scheduler +// knows about accounting for the slots THIS pool is holding". ActiveJobs() — +// the global count — cannot answer that on a node that runs more than one +// kind of job, and a mac runs three. +func (s *Scheduler) trackedInPool(pool string) int { + s.mu.Lock() + defer s.mu.Unlock() + n := 0 + for _, rj := range s.running { + if poolOf(rj) == pool { + n++ + } + } + return n +} + +// SlotUsage reports concurrency-slot occupancy for every dispatch pool. +// +// Held is read from the semaphore itself, so it counts slots that are charged +// to an UNTRACKED job — a provision still in flight, or a leak. That is the +// whole point: ActiveJobs() counts len(s.running) and so reported 0 on the +// node that had been wedged for 28 hours. A pool at held == capacity with +// nothing tracked against it is the signal that was missing. +func (s *Scheduler) SlotUsage() []SlotStats { + return []SlotStats{ + {Pool: "local", Held: len(s.sem), Capacity: cap(s.sem)}, + {Pool: "linux", Held: len(s.linuxSem), Capacity: cap(s.linuxSem)}, + {Pool: "macos", Held: len(s.macSem), Capacity: cap(s.macSem)}, + } +} + +// SlotStats is one dispatch pool's occupancy. +type SlotStats struct { + Pool string `json:"pool"` + Held int `json:"held"` + Capacity int `json:"capacity"` +} + +// HeldSlots is the total number of concurrency slots currently held across +// all pools, tracked or not. +func (s *Scheduler) HeldSlots() int { + total := 0 + for _, p := range s.SlotUsage() { + total += p.Held + } + return total +} + +// SlotCapacity is the total number of concurrency slots across all pools. +func (s *Scheduler) SlotCapacity() int { + total := 0 + for _, p := range s.SlotUsage() { + total += p.Capacity + } + return total +} + +// --------------------------------------------------------------------------- +// Abandoned macOS VMs. +// +// The #196 fix trades a leaked concurrency slot for a leaked goroutine: when a +// Vz stop does not return, we walk away from it and give the slot back. That +// trade is right, but it is not free, and until this registry existed it was +// also UNBOUNDED and INVISIBLE. +// +// What one abandonment actually costs. Three goroutines (the WaitForRunner +// wait, abandonMacProvision's stopVMAsync, and — before the fix below — +// the caller's redundant stopVMBounded blocking inside darwinMacOSVM's +// stopOnce), each retaining a live *darwinMacOSVM, and, far more expensively, +// a STILL-RUNNING macOS guest holding its whole configured RAM +// (MacOSVMConfig.MemoryMB — gigabytes) plus its disk clone. +// +// What used to bound it. The semaphore, by accident: leak the only slot and +// the node provisions zero further macOS VMs, so exactly one guest could ever +// be stranded. Releasing the slot on the failure path is precisely what +// removed that accidental cap, and MaxMacOSVMs no longer charges for abandoned +// VMs because they are not tracked and no longer hold a slot. Left alone, a +// mac with a repeatably wedging guest would keep provisioning until +// Virtualization.framework refused (its concurrent macOS-guest limit is low — +// about two on Apple Silicon) or the host ran out of memory. +// +// So the bound is explicit now: count them, cap them, say so loudly, and +// report the count next to held_slots. A node that has given up on more than a +// couple of guests is broken in a way a restart fixes and no further +// provisioning will; refusing new macOS work there turns unbounded RAM growth +// into a legible, bounded failure that `ephemerd status` names. +// --------------------------------------------------------------------------- + +// defaultMacAbandonedVMCap is how many macOS VMs may be abandoned-but-not- +// confirmed-dead before the macOS pool stops taking work. +// +// Deliberately tiny. Each entry is a live guest holding multiple gigabytes, +// and Virtualization.framework itself only permits about two concurrent macOS +// guests on Apple Silicon — so the useful budget is "one transient hiccup", +// not "keep trying". +const defaultMacAbandonedVMCap = 2 + +// macVMRef identifies one macOS VM instance for the abandoned registry and for +// the log lines about it. VMID is the PER-ATTEMPT instance id (see +// macVMInstanceID), not the job id: two provisioning attempts for the same job +// are two different VMs and must be countable separately. +type macVMRef struct { + JobID int64 + VMID string +} + +// AbandonedMacVM is one macOS VM whose teardown was given up on, as reported +// by status/healthz. +type AbandonedMacVM struct { + JobID int64 `json:"job_id"` + VMID string `json:"vm_id"` + Reason string `json:"reason"` + HeldFor string `json:"held_for"` +} + +type abandonedEntry struct { + ref macVMRef + reason string + since time.Time +} + +// abandonedMacVMs is the registry. Entries are added when a stop is walked +// away from and removed if that stop ever returns — which does happen: a Vz +// stop can be slow rather than wedged, and a node that recovers must be +// allowed to serve macOS jobs again without a restart. +type abandonedMacVMs struct { + mu sync.Mutex + next uint64 + live map[uint64]abandonedEntry +} + +func (a *abandonedMacVMs) charge(ref macVMRef, reason string) (uint64, int) { + a.mu.Lock() + defer a.mu.Unlock() + if a.live == nil { + a.live = make(map[uint64]abandonedEntry) + } + a.next++ + id := a.next + a.live[id] = abandonedEntry{ref: ref, reason: reason, since: time.Now()} + return id, len(a.live) +} + +// discharge removes an entry and reports the remaining count and how long the +// VM was outstanding. +func (a *abandonedMacVMs) discharge(id uint64) (time.Duration, int) { + a.mu.Lock() + defer a.mu.Unlock() + e, ok := a.live[id] + delete(a.live, id) + remaining := len(a.live) + if !ok { + return 0, remaining + } + return time.Since(e.since), remaining +} + +func (a *abandonedMacVMs) count() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.live) +} + +func (a *abandonedMacVMs) snapshot() []AbandonedMacVM { + a.mu.Lock() + defer a.mu.Unlock() + out := make([]AbandonedMacVM, 0, len(a.live)) + for _, e := range a.live { + out = append(out, AbandonedMacVM{ + JobID: e.ref.JobID, + VMID: e.ref.VMID, + Reason: e.reason, + HeldFor: time.Since(e.since).Truncate(time.Second).String(), + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].VMID < out[j].VMID }) + return out +} + +// AbandonedMacOSVMs is how many macOS VMs this daemon has given up on and not +// yet seen die. Non-zero means guests are holding host RAM that nothing in +// this process can reclaim; at or above the cap the macOS pool is cordoned. +func (s *Scheduler) AbandonedMacOSVMs() int { return s.macAbandoned.count() } + +// AbandonedMacOSVMDetails lists them, newest first by instance id. +func (s *Scheduler) AbandonedMacOSVMDetails() []AbandonedMacVM { return s.macAbandoned.snapshot() } + +// abandonedMacVMCap is the effective cap; the field exists so tests can set it +// to 1 instead of staging two wedged guests. +func (s *Scheduler) abandonedMacVMCap() int { + if s.macAbandonedCap > 0 { + return s.macAbandonedCap + } + return defaultMacAbandonedVMCap +} + +// macPoolCordoned reports whether the macOS pool has given up on so many VMs +// that it must stop provisioning, and how many. +// +// Checked twice on the dispatch path for the same reason the #154 cordon is: +// once before blocking on the semaphore, and once after — a dispatch parked on +// a full pool can sit there for the entire life of the job ahead of it, and +// the node can break underneath it in that window. +func (s *Scheduler) macPoolCordoned() (int, int, bool) { + n, limit := s.macAbandoned.count(), s.abandonedMacVMCap() + return n, limit, n >= limit +} + +// abandonVM charges a walked-away-from teardown to the registry and arranges +// for it to be discharged if the stop ever returns. +// +// done is the channel stopVMAsync returns. If it is already closed the stop +// landed in the race between the caller's deadline and this call, and nothing +// is abandoned — that distinction matters, because charging a VM that is in +// fact dead would cordon a healthy node. +func (s *Scheduler) abandonVM(done <-chan struct{}, ref macVMRef, reason string, log *slog.Logger) { + select { + case <-done: + return // teardown finished after all; nothing was abandoned + default: + } + + id, n := s.macAbandoned.charge(ref, reason) + limit := s.abandonedMacVMCap() + log.Error("abandoned a macOS VM whose teardown never returned; its guest is still holding host RAM", + "vm", ref.VMID, + "job_id", ref.JobID, + "reason", reason, + "abandoned_macos_vms", n, + "cap", limit, + "detail", "walking away is what keeps the concurrency slot from leaking (#196), but the guest is not gone — only a daemon restart reclaims it") + if n >= limit { + log.Error("macOS pool cordoned: too many macOS VMs abandoned without confirmed death — refusing further macOS provisioning on this node", + "abandoned_macos_vms", n, + "cap", limit, + "detail", "each abandoned guest holds its full configured RAM and Virtualization.framework allows only a couple of concurrent macOS guests; provisioning past this point buys nothing and can OOM the host. Restart ephemerd to clear them.") + } + + // Discharge if it ever finishes. A Vz stop can be merely slow, and a node + // that recovers on its own must not need a restart to serve macOS again. + go func() { + <-done + held, remaining := s.macAbandoned.discharge(id) + log.Warn("a previously abandoned macOS VM finally finished stopping", + "vm", ref.VMID, + "job_id", ref.JobID, + "held_for", held.Truncate(time.Second), + "abandoned_macos_vms", remaining) + }() +} + +// provisionUnwindGrace and teardownGrace are the effective bounds on macOS VM +// teardown. They read from the scheduler so tests can compress them, and fall +// back to the package defaults for a zero-valued Scheduler. +func (s *Scheduler) provisionUnwindGrace() time.Duration { + if s.macUnwindGrace > 0 { + return s.macUnwindGrace + } + return macProvisionUnwindGrace +} + +func (s *Scheduler) teardownGrace() time.Duration { + if s.macStopGrace > 0 { + return s.macStopGrace + } + return macTeardownGrace +} + +// awaitUnwind waits for an abandoned provisioning attempt's teardown +// (stopped) and its wait goroutine (unwound) to finish, but only for grace. +// It reports whether both arrived in time. +// +// This bound is the issue #196 fix in one function. The previous code did the +// moral equivalent of `<-resCh` with no deadline, on the reasoning that +// force-stopping the VM drops the guest connection and therefore unblocks the +// reachability wait. On metal that reasoning failed: the VM's own stop path +// timed out ("macOS VM did not stop gracefully, forcing") and the guest wait +// never returned, so the dispatch goroutine parked there permanently — still +// holding the macOS slot it had just logged that it was reclaiming. +// +// A wedged guest wait is already a leaked goroutine and nothing in this +// process can kill it. Blocking on it converts that leaked goroutine into a +// leaked concurrency slot, which is the difference between a wasted job and a +// dead node. So: give teardown a bounded chance to be tidy, then walk away. +func awaitUnwind(stopped, unwound <-chan struct{}, grace time.Duration) bool { + timer := time.NewTimer(grace) + defer timer.Stop() + for stopped != nil || unwound != nil { + // Prefer an already-finished signal over the deadline: with both + // ready, a plain select picks at random and would report a clean + // unwind as a timeout. + select { + case <-stopped: + stopped = nil + continue + case <-unwound: + unwound = nil + continue + default: + } + select { + case <-stopped: + stopped = nil + case <-unwound: + unwound = nil + case <-timer.C: + return false + } + } + return true +} diff --git a/pkg/scheduler/slots_test.go b/pkg/scheduler/slots_test.go new file mode 100644 index 0000000..3c21f46 --- /dev/null +++ b/pkg/scheduler/slots_test.go @@ -0,0 +1,1303 @@ +package scheduler + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + apiv1 "github.com/ephpm/ephemerd/api/v1" + "github.com/ephpm/ephemerd/pkg/providers" + "github.com/ephpm/ephemerd/pkg/vm" +) + +// wedgedMacVM is the macOS VM from issue #196 and the reason +// blockingMacVM (macos_provision_test.go) was not enough: its WaitForRunner +// never returns AT ALL, not even when the VM is stopped, and its Stop() never +// returns either. +// +// That is what the fleet's mac actually did. The daemon logged +// +// ERROR macOS VM stuck in provisioning past deadline; force-stopping VM to reclaim the slot +// WARN macOS VM did not stop gracefully, forcing +// INFO macOS VM destroyed +// +// and then nothing for 28 hours: the guest wait never unwound, so the +// watchdog's `<-resCh` never returned, so the "reclaim the slot" it had just +// announced never happened. Every assertion below is about the dispatch path +// surviving a VM it cannot kill. +type wedgedMacVM struct { + stopCalls atomic.Int32 + // stopReturns, when closed, lets Stop() return. Left open to model a + // teardown wedged inside Virtualization.framework. + stopReturns chan struct{} +} + +// newWedgedMacVM registers cleanup that lets the wedged Stop() goroutines +// finish, so the test binary does not accumulate them across subtests. +func newWedgedMacVM(t *testing.T) *wedgedMacVM { + t.Helper() + m := &wedgedMacVM{stopReturns: make(chan struct{})} + t.Cleanup(func() { close(m.stopReturns) }) + return m +} + +func (m *wedgedMacVM) WriteJITConfig(string) error { return nil } +func (m *wedgedMacVM) Start(_ context.Context) error { return nil } +func (m *wedgedMacVM) RunnerAddress() string { return "" } +func (m *wedgedMacVM) Wait(context.Context) (int, error) { + select {} //nolint:staticcheck // models a VM whose wait never returns +} + +// WaitForRunner never returns — it ignores ctx exactly as a blocked +// golang.org/x/crypto/ssh session call does. +func (m *wedgedMacVM) WaitForRunner(context.Context) (string, error) { + select {} //nolint:staticcheck // models the wedged guest wait +} + +func (m *wedgedMacVM) Stop() { + m.stopCalls.Add(1) + <-m.stopReturns +} + +// newSlotTestScheduler builds a scheduler with a single macOS slot and +// millisecond teardown bounds, so the 30-second production graces do not turn +// every assertion into a 30-second test. +func newSlotTestScheduler(t *testing.T, prov providers.Provider, mac vm.MacOSVM) *Scheduler { + t.Helper() + s := New(Config{ + Providers: []providers.Provider{prov}, + MacOSVMConfig: &vm.MacOSVMConfig{}, + MaxMacOSVMs: 1, // the incident's max_concurrent = 1 + MacOSProvisionTimeout: 50 * time.Millisecond, + Log: quietLogger(), + }) + s.macUnwindGrace = 50 * time.Millisecond + s.macStopGrace = 50 * time.Millisecond + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { return mac, nil } + return s +} + +func macEvent(prov providers.Provider, jobID int64) providers.JobEvent { + return providers.JobEvent{ + Provider: prov, + Action: "queued", + Repo: "repo", + JobID: jobID, + Labels: []string{"self-hosted", "macos"}, + } +} + +// runMacDispatch runs handleMacOSJob and fails (rather than hanging) if it has +// not returned by deadline. A regression here must be a FAILURE, not a test +// that sits there until the package timeout. +func runMacDispatch(t *testing.T, s *Scheduler, event providers.JobEvent, deadline time.Duration) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + s.handleMacOSJob(context.Background(), event) + }() + select { + case <-done: + case <-time.After(deadline): + t.Fatalf("handleMacOSJob did not return within %s: the dispatch path is blocked on VM teardown while holding a slot", deadline) + } +} + +// assertSlotFree asserts the pool has a slot available right now. +func assertSlotFree(t *testing.T, sem chan struct{}, what string) { + t.Helper() + select { + case sem <- struct{}{}: + <-sem + default: + t.Fatalf("%s: no free slot (held=%d capacity=%d)", what, len(sem), cap(sem)) + } +} + +// TestHandleMacOSJob_UnkillableVMStillFreesSlot is the issue #196 regression +// test. The VM cannot be killed and its reachability wait never unwinds; the +// dispatch must still return and give the macOS slot back, because otherwise +// the node silently serves zero macOS jobs forever while reporting healthy. +// +// Without the fix this test FAILS on the deadline in runMacDispatch: +// waitForMacRunnerBounded parks on `<-resCh` after force-stopping the VM and +// handleMacOSJob never reaches its release. +func TestHandleMacOSJob_UnkillableVMStillFreesSlot(t *testing.T) { + prov := newMockProvider("mac-wedged") + defer func() { _ = prov.Stop(context.Background()) }() + + macVM := newWedgedMacVM(t) + s := newSlotTestScheduler(t, prov, macVM) + + runMacDispatch(t, s, macEvent(prov, 100817345293), 5*time.Second) + + if macVM.stopCalls.Load() == 0 { + t.Error("expected the wedged macOS VM to be force-stopped on the provisioning deadline (#178 behaviour must not regress)") + } + assertSlotFree(t, s.macSem, "after a provisioning deadline on an unkillable VM") + if held := len(s.macSem); held != 0 { + t.Errorf("macSem holds %d slots after the dispatch returned, want 0", held) + } + + // The next macOS job must be able to run. This is the assertion the + // production node failed for 28 hours. + second := newWedgedMacVM(t) + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { return second, nil } + runMacDispatch(t, s, macEvent(prov, 100817345294), 5*time.Second) + if second.stopCalls.Load() == 0 { + t.Error("the job queued behind the wedged one never reached provisioning: the slot was never actually reclaimed") + } + + // Nothing may be left tracked, and the ghost JIT runners must be gone. + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d after two abandoned provisions, want 0", n) + } + prov.mu.Lock() + releases := len(prov.releases) + prov.mu.Unlock() + if releases != 2 { + t.Errorf("ReleaseJob called %d times, want 2 (one ghost runner per abandoned provision)", releases) + } +} + +// TestHandleMacOSJob_ReleasesExactlyOncePerOutcome walks the distinct exits +// from the macOS dispatch and asserts each one leaves the pool exactly as it +// found it. "Exactly once" matters in both directions: a missed release +// starves the node, a double release hands out a slot somebody still holds. +func TestHandleMacOSJob_ReleasesExactlyOncePerOutcome(t *testing.T) { + prov := newMockProvider("mac-outcomes") + defer func() { _ = prov.Stop(context.Background()) }() + + t.Run("provisioning deadline", func(t *testing.T) { + s := newSlotTestScheduler(t, prov, newWedgedMacVM(t)) + runMacDispatch(t, s, macEvent(prov, 1), 5*time.Second) + if held := len(s.macSem); held != 0 { + t.Fatalf("held=%d after the deadline path, want 0", held) + } + // Capacity must be intact, i.e. the slot came back once and not twice. + assertCapacityIntact(t, s.macSem) + }) + + t.Run("vm create failure", func(t *testing.T) { + s := newSlotTestScheduler(t, prov, nil) + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { + return nil, errFakeCreate + } + runMacDispatch(t, s, macEvent(prov, 2), 5*time.Second) + if held := len(s.macSem); held != 0 { + t.Fatalf("held=%d after a VM create failure, want 0", held) + } + assertCapacityIntact(t, s.macSem) + }) + + t.Run("context cancelled before a slot is free", func(t *testing.T) { + s := newSlotTestScheduler(t, prov, newWedgedMacVM(t)) + // Fill the sole slot so the dispatch has to wait for it. + s.macSem <- struct{}{} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + s.handleMacOSJob(ctx, macEvent(prov, 3)) + }() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("handleMacOSJob did not return after its context was cancelled") + } + // It must not have taken a second slot, and must not have released + // the one it never acquired. + if held := len(s.macSem); held != 1 { + t.Fatalf("held=%d after a cancelled dispatch, want 1 (the pre-existing holder)", held) + } + <-s.macSem + assertCapacityIntact(t, s.macSem) + }) + + t.Run("successful job", func(t *testing.T) { + var stops atomic.Int32 + s := newSlotTestScheduler(t, prov, &fastMacVM{ip: "192.168.64.5", stops: &stops}) + runMacDispatch(t, s, macEvent(prov, 4), 5*time.Second) + // The wait-goroutine owns the slot now; fastMacVM.Wait returns + // immediately, so it comes back promptly. + waitForSlotFree(t, s.macSem, 5*time.Second) + assertCapacityIntact(t, s.macSem) + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d after the job finished, want 0", n) + } + }) +} + +var errFakeCreate = errors.New("vz: no macOS image installed") + +// assertCapacityIntact proves the pool can still be filled to capacity and no +// further — i.e. releases and acquires balanced exactly. +func assertCapacityIntact(t *testing.T, sem chan struct{}) { + t.Helper() + n := 0 + for i := 0; i < cap(sem); i++ { + select { + case sem <- struct{}{}: + n++ + default: + } + } + if n != cap(sem) { + t.Errorf("pool accepted %d of %d slots; a release was missed", n, cap(sem)) + } + select { + case sem <- struct{}{}: + t.Error("pool accepted more than its capacity; a slot was released twice") + default: + } + for i := 0; i < n; i++ { + <-sem + } +} + +func waitForSlotFree(t *testing.T, sem chan struct{}, within time.Duration) { + t.Helper() + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if len(sem) == 0 { + return + } + time.Sleep(2 * time.Millisecond) + } + t.Fatalf("slot was not released within %s (held=%d)", within, len(sem)) +} + +// TestSlotTokenReleaseIsIdempotent covers the property the whole fix leans on: +// the explicit release that lets the next job in, and the deferred backstop +// that catches a forgotten path, are the same release. +func TestSlotTokenReleaseIsIdempotent(t *testing.T) { + sem := make(chan struct{}, 2) + sem <- struct{}{} + tok := &slotToken{sem: sem} + + tok.release() + if len(sem) != 0 { + t.Fatalf("held=%d after release, want 0", len(sem)) + } + + // A second holder takes the freed slot; further releases of the spent + // token must not steal it. + sem <- struct{}{} + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + tok.release() + }() + } + wg.Wait() + if len(sem) != 1 { + t.Errorf("held=%d after repeated releases of a spent token, want 1", len(sem)) + } + + // A nil token is a no-op, so a path that never acquired can still defer. + var nilTok *slotToken + nilTok.release() +} + +// TestAcquireSlotBlocksAndLogs checks the observability gap that let #196 hide: +// a dispatch parked on a full pool used to log nothing whatsoever. +func TestAcquireSlotBlocksAndLogs(t *testing.T) { + after, every, leak := slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter + defer func() { slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter = after, every, leak }() + slotWaitLogAfter = 10 * time.Millisecond + slotWaitLogEvery = 10 * time.Millisecond + slotLeakSuspectAfter = time.Hour // stay at Warn for this test + + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + s.macSem <- struct{}{} // pool full + + got := make(chan *slotToken, 1) + go func() { got <- s.acquireSlot(context.Background(), s.macSem, "macos", log) }() + + // Give the waiter time to block and log at least once, then free the slot. + time.Sleep(60 * time.Millisecond) + <-s.macSem + + select { + case tok := <-got: + if tok == nil { + t.Fatal("acquireSlot returned nil despite a slot becoming free") + } + tok.release() + case <-time.After(5 * time.Second): + t.Fatal("acquireSlot never returned after the slot was freed") + } + + out := buf.String() + if !strings.Contains(out, "waiting for a free concurrency slot") { + t.Errorf("blocked acquire logged nothing about waiting; got:\n%s", out) + } + if !strings.Contains(out, "pool=macos") { + t.Errorf("wait log did not name the pool; got:\n%s", out) + } + if !strings.Contains(out, "capacity=1") { + t.Errorf("wait log did not report the pool capacity; got:\n%s", out) + } +} + +// TestAcquireSlotFastPathIsSilent: the common case must not narrate itself. +func TestAcquireSlotFastPathIsSilent(t *testing.T) { + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + tok := s.acquireSlot(context.Background(), s.macSem, "macos", log) + if tok == nil { + t.Fatal("acquireSlot returned nil on an empty pool") + } + tok.release() + if buf.Len() != 0 { + t.Errorf("uncontended acquire logged %q, want silence", buf.String()) + } +} + +// TestAcquireSlotHonoursContext: a cancelled dispatch takes no slot. +func TestAcquireSlotHonoursContext(t *testing.T) { + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + s.macSem <- struct{}{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if tok := s.acquireSlot(ctx, s.macSem, "macos", quietLogger()); tok != nil { + t.Error("acquireSlot handed out a slot under a cancelled context") + } + if len(s.macSem) != 1 { + t.Errorf("held=%d, want 1 (the cancelled waiter must not have taken one)", len(s.macSem)) + } +} + +// TestSlotWaitSeverity pins the leak watchdog's escalation rule: held slots +// with nothing tracked, waited on long enough, is an Error — that combination +// is exactly what `ephemerd status` showed on the wedged mac. +func TestSlotWaitSeverity(t *testing.T) { + tests := []struct { + name string + waited time.Duration + held int + tracked int + wantLevel slog.Level + wantSuspect bool + }{ + {"this pool has a tracked job, short wait", time.Minute, 1, 1, slog.LevelWarn, false}, + // The third column is jobs tracked IN THIS POOL, not on the node. + // It used to be s.ActiveJobs() (every pool), which meant a Linux job + // on the mac explained away a leaked macOS slot; see + // TestSlotLeakEscalationIsScopedToThePool. + {"this pool has a tracked job, long wait", 2 * time.Hour, 1, 1, slog.LevelWarn, false}, + {"untracked holder, short wait", time.Minute, 1, 0, slog.LevelWarn, false}, + {"untracked holder, long wait", slotLeakSuspectAfter, 1, 0, slog.LevelError, true}, + {"nothing held, long wait", 2 * time.Hour, 0, 0, slog.LevelWarn, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + level, msg := slotWaitSeverity(tc.waited, tc.held, tc.tracked) + if level != tc.wantLevel { + t.Errorf("level = %v, want %v", level, tc.wantLevel) + } + if suspect := strings.Contains(msg, "suspected slot leak"); suspect != tc.wantSuspect { + t.Errorf("message %q: leak-suspected = %v, want %v", msg, suspect, tc.wantSuspect) + } + }) + } +} + +// TestAwaitUnwind covers the bound that replaced the unbounded `<-resCh`. +func TestAwaitUnwind(t *testing.T) { + closed := func() chan struct{} { + c := make(chan struct{}) + close(c) + return c + } + + t.Run("both already finished", func(t *testing.T) { + if !awaitUnwind(closed(), closed(), time.Millisecond) { + t.Error("awaitUnwind reported a timeout when both signals were already done") + } + }) + + t.Run("both finish in time", func(t *testing.T) { + stopped, unwound := make(chan struct{}), make(chan struct{}) + go func() { time.Sleep(5 * time.Millisecond); close(stopped) }() + go func() { time.Sleep(10 * time.Millisecond); close(unwound) }() + if !awaitUnwind(stopped, unwound, 5*time.Second) { + t.Error("awaitUnwind reported a timeout for a teardown that completed") + } + }) + + t.Run("wait never unwinds", func(t *testing.T) { + start := time.Now() + if awaitUnwind(closed(), make(chan struct{}), 20*time.Millisecond) { + t.Error("awaitUnwind claimed a clean unwind for a wait that never returned") + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("awaitUnwind took %s to give up; it must be bounded by grace", elapsed) + } + }) + + t.Run("stop never returns", func(t *testing.T) { + if awaitUnwind(make(chan struct{}), closed(), 20*time.Millisecond) { + t.Error("awaitUnwind claimed a clean unwind while Stop was still wedged") + } + }) +} + +// TestStopVMBoundedDoesNotHangOnAWedgedStop: teardown is best-effort, +// capacity is not. +func TestStopVMBoundedDoesNotHangOnAWedgedStop(t *testing.T) { + macVM := newWedgedMacVM(t) + s := New(Config{MaxMacOSVMs: 1, Log: quietLogger()}) + s.macStopGrace = 20 * time.Millisecond + done := make(chan struct{}) + go func() { + defer close(done) + s.stopVMBounded(macVM, macVMRef{JobID: 1, VMID: "1-wedged"}, quietLogger()) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("stopVMBounded blocked on a Stop() that never returns") + } + if macVM.stopCalls.Load() != 1 { + t.Errorf("Stop called %d times, want 1", macVM.stopCalls.Load()) + } + // The VM it walked away from must be charged, or nothing bounds how many + // of them a node can strand. + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d after walking away from a wedged stop, want 1", n) + } +} + +// TestSlotUsageReportsHeldSlots is the reporting gap from the issue: a node +// with every slot held must not look idle. +func TestSlotUsageReportsHeldSlots(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + + if got := s.HeldSlots(); got != 0 { + t.Errorf("HeldSlots() = %d on a fresh scheduler, want 0", got) + } + if got := s.SlotCapacity(); got != 5 { // local 2 + linux 2 + macos 1 + t.Errorf("SlotCapacity() = %d, want 5", got) + } + + // Hold the macOS slot without tracking a job — the #196 state exactly. + s.macSem <- struct{}{} + defer func() { <-s.macSem }() + + if got := s.ActiveJobs(); got != 0 { + t.Fatalf("ActiveJobs() = %d, want 0 (an untracked holder is the whole point)", got) + } + if got := s.HeldSlots(); got != 1 { + t.Errorf("HeldSlots() = %d while the macOS slot is held, want 1", got) + } + + byPool := map[string]SlotStats{} + for _, p := range s.SlotUsage() { + byPool[p.Pool] = p + } + if mac := byPool["macos"]; mac.Held != 1 || mac.Capacity != 1 { + t.Errorf("macos pool = %+v, want held 1 capacity 1", mac) + } + if local := byPool["local"]; local.Held != 0 || local.Capacity != 2 { + t.Errorf("local pool = %+v, want held 0 capacity 2", local) + } +} + +// TestStatusRPCExposesHeldSlots is the same check for the control socket, +// which is what `ephemerd status` and mayfly actually read. +func TestStatusRPCExposesHeldSlots(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + cs := &controlServer{sched: s, log: quietLogger()} + + s.macSem <- struct{}{} + defer func() { <-s.macSem }() + + resp, err := cs.Status(context.Background(), &apiv1.StatusRequest{}) + if err != nil { + t.Fatalf("Status() error: %v", err) + } + // The pre-existing contract is unchanged. + if resp.Status != "ok" || resp.ActiveJobs != 0 || resp.MaxConcurrent != 2 { + t.Errorf("existing Status fields changed: %+v", resp) + } + if resp.HeldSlots != 1 { + t.Errorf("HeldSlots = %d, want 1 (the untracked macOS holder)", resp.HeldSlots) + } + if resp.SlotCapacity != 5 { + t.Errorf("SlotCapacity = %d, want 5", resp.SlotCapacity) + } + var sawMac bool + for _, p := range resp.SlotPools { + if p.Pool == "macos" { + sawMac = true + if p.Held != 1 || p.Capacity != 1 { + t.Errorf("macos pool = %+v, want held 1 capacity 1", p) + } + } + } + if !sawMac { + t.Error("Status did not break the macos pool out in SlotPools") + } +} + +// TestHealthzExposesHeldSlots checks the endpoint an operator (and the fleet +// monitor) actually reads. active_jobs keeps its old meaning; held_slots is +// what would have shown the wedged mac. +func TestHealthzExposesHeldSlots(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + s.macSem <- struct{}{} + defer func() { <-s.macSem }() + + rec := httptest.NewRecorder() + s.handleHealthz(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("healthz status = %d, want 200", rec.Code) + } + + var body struct { + Status string `json:"status"` + ActiveJobs int `json:"active_jobs"` + MaxConcurrent int `json:"max_concurrent"` + HeldSlots int `json:"held_slots"` + SlotCapacity int `json:"slot_capacity"` + Slots []SlotStats `json:"slots"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decoding healthz body %q: %v", rec.Body.String(), err) + } + // Backwards compatibility: the pre-existing fields must be untouched. + if body.Status != "ok" || body.ActiveJobs != 0 || body.MaxConcurrent != 2 { + t.Errorf("existing healthz fields changed: %+v", body) + } + if body.HeldSlots != 1 { + t.Errorf("held_slots = %d, want 1", body.HeldSlots) + } + if body.SlotCapacity != 5 { + t.Errorf("slot_capacity = %d, want 5", body.SlotCapacity) + } + var sawMac bool + for _, p := range body.Slots { + if p.Pool == "macos" { + sawMac = true + if p.Held != 1 || p.Capacity != 1 { + t.Errorf("macos pool in healthz = %+v, want held 1 capacity 1", p) + } + } + } + if !sawMac { + t.Error("healthz slots did not break out the macos pool") + } +} + +// --------------------------------------------------------------------------- +// MAJOR 1: no macOS Stop() may block its caller — least of all the event loop. +// --------------------------------------------------------------------------- + +// seedMacOSRunner files a tracked macOS job plus its runner-ledger entry, the +// state the orphan sweep, handleCompleted and destroyAll all operate on. +func seedMacOSRunner(s *Scheduler, prov providers.Provider, jobID int64, name string, dispatchedAt time.Time, macVM vm.MacOSVM) jobKey { + key := jobKey{Provider: prov.Name(), JobID: jobID} + _, cancel := context.WithCancel(context.Background()) + s.mu.Lock() + s.running[key] = &runningJob{ + provider: prov, + claim: &providers.Claim{RunnerID: jobID * 10, RunnerName: name, Repo: "myrepo"}, + repo: "myrepo", + cancel: cancel, + macosVM: macVM, + macVMID: fmt.Sprintf("%d-seeded", jobID), + startedAt: dispatchedAt, + } + s.runners[name] = &runnerBinding{ + intentKey: key, + dispatchedAt: dispatchedAt, + observable: true, + labelSet: labelSetKey([]string{"self-hosted", "macos"}), + } + s.mu.Unlock() + return key +} + +// newTeardownTestScheduler builds a scheduler whose macOS teardown bounds are +// milliseconds, for the paths that tear a TRACKED macOS job down. +func newTeardownTestScheduler(t *testing.T, prov providers.Provider, sweep bool) *Scheduler { + t.Helper() + s := New(Config{ + Providers: []providers.Provider{prov}, + MacOSVMConfig: &vm.MacOSVMConfig{}, + MaxMacOSVMs: 1, + ShutdownTimeout: 50 * time.Millisecond, + OrphanSweep: OrphanSweepConfig{Enabled: sweep, Grace: 10 * time.Minute}, + Log: quietLogger(), + }) + s.webhookMode = true + s.busyProbe = idleProbe + s.macStopGrace = 50 * time.Millisecond + s.macUnwindGrace = 50 * time.Millisecond + return s +} + +// mustReturnWithin fails (rather than hangs) if fn has not returned by d. +func mustReturnWithin(t *testing.T, d time.Duration, what string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + fn() + }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("%s did not return within %s: a macOS VM Stop() is blocking its caller", what, d) + } +} + +// TestSweepOrphanRunners_WedgedMacOSVMCannotBlockTheEventLoop is the worst of +// the three unbounded Stop() sites the #196 fix left behind. +// +// sweepOrphanRunners is called SYNCHRONOUSLY from the cleanupTicker arm of +// Run's select — the scheduler's one and only event loop. A tracked macOS job +// nominated as an orphan whose Vz stop wedges (exactly the failure that +// produced the 28-hour outage) parked that loop forever: no queued, +// in_progress or completed handling for any platform, and no drain on SIGTERM. +// That is strictly worse than the leaked slot the rest of the PR fixes. +// +// Without the fix this test FAILS on mustReturnWithin: the sweep sits inside +// wedgedMacVM.Stop() until the test binary's own timeout. +func TestSweepOrphanRunners_WedgedMacOSVMCannotBlockTheEventLoop(t *testing.T) { + prov := &reportingProvider{claimCountingProvider: newClaimCountingProvider("mac-sweep")} + s := newTeardownTestScheduler(t, prov, true) + + macVM := newWedgedMacVM(t) + key := seedMacOSRunner(s, prov, 4242, "mac-r-1", time.Now().Add(-30*time.Minute), macVM) + + mustReturnWithin(t, 5*time.Second, "sweepOrphanRunners", s.sweepOrphanRunners) + + if macVM.stopCalls.Load() == 0 { + t.Error("the orphaned macOS VM was never stopped; the sweep must still tear it down") + } + s.mu.Lock() + _, stillRunning := s.running[key] + s.mu.Unlock() + if stillRunning { + t.Error("orphaned macOS job is still tracked after the sweep") + } + // A stop it walked away from has to be counted, or nothing bounds them. + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d after the sweep abandoned a wedged stop, want 1", n) + } +} + +// TestHandleCompleted_WedgedMacOSVMCannotBlockTheHandler: the completed-event +// handler also ran a bare Stop(). A wedge there parks the handler forever, and +// with it the deferred sweepOrphanRunners that reconciles the ledger after +// every completion. +// +// Without the fix this FAILS on mustReturnWithin. +func TestHandleCompleted_WedgedMacOSVMCannotBlockTheHandler(t *testing.T) { + prov := &reportingProvider{claimCountingProvider: newClaimCountingProvider("mac-completed")} + s := newTeardownTestScheduler(t, prov, false) + + macVM := newWedgedMacVM(t) + key := seedMacOSRunner(s, prov, 5150, "mac-r-2", time.Now(), macVM) + + event := providers.JobEvent{ + Provider: prov, + Action: "completed", + Repo: "myrepo", + JobID: 5150, + RunnerName: "mac-r-2", + Conclusion: "success", + } + mustReturnWithin(t, 5*time.Second, "handleCompleted", func() { + s.handleCompleted(context.Background(), event) + }) + + if macVM.stopCalls.Load() == 0 { + t.Error("the completed job's macOS VM was never stopped") + } + s.mu.Lock() + _, stillRunning := s.running[key] + s.mu.Unlock() + if stillRunning { + t.Error("completed macOS job is still tracked") + } +} + +// TestDrain_WedgedMacOSVMCannotBlockShutdown: destroyAll is the force-kill arm +// of drain(), reached only after ShutdownTimeout has already expired. A bare +// Stop() there meant a wedged guest kept the process alive indefinitely — +// drain never returns, Run never returns, and SIGTERM has to be escalated to +// SIGKILL by hand. The shutdown timeout exists to bound shutdown; a teardown +// that ignores it defeats the entire mechanism. +// +// Without the fix this FAILS on mustReturnWithin. +func TestDrain_WedgedMacOSVMCannotBlockShutdown(t *testing.T) { + prov := &reportingProvider{claimCountingProvider: newClaimCountingProvider("mac-drain")} + s := newTeardownTestScheduler(t, prov, false) + + macVM := newWedgedMacVM(t) + seedMacOSRunner(s, prov, 6006, "mac-r-3", time.Now(), macVM) + + mustReturnWithin(t, 10*time.Second, "drain", s.drain) + + if macVM.stopCalls.Load() == 0 { + t.Error("the running macOS VM was never stopped on shutdown") + } + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d after drain force-killed everything, want 0", n) + } +} + +// --------------------------------------------------------------------------- +// MAJOR 2: a re-dispatch of the same job must not share state with an +// abandoned attempt at that job. +// --------------------------------------------------------------------------- + +// pathedMacVM owns real files, named the way darwinMacOSVM names them: +// +// /vm/macos/jobs/ the virtio-fs share (holds .jit_config) +// /vm/macos/jobs/.img the APFS disk clone +// +// and its Stop() deletes both, exactly as darwinMacOSVM.stop does +// (os.Remove(clonePath) + os.RemoveAll(jobDir)). The point of the fake is that +// the paths come from the id string the SCHEDULER chooses, so the test can +// observe whether two attempts at the same job collide on disk. +type pathedMacVM struct { + id string + jobDir string + clone string + + wedge bool // WaitForRunner never returns (the #196 guest) + gate chan struct{} // Stop blocks here before deleting; nil to not block + held chan struct{} // closed once Stop has finished its deletions + done chan struct{} // Wait blocks here; nil returns immediately + + stops atomic.Int32 + stopOnce sync.Once +} + +func newPathedMacVM(t *testing.T, dataDir, id string, wedge bool) *pathedMacVM { + t.Helper() + jobsDir := filepath.Join(dataDir, "vm", "macos", "jobs") + return &pathedMacVM{ + id: id, + jobDir: filepath.Join(jobsDir, id), + clone: filepath.Join(jobsDir, id+".img"), + wedge: wedge, + held: make(chan struct{}), + } +} + +func (m *pathedMacVM) jitPath() string { return filepath.Join(m.jobDir, ".jit_config") } + +func (m *pathedMacVM) WriteJITConfig(cfg string) error { + if err := os.MkdirAll(m.jobDir, 0o755); err != nil { + return err + } + return os.WriteFile(m.jitPath(), []byte(cfg), 0o600) +} + +func (m *pathedMacVM) Start(context.Context) error { + if err := os.MkdirAll(filepath.Dir(m.clone), 0o755); err != nil { + return err + } + return os.WriteFile(m.clone, []byte("disk-"+m.id), 0o600) +} + +func (m *pathedMacVM) RunnerAddress() string { return "192.168.64.9" } + +func (m *pathedMacVM) WaitForRunner(context.Context) (string, error) { + if m.wedge { + select {} //nolint:staticcheck // models the wedged guest wait + } + return "192.168.64.9", nil +} + +func (m *pathedMacVM) Wait(context.Context) (int, error) { + if m.done != nil { + <-m.done + } + return 0, nil +} + +// Stop mirrors darwinMacOSVM.Stop: a sync.Once wrapping a teardown that ends +// by deleting the VM's clone and its shared directory. +func (m *pathedMacVM) Stop() { + m.stops.Add(1) + m.stopOnce.Do(func() { + if m.gate != nil { + <-m.gate + } + _ = os.RemoveAll(m.jobDir) + _ = os.Remove(m.clone) + close(m.held) + }) +} + +func assertFileExists(t *testing.T, path, what string) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Errorf("%s is gone (%s): %v", what, path, err) + } +} + +// TestHandleMacOSJob_RedispatchOfTheSameJobGetsDisjointResources is the race +// that release-before-teardown opened. +// +// Every macOS resource used to be keyed by job ID alone and nothing else: the +// VM's shared directory, its .img/.aux clones, its artifacts dir. The failure +// paths now release the slot and clear seen/pending BEFORE teardown, so the +// same job ID can be dispatched again — the poll re-emits a still-queued job +// about every 30s, and maxProvisionAttempts allows several passes — while the +// previous attempt's VM is still alive with a stop wedged inside +// Virtualization.framework. +// +// Attempt 2 then writes its JIT config and clones its disk over attempt 1's +// live files, attempt 1's abandoned stop returns minutes later and runs +// os.Remove(clone) + os.RemoveAll(jobDir), and attempt 2 loses its disk and its +// .jit_config to a teardown for a VM it has nothing to do with. Its runner can +// never register, so it times out too, and strands another guest doing it — +// self-sustaining. +// +// Without the per-attempt VM instance id this test FAILS: both attempts get +// the same id, and attempt 2's files are gone after attempt 1's late teardown. +func TestHandleMacOSJob_RedispatchOfTheSameJobGetsDisjointResources(t *testing.T) { + prov := newMockProvider("mac-redispatch") + defer func() { _ = prov.Stop(context.Background()) }() + + dataDir := t.TempDir() + s := newSlotTestScheduler(t, prov, nil) + s.cfg.DataDir = dataDir + s.macAbandonedCap = 5 // the cap is tested elsewhere; don't trip it here + + var mu sync.Mutex + var made []*pathedMacVM + s.newMacOSVM = func(_ vm.MacOSVMConfig, id string) (vm.MacOSVM, error) { + mu.Lock() + defer mu.Unlock() + // The first attempt is the #196 guest: it never becomes reachable and + // its Stop() does not return until the test lets it. + first := len(made) == 0 + m := newPathedMacVM(t, dataDir, id, first) + if first { + m.gate = make(chan struct{}) + } else { + m.done = make(chan struct{}) // stays "running" until released + } + made = append(made, m) + return m, nil + } + + // The same job id both times: this is a RE-dispatch, not a second job. + const jobID = int64(100817345293) + event := macEvent(prov, jobID) + + // Attempt 1 hits the provisioning deadline and is abandoned with its Stop + // still pending. + runMacDispatch(t, s, event, 5*time.Second) + + // Attempt 2 takes the freed slot. + runMacDispatch(t, s, event, 5*time.Second) + + mu.Lock() + n := len(made) + mu.Unlock() + if n != 2 { + t.Fatalf("expected 2 provisioning attempts, got %d", n) + } + mu.Lock() + one, two := made[0], made[1] + mu.Unlock() + t.Cleanup(func() { + if two.done != nil { + close(two.done) + } + }) + + if one.id == two.id { + t.Fatalf("both attempts at job %d were given the same VM instance id %q — "+ + "their shared dir and disk clone are the same files, so attempt 1's "+ + "late teardown will delete attempt 2's", jobID, one.id) + } + + assertFileExists(t, two.jitPath(), "attempt 2's JIT config") + assertFileExists(t, two.clone, "attempt 2's disk clone") + + // Attempt 1's abandoned teardown finally lands, minutes late in + // production. It must only ever delete its own files. + close(one.gate) + select { + case <-one.held: + case <-time.After(5 * time.Second): + t.Fatal("the abandoned teardown never completed") + } + + assertFileExists(t, two.jitPath(), "attempt 2's JIT config after attempt 1's late teardown") + assertFileExists(t, two.clone, "attempt 2's disk clone after attempt 1's late teardown") + + if got := s.ActiveJobs(); got != 1 { + t.Errorf("ActiveJobs() = %d, want 1 (attempt 2 must still be running)", got) + } +} + +// TestMacVMInstanceIDDiscriminatesAttempts: the id keeps the job number (it is +// what an operator greps for) and adds a per-attempt suffix. +func TestMacVMInstanceIDDiscriminatesAttempts(t *testing.T) { + const jobID = int64(100817345293) + seen := map[string]bool{} + for i := 0; i < 64; i++ { + id := macVMInstanceID(jobID) + if !strings.HasPrefix(id, "100817345293-") { + t.Fatalf("instance id %q does not lead with the job id", id) + } + if seen[id] { + t.Fatalf("macVMInstanceID repeated %q; two attempts would share a disk clone", id) + } + seen[id] = true + } +} + +// --------------------------------------------------------------------------- +// MAJOR 3: the abandoned-VM trade is bounded and legible. +// --------------------------------------------------------------------------- + +// TestMacOSPoolCordonsPastTheAbandonedVMCap. +// +// The semaphore used to be the accidental limiter on stranded guests: leak the +// only slot and the node provisions zero more, so at most one guest could ever +// be abandoned. Releasing the slot on the failure path is exactly what removed +// that, and an abandoned VM is not tracked and no longer charged against +// MaxMacOSVMs — so without an explicit cap a repeatably-wedging guest lets the +// node keep provisioning until Virtualization.framework refuses (about two +// concurrent macOS guests on Apple Silicon) or the host runs out of RAM. +// +// Without the cap this test FAILS: the second dispatch constructs and boots +// another VM instead of being refused. +func TestMacOSPoolCordonsPastTheAbandonedVMCap(t *testing.T) { + prov := newMockProvider("mac-cap") + defer func() { _ = prov.Stop(context.Background()) }() + + // Constructed by hand rather than via newWedgedMacVM: this test needs to + // release the wedged Stop() mid-test to prove the cordon lifts. + first := &wedgedMacVM{stopReturns: make(chan struct{})} + s := newSlotTestScheduler(t, prov, first) + s.macAbandonedCap = 1 + + runMacDispatch(t, s, macEvent(prov, 1), 5*time.Second) + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Fatalf("AbandonedMacOSVMs() = %d after one unkillable VM, want 1", n) + } + + // Second job: must be refused before anything is constructed or booted. + var created atomic.Int32 + s.newMacOSVM = func(vm.MacOSVMConfig, string) (vm.MacOSVM, error) { + created.Add(1) + return &wedgedMacVM{stopReturns: make(chan struct{})}, nil + } + runMacDispatch(t, s, macEvent(prov, 2), 5*time.Second) + if got := created.Load(); got != 0 { + t.Errorf("provisioned %d macOS VMs past the abandoned cap; the point of the cap is that it provisions none", got) + } + assertCapacityIntact(t, s.macSem) + if n := s.ActiveJobs(); n != 0 { + t.Errorf("ActiveJobs() = %d, want 0", n) + } + + // The count is reportable, not just internal. + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d, want 1", n) + } + if d := s.AbandonedMacOSVMDetails(); len(d) != 1 || d[0].JobID != 1 { + t.Errorf("AbandonedMacOSVMDetails() = %+v, want one entry for job 1", d) + } + + // The cordon is not permanent: a merely-slow Vz stop that eventually + // returns discharges its entry and the pool takes work again. A node that + // recovers on its own must not need a restart. + close(first.stopReturns) + deadline := time.Now().Add(5 * time.Second) + for s.AbandonedMacOSVMs() != 0 && time.Now().Before(deadline) { + time.Sleep(2 * time.Millisecond) + } + if n := s.AbandonedMacOSVMs(); n != 0 { + t.Fatalf("AbandonedMacOSVMs() = %d after the abandoned stop finally returned, want 0", n) + } + + runMacDispatch(t, s, macEvent(prov, 3), 5*time.Second) + if created.Load() == 0 { + t.Error("the macOS pool stayed cordoned after its abandoned VMs were confirmed dead") + } +} + +// TestStatusAndHealthzReportAbandonedMacOSVMs: held_slots showed the outage; +// abandoned_macos_vms shows what the fix for it costs. An operator has to be +// able to see stranded guests without reading the log. +func TestStatusAndHealthzReportAbandonedMacOSVMs(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 1, Log: quietLogger()}) + s.macAbandoned.charge(macVMRef{JobID: 77, VMID: "77-deadbeef"}, "test") + + cs := &controlServer{sched: s, log: quietLogger()} + resp, err := cs.Status(context.Background(), &apiv1.StatusRequest{}) + if err != nil { + t.Fatalf("Status() error: %v", err) + } + if resp.AbandonedMacosVms != 1 { + t.Errorf("Status.AbandonedMacosVms = %d, want 1", resp.AbandonedMacosVms) + } + if resp.AbandonedMacosVmCap != defaultMacAbandonedVMCap { + t.Errorf("Status.AbandonedMacosVmCap = %d, want %d", resp.AbandonedMacosVmCap, defaultMacAbandonedVMCap) + } + // The pre-existing contract is untouched. + if resp.Status != "ok" || resp.ActiveJobs != 0 || resp.MaxConcurrent != 2 { + t.Errorf("existing Status fields changed: %+v", resp) + } + + rec := httptest.NewRecorder() + s.handleHealthz(rec, httptest.NewRequest(http.MethodGet, "/healthz", nil)) + var body struct { + ActiveJobs int `json:"active_jobs"` + HeldSlots int `json:"held_slots"` + AbandonedMacOSVMs int `json:"abandoned_macos_vms"` + AbandonedCap int `json:"abandoned_macos_vm_cap"` + Details []AbandonedMacVM `json:"abandoned_macos_vm_details"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decoding healthz body %q: %v", rec.Body.String(), err) + } + if body.AbandonedMacOSVMs != 1 { + t.Errorf("healthz abandoned_macos_vms = %d, want 1", body.AbandonedMacOSVMs) + } + if body.AbandonedCap != defaultMacAbandonedVMCap { + t.Errorf("healthz abandoned_macos_vm_cap = %d, want %d", body.AbandonedCap, defaultMacAbandonedVMCap) + } + if len(body.Details) != 1 || body.Details[0].VMID != "77-deadbeef" { + t.Errorf("healthz abandoned_macos_vm_details = %+v, want the stranded VM named", body.Details) + } +} + +// --------------------------------------------------------------------------- +// MINOR 5: the deadline path already stopped the VM; do not stop it again. +// --------------------------------------------------------------------------- + +// TestHandleMacOSJob_DeadlinePathStopsTheVMExactlyOnce. +// +// waitForMacRunnerBounded force-stops the VM when the deadline fires (#178, +// unchanged). The caller's error path then called stopVMBounded on the very +// same VM. Against darwinMacOSVM's stopOnce that is never a cheap no-op: the +// second caller BLOCKS until the first invocation returns, so on the wedged +// guest this whole PR is about it burned a full teardown grace and parked a +// third goroutine inside the Once — while a ghost JIT runner still needed +// deregistering. +// +// Without the fix this test FAILS: Stop is called twice and the second +// abandonment is charged against the cap as well, so a single wedged VM +// consumes the node's entire abandoned-VM budget. +func TestHandleMacOSJob_DeadlinePathStopsTheVMExactlyOnce(t *testing.T) { + prov := newMockProvider("mac-double-stop") + defer func() { _ = prov.Stop(context.Background()) }() + + macVM := newWedgedMacVM(t) + s := newSlotTestScheduler(t, prov, macVM) + s.macAbandonedCap = 99 // not the subject here + + runMacDispatch(t, s, macEvent(prov, 999), 5*time.Second) + + if got := macVM.stopCalls.Load(); got != 1 { + t.Errorf("Stop called %d times on the deadline path, want 1 — the second call re-enters a sync.Once that is still wedged", got) + } + if n := s.AbandonedMacOSVMs(); n != 1 { + t.Errorf("AbandonedMacOSVMs() = %d for ONE wedged VM, want 1 — a redundant stop must not double-charge the cap", n) + } +} + +// --------------------------------------------------------------------------- +// MINOR 4: the leak escalation is per-pool, not per-node. +// --------------------------------------------------------------------------- + +// TestTrackedInPool: jobs are counted against the pool whose slot they hold. +func TestTrackedInPool(t *testing.T) { + s := New(Config{MaxConcurrent: 2, MaxMacOSVMs: 2, Log: quietLogger()}) + s.mu.Lock() + s.running[jobKey{Provider: "p", JobID: 1}] = &runningJob{} // local + s.running[jobKey{Provider: "p", JobID: 2}] = &runningJob{dispatched: "r-2"} // linux + s.running[jobKey{Provider: "p", JobID: 3}] = &runningJob{ + macosVM: &fastMacVM{stops: new(atomic.Int32)}, + } // macos + s.mu.Unlock() + + for pool, want := range map[string]int{"local": 1, "linux": 1, "macos": 1} { + if got := s.trackedInPool(pool); got != want { + t.Errorf("trackedInPool(%q) = %d, want %d", pool, got, want) + } + } + if got := s.ActiveJobs(); got != 3 { + t.Errorf("ActiveJobs() = %d, want 3 (the global count keeps its meaning)", got) + } +} + +// TestSlotLeakEscalationIsScopedToThePool is the reason trackedInPool exists. +// +// The escalation compared a PER-POOL held count against s.ActiveJobs(), which +// is len(s.running) across every pool. The #196 node is a mac: it serves Linux +// jobs from its embedded VM continuously, so there was almost always a tracked +// job somewhere — and one was enough to make a leaked macOS slot look +// accounted for and hold the report at Warn forever. The one signal that would +// have named the outage was suppressed by unrelated work. +// +// Without the fix this test FAILS: the log says "waiting for a free +// concurrency slot" at Warn instead of naming the suspected leak. +func TestSlotLeakEscalationIsScopedToThePool(t *testing.T) { + after, every, leak := slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter + defer func() { slotWaitLogAfter, slotWaitLogEvery, slotLeakSuspectAfter = after, every, leak }() + slotWaitLogAfter = 10 * time.Millisecond + slotWaitLogEvery = 10 * time.Millisecond + slotLeakSuspectAfter = 10 * time.Millisecond + + s := New(Config{MaxConcurrent: 1, MaxMacOSVMs: 1, Log: quietLogger()}) + + // A perfectly healthy LOCAL job, tracked, doing real work. It has nothing + // to do with the macOS pool. + s.mu.Lock() + s.running[jobKey{Provider: "github", JobID: 1}] = &runningJob{repo: "myrepo"} + s.mu.Unlock() + + // The macOS slot is held by nothing tracked: the #196 state exactly. + s.macSem <- struct{}{} + + var buf bytes.Buffer + log := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + got := make(chan *slotToken, 1) + go func() { got <- s.acquireSlot(context.Background(), s.macSem, "macos", log) }() + + time.Sleep(60 * time.Millisecond) + <-s.macSem // let the waiter through so the goroutine (and the log) settle + select { + case tok := <-got: + tok.release() + case <-time.After(5 * time.Second): + t.Fatal("acquireSlot never returned after the slot was freed") + } + + out := buf.String() + if !strings.Contains(out, "suspected slot leak") { + t.Errorf("a leaked macOS slot was not escalated because an unrelated LOCAL job was tracked; got:\n%s", out) + } + if !strings.Contains(out, "level=ERROR") { + t.Errorf("leak report was not logged at ERROR; got:\n%s", out) + } + if !strings.Contains(out, "pool_tracked_jobs=0") { + t.Errorf("wait log did not report the POOL's tracked-job count; got:\n%s", out) + } +} + +// --------------------------------------------------------------------------- +// MINOR 6: the slot must be free WHILE teardown is still in flight. +// --------------------------------------------------------------------------- + +// TestSlotIsFreeWhileTeardownIsStillInFlight pins the ORDERING, which nothing +// else in the suite did. +// +// Every teardown bound in this package is compressed to milliseconds in tests +// and mockProvider.ReleaseJob returns instantly, so deleting the explicit +// slot.release() and leaning on the deferred backstop still passed everything +// — even though in production the difference is a GitHub API call on +// context.Background() with no deadline, plus a VM stop that may never return, +// standing between the job ending and the next macOS job starting. This test +// makes ReleaseJob slow on purpose and asserts the slot is already back. +func TestSlotIsFreeWhileTeardownIsStillInFlight(t *testing.T) { + t.Run("provisioning failure path", func(t *testing.T) { + prov := newMockProvider("mac-order-fail") + defer func() { _ = prov.Stop(context.Background()) }() + + s := newSlotTestScheduler(t, prov, newWedgedMacVM(t)) + + observed := make(chan int, 4) + proceed := make(chan struct{}) + var once sync.Once + prov.releaseHook = func(*providers.Claim) { + once.Do(func() { + observed <- len(s.macSem) + <-proceed + }) + } + + done := make(chan struct{}) + go func() { + defer close(done) + s.handleMacOSJob(context.Background(), macEvent(prov, 11)) + }() + + select { + case held := <-observed: + if held != 0 { + t.Errorf("macSem held=%d while the ghost-runner deregistration was still in flight, want 0: "+ + "the slot is being held across a GitHub API call that has no deadline", held) + } + case <-time.After(5 * time.Second): + t.Fatal("ReleaseJob was never reached on the provisioning failure path") + } + close(proceed) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("handleMacOSJob did not return") + } + }) + + t.Run("job wait-goroutine", func(t *testing.T) { + prov := newMockProvider("mac-order-wait") + defer func() { _ = prov.Stop(context.Background()) }() + + var stops atomic.Int32 + s := newSlotTestScheduler(t, prov, &fastMacVM{ip: "192.168.64.5", stops: &stops}) + + observed := make(chan int, 4) + proceed := make(chan struct{}) + var once sync.Once + prov.releaseHook = func(*providers.Claim) { + once.Do(func() { + observed <- len(s.macSem) + <-proceed + }) + } + + runMacDispatch(t, s, macEvent(prov, 12), 5*time.Second) + + select { + case held := <-observed: + if held != 0 { + t.Errorf("macSem held=%d while the finished job's runner was still being deregistered, want 0: "+ + "the next macOS job is waiting on a teardown that cannot delay it", held) + } + case <-time.After(5 * time.Second): + t.Fatal("the wait-goroutine never reached ReleaseJob") + } + close(proceed) + waitForSlotFree(t, s.macSem, 5*time.Second) + assertCapacityIntact(t, s.macSem) + }) +}