diff --git a/pkg/controller/helper/event.go b/pkg/controller/helper/event.go index 8d7e492ef..8ce31719d 100644 --- a/pkg/controller/helper/event.go +++ b/pkg/controller/helper/event.go @@ -28,6 +28,7 @@ const ( DeleteTimestampChanged = "DeleteTimestampChanged" PreservedOnDelete = "PreservedOnDelete" FeatureNotSupported = "FeatureNotSupported" + GracefulShutdownDisabled = "GracefulShutdownDisabled" ) // NodeEventReason diff --git a/pkg/controller/service/clbv1/vgroups.go b/pkg/controller/service/clbv1/vgroups.go index b3dbbb80b..8b9615a54 100644 --- a/pkg/controller/service/clbv1/vgroups.go +++ b/pkg/controller/service/clbv1/vgroups.go @@ -705,6 +705,8 @@ func (mgr *VGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.RequestCo return nil, false, nil } + gracefulShutdownEnabled := backend.IsGracefulShutdownEnabled(reqCtx, candidates.TrafficPolicy) + for _, es := range candidates.EndpointSlices { var backendPort int if vgroup.ServicePort.TargetPort.Type == intstr.Int { @@ -726,9 +728,11 @@ func (mgr *VGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.RequestCo } for _, ep := range es.Endpoints { - // ignore terminating pods - if ep.Conditions.Terminating != nil && *ep.Conditions.Terminating { - continue + isTerminating := ep.Conditions.Terminating != nil && *ep.Conditions.Terminating + if isTerminating { + if !gracefulShutdownEnabled || ep.Conditions.Serving == nil || !*ep.Conditions.Serving { + continue + } } for _, addr := range ep.Addresses { @@ -736,7 +740,7 @@ func (mgr *VGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.RequestCo continue } - if ep.Conditions.Ready != nil && *ep.Conditions.Ready { + if isTerminating || (ep.Conditions.Ready != nil && *ep.Conditions.Ready) { endpointMap[addr] = true backends = append(backends, model.BackendAttribute{ NodeName: ep.NodeName, @@ -746,6 +750,7 @@ func (mgr *VGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.RequestCo Port: backendPort, Description: vgroup.VGroupName, TargetRef: ep.TargetRef, + Terminating: isTerminating, }) continue } @@ -962,11 +967,17 @@ func setWeightBackends(mode helper.TrafficPolicy, backends []model.BackendAttrib // use default if weight == nil { defaultWeight := pointer.IntDeref(defaultWeight, DefaultServerWeight) - return podNumberAlgorithm(mode, backends, defaultWeight) + backends = podNumberAlgorithm(mode, backends, defaultWeight) + } else { + backends = podPercentAlgorithm(mode, backends, *weight) } - return podPercentAlgorithm(mode, backends, *weight) - + for i := range backends { + if backends[i].Terminating { + backends[i].Weight = 0 + } + } + return backends } // weight algorithm diff --git a/pkg/controller/service/clbv1/vgroups_test.go b/pkg/controller/service/clbv1/vgroups_test.go index bbc204bb0..2bc321f35 100644 --- a/pkg/controller/service/clbv1/vgroups_test.go +++ b/pkg/controller/service/clbv1/vgroups_test.go @@ -685,6 +685,310 @@ func TestSetBackendsFromEndpointSlices(t *testing.T) { assert.Equal(t, 0, len(backends)) }) + t.Run("graceful-shutdown: terminating+serving kept in ENI mode", func(t *testing.T) { + terminating := true + serving := true + nodeName := "node-1" + portName := "http" + port := int32(8080) + protocol := v1.ProtocolTCP + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: klogr.New(), + } + candidates := &backend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "default", + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.10"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Port: &port, + Protocol: &protocol, + }, + }, + }, + }, + } + vgroup := model.VServerGroup{ + VGroupName: "test-vgroup", + ServicePort: v1.ServicePort{ + Port: 80, + TargetPort: intstr.FromInt(8080), + Name: "http", + }, + } + backends, containsPotential, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, vgroup) + assert.NoError(t, err) + assert.False(t, containsPotential) + assert.Equal(t, 1, len(backends)) + assert.Equal(t, "10.0.0.10", backends[0].ServerIp) + assert.True(t, backends[0].Terminating) + }) + + t.Run("graceful-shutdown: ignored when ignore-weight-update is on", func(t *testing.T) { + terminating := true + serving := true + nodeName := "node-1" + portName := "http" + port := int32(8080) + protocol := v1.ProtocolTCP + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + annotation.Annotation(annotation.IgnoreWeightUpdate): string(model.OnFlag), + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: klogr.New(), + } + candidates := &backend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "default", + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.14"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Port: &port, + Protocol: &protocol, + }, + }, + }, + }, + } + vgroup := model.VServerGroup{ + VGroupName: "test-vgroup", + ServicePort: v1.ServicePort{ + Port: 80, + TargetPort: intstr.FromInt(8080), + Name: "http", + }, + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, vgroup) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + + t.Run("graceful-shutdown: terminating+not-serving removed", func(t *testing.T) { + terminating := true + serving := false + nodeName := "node-1" + portName := "http" + port := int32(8080) + protocol := v1.ProtocolTCP + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: klogr.New(), + } + candidates := &backend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "default", + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.11"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Port: &port, + Protocol: &protocol, + }, + }, + }, + }, + } + vgroup := model.VServerGroup{ + VGroupName: "test-vgroup", + ServicePort: v1.ServicePort{ + Port: 80, + TargetPort: intstr.FromInt(8080), + Name: "http", + }, + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, vgroup) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + + t.Run("graceful-shutdown: terminating ignored without annotation", func(t *testing.T) { + terminating := true + serving := true + nodeName := "node-1" + portName := "http" + port := int32(8080) + protocol := v1.ProtocolTCP + candidates := &backend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "default", + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.12"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Port: &port, + Protocol: &protocol, + }, + }, + }, + }, + } + vgroup := model.VServerGroup{ + VGroupName: "test-vgroup", + ServicePort: v1.ServicePort{ + Port: 80, + TargetPort: intstr.FromInt(8080), + Name: "http", + }, + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtx, candidates, vgroup) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + + t.Run("graceful-shutdown: terminating ignored in non-ENI mode", func(t *testing.T) { + terminating := true + serving := true + nodeName := "node-1" + portName := "http" + port := int32(8080) + protocol := v1.ProtocolTCP + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: klogr.New(), + } + candidates := &backend.EndpointWithENI{ + TrafficPolicy: helper.LocalTrafficPolicy, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-slice", + Namespace: "default", + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.13"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &portName, + Port: &port, + Protocol: &protocol, + }, + }, + }, + }, + } + vgroup := model.VServerGroup{ + VGroupName: "test-vgroup", + ServicePort: v1.ServicePort{ + Port: 80, + TargetPort: intstr.FromInt(8080), + Name: "http", + }, + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, vgroup) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + t.Run("duplicate endpoint addresses", func(t *testing.T) { ready := true nodeName := "node-1" diff --git a/pkg/controller/service/nlbv2/server_groups.go b/pkg/controller/service/nlbv2/server_groups.go index c1426a1dc..6d4f5949b 100644 --- a/pkg/controller/service/nlbv2/server_groups.go +++ b/pkg/controller/service/nlbv2/server_groups.go @@ -728,6 +728,8 @@ func (mgr *ServerGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.Requ return nil, false, nil } + gracefulShutdownEnabled := reconbackend.IsGracefulShutdownEnabled(reqCtx, candidates.TrafficPolicy) + for _, es := range candidates.EndpointSlices { var backendPort int32 if sg.ServicePort.TargetPort.Type == intstr.Int { @@ -749,9 +751,11 @@ func (mgr *ServerGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.Requ } for _, ep := range es.Endpoints { - // ignore terminating pods - if ep.Conditions.Terminating != nil && *ep.Conditions.Terminating { - continue + isTerminating := ep.Conditions.Terminating != nil && *ep.Conditions.Terminating + if isTerminating { + if !gracefulShutdownEnabled || ep.Conditions.Serving == nil || !*ep.Conditions.Serving { + continue + } } for _, addr := range ep.Addresses { @@ -759,7 +763,7 @@ func (mgr *ServerGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.Requ continue } - if ep.Conditions.Ready != nil && *ep.Conditions.Ready { + if isTerminating || (ep.Conditions.Ready != nil && *ep.Conditions.Ready) { endpointMap[addr] = true backends = append(backends, nlbmodel.ServerGroupServer{ NodeName: ep.NodeName, @@ -771,6 +775,7 @@ func (mgr *ServerGroupManager) setBackendsFromEndpointSlices(reqCtx *svcCtx.Requ Description: sg.ServerGroupName, TargetRef: ep.TargetRef, IPVersion: addressTypeToIPVersionType(es.AddressType), + Terminating: isTerminating, }) continue } @@ -1059,11 +1064,17 @@ func setWeightBackends(mode helper.TrafficPolicy, backends []nlbmodel.ServerGrou // use default if weight == nil { defaultWeight := pointer.IntDeref(defaultWeight, DefaultServerWeight) - return podNumberAlgorithm(mode, backends, defaultWeight) + backends = podNumberAlgorithm(mode, backends, defaultWeight) + } else { + backends = podPercentAlgorithm(mode, backends, *weight) } - return podPercentAlgorithm(mode, backends, *weight) - + for i := range backends { + if backends[i].Terminating { + backends[i].Weight = 0 + } + } + return backends } // weight algorithm diff --git a/pkg/controller/service/nlbv2/server_groups_test.go b/pkg/controller/service/nlbv2/server_groups_test.go index a909dbfb9..de1c6eabf 100644 --- a/pkg/controller/service/nlbv2/server_groups_test.go +++ b/pkg/controller/service/nlbv2/server_groups_test.go @@ -2803,6 +2803,261 @@ func TestServerGroupManager_SetBackendsFromEndpointSlices(t *testing.T) { tt.validate(t, backends, containsPotentialReady) }) } + + // graceful-shutdown tests require ENI traffic policy and annotation, so run separately + t.Run("graceful-shutdown: terminating+serving kept in ENI mode", func(t *testing.T) { + terminating := true + serving := true + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "test-svc", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: util.NLBLog.WithValues("service", util.Key(svcWithAnno)), + } + candidates := &reconbackend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + AddressIPVersion: model.IPv4, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-svc-1", + Namespace: v1.NamespaceDefault, + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.10"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + {Name: &portName, Port: &port, Protocol: &protocol}, + }, + AddressType: discovery.AddressTypeIPv4, + }, + }, + } + sg := nlbmodel.ServerGroup{ + ServicePort: &v1.ServicePort{Name: "tcp", Port: 80, TargetPort: intstr.FromInt(8080)}, + ServerGroupName: "test-sg", + } + backends, containsPotentialReady, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, sg) + assert.NoError(t, err) + assert.False(t, containsPotentialReady) + assert.Equal(t, 1, len(backends)) + assert.Equal(t, "10.0.0.10", backends[0].ServerIp) + assert.True(t, backends[0].Terminating) + }) + + t.Run("graceful-shutdown: ignored when ignore-weight-update is on", func(t *testing.T) { + terminating := true + serving := true + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "test-svc", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + annotation.Annotation(annotation.IgnoreWeightUpdate): string(model.OnFlag), + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: util.NLBLog.WithValues("service", util.Key(svcWithAnno)), + } + candidates := &reconbackend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + AddressIPVersion: model.IPv4, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-svc-1", + Namespace: v1.NamespaceDefault, + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.14"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + {Name: &portName, Port: &port, Protocol: &protocol}, + }, + AddressType: discovery.AddressTypeIPv4, + }, + }, + } + sg := nlbmodel.ServerGroup{ + ServicePort: &v1.ServicePort{Name: "tcp", Port: 80, TargetPort: intstr.FromInt(8080)}, + ServerGroupName: "test-sg", + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, sg) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + + t.Run("graceful-shutdown: terminating+not-serving removed", func(t *testing.T) { + terminating := true + serving := false + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "test-svc", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: util.NLBLog.WithValues("service", util.Key(svcWithAnno)), + } + candidates := &reconbackend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + AddressIPVersion: model.IPv4, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-svc-1", + Namespace: v1.NamespaceDefault, + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.11"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + {Name: &portName, Port: &port, Protocol: &protocol}, + }, + AddressType: discovery.AddressTypeIPv4, + }, + }, + } + sg := nlbmodel.ServerGroup{ + ServicePort: &v1.ServicePort{Name: "tcp", Port: 80, TargetPort: intstr.FromInt(8080)}, + ServerGroupName: "test-sg", + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, sg) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + + t.Run("graceful-shutdown: terminating ignored without annotation", func(t *testing.T) { + terminating := true + serving := true + candidates := &reconbackend.EndpointWithENI{ + TrafficPolicy: helper.ENITrafficPolicy, + AddressIPVersion: model.IPv4, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-svc-1", + Namespace: v1.NamespaceDefault, + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.12"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + {Name: &portName, Port: &port, Protocol: &protocol}, + }, + AddressType: discovery.AddressTypeIPv4, + }, + }, + } + sg := nlbmodel.ServerGroup{ + ServicePort: &v1.ServicePort{Name: "tcp", Port: 80, TargetPort: intstr.FromInt(8080)}, + ServerGroupName: "test-sg", + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtx, candidates, sg) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) + + t.Run("graceful-shutdown: terminating ignored in non-ENI mode", func(t *testing.T) { + terminating := true + serving := true + svcWithAnno := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "test-svc", + Annotations: map[string]string{ + annotation.Annotation(annotation.GracefulShutdown): "true", + }, + }, + } + reqCtxWithAnno := &svcCtx.RequestContext{ + Ctx: context.TODO(), + Service: svcWithAnno, + Anno: annotation.NewAnnotationRequest(svcWithAnno), + Log: util.NLBLog.WithValues("service", util.Key(svcWithAnno)), + } + candidates := &reconbackend.EndpointWithENI{ + TrafficPolicy: helper.LocalTrafficPolicy, + AddressIPVersion: model.IPv4, + EndpointSlices: []discovery.EndpointSlice{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-svc-1", + Namespace: v1.NamespaceDefault, + }, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{"10.0.0.13"}, + Conditions: discovery.EndpointConditions{ + Terminating: &terminating, + Serving: &serving, + }, + NodeName: &nodeName, + }, + }, + Ports: []discovery.EndpointPort{ + {Name: &portName, Port: &port, Protocol: &protocol}, + }, + AddressType: discovery.AddressTypeIPv4, + }, + }, + } + sg := nlbmodel.ServerGroup{ + ServicePort: &v1.ServicePort{Name: "tcp", Port: 80, TargetPort: intstr.FromInt(8080)}, + ServerGroupName: "test-sg", + } + backends, _, err := mgr.setBackendsFromEndpointSlices(reqCtxWithAnno, candidates, sg) + assert.NoError(t, err) + assert.Equal(t, 0, len(backends)) + }) } func TestServerGroupManager_BuildENIBackends(t *testing.T) { diff --git a/pkg/controller/service/reconcile/annotation/annotations.go b/pkg/controller/service/reconcile/annotation/annotations.go index 9141e8904..6cdd9e1fc 100644 --- a/pkg/controller/service/reconcile/annotation/annotations.go +++ b/pkg/controller/service/reconcile/annotation/annotations.go @@ -44,6 +44,7 @@ const ( ConnectionDrain = AnnotationLoadBalancerPrefix + "connection-drain" // ConnectionDrain connection drain ConnectionDrainTimeout = AnnotationLoadBalancerPrefix + "connection-drain-timeout" // ConnectionDrainTimeout connection drain timeout IgnoreWeightUpdate = AnnotationLoadBalancerPrefix + "ignore-weight-update" + GracefulShutdown = AnnotationLoadBalancerPrefix + "graceful-shutdown" PreserveLBOnDelete = AnnotationLoadBalancerPrefix + "preserve-lb-on-delete" ) diff --git a/pkg/controller/service/reconcile/backend/backend.go b/pkg/controller/service/reconcile/backend/backend.go index 90bdd8b92..a68c79f5e 100644 --- a/pkg/controller/service/reconcile/backend/backend.go +++ b/pkg/controller/service/reconcile/backend/backend.go @@ -304,3 +304,20 @@ func Batch[T any](target []T, batchSize int, f Func[T]) error { return utilerrors.NewAggregate(errs) } + +func IsGracefulShutdownEnabled(reqCtx *svcCtx.RequestContext, trafficPolicy helper.TrafficPolicy) bool { + if trafficPolicy != helper.ENITrafficPolicy { + return false + } + if reqCtx.Anno.Get(annotation.GracefulShutdown) != "true" { + return false + } + if strings.EqualFold(reqCtx.Anno.Get(annotation.IgnoreWeightUpdate), string(model.OnFlag)) { + if reqCtx.Recorder != nil { + reqCtx.Recorder.Event(reqCtx.Service, v1.EventTypeWarning, helper.GracefulShutdownDisabled, + "Disable graceful-shutdown because ignore-weight-update is enabled") + } + return false + } + return true +} diff --git a/pkg/model/load_balancer.go b/pkg/model/load_balancer.go index eaef94042..f7394ad76 100644 --- a/pkg/model/load_balancer.go +++ b/pkg/model/load_balancer.go @@ -246,7 +246,8 @@ type BackendAttribute struct { TargetRef *v1.ObjectReference // Invalid represents that the backend is un - Invalid bool + Invalid bool + Terminating bool } type CertAttribute struct { diff --git a/pkg/model/nlb/nlb.go b/pkg/model/nlb/nlb.go index de66e5318..4787cd33d 100644 --- a/pkg/model/nlb/nlb.go +++ b/pkg/model/nlb/nlb.go @@ -264,8 +264,9 @@ type ServerGroupServer struct { Status string IPVersion model.AddressIPVersionType - TargetRef *v1.ObjectReference - Invalid bool + TargetRef *v1.ObjectReference + Invalid bool + Terminating bool } type ZoneMapping struct { diff --git a/test/e2e/client/kubeclient.go b/test/e2e/client/kubeclient.go index e07f3eb17..4ccf5a364 100644 --- a/test/e2e/client/kubeclient.go +++ b/test/e2e/client/kubeclient.go @@ -824,6 +824,117 @@ func (client *KubeClient) GetDeploymentPods() ([]v1.Pod, error) { return podList.Items, nil } +func (client *KubeClient) DeletePod(name string) error { + return client.CoreV1().Pods(Namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) +} + +func (client *KubeClient) CreateGracefulShutdownDeployment() error { + var replica int32 = 3 + var gracePeriod int64 = 60 + name := "nginx-graceful" + deploy := &appv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: Namespace, + Labels: map[string]string{"run": name}, + }, + Spec: appv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"run": name}, + }, + Replicas: &replica, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"run": name}, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{ + { + Name: "nginx", + Image: "registry.cn-hangzhou.aliyuncs.com/acs-sample/nginx:latest", + ImagePullPolicy: "Always", + Ports: []v1.ContainerPort{ + {Name: "http", ContainerPort: 80, Protocol: v1.ProtocolTCP}, + {Name: "https", ContainerPort: 443, Protocol: v1.ProtocolTCP}, + }, + Lifecycle: &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{Command: []string{"sleep", "30"}}, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := client.AppsV1().Deployments(Namespace).Create(context.Background(), deploy, metav1.CreateOptions{}) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create %s error: %s", name, err.Error()) + } + if err := client.DeleteGracefulShutdownDeployment(); err != nil { + return fmt.Errorf("delete existing %s error: %s", name, err.Error()) + } + _, err = client.AppsV1().Deployments(Namespace).Create(context.Background(), deploy, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create %s error: %s", name, err.Error()) + } + } + return wait.Poll(5*time.Second, 2*time.Minute, func() (done bool, err error) { + pods, err := client.CoreV1().Pods(Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: "run=" + name}) + if err != nil { + return false, nil + } + if len(pods.Items) != int(replica) { + return false, nil + } + for _, pod := range pods.Items { + if pod.Status.Phase != v1.PodRunning { + return false, nil + } + } + return true, nil + }) +} + +func (client *KubeClient) DeleteGracefulShutdownDeployment() error { + name := "nginx-graceful" + propagation := metav1.DeletePropagationForeground + err := client.AppsV1().Deployments(Namespace).Delete(context.TODO(), name, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + }) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + return wait.PollImmediate(5*time.Second, 2*time.Minute, func() (done bool, err error) { + _, err = client.AppsV1().Deployments(Namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }) +} + +func (client *KubeClient) GetGracefulShutdownPods() ([]v1.Pod, error) { + podList, err := client.CoreV1().Pods(Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "run=nginx-graceful", + }) + if err != nil { + return nil, err + } + return podList.Items, nil +} + func (client *KubeClient) ListServiceEvents(svc *v1.Service, reason string) ([]v1.Event, error) { selector := fmt.Sprintf("involvedObject.name=%s", svc.Name) if reason != "" { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index bcf809d1e..e3d70dc73 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -73,6 +73,7 @@ func AddControllerTests(f *framework.Framework) { clbv1.RunLoadBalancerTestCases(f) clbv1.RunListenerTestCases(f) clbv1.RunBackendTestCases(f) + clbv1.RunGracefulShutdownTestCases(f) }) if options.TestConfig.NLBZoneMaps != "" { @@ -80,6 +81,7 @@ func AddControllerTests(f *framework.Framework) { nlbv2.RunLoadBalancerTestCases(f) nlbv2.RunListenerTestCases(f) nlbv2.RunBackendTestCases(f) + nlbv2.RunGracefulShutdownTestCases(f) }) } else { klog.Warningf("NLBZoneMaps is empty, skip NLB service tests") diff --git a/test/e2e/framework/expect.go b/test/e2e/framework/expect.go index 005be0223..4ff5144ed 100644 --- a/test/e2e/framework/expect.go +++ b/test/e2e/framework/expect.go @@ -1643,3 +1643,65 @@ func domainExtensionsEqual(a, b []model.DomainExtension) bool { } return true } + +func (f *Framework) WaitForBackendWeight(svc *v1.Service, serverIp string, expectedWeight int) error { + var retErr error + _ = wait.PollImmediate(5*time.Second, 60*time.Second, func() (done bool, err error) { + remote, err := buildRemoteModel(f, svc) + if err != nil { + retErr = fmt.Errorf("build remote model: %s", err.Error()) + return false, nil + } + port := svc.Spec.Ports[0] + name := getVGroupName(svc, port) + for _, vg := range remote.VServerGroups { + if vg.VGroupName != name { + continue + } + for _, b := range vg.Backends { + if b.ServerIp == serverIp { + if b.Weight == expectedWeight { + retErr = nil + return true, nil + } + retErr = fmt.Errorf("backend %s weight: expect %d, got %d", serverIp, expectedWeight, b.Weight) + return false, nil + } + } + retErr = fmt.Errorf("backend %s not found in vgroup %s", serverIp, vg.VGroupId) + return false, nil + } + retErr = fmt.Errorf("vgroup %s not found", name) + return false, nil + }) + return retErr +} + +func (f *Framework) WaitForBackendRemoved(svc *v1.Service, serverIp string) error { + var retErr error + _ = wait.PollImmediate(5*time.Second, 120*time.Second, func() (done bool, err error) { + remote, err := buildRemoteModel(f, svc) + if err != nil { + retErr = fmt.Errorf("build remote model: %s", err.Error()) + return false, nil + } + port := svc.Spec.Ports[0] + name := getVGroupName(svc, port) + for _, vg := range remote.VServerGroups { + if vg.VGroupName != name { + continue + } + for _, b := range vg.Backends { + if b.ServerIp == serverIp { + retErr = fmt.Errorf("backend %s still present with weight %d", serverIp, b.Weight) + return false, nil + } + } + retErr = nil + return true, nil + } + retErr = nil + return true, nil + }) + return retErr +} diff --git a/test/e2e/framework/expect_nlb.go b/test/e2e/framework/expect_nlb.go index 511c7d1cb..5164fb3fe 100644 --- a/test/e2e/framework/expect_nlb.go +++ b/test/e2e/framework/expect_nlb.go @@ -1530,3 +1530,51 @@ func nlbSourceRangesSecurityGroupPermissionsEqual(svc *v1.Service, sg *ecsmodel. return nil } + +func (f *Framework) WaitForNLBBackendWeight(svc *v1.Service, serverIp string, expectedWeight int32) error { + var retErr error + _ = wait.PollImmediate(5*time.Second, 60*time.Second, func() (done bool, err error) { + remote, err := buildNLBRemoteModel(f, svc) + if err != nil { + retErr = fmt.Errorf("build nlb remote model: %s", err.Error()) + return false, nil + } + for _, sg := range remote.ServerGroups { + for _, b := range sg.Servers { + if b.ServerIp == serverIp { + if b.Weight == expectedWeight { + retErr = nil + return true, nil + } + retErr = fmt.Errorf("nlb backend %s weight: expect %d, got %d", serverIp, expectedWeight, b.Weight) + return false, nil + } + } + } + retErr = fmt.Errorf("nlb backend %s not found", serverIp) + return false, nil + }) + return retErr +} + +func (f *Framework) WaitForNLBBackendRemoved(svc *v1.Service, serverIp string) error { + var retErr error + _ = wait.PollImmediate(5*time.Second, 120*time.Second, func() (done bool, err error) { + remote, err := buildNLBRemoteModel(f, svc) + if err != nil { + retErr = fmt.Errorf("build nlb remote model: %s", err.Error()) + return false, nil + } + for _, sg := range remote.ServerGroups { + for _, b := range sg.Servers { + if b.ServerIp == serverIp { + retErr = fmt.Errorf("nlb backend %s still present with weight %d", serverIp, b.Weight) + return false, nil + } + } + } + retErr = nil + return true, nil + }) + return retErr +} diff --git a/test/e2e/testcase/service/clbv1/backend.go b/test/e2e/testcase/service/clbv1/backend.go index 2d703ccea..64467b9cf 100644 --- a/test/e2e/testcase/service/clbv1/backend.go +++ b/test/e2e/testcase/service/clbv1/backend.go @@ -851,3 +851,57 @@ func RunBackendTestCases(f *framework.Framework) { }) }) } + +func RunGracefulShutdownTestCases(f *framework.Framework) { + if options.TestConfig.Network != options.Terway { + return + } + + ginkgo.Describe("clb service controller: graceful-shutdown", func() { + ginkgo.AfterEach(func() { + ginkgo.By("delete service") + err := f.AfterEach() + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.It("terminating pod weight should be 0", func() { + ginkgo.By("creating dedicated deployment with preStop hook") + err := f.Client.KubeClient.CreateGracefulShutdownDeployment() + gomega.Expect(err).To(gomega.BeNil()) + defer func() { + _ = f.Client.KubeClient.DeleteGracefulShutdownDeployment() + }() + + rawSvc := f.Client.KubeClient.DefaultService() + rawSvc.Spec.Selector = map[string]string{"run": "nginx-graceful"} + rawSvc.Annotations = map[string]string{ + annotation.BackendType: model.ENIBackendType, + annotation.Annotation(annotation.GracefulShutdown): "true", + } + svc, err := f.Client.KubeClient.CreateService(rawSvc) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectLoadBalancerEqual(svc) + gomega.Expect(err).To(gomega.BeNil()) + + pods, err := f.Client.KubeClient.GetGracefulShutdownPods() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(pods)).To(gomega.BeNumerically(">=", 1)) + + targetPod := pods[0] + targetIP := targetPod.Status.PodIP + ginkgo.By(fmt.Sprintf("deleting pod %s (ip: %s)", targetPod.Name, targetIP)) + + err = f.Client.KubeClient.DeletePod(targetPod.Name) + gomega.Expect(err).To(gomega.BeNil()) + + ginkgo.By("waiting for backend weight=0") + err = f.WaitForBackendWeight(svc, targetIP, 0) + gomega.Expect(err).To(gomega.BeNil()) + + ginkgo.By("waiting for backend removal after termination") + err = f.WaitForBackendRemoved(svc, targetIP) + gomega.Expect(err).To(gomega.BeNil()) + }) + }) +} diff --git a/test/e2e/testcase/service/nlbv2/backend.go b/test/e2e/testcase/service/nlbv2/backend.go index 2eaca022a..a4266d8e5 100644 --- a/test/e2e/testcase/service/nlbv2/backend.go +++ b/test/e2e/testcase/service/nlbv2/backend.go @@ -1260,3 +1260,58 @@ func RunBackendTestCases(f *framework.Framework) { }) }) } + +func RunGracefulShutdownTestCases(f *framework.Framework) { + if options.TestConfig.Network != options.Terway { + return + } + + ginkgo.Describe("nlb service controller: graceful-shutdown", func() { + ginkgo.AfterEach(func() { + ginkgo.By("delete service") + err := f.AfterEach() + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.It("terminating pod weight should be 0", func() { + ginkgo.By("creating dedicated deployment with preStop hook") + err := f.Client.KubeClient.CreateGracefulShutdownDeployment() + gomega.Expect(err).To(gomega.BeNil()) + defer func() { + _ = f.Client.KubeClient.DeleteGracefulShutdownDeployment() + }() + + rawSvc := f.Client.KubeClient.DefaultNLBService() + rawSvc.Spec.Selector = map[string]string{"run": "nginx-graceful"} + rawSvc.Annotations = map[string]string{ + annotation.BackendType: model.ENIBackendType, + annotation.Annotation(annotation.GracefulShutdown): "true", + annotation.Annotation(annotation.ZoneMaps): options.TestConfig.NLBZoneMaps, + } + svc, err := f.Client.KubeClient.CreateService(rawSvc) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectNetworkLoadBalancerEqual(svc) + gomega.Expect(err).To(gomega.BeNil()) + + pods, err := f.Client.KubeClient.GetGracefulShutdownPods() + gomega.Expect(err).To(gomega.BeNil()) + gomega.Expect(len(pods)).To(gomega.BeNumerically(">=", 1)) + + targetPod := pods[0] + targetIP := targetPod.Status.PodIP + ginkgo.By(fmt.Sprintf("deleting pod %s (ip: %s)", targetPod.Name, targetIP)) + + err = f.Client.KubeClient.DeletePod(targetPod.Name) + gomega.Expect(err).To(gomega.BeNil()) + + ginkgo.By("waiting for backend weight=0") + err = f.WaitForNLBBackendWeight(svc, targetIP, 0) + gomega.Expect(err).To(gomega.BeNil()) + + ginkgo.By("waiting for backend removal after termination") + err = f.WaitForNLBBackendRemoved(svc, targetIP) + gomega.Expect(err).To(gomega.BeNil()) + }) + }) +}