Add job to retry stuck service binding delete operations

Detect credential-binding and service-key delete operations stuck in
'in progress' with a permanently-failed polling job, and internally
retry by re-enqueuing the original DeleteBindingJob. Mirrors the
existing service-instance delete-retry job. Route bindings are skipped.

Author: kathap

app/jobs/runtime/service_operations_binding_delete_stuck_in_progress_retry.rb

@@ -0,0 +1,109 @@
+module VCAP::CloudController
+  module Jobs
+    module Runtime
+      class ServiceOperationsBindingDeleteStuckInProgressRetry < VCAP::CloudController::Jobs::CCJob
+        BATCH_SIZE = 10
+
+        def perform
+          logger.info("Retrying stuck binding 'delete' operations")
+          retry_stuck(ServiceBindingOperation, ServiceBinding, :service_binding_id, 'service_bindings.delete')
+          retry_stuck(ServiceKeyOperation,     ServiceKey,     :service_key_id,     'service_keys.delete')
+        end
+
+        def max_attempts
+          1
+        end
+
+        private
+
+        def retry_stuck(operation_model, instance_model, foreign_key, jobs_operation)
+          # Find stuck binding 'delete' operations where the broker may still be working
+          # but CC's polling job has permanently failed due to a transient error (e.g. brief db connection flip).
+          #
+          # Unlike create we do not mark the operation failed and do not mitigate orphans: for a delete we
+          # re-enqueue the original polling job so the unbind is driven to completion. The original delayed_job's
+          # serialized handler is reused, preserving @start_time so the ReoccurringJob max-duration expiry
+          # (which marks the operation failed via handle_timeout) still fires against the original polling window.
+          operation_table = operation_model.table_name
+          instance_table = instance_model.table_name
+
+          stuck = operation_model.
+                  join(instance_table, id: Sequel[operation_table][foreign_key]).
+                  join(:jobs, resource_guid: Sequel[instance_table][:guid]).
+                  join(:delayed_jobs, guid: Sequel[:jobs][:delayed_job_guid]).
+                  where(Sequel[operation_table][:state] => 'in progress').
+                  where(Sequel[operation_table][:type] => 'delete').
+                  where(Sequel.lit("#{operation_table}.created_at > CURRENT_TIMESTAMP - INTERVAL '?' SECOND", default_maximum_duration_seconds.to_i)).
+                  where(Sequel[:jobs][:state] => [PollableJobModel::POLLING_STATE, PollableJobModel::FAILED_STATE]).
+                  where(Sequel[:jobs][:operation] => jobs_operation).
+                  exclude(Sequel[:delayed_jobs][:failed_at] => nil).
+                  select(
+                    Sequel[:jobs][:guid].as(:pollable_guid),
+                    Sequel[operation_table][:id].as(:op_id),
+                    Sequel[operation_table][foreign_key].as(:resource_id)
+                  ).
+                  order(Sequel[operation_table][:created_at]).
+                  limit(BATCH_SIZE)
+
+          stuck.each do |row|
+            resolve_stuck(operation_model, instance_model, row[:op_id], row[:resource_id], row[:pollable_guid])
+          end
+        end
+
+        def resolve_stuck(operation_model, instance_model, op_id, resource_id, pollable_guid)
+          operation_model.db.transaction do
+            operation = operation_model.where(id: op_id, state: 'in progress').for_update.skip_locked.first
+            return unless operation
+
+            binding = instance_model.first(id: resource_id)
+            return unless binding
+
+            pollable = PollableJobModel.first(guid: pollable_guid)
+            return unless pollable
+
+            handler = deserialize_handler(pollable)
+            return unless handler
+
+            binding_type = instance_model.to_s.split('::').last
+
+            logger.info(
+              "#{binding_type} #{binding.guid} delete operation is stuck in 'in progress'. Re-enqueuing the polling job.",
+              binding_type: binding_type,
+              binding_guid: binding.guid,
+              operation_id: op_id,
+              pollable_job_guid: pollable_guid
+            )
+
+            pollable.update(state: PollableJobModel::POLLING_STATE, cf_api_error: nil)
+            Jobs::GenericEnqueuer.shared.enqueue_pollable(handler, existing_guid: pollable.guid, preserve_priority: true)
+          end
+        end
+
+        # Reuse the original delete polling job by deserializing the failed delayed_job's handler and unwrapping
+        # the wrapper chain (LoggingContextJob → TimeoutJob → PollableJobWrapper → DeleteBindingJob).
+        # This preserves the original @user_audit_info, @start_time and the binding @type.
+        def deserialize_handler(pollable)
+          delayed_job = Delayed::Job[guid: pollable.delayed_job_guid]
+          return unless delayed_job
+
+          Jobs::Enqueuer.unwrap_job(delayed_job.payload_object)
+        rescue StandardError => e
+          logger.error("Could not deserialize delayed job '#{pollable.delayed_job_guid}' for pollable '#{pollable.guid}': #{e.class}: #{e.message}")
+          nil
+        end
+
+        def default_maximum_duration_seconds
+          Config.config.get(:broker_client_max_async_poll_duration_minutes).minutes
+        end
+
+        def logger
+          @logger ||= Steno.logger('cc.background.service-operations-binding-delete-stuck-in-progress-retry')
+        end
+
+        def job_name_in_configuration
+          :service_operations_binding_delete_stuck_in_progress_retry
+        end
+      end
+    end
+  end
+end

config/cloud_controller.yml

@@ -64,6 +64,9 @@ service_operations_update_stuck_in_progress_failed:
 service_operations_delete_stuck_in_progress_retry:
   frequency_in_seconds: 3600 #1h
 
+service_operations_binding_delete_stuck_in_progress_retry:
+  frequency_in_seconds: 3600 #1h
+
 # One-off backfill - to be removed in a future version.
 lifecycle_type_backfill:
   frequency_in_seconds: 3600 #1h

lib/cloud_controller/clock/scheduler.rb

@@ -28,6 +28,7 @@ class Scheduler
       { name: 'service_operations_create_in_progress_cleanup', class: Jobs::Runtime::ServiceOperationsCreateInProgressCleanup },
       { name: 'service_operations_update_stuck_in_progress_failed', class: Jobs::Runtime::ServiceOperationsUpdateStuckInProgressFailed },
       { name: 'service_operations_delete_stuck_in_progress_retry', class: Jobs::Runtime::ServiceOperationsDeleteStuckInProgressRetry },
+      { name: 'service_operations_binding_delete_stuck_in_progress_retry', class: Jobs::Runtime::ServiceOperationsBindingDeleteStuckInProgressRetry },
       # One-off backfill - to be removed in a future version.
       { name: 'lifecycle_type_backfill', class: Jobs::Runtime::LifecycleTypeBackfill }
     ].freeze

lib/cloud_controller/config_schemas/clock_schema.rb

@@ -43,6 +43,9 @@ class ClockSchema < VCAP::Config
           service_operations_delete_stuck_in_progress_retry: {
             frequency_in_seconds: Integer
           },
+          service_operations_binding_delete_stuck_in_progress_retry: {
+            frequency_in_seconds: Integer
+          },
           # One-off backfill - to be removed in a future version.
           lifecycle_type_backfill: {
             frequency_in_seconds: Integer

lib/cloud_controller/jobs.rb

@@ -28,6 +28,7 @@
 require 'jobs/runtime/service_operations_create_in_progress_cleanup'
 require 'jobs/runtime/service_operations_update_stuck_in_progress_failed'
 require 'jobs/runtime/service_operations_delete_stuck_in_progress_retry'
+require 'jobs/runtime/service_operations_binding_delete_stuck_in_progress_retry'
 require 'jobs/runtime/failed_jobs_cleanup'
 require 'jobs/runtime/service_operations_initial_cleanup'
 require 'jobs/runtime/legacy_jobs'

spec/unit/jobs/runtime/service_operations_binding_delete_stuck_in_progress_retry_spec.rb

@@ -0,0 +1,221 @@
+require 'spec_helper'
+
+module VCAP::CloudController
+  module Jobs::Runtime
+    RSpec.describe ServiceOperationsBindingDeleteStuckInProgressRetry, job_context: :worker do
+      subject(:job) { ServiceOperationsBindingDeleteStuckInProgressRetry.new }
+
+      let(:fake_logger) { instance_double(Steno::Logger, info: nil, warn: nil, error: nil) }
+      let(:max_poll_duration_minutes) { 60 }
+      let(:user_audit_info) { UserAuditInfo.new(user_guid: create(:user).guid, user_email: 'foo@example.com') }
+      let(:enqueuer) { instance_double(Jobs::GenericEnqueuer, enqueue_pollable: nil) }
+
+      before do
+        allow(Steno).to receive(:logger).and_return(fake_logger)
+        TestConfig.override(broker_client_max_async_poll_duration_minutes: max_poll_duration_minutes)
+        allow(Jobs::GenericEnqueuer).to receive(:shared).and_return(enqueuer)
+      end
+
+      # Enqueue a real DeleteBindingJob so the delayed_job carries a genuine serialized handler,
+      # then simulate the permanent failure (failed_at set) that leaves the operation stuck in progress.
+      def prepare_stuck_binding(
+        binding_type:,
+        operation_state: 'in progress',
+        operation_type: 'delete',
+        operation_created_at: Time.now,
+        pollable_job_state: PollableJobModel::FAILED_STATE,
+        pollable_job_operation: nil,
+        delayed_job_failed_at: Time.now
+      )
+        if binding_type == :credential
+          binding = create(:service_binding)
+          create(:service_binding_operation, service_binding_id: binding.id, type: operation_type, state: operation_state, created_at: operation_created_at)
+          default_operation = 'service_bindings.delete'
+          resource_type = 'service_bindings'
+        else
+          binding = create(:service_key)
+          create(:service_key_operation, service_key_id: binding.id, type: operation_type, state: operation_state, created_at: operation_created_at)
+          default_operation = 'service_keys.delete'
+          resource_type = 'service_keys'
+        end
+
+        delete_job = V3::DeleteBindingJob.new(binding_type, binding.guid, user_audit_info: user_audit_info)
+        pjob = Jobs::Enqueuer.new(queue: Jobs::Queues.generic).enqueue_pollable(delete_job)
+        pjob.update(state: pollable_job_state, operation: pollable_job_operation || default_operation, resource_type: resource_type)
+
+        dj = Delayed::Job[guid: pjob.delayed_job_guid]
+        dj.update(failed_at: delayed_job_failed_at)
+
+        { binding: binding, pjob: pjob, delayed_job: dj }
+      end
+
+      it { is_expected.to be_a_valid_job }
+
+      %i[credential key].each do |binding_type|
+        describe "#perform for #{binding_type} bindings" do
+          shared_examples 'does not retry the operation' do
+            it 'leaves the operation in progress, the pollable job untouched, and does not re-enqueue' do
+              scenario = subject_scenario
+              original_pollable_state = scenario[:pjob].state
+              job.perform
+              expect(scenario[:binding].last_operation.reload.state).to eq('in progress')
+              expect(scenario[:pjob].reload.state).to eq(original_pollable_state)
+              expect(enqueuer).not_to have_received(:enqueue_pollable)
+            end
+          end
+
+          context 'when operation state is not in progress' do
+            it 'does not retry when state is succeeded' do
+              scenario = prepare_stuck_binding(binding_type: binding_type, operation_state: 'succeeded')
+              job.perform
+              expect(scenario[:binding].last_operation.reload.state).to eq('succeeded')
+              expect(enqueuer).not_to have_received(:enqueue_pollable)
+            end
+
+            it 'does not retry when state is failed' do
+              scenario = prepare_stuck_binding(binding_type: binding_type, operation_state: 'failed')
+              job.perform
+              expect(scenario[:binding].last_operation.reload.state).to eq('failed')
+              expect(enqueuer).not_to have_received(:enqueue_pollable)
+            end
+          end
+
+          context 'when operation type is not delete' do
+            let(:subject_scenario) do
+              prepare_stuck_binding(binding_type: binding_type, operation_type: 'create',
+                                    pollable_job_operation: binding_type == :credential ? 'service_bindings.create' : 'service_keys.create')
+            end
+
+            it_behaves_like 'does not retry the operation'
+          end
+
+          context 'when operation created_at is beyond the max polling window' do
+            let(:subject_scenario) { prepare_stuck_binding(binding_type: binding_type, operation_created_at: Time.now - (max_poll_duration_minutes + 1).minutes) }
+
+            it_behaves_like 'does not retry the operation'
+          end
+
+          context 'when delayed_job.failed_at is nil (job still running or locked)' do
+            let(:subject_scenario) { prepare_stuck_binding(binding_type: binding_type, delayed_job_failed_at: nil) }
+
+            it_behaves_like 'does not retry the operation'
+          end
+
+          context 'when pollable job state is COMPLETE' do
+            let(:subject_scenario) { prepare_stuck_binding(binding_type: binding_type, pollable_job_state: PollableJobModel::COMPLETE_STATE) }
+
+            it_behaves_like 'does not retry the operation'
+          end
+
+          context 'when pollable job state is PROCESSING' do
+            let(:subject_scenario) { prepare_stuck_binding(binding_type: binding_type, pollable_job_state: PollableJobModel::PROCESSING_STATE) }
+
+            it_behaves_like 'does not retry the operation'
+          end
+
+          context 'when pollable job operation does not match the delete operation' do
+            let(:subject_scenario) do
+              prepare_stuck_binding(binding_type: binding_type,
+                                    pollable_job_operation: binding_type == :credential ? 'service_bindings.create' : 'service_keys.create')
+            end
+
+            it_behaves_like 'does not retry the operation'
+          end
+
+          context 'when a binding delete job is stuck with state FAILED' do
+            it 'resets the pollable job to POLLING and re-enqueues the original delete job' do
+              scenario = prepare_stuck_binding(binding_type: binding_type)
+              job.perform
+
+              expect(scenario[:binding].last_operation.reload.state).to eq('in progress')
+              expect(scenario[:pjob].reload.state).to eq(PollableJobModel::POLLING_STATE)
+              expect(enqueuer).to have_received(:enqueue_pollable).with(
+                an_instance_of(V3::DeleteBindingJob),
+                hash_including(existing_guid: scenario[:pjob].guid, preserve_priority: true)
+              )
+            end
+          end
+
+          context 'when a binding delete job is stuck with state POLLING (DB flip before failure hook)' do
+            it 'resets the pollable job to POLLING and re-enqueues the original delete job' do
+              scenario = prepare_stuck_binding(binding_type: binding_type, pollable_job_state: PollableJobModel::POLLING_STATE)
+              job.perform
+
+              expect(scenario[:pjob].reload.state).to eq(PollableJobModel::POLLING_STATE)
+              expect(enqueuer).to have_received(:enqueue_pollable).with(
+                an_instance_of(V3::DeleteBindingJob),
+                hash_including(existing_guid: scenario[:pjob].guid)
+              )
+            end
+          end
+
+          context 'when there are multiple stuck jobs within the batch size' do
+            it 'retries each one' do
+              3.times { prepare_stuck_binding(binding_type: binding_type) }
+              job.perform
+              expect(enqueuer).to have_received(:enqueue_pollable).exactly(3).times
+            end
+          end
+
+          context 'when there are more stuck jobs than the batch size' do
+            it 'processes only up to BATCH_SIZE jobs per run' do
+              (ServiceOperationsBindingDeleteStuckInProgressRetry::BATCH_SIZE + 1).times { prepare_stuck_binding(binding_type: binding_type) }
+              job.perform
+              expect(enqueuer).to have_received(:enqueue_pollable).exactly(ServiceOperationsBindingDeleteStuckInProgressRetry::BATCH_SIZE).times
+            end
+          end
+        end
+      end
+
+      describe '#perform cross-type isolation' do
+        it 'retries both a stuck credential-binding delete and a stuck key delete' do
+          prepare_stuck_binding(binding_type: :credential)
+          prepare_stuck_binding(binding_type: :key)
+          job.perform
+          expect(enqueuer).to have_received(:enqueue_pollable).exactly(2).times
+        end
+      end
+
+      describe '#resolve_stuck' do
+        context 'when another process already resolved it (skip_locked returns nil)' do
+          it 'does nothing and does not re-enqueue' do
+            scenario = prepare_stuck_binding(binding_type: :credential)
+
+            expect do
+              job.send(:resolve_stuck, ServiceBindingOperation, ServiceBinding,
+                       -1, scenario[:binding].id, scenario[:pjob].guid)
+            end.not_to raise_error
+            expect(scenario[:pjob].reload.state).to eq(PollableJobModel::FAILED_STATE)
+            expect(enqueuer).not_to have_received(:enqueue_pollable)
+          end
+        end
+
+        context 'when the delayed job handler cannot be deserialized' do
+          it 'does not re-enqueue and leaves the pollable job untouched' do
+            scenario = prepare_stuck_binding(binding_type: :credential)
+            Delayed::Job[guid: scenario[:pjob].delayed_job_guid].update(handler: 'not-valid-yaml: ]')
+            op = scenario[:binding].last_operation
+
+            job.send(:resolve_stuck, ServiceBindingOperation, ServiceBinding,
+                     op.id, scenario[:binding].id, scenario[:pjob].guid)
+
+            expect(scenario[:pjob].reload.state).to eq(PollableJobModel::FAILED_STATE)
+            expect(enqueuer).not_to have_received(:enqueue_pollable)
+          end
+        end
+
+        context 'when the operation is stuck in progress' do
+          it 'resets the pollable job from its failed state to POLLING' do
+            scenario = prepare_stuck_binding(binding_type: :credential)
+            op = scenario[:binding].last_operation
+
+            expect do
+              job.send(:resolve_stuck, ServiceBindingOperation, ServiceBinding,
+                       op.id, scenario[:binding].id, scenario[:pjob].guid)
+            end.to change { scenario[:pjob].reload.state }.from(PollableJobModel::FAILED_STATE).to(PollableJobModel::POLLING_STATE)
+          end
+        end
+      end
+    end
+  end
+end

spec/unit/lib/cloud_controller/clock/scheduler_spec.rb

@@ -24,6 +24,7 @@ module VCAP::CloudController
           service_operations_create_in_progress_cleanup: { frequency_in_seconds: 600 },
           service_operations_update_stuck_in_progress_failed: { frequency_in_seconds: 600 },
           service_operations_delete_stuck_in_progress_retry: { frequency_in_seconds: 600 },
+          service_operations_binding_delete_stuck_in_progress_retry: { frequency_in_seconds: 600 },
           lifecycle_type_backfill: { frequency_in_seconds: 500 },
           service_usage_events: { cutoff_age_in_days: 5 },
           completed_tasks: { cutoff_age_in_days: 6 },
@@ -183,6 +184,12 @@ module VCAP::CloudController
           expect(block.call).to be_instance_of(Jobs::Runtime::ServiceOperationsDeleteStuckInProgressRetry)
         end
 
+        expect(clock).to receive(:schedule_frequent_worker_job) do |args, &block|
+          expect(args).to eql(name: 'service_operations_binding_delete_stuck_in_progress_retry', interval: 600)
+          expect(Jobs::Runtime::ServiceOperationsBindingDeleteStuckInProgressRetry).to receive(:new).and_call_original
+          expect(block.call).to be_instance_of(Jobs::Runtime::ServiceOperationsBindingDeleteStuckInProgressRetry)
+        end
+
         expect(clock).to receive(:schedule_frequent_worker_job) do |args, &block|
           expect(args).to eql(name: 'lifecycle_type_backfill', interval: 500)
           expect(Jobs::Runtime::LifecycleTypeBackfill).to receive(:new).and_call_original