|
| 1 | +require 'spec_helper' |
| 2 | + |
| 3 | +module VCAP::CloudController |
| 4 | + module Jobs::Runtime |
| 5 | + RSpec.describe ServiceOperationsUpdateInProgressCleanup, job_context: :worker do |
| 6 | + subject(:job) { ServiceOperationsUpdateInProgressCleanup.new } |
| 7 | + |
| 8 | + let(:fake_logger) { instance_double(Steno::Logger, info: nil, warn: nil) } |
| 9 | + let(:max_poll_duration_minutes) { 60 } |
| 10 | + |
| 11 | + before do |
| 12 | + allow(Steno).to receive(:logger).and_return(fake_logger) |
| 13 | + TestConfig.override(broker_client_max_async_poll_duration_minutes: max_poll_duration_minutes) |
| 14 | + end |
| 15 | + |
| 16 | + def prepare_stuck_service_instance( |
| 17 | + service_instance_state: 'in progress', |
| 18 | + service_instance_type: 'update', |
| 19 | + service_instance_created_at: Time.now, |
| 20 | + pollable_job_state: PollableJobModel::FAILED_STATE, |
| 21 | + pollable_job_operation: 'service_instance.update', |
| 22 | + delayed_job_failed_at: Time.now |
| 23 | + ) |
| 24 | + service_instance = create(:managed_service_instance) |
| 25 | + |
| 26 | + create(:service_instance_operation, |
| 27 | + service_instance_id: service_instance.id, |
| 28 | + type: service_instance_type, |
| 29 | + state: service_instance_state, |
| 30 | + created_at: service_instance_created_at) |
| 31 | + |
| 32 | + dj = Delayed::Job.create!( |
| 33 | + guid: SecureRandom.uuid, |
| 34 | + handler: 'fake', |
| 35 | + run_at: Time.now, |
| 36 | + failed_at: delayed_job_failed_at, |
| 37 | + queue: 'cc-generic' |
| 38 | + ) |
| 39 | + |
| 40 | + pjob = create(:pollable_job_model, |
| 41 | + state: pollable_job_state, |
| 42 | + operation: pollable_job_operation, |
| 43 | + resource_guid: service_instance.guid, |
| 44 | + resource_type: 'service_instances', |
| 45 | + delayed_job_guid: dj.guid) |
| 46 | + |
| 47 | + { service_instance: service_instance, pjob: pjob, delayed_job: dj } |
| 48 | + end |
| 49 | + |
| 50 | + shared_examples 'does not resolve the operation' do |
| 51 | + it 'leaves the operation in progress and the pollable job untouched' do |
| 52 | + scenario = subject_scenario |
| 53 | + job.perform |
| 54 | + expect(scenario[:service_instance].last_operation.reload.state).to eq('in progress') |
| 55 | + expect(scenario[:pjob].reload.state).to eq(scenario[:pjob].state) |
| 56 | + end |
| 57 | + end |
| 58 | + |
| 59 | + it { is_expected.to be_a_valid_job } |
| 60 | + |
| 61 | + describe '#perform' do |
| 62 | + context 'when sio state is not in progress' do |
| 63 | + it 'does not resolve when state is succeeded' do |
| 64 | + scenario = prepare_stuck_service_instance(service_instance_state: 'succeeded') |
| 65 | + job.perform |
| 66 | + expect(scenario[:service_instance].last_operation.reload.state).to eq('succeeded') |
| 67 | + end |
| 68 | + |
| 69 | + it 'does not resolve when state is failed' do |
| 70 | + scenario = prepare_stuck_service_instance(service_instance_state: 'failed') |
| 71 | + job.perform |
| 72 | + expect(scenario[:service_instance].last_operation.reload.state).to eq('failed') |
| 73 | + end |
| 74 | + end |
| 75 | + |
| 76 | + context 'when sio type is not update' do |
| 77 | + let(:subject_scenario) { prepare_stuck_service_instance(service_instance_type: 'create') } |
| 78 | + |
| 79 | + it_behaves_like 'does not resolve the operation' |
| 80 | + end |
| 81 | + |
| 82 | + context 'when sio created_at is beyond the max polling window' do |
| 83 | + let(:subject_scenario) { prepare_stuck_service_instance(service_instance_created_at: Time.now - (max_poll_duration_minutes + 1).minutes) } |
| 84 | + |
| 85 | + it_behaves_like 'does not resolve the operation' |
| 86 | + end |
| 87 | + |
| 88 | + context 'when delayed_job.failed_at is nil (job still running or locked)' do |
| 89 | + let(:subject_scenario) { prepare_stuck_service_instance(delayed_job_failed_at: nil) } |
| 90 | + |
| 91 | + it_behaves_like 'does not resolve the operation' |
| 92 | + end |
| 93 | + |
| 94 | + context 'when pollable job state is COMPLETE' do |
| 95 | + let(:subject_scenario) { prepare_stuck_service_instance(pollable_job_state: PollableJobModel::COMPLETE_STATE) } |
| 96 | + |
| 97 | + it_behaves_like 'does not resolve the operation' |
| 98 | + end |
| 99 | + |
| 100 | + context 'when pollable job state is PROCESSING' do |
| 101 | + let(:subject_scenario) { prepare_stuck_service_instance(pollable_job_state: PollableJobModel::PROCESSING_STATE) } |
| 102 | + |
| 103 | + it_behaves_like 'does not resolve the operation' |
| 104 | + end |
| 105 | + |
| 106 | + context 'when pollable job operation is not service_instance.update' do |
| 107 | + let(:subject_scenario) { prepare_stuck_service_instance(pollable_job_operation: 'service_instance.create') } |
| 108 | + |
| 109 | + it_behaves_like 'does not resolve the operation' |
| 110 | + end |
| 111 | + |
| 112 | + context 'when a service instance update job is stuck with state FAILED' do |
| 113 | + it 'sets operation to failed and pollable job to FAILED' do |
| 114 | + scenario = prepare_stuck_service_instance |
| 115 | + job.perform |
| 116 | + expect(scenario[:service_instance].last_operation.reload.state).to eq('failed') |
| 117 | + expect(scenario[:pjob].reload.state).to eq(PollableJobModel::FAILED_STATE) |
| 118 | + end |
| 119 | + end |
| 120 | + |
| 121 | + context 'when a service instance update job is stuck with state POLLING (DB flip before failure hook)' do |
| 122 | + it 'sets operation to failed and pollable job to FAILED' do |
| 123 | + scenario = prepare_stuck_service_instance(pollable_job_state: PollableJobModel::POLLING_STATE) |
| 124 | + job.perform |
| 125 | + expect(scenario[:service_instance].last_operation.reload.state).to eq('failed') |
| 126 | + expect(scenario[:pjob].reload.state).to eq(PollableJobModel::FAILED_STATE) |
| 127 | + end |
| 128 | + end |
| 129 | + |
| 130 | + context 'when there are multiple stuck jobs within the batch size' do |
| 131 | + it 'resolves each one' do |
| 132 | + 3.times { prepare_stuck_service_instance } |
| 133 | + job.perform |
| 134 | + expect(ServiceInstanceOperation.where(state: 'failed').count).to eq(3) |
| 135 | + end |
| 136 | + end |
| 137 | + |
| 138 | + context 'when there are more stuck jobs than the batch size' do |
| 139 | + it 'processes only up to BATCH_SIZE jobs per run' do |
| 140 | + (ServiceOperationsUpdateInProgressCleanup::BATCH_SIZE + 1).times { prepare_stuck_service_instance } |
| 141 | + job.perform |
| 142 | + expect(ServiceInstanceOperation.where(state: 'failed').count).to eq(ServiceOperationsUpdateInProgressCleanup::BATCH_SIZE) |
| 143 | + end |
| 144 | + end |
| 145 | + end |
| 146 | + |
| 147 | + describe '#resolve_stuck' do |
| 148 | + context 'when another process already resolved it (skip_locked returns nil)' do |
| 149 | + it 'does nothing' do |
| 150 | + scenario = prepare_stuck_service_instance |
| 151 | + |
| 152 | + expect do |
| 153 | + job.send(:resolve_stuck, ServiceInstanceOperation, ServiceInstance, |
| 154 | + -1, scenario[:service_instance].id, scenario[:pjob].guid) |
| 155 | + end.not_to raise_error |
| 156 | + expect(scenario[:service_instance].last_operation.reload.state).to eq('in progress') |
| 157 | + end |
| 158 | + end |
| 159 | + |
| 160 | + context 'when the operation is stuck in progress' do |
| 161 | + it 'sets the operation state from in progress to failed' do |
| 162 | + scenario = prepare_stuck_service_instance |
| 163 | + op = scenario[:service_instance].last_operation |
| 164 | + |
| 165 | + expect do |
| 166 | + job.send(:resolve_stuck, ServiceInstanceOperation, ServiceInstance, |
| 167 | + op.id, scenario[:service_instance].id, scenario[:pjob].guid) |
| 168 | + end.to change { op.reload.state }.from('in progress').to('failed') |
| 169 | + end |
| 170 | + |
| 171 | + it 'sets the pollable job state to FAILED' do |
| 172 | + scenario = prepare_stuck_service_instance(pollable_job_state: PollableJobModel::POLLING_STATE) |
| 173 | + op = scenario[:service_instance].last_operation |
| 174 | + |
| 175 | + expect do |
| 176 | + job.send(:resolve_stuck, ServiceInstanceOperation, ServiceInstance, |
| 177 | + op.id, scenario[:service_instance].id, scenario[:pjob].guid) |
| 178 | + end.to change { scenario[:pjob].reload.state }.from(PollableJobModel::POLLING_STATE).to(PollableJobModel::FAILED_STATE) |
| 179 | + end |
| 180 | + end |
| 181 | + end |
| 182 | + end |
| 183 | + end |
| 184 | +end |
0 commit comments