Add lifecycle_type backfill clock job and rake task (#5196)

A one-off clock job that backfills the lifecycle_type column on apps,
droplets, and builds for rows that pre-date the column's introduction.
Once all installations have run it long enough to drain those rows, this
job will be removed.

The job processes up to 1000 (BATCH_SIZE) * 10 (BATCHES_PER_RUN) rows
per table in short transactions, idempotent and bounded. It is defensive
against missing columns: schema(:table) is checked on each invocation,
so it would be safe to be shipped together with the migration that
introduces the column.

Operators can also run `rake db:lifecycle_type_backfill[batch_size,
batches_per_run]` manually; passing -1 for batches_per_run drains until
no NULL rows remain.

Author: philippthun

app/jobs/runtime/lifecycle_type_backfill.rb

@@ -0,0 +1,98 @@
+# NOTE: This is a one-off backfill job. It populates the `lifecycle_type`
+# column on `apps`, `droplets`, and `builds` for rows that pre-date the
+# column's introduction. Once all installations have run it long enough to
+# drain those rows, this job will be removed.
+#
+# Operators can also run `rake db:lifecycle_type_backfill` manually.
+
+module VCAP::CloudController
+  module Jobs
+    module Runtime
+      class LifecycleTypeBackfill < VCAP::CloudController::Jobs::CCJob
+        BATCH_SIZE      = 1000
+        BATCHES_PER_RUN = 10
+
+        TABLES = [
+          { table: :apps,     guid_column: :app_guid     },
+          { table: :droplets, guid_column: :droplet_guid },
+          { table: :builds,   guid_column: :build_guid   }
+        ].freeze
+
+        # Pass -1 for +batches_per_run+ to drain until no rows remain.
+        def initialize(batch_size: BATCH_SIZE, batches_per_run: BATCHES_PER_RUN)
+          super()
+          @batch_size      = batch_size
+          @batches_per_run = batches_per_run
+        end
+
+        def perform
+          TABLES.each { |t| backfill(**t) }
+        end
+
+        def job_name_in_configuration
+          :lifecycle_type_backfill
+        end
+
+        def max_attempts
+          1
+        end
+
+        private
+
+        def backfill(table:, guid_column:)
+          return unless column_exists?(table, :lifecycle_type)
+
+          total_rows = 0
+          remaining_batches = @batches_per_run
+          while remaining_batches != 0 # -1 means: drain until no rows remain
+            updated_rows = update_batch(table, guid_column)
+            total_rows += updated_rows
+            break if updated_rows < @batch_size
+
+            remaining_batches -= 1 if remaining_batches > 0
+          end
+          logger.info("lifecycle_type_backfill: updated #{total_rows} rows in #{table}") if total_rows > 0
+        end
+
+        def update_batch(table, guid_column)
+          guids = db[table].where(lifecycle_type: nil).limit(@batch_size).select_map(:guid)
+          return 0 if guids.empty?
+
+          # If a row appears in both *_lifecycle_data tables (which it shouldn't), buildpack wins
+          # (matches the runtime fallback in {app,build,droplet}_model.rb#lifecycle_type).
+          guids_with_buildpack_lifecycle_data = db[:buildpack_lifecycle_data].where(guid_column => guids).select_map(guid_column)
+          guids_with_cnb_lifecycle_data       = db[:cnb_lifecycle_data].where(guid_column => guids).select_map(guid_column) - guids_with_buildpack_lifecycle_data
+          guids_without_lifecycle_data        = guids - guids_with_buildpack_lifecycle_data - guids_with_cnb_lifecycle_data
+
+          db.transaction do
+            update_lifecycle(table, guids_with_buildpack_lifecycle_data, BuildpackLifecycleDataModel::LIFECYCLE_TYPE)
+            update_lifecycle(table, guids_with_cnb_lifecycle_data,       CNBLifecycleDataModel::LIFECYCLE_TYPE)
+            update_lifecycle(table, guids_without_lifecycle_data,        DockerLifecycleDataModel::LIFECYCLE_TYPE)
+          end
+
+          guids.size
+        end
+
+        def update_lifecycle(table, guids, value)
+          return if guids.empty?
+
+          db[table].where(guid: guids, lifecycle_type: nil).update(lifecycle_type: value)
+        end
+
+        def column_exists?(table, column)
+          db.schema(table, reload: true).map(&:first).include?(column)
+        rescue Sequel::Error
+          false
+        end
+
+        def db
+          Sequel::Model.db
+        end
+
+        def logger
+          @logger ||= Steno.logger('cc.background.lifecycle-type-backfill')
+        end
+      end
+    end
+  end
+end

config/cloud_controller.yml

@@ -58,6 +58,10 @@ service_operations_initial_cleanup:
 service_operations_create_in_progress_cleanup:
   frequency_in_seconds: 3600 #1h
 
+# One-off backfill - to be removed in a future version.
+lifecycle_type_backfill:
+  frequency_in_seconds: 3600 #1h
+
 completed_tasks:
   cutoff_age_in_days: 31
 

lib/cloud_controller/clock/scheduler.rb

@@ -25,7 +25,9 @@ class Scheduler
       { name: 'pending_builds', class: Jobs::Runtime::PendingBuildCleanup },
       { name: 'failed_jobs', class: Jobs::Runtime::FailedJobsCleanup },
       { name: 'service_operations_initial_cleanup', class: Jobs::Runtime::ServiceOperationsInitialCleanup },
-      { name: 'service_operations_create_in_progress_cleanup', class: Jobs::Runtime::ServiceOperationsCreateInProgressCleanup }
+      { name: 'service_operations_create_in_progress_cleanup', class: Jobs::Runtime::ServiceOperationsCreateInProgressCleanup },
+      # One-off backfill - to be removed in a future version.
+      { name: 'lifecycle_type_backfill', class: Jobs::Runtime::LifecycleTypeBackfill }
     ].freeze
 
     def initialize(config)

lib/cloud_controller/config_schemas/clock_schema.rb

@@ -37,6 +37,10 @@ class ClockSchema < VCAP::Config
           service_operations_create_in_progress_cleanup: {
             frequency_in_seconds: Integer
           },
+          # One-off backfill - to be removed in a future version.
+          lifecycle_type_backfill: {
+            frequency_in_seconds: Integer
+          },
           default_health_check_timeout: Integer,
 
           uaa: {

lib/cloud_controller/jobs.rb

@@ -38,6 +38,7 @@
 require 'jobs/runtime/prune_completed_deployments'
 require 'jobs/runtime/prune_completed_builds'
 require 'jobs/runtime/prune_excess_app_revisions'
+require 'jobs/runtime/lifecycle_type_backfill'
 
 require 'jobs/v2/services/service_usage_events_cleanup'
 

lib/tasks/db.rake

@@ -192,6 +192,27 @@ namespace :db do
     VCAP::BigintMigration.backfill(logger, db, args.table.to_sym, batch_size: args.batch_size.to_i, iterations: args.iterations.to_i)
   end
 
+  # One-off backfill - to be removed in a future version.
+  desc 'Backfill lifecycle_type column on apps, droplets, and builds (pass -1 for batches_per_run to drain)'
+  task :lifecycle_type_backfill, %i[batch_size batches_per_run] => :environment do |_t, args|
+    args.with_defaults(batch_size: 1_000, batches_per_run: 10)
+
+    RakeConfig.context = :api
+
+    batch_size      = args.batch_size.to_i
+    batches_per_run = args.batches_per_run.to_i
+    BackgroundJobEnvironment.new(RakeConfig.config).setup_environment do
+      # Ensure we always log to stdout (regardless of `stdout_sink_enabled`).
+      VCAP::CloudController::StenoConfigurer.new(RakeConfig.config.get(:logging)).configure do |steno_config_hash|
+        steno_config_hash[:sinks] << Steno::Sink::IO.new($stdout)
+      end
+      logger = Steno.logger('cc.db.lifecycle_type_backfill')
+      logger.info("starting lifecycle_type backfill (batch_size: #{batch_size}, batches_per_run: #{batches_per_run})")
+      VCAP::CloudController::Jobs::Runtime::LifecycleTypeBackfill.new(batch_size:, batches_per_run:).perform
+      logger.info('finished lifecycle_type backfill')
+    end
+  end
+
   namespace :dev do
     desc 'Migrate the database set in spec/support/bootstrap/db_config'
     task migrate: :environment do

spec/unit/jobs/runtime/lifecycle_type_backfill_spec.rb

@@ -0,0 +1,184 @@
+require 'spec_helper'
+
+module VCAP::CloudController
+  module Jobs::Runtime
+    RSpec.describe LifecycleTypeBackfill, job_context: :worker do
+      subject(:job) { LifecycleTypeBackfill.new }
+
+      let(:db) { Sequel::Model.db }
+
+      it { is_expected.to be_a_valid_job }
+
+      it 'knows its job name' do
+        expect(job.job_name_in_configuration).to eq(:lifecycle_type_backfill)
+      end
+
+      it 'has max_attempts of 1' do
+        expect(job.max_attempts).to eq(1)
+      end
+
+      describe '#perform' do
+        context 'when the lifecycle_type column is missing on every table' do
+          let!(:app) { AppModel.make }
+          let!(:droplet) { DropletModel.make }
+          let!(:build) { BuildModel.make }
+
+          before do
+            db[:apps].where(guid: app.guid).update(lifecycle_type: nil)
+            db[:droplets].where(guid: droplet.guid).update(lifecycle_type: nil)
+            db[:builds].where(guid: build.guid).update(lifecycle_type: nil)
+            allow(db).to receive(:schema).and_call_original
+            %i[apps droplets builds].each do |table|
+              allow(db).to receive(:schema).with(table, reload: true).and_return(
+                [[:guid, {}], [:name, {}]]
+              )
+            end
+          end
+
+          it 'does not issue any UPDATE statements' do
+            expect { job.perform }.to have_queried_db_times(/update .(apps|droplets|builds). set/i, 0)
+          end
+
+          it 'leaves NULL rows untouched' do
+            job.perform
+            expect(db[:apps].where(guid: app.guid).get(:lifecycle_type)).to be_nil
+            expect(db[:droplets].where(guid: droplet.guid).get(:lifecycle_type)).to be_nil
+            expect(db[:builds].where(guid: build.guid).get(:lifecycle_type)).to be_nil
+          end
+        end
+
+        context 'when no rows have NULL lifecycle_type on any table' do
+          before do
+            AppModel.make
+            DropletModel.make
+            BuildModel.make
+          end
+
+          it 'does not issue any UPDATE statements' do
+            expect { job.perform }.to have_queried_db_times(/update .(apps|droplets|builds). set/i, 0)
+          end
+        end
+
+        context 'when there are apps with NULL lifecycle_type' do
+          let(:buildpack_app) { AppModel.make }
+          let(:cnb_app) { AppModel.make(:cnb) }
+          let(:docker_app) { AppModel.make(:docker) }
+
+          before do
+            db[:apps].where(guid: [buildpack_app.guid, cnb_app.guid, docker_app.guid]).update(lifecycle_type: nil)
+          end
+
+          it 'sets lifecycle_type accordingly' do
+            job.perform
+            expect(db[:apps].where(guid: buildpack_app.guid).get(:lifecycle_type)).to eq(BuildpackLifecycleDataModel::LIFECYCLE_TYPE)
+            expect(db[:apps].where(guid: cnb_app.guid).get(:lifecycle_type)).to eq(CNBLifecycleDataModel::LIFECYCLE_TYPE)
+            expect(db[:apps].where(guid: docker_app.guid).get(:lifecycle_type)).to eq(DockerLifecycleDataModel::LIFECYCLE_TYPE)
+          end
+
+          it 'does not touch updated_at' do
+            original_updated_at = db[:apps].where(guid: [buildpack_app.guid, cnb_app.guid, docker_app.guid]).select_map(%i[guid updated_at]).to_h
+            job.perform
+            expect(db[:apps].where(guid: [buildpack_app.guid, cnb_app.guid, docker_app.guid]).select_map(%i[guid updated_at]).to_h).to eq(original_updated_at)
+          end
+        end
+
+        context 'when there are droplets with NULL lifecycle_type' do
+          let(:buildpack_droplet) { DropletModel.make }
+          let(:cnb_droplet) { DropletModel.make(:cnb) }
+          let(:docker_droplet) { DropletModel.make(:docker) }
+
+          before do
+            db[:droplets].where(guid: [buildpack_droplet.guid, cnb_droplet.guid, docker_droplet.guid]).update(lifecycle_type: nil)
+          end
+
+          it 'sets lifecycle_type accordingly' do
+            job.perform
+            expect(db[:droplets].where(guid: buildpack_droplet.guid).get(:lifecycle_type)).to eq(BuildpackLifecycleDataModel::LIFECYCLE_TYPE)
+            expect(db[:droplets].where(guid: cnb_droplet.guid).get(:lifecycle_type)).to eq(CNBLifecycleDataModel::LIFECYCLE_TYPE)
+            expect(db[:droplets].where(guid: docker_droplet.guid).get(:lifecycle_type)).to eq(DockerLifecycleDataModel::LIFECYCLE_TYPE)
+          end
+
+          it 'does not touch updated_at' do
+            original_updated_at = db[:droplets].where(guid: [buildpack_droplet.guid, cnb_droplet.guid, docker_droplet.guid]).select_map(%i[guid updated_at]).to_h
+            job.perform
+            expect(db[:droplets].where(guid: [buildpack_droplet.guid, cnb_droplet.guid, docker_droplet.guid]).select_map(%i[guid updated_at]).to_h).to eq(original_updated_at)
+          end
+        end
+
+        context 'when there are builds with NULL lifecycle_type' do
+          let(:buildpack_build) { BuildModel.make }
+          let(:cnb_build) { BuildModel.make(:cnb) }
+          let(:docker_build) { BuildModel.make(:docker) }
+
+          before do
+            db[:builds].where(guid: [buildpack_build.guid, cnb_build.guid, docker_build.guid]).update(lifecycle_type: nil)
+          end
+
+          it 'sets lifecycle_type accordingly' do
+            job.perform
+            expect(db[:builds].where(guid: buildpack_build.guid).get(:lifecycle_type)).to eq(BuildpackLifecycleDataModel::LIFECYCLE_TYPE)
+            expect(db[:builds].where(guid: cnb_build.guid).get(:lifecycle_type)).to eq(CNBLifecycleDataModel::LIFECYCLE_TYPE)
+            expect(db[:builds].where(guid: docker_build.guid).get(:lifecycle_type)).to eq(DockerLifecycleDataModel::LIFECYCLE_TYPE)
+          end
+
+          it 'does not touch updated_at' do
+            original_updated_at = db[:builds].where(guid: [buildpack_build.guid, cnb_build.guid, docker_build.guid]).select_map(%i[guid updated_at]).to_h
+            job.perform
+            expect(db[:builds].where(guid: [buildpack_build.guid, cnb_build.guid, docker_build.guid]).select_map(%i[guid updated_at]).to_h).to eq(original_updated_at)
+          end
+        end
+
+        context 'with more rows than batch_size * batches_per_run' do
+          subject(:job) { LifecycleTypeBackfill.new(batch_size: 2, batches_per_run: 2) }
+
+          before do
+            5.times { AppModel.make }
+            db[:apps].update(lifecycle_type: nil)
+          end
+
+          it 'updates at most batch_size * batches_per_run rows in a single perform' do
+            expect { job.perform }.to change { db[:apps].where(lifecycle_type: nil).count }.from(5).to(1)
+          end
+
+          it 'processes the remainder on the next perform' do
+            job.perform
+            job.perform
+            expect(db[:apps].where(lifecycle_type: nil).count).to eq(0)
+          end
+        end
+
+        context 'with fewer rows than batch_size' do
+          subject(:job) { LifecycleTypeBackfill.new(batch_size: 2, batches_per_run: 2) }
+
+          before do
+            AppModel.make
+            db[:apps].update(lifecycle_type: nil)
+          end
+
+          it 'updates every NULL row in a single perform' do
+            job.perform
+            expect(db[:apps].where(lifecycle_type: nil).count).to eq(0)
+          end
+
+          it 'issues exactly one SELECT for guids, subsequent batch is skipped' do
+            expect { job.perform }.to have_queried_db_times(/select .guid. from .apps. where \(.lifecycle_type. is null\)/i, 1)
+          end
+        end
+
+        context 'when batches_per_run is -1 (drain mode)' do
+          subject(:job) { LifecycleTypeBackfill.new(batch_size: 2, batches_per_run: -1) }
+
+          before do
+            5.times { AppModel.make }
+            db[:apps].update(lifecycle_type: nil)
+          end
+
+          it 'keeps batching until no NULL rows remain' do
+            job.perform
+            expect(db[:apps].where(lifecycle_type: nil).count).to eq(0)
+          end
+        end
+      end
+    end
+  end
+end

spec/unit/lib/cloud_controller/clock/scheduler_spec.rb

@@ -22,6 +22,7 @@ module VCAP::CloudController
           pollable_jobs: { cutoff_age_in_days: 2 },
           service_operations_initial_cleanup: { frequency_in_seconds: 600 },
           service_operations_create_in_progress_cleanup: { frequency_in_seconds: 600 },
+          lifecycle_type_backfill: { frequency_in_seconds: 500 },
           service_usage_events: { cutoff_age_in_days: 5 },
           completed_tasks: { cutoff_age_in_days: 6 },
           pending_droplets: { frequency_in_seconds: 300, expiration_in_seconds: 600 },
@@ -168,6 +169,12 @@ module VCAP::CloudController
           expect(block.call).to be_instance_of(Jobs::Runtime::ServiceOperationsCreateInProgressCleanup)
         end
 
+        expect(clock).to receive(:schedule_frequent_worker_job) do |args, &block|
+          expect(args).to eql(name: 'lifecycle_type_backfill', interval: 500)
+          expect(Jobs::Runtime::LifecycleTypeBackfill).to receive(:new).and_call_original
+          expect(block.call).to be_instance_of(Jobs::Runtime::LifecycleTypeBackfill)
+        end
+
         schedule.start
       end