Disable MySQL TLS peer verification when no ca_cert is set (#683)

MariaDB Connector/C 3.4.x (shipped since capi-release 1.242.0) verifies
the server certificate by default. When Cloud Controller connects to a
MySQL database without ccdb.ca_cert - as use-external-dbs.yml does after
removing the CA, e.g. against GCP Cloud SQL's self-signed CA - there is
no CA to verify against, so the connection fails with "unable to get
local issuer certificate".

Set MARIADB_TLS_DISABLE_PEER_VERIFICATION=1 for every path that opens a
ccdb connection, but only when the scheme is mysql and no ca_cert is
configured:
- the bpm environment of the long-running CC processes
  (cloud_controller_ng and its local_worker, cloud_controller_worker,
  cloud_controller_clock, cc_deployment_updater);
- the cloud_controller_ng pre-start hook, which runs DB migrations,
  seeding and encryption-key validation outside bpm (chpst preserves the
  exported variable for those scripts);
- the rotate_cc_database_key and blobstore_benchmark errands, which boot
  CC and connect to the database.

This restores the pre-3.4.x encrypt-without-verify behavior for that
path. When a ca_cert is set, verification stays on. The variable is only
set for MySQL; Postgres uses libpq and is unaffected.

Add template rendering tests covering the three paths (mysql without
ca_cert, mysql with ca_cert, postgres) for the bpm templates of all four
jobs, the cloud_controller_ng pre-start hook, and the
rotate_cc_database_key and blobstore_benchmark errands.

Author: philippthun

jobs/blobstore_benchmark/templates/bin/run.erb

@@ -11,6 +11,17 @@ export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
 
 perform_blobstore_benchmarks() {
   export CLOUD_CONTROLLER_NG_CONFIG=/var/vcap/jobs/blobstore_benchmark/config/cloud_controller_ng.yml
+<%
+  # This errand boots CCNG and opens a ccdb connection. Disable MySQL TLS peer
+  # verification when no ccdb.ca_cert is set. See the cloud_controller_ng job's
+  # config/bpm.yml template for the rationale. DB config comes from the
+  # cloud_controller_db link.
+  ca_cert_configured = false
+  link("cloud_controller_db").if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  if link("cloud_controller_db").p("ccdb.db_scheme") == "mysql" && !ca_cert_configured
+%>
+  export MARIADB_TLS_DISABLE_PEER_VERIFICATION="1"
+<% end %>
   source "${JOB_DIR}/bin/ruby_version.sh"
   cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng
 

jobs/cc_deployment_updater/templates/bpm.yml.erb

@@ -9,3 +9,12 @@ processes:
     LANG: en_US.UTF-8
     LIBRARY_PATH: /var/vcap/packages/libpq/lib
     NEWRELIC_ENABLE: false
+<%
+  # Disable MySQL TLS peer verification when no ccdb.ca_cert is set. See the
+  # cloud_controller_ng job for the rationale.
+  ca_cert_configured = false
+  if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  if p("ccdb.db_scheme") == "mysql" && !ca_cert_configured
+%>
+    MARIADB_TLS_DISABLE_PEER_VERIFICATION: "1"
+<% end %>

jobs/cloud_controller_clock/templates/bpm.yml.erb

@@ -9,3 +9,12 @@ processes:
     LANG: en_US.UTF-8
     LIBRARY_PATH: /var/vcap/packages/libpq/lib
     NEWRELIC_ENABLE: false
+<%
+  # Disable MySQL TLS peer verification when no ccdb.ca_cert is set. See the
+  # cloud_controller_ng job for the rationale.
+  ca_cert_configured = false
+  if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  if p("ccdb.db_scheme") == "mysql" && !ca_cert_configured
+%>
+    MARIADB_TLS_DISABLE_PEER_VERIFICATION: "1"
+<% end %>

jobs/cloud_controller_ng/templates/bpm.yml.erb

@@ -20,6 +20,21 @@ def mount_valkey_volume!(config)
   }
 end
 
+# MariaDB Connector/C 3.4.x verifies the server certificate by default. When
+# CC talks to a MySQL database without a ccdb.ca_cert (e.g. use-external-dbs.yml
+# strips it, as with GCP Cloud SQL's self-signed CA), there is no CA to verify
+# against, so the connection can only fail. Disable peer verification in that
+# case to restore the pre-3.4.x encrypt-without-verify behavior. When a ca_cert
+# is configured, verification stays on. Only set for MySQL; Postgres uses libpq,
+# which this variable does not affect.
+def disable_mysql_tls_peer_verification!(config)
+  return unless p("ccdb.db_scheme") == "mysql"
+  ca_cert_configured = false
+  if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  return if ca_cert_configured
+  config["env"]["MARIADB_TLS_DISABLE_PEER_VERIFICATION"] = "1"
+end
+
 cloud_controller_ng_config = {
   "name" => "cloud_controller_ng",
   "executable" => "/var/vcap/jobs/cloud_controller_ng/bin/cloud_controller_ng",
@@ -59,6 +74,7 @@ if properties.env
 end
 mount_nfs_volume!(cloud_controller_ng_config)
 mount_valkey_volume!(cloud_controller_ng_config)
+disable_mysql_tls_peer_verification!(cloud_controller_ng_config)
 
 nginx_config = {
   "name" => "nginx",
@@ -114,12 +130,12 @@ config = {
     }
   }
   mount_nfs_volume!(local_worker_config)
+  disable_mysql_tls_peer_verification!(local_worker_config)
 
   if !!properties.cc.newrelic.license_key
    local_worker_config["env"]["NEWRELIC_ENABLE"] = true
   end
 
-
   config["processes"] << local_worker_config
 end
 

jobs/cloud_controller_ng/templates/pre-start.sh.erb

@@ -10,6 +10,19 @@ CC_JOB_DIR="/var/vcap/jobs/cloud_controller_ng"
 CONFIG_DIR="${CC_JOB_DIR}/config"
 export CLOUD_CONTROLLER_NG_CONFIG="${CONFIG_DIR}/cloud_controller_ng.yml"
 
+<%
+  # DB migrations, seeding and encryption-key validation run here in pre-start
+  # (outside bpm) and open a ccdb connection. Disable MySQL TLS peer
+  # verification when no ccdb.ca_cert is set. See the config/bpm.yml template
+  # for the rationale. chpst preserves this exported env for the migrate/seed/
+  # validate scripts.
+  ca_cert_configured = false
+  if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  if p("ccdb.db_scheme") == "mysql" && !ca_cert_configured
+%>
+export MARIADB_TLS_DISABLE_PEER_VERIFICATION="1"
+<% end %>
+
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 source "${SCRIPT_DIR}/ruby_version.sh"
 

jobs/cloud_controller_worker/templates/bpm.yml.erb

@@ -2,6 +2,16 @@
 
 config = { "processes" => [] }
 
+# Disable MySQL TLS peer verification when no ccdb.ca_cert is set. See the
+# cloud_controller_ng job for the rationale.
+def disable_mysql_tls_peer_verification!(config)
+  return unless p("ccdb.db_scheme") == "mysql"
+  ca_cert_configured = false
+  if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  return if ca_cert_configured
+  config["env"]["MARIADB_TLS_DISABLE_PEER_VERIFICATION"] = "1"
+end
+
 (1..(p("cc.jobs.generic.number_of_workers"))).each do |index|
   worker_config = {
     "name" => "worker_#{index}",
@@ -30,6 +40,8 @@ config = { "processes" => [] }
     ]
   end
 
+  disable_mysql_tls_peer_verification!(worker_config)
+
   if !!properties.cc.newrelic.license_key
    worker_config["env"]["NEWRELIC_ENABLE"] = true
   end

jobs/rotate_cc_database_key/templates/bin/run.erb

@@ -6,6 +6,17 @@ export LD_PRELOAD=/var/vcap/packages/jemalloc/lib/libjemalloc.so
 
 rotate() {
   export CLOUD_CONTROLLER_NG_CONFIG=/var/vcap/jobs/rotate_cc_database_key/config/cloud_controller_ng.yml
+<%
+  # This errand opens a ccdb connection to rotate the encryption key. Disable
+  # MySQL TLS peer verification when no ccdb.ca_cert is set. See the
+  # cloud_controller_ng job's config/bpm.yml template for the rationale. DB
+  # config comes from the cloud_controller_db link.
+  ca_cert_configured = false
+  link("cloud_controller_db").if_p("ccdb.ca_cert") { |ca_cert| ca_cert_configured = !ca_cert.to_s.strip.empty? }
+  if link("cloud_controller_db").p("ccdb.db_scheme") == "mysql" && !ca_cert_configured
+%>
+  export MARIADB_TLS_DISABLE_PEER_VERIFICATION="1"
+<% end %>
   source /var/vcap/jobs/rotate_cc_database_key/bin/ruby_version.sh
   cd /var/vcap/packages/cloud_controller_ng/cloud_controller_ng
   exec bundle exec rake rotate_cc_database_key:perform

spec/blobstore_benchmark/blobstore_benchmark_spec.rb

@@ -177,6 +177,43 @@ module Test
             end
           end
         end
+
+        describe 'bin/run' do
+          let(:template) { job.template('bin/run') }
+
+          def disables_peer_verification?(db_scheme:, ca_cert:)
+            ccdb = {
+              'db_scheme' => db_scheme,
+              'databases' => [{ 'tag' => 'cc', 'name' => 'ccdb' }],
+              'roles' => [{ 'tag' => 'admin', 'name' => 'u', 'password' => 'p' }],
+              'port' => 3306
+            }
+            ccdb['ca_cert'] = ca_cert unless ca_cert.nil?
+            db_link = Link.new(name: 'cloud_controller_db', properties: { 'ccdb' => ccdb },
+                               instances: [LinkInstance.new(address: 'ccdb')])
+            rendered = template.render({ 'blobstore_benchmark' => { 'mode' => 'all' } },
+                                       consumes: [cloud_controller_internal_link, db_link, database_link])
+            rendered.include?('export MARIADB_TLS_DISABLE_PEER_VERIFICATION="1"')
+          end
+
+          context 'when the database is mysql and no ca_cert is configured' do
+            it 'disables peer verification' do
+              expect(disables_peer_verification?(db_scheme: 'mysql', ca_cert: nil)).to be(true)
+            end
+          end
+
+          context 'when the database is mysql and a ca_cert is configured' do
+            it 'does not disable peer verification' do
+              expect(disables_peer_verification?(db_scheme: 'mysql', ca_cert: 'a-ca-cert')).to be(false)
+            end
+          end
+
+          context 'when the database is postgres' do
+            it 'does not disable peer verification' do
+              expect(disables_peer_verification?(db_scheme: 'postgres', ca_cert: nil)).to be(false)
+            end
+          end
+        end
       end
     end
   end

spec/cc_deployment_updater/bpm_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'bosh/template/test'
+require 'yaml'
+
+module Bosh
+  module Template
+    module Test
+      describe 'cc_deployment_updater bpm job template rendering' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release) { ReleaseDir.new(release_path) }
+        let(:job) { release.job('cc_deployment_updater') }
+
+        describe 'config/bpm.yml' do
+          let(:template) { job.template('config/bpm.yml') }
+
+          describe 'MySQL TLS peer verification' do
+            def updater_env(properties)
+              template_hash = YAML.safe_load(template.render(properties, consumes: {}))
+              updater = template_hash['processes'].find { |p| p['name'] == 'cc_deployment_updater' }
+              expect(updater).not_to be_nil
+              updater['env']
+            end
+
+            context 'when the database is mysql and no ca_cert is configured' do
+              let(:properties) { { 'ccdb' => { 'db_scheme' => 'mysql' } } }
+
+              it 'disables peer verification' do
+                expect(updater_env(properties)['MARIADB_TLS_DISABLE_PEER_VERIFICATION']).to eq('1')
+              end
+            end
+
+            context 'when the database is mysql and a ca_cert is configured' do
+              let(:properties) { { 'ccdb' => { 'db_scheme' => 'mysql', 'ca_cert' => 'a-ca-cert' } } }
+
+              it 'does not disable peer verification' do
+                expect(updater_env(properties)).not_to have_key('MARIADB_TLS_DISABLE_PEER_VERIFICATION')
+              end
+            end
+
+            context 'when the database is postgres' do
+              let(:properties) { { 'ccdb' => { 'db_scheme' => 'postgres' } } }
+
+              it 'does not disable peer verification' do
+                expect(updater_env(properties)).not_to have_key('MARIADB_TLS_DISABLE_PEER_VERIFICATION')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/cloud_controller_clock/bpm_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'bosh/template/test'
+require 'yaml'
+
+module Bosh
+  module Template
+    module Test
+      describe 'cloud_controller_clock bpm job template rendering' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release) { ReleaseDir.new(release_path) }
+        let(:job) { release.job('cloud_controller_clock') }
+
+        describe 'config/bpm.yml' do
+          let(:template) { job.template('config/bpm.yml') }
+
+          describe 'MySQL TLS peer verification' do
+            def clock_env(properties)
+              template_hash = YAML.safe_load(template.render(properties, consumes: {}))
+              clock = template_hash['processes'].find { |p| p['name'] == 'cloud_controller_clock' }
+              expect(clock).not_to be_nil
+              clock['env']
+            end
+
+            context 'when the database is mysql and no ca_cert is configured' do
+              let(:properties) { { 'ccdb' => { 'db_scheme' => 'mysql' } } }
+
+              it 'disables peer verification' do
+                expect(clock_env(properties)['MARIADB_TLS_DISABLE_PEER_VERIFICATION']).to eq('1')
+              end
+            end
+
+            context 'when the database is mysql and a ca_cert is configured' do
+              let(:properties) { { 'ccdb' => { 'db_scheme' => 'mysql', 'ca_cert' => 'a-ca-cert' } } }
+
+              it 'does not disable peer verification' do
+                expect(clock_env(properties)).not_to have_key('MARIADB_TLS_DISABLE_PEER_VERIFICATION')
+              end
+            end
+
+            context 'when the database is postgres' do
+              let(:properties) { { 'ccdb' => { 'db_scheme' => 'postgres' } } }
+
+              it 'does not disable peer verification' do
+                expect(clock_env(properties)).not_to have_key('MARIADB_TLS_DISABLE_PEER_VERIFICATION')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end