From b2afaa2e07437c9e0096c1655e0221f3e40adbaa Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 15 Feb 2018 17:41:36 -0700 Subject: [PATCH 01/12] first attempt at init awareness --- manifests/config.pp | 6 ++++-- manifests/params.pp | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/manifests/config.pp b/manifests/config.pp index da83832..9fd983f 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -24,8 +24,10 @@ notify => $notify_service, } - file { '/etc/default/kafka': - content => template('confluent_kafka/kafka.defaults.erb'), + if { $::confluent_kafka::params::initstyle == 'init'} { + file { '/etc/default/kafka': + content => template('confluent_kafka/kafka.defaults.erb'), + } } file { '/etc/kafka/server.properties': diff --git a/manifests/params.pp b/manifests/params.pp index 7275a3a..9caf126 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -4,6 +4,26 @@ # It sets variables according to platform. # class confluent_kafka::params { + case $::osfamily { + 'Debian': { + case $::operatingsystem { + 'Debian': { + case $::operatingsystemmajrelease { + '7': { $initstyle = 'init' } + '8': { $initstyle = 'systemd' } + default: { $initstyle = undef } + } + } + 'Ubuntu': { + case $::operatingsystemmajrelease { + '14.04': { $initstyle = 'upstart' } + '16.04': { $initstyle = 'systemd' } + default: { $initstyle = undef } + } + } + default: { $initstyle = 'init' } + } + $scala_version = '2.10.4' $platform_version = '1.0' $service_name = 'kafka' From a908da4dc82bcf7196c07e53a081cf156c88522a Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 15 Feb 2018 17:47:29 -0700 Subject: [PATCH 02/12] forgot closing braces --- manifests/params.pp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manifests/params.pp b/manifests/params.pp index 9caf126..a8df86f 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -23,6 +23,8 @@ } default: { $initstyle = 'init' } } + } + } $scala_version = '2.10.4' $platform_version = '1.0' From 95ab411b61e36cf7393ea57319fb3843a7bf5f52 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 15 Feb 2018 17:55:38 -0700 Subject: [PATCH 03/12] correction for if --- manifests/config.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/config.pp b/manifests/config.pp index 9fd983f..d65c1d5 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -24,7 +24,7 @@ notify => $notify_service, } - if { $::confluent_kafka::params::initstyle == 'init'} { + if ($::confluent_kafka::params::initstyle == 'init') { file { '/etc/default/kafka': content => template('confluent_kafka/kafka.defaults.erb'), } From bee4d624ea7e9a270cb3ad740eeaac2e2208a4b7 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Fri, 16 Feb 2018 11:34:42 -0700 Subject: [PATCH 04/12] systemd defaults file --- manifests/config.pp | 5 +++++ templates/kafka.defaults_systemd.erb | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 templates/kafka.defaults_systemd.erb diff --git a/manifests/config.pp b/manifests/config.pp index d65c1d5..82cff27 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -29,6 +29,11 @@ content => template('confluent_kafka/kafka.defaults.erb'), } } + else { + file { '/etc/default/kafka': + content => template('confluent_kafka/kafka.defaults_systemd.erb'), + } + } file { '/etc/kafka/server.properties': content => template('confluent_kafka/server.properties.erb'), diff --git a/templates/kafka.defaults_systemd.erb b/templates/kafka.defaults_systemd.erb new file mode 100644 index 0000000..e16a139 --- /dev/null +++ b/templates/kafka.defaults_systemd.erb @@ -0,0 +1,8 @@ +# Have to export these log variables since currently a bug in kafka-server-start +# that doesnt export these variables +LOG_DIR="<%= scope.lookupvar('confluent_kafka::app_log_dir') %>" +KAFKA_LOG4J_OPTS="<%= scope.lookupvar('confluent_kafka::log4j_opts') %>" + +KAFKA_HEAP_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_heap_mem') %>" +KAFKA_JMX_OPTS="<%= scope.lookupvar('confluent_kafka::jmx_opts') %>" +KAFKA_JVM_PERFORMANCE_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_perf_opts')%>" From fdada438fa26003da79f74c032d0d01daaf9a254 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Fri, 16 Feb 2018 13:57:14 -0700 Subject: [PATCH 05/12] systemd install --- manifests/install.pp | 19 ++++++++++++++----- templates/kafka.systemd.erb | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 templates/kafka.systemd.erb diff --git a/manifests/install.pp b/manifests/install.pp index 3fd8976..80c400d 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -61,12 +61,21 @@ } if $::confluent_kafka::install_service { - file { "/etc/init.d/${::confluent_kafka::service_name}": - mode => '0755', - owner => 'root', - group => 'root', - source => 'puppet:///modules/confluent_kafka/kafka.init', + if ($::confluent_kafka::params::initstyle == 'init') { + file { "/etc/init.d/${::confluent_kafka::service_name}": + mode => '0755', + owner => 'root', + group => 'root', + source => 'puppet:///modules/confluent_kafka/kafka.init', + } } + else { + file { "/lib/systemd/system/${::confluent_kafka::service_name}.service": + mode => '0644', + owner => 'root', + group => 'root', + content => 'puppet:///modules/confluent_kafka/kafka.systemd.erb', + } } } diff --git a/templates/kafka.systemd.erb b/templates/kafka.systemd.erb new file mode 100644 index 0000000..23e99af --- /dev/null +++ b/templates/kafka.systemd.erb @@ -0,0 +1,20 @@ +[Unit] +Description=Kafka server +After=network.target network-online.target + +[Service] +Type=forking +User=kafka +Group=kafka +EnvironmentFile=-/etc/default/kafka +ExecStart=/usr/bin/kafka-server-start -daemon /etc/kafka/server.properties +ExecStop=/usr/bin/kafka-server-stop +TimeoutStopSec=60s +Restart=on-failure +StartLimitInterval=0 +RestartSec=10s +SuccessExitStatus=143 +LimitNOFILE=<%= scope.lookupvar('confluent_kafka::params::max_nofiles')%> + +[Install] +WantedBy=multi-user.target From 04d80d74697b83ddc38e96051e3853359ae6f756 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Fri, 16 Feb 2018 14:07:11 -0700 Subject: [PATCH 06/12] added missing closing brace --- manifests/install.pp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/manifests/install.pp b/manifests/install.pp index 80c400d..f7fa2ca 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -35,8 +35,8 @@ exec { 'apt-get update': - command => "/usr/bin/apt-get update", - alias => "apt-update", + command => '/usr/bin/apt-get update', + alias => 'apt-update', } if $::confluent_kafka::install_java { @@ -46,8 +46,8 @@ } package { "${::confluent_kafka::package_name}-${::confluent_kafka::scala_version}": + ensure => $::confluent_kafka::version, require => Exec[apt-update], - ensure => $::confluent_kafka::version, } group { 'kafka': @@ -76,6 +76,7 @@ group => 'root', content => 'puppet:///modules/confluent_kafka/kafka.systemd.erb', } + } } } From b45b3e9983c7ef6c394823100f3557ea83f4e611 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Fri, 16 Feb 2018 14:25:26 -0700 Subject: [PATCH 07/12] corrected template --- manifests/install.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/install.pp b/manifests/install.pp index f7fa2ca..d8c0cd7 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -74,7 +74,7 @@ mode => '0644', owner => 'root', group => 'root', - content => 'puppet:///modules/confluent_kafka/kafka.systemd.erb', + content => template('confluent_kafka/kafka.systemd.erb'), } } } From 0ffa90538c06f95fc123b80a2064f3771524c2b8 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Fri, 16 Feb 2018 14:28:50 -0700 Subject: [PATCH 08/12] systemd refresh --- manifests/install.pp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/manifests/install.pp b/manifests/install.pp index d8c0cd7..27bb49d 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -75,6 +75,10 @@ owner => 'root', group => 'root', content => template('confluent_kafka/kafka.systemd.erb'), + } ~> + exec { 'systemctl daemon-reload # for kafka': + refreshonly => true, + notify => Service[$::confluent_kafka::service_name] } } } From e08aeae7d3cf3f0f1eb5d33d69dba48927b4a6e7 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 26 Apr 2018 11:16:27 -0600 Subject: [PATCH 09/12] added systemd extra_args --- manifests/init.pp | 4 ++++ templates/kafka.defaults_systemd.erb | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/manifests/init.pp b/manifests/init.pp index bdc0bcb..dbc1daa 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -61,6 +61,9 @@ # [*log4j_opts*] # Override Log4J file # +# [extra_args*] +# Override Log4J file +# class confluent_kafka ( $package_name = $::confluent_kafka::params::package_name, @@ -82,6 +85,7 @@ $jvm_perf_opts = $::confluent_kafka::params::jvm_perf_opts, $jmx_opts = $::confluent_kafka::params::jmx_opts, $log4j_opts = $::confluent_kafka::params::log4j_opts, + $extra_args = $::confluent_kafka::params::extra_args, $platform_version = $::confluent_kafka::params::platform_version, ) inherits ::confluent_kafka::params { diff --git a/templates/kafka.defaults_systemd.erb b/templates/kafka.defaults_systemd.erb index e16a139..c3da660 100644 --- a/templates/kafka.defaults_systemd.erb +++ b/templates/kafka.defaults_systemd.erb @@ -6,3 +6,7 @@ KAFKA_LOG4J_OPTS="<%= scope.lookupvar('confluent_kafka::log4j_opts') %>" KAFKA_HEAP_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_heap_mem') %>" KAFKA_JMX_OPTS="<%= scope.lookupvar('confluent_kafka::jmx_opts') %>" KAFKA_JVM_PERFORMANCE_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_perf_opts')%>" +<%if scope.lookupvar('confluent_kafka::extra_args') != "" -%> +EXTRA_ARGS="scope.lookupvar('confluent_kafka::jmx_opts')" +<% end -%> + From 3675dc97c8bcc66971ed7ce56a08f6ba4579dfb1 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 26 Apr 2018 11:23:44 -0600 Subject: [PATCH 10/12] updated erb tempalte --- templates/kafka.defaults_systemd.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/kafka.defaults_systemd.erb b/templates/kafka.defaults_systemd.erb index c3da660..ed9bfdd 100644 --- a/templates/kafka.defaults_systemd.erb +++ b/templates/kafka.defaults_systemd.erb @@ -6,7 +6,7 @@ KAFKA_LOG4J_OPTS="<%= scope.lookupvar('confluent_kafka::log4j_opts') %>" KAFKA_HEAP_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_heap_mem') %>" KAFKA_JMX_OPTS="<%= scope.lookupvar('confluent_kafka::jmx_opts') %>" KAFKA_JVM_PERFORMANCE_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_perf_opts')%>" -<%if scope.lookupvar('confluent_kafka::extra_args') != "" -%> -EXTRA_ARGS="scope.lookupvar('confluent_kafka::jmx_opts')" +<% if ! scope.lookupvar('confluent_kafka::extra_args').empty? then -%> +EXTRA_ARGS="<%= scope.lookupvar('confluent_kafka::extra_args') %>" <% end -%> From 02a012598bb1629fbf84ff111fc46d5c66428a21 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 26 Apr 2018 11:32:26 -0600 Subject: [PATCH 11/12] added default extra_args value --- manifests/params.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/params.pp b/manifests/params.pp index a8df86f..31b6f5b 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -45,6 +45,7 @@ $jvm_perf_opts = '-XX:PermSize=48m -XX:MaxPermSize=48m -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35' $jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.net.preferIPv4Stack=true -Dcom.sun.management.jmxremote.port=9999' $log4j_opts = '-Dlog4j.configuration=file:/etc/kafka/log4j.properties' + $extra_args = '' $brokers = { 'localhost' => 0, From 1e5869dbd640a2b3e65238666553a566983d2ab1 Mon Sep 17 00:00:00 2001 From: "Tom E. Tomlinson" Date: Thu, 26 Apr 2018 13:34:50 -0600 Subject: [PATCH 12/12] added EXTRA_ARGS to init defaults --- templates/kafka.defaults.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/kafka.defaults.erb b/templates/kafka.defaults.erb index 41e5d78..fd890d1 100644 --- a/templates/kafka.defaults.erb +++ b/templates/kafka.defaults.erb @@ -7,3 +7,6 @@ export KAFKA_HEAP_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_heap_mem') %>" export KAFKA_JMX_OPTS="<%= scope.lookupvar('confluent_kafka::jmx_opts') %>" export KAFKA_JVM_PERFORMANCE_OPTS="<%= scope.lookupvar('confluent_kafka::jvm_perf_opts')%>" ulimit -n <%= scope.lookupvar('confluent_kafka::params::max_nofiles')%> +<% if ! scope.lookupvar('confluent_kafka::extra_args').empty? then -%> +export EXTRA_ARGS="<%= scope.lookupvar('confluent_kafka::extra_args') %>" +<% end -%>