diff --git a/Gemfile b/Gemfile index 9340366..8b4adb5 100644 --- a/Gemfile +++ b/Gemfile @@ -1,82 +1,86 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" def location_for(place, fake_version = nil) if place =~ /^(git[:@][^#]*)#(.*)/ [fake_version, { :git => $1, :branch => $2, :require => false }].compact elsif place =~ /^file:\/\/(.*)/ ['>= 0', { :path => File.expand_path($1), :require => false }] else [place, { :require => false }] end end group :test do gem 'puppetlabs_spec_helper', '>= 2.14.0', :require => false gem 'rspec-puppet-facts', '>= 1.9.5', :require => false gem 'rspec-puppet-utils', :require => false gem 'puppet-lint-leading_zero-check', :require => false gem 'puppet-lint-trailing_comma-check', :require => false gem 'puppet-lint-version_comparison-check', :require => false gem 'puppet-lint-classes_and_types_beginning_with_digits-check', :require => false gem 'puppet-lint-unquoted_string-check', :require => false gem 'puppet-lint-variable_contains_upcase', :require => false + gem 'puppet-lint-absolute_classname-check', '>= 2.0.0', :require => false + gem 'puppet-lint-topscope-variable-check', :require => false + gem 'puppet-lint-legacy_facts-check', :require => false + gem 'puppet-lint-anchor-check', :require => false gem 'metadata-json-lint', :require => false gem 'redcarpet', :require => false gem 'rubocop', '~> 0.49.1', :require => false gem 'rubocop-rspec', '~> 1.15.0', :require => false gem 'mocha', '~> 1.4.0', :require => false gem 'coveralls', :require => false gem 'simplecov-console', :require => false gem 'parallel_tests', :require => false end group :development do gem 'travis', :require => false gem 'travis-lint', :require => false gem 'guard-rake', :require => false gem 'overcommit', '>= 0.39.1', :require => false end group :system_tests do gem 'winrm', :require => false if beaker_version = ENV['BEAKER_VERSION'] gem 'beaker', *location_for(beaker_version) else gem 'beaker', '>= 4.2.0', :require => false end if beaker_rspec_version = ENV['BEAKER_RSPEC_VERSION'] gem 'beaker-rspec', *location_for(beaker_rspec_version) else gem 'beaker-rspec', :require => false end gem 'serverspec', :require => false gem 'beaker-hostgenerator', '>= 1.1.22', :require => false gem 'beaker-docker', :require => false gem 'beaker-puppet', :require => false gem 'beaker-puppet_install_helper', :require => false gem 'beaker-module_install_helper', :require => false gem 'rbnacl', '>= 4', :require => false gem 'rbnacl-libsodium', :require => false gem 'bcrypt_pbkdf', :require => false gem 'ed25519', :require => false end group :release do gem 'github_changelog_generator', :require => false, :git => 'https://github.com/voxpupuli/github-changelog-generator', :branch => 'voxpupuli_essential_fixes' gem 'puppet-blacksmith', :require => false gem 'voxpupuli-release', :require => false gem 'puppet-strings', '>= 2.2', :require => false end if facterversion = ENV['FACTER_GEM_VERSION'] gem 'facter', facterversion.to_s, :require => false, :groups => [:test] else gem 'facter', :require => false, :groups => [:test] end ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 6.0' : puppetversion = ENV['PUPPET_VERSION'].to_s gem 'puppet', puppetversion, :require => false, :groups => [:test] # vim: syntax=ruby diff --git a/manifests/broker.pp b/manifests/broker.pp index f32fa40..fae44a6 100644 --- a/manifests/broker.pp +++ b/manifests/broker.pp @@ -1,155 +1,155 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::broker # # This class will install kafka with the broker role. # # === Requirements/Dependencies # # Currently requires the puppetlabs/stdlib module on the Puppet Forge in # order to validate much of the the provided configuration. # # === Parameters # # [*version*] # The version of kafka that should be installed. # # [*scala_version*] # The scala version what kafka was built with. # # [*install_dir*] # The directory to install kafka to. # # [*mirror_url*] # The url where the kafka is downloaded from. # # [*install_java*] # Install java if it's not already installed. # # [*package_dir*] # The directory to install kafka. # # [*package_name*] # Package name, when installing kafka from a package. # # [*package_ensure*] # Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. # # [*user*] # User to run kafka as. # # [*group*] # Group to run kafka as. # # [*user_id*] # Create the kafka user with this ID. # # [*group_id*] # Create the kafka group with this ID. # # [*manage_user*] # Create the kafka user if it's not already present. # # [*manage_group*] # Create the kafka group if it's not already present. # # [*config_mode*] # The permissions for the config files. # # [*config_dir*] # The directory to create the kafka config files to. # # [*log_dir*] # The directory for kafka log files. # # [*bin_dir*] # The directory where the kafka scripts are. # # [*service_name*] # Set the name of the service. # # [*service_install*] # Install the init.d or systemd service. # # [*service_ensure*] # Set the ensure state of the service to 'stopped' or 'running'. # # [*service_restart*] # Whether the configuration files should trigger a service restart. # # [*service_requires*] # Set the list of services required to be running before Kafka. # # [*limit_nofile*] # Set the 'LimitNOFILE' option of the systemd service. # # [*limit_core*] # Set the 'LimitCORE' option of the systemd service. # # [*timeout_stop*] # Set the 'TimeoutStopSec' option of the systemd service. # # [*exec_stop*] # Set the 'ExecStop' option of the systemd service to 'kafka-server-stop.sh'. # # [*daemon_start*] # Use the '-daemon' option when starting Kafka with 'kafka-server-start.sh'. # # [*env*] # A hash of the environment variables to set. # # [*config*] # A hash of the configuration options. # # === Examples # # Create a single broker instance which talks to a local zookeeper instance. # # class { 'kafka::broker': # config => { 'broker.id' => '0', 'zookeeper.connect' => 'localhost:2181' } # } # class kafka::broker ( String $version = $kafka::params::version, String $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $install_java = $kafka::params::install_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String] $package_name = $kafka::params::package_name, String $package_ensure = $kafka::params::package_ensure, String $user = $kafka::params::user, String $group = $kafka::params::group, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String $service_name = 'kafka', Boolean $service_install = $kafka::params::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String] $service_requires = $kafka::params::service_requires, Optional[String] $limit_nofile = $kafka::params::limit_nofile, Optional[String] $limit_core = $kafka::params::limit_core, Optional[String] $timeout_stop = $kafka::params::timeout_stop, Boolean $exec_stop = $kafka::params::exec_stop, Boolean $daemon_start = $kafka::params::daemon_start, Hash $env = {}, Hash $config = {}, String $heap_opts = $kafka::params::broker_heap_opts, String $jmx_opts = $kafka::params::broker_jmx_opts, String $log4j_opts = $kafka::params::broker_log4j_opts, $opts = $kafka::params::broker_opts, ) inherits kafka::params { - class { '::kafka::broker::install': } - -> class { '::kafka::broker::config': } - -> class { '::kafka::broker::service': } + class { 'kafka::broker::install': } + -> class { 'kafka::broker::config': } + -> class { 'kafka::broker::service': } -> Class['kafka::broker'] } diff --git a/manifests/broker/install.pp b/manifests/broker/install.pp index 74400a5..454e9e1 100644 --- a/manifests/broker/install.pp +++ b/manifests/broker/install.pp @@ -1,36 +1,36 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::broker::install # # This private class is meant to be called from `kafka::broker`. # It downloads the package and installs it. # class kafka::broker::install { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } - if !defined(Class['::kafka']) { - class { '::kafka': + if !defined(Class['kafka']) { + class { 'kafka': version => $kafka::broker::version, scala_version => $kafka::broker::scala_version, install_dir => $kafka::broker::install_dir, mirror_url => $kafka::broker::mirror_url, install_java => $kafka::broker::install_java, package_dir => $kafka::broker::package_dir, package_name => $kafka::broker::package_name, package_ensure => $kafka::broker::package_ensure, user => $kafka::broker::user, group => $kafka::broker::group, user_id => $kafka::broker::user_id, group_id => $kafka::broker::group_id, manage_user => $kafka::broker::manage_user, manage_group => $kafka::broker::manage_group, config_dir => $kafka::broker::config_dir, log_dir => $kafka::broker::log_dir, } } } diff --git a/manifests/broker/service.pp b/manifests/broker/service.pp index 54fdaee..defe4ee 100644 --- a/manifests/broker/service.pp +++ b/manifests/broker/service.pp @@ -1,79 +1,79 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::broker::service # # This private class is meant to be called from `kafka::broker`. # It manages the kafka service # class kafka::broker::service( String $user = $kafka::broker::user, String $group = $kafka::broker::group, Stdlib::Absolutepath $config_dir = $kafka::broker::config_dir, Stdlib::Absolutepath $log_dir = $kafka::broker::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::broker::bin_dir, String $service_name = $kafka::broker::service_name, Boolean $service_install = $kafka::broker::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::broker::service_ensure, Array[String] $service_requires = $kafka::broker::service_requires, Optional[String] $limit_nofile = $kafka::broker::limit_nofile, Optional[String] $limit_core = $kafka::broker::limit_core, Optional[String] $timeout_stop = $kafka::broker::timeout_stop, Boolean $exec_stop = $kafka::broker::exec_stop, Boolean $daemon_start = $kafka::broker::daemon_start, Hash $env = $kafka::broker::env, String $heap_opts = $kafka::broker::heap_opts, String $jmx_opts = $kafka::broker::jmx_opts, String $log4j_opts = $kafka::broker::log4j_opts, $opts = $kafka::broker::opts, ) { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } if $service_install { $env_defaults = { 'KAFKA_HEAP_OPTS' => $heap_opts, 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, 'KAFKA_OPTS' => $opts, 'LOG_DIR' => $log_dir, } $environment = deep_merge($env_defaults, $env) if $::service_provider == 'systemd' { - include ::systemd + include systemd file { "/etc/systemd/system/${service_name}.service": ensure => file, mode => '0644', content => template('kafka/unit.erb'), } file { "/etc/init.d/${service_name}": ensure => absent, } File["/etc/systemd/system/${service_name}.service"] ~> Exec['systemctl-daemon-reload'] -> Service[$service_name] } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/consumer.pp b/manifests/consumer.pp index 38fa0a4..eb2c07e 100644 --- a/manifests/consumer.pp +++ b/manifests/consumer.pp @@ -1,144 +1,144 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::consumer # # This class will install kafka with the consumer role. # # === Requirements/Dependencies # # Currently requires the puppetlabs/stdlib module on the Puppet Forge in # order to validate much of the the provided configuration. # # === Parameters # # [*version*] # The version of kafka that should be installed. # # [*scala_version*] # The scala version what kafka was built with. # # [*install_dir*] # The directory to install kafka to. # # [*mirror_url*] # The url where the kafka is downloaded from. # # [*install_java*] # Install java if it's not already installed. # # [*package_dir*] # The directory to install kafka. # # [*package_name*] # Package name, when installing kafka from a package. # # [*package_ensure*] # Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. # # [*user*] # User to run kafka as. # # [*group*] # Group to run kafka as. # # [*user_id*] # Create the kafka user with this ID. # # [*group_id*] # Create the kafka group with this ID. # # [*manage_user*] # Create the kafka user if it's not already present. # # [*manage_group*] # Create the kafka group if it's not already present. # # [*config_mode*] # The permissions for the config files. # # [*config_dir*] # The directory to create the kafka config files to. # # [*log_dir*] # The directory for kafka log files. # # [*bin_dir*] # The directory where the kafka scripts are. # # [*service_name*] # Set the name of the service. # # [*service_install*] # Install the init.d or systemd service. # # [*service_ensure*] # Set the ensure state of the service to 'stopped' or 'running'. # # [*service_restart*] # Whether the configuration files should trigger a service restart. # # [*service_requires*] # Set the list of services required to be running before Kafka. # # [*limit_nofile*] # Set the 'LimitNOFILE' option of the systemd service. # # [*limit_core*] # Set the 'LimitCORE' option of the systemd service. # # [*env*] # A hash of the environment variables to set. # # [*config*] # A hash of the consumer configuration options. # # [*service_config*] # A hash of the `kafka-console-consumer.sh` script options. # # === Examples # # Create the consumer service connecting to a local zookeeper # # class { 'kafka::consumer': # config => { 'client.id' => '0', 'zookeeper.connect' => 'localhost:2181' } # } class kafka::consumer ( String $version = $kafka::params::version, String $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $install_java = $kafka::params::install_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String] $package_name = $kafka::params::package_name, String $package_ensure = $kafka::params::package_ensure, String $user = $kafka::params::user, String $group = $kafka::params::group, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String $service_name = 'kafka-consumer', Boolean $service_install = $kafka::params::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String] $service_requires = $kafka::params::service_requires, Optional[String] $limit_nofile = $kafka::params::limit_nofile, Optional[String] $limit_core = $kafka::params::limit_core, Hash $env = {}, Hash $config = {}, Hash $service_config = {}, String $jmx_opts = $kafka::params::consumer_jmx_opts, String $log4j_opts = $kafka::params::consumer_log4j_opts, ) inherits kafka::params { - class { '::kafka::consumer::install': } - -> class { '::kafka::consumer::config': } - -> class { '::kafka::consumer::service': } + class { 'kafka::consumer::install': } + -> class { 'kafka::consumer::config': } + -> class { 'kafka::consumer::service': } -> Class['kafka::consumer'] } diff --git a/manifests/consumer/install.pp b/manifests/consumer/install.pp index f99d038..8b2c7ea 100644 --- a/manifests/consumer/install.pp +++ b/manifests/consumer/install.pp @@ -1,36 +1,36 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::consumer::install # # This private class is meant to be called from `kafka::consumer`. # It downloads the package and installs it. # class kafka::consumer::install { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } - if !defined(Class['::kafka']) { - class { '::kafka': + if !defined(Class['kafka']) { + class { 'kafka': version => $kafka::consumer::version, scala_version => $kafka::consumer::scala_version, install_dir => $kafka::consumer::install_dir, mirror_url => $kafka::consumer::mirror_url, install_java => $kafka::consumer::install_java, package_dir => $kafka::consumer::package_dir, package_name => $kafka::consumer::package_name, package_ensure => $kafka::consumer::package_ensure, user => $kafka::consumer::user, group => $kafka::consumer::group, user_id => $kafka::consumer::user_id, group_id => $kafka::consumer::group_id, manage_user => $kafka::consumer::manage_user, manage_group => $kafka::consumer::manage_group, config_dir => $kafka::consumer::config_dir, log_dir => $kafka::consumer::log_dir, } } } diff --git a/manifests/consumer/service.pp b/manifests/consumer/service.pp index 0364740..5fff83b 100644 --- a/manifests/consumer/service.pp +++ b/manifests/consumer/service.pp @@ -1,80 +1,80 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::consumer::service # # This private class is meant to be called from `kafka::consumer`. # It manages the kafka-consumer service # class kafka::consumer::service( String $user = $kafka::consumer::user, String $group = $kafka::consumer::group, Stdlib::Absolutepath $config_dir = $kafka::consumer::config_dir, Stdlib::Absolutepath $log_dir = $kafka::consumer::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::consumer::bin_dir, String $service_name = $kafka::consumer::service_name, Boolean $service_install = $kafka::consumer::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::consumer::service_ensure, Array[String] $service_requires = $kafka::consumer::service_requires, Optional[String] $limit_nofile = $kafka::consumer::limit_nofile, Optional[String] $limit_core = $kafka::consumer::limit_core, Hash $env = $kafka::consumer::env, String $jmx_opts = $kafka::consumer::jmx_opts, String $log4j_opts = $kafka::consumer::log4j_opts, Hash $service_config = $kafka::consumer::service_config, ) { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } if $service_install { if $service_config['topic'] == '' { fail('[Consumer] You need to specify a value for topic') } if $service_config['zookeeper'] == '' { fail('[Consumer] You need to specify a value for zookeeper') } $env_defaults = { 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, } $environment = deep_merge($env_defaults, $env) if $::service_provider == 'systemd' { - include ::systemd + include systemd file { "/etc/systemd/system/${service_name}.service": ensure => file, mode => '0644', content => template('kafka/unit.erb'), } file { "/etc/init.d/${service_name}": ensure => absent, } File["/etc/systemd/system/${service_name}.service"] ~> Exec['systemctl-daemon-reload'] -> Service[$service_name] } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/init.pp b/manifests/init.pp index 5bb9073..7a70952 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,230 +1,230 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka # # This class will install kafka binaries # # === Requirements/Dependencies # # Currently requires the puppetlabs/stdlib module on the Puppet Forge in # order to validate much of the the provided configuration. # # === Parameters # # [*version*] # The version of kafka that should be installed. # # [*scala_version*] # The scala version what kafka was built with. # # [*install_dir*] # The directory to install kafka to. # # [*mirror_url*] # The url where the kafka is downloaded from. # # [*install_java*] # Install java if it's not already installed. # # [*package_dir*] # The directory to install kafka. # # [*package_name*] # Package name, when installing kafka from a package. # # [*package_ensure*] # Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. # # [*user*] # User to run kafka as. # # [*group*] # Group to run kafka as. # # [*user_id*] # Create the kafka user with this ID. # # [*system_user*] # Whether the kafka user is a system user or not. # # [*group_id*] # Create the kafka group with this ID. # # [*system_group*] # Whether the kafka group is a system group or not. # # [*manage_user*] # Create the kafka user if it's not already present. # # [*manage_group*] # Create the kafka group if it's not already present. # # [*config_dir*] # The directory to create the kafka config files to. # # [*log_dir*] # The directory for kafka log files. # # === Examples # # class kafka ( String $version = $kafka::params::version, String $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $install_java = $kafka::params::install_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String] $package_name = $kafka::params::package_name, Optional[String] $mirror_subpath = $kafka::params::mirror_subpath, Optional[String] $proxy_server = $kafka::params::proxy_server, Optional[String] $proxy_port = $kafka::params::proxy_port, Optional[String] $proxy_host = $kafka::params::proxy_host, Optional[String] $proxy_type = $kafka::params::proxy_type, String $package_ensure = $kafka::params::package_ensure, String $user = $kafka::params::user, String $group = $kafka::params::group, Boolean $system_user = $kafka::params::system_user, Boolean $system_group = $kafka::params::system_group, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Optional[String] $install_mode = $kafka::params::install_mode, ) inherits kafka::params { if $install_java { - class { '::java': + class { 'java': distribution => 'jdk', } } if $manage_group { group { $group: ensure => present, gid => $group_id, system => $system_group, } } if $manage_user { user { $user: ensure => present, shell => '/bin/bash', require => Group[$group], uid => $user_id, system => $system_user, } } file { $config_dir: ensure => directory, owner => 'root', group => 'root', } file { $log_dir: ensure => directory, owner => $user, group => $group, require => [ Group[$group], User[$user], ], } if $package_name == undef { - include ::archive + include archive $mirror_path = $mirror_subpath ? { # if mirror_subpath was not changed, # we adapt it for the version $kafka::params::mirror_subpath => "kafka/${version}", # else, we just take whatever was supplied: default => $mirror_subpath, } $basefilename = "kafka_${scala_version}-${version}.tgz" $package_url = "${mirror_url}${mirror_path}/${basefilename}" $source = $mirror_url ?{ /tgz$/ => $mirror_url, default => $package_url, } $install_directory = $install_dir ? { # if install_dir was not changed, # we adapt it for the scala_version and the version $kafka::params::install_dir => "/opt/kafka-${scala_version}-${version}", # else, we just take whatever was supplied: default => $install_dir, } file { $package_dir: ensure => directory, owner => $user, group => $group, require => [ Group[$group], User[$user], ], } file { $install_directory: ensure => directory, owner => $user, group => $group, mode => $install_mode, require => [ Group[$group], User[$user], ], } file { '/opt/kafka': ensure => link, target => $install_directory, require => File[$install_directory], } if $proxy_server == undef and $proxy_host != undef and $proxy_port != undef { $final_proxy_server = "${proxy_host}:${proxy_port}" } else { $final_proxy_server = $proxy_server } archive { "${package_dir}/${basefilename}": ensure => present, extract => true, extract_command => 'tar xfz %s --strip-components=1', extract_path => $install_directory, source => $source, creates => "${install_directory}/config", cleanup => true, proxy_server => $final_proxy_server, proxy_type => $proxy_type, user => $user, group => $group, require => [ File[$package_dir], File[$install_directory], Group[$group], User[$user], ], before => File[$config_dir], } } else { package { $package_name: ensure => $package_ensure, before => File[$config_dir], } } } diff --git a/manifests/mirror.pp b/manifests/mirror.pp index 2c74ff2..7bc291d 100644 --- a/manifests/mirror.pp +++ b/manifests/mirror.pp @@ -1,147 +1,147 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::mirror # # This class will install kafka with the mirror role. # # === Requirements/Dependencies # # Currently requires the puppetlabs/stdlib module on the Puppet Forge in # order to validate much of the the provided configuration. # # === Parameters # # [*version*] # The version of kafka that should be installed. # # [*scala_version*] # The scala version what kafka was built with. # # [*install_dir*] # The directory to install kafka to. # # [*mirror_url*] # The url where the kafka is downloaded from. # # [*install_java*] # Install java if it's not already installed. # # [*package_dir*] # The directory to install kafka. # # [*package_name*] # Package name, when installing kafka from a package. # # [*package_ensure*] # Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. # # [*user*] # User to run kafka as. # # [*group*] # Group to run kafka as. # # [*user_id*] # Create the kafka user with this ID. # # [*group_id*] # Create the kafka group with this ID. # # [*manage_user*] # Create the kafka user if it's not already present. # # [*manage_group*] # Create the kafka group if it's not already present. # # [*config_dir*] # The directory to create the kafka config files to. # # [*log_dir*] # The directory for kafka log files. # # [*bin_dir*] # The directory where the kafka scripts are. # # [*service_name*] # Set the name of the service. # # [*service_install*] # Install the init.d or systemd service. # # [*service_ensure*] # Set the ensure state of the service to 'stopped' or 'running'. # # [*service_restart*] # Whether the configuration files should trigger a service restart. # # [*service_requires*] # Set the list of services required to be running before Kafka. # # [*limit_nofile*] # Set the 'LimitNOFILE' option of the systemd service. # # [*limit_core*] # Set the 'LimitCORE' option of the systemd service. # # [*env*] # A hash of the environment variables to set. # # [*consumer_config*] # A hash of the consumer configuration options. # # [*producer_config*] # A hash of the producer configuration options. # # [*service_config*] # A hash of the mirror script options. # # === Examples # # Create the mirror service connecting to a local zookeeper # # class { 'kafka::mirror': # consumer_config => { 'client.id' => '0', 'zookeeper.connect' => 'localhost:2181' } # } # class kafka::mirror ( String $version = $kafka::params::version, String $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $install_java = $kafka::params::install_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String] $package_name = $kafka::params::package_name, String $package_ensure = $kafka::params::package_ensure, String $user = $kafka::params::user, String $group = $kafka::params::group, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String $service_name = 'kafka-mirror', Boolean $service_install = $kafka::params::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String] $service_requires = $kafka::params::service_requires, Optional[String] $limit_nofile = $kafka::params::limit_nofile, Optional[String] $limit_core = $kafka::params::limit_core, Hash $env = {}, Hash $consumer_config = {}, Hash $producer_config = {}, Hash $service_config = {}, String $heap_opts = $kafka::params::mirror_heap_opts, String $jmx_opts = $kafka::params::mirror_jmx_opts, String $log4j_opts = $kafka::params::mirror_log4j_opts, ) inherits kafka::params { - class { '::kafka::mirror::install': } - -> class { '::kafka::mirror::config': } - -> class { '::kafka::mirror::service': } + class { 'kafka::mirror::install': } + -> class { 'kafka::mirror::config': } + -> class { 'kafka::mirror::service': } -> Class['kafka::mirror'] } diff --git a/manifests/mirror/config.pp b/manifests/mirror/config.pp index 73c288f..91a4406 100644 --- a/manifests/mirror/config.pp +++ b/manifests/mirror/config.pp @@ -1,54 +1,54 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::mirror::config # # This private class is meant to be called from `kafka::mirror`. # It manages the mirror-maker config files # class kafka::mirror::config( Stdlib::Absolutepath $config_dir = $kafka::mirror::config_dir, String $service_name = $kafka::mirror::service_name, Boolean $service_install = $kafka::mirror::service_install, Boolean $service_restart = $kafka::mirror::service_restart, Hash $consumer_config = $kafka::mirror::consumer_config, Hash $producer_config = $kafka::mirror::producer_config, Stdlib::Filemode $config_mode = $kafka::mirror::config_mode, String $group = $kafka::mirror::group, ) { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } if $consumer_config['group.id'] == '' { fail('[Consumer] You need to specify a value for group.id') } if $consumer_config['zookeeper.connect'] == '' { fail('[Consumer] You need to specify a value for zookeeper.connect') } if $producer_config['bootstrap.servers'] == '' { fail('[Producer] You need to specify a value for bootstrap.servers') } - class { '::kafka::consumer::config': + class { 'kafka::consumer::config': config_dir => $config_dir, config_mode => $config_mode, service_name => $service_name, service_install => $service_install, service_restart => $service_restart, config => $consumer_config, group => $group, } - class { '::kafka::producer::config': + class { 'kafka::producer::config': config_dir => $config_dir, config_mode => $config_mode, service_name => $service_name, service_install => $service_install, service_restart => $service_restart, config => $producer_config, group => $group, } } diff --git a/manifests/mirror/install.pp b/manifests/mirror/install.pp index da7e763..90b2e4a 100644 --- a/manifests/mirror/install.pp +++ b/manifests/mirror/install.pp @@ -1,36 +1,36 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::mirror::install # # This private class is meant to be called from `kafka::mirror`. # It downloads the package and installs it. # class kafka::mirror::install { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } - if !defined(Class['::kafka']) { - class { '::kafka': + if !defined(Class['kafka']) { + class { 'kafka': version => $kafka::mirror::version, scala_version => $kafka::mirror::scala_version, install_dir => $kafka::mirror::install_dir, mirror_url => $kafka::mirror::mirror_url, install_java => $kafka::mirror::install_java, package_dir => $kafka::mirror::package_dir, package_name => $kafka::mirror::package_name, package_ensure => $kafka::mirror::package_ensure, user => $kafka::mirror::user, group => $kafka::mirror::group, user_id => $kafka::mirror::user_id, group_id => $kafka::mirror::group_id, manage_user => $kafka::mirror::manage_user, manage_group => $kafka::mirror::manage_group, config_dir => $kafka::mirror::config_dir, log_dir => $kafka::mirror::log_dir, } } } diff --git a/manifests/mirror/service.pp b/manifests/mirror/service.pp index 0819099..ff58b5c 100644 --- a/manifests/mirror/service.pp +++ b/manifests/mirror/service.pp @@ -1,76 +1,76 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::mirror::service # # This private class is meant to be called from `kafka::mirror`. # It manages the kafka-mirror service # class kafka::mirror::service( String $user = $kafka::mirror::user, String $group = $kafka::mirror::group, Stdlib::Absolutepath $config_dir = $kafka::mirror::config_dir, Stdlib::Absolutepath $log_dir = $kafka::mirror::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::mirror::bin_dir, String $service_name = $kafka::mirror::service_name, Boolean $service_install = $kafka::mirror::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::mirror::service_ensure, Array[String] $service_requires = $kafka::mirror::service_requires, Optional[String] $limit_nofile = $kafka::mirror::limit_nofile, Optional[String] $limit_core = $kafka::mirror::limit_core, Hash $env = $kafka::mirror::env, Hash $consumer_config = $kafka::mirror::consumer_config, Hash $producer_config = $kafka::mirror::producer_config, Hash $service_config = $kafka::mirror::service_config, String $heap_opts = $kafka::mirror::heap_opts, String $jmx_opts = $kafka::mirror::jmx_opts, String $log4j_opts = $kafka::mirror::log4j_opts, ) { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } if $service_install { $env_defaults = { 'KAFKA_HEAP_OPTS' => $heap_opts, 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, } $environment = deep_merge($env_defaults, $env) if $::service_provider == 'systemd' { - include ::systemd + include systemd file { "/etc/systemd/system/${service_name}.service": ensure => file, mode => '0644', content => template('kafka/unit.erb'), } file { "/etc/init.d/${service_name}": ensure => absent, } File["/etc/systemd/system/${service_name}.service"] ~> Exec['systemctl-daemon-reload'] -> Service[$service_name] } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/producer.pp b/manifests/producer.pp index 913bec9..d083824 100644 --- a/manifests/producer.pp +++ b/manifests/producer.pp @@ -1,146 +1,146 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::producer # # This class will install kafka with the producer role. # # === Requirements/Dependencies # # Currently requires the puppetlabs/stdlib module on the Puppet Forge in # order to validate much of the the provided configuration. # # === Parameters # # [*version*] # The version of kafka that should be installed. # # [*scala_version*] # The scala version what kafka was built with. # # [*install_dir*] # The directory to install kafka to. # # [*mirror_url*] # The url where the kafka is downloaded from. # # [*install_java*] # Install java if it's not already installed. # # [*package_dir*] # The directory to install kafka. # # [*package_name*] # Package name, when installing kafka from a package. # # [*package_ensure*] # Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. # # [*user*] # User to run kafka as. # # [*group*] # Group to run kafka as. # # [*user_id*] # Create the kafka user with this ID. # # [*group_id*] # Create the kafka group with this ID. # # [*manage_user*] # Create the kafka user if it's not already present. # # [*manage_group*] # Create the kafka group if it's not already present. # # [*config_mode*] # The permissions for the config files. # # [*config_dir*] # The directory to create the kafka config files to. # # [*log_dir*] # The directory for kafka log files. # # [*bin_dir*] # The directory where the kafka scripts are. # # [*service_name*] # Set the name of the service. # # [*service_install*] # Install the init.d or systemd service. # # [*service_ensure*] # Set the ensure state of the service to 'stopped' or 'running'. # # [*service_restart*] # Whether the configuration files should trigger a service restart. # # [*service_requires*] # Set the list of services required to be running before Kafka. # # [*limit_nofile*] # Set the 'LimitNOFILE' option of the systemd service. # # [*limit_core*] # Set the 'LimitCORE' option of the systemd service. # # [*env*] # A hash of the environment variables to set. # # [*config*] # A hash of the producer configuration options. # # [*service_config*] # A hash of the `kafka-console-producer.sh` script options. # # === Examples # # Create the producer service connecting to a local zookeeper # # class { 'kafka::producer': # config => { 'client.id' => '0', 'zookeeper.connect' => 'localhost:2181' } # } # class kafka::producer ( $input, String $version = $kafka::params::version, String $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $install_java = $kafka::params::install_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String] $package_name = $kafka::params::package_name, String $package_ensure = $kafka::params::package_ensure, String $user = $kafka::params::user, String $group = $kafka::params::group, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String $service_name = 'kafka-producer', Boolean $service_install = $kafka::params::service_install, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String] $service_requires = $kafka::params::service_requires, Optional[String] $limit_nofile = $kafka::params::limit_nofile, Optional[String] $limit_core = $kafka::params::limit_core, Hash $env = {}, Hash $config = {}, Hash $service_config = {}, String $jmx_opts = $kafka::params::producer_jmx_opts, String $log4j_opts = $kafka::params::producer_log4j_opts, ) inherits kafka::params { - class { '::kafka::producer::install': } - -> class { '::kafka::producer::config': } - -> class { '::kafka::producer::service': } + class { 'kafka::producer::install': } + -> class { 'kafka::producer::config': } + -> class { 'kafka::producer::service': } -> Class['kafka::producer'] } diff --git a/manifests/producer/install.pp b/manifests/producer/install.pp index 1ae7a58..4615db8 100644 --- a/manifests/producer/install.pp +++ b/manifests/producer/install.pp @@ -1,36 +1,36 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class: kafka::producer::install # # This private class is meant to be called from `kafka::producer`. # It downloads the package and installs it. # class kafka::producer::install { if $caller_module_name != $module_name { fail("Use of private class ${name} by ${caller_module_name}") } - if !defined(Class['::kafka']) { - class { '::kafka': + if !defined(Class['kafka']) { + class { 'kafka': version => $kafka::producer::version, scala_version => $kafka::producer::scala_version, install_dir => $kafka::producer::install_dir, mirror_url => $kafka::producer::mirror_url, install_java => $kafka::producer::install_java, package_dir => $kafka::producer::package_dir, package_name => $kafka::producer::package_name, package_ensure => $kafka::producer::package_ensure, user => $kafka::producer::user, group => $kafka::producer::group, user_id => $kafka::producer::user_id, group_id => $kafka::producer::group_id, manage_user => $kafka::producer::manage_user, manage_group => $kafka::producer::manage_group, config_dir => $kafka::producer::config_dir, log_dir => $kafka::producer::log_dir, } } }