diff --git a/Gemfile b/Gemfile index 87db4f9..abd6226 100644 --- a/Gemfile +++ b/Gemfile @@ -1,48 +1,49 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" def location_for(place, fake_version = nil) if place =~ /^(git[:@][^#]*)#(.*)/ [fake_version, { :git => $1, :branch => $2, :require => false }].compact elsif place =~ /^file:\/\/(.*)/ ['>= 0', { :path => File.expand_path($1), :require => false }] else [place, { :require => false }] end end group :test do gem 'voxpupuli-test', '>= 1.4.0', :require => false gem 'coveralls', :require => false gem 'simplecov-console', :require => false end group :development do gem 'travis', :require => false gem 'travis-lint', :require => false gem 'guard-rake', :require => false gem 'overcommit', '>= 0.39.1', :require => false end group :system_tests do + gem 'beaker', '4.22.1', :require => true gem 'voxpupuli-acceptance', :require => false end group :release do gem 'github_changelog_generator', :require => false, :git => 'https://github.com/voxpupuli/github-changelog-generator', :branch => 'voxpupuli_essential_fixes' gem 'puppet-blacksmith', :require => false gem 'voxpupuli-release', :require => false gem 'puppet-strings', '>= 2.2', :require => false end if facterversion = ENV['FACTER_GEM_VERSION'] gem 'facter', facterversion.to_s, :require => false, :groups => [:test] else gem 'facter', :require => false, :groups => [:test] end ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 6.0' : puppetversion = ENV['PUPPET_VERSION'].to_s gem 'puppet', puppetversion, :require => false, :groups => [:test] # vim: syntax=ruby diff --git a/README.md b/README.md index fc6d0aa..c52b74d 100644 --- a/README.md +++ b/README.md @@ -1,360 +1,133 @@ # Kafka module for Puppet [![Build Status](https://travis-ci.org/voxpupuli/puppet-kafka.png?branch=master)](https://travis-ci.org/voxpupuli/puppet-kafka) [![Puppet Forge](https://img.shields.io/puppetforge/v/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) [![Puppet Forge - downloads](https://img.shields.io/puppetforge/dt/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) [![Puppet Forge - endorsement](https://img.shields.io/puppetforge/e/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) [![Puppet Forge - scores](https://img.shields.io/puppetforge/f/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) -#### Table of Contents +## Table of Contents 1. [Overview](#overview) -2. [Module Description - What the module does and why it is useful](#module-description) -3. [Setup - The basics of getting started with kafka](#setup) - * [What kafka affects](#what-kafka-affects) +1. [Module Description - What the module does and why it is useful](#module-description) +1. [Setup - The basics of getting started with Kafka](#setup) + * [What Kafka affects](#what-kafka-affects) * [Setup requirements](#setup-requirements) - * [Beginning with kafka](#beginning-with-kafka) -4. [Usage - Configuration options and additional functionality](#usage) -5. [Reference - An under-the-hood peek at what the module is doing and how](#reference) -5. [Limitations - OS compatibility, etc.](#limitations) -6. [Development - Guide for contributing to the module](#development) + * [Beginning with Kafka](#beginning-with-kafka) +1. [Usage - Configuration options and additional functionality](#usage) +1. [Reference - An under-the-hood peek at what the module is doing and how](#reference) +1. [Limitations - OS compatibility, etc.](#limitations) +1. [Development - Guide for contributing to the module](#development) ## Overview -The Kafka module for managing the installation and configuration of [Apache Kafka](http://kafka.apache.org) +The Kafka module for managing the installation and configuration of [Apache Kafka](http://kafka.apache.org). ## Module Description The Kafka module for managing the installation and configuration of Apache Kafka: it's brokers, producers and consumers. ## Setup -### What kafka affects +### What Kafka affects Installs the Kafka package and creates a new service. +### Setup requirements + +This module has the following dependencies: + +* [deric/zookeeper](https://github.com/deric/puppet-zookeeper) +* [camptocamp/systemd](https://github.com/camptocamp/puppet-systemd) +* [puppet/archive](https://github.com/voxpupuli/puppet-archive) +* [puppetlabs/java](https://github.com/puppetlabs/puppetlabs-java) +* [puppetlabs/stdlib](https://github.com/puppetlabs/puppetlabs-stdlib) + ### Beginning with Kafka To successfully install Kafka using this module you need to have Apache ZooKeeper already running at localhost:2181. You can specify another ZooKeeper host:port -configuration using the config hash of the kafka:broker class. +configuration using the config hash of the kafka::broker class. -The default configuration installs Kafka 0.11.0.3 binaries with Scala 2.11 +The default configuration installs Kafka 0.11.0.3 binaries with Scala 2.11: ```puppet class { 'kafka': } ``` If you want a Kafka broker server that connects to ZooKeeper listening on port 2181: ```puppet class { 'kafka::broker': - config => { 'broker.id' => '0', 'zookeeper.connect' => 'localhost:2181' } + config => { + 'broker.id' => '0', + 'zookeeper.connect' => 'localhost:2181' + } } ``` ## Usage You can specify different Kafka binaries packages versions to install. Please take a look at the different Scala and Kafka versions combinations at the [Apache Kafka Website](http://kafka.apache.org/downloads.html) ### Installing Kafka version 1.1.0 with scala 2.12 We first install the binary package with: ```puppet class { 'kafka': version => '1.1.0', scala_version => '2.12' } ``` Then we set a minimal Kafka broker configuration with: ```puppet class { 'kafka::broker': - config => { 'broker.id' => '0', 'zookeeper.connect' => 'localhost:2181' } + config => { + 'broker.id' => '0', + 'zookeeper.connect' => 'localhost:2181' + } } ``` -### Classes and Defined Types - -#### Class: `kafka` - -One of the primary classes of the kafka module. This class will install the -kafka binaries - -**Parameters within `kafka`:** - -##### `version` - -The version of kafka that should be installed. - -##### `scala_version` - -The scala version what kafka was built with. - -##### `install_dir` - -The directory to install kafka to. - -##### `mirror_url` - -The url where the kafka is downloaded from. - -##### `mirror_subpath` - -The url subpath where the kafka is downloaded from (default value: `/kafka/{version}`). - -##### `proxy_port` - -The proxy port where the kafka is downloaded from. - -##### `proxy_host` - -The proxy host where the kafka is downloaded from. - -##### `proxy_server` - -The proxy server where the kafka is downloaded from (to use instead of `proxy_port` and `proxy_host` if you need to set an url as proxy for example). - -##### `proxy_type` - -The proxy type where the kafka is downloaded from (`http` for example). - -##### `install_java` - -Install java if it's not already installed. - -##### `package_dir` - -The directory to install kafka. - -#### `package_name` - -Package name, when installing kafka from a package. - -#### `package_ensure` - -Package version (or 'present', 'absent', 'latest'), when installing kafka from -a package. - -#### `group_id` - -Create kafka group with this ID - -#### `user_id` - -Create kafka user with this ID - -#### `user` - -User to install kafka as. Defaults to the kafka user. - -#### `group` - -Group to install kafka as. Defaults to the kafka group. - -#### `config_dir` - -Directory for kafka config files. Defaults to /opt/kafka/config. - -#### `log_dir` - -Directory for kafka log files. Defaults to /var/log/kafka. - -#### Class: `kafka::broker` - -One of the primary classes of the kafka module. This class will install a kafka broker. - -**Parameters within `kafka::broker`:** - -##### `version` - -The version of kafka that should be installed. - -##### `scala_version` - -The scala version what kafka was built with. - -##### `install_dir` - -The directory to install kafka to. - -##### `mirror_url` - -The url where the kafka is downloaded from. - -##### `config` - -A hash of the configuration options. All values are used in the -`server.properties` file directly. - -##### `install_java` - -Install java if it's not already installed. - -##### `service_ensure` - -Sets the ensure state of the broker service to stopped or running. - -##### `service_install` - -Install the init.d service. - -##### `service_restart` - -Whether the configuration files should trigger a service restart - -##### `package_dir` - -The directory to install kafka. - -#### Class: `kafka::consumer` - -One of the primary classes of the kafka module. This class will install a kafka consumer. - -**Parameters within `kafka::consumer`:** - -#### `version` - -The version of kafka that should be installed. - -#### `scala_version` - -The scala version that kafka was built with. - -#### `install_dir` - -The directory to install kafka to. - -#### `mirror_url` - -The url where the kafka is downloaded from. - -#### `install_java` - -Install java if it's not already installed. - -#### `package_dir` - -The directory to install kafka. - -#### Class: `kafka::mirror` - -One of the primary classes of the kafka module. This class will install a kafka mirror. - -**Parameters within `kafka::mirror`:** - -#### `version` - -The version of kafka that should be installed. - -#### `scala_version` - -The scala version that kafka was built with. - -#### `install_dir` - -The directory to install kafka to. - -#### `mirror_url` - -The url where the kafka is downloaded from. - -#### `install_java` - -Install java if it's not already installed. - -#### `package_dir` - -The directory to install kafka. - -#### Class: `kafka::producer` - -One of the primary classes of the kafka module. This class will install a kafka producer. - -**Parameters within `kafka::producer`:** - -#### `version` - -The version of kafka that should be installed. - -#### `scala_version` - -The scala version that kafka was built with. - -#### `install_dir` - -The directory to install kafka to. - -#### `mirror_url` - -The url where the kafka is downloaded from. - -#### `install_java` - -Install java if it's not already installed. - -#### `package_dir` - -The directory to install kafka. - -#### Define: `kafka::topic` - -Defined type that creates Kafka topics. - ## Reference -### Classes - -#### Public Classes - -* `kafka`: Guides the basic installation of kafka binaries -* `kafka::broker`: Guides the basic installation of a kafka broker -* `kafka::consumer`: Guides the basic installation of a kafka consumer -* `kafka::mirror`: Guides the basic installation of a kafka mirror -* `kafka::producer`: Guides the basic installation of a kafka producer - -#### Private Classes - -* [`kafka::broker::config`] Manages all the default configuration of the kafka broker application -* [`kafka::broker::install`] Manages the installation of the kafka packages -* [`kafka::broker::service`] Manages the kafka server service -* [`kafka::consumer::config`] Manages all the default configuration of the kafka consumer application -* [`kafka::consumer::install`] Manages the installation of the kafka packages -* [`kafka::consumer::service`] Manages the kafka server service -* [`kafka::mirror::config`] Manages all the default configuration of the kafka mirror application -* [`kafka::mirror::install`] Manages the installation of the kafka packages -* [`kafka::mirror::service`] Manages the kafka server service -* [`kafka::producer::config`] Manages all the default configuration of the kafka producer application -* [`kafka::producer::install`] Manages the installation of the kafka packages -* [`kafka::producer::service`] Manages the kafka server service - -#### Defined Types - -* `kafka::topic` Create kafka topics +The [reference][1] documentation of this module is generated using [puppetlabs/puppetlabs-strings][2]. ## Limitations This module only supports Kafka >= 0.9.0.0. This module is tested on the following platforms: * Debian 8 * Debian 9 * Debian 10 * Ubuntu 16.04 * Ubuntu 18.04 * CentOS 6 * CentOS 7 It is tested with the OSS version of Puppet (>= 5.5) only. ## Development -### Contributing +This module has grown over time based on a range of contributions from people +using it. If you follow these [contributing][3] guidelines your patch will +likely make it into a release a little more quickly. + +## Author + +This module is maintained by [Vox Pupuli][4]. It was originally written and +maintained by [Liam Bennett][5]. -Please read CONTRIBUTING.md for full details on contributing to this project. +[1]: https://github.com/voxpupuli/puppet-kafka/blob/master/REFERENCE.md +[2]: https://github.com/puppetlabs/puppetlabs-strings +[3]: https://github.com/voxpupuli/puppet-kafka/blob/master/.github/CONTRIBUTING.md +[4]: https://voxpupuli.org +[5]: https://www.opentable.com \ No newline at end of file diff --git a/REFERENCE.md b/REFERENCE.md new file mode 100644 index 0000000..b9bcdc5 --- /dev/null +++ b/REFERENCE.md @@ -0,0 +1,1572 @@ +# Reference + + +## Table of Contents + +**Classes** + +_Public Classes_ + +* [`kafka`](#kafka): This class handles the Kafka requirements. +* [`kafka::broker`](#kafkabroker): This class handles the Kafka (broker). +* [`kafka::consumer`](#kafkaconsumer): This class handles the Kafka (consumer). +* [`kafka::consumer::config`](#kafkaconsumerconfig): This class handles the Kafka (consumer) config. +* [`kafka::mirror`](#kafkamirror): This class handles the Kafka (mirror). +* [`kafka::params`](#kafkaparams): This class provides default parameters. +* [`kafka::producer`](#kafkaproducer): This class handles the Kafka (producer). +* [`kafka::producer::config`](#kafkaproducerconfig): This class handles the Kafka (producer) config. + +_Private Classes_ + +* `kafka::broker::config`: This class handles the Kafka (broker) config. +* `kafka::broker::install`: This class handles the Kafka (broker) package. +* `kafka::broker::service`: This class handles the Kafka (broker) service. +* `kafka::consumer::install`: This class handles the Kafka (consumer) package. +* `kafka::consumer::service`: This class handles the Kafka (consumer) service. +* `kafka::mirror::config`: This class handles the Kafka (mirror) config. +* `kafka::mirror::install`: This class handles the Kafka (mirror) package. +* `kafka::mirror::service`: This class handles the Kafka (mirror) service. +* `kafka::producer::install`: This class handles the Kafka (producer) package. +* `kafka::producer::service`: This class handles the Kafka (producer) service. + +**Defined types** + +* [`kafka::topic`](#kafkatopic): This defined type handles the creation of Kafka topics. + +## Classes + +### kafka + +This class handles the Kafka requirements. + +#### Examples + +##### Basic usage + +```puppet +class { 'kafka': } +``` + +#### Parameters + +The following parameters are available in the `kafka` class. + +##### `kafka_version` + +Data type: `String[1]` + +The version of Kafka that should be installed. + +Default value: $kafka::params::kafka_version + +##### `scala_version` + +Data type: `String[1]` + +The scala version what Kafka was built with. + +Default value: $kafka::params::scala_version + +##### `install_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka to. + +Default value: $kafka::params::install_dir + +##### `mirror_url` + +Data type: `Stdlib::HTTPUrl` + +The url where the Kafka is downloaded from. + +Default value: $kafka::params::mirror_url + +##### `manage_java` + +Data type: `Boolean` + +Install java if it's not already installed. + +Default value: $kafka::params::manage_java + +##### `package_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka. + +Default value: $kafka::params::package_dir + +##### `package_name` + +Data type: `Optional[String[1]]` + +Package name, when installing Kafka from a package. + +Default value: $kafka::params::package_name + +##### `mirror_subpath` + +Data type: `Optional[String[1]]` + +The sub directory where the source is downloaded from. + +Default value: $kafka::params::mirror_subpath + +##### `proxy_server` + +Data type: `Optional[String[1]]` + +Set proxy server, when installing Kafka from source. + +Default value: $kafka::params::proxy_server + +##### `proxy_port` + +Data type: `Optional[String[1]]` + +Set proxy port, when installing Kafka from source. + +Default value: $kafka::params::proxy_port + +##### `proxy_host` + +Data type: `Optional[String[1]]` + +Set proxy host, when installing Kafka from source. + +Default value: $kafka::params::proxy_host + +##### `proxy_type` + +Data type: `Optional[String[1]]` + +Set proxy type, when installing Kafka from source. + +Default value: $kafka::params::proxy_type + +##### `package_ensure` + +Data type: `String[1]` + +Package version or ensure state, when installing Kafka from a package. + +Default value: $kafka::params::package_ensure + +##### `user_name` + +Data type: `String[1]` + +User to run Kafka as. + +Default value: $kafka::params::user_name + +##### `group_name` + +Data type: `String[1]` + +Group to run Kafka as. + +Default value: $kafka::params::group_name + +##### `system_user` + +Data type: `Boolean` + +Whether the Kafka user is a system user or not. + +Default value: $kafka::params::system_user + +##### `system_group` + +Data type: `Boolean` + +Whether the Kafka group is a system group or not. + +Default value: $kafka::params::system_group + +##### `user_id` + +Data type: `Optional[Integer]` + +Create the Kafka user with this ID. + +Default value: $kafka::params::user_id + +##### `group_id` + +Data type: `Optional[Integer]` + +Create the Kafka group with this ID. + +Default value: $kafka::params::group_id + +##### `manage_user` + +Data type: `Boolean` + +Create the Kafka user if it's not already present. + +Default value: $kafka::params::manage_user + +##### `manage_group` + +Data type: `Boolean` + +Create the Kafka group if it's not already present. + +Default value: $kafka::params::manage_group + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to create the Kafka config files to. + +Default value: $kafka::params::config_dir + +##### `log_dir` + +Data type: `Stdlib::Absolutepath` + +The directory for Kafka log files. + +Default value: $kafka::params::log_dir + +##### `install_mode` + +Data type: `Stdlib::Filemode` + +The permissions for the install directory. + +Default value: $kafka::params::install_mode + +### kafka::broker + +This class handles the Kafka (broker). + +#### Examples + +##### Basic usage + +```puppet +class { 'kafka::broker': + config => { + 'broker.id' => '0', + 'zookeeper.connect' => 'localhost:2181' + } +} +``` + +#### Parameters + +The following parameters are available in the `kafka::broker` class. + +##### `kafka_version` + +Data type: `String[1]` + +The version of Kafka that should be installed. + +Default value: $kafka::params::kafka_version + +##### `scala_version` + +Data type: `String[1]` + +The scala version what Kafka was built with. + +Default value: $kafka::params::scala_version + +##### `install_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka to. + +Default value: $kafka::params::install_dir + +##### `mirror_url` + +Data type: `Stdlib::HTTPUrl` + +The url where the Kafka is downloaded from. + +Default value: $kafka::params::mirror_url + +##### `manage_java` + +Data type: `Boolean` + +Install java if it's not already installed. + +Default value: $kafka::params::manage_java + +##### `package_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka. + +Default value: $kafka::params::package_dir + +##### `package_name` + +Data type: `Optional[String[1]]` + +Package name, when installing Kafka from a package. + +Default value: $kafka::params::package_name + +##### `package_ensure` + +Data type: `String[1]` + +Package version or ensure state, when installing Kafka from a package. + +Default value: $kafka::params::package_ensure + +##### `user_name` + +Data type: `String[1]` + +User to run Kafka as. + +Default value: $kafka::params::user_name + +##### `group_name` + +Data type: `String[1]` + +Group to run Kafka as. + +Default value: $kafka::params::group_name + +##### `user_id` + +Data type: `Optional[Integer]` + +Create the Kafka user with this ID. + +Default value: $kafka::params::user_id + +##### `group_id` + +Data type: `Optional[Integer]` + +Create the Kafka group with this ID. + +Default value: $kafka::params::group_id + +##### `manage_user` + +Data type: `Boolean` + +Create the Kafka user if it's not already present. + +Default value: $kafka::params::manage_user + +##### `manage_group` + +Data type: `Boolean` + +Create the Kafka group if it's not already present. + +Default value: $kafka::params::manage_group + +##### `config_mode` + +Data type: `Stdlib::Filemode` + +The permissions for the config files. + +Default value: $kafka::params::config_mode + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to create the Kafka config files to. + +Default value: $kafka::params::config_dir + +##### `log_dir` + +Data type: `Stdlib::Absolutepath` + +The directory for Kafka log files. + +Default value: $kafka::params::log_dir + +##### `bin_dir` + +Data type: `Stdlib::Absolutepath` + +The directory where the Kafka scripts are. + +Default value: $kafka::params::bin_dir + +##### `service_name` + +Data type: `String[1]` + +Set the name of the service. + +Default value: 'kafka' + +##### `manage_service` + +Data type: `Boolean` + +Install the init.d or systemd service. + +Default value: $kafka::params::manage_service + +##### `service_ensure` + +Data type: `Enum['running', 'stopped']` + +Set the ensure state of the service. + +Default value: $kafka::params::service_ensure + +##### `service_restart` + +Data type: `Boolean` + +Whether the configuration files should trigger a service restart. + +Default value: $kafka::params::service_restart + +##### `service_requires` + +Data type: `Array[String[1]]` + +Set the list of services required to be running before Kafka. + +Default value: $kafka::params::service_requires + +##### `limit_nofile` + +Data type: `Optional[String[1]]` + +Set the 'LimitNOFILE' option of the systemd service. + +Default value: $kafka::params::limit_nofile + +##### `limit_core` + +Data type: `Optional[String[1]]` + +Set the 'LimitCORE' option of the systemd service. + +Default value: $kafka::params::limit_core + +##### `timeout_stop` + +Data type: `Optional[String[1]]` + +Set the 'TimeoutStopSec' option of the systemd service. + +Default value: $kafka::params::timeout_stop + +##### `exec_stop` + +Data type: `Boolean` + +Set the 'ExecStop' option of the systemd service to 'kafka-server-stop.sh'. + +Default value: $kafka::params::exec_stop + +##### `daemon_start` + +Data type: `Boolean` + +Use the '-daemon' option when starting Kafka with 'kafka-server-start.sh'. + +Default value: $kafka::params::daemon_start + +##### `env` + +Data type: `Hash` + +A hash of the environment variables to set. + +Default value: {} + +##### `config` + +Data type: `Hash[String[1], Any]` + +A hash of the broker configuration options. + +Default value: {} + +##### `heap_opts` + +Data type: `String[1]` + +Set the Java heap size. + +Default value: $kafka::params::broker_heap_opts + +##### `jmx_opts` + +Data type: `String[1]` + +Set the JMX options. + +Default value: $kafka::params::broker_jmx_opts + +##### `log4j_opts` + +Data type: `String[1]` + +Set the Log4j options. + +Default value: $kafka::params::broker_log4j_opts + +##### `opts` + +Data type: `String[0]` + +Set the Kafka options. + +Default value: $kafka::params::broker_opts + +### kafka::consumer + +This class handles the Kafka (consumer). + +#### Examples + +##### Basic usage + +```puppet +class { 'kafka::consumer': + config => { + 'client.id' => '0', + 'zookeeper.connect' => 'localhost:2181' + } +} +``` + +#### Parameters + +The following parameters are available in the `kafka::consumer` class. + +##### `kafka_version` + +Data type: `String[1]` + +The version of Kafka that should be installed. + +Default value: $kafka::params::kafka_version + +##### `scala_version` + +Data type: `String[1]` + +The scala version what Kafka was built with. + +Default value: $kafka::params::scala_version + +##### `install_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka to. + +Default value: $kafka::params::install_dir + +##### `mirror_url` + +Data type: `Stdlib::HTTPUrl` + +The url where the Kafka is downloaded from. + +Default value: $kafka::params::mirror_url + +##### `manage_java` + +Data type: `Boolean` + +Install java if it's not already installed. + +Default value: $kafka::params::manage_java + +##### `package_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka. + +Default value: $kafka::params::package_dir + +##### `package_name` + +Data type: `Optional[String[1]]` + +Package name, when installing Kafka from a package. + +Default value: $kafka::params::package_name + +##### `package_ensure` + +Data type: `String[1]` + +Package version or ensure state, when installing Kafka from a package. + +Default value: $kafka::params::package_ensure + +##### `user_name` + +Data type: `String[1]` + +User to run Kafka as. + +Default value: $kafka::params::user_name + +##### `group_name` + +Data type: `String[1]` + +Group to run Kafka as. + +Default value: $kafka::params::group_name + +##### `user_id` + +Data type: `Optional[Integer]` + +Create the Kafka user with this ID. + +Default value: $kafka::params::user_id + +##### `group_id` + +Data type: `Optional[Integer]` + +Create the Kafka group with this ID. + +Default value: $kafka::params::group_id + +##### `manage_user` + +Data type: `Boolean` + +Create the Kafka user if it's not already present. + +Default value: $kafka::params::manage_user + +##### `manage_group` + +Data type: `Boolean` + +Create the Kafka group if it's not already present. + +Default value: $kafka::params::manage_group + +##### `config_mode` + +Data type: `Stdlib::Filemode` + +The permissions for the config files. + +Default value: $kafka::params::config_mode + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to create the Kafka config files to. + +Default value: $kafka::params::config_dir + +##### `log_dir` + +Data type: `Stdlib::Absolutepath` + +The directory for Kafka log files. + +Default value: $kafka::params::log_dir + +##### `bin_dir` + +Data type: `Stdlib::Absolutepath` + +The directory where the Kafka scripts are. + +Default value: $kafka::params::bin_dir + +##### `service_name` + +Data type: `String[1]` + +Set the name of the service. + +Default value: 'kafka-consumer' + +##### `manage_service` + +Data type: `Boolean` + +Install the init.d or systemd service. + +Default value: $kafka::params::manage_service + +##### `service_ensure` + +Data type: `Enum['running', 'stopped']` + +Set the ensure state of the service. + +Default value: $kafka::params::service_ensure + +##### `service_restart` + +Data type: `Boolean` + +Whether the configuration files should trigger a service restart. + +Default value: $kafka::params::service_restart + +##### `service_requires` + +Data type: `Array[String[1]]` + +Set the list of services required to be running before Kafka. + +Default value: $kafka::params::service_requires + +##### `limit_nofile` + +Data type: `Optional[String[1]]` + +Set the 'LimitNOFILE' option of the systemd service. + +Default value: $kafka::params::limit_nofile + +##### `limit_core` + +Data type: `Optional[String[1]]` + +Set the 'LimitCORE' option of the systemd service. + +Default value: $kafka::params::limit_core + +##### `env` + +Data type: `Hash` + +A hash of the environment variables to set. + +Default value: {} + +##### `config` + +Data type: `Hash[String[1], Any]` + +A hash of the consumer configuration options. + +Default value: {} + +##### `service_config` + +Data type: `Hash[String[1],String[1]]` + +A hash of the `kafka-console-consumer.sh` script options. + +Default value: {} + +##### `jmx_opts` + +Data type: `String[1]` + +Set the JMX options. + +Default value: $kafka::params::consumer_jmx_opts + +##### `log4j_opts` + +Data type: `String[1]` + +Set the Log4j options. + +Default value: $kafka::params::consumer_log4j_opts + +### kafka::consumer::config + +This class handles the Kafka (consumer) config. + +#### Parameters + +The following parameters are available in the `kafka::consumer::config` class. + +##### `manage_service` + +Data type: `Boolean` + + + +Default value: $kafka::consumer::manage_service + +##### `service_name` + +Data type: `String[1]` + + + +Default value: $kafka::consumer::service_name + +##### `service_restart` + +Data type: `Boolean` + + + +Default value: $kafka::consumer::service_restart + +##### `config` + +Data type: `Hash[String[1], Any]` + + + +Default value: $kafka::consumer::config + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + + + +Default value: $kafka::consumer::config_dir + +##### `user_name` + +Data type: `String[1]` + + + +Default value: $kafka::consumer::user_name + +##### `group_name` + +Data type: `String[1]` + + + +Default value: $kafka::consumer::group_name + +##### `config_mode` + +Data type: `Stdlib::Filemode` + + + +Default value: $kafka::consumer::config_mode + +### kafka::mirror + +This class handles the Kafka (mirror). + +#### Examples + +##### Basic usage + +```puppet +class { 'kafka::mirror': + consumer_config => { + 'group.id' => 'kafka-mirror', + 'zookeeper.connect' => 'localhost:2181' + }, + producer_config => { + 'zookeeper.connect' => 'localhost:2181', + }, + service_config => { + 'whitelist' => '.*', + } +} +``` + +#### Parameters + +The following parameters are available in the `kafka::mirror` class. + +##### `kafka_version` + +Data type: `String[1]` + +The version of Kafka that should be installed. + +Default value: $kafka::params::kafka_version + +##### `scala_version` + +Data type: `String[1]` + +The scala version what Kafka was built with. + +Default value: $kafka::params::scala_version + +##### `install_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka to. + +Default value: $kafka::params::install_dir + +##### `mirror_url` + +Data type: `Stdlib::HTTPUrl` + +The url where the Kafka is downloaded from. + +Default value: $kafka::params::mirror_url + +##### `manage_java` + +Data type: `Boolean` + +Install java if it's not already installed. + +Default value: $kafka::params::manage_java + +##### `package_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka. + +Default value: $kafka::params::package_dir + +##### `package_name` + +Data type: `Optional[String[1]]` + +Package name, when installing Kafka from a package. + +Default value: $kafka::params::package_name + +##### `package_ensure` + +Data type: `String[1]` + +Package version or ensure state, when installing Kafka from a package. + +Default value: $kafka::params::package_ensure + +##### `user_name` + +Data type: `String[1]` + +User to run Kafka as. + +Default value: $kafka::params::user_name + +##### `group_name` + +Data type: `String[1]` + +Group to run Kafka as. + +Default value: $kafka::params::group_name + +##### `user_id` + +Data type: `Optional[Integer]` + +Create the Kafka user with this ID. + +Default value: $kafka::params::user_id + +##### `group_id` + +Data type: `Optional[Integer]` + +Create the Kafka group with this ID. + +Default value: $kafka::params::group_id + +##### `manage_user` + +Data type: `Boolean` + +Create the Kafka user if it's not already present. + +Default value: $kafka::params::manage_user + +##### `manage_group` + +Data type: `Boolean` + +Create the Kafka group if it's not already present. + +Default value: $kafka::params::manage_group + +##### `config_mode` + +Data type: `Stdlib::Filemode` + +The permissions for the config files. + +Default value: $kafka::params::config_mode + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to create the Kafka config files to. + +Default value: $kafka::params::config_dir + +##### `log_dir` + +Data type: `Stdlib::Absolutepath` + +The directory for Kafka log files. + +Default value: $kafka::params::log_dir + +##### `bin_dir` + +Data type: `Stdlib::Absolutepath` + +The directory where the Kafka scripts are. + +Default value: $kafka::params::bin_dir + +##### `service_name` + +Data type: `String[1]` + +Set the name of the service. + +Default value: 'kafka-mirror' + +##### `manage_service` + +Data type: `Boolean` + +Install the init.d or systemd service. + +Default value: $kafka::params::manage_service + +##### `service_ensure` + +Data type: `Enum['running', 'stopped']` + +Set the ensure state of the service. + +Default value: $kafka::params::service_ensure + +##### `service_restart` + +Data type: `Boolean` + +Whether the configuration files should trigger a service restart. + +Default value: $kafka::params::service_restart + +##### `service_requires` + +Data type: `Array[String[1]]` + +Set the list of services required to be running before Kafka. + +Default value: $kafka::params::service_requires + +##### `limit_nofile` + +Data type: `Optional[String[1]]` + +Set the 'LimitNOFILE' option of the systemd service. + +Default value: $kafka::params::limit_nofile + +##### `limit_core` + +Data type: `Optional[String[1]]` + +Set the 'LimitCORE' option of the systemd service. + +Default value: $kafka::params::limit_core + +##### `env` + +Data type: `Hash` + +A hash of the environment variables to set. + +Default value: {} + +##### `consumer_config` + +Data type: `Hash[String[1],String[1]]` + +A hash of the consumer configuration options. + +Default value: {} + +##### `producer_config` + +Data type: `Hash[String[1],String[1]]` + +A hash of the producer configuration options. + +Default value: {} + +##### `service_config` + +Data type: `Hash[String[1],String[1]]` + +A hash of the mirror script options. + +Default value: {} + +##### `heap_opts` + +Data type: `String[1]` + +Set the Java heap size. + +Default value: $kafka::params::mirror_heap_opts + +##### `jmx_opts` + +Data type: `String[1]` + +Set the JMX options. + +Default value: $kafka::params::mirror_jmx_opts + +##### `log4j_opts` + +Data type: `String[1]` + +Set the Log4j options. + +Default value: $kafka::params::mirror_log4j_opts + +### kafka::params + +This class provides default parameters. + +### kafka::producer + +This class handles the Kafka (producer). + +#### Examples + +##### Basic usage + +```puppet +class { 'kafka::producer': + config => { + 'client.id' => '0', + 'zookeeper.connect' => 'localhost:2181' + } +} +``` + +#### Parameters + +The following parameters are available in the `kafka::producer` class. + +##### `input` + +Data type: `Optional[String[1]]` + +Set named pipe as input. + +##### `kafka_version` + +Data type: `String[1]` + +The version of Kafka that should be installed. + +Default value: $kafka::params::kafka_version + +##### `scala_version` + +Data type: `String[1]` + +The scala version what Kafka was built with. + +Default value: $kafka::params::scala_version + +##### `install_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka to. + +Default value: $kafka::params::install_dir + +##### `mirror_url` + +Data type: `Stdlib::HTTPUrl` + +The url where the Kafka is downloaded from. + +Default value: $kafka::params::mirror_url + +##### `manage_java` + +Data type: `Boolean` + +Install java if it's not already installed. + +Default value: $kafka::params::manage_java + +##### `package_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to install Kafka. + +Default value: $kafka::params::package_dir + +##### `package_name` + +Data type: `Optional[String[1]]` + +Package name, when installing Kafka from a package. + +Default value: $kafka::params::package_name + +##### `package_ensure` + +Data type: `String[1]` + +Package version or ensure state, when installing Kafka from a package. + +Default value: $kafka::params::package_ensure + +##### `user_name` + +Data type: `String[1]` + +User to run Kafka as. + +Default value: $kafka::params::user_name + +##### `group_name` + +Data type: `String[1]` + +Group to run Kafka as. + +Default value: $kafka::params::group_name + +##### `user_id` + +Data type: `Optional[Integer]` + +Create the Kafka user with this ID. + +Default value: $kafka::params::user_id + +##### `group_id` + +Data type: `Optional[Integer]` + +Create the Kafka group with this ID. + +Default value: $kafka::params::group_id + +##### `manage_user` + +Data type: `Boolean` + +Create the Kafka user if it's not already present. + +Default value: $kafka::params::manage_user + +##### `manage_group` + +Data type: `Boolean` + +Create the Kafka group if it's not already present. + +Default value: $kafka::params::manage_group + +##### `config_mode` + +Data type: `Stdlib::Filemode` + +The permissions for the config files. + +Default value: $kafka::params::config_mode + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + +The directory to create the Kafka config files to. + +Default value: $kafka::params::config_dir + +##### `log_dir` + +Data type: `Stdlib::Absolutepath` + +The directory for Kafka log files. + +Default value: $kafka::params::log_dir + +##### `bin_dir` + +Data type: `Stdlib::Absolutepath` + +The directory where the Kafka scripts are. + +Default value: $kafka::params::bin_dir + +##### `service_name` + +Data type: `String[1]` + +Set the name of the service. + +Default value: 'kafka-producer' + +##### `manage_service` + +Data type: `Boolean` + +Install the init.d or systemd service. + +Default value: $kafka::params::manage_service + +##### `service_ensure` + +Data type: `Enum['running', 'stopped']` + +Set the ensure state of the service. + +Default value: $kafka::params::service_ensure + +##### `service_restart` + +Data type: `Boolean` + +Whether the configuration files should trigger a service restart. + +Default value: $kafka::params::service_restart + +##### `service_requires` + +Data type: `Array[String[1]]` + +Set the list of services required to be running before Kafka. + +Default value: $kafka::params::service_requires + +##### `limit_nofile` + +Data type: `Optional[String[1]]` + +Set the 'LimitNOFILE' option of the systemd service. + +Default value: $kafka::params::limit_nofile + +##### `limit_core` + +Data type: `Optional[String[1]]` + +Set the 'LimitCORE' option of the systemd service. + +Default value: $kafka::params::limit_core + +##### `env` + +Data type: `Hash` + +A hash of the environment variables to set. + +Default value: {} + +##### `config` + +Data type: `Hash[String[1], Any]` + +A hash of the producer configuration options. + +Default value: {} + +##### `service_config` + +Data type: `Hash[String[1],String[1]]` + +A hash of the `kafka-console-producer.sh` script options. + +Default value: {} + +##### `jmx_opts` + +Data type: `String[1]` + +Set the JMX options. + +Default value: $kafka::params::producer_jmx_opts + +##### `log4j_opts` + +Data type: `String[1]` + +Set the Log4j options. + +Default value: $kafka::params::producer_log4j_opts + +### kafka::producer::config + +This class handles the Kafka (producer) config. + +#### Parameters + +The following parameters are available in the `kafka::producer::config` class. + +##### `manage_service` + +Data type: `Boolean` + + + +Default value: $kafka::producer::manage_service + +##### `service_name` + +Data type: `String[1]` + + + +Default value: $kafka::producer::service_name + +##### `service_restart` + +Data type: `Boolean` + + + +Default value: $kafka::producer::service_restart + +##### `config` + +Data type: `Hash[String[1], Any]` + + + +Default value: $kafka::producer::config + +##### `config_dir` + +Data type: `Stdlib::Absolutepath` + + + +Default value: $kafka::producer::config_dir + +##### `user_name` + +Data type: `String[1]` + + + +Default value: $kafka::producer::user_name + +##### `group_name` + +Data type: `String[1]` + + + +Default value: $kafka::producer::group_name + +##### `config_mode` + +Data type: `Stdlib::Filemode` + + + +Default value: $kafka::producer::config_mode + +## Defined types + +### kafka::topic + +This defined type handles the creation of Kafka topics. + +#### Examples + +##### Basic usage + +```puppet +kafka::topic { 'test': + ensure => present, + zookeeper => 'localhost:2181', + replication_factor => 1, + partitions => 1, +} +``` + +#### Parameters + +The following parameters are available in the `kafka::topic` defined type. + +##### `ensure` + +Data type: `String[1]` + +Should the topic be created. + +Default value: '' + +##### `zookeeper` + +Data type: `String[1]` + +The connection string for the ZooKeeper connection in the form host:port. +Multiple hosts can be given to allow fail-over. + +Default value: '' + +##### `replication_factor` + +Data type: `Variant[Integer,String[1]]` + +The replication factor for each partition in the topic being created. If +not supplied, defaults to the cluster default. + +Default value: 1 + +##### `partitions` + +Data type: `Variant[Integer,String[1]]` + +The number of partitions for the topic being created or altered. If not +supplied for create, defaults to the cluster default. + +Default value: 1 + +##### `bin_dir` + +Data type: `String[1]` + +The directory where the file kafka-topics.sh is located. + +Default value: '/opt/kafka/bin' + +##### `config` + +Data type: `Optional[Hash[String[1],String[1]]]` + +A topic configuration override for the topic being created or altered. +See the Kafka documentation for full details on the topic configs. + +Default value: `undef` + diff --git a/manifests/broker.pp b/manifests/broker.pp index b52347a..3146026 100644 --- a/manifests/broker.pp +++ b/manifests/broker.pp @@ -1,155 +1,155 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::broker -# -# This class will install kafka with the broker role. -# -# === Requirements/Dependencies +# @summary +# This class handles the Kafka (broker). # -# Currently requires the puppetlabs/stdlib module on the Puppet Forge in -# order to validate much of the the provided configuration. +# @example Basic usage +# class { 'kafka::broker': +# config => { +# 'broker.id' => '0', +# 'zookeeper.connect' => 'localhost:2181' +# } +# } # -# === Parameters +# @param kafka_version +# The version of Kafka that should be installed. # -# [*kafka_version*] -# The version of kafka that should be installed. +# @param scala_version +# The scala version what Kafka was built with. # -# [*scala_version*] -# The scala version what kafka was built with. +# @param install_dir +# The directory to install Kafka to. # -# [*install_dir*] -# The directory to install kafka to. +# @param mirror_url +# The url where the Kafka is downloaded from. # -# [*mirror_url*] -# The url where the kafka is downloaded from. +# @param manage_java +# Install java if it's not already installed. # -# [*manage_java*] -# Install java if it's not already installed. +# @param package_dir +# The directory to install Kafka. # -# [*package_dir*] -# The directory to install kafka. +# @param package_name +# Package name, when installing Kafka from a package. # -# [*package_name*] -# Package name, when installing kafka from a package. +# @param package_ensure +# Package version or ensure state, when installing Kafka from a package. # -# [*package_ensure*] -# Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. +# @param user_name +# User to run Kafka as. # -# [*user_name*] -# User to run kafka as. +# @param group_name +# Group to run Kafka as. # -# [*group_name*] -# Group to run kafka as. +# @param user_id +# Create the Kafka user with this ID. # -# [*user_id*] -# Create the kafka user with this ID. +# @param group_id +# Create the Kafka group with this ID. # -# [*group_id*] -# Create the kafka group with this ID. +# @param manage_user +# Create the Kafka user if it's not already present. # -# [*manage_user*] -# Create the kafka user if it's not already present. +# @param manage_group +# Create the Kafka group if it's not already present. # -# [*manage_group*] -# Create the kafka group if it's not already present. +# @param config_mode +# The permissions for the config files. # -# [*config_mode*] -# The permissions for the config files. +# @param config_dir +# The directory to create the Kafka config files to. # -# [*config_dir*] -# The directory to create the kafka config files to. +# @param log_dir +# The directory for Kafka log files. # -# [*log_dir*] -# The directory for kafka log files. +# @param bin_dir +# The directory where the Kafka scripts are. # -# [*bin_dir*] -# The directory where the kafka scripts are. +# @param service_name +# Set the name of the service. # -# [*service_name*] -# Set the name of the service. +# @param manage_service +# Install the init.d or systemd service. # -# [*manage_service*] -# Install the init.d or systemd service. +# @param service_ensure +# Set the ensure state of the service. # -# [*service_ensure*] -# Set the ensure state of the service to 'stopped' or 'running'. +# @param service_restart +# Whether the configuration files should trigger a service restart. # -# [*service_restart*] -# Whether the configuration files should trigger a service restart. +# @param service_requires +# Set the list of services required to be running before Kafka. # -# [*service_requires*] -# Set the list of services required to be running before Kafka. +# @param limit_nofile +# Set the 'LimitNOFILE' option of the systemd service. # -# [*limit_nofile*] -# Set the 'LimitNOFILE' option of the systemd service. +# @param limit_core +# Set the 'LimitCORE' option of the systemd service. # -# [*limit_core*] -# Set the 'LimitCORE' option of the systemd service. +# @param timeout_stop +# Set the 'TimeoutStopSec' option of the systemd service. # -# [*timeout_stop*] -# Set the 'TimeoutStopSec' option of the systemd service. +# @param exec_stop +# Set the 'ExecStop' option of the systemd service to 'kafka-server-stop.sh'. # -# [*exec_stop*] -# Set the 'ExecStop' option of the systemd service to 'kafka-server-stop.sh'. +# @param daemon_start +# Use the '-daemon' option when starting Kafka with 'kafka-server-start.sh'. # -# [*daemon_start*] -# Use the '-daemon' option when starting Kafka with 'kafka-server-start.sh'. +# @param env +# A hash of the environment variables to set. # -# [*env*] -# A hash of the environment variables to set. +# @param config +# A hash of the broker configuration options. # -# [*config*] -# A hash of the configuration options. +# @param heap_opts +# Set the Java heap size. # -# === Examples +# @param jmx_opts +# Set the JMX options. # -# Create a single broker instance which talks to a local zookeeper instance. +# @param log4j_opts +# Set the Log4j options. # -# class { 'kafka::broker': -# config => { 'broker.id' => '0', 'zookeeper.connect' => 'localhost:2181' } -# } +# @param opts +# Set the Kafka options. # class kafka::broker ( String[1] $kafka_version = $kafka::params::kafka_version, String[1] $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $manage_java = $kafka::params::manage_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String[1]] $package_name = $kafka::params::package_name, String[1] $package_ensure = $kafka::params::package_ensure, String[1] $user_name = $kafka::params::user_name, String[1] $group_name = $kafka::params::group_name, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String[1] $service_name = 'kafka', Boolean $manage_service = $kafka::params::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String[1]] $service_requires = $kafka::params::service_requires, Optional[String[1]] $limit_nofile = $kafka::params::limit_nofile, Optional[String[1]] $limit_core = $kafka::params::limit_core, Optional[String[1]] $timeout_stop = $kafka::params::timeout_stop, Boolean $exec_stop = $kafka::params::exec_stop, Boolean $daemon_start = $kafka::params::daemon_start, Hash $env = {}, Hash[String[1], Any] $config = {}, String[1] $heap_opts = $kafka::params::broker_heap_opts, String[1] $jmx_opts = $kafka::params::broker_jmx_opts, String[1] $log4j_opts = $kafka::params::broker_log4j_opts, String[0] $opts = $kafka::params::broker_opts, ) inherits kafka::params { class { 'kafka::broker::install': } -> class { 'kafka::broker::config': } -> class { 'kafka::broker::service': } -> Class['kafka::broker'] } diff --git a/manifests/broker/config.pp b/manifests/broker/config.pp index 32b62d0..1602351 100644 --- a/manifests/broker/config.pp +++ b/manifests/broker/config.pp @@ -1,39 +1,35 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::broker::config +# @summary +# This class handles the Kafka (broker) config. # -# This private class is meant to be called from `kafka::broker`. -# It manages the broker config files +# @api private # class kafka::broker::config( Boolean $manage_service = $kafka::broker::manage_service, String[1] $service_name = $kafka::broker::service_name, Boolean $service_restart = $kafka::broker::service_restart, Hash[String[1], Any] $config = $kafka::broker::config, Stdlib::Absolutepath $config_dir = $kafka::broker::config_dir, String[1] $user_name = $kafka::broker::user_name, String[1] $group_name = $kafka::broker::group_name, Stdlib::Filemode $config_mode = $kafka::broker::config_mode, ) { assert_private() if ($manage_service and $service_restart) { $config_notify = Service[$service_name] } else { $config_notify = undef } $doctag = 'brokerconfigs' file { "${config_dir}/server.properties": ensure => present, owner => $user_name, group => $group_name, mode => $config_mode, content => template('kafka/properties.erb'), notify => $config_notify, require => File[$config_dir], } } diff --git a/manifests/broker/install.pp b/manifests/broker/install.pp index 6c15e0f..b09f511 100644 --- a/manifests/broker/install.pp +++ b/manifests/broker/install.pp @@ -1,34 +1,30 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::broker::install +# @summary +# This class handles the Kafka (broker) package. # -# This private class is meant to be called from `kafka::broker`. -# It downloads the package and installs it. +# @api private # class kafka::broker::install { assert_private() if !defined(Class['kafka']) { class { 'kafka': manage_java => $kafka::broker::manage_java, manage_group => $kafka::broker::manage_group, group_id => $kafka::broker::group_id, group_name => $kafka::broker::group_name, manage_user => $kafka::broker::manage_user, user_id => $kafka::broker::user_id, user_name => $kafka::broker::user_name, config_dir => $kafka::broker::config_dir, log_dir => $kafka::broker::log_dir, mirror_url => $kafka::broker::mirror_url, kafka_version => $kafka::broker::kafka_version, scala_version => $kafka::broker::scala_version, install_dir => $kafka::broker::install_dir, package_dir => $kafka::broker::package_dir, package_ensure => $kafka::broker::package_ensure, package_name => $kafka::broker::package_name, } } } diff --git a/manifests/broker/service.pp b/manifests/broker/service.pp index d6e12d3..a2f66fa 100644 --- a/manifests/broker/service.pp +++ b/manifests/broker/service.pp @@ -1,77 +1,72 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::broker::service +# @summary +# This class handles the Kafka (broker) service. # -# This private class is meant to be called from `kafka::broker`. -# It manages the kafka service +# @api private # class kafka::broker::service( Boolean $manage_service = $kafka::broker::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::broker::service_ensure, String[1] $service_name = $kafka::broker::service_name, String[1] $user_name = $kafka::broker::user_name, String[1] $group_name = $kafka::broker::group_name, Stdlib::Absolutepath $config_dir = $kafka::broker::config_dir, Stdlib::Absolutepath $log_dir = $kafka::broker::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::broker::bin_dir, Array[String[1]] $service_requires = $kafka::broker::service_requires, Optional[String[1]] $limit_nofile = $kafka::broker::limit_nofile, Optional[String[1]] $limit_core = $kafka::broker::limit_core, Optional[String[1]] $timeout_stop = $kafka::broker::timeout_stop, Boolean $exec_stop = $kafka::broker::exec_stop, Boolean $daemon_start = $kafka::broker::daemon_start, Hash $env = $kafka::broker::env, String[1] $heap_opts = $kafka::broker::heap_opts, String[1] $jmx_opts = $kafka::broker::jmx_opts, String[1] $log4j_opts = $kafka::broker::log4j_opts, String[0] $opts = $kafka::broker::opts, ) { assert_private() if $manage_service { $env_defaults = { 'KAFKA_HEAP_OPTS' => $heap_opts, 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, 'KAFKA_OPTS' => $opts, 'LOG_DIR' => $log_dir, } $environment = deep_merge($env_defaults, $env) if $facts['service_provider'] == 'systemd' { include systemd file { "/etc/systemd/system/${service_name}.service": ensure => file, mode => '0644', content => template('kafka/unit.erb'), } file { "/etc/init.d/${service_name}": ensure => absent, } File["/etc/systemd/system/${service_name}.service"] ~> Exec['systemctl-daemon-reload'] -> Service[$service_name] - } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/consumer.pp b/manifests/consumer.pp index 6c6bd94..a496688 100644 --- a/manifests/consumer.pp +++ b/manifests/consumer.pp @@ -1,144 +1,139 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::consumer -# -# This class will install kafka with the consumer role. -# -# === Requirements/Dependencies -# -# Currently requires the puppetlabs/stdlib module on the Puppet Forge in -# order to validate much of the the provided configuration. +# @summary +# This class handles the Kafka (consumer). # -# === Parameters +# @example Basic usage +# class { 'kafka::consumer': +# config => { +# 'client.id' => '0', +# 'zookeeper.connect' => 'localhost:2181' +# } +# } # -# [*kafka_version*] -# The version of kafka that should be installed. +# @param kafka_version +# The version of Kafka that should be installed. # -# [*scala_version*] -# The scala version what kafka was built with. +# @param scala_version +# The scala version what Kafka was built with. # -# [*install_dir*] -# The directory to install kafka to. +# @param install_dir +# The directory to install Kafka to. # -# [*mirror_url*] -# The url where the kafka is downloaded from. +# @param mirror_url +# The url where the Kafka is downloaded from. # -# [*manage_java*] -# Install java if it's not already installed. +# @param manage_java +# Install java if it's not already installed. # -# [*package_dir*] -# The directory to install kafka. +# @param package_dir +# The directory to install Kafka. # -# [*package_name*] -# Package name, when installing kafka from a package. +# @param package_name +# Package name, when installing Kafka from a package. # -# [*package_ensure*] -# Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. +# @param package_ensure +# Package version or ensure state, when installing Kafka from a package. # -# [*user_name*] -# User to run kafka as. +# @param user_name +# User to run Kafka as. # -# [*group_name*] -# Group to run kafka as. +# @param group_name +# Group to run Kafka as. # -# [*user_id*] -# Create the kafka user with this ID. +# @param user_id +# Create the Kafka user with this ID. # -# [*group_id*] -# Create the kafka group with this ID. +# @param group_id +# Create the Kafka group with this ID. # -# [*manage_user*] -# Create the kafka user if it's not already present. +# @param manage_user +# Create the Kafka user if it's not already present. # -# [*manage_group*] -# Create the kafka group if it's not already present. +# @param manage_group +# Create the Kafka group if it's not already present. # -# [*config_mode*] -# The permissions for the config files. +# @param config_mode +# The permissions for the config files. # -# [*config_dir*] -# The directory to create the kafka config files to. +# @param config_dir +# The directory to create the Kafka config files to. # -# [*log_dir*] -# The directory for kafka log files. +# @param log_dir +# The directory for Kafka log files. # -# [*bin_dir*] -# The directory where the kafka scripts are. +# @param bin_dir +# The directory where the Kafka scripts are. # -# [*service_name*] -# Set the name of the service. +# @param service_name +# Set the name of the service. # -# [*manage_service*] -# Install the init.d or systemd service. +# @param manage_service +# Install the init.d or systemd service. # -# [*service_ensure*] -# Set the ensure state of the service to 'stopped' or 'running'. +# @param service_ensure +# Set the ensure state of the service. # -# [*service_restart*] -# Whether the configuration files should trigger a service restart. +# @param service_restart +# Whether the configuration files should trigger a service restart. # -# [*service_requires*] -# Set the list of services required to be running before Kafka. +# @param service_requires +# Set the list of services required to be running before Kafka. # -# [*limit_nofile*] -# Set the 'LimitNOFILE' option of the systemd service. +# @param limit_nofile +# Set the 'LimitNOFILE' option of the systemd service. # -# [*limit_core*] -# Set the 'LimitCORE' option of the systemd service. +# @param limit_core +# Set the 'LimitCORE' option of the systemd service. # -# [*env*] -# A hash of the environment variables to set. +# @param env +# A hash of the environment variables to set. # -# [*config*] -# A hash of the consumer configuration options. +# @param config +# A hash of the consumer configuration options. # -# [*service_config*] -# A hash of the `kafka-console-consumer.sh` script options. +# @param service_config +# A hash of the `kafka-console-consumer.sh` script options. # -# === Examples +# @param jmx_opts +# Set the JMX options. # -# Create the consumer service connecting to a local zookeeper +# @param log4j_opts +# Set the Log4j options. # -# class { 'kafka::consumer': -# config => { 'client.id' => '0', 'zookeeper.connect' => 'localhost:2181' } -# } class kafka::consumer ( String[1] $kafka_version = $kafka::params::kafka_version, String[1] $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $manage_java = $kafka::params::manage_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String[1]] $package_name = $kafka::params::package_name, String[1] $package_ensure = $kafka::params::package_ensure, String[1] $user_name = $kafka::params::user_name, String[1] $group_name = $kafka::params::group_name, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String[1] $service_name = 'kafka-consumer', Boolean $manage_service = $kafka::params::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String[1]] $service_requires = $kafka::params::service_requires, Optional[String[1]] $limit_nofile = $kafka::params::limit_nofile, Optional[String[1]] $limit_core = $kafka::params::limit_core, Hash $env = {}, Hash[String[1], Any] $config = {}, Hash[String[1],String[1]] $service_config = {}, String[1] $jmx_opts = $kafka::params::consumer_jmx_opts, String[1] $log4j_opts = $kafka::params::consumer_log4j_opts, ) inherits kafka::params { class { 'kafka::consumer::install': } -> class { 'kafka::consumer::config': } -> class { 'kafka::consumer::service': } -> Class['kafka::consumer'] } diff --git a/manifests/consumer/config.pp b/manifests/consumer/config.pp index b2d62ed..6e92d9d 100644 --- a/manifests/consumer/config.pp +++ b/manifests/consumer/config.pp @@ -1,37 +1,31 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::consumer::config -# -# This private class is meant to be called from `kafka::consumer`. -# It manages the consumer config files +# @summary +# This class handles the Kafka (consumer) config. # class kafka::consumer::config( Boolean $manage_service = $kafka::consumer::manage_service, String[1] $service_name = $kafka::consumer::service_name, Boolean $service_restart = $kafka::consumer::service_restart, Hash[String[1], Any] $config = $kafka::consumer::config, Stdlib::Absolutepath $config_dir = $kafka::consumer::config_dir, String[1] $user_name = $kafka::consumer::user_name, String[1] $group_name = $kafka::consumer::group_name, Stdlib::Filemode $config_mode = $kafka::consumer::config_mode, ) { if ($manage_service and $service_restart) { $config_notify = Service[$service_name] } else { $config_notify = undef } $doctag = 'consumerconfigs' file { "${config_dir}/consumer.properties": ensure => present, owner => $user_name, group => $group_name, mode => $config_mode, content => template('kafka/properties.erb'), notify => $config_notify, require => File[$config_dir], } } diff --git a/manifests/consumer/install.pp b/manifests/consumer/install.pp index 724baaa..ed9f290 100644 --- a/manifests/consumer/install.pp +++ b/manifests/consumer/install.pp @@ -1,34 +1,30 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::consumer::install +# @summary +# This class handles the Kafka (consumer) package. # -# This private class is meant to be called from `kafka::consumer`. -# It downloads the package and installs it. +# @api private # class kafka::consumer::install { assert_private() if !defined(Class['kafka']) { class { 'kafka': manage_java => $kafka::consumer::manage_java, manage_group => $kafka::consumer::manage_group, group_id => $kafka::consumer::group_id, group_name => $kafka::consumer::group_name, manage_user => $kafka::consumer::manage_user, user_id => $kafka::consumer::user_id, user_name => $kafka::consumer::user_name, config_dir => $kafka::consumer::config_dir, log_dir => $kafka::consumer::log_dir, mirror_url => $kafka::consumer::mirror_url, kafka_version => $kafka::consumer::kafka_version, scala_version => $kafka::consumer::scala_version, install_dir => $kafka::consumer::install_dir, package_dir => $kafka::consumer::package_dir, package_ensure => $kafka::consumer::package_ensure, package_name => $kafka::consumer::package_name, } } } diff --git a/manifests/consumer/service.pp b/manifests/consumer/service.pp index da8e60a..287ae32 100644 --- a/manifests/consumer/service.pp +++ b/manifests/consumer/service.pp @@ -1,78 +1,73 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::consumer::service +# @summary +# This class handles the Kafka (consumer) service. # -# This private class is meant to be called from `kafka::consumer`. -# It manages the kafka-consumer service +# @api private # class kafka::consumer::service( Boolean $manage_service = $kafka::consumer::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::consumer::service_ensure, String[1] $service_name = $kafka::consumer::service_name, String[1] $user_name = $kafka::consumer::user_name, String[1] $group_name = $kafka::consumer::group_name, Stdlib::Absolutepath $config_dir = $kafka::consumer::config_dir, Stdlib::Absolutepath $log_dir = $kafka::consumer::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::consumer::bin_dir, Array[String[1]] $service_requires = $kafka::consumer::service_requires, Optional[String[1]] $limit_nofile = $kafka::consumer::limit_nofile, Optional[String[1]] $limit_core = $kafka::consumer::limit_core, Hash $env = $kafka::consumer::env, String[1] $jmx_opts = $kafka::consumer::jmx_opts, String[1] $log4j_opts = $kafka::consumer::log4j_opts, Hash[String[1],String[1]] $service_config = $kafka::consumer::service_config, ) { assert_private() if $manage_service { if $service_config['topic'] == '' { fail('[Consumer] You need to specify a value for topic') } if $service_config['zookeeper'] == '' { fail('[Consumer] You need to specify a value for zookeeper') } $env_defaults = { 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, } $environment = deep_merge($env_defaults, $env) if $facts['service_provider'] == 'systemd' { include systemd file { "/etc/systemd/system/${service_name}.service": ensure => file, mode => '0644', content => template('kafka/unit.erb'), } file { "/etc/init.d/${service_name}": ensure => absent, } File["/etc/systemd/system/${service_name}.service"] ~> Exec['systemctl-daemon-reload'] -> Service[$service_name] - } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/init.pp b/manifests/init.pp index a9e1fa8..0c76be4 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,230 +1,232 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka +# @summary +# This class handles the Kafka requirements. +# +# @example Basic usage +# class { 'kafka': } # -# This class will install kafka binaries +# @param kafka_version +# The version of Kafka that should be installed. # -# === Requirements/Dependencies +# @param scala_version +# The scala version what Kafka was built with. # -# Currently requires the puppetlabs/stdlib module on the Puppet Forge in -# order to validate much of the the provided configuration. +# @param install_dir +# The directory to install Kafka to. # -# === Parameters +# @param mirror_url +# The url where the Kafka is downloaded from. # -# [*kafka_version*] -# The version of kafka that should be installed. +# @param manage_java +# Install java if it's not already installed. # -# [*scala_version*] -# The scala version what kafka was built with. +# @param package_dir +# The directory to install Kafka. # -# [*install_dir*] -# The directory to install kafka to. +# @param package_name +# Package name, when installing Kafka from a package. # -# [*mirror_url*] -# The url where the kafka is downloaded from. +# @param mirror_subpath +# The sub directory where the source is downloaded from. # -# [*manage_java*] -# Install java if it's not already installed. +# @param proxy_server +# Set proxy server, when installing Kafka from source. # -# [*package_dir*] -# The directory to install kafka. +# @param proxy_port +# Set proxy port, when installing Kafka from source. # -# [*package_name*] -# Package name, when installing kafka from a package. +# @param proxy_host +# Set proxy host, when installing Kafka from source. # -# [*package_ensure*] -# Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. +# @param proxy_type +# Set proxy type, when installing Kafka from source. # -# [*user_name*] -# User to run kafka as. +# @param package_ensure +# Package version or ensure state, when installing Kafka from a package. # -# [*group_name*] -# Group to run kafka as. +# @param user_name +# User to run Kafka as. # -# [*user_id*] -# Create the kafka user with this ID. +# @param group_name +# Group to run Kafka as. # -# [*system_user*] -# Whether the kafka user is a system user or not. +# @param system_user +# Whether the Kafka user is a system user or not. # -# [*group_id*] -# Create the kafka group with this ID. +# @param system_group +# Whether the Kafka group is a system group or not. # -# [*system_group*] -# Whether the kafka group is a system group or not. +# @param user_id +# Create the Kafka user with this ID. # -# [*manage_user*] -# Create the kafka user if it's not already present. +# @param group_id +# Create the Kafka group with this ID. # -# [*manage_group*] -# Create the kafka group if it's not already present. +# @param manage_user +# Create the Kafka user if it's not already present. # -# [*config_dir*] -# The directory to create the kafka config files to. +# @param manage_group +# Create the Kafka group if it's not already present. # -# [*log_dir*] -# The directory for kafka log files. +# @param config_dir +# The directory to create the Kafka config files to. # -# === Examples +# @param log_dir +# The directory for Kafka log files. # +# @param install_mode +# The permissions for the install directory. # class kafka ( String[1] $kafka_version = $kafka::params::kafka_version, String[1] $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $manage_java = $kafka::params::manage_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String[1]] $package_name = $kafka::params::package_name, Optional[String[1]] $mirror_subpath = $kafka::params::mirror_subpath, Optional[String[1]] $proxy_server = $kafka::params::proxy_server, Optional[String[1]] $proxy_port = $kafka::params::proxy_port, Optional[String[1]] $proxy_host = $kafka::params::proxy_host, Optional[String[1]] $proxy_type = $kafka::params::proxy_type, String[1] $package_ensure = $kafka::params::package_ensure, String[1] $user_name = $kafka::params::user_name, String[1] $group_name = $kafka::params::group_name, Boolean $system_user = $kafka::params::system_user, Boolean $system_group = $kafka::params::system_group, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, - Optional[String[1]] $install_mode = $kafka::params::install_mode, + Stdlib::Filemode $install_mode = $kafka::params::install_mode, ) inherits kafka::params { if $manage_java { class { 'java': distribution => 'jdk', } } if $manage_group { group { $group_name: ensure => present, gid => $group_id, system => $system_group, } } if $manage_user { user { $user_name: ensure => present, shell => '/bin/bash', require => Group[$group_name], uid => $user_id, system => $system_user, } } file { $config_dir: ensure => directory, owner => $user_name, group => $group_name, } file { $log_dir: ensure => directory, owner => $user_name, group => $group_name, require => [ Group[$group_name], User[$user_name], ], } if $package_name == undef { - include archive $mirror_path = $mirror_subpath ? { # if mirror_subpath was not changed, # we adapt it for the version $kafka::params::mirror_subpath => "kafka/${kafka_version}", # else, we just take whatever was supplied: default => $mirror_subpath, } $basefilename = "kafka_${scala_version}-${kafka_version}.tgz" $package_url = "${mirror_url}${mirror_path}/${basefilename}" $source = $mirror_url ?{ /tgz$/ => $mirror_url, default => $package_url, } $install_directory = $install_dir ? { # if install_dir was not changed, # we adapt it for the scala_version and the version $kafka::params::install_dir => "/opt/kafka-${scala_version}-${kafka_version}", # else, we just take whatever was supplied: default => $install_dir, } file { $package_dir: ensure => directory, owner => $user_name, group => $group_name, require => [ Group[$group_name], User[$user_name], ], } file { $install_directory: ensure => directory, owner => $user_name, group => $group_name, mode => $install_mode, require => [ Group[$group_name], User[$user_name], ], } file { '/opt/kafka': ensure => link, target => $install_directory, require => File[$install_directory], } if $proxy_server == undef and $proxy_host != undef and $proxy_port != undef { $final_proxy_server = "${proxy_host}:${proxy_port}" } else { $final_proxy_server = $proxy_server } archive { "${package_dir}/${basefilename}": ensure => present, extract => true, extract_command => 'tar xfz %s --strip-components=1', extract_path => $install_directory, source => $source, creates => "${install_directory}/config", cleanup => true, proxy_server => $final_proxy_server, proxy_type => $proxy_type, user => $user_name, group => $group_name, require => [ File[$package_dir], File[$install_directory], Group[$group_name], User[$user_name], ], before => File[$config_dir], } - } else { - package { $package_name: ensure => $package_ensure, before => File[$config_dir], } - } } diff --git a/manifests/mirror.pp b/manifests/mirror.pp index 939b56e..674db86 100644 --- a/manifests/mirror.pp +++ b/manifests/mirror.pp @@ -1,147 +1,153 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::mirror -# -# This class will install kafka with the mirror role. -# -# === Requirements/Dependencies +# @summary +# This class handles the Kafka (mirror). # -# Currently requires the puppetlabs/stdlib module on the Puppet Forge in -# order to validate much of the the provided configuration. +# @example Basic usage +# class { 'kafka::mirror': +# consumer_config => { +# 'group.id' => 'kafka-mirror', +# 'zookeeper.connect' => 'localhost:2181' +# }, +# producer_config => { +# 'zookeeper.connect' => 'localhost:2181', +# }, +# service_config => { +# 'whitelist' => '.*', +# } +# } # -# === Parameters +# @param kafka_version +# The version of Kafka that should be installed. # -# [*kafka_version*] -# The version of kafka that should be installed. +# @param scala_version +# The scala version what Kafka was built with. # -# [*scala_version*] -# The scala version what kafka was built with. +# @param install_dir +# The directory to install Kafka to. # -# [*install_dir*] -# The directory to install kafka to. +# @param mirror_url +# The url where the Kafka is downloaded from. # -# [*mirror_url*] -# The url where the kafka is downloaded from. +# @param manage_java +# Install java if it's not already installed. # -# [*manage_java*] -# Install java if it's not already installed. +# @param package_dir +# The directory to install Kafka. # -# [*package_dir*] -# The directory to install kafka. +# @param package_name +# Package name, when installing Kafka from a package. # -# [*package_name*] -# Package name, when installing kafka from a package. +# @param package_ensure +# Package version or ensure state, when installing Kafka from a package. # -# [*package_ensure*] -# Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. +# @param user_name +# User to run Kafka as. # -# [*user_name*] -# User to run kafka as. +# @param group_name +# Group to run Kafka as. # -# [*group_name*] -# Group to run kafka as. +# @param user_id +# Create the Kafka user with this ID. # -# [*user_id*] -# Create the kafka user with this ID. +# @param group_id +# Create the Kafka group with this ID. # -# [*group_id*] -# Create the kafka group with this ID. +# @param manage_user +# Create the Kafka user if it's not already present. # -# [*manage_user*] -# Create the kafka user if it's not already present. +# @param manage_group +# Create the Kafka group if it's not already present. # -# [*manage_group*] -# Create the kafka group if it's not already present. +# @param config_mode +# The permissions for the config files. # -# [*config_dir*] -# The directory to create the kafka config files to. +# @param config_dir +# The directory to create the Kafka config files to. # -# [*log_dir*] -# The directory for kafka log files. +# @param log_dir +# The directory for Kafka log files. # -# [*bin_dir*] -# The directory where the kafka scripts are. +# @param bin_dir +# The directory where the Kafka scripts are. # -# [*service_name*] -# Set the name of the service. +# @param service_name +# Set the name of the service. # -# [*manage_service*] -# Install the init.d or systemd service. +# @param manage_service +# Install the init.d or systemd service. # -# [*service_ensure*] -# Set the ensure state of the service to 'stopped' or 'running'. +# @param service_ensure +# Set the ensure state of the service. # -# [*service_restart*] -# Whether the configuration files should trigger a service restart. +# @param service_restart +# Whether the configuration files should trigger a service restart. # -# [*service_requires*] -# Set the list of services required to be running before Kafka. +# @param service_requires +# Set the list of services required to be running before Kafka. # -# [*limit_nofile*] -# Set the 'LimitNOFILE' option of the systemd service. +# @param limit_nofile +# Set the 'LimitNOFILE' option of the systemd service. # -# [*limit_core*] -# Set the 'LimitCORE' option of the systemd service. +# @param limit_core +# Set the 'LimitCORE' option of the systemd service. # -# [*env*] -# A hash of the environment variables to set. +# @param env +# A hash of the environment variables to set. # -# [*consumer_config*] -# A hash of the consumer configuration options. +# @param consumer_config +# A hash of the consumer configuration options. # -# [*producer_config*] -# A hash of the producer configuration options. +# @param producer_config +# A hash of the producer configuration options. # -# [*service_config*] -# A hash of the mirror script options. +# @param service_config +# A hash of the mirror script options. # -# === Examples +# @param heap_opts +# Set the Java heap size. # -# Create the mirror service connecting to a local zookeeper +# @param jmx_opts +# Set the JMX options. # -# class { 'kafka::mirror': -# consumer_config => { 'client.id' => '0', 'zookeeper.connect' => 'localhost:2181' } -# } +# @param log4j_opts +# Set the Log4j options. # class kafka::mirror ( String[1] $kafka_version = $kafka::params::kafka_version, String[1] $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $manage_java = $kafka::params::manage_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String[1]] $package_name = $kafka::params::package_name, String[1] $package_ensure = $kafka::params::package_ensure, String[1] $user_name = $kafka::params::user_name, String[1] $group_name = $kafka::params::group_name, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String[1] $service_name = 'kafka-mirror', Boolean $manage_service = $kafka::params::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String[1]] $service_requires = $kafka::params::service_requires, Optional[String[1]] $limit_nofile = $kafka::params::limit_nofile, Optional[String[1]] $limit_core = $kafka::params::limit_core, Hash $env = {}, Hash[String[1],String[1]] $consumer_config = {}, Hash[String[1],String[1]] $producer_config = {}, Hash[String[1],String[1]] $service_config = {}, String[1] $heap_opts = $kafka::params::mirror_heap_opts, String[1] $jmx_opts = $kafka::params::mirror_jmx_opts, String[1] $log4j_opts = $kafka::params::mirror_log4j_opts, ) inherits kafka::params { class { 'kafka::mirror::install': } -> class { 'kafka::mirror::config': } -> class { 'kafka::mirror::service': } -> Class['kafka::mirror'] } diff --git a/manifests/mirror/config.pp b/manifests/mirror/config.pp index 076bfe6..38a667a 100644 --- a/manifests/mirror/config.pp +++ b/manifests/mirror/config.pp @@ -1,55 +1,51 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::mirror::config +# @summary +# This class handles the Kafka (mirror) config. # -# This private class is meant to be called from `kafka::mirror`. -# It manages the mirror-maker config files +# @api private # class kafka::mirror::config( Boolean $manage_service = $kafka::mirror::manage_service, String[1] $service_name = $kafka::mirror::service_name, Boolean $service_restart = $kafka::mirror::service_restart, Hash[String[1],String[1]] $consumer_config = $kafka::mirror::consumer_config, Hash[String[1],String[1]] $producer_config = $kafka::mirror::producer_config, Stdlib::Absolutepath $config_dir = $kafka::mirror::config_dir, String[1] $user_name = $kafka::mirror::user_name, String[1] $group_name = $kafka::mirror::group_name, Stdlib::Filemode $config_mode = $kafka::mirror::config_mode, ) { assert_private() if $consumer_config['group.id'] == '' { fail('[Consumer] You need to specify a value for group.id') } if $consumer_config['zookeeper.connect'] == '' { fail('[Consumer] You need to specify a value for zookeeper.connect') } if $producer_config['bootstrap.servers'] == '' { fail('[Producer] You need to specify a value for bootstrap.servers') } class { 'kafka::consumer::config': manage_service => $manage_service, service_name => $service_name, service_restart => $service_restart, config => $consumer_config, config_dir => $config_dir, user_name => $user_name, group_name => $group_name, config_mode => $config_mode, } class { 'kafka::producer::config': manage_service => $manage_service, service_name => $service_name, service_restart => $service_restart, config => $producer_config, config_dir => $config_dir, user_name => $user_name, group_name => $group_name, config_mode => $config_mode, } } diff --git a/manifests/mirror/install.pp b/manifests/mirror/install.pp index cbc030b..1d3f741 100644 --- a/manifests/mirror/install.pp +++ b/manifests/mirror/install.pp @@ -1,34 +1,30 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::mirror::install +# @summary +# This class handles the Kafka (mirror) package. # -# This private class is meant to be called from `kafka::mirror`. -# It downloads the package and installs it. +# @api private # class kafka::mirror::install { assert_private() if !defined(Class['kafka']) { class { 'kafka': manage_java => $kafka::mirror::manage_java, manage_group => $kafka::mirror::manage_group, group_id => $kafka::mirror::group_id, group_name => $kafka::mirror::group_name, manage_user => $kafka::mirror::manage_user, user_id => $kafka::mirror::user_id, user_name => $kafka::mirror::user_name, config_dir => $kafka::mirror::config_dir, log_dir => $kafka::mirror::log_dir, mirror_url => $kafka::mirror::mirror_url, kafka_version => $kafka::mirror::kafka_version, scala_version => $kafka::mirror::scala_version, install_dir => $kafka::mirror::install_dir, package_dir => $kafka::mirror::package_dir, package_ensure => $kafka::mirror::package_ensure, package_name => $kafka::mirror::package_name, } } } diff --git a/manifests/mirror/service.pp b/manifests/mirror/service.pp index 834172e..b4132cb 100644 --- a/manifests/mirror/service.pp +++ b/manifests/mirror/service.pp @@ -1,74 +1,69 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::mirror::service +# @summary +# This class handles the Kafka (mirror) service. # -# This private class is meant to be called from `kafka::mirror`. -# It manages the kafka-mirror service +# @api private # class kafka::mirror::service( Boolean $manage_service = $kafka::mirror::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::mirror::service_ensure, String[1] $service_name = $kafka::mirror::service_name, String[1] $user_name = $kafka::mirror::user_name, String[1] $group_name = $kafka::mirror::group_name, Stdlib::Absolutepath $config_dir = $kafka::mirror::config_dir, Stdlib::Absolutepath $log_dir = $kafka::mirror::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::mirror::bin_dir, Array[String[1]] $service_requires = $kafka::mirror::service_requires, Optional[String[1]] $limit_nofile = $kafka::mirror::limit_nofile, Optional[String[1]] $limit_core = $kafka::mirror::limit_core, Hash $env = $kafka::mirror::env, Hash[String[1],String[1]] $consumer_config = $kafka::mirror::consumer_config, Hash[String[1],String[1]] $producer_config = $kafka::mirror::producer_config, Hash[String[1],String[1]] $service_config = $kafka::mirror::service_config, String[1] $heap_opts = $kafka::mirror::heap_opts, String[1] $jmx_opts = $kafka::mirror::jmx_opts, String[1] $log4j_opts = $kafka::mirror::log4j_opts, ) { assert_private() if $manage_service { $env_defaults = { 'KAFKA_HEAP_OPTS' => $heap_opts, 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, } $environment = deep_merge($env_defaults, $env) if $::service_provider == 'systemd' { include systemd file { "/etc/systemd/system/${service_name}.service": ensure => file, mode => '0644', content => template('kafka/unit.erb'), } file { "/etc/init.d/${service_name}": ensure => absent, } File["/etc/systemd/system/${service_name}.service"] ~> Exec['systemctl-daemon-reload'] -> Service[$service_name] - } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/params.pp b/manifests/params.pp index b3f08d9..5a678bb 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -1,73 +1,64 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class kafka::params -# -# This class is meant to be called from kafka::broker -# It sets variables according to platform +# @summary +# This class provides default parameters. # class kafka::params { - - # this is all only tested on Debian and RedHat - # params gets included everywhere so we can do the validation here unless $facts['os']['family'] =~ /(RedHat|Debian)/ { warning("${facts['os']['family']} is not supported") } + $kafka_version = '2.4.1' $scala_version = '2.12' $install_dir = "/opt/kafka-${scala_version}-${kafka_version}" $config_dir = '/opt/kafka/config' $bin_dir = '/opt/kafka/bin' $log_dir = '/var/log/kafka' $mirror_url = 'https://www.apache.org/dyn/closer.lua?action=download&filename=' $mirror_subpath = "kafka/${kafka_version}" $manage_java = false $package_dir = '/var/tmp/kafka' $package_name = undef $proxy_server = undef $proxy_host = undef $proxy_port = undef $proxy_type = undef $package_ensure = 'present' $user_name = 'kafka' $group_name = 'kafka' $user_id = undef $group_id = undef $system_user = false $system_group = false $manage_user = true $manage_group = true $config_mode = '0644' $install_mode = '0755' $manage_service = true $service_ensure = 'running' $service_restart = true $service_requires = $facts['os']['family'] ? { 'RedHat' => ['network.target', 'syslog.target'], default => [], } $limit_nofile = undef $limit_core = undef $timeout_stop = undef $exec_stop = false $daemon_start = false $broker_heap_opts = '-Xmx1G -Xms1G' $broker_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false \ -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9990' $broker_log4j_opts = "-Dlog4j.configuration=file:${config_dir}/log4j.properties" $broker_opts = '' $mirror_heap_opts = '-Xmx256M' $mirror_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9991' $mirror_log4j_opts = $broker_log4j_opts $producer_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9992' $producer_log4j_opts = $broker_log4j_opts $consumer_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9993' $consumer_log4j_opts = $broker_log4j_opts - } diff --git a/manifests/producer.pp b/manifests/producer.pp index 7f46b3e..5ade012 100644 --- a/manifests/producer.pp +++ b/manifests/producer.pp @@ -1,146 +1,143 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::producer -# -# This class will install kafka with the producer role. -# -# === Requirements/Dependencies -# -# Currently requires the puppetlabs/stdlib module on the Puppet Forge in -# order to validate much of the the provided configuration. +# @summary +# This class handles the Kafka (producer). # -# === Parameters +# @example Basic usage +# class { 'kafka::producer': +# config => { +# 'client.id' => '0', +# 'zookeeper.connect' => 'localhost:2181' +# } +# } # -# [*kafka_version*] -# The version of kafka that should be installed. +# @param input +# Set named pipe as input. # -# [*scala_version*] -# The scala version what kafka was built with. +# @param kafka_version +# The version of Kafka that should be installed. # -# [*install_dir*] -# The directory to install kafka to. +# @param scala_version +# The scala version what Kafka was built with. # -# [*mirror_url*] -# The url where the kafka is downloaded from. +# @param install_dir +# The directory to install Kafka to. # -# [*manage_java*] -# Install java if it's not already installed. +# @param mirror_url +# The url where the Kafka is downloaded from. # -# [*package_dir*] -# The directory to install kafka. +# @param manage_java +# Install java if it's not already installed. # -# [*package_name*] -# Package name, when installing kafka from a package. +# @param package_dir +# The directory to install Kafka. # -# [*package_ensure*] -# Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. +# @param package_name +# Package name, when installing Kafka from a package. # -# [*user_name*] -# User to run kafka as. +# @param package_ensure +# Package version or ensure state, when installing Kafka from a package. # -# [*group_name*] -# Group to run kafka as. +# @param user_name +# User to run Kafka as. # -# [*user_id*] -# Create the kafka user with this ID. +# @param group_name +# Group to run Kafka as. # -# [*group_id*] -# Create the kafka group with this ID. +# @param user_id +# Create the Kafka user with this ID. # -# [*manage_user*] -# Create the kafka user if it's not already present. +# @param group_id +# Create the Kafka group with this ID. # -# [*manage_group*] -# Create the kafka group if it's not already present. +# @param manage_user +# Create the Kafka user if it's not already present. # -# [*config_mode*] -# The permissions for the config files. +# @param manage_group +# Create the Kafka group if it's not already present. # -# [*config_dir*] -# The directory to create the kafka config files to. +# @param config_mode +# The permissions for the config files. # -# [*log_dir*] -# The directory for kafka log files. +# @param config_dir +# The directory to create the Kafka config files to. # -# [*bin_dir*] -# The directory where the kafka scripts are. +# @param log_dir +# The directory for Kafka log files. # -# [*service_name*] -# Set the name of the service. +# @param bin_dir +# The directory where the Kafka scripts are. # -# [*manage_service*] -# Install the init.d or systemd service. +# @param service_name +# Set the name of the service. # -# [*service_ensure*] -# Set the ensure state of the service to 'stopped' or 'running'. +# @param manage_service +# Install the init.d or systemd service. # -# [*service_restart*] -# Whether the configuration files should trigger a service restart. +# @param service_ensure +# Set the ensure state of the service. # -# [*service_requires*] -# Set the list of services required to be running before Kafka. +# @param service_restart +# Whether the configuration files should trigger a service restart. # -# [*limit_nofile*] -# Set the 'LimitNOFILE' option of the systemd service. +# @param service_requires +# Set the list of services required to be running before Kafka. # -# [*limit_core*] -# Set the 'LimitCORE' option of the systemd service. +# @param limit_nofile +# Set the 'LimitNOFILE' option of the systemd service. # -# [*env*] -# A hash of the environment variables to set. +# @param limit_core +# Set the 'LimitCORE' option of the systemd service. # -# [*config*] -# A hash of the producer configuration options. +# @param env +# A hash of the environment variables to set. # -# [*service_config*] -# A hash of the `kafka-console-producer.sh` script options. +# @param config +# A hash of the producer configuration options. # -# === Examples +# @param service_config +# A hash of the `kafka-console-producer.sh` script options. # -# Create the producer service connecting to a local zookeeper +# @param jmx_opts +# Set the JMX options. # -# class { 'kafka::producer': -# config => { 'client.id' => '0', 'zookeeper.connect' => 'localhost:2181' } -# } +# @param log4j_opts +# Set the Log4j options. # class kafka::producer ( Optional[String[1]] $input, String[1] $kafka_version = $kafka::params::kafka_version, String[1] $scala_version = $kafka::params::scala_version, Stdlib::Absolutepath $install_dir = $kafka::params::install_dir, Stdlib::HTTPUrl $mirror_url = $kafka::params::mirror_url, Boolean $manage_java = $kafka::params::manage_java, Stdlib::Absolutepath $package_dir = $kafka::params::package_dir, Optional[String[1]] $package_name = $kafka::params::package_name, String[1] $package_ensure = $kafka::params::package_ensure, String[1] $user_name = $kafka::params::user_name, String[1] $group_name = $kafka::params::group_name, Optional[Integer] $user_id = $kafka::params::user_id, Optional[Integer] $group_id = $kafka::params::group_id, Boolean $manage_user = $kafka::params::manage_user, Boolean $manage_group = $kafka::params::manage_group, Stdlib::Filemode $config_mode = $kafka::params::config_mode, Stdlib::Absolutepath $config_dir = $kafka::params::config_dir, Stdlib::Absolutepath $log_dir = $kafka::params::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::params::bin_dir, String[1] $service_name = 'kafka-producer', Boolean $manage_service = $kafka::params::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::params::service_ensure, Boolean $service_restart = $kafka::params::service_restart, Array[String[1]] $service_requires = $kafka::params::service_requires, Optional[String[1]] $limit_nofile = $kafka::params::limit_nofile, Optional[String[1]] $limit_core = $kafka::params::limit_core, Hash $env = {}, Hash[String[1], Any] $config = {}, Hash[String[1],String[1]] $service_config = {}, String[1] $jmx_opts = $kafka::params::producer_jmx_opts, String[1] $log4j_opts = $kafka::params::producer_log4j_opts, ) inherits kafka::params { class { 'kafka::producer::install': } -> class { 'kafka::producer::config': } -> class { 'kafka::producer::service': } -> Class['kafka::producer'] } diff --git a/manifests/producer/config.pp b/manifests/producer/config.pp index 99b1173..36a8c13 100644 --- a/manifests/producer/config.pp +++ b/manifests/producer/config.pp @@ -1,37 +1,31 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::producer::config -# -# This private class is meant to be called from `kafka::producer`. -# It manages the producer config files +# @summary +# This class handles the Kafka (producer) config. # class kafka::producer::config( Boolean $manage_service = $kafka::producer::manage_service, String[1] $service_name = $kafka::producer::service_name, Boolean $service_restart = $kafka::producer::service_restart, Hash[String[1], Any] $config = $kafka::producer::config, Stdlib::Absolutepath $config_dir = $kafka::producer::config_dir, String[1] $user_name = $kafka::producer::user_name, String[1] $group_name = $kafka::producer::group_name, Stdlib::Filemode $config_mode = $kafka::producer::config_mode, ) { if ($manage_service and $service_restart) { $config_notify = Service[$service_name] } else { $config_notify = undef } $doctag = 'producerconfigs' file { "${config_dir}/producer.properties": ensure => present, owner => $user_name, group => $group_name, mode => $config_mode, content => template('kafka/properties.erb'), notify => $config_notify, require => File[$config_dir], } } diff --git a/manifests/producer/install.pp b/manifests/producer/install.pp index 23d5c9b..f3cd3ce 100644 --- a/manifests/producer/install.pp +++ b/manifests/producer/install.pp @@ -1,34 +1,30 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::producer::install +# @summary +# This class handles the Kafka (producer) package. # -# This private class is meant to be called from `kafka::producer`. -# It downloads the package and installs it. +# @api private # class kafka::producer::install { assert_private() if !defined(Class['kafka']) { class { 'kafka': manage_java => $kafka::producer::manage_java, manage_group => $kafka::producer::manage_group, group_id => $kafka::producer::group_id, group_name => $kafka::producer::group_name, manage_user => $kafka::producer::manage_user, user_id => $kafka::producer::user_id, user_name => $kafka::producer::user_name, config_dir => $kafka::producer::config_dir, log_dir => $kafka::producer::log_dir, mirror_url => $kafka::producer::mirror_url, kafka_version => $kafka::producer::kafka_version, scala_version => $kafka::producer::scala_version, install_dir => $kafka::producer::install_dir, package_dir => $kafka::producer::package_dir, package_ensure => $kafka::producer::package_ensure, package_name => $kafka::producer::package_name, } } } diff --git a/manifests/producer/service.pp b/manifests/producer/service.pp index 18a0956..7d8b4a9 100644 --- a/manifests/producer/service.pp +++ b/manifests/producer/service.pp @@ -1,64 +1,60 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Class: kafka::producer::service +# @summary +# This class handles the Kafka (producer) service. # -# This private class is meant to be called from `kafka::producer`. -# It manages the kafka-producer service +# @api private # class kafka::producer::service( Boolean $manage_service = $kafka::producer::manage_service, Enum['running', 'stopped'] $service_ensure = $kafka::producer::service_ensure, String[1] $service_name = $kafka::producer::service_name, - String[1]$user_name = $kafka::producer::user_name, + String[1] $user_name = $kafka::producer::user_name, String[1] $group_name = $kafka::producer::group_name, Stdlib::Absolutepath $config_dir = $kafka::producer::config_dir, Stdlib::Absolutepath $log_dir = $kafka::producer::log_dir, Stdlib::Absolutepath $bin_dir = $kafka::producer::bin_dir, Array[String[1]] $service_requires = $kafka::producer::service_requires, Optional[String[1]] $limit_nofile = $kafka::producer::limit_nofile, Optional[String[1]] $limit_core = $kafka::producer::limit_core, Hash $env = $kafka::producer::env, Optional[String[1]] $input = $kafka::producer::input, String[1] $jmx_opts = $kafka::producer::jmx_opts, String[1] $log4j_opts = $kafka::producer::log4j_opts, Hash[String[1],String[1]] $service_config = $kafka::producer::service_config, ) { assert_private() if $manage_service { if $service_config['broker-list'] == '' { fail('[Producer] You need to specify a value for broker-list') } if $service_config['topic'] == '' { fail('[Producer] You need to specify a value for topic') } $env_defaults = { 'KAFKA_JMX_OPTS' => $jmx_opts, 'KAFKA_LOG4J_OPTS' => $log4j_opts, } $environment = deep_merge($env_defaults, $env) if $facts['service_provider'] == 'systemd' { fail('Console Producer is not supported on systemd, because the stdin of the process cannot be redirected') } else { file { "/etc/init.d/${service_name}": ensure => file, mode => '0755', content => template('kafka/init.erb'), before => Service[$service_name], } } service { $service_name: ensure => $service_ensure, enable => true, hasstatus => true, hasrestart => true, } } } diff --git a/manifests/topic.pp b/manifests/topic.pp index de3177b..fc0f8d5 100644 --- a/manifests/topic.pp +++ b/manifests/topic.pp @@ -1,43 +1,68 @@ -# Author:: Liam Bennett (mailto:lbennett@opentable.com) -# Copyright:: Copyright (c) 2013 OpenTable Inc -# License:: MIT - -# == Define: kafka::topic +# @summary +# This defined type handles the creation of Kafka topics. +# +# @example Basic usage +# kafka::topic { 'test': +# ensure => present, +# zookeeper => 'localhost:2181', +# replication_factor => 1, +# partitions => 1, +# } +# +# @param ensure +# Should the topic be created. +# +# @param zookeeper +# The connection string for the ZooKeeper connection in the form host:port. +# Multiple hosts can be given to allow fail-over. +# +# @param replication_factor +# The replication factor for each partition in the topic being created. If +# not supplied, defaults to the cluster default. +# +# @param partitions +# The number of partitions for the topic being created or altered. If not +# supplied for create, defaults to the cluster default. +# +# @param bin_dir +# The directory where the file kafka-topics.sh is located. # -# This defined type is used to manage the creation of kafka topics. +# @param config +# A topic configuration override for the topic being created or altered. +# See the Kafka documentation for full details on the topic configs. # define kafka::topic( String[1] $ensure = '', String[1] $zookeeper = '', Variant[Integer,String[1]] $replication_factor = 1, Variant[Integer,String[1]] $partitions = 1, String[1] $bin_dir = '/opt/kafka/bin', Optional[Hash[String[1],String[1]]] $config = undef, ) { if is_string($replication_factor) { deprecation('kafka::topic', 'Please use Integer type, not String, for paramter replication_factor') } if is_string($partitions) { deprecation('kafka::topic', 'Please use Integer type, not String, for paramter partitions') } $_zookeeper = "--zookeeper ${zookeeper}" $_replication_factor = "--replication-factor ${replication_factor}" $_partitions = "--partitions ${partitions}" if $config { $_config_array = $config.map |$key, $value| { "--config ${key}=${value}" } $_config = join($_config_array, ' ') } else { $_config = '' } if $ensure == 'present' { exec { "create topic ${name}": path => "/usr/bin:/usr/sbin/:/bin:/sbin:${bin_dir}", command => "kafka-topics.sh --create ${_zookeeper} ${_replication_factor} ${_partitions} --topic ${name} ${_config}", unless => "kafka-topics.sh --list ${_zookeeper} | grep -x ${name}", } } }