diff --git a/.travis.yml b/.travis.yml index 943f516..2175e81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,73 +1,106 @@ --- sudo: false dist: trusty language: ruby cache: bundler before_install: - rm -f Gemfile.lock script: - 'bundle exec rake $CHECK' matrix: fast_finish: true include: - - rvm: 2.1.9 - bundler_args: --without system_tests development release - env: PUPPET_VERSION="~> 4.0" CHECK=test PARALLEL_TEST_PROCESSORS=12 - rvm: 2.4.4 bundler_args: --without system_tests development release env: PUPPET_VERSION="~> 5.0" CHECK=test - rvm: 2.5.1 bundler_args: --without system_tests development release env: PUPPET_VERSION="~> 5.0" CHECK=test_with_coveralls - rvm: 2.4.4 bundler_args: --without system_tests development release env: PUPPET_VERSION="~> 5.0" CHECK=rubocop - rvm: 2.5.1 bundler_args: --without system_tests development release env: PUPPET_VERSION="~> 5.0" CHECK=build DEPLOY_TO_FORGE=yes - rvm: 2.5.1 bundler_args: --without development release dist: trusty env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet5 BEAKER_debug=true BEAKER_setfile=ubuntu1604-64{hypervisor=docker} CHECK=beaker services: docker sudo: required + - rvm: 2.5.1 + bundler_args: --without development release + dist: trusty + env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet5 BEAKER_debug=true BEAKER_setfile=centos6-64{hypervisor=docker} CHECK=beaker + services: docker + sudo: required - rvm: 2.5.1 bundler_args: --without development release dist: trusty env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet5 BEAKER_debug=true BEAKER_setfile=centos7-64{hypervisor=docker} CHECK=beaker services: docker sudo: required - rvm: 2.5.1 bundler_args: --without development release dist: trusty env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet5 BEAKER_debug=true BEAKER_setfile=debian8-64{hypervisor=docker} CHECK=beaker services: docker sudo: required - rvm: 2.5.1 bundler_args: --without development release dist: trusty env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet5 BEAKER_debug=true BEAKER_setfile=debian9-64{hypervisor=docker} CHECK=beaker services: docker sudo: required + - rvm: 2.5.1 + bundler_args: --without development release + dist: trusty + env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet6 BEAKER_debug=true BEAKER_setfile=ubuntu1604-64{hypervisor=docker} CHECK=beaker + services: docker + sudo: required + - rvm: 2.5.1 + bundler_args: --without development release + dist: trusty + env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet6 BEAKER_debug=true BEAKER_setfile=centos6-64{hypervisor=docker} CHECK=beaker + services: docker + sudo: required + - rvm: 2.5.1 + bundler_args: --without development release + dist: trusty + env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet6 BEAKER_debug=true BEAKER_setfile=centos7-64{hypervisor=docker} CHECK=beaker + services: docker + sudo: required + - rvm: 2.5.1 + bundler_args: --without development release + dist: trusty + env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet6 BEAKER_debug=true BEAKER_setfile=debian8-64{hypervisor=docker} CHECK=beaker + services: docker + sudo: required + - rvm: 2.5.1 + bundler_args: --without development release + dist: trusty + env: PUPPET_INSTALL_TYPE=agent BEAKER_PUPPET_COLLECTION=puppet6 BEAKER_debug=true BEAKER_setfile=debian9-64{hypervisor=docker} CHECK=beaker + services: docker + sudo: required branches: only: - master - /^v\d/ notifications: email: false irc: on_success: always on_failure: always channels: - "chat.freenode.org#voxpupuli-notifications" deploy: provider: puppetforge user: puppet password: secure: "j/Db/NnuJUwyFWGVwZEciC/0Xrhaes647UK49ZnlZTjUppUeTsqY/rKE8Pc4jpiW8DsfeGijCYP1O02tquH+KSKSwiwxIBjbToFjhNNJ6Qgh0DGIR29VZkiyirh5ZkK1yLMx9Ciyn8opwOXHqTRMk6JwAY05Gux1sD2T7Eu2c4w=" on: tags: true # all_branches is required to use tags all_branches: true # Only publish the build marked with "DEPLOY_TO_FORGE" condition: "$DEPLOY_TO_FORGE = yes" diff --git a/Gemfile b/Gemfile index 8da95fb..9340366 100644 --- a/Gemfile +++ b/Gemfile @@ -1,81 +1,82 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" def location_for(place, fake_version = nil) if place =~ /^(git[:@][^#]*)#(.*)/ [fake_version, { :git => $1, :branch => $2, :require => false }].compact elsif place =~ /^file:\/\/(.*)/ ['>= 0', { :path => File.expand_path($1), :require => false }] else [place, { :require => false }] end end group :test do - gem 'puppetlabs_spec_helper', '~> 2.6', :require => false - gem 'rspec-puppet', '~> 2.5', :require => false - gem 'rspec-puppet-facts', :require => false + gem 'puppetlabs_spec_helper', '>= 2.14.0', :require => false + gem 'rspec-puppet-facts', '>= 1.9.5', :require => false gem 'rspec-puppet-utils', :require => false gem 'puppet-lint-leading_zero-check', :require => false gem 'puppet-lint-trailing_comma-check', :require => false gem 'puppet-lint-version_comparison-check', :require => false gem 'puppet-lint-classes_and_types_beginning_with_digits-check', :require => false gem 'puppet-lint-unquoted_string-check', :require => false gem 'puppet-lint-variable_contains_upcase', :require => false gem 'metadata-json-lint', :require => false gem 'redcarpet', :require => false - gem 'rubocop', '~> 0.49.1', :require => false if RUBY_VERSION >= '2.3.0' - gem 'rubocop-rspec', '~> 1.15.0', :require => false if RUBY_VERSION >= '2.3.0' + gem 'rubocop', '~> 0.49.1', :require => false + gem 'rubocop-rspec', '~> 1.15.0', :require => false gem 'mocha', '~> 1.4.0', :require => false gem 'coveralls', :require => false gem 'simplecov-console', :require => false - gem 'rack', '~> 1.0', :require => false if RUBY_VERSION < '2.2.2' gem 'parallel_tests', :require => false end group :development do gem 'travis', :require => false gem 'travis-lint', :require => false gem 'guard-rake', :require => false gem 'overcommit', '>= 0.39.1', :require => false end group :system_tests do gem 'winrm', :require => false if beaker_version = ENV['BEAKER_VERSION'] gem 'beaker', *location_for(beaker_version) else - gem 'beaker', '>= 3.9.0', :require => false + gem 'beaker', '>= 4.2.0', :require => false end if beaker_rspec_version = ENV['BEAKER_RSPEC_VERSION'] gem 'beaker-rspec', *location_for(beaker_rspec_version) else gem 'beaker-rspec', :require => false end gem 'serverspec', :require => false - gem 'beaker-hostgenerator', '>= 1.1.10', :require => false + gem 'beaker-hostgenerator', '>= 1.1.22', :require => false + gem 'beaker-docker', :require => false + gem 'beaker-puppet', :require => false gem 'beaker-puppet_install_helper', :require => false gem 'beaker-module_install_helper', :require => false - gem 'rbnacl', '~> 4', :require => false if RUBY_VERSION >= '2.2.6' - gem 'rbnacl-libsodium', :require => false if RUBY_VERSION >= '2.2.6' + gem 'rbnacl', '>= 4', :require => false + gem 'rbnacl-libsodium', :require => false gem 'bcrypt_pbkdf', :require => false + gem 'ed25519', :require => false end group :release do - gem 'github_changelog_generator', :require => false, :git => 'https://github.com/skywinder/github-changelog-generator' if RUBY_VERSION >= '2.2.2' + gem 'github_changelog_generator', :require => false, :git => 'https://github.com/voxpupuli/github-changelog-generator', :branch => 'voxpupuli_essential_fixes' gem 'puppet-blacksmith', :require => false - gem 'voxpupuli-release', :require => false, :git => 'https://github.com/voxpupuli/voxpupuli-release-gem' - gem 'puppet-strings', '>= 1.0', :require => false + gem 'voxpupuli-release', :require => false + gem 'puppet-strings', '>= 2.2', :require => false end if facterversion = ENV['FACTER_GEM_VERSION'] gem 'facter', facterversion.to_s, :require => false, :groups => [:test] else gem 'facter', :require => false, :groups => [:test] end -ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 5.0' : puppetversion = ENV['PUPPET_VERSION'].to_s +ENV['PUPPET_VERSION'].nil? ? puppetversion = '~> 6.0' : puppetversion = ENV['PUPPET_VERSION'].to_s gem 'puppet', puppetversion, :require => false, :groups => [:test] # vim: syntax=ruby diff --git a/README.md b/README.md index 131457c..6d7d75c 100644 --- a/README.md +++ b/README.md @@ -1,357 +1,358 @@ # Kafka module for Puppet [![Build Status](https://travis-ci.org/voxpupuli/puppet-kafka.png?branch=master)](https://travis-ci.org/voxpupuli/puppet-kafka) [![Puppet Forge](https://img.shields.io/puppetforge/v/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) [![Puppet Forge - downloads](https://img.shields.io/puppetforge/dt/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) [![Puppet Forge - endorsement](https://img.shields.io/puppetforge/e/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) [![Puppet Forge - scores](https://img.shields.io/puppetforge/f/puppet/kafka.svg)](https://forge.puppetlabs.com/puppet/kafka) #### Table of Contents 1. [Overview](#overview) 2. [Module Description - What the module does and why it is useful](#module-description) 3. [Setup - The basics of getting started with kafka](#setup) * [What kafka affects](#what-kafka-affects) * [Setup requirements](#setup-requirements) * [Beginning with kafka](#beginning-with-kafka) 4. [Usage - Configuration options and additional functionality](#usage) 5. [Reference - An under-the-hood peek at what the module is doing and how](#reference) 5. [Limitations - OS compatibility, etc.](#limitations) 6. [Development - Guide for contributing to the module](#development) ## Overview The Kafka module for managing the installation and configuration of [Apache Kafka](http://kafka.apache.org) ## Module Description The Kafka module for managing the installation and configuration of Apache Kafka: it's brokers, producers and consumers. ## Setup ### What kafka affects Installs the Kafka package and creates a new service. ### Beginning with Kafka To successfully install Kafka using this module you need to have Apache ZooKeeper already running at localhost:2181. You can specify another ZooKeeper host:port configuration using the config hash of the kafka:broker class. The default configuration installs Kafka 0.11.0.3 binaries with Scala 2.11 ```puppet class { 'kafka': } ``` If you want a Kafka broker server that connects to ZooKeeper listening on port 2181: ```puppet class { 'kafka::broker': config => { 'broker.id' => '0', 'zookeeper.connect' => 'localhost:2181' } } ``` ## Usage You can specify different Kafka binaries packages versions to install. Please take a look at the different Scala and Kafka versions combinations at the [Apache Kafka Website](http://kafka.apache.org/downloads.html) ### Installing Kafka version 1.1.0 with scala 2.12 We first install the binary package with: ```puppet class { 'kafka': version => '1.1.0', scala_version => '2.12' } ``` Then we set a minimal Kafka broker configuration with: ```puppet class { 'kafka::broker': config => { 'broker.id' => '0', 'zookeeper.connect' => 'localhost:2181' } } ``` ### Classes and Defined Types #### Class: `kafka` One of the primary classes of the kafka module. This class will install the kafka binaries **Parameters within `kafka`:** ##### `version` The version of kafka that should be installed. ##### `scala_version` The scala version what kafka was built with. ##### `install_dir` The directory to install kafka to. ##### `mirror_url` The url where the kafka is downloaded from. ##### `mirror_subpath` The url subpath where the kafka is downloaded from (default value: `/kafka/{version}`). ##### `proxy_port` The proxy port where the kafka is downloaded from. ##### `proxy_host` The proxy host where the kafka is downloaded from. ##### `proxy_server` The proxy server where the kafka is downloaded from (to use instead of `proxy_port` and `proxy_host` if you need to set an url as proxy for example). ##### `proxy_type` The proxy type where the kafka is downloaded from (`http` for example). ##### `install_java` Install java if it's not already installed. ##### `package_dir` The directory to install kafka. #### `package_name` Package name, when installing kafka from a package. #### `package_ensure` Package version (or 'present', 'absent', 'latest'), when installing kafka from a package. #### `group_id` Create kafka group with this ID #### `user_id` Create kafka user with this ID #### `user` User to install kafka as. Defaults to the kafka user. #### `group` Group to install kafka as. Defaults to the kafka group. #### `config_dir` Directory for kafka config files. Defaults to /opt/kafka/config. #### `log_dir` Directory for kafka log files. Defaults to /var/log/kafka. #### Class: `kafka::broker` One of the primary classes of the kafka module. This class will install a kafka broker. **Parameters within `kafka::broker`:** ##### `version` The version of kafka that should be installed. ##### `scala_version` The scala version what kafka was built with. ##### `install_dir` The directory to install kafka to. ##### `mirror_url` The url where the kafka is downloaded from. ##### `config` A hash of the configuration options. All values are used in the `server.properties` file directly. ##### `install_java` Install java if it's not already installed. ##### `service_ensure` Sets the ensure state of the broker service to stopped or running. ##### `service_install` Install the init.d service. ##### `service_restart` Whether the configuration files should trigger a service restart ##### `package_dir` The directory to install kafka. #### Class: `kafka::consumer` One of the primary classes of the kafka module. This class will install a kafka consumer. **Parameters within `kafka::consumer`:** #### `version` The version of kafka that should be installed. #### `scala_version` The scala version that kafka was built with. #### `install_dir` The directory to install kafka to. #### `mirror_url` The url where the kafka is downloaded from. #### `install_java` Install java if it's not already installed. #### `package_dir` The directory to install kafka. #### Class: `kafka::mirror` One of the primary classes of the kafka module. This class will install a kafka mirror. **Parameters within `kafka::mirror`:** #### `version` The version of kafka that should be installed. #### `scala_version` The scala version that kafka was built with. #### `install_dir` The directory to install kafka to. #### `mirror_url` The url where the kafka is downloaded from. #### `install_java` Install java if it's not already installed. #### `package_dir` The directory to install kafka. #### Class: `kafka::producer` One of the primary classes of the kafka module. This class will install a kafka producer. **Parameters within `kafka::producer`:** #### `version` The version of kafka that should be installed. #### `scala_version` The scala version that kafka was built with. #### `install_dir` The directory to install kafka to. #### `mirror_url` The url where the kafka is downloaded from. #### `install_java` Install java if it's not already installed. #### `package_dir` The directory to install kafka. #### Define: `kafka::topic` Defined type that creates Kafka topics. ## Reference ### Classes #### Public Classes * `kafka`: Guides the basic installation of kafka binaries * `kafka::broker`: Guides the basic installation of a kafka broker * `kafka::consumer`: Guides the basic installation of a kafka consumer * `kafka::mirror`: Guides the basic installation of a kafka mirror * `kafka::producer`: Guides the basic installation of a kafka producer #### Private Classes * [`kafka::broker::config`] Manages all the default configuration of the kafka broker application * [`kafka::broker::install`] Manages the installation of the kafka packages * [`kafka::broker::service`] Manages the kafka server service * [`kafka::consumer::config`] Manages all the default configuration of the kafka consumer application * [`kafka::consumer::install`] Manages the installation of the kafka packages * [`kafka::consumer::service`] Manages the kafka server service * [`kafka::mirror::config`] Manages all the default configuration of the kafka mirror application * [`kafka::mirror::install`] Manages the installation of the kafka packages * [`kafka::mirror::service`] Manages the kafka server service * [`kafka::producer::config`] Manages all the default configuration of the kafka producer application * [`kafka::producer::install`] Manages the installation of the kafka packages * [`kafka::producer::service`] Manages the kafka server service #### Defined Types * `kafka::topic` Create kafka topics ## Limitations This module only supports Kafka >= 0.9.0.0. This module is tested on the following platforms: -* CentOS 5 * CentOS 6 -* Ubuntu 12.04 -* Ubuntu 14.04 +* CentOS 7 +* Ubuntu 16.04 +* Debian 8 +* Debian 9 -It is tested with the OSS version of Puppet (>= 4.7.0) only. +It is tested with the OSS version of Puppet (>= 5.5) only. ## Development ### Contributing Please read CONTRIBUTING.md for full details on contributing to this project. diff --git a/manifests/params.pp b/manifests/params.pp index 66760da..23e9be0 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -1,76 +1,73 @@ # Author:: Liam Bennett (mailto:lbennett@opentable.com) # Copyright:: Copyright (c) 2013 OpenTable Inc # License:: MIT # == Class kafka::params # # This class is meant to be called from kafka::broker # It sets variables according to platform # class kafka::params { # this is all only tested on Debian and RedHat # params gets included everywhere so we can do the validation here unless $facts['os']['family'] =~ /(RedHat|Debian)/ { warning("${facts['os']['family']} is not supported") } - $version = '0.11.0.3' - $scala_version = '2.11' + $version = '2.4.1' + $scala_version = '2.12' $install_dir = "/opt/kafka-${scala_version}-${version}" $config_dir = '/opt/kafka/config' $bin_dir = '/opt/kafka/bin' $log_dir = '/var/log/kafka' $mirror_url = 'https://www.apache.org/dyn/closer.lua?action=download&filename=' $mirror_subpath = "kafka/${version}" $install_java = false $package_dir = '/var/tmp/kafka' $package_name = undef $proxy_server = undef $proxy_host = undef $proxy_port = undef $proxy_type = undef $package_ensure = 'present' $user = 'kafka' $group = 'kafka' $user_id = undef $group_id = undef $system_user = false $system_group = false $manage_user = true $manage_group = true $config_mode = '0644' $install_mode = '0755' $service_install = true $service_ensure = 'running' $service_restart = true $service_requires = $facts['os']['family'] ? { 'RedHat' => ['network.target', 'syslog.target'], default => [], } $limit_nofile = undef $limit_core = undef $timeout_stop = undef $exec_stop = false $daemon_start = false $broker_heap_opts = '-Xmx1G -Xms1G' $broker_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false \ -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9990' $broker_log4j_opts = "-Dlog4j.configuration=file:${config_dir}/log4j.properties" $broker_opts = '' $mirror_heap_opts = '-Xmx256M' - $mirror_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false \ - -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9991' + $mirror_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9991' $mirror_log4j_opts = $broker_log4j_opts - $producer_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false \ - -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9992' + $producer_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9992' $producer_log4j_opts = $broker_log4j_opts - $consumer_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false \ - -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9993' + $consumer_jmx_opts = '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9993' $consumer_log4j_opts = $broker_log4j_opts } diff --git a/metadata.json b/metadata.json index 87e8d2b..4c71957 100644 --- a/metadata.json +++ b/metadata.json @@ -1,71 +1,71 @@ { "name": "puppet-kafka", "version": "5.3.1-rc0", "author": "Vox Pupuli", "summary": "Puppet module for Kafka", "license": "MIT", "source": "https://github.com/voxpupuli/puppet-kafka", "project_page": "https://github.com/voxpupuli/puppet-kafka", "issues_url": "https://github.com/voxpupuli/puppet-kafka/issues", "dependencies": [ { "name": "puppet/archive", "version_requirement": ">= 1.0.0 < 5.0.0" }, { "name": "puppetlabs/java", "version_requirement": ">= 1.4.2 < 6.0.0" }, { "name": "puppetlabs/stdlib", "version_requirement": ">= 4.22.0 < 7.0.0" }, { "name": "deric/zookeeper", "version_requirement": ">= 0.5.1 < 1.0.0" }, { "name": "camptocamp/systemd", "version_requirement": ">= 0.4.0 < 3.0.0" } ], "operatingsystem_support": [ { "operatingsystem": "CentOS", "operatingsystemrelease": [ "6", "7" ] }, { "operatingsystem": "RedHat", "operatingsystemrelease": [ "6", "7" ] }, { "operatingsystem": "Ubuntu", "operatingsystemrelease": [ "16.04" ] }, { "operatingsystem": "Debian", "operatingsystemrelease": [ "8", "9" ] } ], "requirements": [ { "name": "puppet", - "version_requirement": ">= 4.10.0 < 6.0.0" + "version_requirement": ">= 4.10.0 < 7.0.0" } ], "tags": [ "kafka", "pubsub" ] } diff --git a/spec/acceptance/01_zookeeper_spec.rb b/spec/acceptance/01_zookeeper_spec.rb new file mode 100644 index 0000000..d5c7087 --- /dev/null +++ b/spec/acceptance/01_zookeeper_spec.rb @@ -0,0 +1,41 @@ +require 'spec_helper_acceptance' + +describe 'zookeeper prereq' do + zookeeper = <<-EOS + if $::osfamily == 'RedHat' { + class { 'java' : + package => 'java-1.8.0-openjdk-devel', + } + + exec { 'create pid dir': + command => '/bin/mkdir -p /var/run/', + creates => '/var/run/', + } + + file { '/var/run/zookeeper/': + ensure => directory, + owner => 'zookeeper', + group => 'zookeeper', + } + + $zookeeper_service_provider = $facts['os']['release']['major'] ? { + '6' => 'redhat', + '7' => 'systemd', + } + + class { 'zookeeper': + install_method => 'archive', + archive_version => '3.6.0', + service_provider => $zookeeper_service_provider, + manage_service_file => true, + } + } else { + include zookeeper + } + EOS + + it 'installs zookeeper with no errors' do + apply_manifest(zookeeper, catch_failures: true) + apply_manifest(zookeeper, catch_changes: true) + end +end diff --git a/spec/acceptance/broker_spec.rb b/spec/acceptance/broker_spec.rb index 4ffe75a..4cfd990 100644 --- a/spec/acceptance/broker_spec.rb +++ b/spec/acceptance/broker_spec.rb @@ -1,264 +1,224 @@ require 'spec_helper_acceptance' describe 'kafka::broker' do - zookeeper = <<-EOS - if $::osfamily == 'RedHat' { - class { 'java': } - - exec { 'create pid dir': - command => '/bin/mkdir -p /var/run/', - creates => '/var/run/', - } - - file { '/var/run/zookeeper/': - ensure => directory, - owner => 'zookeeper', - group => 'zookeeper', - } - - class { 'zookeeper': - repo => 'cloudera', - cdhver => '5', - initialize_datastore => true, - } - - } else { - class { 'zookeeper': } - } - EOS - it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS class { 'kafka::broker': config => { 'zookeeper.connect' => 'localhost:2181', }, } -> kafka::topic { 'demo': ensure => present, zookeeper => 'localhost:2181', } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe 'kafka::broker::install' do context 'with default parameters' do it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS class { 'kafka::broker': config => { 'zookeeper.connect' => 'localhost:2181', }, } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.12-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.1') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::broker::config' do context 'with default parameters' do it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS class { 'kafka::broker': config => { 'zookeeper.connect' => 'localhost:2181', }, } EOS apply_manifest(pp, catch_failures: true) end describe file('/opt/kafka/config/server.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } it { is_expected.to contain 'zookeeper.connect=localhost:2181' } end end context 'with custom config dir' do it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS class { 'kafka::broker': config => { 'zookeeper.connect' => 'localhost:2181', }, config_dir => '/opt/kafka/custom_config' } EOS apply_manifest(pp, catch_failures: true) end describe file('/opt/kafka/custom_config/server.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } it { is_expected.to contain 'zookeeper.connect=localhost:2181' } end end context 'with specific version' do it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS class { 'kafka::broker': - version => '1.1.0', + version => '2.4.0', config => { 'zookeeper.connect' => 'localhost:2181', }, } EOS apply_manifest(pp, catch_failures: true) end describe file('/opt/kafka/config/server.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::broker::service' do context 'with default parameters' do it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS class { 'kafka::broker': config => { 'zookeeper.connect' => 'localhost:2181', }, } EOS apply_manifest(pp, catch_failures: true) end describe file('/etc/init.d/kafka'), if: (fact('operatingsystemmajrelease') =~ %r{(5|6)} && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/etc/systemd/system/kafka.service'), if: (fact('operatingsystemmajrelease') == '7' && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe service('kafka') do it { is_expected.to be_running } it { is_expected.to be_enabled } end end end describe 'kafka::broker::service' do context 'with log4j/jmx parameters' do it 'works with no errors' do - apply_manifest(zookeeper, catch_failures: true) - pp = <<-EOS exec { 'create log dir': command => '/bin/mkdir -p /some/path/to/logs', creates => '/some/path/to/logs', } -> class { 'kafka::broker': config => { 'zookeeper.connect' => 'localhost:2181', }, heap_opts => '-Xmx512M -Xmx512M', log4j_opts => '-Dlog4j.configuration=file:/tmp/log4j.properties', jmx_opts => '-Dcom.sun.management.jmxremote', opts => '-Djava.security.policy=/some/path/my.policy', log_dir => '/some/path/to/logs' } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe file('/etc/init.d/kafka'), if: (fact('operatingsystemmajrelease') =~ %r{(5|6)} && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } - it { is_expected.to contain 'export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9999"' } + it { is_expected.to contain 'export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote"' } it { is_expected.to contain 'export KAFKA_HEAP_OPTS="-Xmx512M -Xmx512M"' } - it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"' } + it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:/tmp/log4j.properties"' } end describe file('/etc/systemd/system/kafka.service'), if: (fact('operatingsystemmajrelease') == '7' && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } it { is_expected.to contain "Environment='KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote'" } it { is_expected.to contain "Environment='KAFKA_HEAP_OPTS=-Xmx512M -Xmx512M'" } it { is_expected.to contain "Environment='KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/tmp/log4j.properties'" } it { is_expected.to contain "Environment='KAFKA_OPTS=-Djava.security.policy=/some/path/my.policy'" } it { is_expected.to contain "Environment='LOG_DIR=/some/path/to/logs'" } end describe service('kafka') do it { is_expected.to be_running } it { is_expected.to be_enabled } end end end end diff --git a/spec/acceptance/consumer_spec.rb b/spec/acceptance/consumer_spec.rb index 75e6048..f346f2c 100644 --- a/spec/acceptance/consumer_spec.rb +++ b/spec/acceptance/consumer_spec.rb @@ -1,162 +1,157 @@ require 'spec_helper_acceptance' describe 'kafka::consumer' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::consumer': service_config => { - topic => 'demo', - zookeeper => 'localhost:2181', + topic => 'demo', + bootstrap-server => 'localhost:9092', }, } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe 'kafka::consumer::install' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::consumer': service_config => { - topic => 'demo', - zookeeper => 'localhost:2181', + topic => 'demo', + bootstrap-server => 'localhost:9092', }, } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.12-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.1') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::consumer::config' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::consumer': service_config => { - topic => 'demo', - zookeeper => 'localhost:2181', + topic => 'demo', + bootstrap-server => 'localhost:9092', }, } EOS apply_manifest(pp, catch_failures: true) end describe file('/opt/kafka/config/consumer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::consumer::config' do context 'with custom config_dir' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::consumer': service_config => { - topic => 'demo', - zookeeper => 'localhost:2181', + topic => 'demo', + bootstrap-server => 'localhost:9092', }, config_dir => '/opt/kafka/custom_config', } EOS apply_manifest(pp, catch_failures: true) end describe file('/opt/kafka/custom_config/consumer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::consumer::service' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::consumer': service_config => { - topic => 'demo', - zookeeper => 'localhost:2181', + topic => 'demo', + bootstrap-server => 'localhost:9092', }, } EOS apply_manifest(pp, catch_failures: true) end describe file('/etc/init.d/kafka-consumer'), if: (fact('operatingsystemmajrelease') =~ %r{(5|6)} && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } - it { is_expected.to contain 'export KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false' } - it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"' } + it { is_expected.to contain 'export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9993"' } + it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:/opt/kafka/config/log4j.properties"' } end describe file('/etc/systemd/system/kafka-consumer.service'), if: (fact('operatingsystemmajrelease') == '7' && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } - it { is_expected.to contain 'Environment=\'KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false' } + it { is_expected.to contain 'Environment=\'KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9993\'' } it { is_expected.to contain 'Environment=\'KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/opt/kafka/config/log4j.properties\'' } end describe service('kafka-consumer') do it { is_expected.to be_running } it { is_expected.to be_enabled } end end end end diff --git a/spec/acceptance/hieradata/Debian.yaml b/spec/acceptance/hieradata/Debian.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/spec/acceptance/hieradata/RedHat.yaml b/spec/acceptance/hieradata/RedHat.yaml deleted file mode 100644 index 5a450a1..0000000 --- a/spec/acceptance/hieradata/RedHat.yaml +++ /dev/null @@ -1,4 +0,0 @@ -zookeeper::repo: 'cloudera' -zookeeper::cdhver: 5 -zookeeper::initialize_datastore: true - diff --git a/spec/acceptance/init_spec.rb b/spec/acceptance/init_spec.rb index 835651c..5b31550 100644 --- a/spec/acceptance/init_spec.rb +++ b/spec/acceptance/init_spec.rb @@ -1,212 +1,212 @@ require 'spec_helper_acceptance' describe 'kafka' do it 'works with no errors' do pp = <<-EOS class { 'kafka': } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe 'kafka::init' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS class { 'kafka': } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.12-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.1') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end context 'with specific kafka version' do it 'works with no errors' do pp = <<-EOS class { 'kafka': - version => '1.1.0', + version => '2.4.0', } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-1.1.0') do + describe file('/opt/kafka-2.12-2.4.0') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-1.1.0') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.0') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end context 'with specific scala version' do it 'works with no errors' do pp = <<-EOS class { 'kafka': - scala_version => '2.11', + scala_version => '2.13', } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.13-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.13-2.4.1') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end context 'with specific config dir' do it 'works with no errors' do pp = <<-EOS class { 'kafka': config_dir => '/opt/kafka/custom_config', } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.12-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.1') } end describe file('/opt/kafka/custom_config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end end end diff --git a/spec/acceptance/mirror_spec.rb b/spec/acceptance/mirror_spec.rb index da41995..aa44f84 100644 --- a/spec/acceptance/mirror_spec.rb +++ b/spec/acceptance/mirror_spec.rb @@ -1,219 +1,235 @@ require 'spec_helper_acceptance' describe 'kafka::mirror' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::mirror': consumer_config => { 'group.id' => 'kafka-mirror', - 'zookeeper.connect' => 'localhost:2181', + 'bootstrap.servers' => 'localhost:9092', }, producer_config => { 'bootstrap.servers' => 'localhost:9092', }, + service_config => { + 'whitelist' => '.*', + }, } EOS apply_manifest(pp, catch_failures: true) - # apply_manifest(pp, catch_changes: true) TODO currently not working properly + apply_manifest(pp, catch_changes: true) end describe 'kafka::mirror::install' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::mirror': consumer_config => { 'group.id' => 'kafka-mirror', - 'zookeeper.connect' => 'localhost:2181', + 'bootstrap.servers' => 'localhost:9092', }, producer_config => { 'bootstrap.servers' => 'localhost:9092', }, + service_config => { + 'whitelist' => '.*', + }, } EOS apply_manifest(pp, catch_failures: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.12-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.1') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::mirror::config' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::mirror': consumer_config => { 'group.id' => 'kafka-mirror', - 'zookeeper.connect' => 'localhost:2181', + 'bootstrap.servers' => 'localhost:9092', }, producer_config => { 'bootstrap.servers' => 'localhost:9092', }, + service_config => { + 'whitelist' => '.*', + }, } EOS apply_manifest(pp, catch_failures: true) + apply_manifest(pp, catch_changes: true) end describe file('/opt/kafka/config/consumer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka/config/producer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end context 'with custom config_dir' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::mirror': consumer_config => { 'group.id' => 'kafka-mirror', - 'zookeeper.connect' => 'localhost:2181', + 'bootstrap.servers' => 'localhost:9092', }, producer_config => { 'bootstrap.servers' => 'localhost:9092', }, - config_dir => '/opt/kafka/custom_config' + service_config => { + 'whitelist' => '.*', + }, + config_dir => '/opt/kafka/custom_config', } EOS apply_manifest(pp, catch_failures: true) + apply_manifest(pp, catch_changes: true) end describe file('/opt/kafka/custom_config/consumer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka/custom_config/producer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end context 'with specific version' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::mirror': - version => '0.11.0.3', + version => '2.4.0', consumer_config => { 'group.id' => 'kafka-mirror', - 'zookeeper.connect' => 'localhost:2181', + 'bootstrap.servers' => 'localhost:9092', }, producer_config => { 'bootstrap.servers' => 'localhost:9092', }, + service_config => { + 'whitelist' => '.*', + }, } EOS apply_manifest(pp, catch_failures: true) + apply_manifest(pp, catch_changes: true) end describe file('/opt/kafka/config/consumer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka/config/producer.properties') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::mirror::service' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::mirror': consumer_config => { 'group.id' => 'kafka-mirror', - 'zookeeper.connect' => 'localhost:2181', + 'bootstrap.servers' => 'localhost:9092', }, producer_config => { 'bootstrap.servers' => 'localhost:9092', }, + service_config => { + 'whitelist' => '.*', + }, } EOS apply_manifest(pp, catch_failures: true) + apply_manifest(pp, catch_changes: true) end describe file('/etc/init.d/kafka-mirror'), if: (fact('operatingsystemmajrelease') =~ %r{(5|6)} && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } - it { is_expected.to contain 'export KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false' } - it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"' } + it { is_expected.to contain 'export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9991"' } + it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:/opt/kafka/config/log4j.properties"' } end describe file('/etc/systemd/system/kafka-mirror.service'), if: (fact('operatingsystemmajrelease') == '7' && fact('osfamily') == 'RedHat') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } - it { is_expected.to contain 'Environment=\'KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false' } + it { is_expected.to contain 'Environment=\'KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9991\'' } it { is_expected.to contain 'Environment=\'KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:/opt/kafka/config/log4j.properties\'' } end describe service('kafka-mirror') do it { is_expected.to be_running } it { is_expected.to be_enabled } end end end end diff --git a/spec/acceptance/producer_spec.rb b/spec/acceptance/producer_spec.rb index bb1b850..3a1c38f 100644 --- a/spec/acceptance/producer_spec.rb +++ b/spec/acceptance/producer_spec.rb @@ -1,151 +1,147 @@ require 'spec_helper_acceptance' -describe 'kafka::producer', if: (fact('operatingsystemmajrelease') == '6' && fact('osfamily') == 'RedHat') do +describe 'kafka::producer', if: (fact('operatingsystemmajrelease') == '6' && fact('osfamily') == 'RedHat') do # systemd systems not supported by kafka::producer::service it 'works with no errors' do pp = <<-EOS exec { 'create fifo': - command => '/bin/mkfifo /tmp/kafka-producer', + command => '/usr/bin/mkfifo /tmp/kafka-producer', user => 'kafka', creates => '/tmp/kafka-producer', } -> - class { 'zookeeper': } -> class { 'kafka::producer': service_config => { 'broker-list' => 'localhost:9092', topic => 'demo', }, input => '3<>/tmp/kafka-producer 0>&3', } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe 'kafka::producer::install' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS exec { 'create fifo': - command => '/bin/mkfifo /tmp/kafka-producer', + command => '/usr/bin/mkfifo /tmp/kafka-producer', user => 'kafka', creates => '/tmp/kafka-producer', } -> - class { 'zookeeper': } -> class { 'kafka::producer': service_config => { 'broker-list' => 'localhost:9092', topic => 'demo', }, input => '3<>/tmp/kafka-producer 0>&3', } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe group('kafka') do it { is_expected.to exist } end describe user('kafka') do it { is_expected.to exist } it { is_expected.to belong_to_group 'kafka' } it { is_expected.to have_login_shell '/bin/bash' } end describe file('/var/tmp/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end - describe file('/opt/kafka-2.11-0.11.0.3') do + describe file('/opt/kafka-2.12-2.4.1') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end describe file('/opt/kafka') do - it { is_expected.to be_linked_to('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to be_linked_to('/opt/kafka-2.12-2.4.1') } end describe file('/opt/kafka/config') do it { is_expected.to be_directory } - it { is_expected.to be_owned_by 'kafka' } - it { is_expected.to be_grouped_into 'kafka' } + it { is_expected.to be_owned_by 'root' } + it { is_expected.to be_grouped_into 'root' } end describe file('/var/log/kafka') do it { is_expected.to be_directory } it { is_expected.to be_owned_by 'kafka' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::producer::config' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS exec { 'create fifo': - command => '/bin/mkfifo /tmp/kafka-producer', + command => '/usr/bin/mkfifo /tmp/kafka-producer', user => 'kafka', creates => '/tmp/kafka-producer', } -> - class { 'zookeeper': } -> class { 'kafka::producer': service_config => { 'broker-list' => 'localhost:9092', topic => 'demo', }, input => '3<>/tmp/kafka-producer 0>&3', } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end describe file('/opt/kafka/config/producer.properties') do it { is_expected.to be_file } - it { is_expected.to be_owned_by 'kafka' } + it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'kafka' } end end end describe 'kafka::producer::service' do context 'with default parameters' do it 'works with no errors' do pp = <<-EOS - class { 'zookeeper': } -> class { 'kafka::producer': service_config => { 'broker-list' => 'localhost:9092', topic => 'demo', }, input => '3<>/tmp/kafka-producer 0>&3', } EOS apply_manifest(pp, catch_failures: true) apply_manifest(pp, catch_changes: true) end - describe file('/etc/init.d/kafka-producer'), if: (fact('operatingsystemmajrelease') =~ %r{(5|6)} && fact('osfamily') == 'RedHat') do + describe file('/etc/init.d/kafka-producer') do it { is_expected.to be_file } it { is_expected.to be_owned_by 'root' } it { is_expected.to be_grouped_into 'root' } - it { is_expected.to contain 'export KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false' } - it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"' } + it { is_expected.to contain 'export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=9992"' } + it { is_expected.to contain 'export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:/opt/kafka/config/log4j.properties"' } end describe service('kafka-producer') do it { is_expected.to be_running } it { is_expected.to be_enabled } end end end end diff --git a/spec/classes/init_spec.rb b/spec/classes/init_spec.rb index 1990509..410d440 100644 --- a/spec/classes/init_spec.rb +++ b/spec/classes/init_spec.rb @@ -1,62 +1,62 @@ require 'spec_helper' describe 'kafka', type: :class do let :facts do { osfamily: 'Debian', os: { family: 'Debian' }, operatingsystem: 'Ubuntu', operatingsystemrelease: '14.04', lsbdistcodename: 'trusty', architecture: 'amd64', service_provider: 'upstart' } end it { is_expected.to contain_class('kafka::params') } context 'on Debian' do describe 'kafka' do context 'defaults' do it { is_expected.to contain_group('kafka') } it { is_expected.to contain_user('kafka') } it { is_expected.to contain_file('/var/tmp/kafka') } - it { is_expected.to contain_file('/opt/kafka-2.11-0.11.0.3') } + it { is_expected.to contain_file('/opt/kafka-2.12-2.4.1') } it { is_expected.to contain_file('/opt/kafka') } it { is_expected.to contain_file('/opt/kafka/config') } it { is_expected.to contain_file('/var/log/kafka') } end end end context 'on Debian' do describe 'kafka' do context 'all (compatible) parameters' do let :params do { version: '0.10.0.1', scala_version: '2.13', install_dir: '/usr/local/kafka', user_id: 9092, group_id: 9092, user: 'mykafka', group: 'mykafka', install_java: false, config_dir: '/opt/kafka/custom_config', log_dir: '/var/log/custom_kafka' } end it { is_expected.to contain_group('mykafka').with(gid: 9092) } it { is_expected.to contain_user('mykafka').with(uid: 9092) } it { is_expected.to contain_file('/var/tmp/kafka') } it { is_expected.to contain_file('/opt/kafka') } it { is_expected.to contain_file('/usr/local/kafka') } it { is_expected.to contain_file('/opt/kafka/custom_config') } it { is_expected.to contain_file('/var/log/custom_kafka') } end end end end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 0779f29..ee91bb1 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -1,24 +1,26 @@ require 'beaker-rspec' require 'beaker-puppet' require 'beaker/puppet_install_helper' require 'beaker/module_install_helper' run_puppet_install_helper unless ENV['BEAKER_provision'] == 'no' install_ca_certs unless ENV['PUPPET_INSTALL_TYPE'] =~ %r{pe}i install_module_on(hosts) install_module_dependencies_on(hosts) RSpec.configure do |c| c.formatter = :documentation c.before :suite do hosts.each do |host| if host[:platform] =~ %r{el-7-x86_64} && host[:hypervisor] =~ %r{docker} on(host, "sed -i '/nodocs/d' /etc/yum.conf") end - - write_hiera_config_on(host, ['%<::osfamily>s']) - copy_hiera_data_to(host, './spec/acceptance/hieradata/') + next unless fact('os.name') == 'Debian' && fact('os.release.major') == '8' + on host, 'echo "deb http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/backports.list' + on host, 'echo \'Acquire::Check-Valid-Until "false";\' > /etc/apt/apt.conf.d/check-valid' + on host, 'DEBIAN_FRONTEND=noninteractive apt-get -y update' + on host, 'DEBIAN_FRONTEND=noninteractive apt-get install -y -t jessie-backports openjdk-8-jdk' end end end diff --git a/templates/init.erb b/templates/init.erb index 1e74f2d..84459f0 100644 --- a/templates/init.erb +++ b/templates/init.erb @@ -1,153 +1,153 @@ #!/bin/sh # # Init file for Apache Kafka <%= @service_name.split(/-/)[1] and @service_name.split(/-/)[1].capitalize %> # <%- if @osfamily == 'Debian' -%> ### BEGIN INIT INFO # Provides: <%- @service_name -%> # Required-Start: <%= @service_requires.join(' ') %> # Required-Stop: # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # X-Interactive: true # Short-Description: Apache Kafka is a distributed publish-subscribe messaging system ### END INIT INFO <%- else -%> # chkconfig: 35 85 15 # description: Apache Kafka is a distributed publish-subscribe messaging system # pidfile: /var/run/<%= @service_name -%>.pid <%- end -%> NAME=<%= @service_name %> <% @environment.sort.map do |k,v| -%> <% unless v.to_s.strip.empty? -%> export <%= k %>="<%= v %>" <% end -%> <% end -%> PID_FILE="/var/run/$NAME.pid" KAFKA_USER=<%= @user %> <%- case @service_name when 'kafka' -%> PGREP_PATTERN=kafka.Kafka DAEMON="<%= @bin_dir %>/kafka-server-start.sh" DAEMON_OPTS="<%= @config_dir %>/server.properties" <%- when 'kafka-consumer' -%> PGREP_PATTERN=kafka.tools.ConsoleConsumer DAEMON="<%= @bin_dir %>/kafka-console-consumer.sh" DAEMON_OPTS="<% @service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%> '<%= v.is_a?(Array) ? v.join(',') : v %>' <% end -%><% end -%>" <%- when 'kafka-mirror' -%> PGREP_PATTERN=kafka.tools.MirrorMaker DAEMON="<%= @bin_dir %>/kafka-run-class.sh" DAEMON_OPTS="kafka.tools.MirrorMaker --consumer.config <%= @config_dir %>/consumer.properties --producer.config <%= @config_dir %>/producer.properties <% @service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%> '<%= v.is_a?(Array) ? v.join(',') : v %>' <% end -%><% end -%>" <%- when 'kafka-producer' -%> PGREP_PATTERN=kafka.tools.ConsoleProducer DAEMON="<%= @bin_dir %>/kafka-console-producer.sh" DAEMON_OPTS="<% @service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%> '<%= v.is_a?(Array) ? v.join(',') : v %>' <% end -%><% end -%>" PRODUCER_INPUT="<%= @input %>" <%- end -%> if [ -f /etc/default/kafka ]; then . /etc/default/kafka fi start() { ulimit -n 65536 ulimit -s 10240 ulimit -c unlimited if [ -f "$PID_FILE" ]; then PID=`cat "$PID_FILE"` - if [ `ps -p "$PID" -o pid= || echo 1` -eq `pgrep -f "$PGREP_PATTERN" || echo 2` ] ; then + if [ `ps -p "$PID" -o pid= || echo 1` -eq `ps ax | grep -i "$PGREP_PATTERN" | grep -v grep | awk '{print $1}' || echo 2` ] ; then echo "$PID_FILE exists, process is already running" exit 0 fi echo "$PID_FILE exists but the process is not running. Deleting $PID_FILE and re-trying" rm -f -- "$PID_FILE" start return $? fi /bin/su "$KAFKA_USER" -c "KAFKA_JMX_OPTS=\"$KAFKA_JMX_OPTS\" $DAEMON $DAEMON_OPTS<%- if @service_name == 'kafka-producer' -%> $PRODUCER_INPUT<%- end -%> >/dev/null 2>&1 &" sleep 2 - PID=`pgrep -f "$PGREP_PATTERN"` + PID=`ps ax | grep -i "$PGREP_PATTERN" | grep -v grep | awk '{print $1}'` if [ -z "$PID" ]; then echo "$NAME could not be started" exit 1 fi echo "$PID" > "$PID_FILE"; echo "$NAME started" return 0 } stop() { if ! [ -f "$PID_FILE" ]; then echo -n "$PID_FILE does not exist" - if PID=`pgrep -f "$PGREP_PATTERN"` ; then + if PID=`ps ax | grep -i "$PGREP_PATTERN" | grep -v grep | awk '{print $1}'` ; then echo -n ", but process is running" echo "$PID" > "$PID_FILE" else echo -n ", and process is not running" return 1 fi fi PID=`cat $PID_FILE` kill $PID; rm -f -- "$PID_FILE"; # wait until the process is finished RETRIES=0 MAX_RETRIES=10 - while [ ! -z `pgrep -f "$PGREP_PATTERN"` ]; do + while [ ! -z `ps ax | grep -i "$PGREP_PATTERN" | grep -v grep | awk '{print $1}'` ]; do sleep 1 RETRIES=$((RETRIES+1)) if [ "$RETRIES" -ge "$MAX_RETRIES" ]; then echo "$NAME service: stop tried $MAX_RETRIES times but process $PID is still running" return 1 fi done echo "$NAME stopped" return 0 } status() { if ! [ -f "$PID_FILE" ]; then echo "$NAME stopped" exit 1 fi PID=`cat "$PID_FILE"` - if ! [ `ps -p "$PID" -o pid= || echo 1` -eq `pgrep -f "$PGREP_PATTERN" || echo 2` ] ; then + if ! [ `ps -p "$PID" -o pid= || echo 1` -eq `ps ax | grep -i "$PGREP_PATTERN" | grep -v grep | awk '{print $1}' || echo 2` ] ; then echo "$NAME stopped but pid file exists" exit 1 fi echo "$NAME running with pid $PID" exit 0 } case "$1" in status) status ;; start) echo "Starting daemon: $NAME" start ;; stop) echo "Stopping daemon: $NAME" stop ;; restart) echo "Restarting daemon: $NAME" stop sleep 2 start ;; *) echo "Usage: "$1" {status|start|stop|restart}" exit 1 esac exit 0