diff --git a/.travis.yml b/.travis.yml index bc3759d..664d430 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,129 +1,174 @@ sudo: required # group: deprecated-2017Q4 services: - docker cache: bundler: true directories: - spec/fixtures/artifacts - spec/fixtures/modules language: ruby script: travis_retry bundle exec rake $TASK jobs: allow_failures: - env: - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - TASK=beaker:centos-6-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-6-x64:snapshot - env: - TASK=beaker:centos-7-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-7-x64:snapshot - env: - TASK=beaker:debian-8-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-8-x64:snapshot - env: - TASK=beaker:debian-9-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-9-x64:snapshot include: - stage: intake env: - TASK=intake - env: - TASK=intake - PUPPET_VERSION='~> 5.0' - stage: acceptance env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:amazonlinux-1-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:amazonlinux-1-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:amazonlinux-2-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:amazonlinux-2-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:centos-6-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:centos-6-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:centos-7-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:centos-7-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:centos-8-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:centos-8-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:oracle-6-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:oracle-6-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:oracle-7-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:oracle-7-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:debian-8-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:debian-8-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:debian-9-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:debian-9-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:debian-10-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:debian-10-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:ubuntu-server-1404-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:ubuntu-server-1404-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:ubuntu-server-1604-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:ubuntu-server-1604-x64:acceptance[7.8.0] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:ubuntu-server-1804-x64:acceptance + - env: + - BEAKER_PUPPET_COLLECTION=puppet6 + - TASK=beaker:ubuntu-server-1804-x64:acceptance[7.8.0] + # - env: + # - BEAKER_PUPPET_COLLECTION=puppet6 + # - TASK=beaker:ubuntu-server-2004-x64:acceptance + # - env: + # - BEAKER_PUPPET_COLLECTION=puppet6 + # - TASK=beaker:ubuntu-server-2004-x64:acceptance[7.8.0] - stage: snapshots env: - TASK=beaker:ubuntu-server-1404-x64:snapshot env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - TASK=beaker:centos-6-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-6-x64:snapshot - env: - TASK=beaker:centos-7-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-7-x64:snapshot - env: - TASK=beaker:debian-8-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-8-x64:snapshot - env: - TASK=beaker:debian-9-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-9-x64:snapshot notifications: slack: secure: T1FO+ttrJNH+bXmNR6349qcttG68Qr1xmMqVVRnUr7+129GQO5174Z8MFC8ck0qOCZGHO7GCNO5seNFflrjF/5EKbdkmVnqhf9gVa9kN7I4psMzxJX9bp29xJA6m3wA4VqCosDKVFSfilDZujAblWT+KDHZLjP8sEWEnHPvCjf69S2XDQEWUoxZan5V9IJQas4XR+hMdIZTA3ChVrEyqRfeehAZImbAr/LH8zChZaTdHZQY7p2rN3+qVNi3+GISV9fNPpOCynnX/ACbdUaRt3+1etxGGaQMPzGmTejN3VlMw4OZRXImb6HQ2rXE+fNCASXiKiwylxTbriQsS0dFv4skxH03YlYM8pqaBpeIOwzf4n45tTzdAQZJMC5cOb+RvwS7qkAwuaVlVxiiA+MWRG/UcFpWS+iNn4KEKxbpBjYP8X1JIP9DlHLME7DNMM2pePv9X6ZjY6eDhVM1gbKi77dXOo5y2Sp0ru8QkLpIKFVXS01O+x7oDHHv2Osvih0jNMgM66Byso3KJYJ6EJ0D2/3Q9ZNpVM4CMuIY5pBQfXf691zqkBHI6JUnU6VMw97cH0k6Gq0ypZoW5trXdnRC5aEg4jKKid84zKmAeTpj/iMuagyb/a5msJstIVboynRtfDHR0J8WWhfSU2wzqKAb6L66iyRe62Fe8OGzLhk2+KNU= env: global: - secure: WFFcjwBIRBG2zyk4c8Ugq0tgI1YaH/+s5eV9h3i2kR1ggobT+nrNqn3hCOkmPtwGYPBNjVj6yp+7qy//MRe8AS2eo1XuMD/P4MYcDGmZiMnqPhz1UsLltGTYlh3y6jl9DJvNujFBQMnAu/ey2g/iWrcHdtl2qninvN3wOrXi2Bs= - secure: bvBaKoV5wBj2eQb4Zx8E2NaBDsMOyuHczRByVLNX5YqeuRWL9kcsUYzAUshFpd2GFa4tzfnSLKCp0+h3T4Uei5e8CjV5dx0VFmijXoZif0OJplRaJ+S3dJSluTV04NoE4u6l5Pg6kkFTMnAaApKVB4je2nSlgvrm/tuavhd9i0M= - secure: akshyW92CqV3Wt+rzQ3ScxIG55ILEaiwQ011rNF1kCXTds5HrHOGy++4VEidaTpems8OQH2+hCLK5r/7FXXgRQEV/TRYRGhp/y9mwqdioyDQ1D0yA3f42NWGNDGg2yOTTbhqQFJg394LDMiLmnevoiajEVIH+Ksr5bV/cIJc4Tc= diff --git a/manifests/config.pp b/manifests/config.pp index 3c808bd..dffda9f 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,252 +1,225 @@ # This class exists to coordinate all configuration related actions, # functionality and logical units in a central place. # # It is not intended to be used directly by external resources like node # definitions or other modules. # # @example importing this class into other classes to use its functionality: # class { 'elasticsearch::config': } # # @author Richard Pijnenburg # @author Tyler Langlois # @author Gavin Williams # class elasticsearch::config { #### Configuration Exec { path => [ '/bin', '/usr/bin', '/usr/local/bin' ], cwd => '/', } $init_defaults = merge( { 'MAX_OPEN_FILES' => '65535', }, $elasticsearch::init_defaults ) if ( $elasticsearch::ensure == 'present' ) { file { $elasticsearch::homedir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user; $elasticsearch::configdir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => 'root', mode => '2750'; $elasticsearch::datadir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, - owner => $elasticsearch::elasticsearch_user; + owner => $elasticsearch::elasticsearch_user, + mode => '2750'; $elasticsearch::logdir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, - mode => '0750'; + mode => '2750'; $elasticsearch::real_plugindir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => 'o+Xr'; "${elasticsearch::homedir}/lib": ensure => 'directory', group => '0', owner => 'root', recurse => true; } - if $elasticsearch::pid_dir { - file { $elasticsearch::pid_dir: - ensure => 'directory', - group => undef, - owner => $elasticsearch::elasticsearch_user, - recurse => true, - } - - if ($elasticsearch::service_provider == 'systemd') { - $group = $elasticsearch::elasticsearch_group - $user = $elasticsearch::elasticsearch_user - $pid_dir = $elasticsearch::pid_dir - - file { '/usr/lib/tmpfiles.d/elasticsearch.conf': - ensure => 'file', - content => template("${module_name}/usr/lib/tmpfiles.d/elasticsearch.conf.erb"), - group => '0', - owner => 'root', - } - } - } - - if $elasticsearch::defaults_location { - augeas { "${elasticsearch::defaults_location}/elasticsearch": - incl => "${elasticsearch::defaults_location}/elasticsearch", - lens => 'Shellvars.lns', - changes => [ - 'rm CONF_FILE', - 'rm CONF_DIR', - 'rm ES_PATH_CONF', - ], - } - - file { "${elasticsearch::defaults_location}/elasticsearch": - ensure => 'file', - group => $elasticsearch::elasticsearch_group, - owner => $elasticsearch::elasticsearch_user, - mode => '0640'; - } - } - # Defaults file, either from file source or from hash to augeas commands if ($elasticsearch::init_defaults_file != undef) { file { "${elasticsearch::defaults_location}/elasticsearch": ensure => $elasticsearch::ensure, source => $elasticsearch::init_defaults_file, owner => 'root', - group => '0', - mode => '0644', + group => $elasticsearch::elasticsearch_group, + mode => '0660', before => Service['elasticsearch'], notify => $elasticsearch::_notify_service, } } else { - augeas { 'init_defaults': + augeas { "${elasticsearch::defaults_location}/elasticsearch": incl => "${elasticsearch::defaults_location}/elasticsearch", lens => 'Shellvars.lns', changes => template("${module_name}/etc/sysconfig/defaults.erb"), before => Service['elasticsearch'], notify => $elasticsearch::_notify_service, } } # Generate config file $_config = deep_implode($elasticsearch::config) # Generate SSL config if $elasticsearch::ssl { if ($elasticsearch::keystore_password == undef) { fail('keystore_password required') } if ($elasticsearch::keystore_path == undef) { $_keystore_path = "${elasticsearch::configdir}/elasticsearch.ks" } else { $_keystore_path = $elasticsearch::keystore_path } - $_tls_config = { - 'xpack.security.transport.ssl.enabled' => true, - 'xpack.security.http.ssl.enabled' => true, - 'xpack.ssl.keystore.path' => $_keystore_path, - 'xpack.ssl.keystore.password' => $elasticsearch::keystore_password, + # Set the correct xpack. settings based on ES version + if (versioncmp($elasticsearch::version, '7') >= 0) { + $_tls_config = { + 'xpack.security.http.ssl.enabled' => true, + 'xpack.security.http.ssl.keystore.path' => $_keystore_path, + 'xpack.security.http.ssl.keystore.password' => $elasticsearch::keystore_password, + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.transport.ssl.keystore.path' => $_keystore_path, + 'xpack.security.transport.ssl.keystore.password' => $elasticsearch::keystore_password, + } + } + else { + $_tls_config = { + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.http.ssl.enabled' => true, + 'xpack.ssl.keystore.path' => $_keystore_path, + 'xpack.ssl.keystore.password' => $elasticsearch::keystore_password, + } } # Trust CA Certificate java_ks { 'elasticsearch_ca': ensure => 'latest', certificate => $elasticsearch::ca_certificate, target => $_keystore_path, password => $elasticsearch::keystore_password, trustcacerts => true, } # Load node certificate and private key java_ks { 'elasticsearch_node': ensure => 'latest', certificate => $elasticsearch::certificate, private_key => $elasticsearch::private_key, target => $_keystore_path, password => $elasticsearch::keystore_password, } } else { $_tls_config = {} } # # Logging file or hash # if ($elasticsearch::logging_file != undef) { # $_log4j_content = undef # } else { # if ($elasticsearch::logging_template != undef ) { # $_log4j_content = template($elasticsearch::logging_template) # } else { # $_log4j_content = template("${module_name}/etc/elasticsearch/log4j2.properties.erb") # } # $_logging_source = undef # } # file { # "${elasticsearch::configdir}/log4j2.properties": # ensure => file, # content => $_log4j_content, # source => $_logging_source, # mode => '0644', # notify => $elasticsearch::_notify_service, # require => Class['elasticsearch::package'], # before => Class['elasticsearch::service'], # } # Generate Elasticsearch config $_es_config = merge( $elasticsearch::config, { 'path.data' => $elasticsearch::datadir }, { 'path.logs' => $elasticsearch::logdir }, $_tls_config ) datacat_fragment { 'main_config': target => "${elasticsearch::configdir}/elasticsearch.yml", data => $_es_config, } datacat { "${elasticsearch::configdir}/elasticsearch.yml": template => "${module_name}/etc/elasticsearch/elasticsearch.yml.erb", notify => $elasticsearch::_notify_service, require => Class['elasticsearch::package'], owner => $elasticsearch::elasticsearch_user, group => $elasticsearch::elasticsearch_group, mode => '0440', } # Add any additional JVM options $elasticsearch::jvm_options.each |String $jvm_option| { file_line { "jvm_option_${jvm_option}": ensure => present, path => "${elasticsearch::configdir}/jvm.options", line => $jvm_option, notify => $elasticsearch::_notify_service, } } if $elasticsearch::system_key != undef { file { "${elasticsearch::configdir}/system_key": ensure => 'file', source => $elasticsearch::system_key, mode => '0400', } } # Add secrets to keystore if $elasticsearch::secrets != undef { elasticsearch_keystore { 'elasticsearch_secrets': configdir => $elasticsearch::configdir, purge => $elasticsearch::purge_secrets, settings => $elasticsearch::secrets, notify => $::elaticsearch::_notify_service, } } } elsif ( $elasticsearch::ensure == 'absent' ) { file { $elasticsearch::real_plugindir: ensure => 'absent', force => true, backup => false, } file { "${elasticsearch::defaults_location}/elasticsearch": ensure => 'absent', subscribe => Service['elasticsearch'], } } } diff --git a/spec/acceptance/nodesets/ubuntu-server-2004-x64.yml b/spec/acceptance/nodesets/ubuntu-server-2004-x64.yml new file mode 100644 index 0000000..c0b13f5 --- /dev/null +++ b/spec/acceptance/nodesets/ubuntu-server-2004-x64.yml @@ -0,0 +1,17 @@ +HOSTS: + ubuntu-20-04: + roles: + - agent + - master + - database + - dashboard + platform: ubuntu-20.04-amd64 + image: ubuntu:20.04 + hypervisor: docker + docker_cmd: ["/sbin/init"] + docker_preserve_image: true + docker_image_commands: + - apt-get update + - apt-get install -yq libssl-dev apt-transport-https openjdk-8-jdk iproute2 +CONFIG: + log_level: warn diff --git a/spec/classes/000_elasticsearch_init_spec.rb b/spec/classes/000_elasticsearch_init_spec.rb index 823c787..c37fe5e 100644 --- a/spec/classes/000_elasticsearch_init_spec.rb +++ b/spec/classes/000_elasticsearch_init_spec.rb @@ -1,442 +1,438 @@ require 'spec_helper' describe 'elasticsearch', :type => 'class' do default_params = { :config => { 'node.name' => 'foo' } } on_supported_os.each do |os, facts| context "on #{os}" do case facts[:os]['family'] when 'Debian' let(:defaults_path) { '/etc/default' } let(:system_service_folder) { '/lib/systemd/system' } let(:pkg_ext) { 'deb' } let(:pkg_prov) { 'dpkg' } let(:version_add) { '' } if (facts[:os]['name'] == 'Debian' and \ facts[:os]['release']['major'].to_i >= 8) or \ (facts[:os]['name'] == 'Ubuntu' and \ facts[:os]['release']['major'].to_i >= 15) let(:systemd_service_path) { '/lib/systemd/system' } test_pid = true else test_pid = false end when 'RedHat' let(:defaults_path) { '/etc/sysconfig' } let(:system_service_folder) { '/lib/systemd/system' } let(:pkg_ext) { 'rpm' } let(:pkg_prov) { 'rpm' } let(:version_add) { '-1' } if facts[:os]['release']['major'].to_i >= 7 let(:systemd_service_path) { '/lib/systemd/system' } test_pid = true else test_pid = false end when 'Suse' let(:defaults_path) { '/etc/sysconfig' } let(:pkg_ext) { 'rpm' } let(:pkg_prov) { 'rpm' } let(:version_add) { '-1' } if facts[:os]['name'] == 'OpenSuSE' and facts[:os]['release']['major'].to_i <= 12 let(:systemd_service_path) { '/lib/systemd/system' } else let(:systemd_service_path) { '/usr/lib/systemd/system' } end end let(:facts) do facts.merge('scenario' => '', 'common' => '') end let(:params) do default_params.merge({}) end # Varies depending on distro it { should contain_augeas("#{defaults_path}/elasticsearch") } - it do - should contain_file("#{defaults_path}/elasticsearch").with( - :ensure => 'file', - :group => 'elasticsearch', - :owner => 'elasticsearch', - :mode => '0640' - ) - end # Systemd-specific files if test_pid == true it { should contain_service('elasticsearch').with( :ensure => 'running', :enable => true ) } - it { should contain_file('/usr/lib/tmpfiles.d/elasticsearch.conf') } end context 'java installation' do let(:pre_condition) do <<-MANIFEST include ::java MANIFEST end it { should contain_class('elasticsearch::config') .that_requires('Class[java]') } end context 'package installation' do context 'via repository' do context 'with specified version' do let(:params) do default_params.merge( :version => '1.0' ) end it { should contain_package('elasticsearch') .with(:ensure => "1.0#{version_add}") } end if facts[:os]['family'] == 'RedHat' context 'Handle special CentOS/RHEL package versioning' do let(:params) do default_params.merge( :version => '1.1-2' ) end it { should contain_package('elasticsearch') .with(:ensure => '1.1-2') } end end end context 'when setting package version and package_url' do let(:params) do default_params.merge( :version => '0.90.10', :package_url => "puppet:///path/to/some/es-0.90.10.#{pkg_ext}" ) end it { expect { should raise_error(Puppet::Error) } } end context 'via package_url setting' do ['file:/', 'ftp://', 'http://', 'https://', 'puppet:///'].each do |schema| context "using #{schema} schema" do let(:params) do default_params.merge( :package_url => "#{schema}domain-or-path/pkg.#{pkg_ext}" ) end unless schema.start_with? 'puppet' it { should contain_exec('create_package_dir_elasticsearch') .with(:command => 'mkdir -p /opt/elasticsearch/swdl') } it { should contain_file('/opt/elasticsearch/swdl') .with( :purge => false, :force => false, :require => 'Exec[create_package_dir_elasticsearch]' ) } end case schema when 'file:/' it { should contain_file( "/opt/elasticsearch/swdl/pkg.#{pkg_ext}" ).with( :source => "/domain-or-path/pkg.#{pkg_ext}", :backup => false ) } when 'puppet:///' it { should contain_file( "/opt/elasticsearch/swdl/pkg.#{pkg_ext}" ).with( :source => "#{schema}domain-or-path/pkg.#{pkg_ext}", :backup => false ) } else [true, false].each do |verify_certificates| context "with download_tool_verify_certificates '#{verify_certificates}'" do let(:params) do default_params.merge( :package_url => "#{schema}domain-or-path/pkg.#{pkg_ext}", :download_tool_verify_certificates => verify_certificates ) end flag = (not verify_certificates) ? ' --no-check-certificate' : '' it { should contain_exec('download_package_elasticsearch') .with( :command => "wget#{flag} -O /opt/elasticsearch/swdl/pkg.#{pkg_ext} #{schema}domain-or-path/pkg.#{pkg_ext} 2> /dev/null", :require => 'File[/opt/elasticsearch/swdl]' ) } end end end it { should contain_package('elasticsearch') .with( :ensure => 'present', :source => "/opt/elasticsearch/swdl/pkg.#{pkg_ext}", :provider => pkg_prov ) } end end context 'using http:// schema with proxy_url' do let(:params) do default_params.merge( :package_url => "http://www.domain.com/package.#{pkg_ext}", :proxy_url => 'http://proxy.example.com:12345/' ) end it { should contain_exec('download_package_elasticsearch') .with( :environment => [ 'use_proxy=yes', 'http_proxy=http://proxy.example.com:12345/', 'https_proxy=http://proxy.example.com:12345/' ] ) } end end end # package context 'when setting the module to absent' do let(:params) do default_params.merge( :ensure => 'absent' ) end case facts[:os]['family'] when 'Suse' it { should contain_package('elasticsearch') .with(:ensure => 'absent') } else it { should contain_package('elasticsearch') .with(:ensure => 'purged') } end + it { should contain_service('elasticsearch') + .with( + :ensure => 'stopped', + :enable => 'false' + ) } it { should contain_file('/usr/share/elasticsearch/plugins') .with(:ensure => 'absent') } + it { should contain_file("#{defaults_path}/elasticsearch") + .with(:ensure => 'absent') } end context 'When managing the repository' do let(:params) do default_params.merge( :manage_repo => true ) end it { should contain_class('elastic_stack::repo') } end context 'When not managing the repository' do let(:params) do default_params.merge( :manage_repo => false ) end it { should compile.with_all_deps } end end end on_supported_os( :hardwaremodels => ['x86_64'], :supported_os => [ { 'operatingsystem' => 'CentOS', 'operatingsystemrelease' => ['7'] } ] ).each do |os, facts| context "on #{os}" do let(:facts) { facts.merge( :scenario => '', :common => '' ) } describe 'main class tests' do # init.pp it { should compile.with_all_deps } it { should contain_class('elasticsearch') } it { should contain_class('elasticsearch::package') } it { should contain_class('elasticsearch::config') .that_requires('Class[elasticsearch::package]') } it { should contain_class('elasticsearch::service') .that_requires('Class[elasticsearch::config]') } # Base directories it { should contain_file('/etc/elasticsearch') } it { should contain_file('/usr/share/elasticsearch') } it { should contain_file('/usr/share/elasticsearch/lib') } it { should contain_file('/var/lib/elasticsearch') } it { should contain_exec('remove_plugin_dir') } end context 'package installation' do describe 'with default package' do it { should contain_package('elasticsearch') .with(:ensure => 'present') } it { should_not contain_package('my-elasticsearch') .with(:ensure => 'present') } end describe 'with specified package name' do let(:params) do default_params.merge( :package_name => 'my-elasticsearch' ) end it { should contain_package('elasticsearch') .with(:ensure => 'present', :name => 'my-elasticsearch') } it { should_not contain_package('elasticsearch') .with(:ensure => 'present', :name => 'elasticsearch') } end describe 'with auto upgrade enabled' do let(:params) do default_params.merge( :autoupgrade => true ) end it { should contain_package('elasticsearch') .with(:ensure => 'latest') } end end describe 'running a a different user' do let(:params) do default_params.merge( :elasticsearch_user => 'myesuser', :elasticsearch_group => 'myesgroup' ) end it { should contain_file('/etc/elasticsearch') .with(:owner => 'root', :group => 'myesgroup') } it { should contain_file('/var/log/elasticsearch') .with(:owner => 'myesuser') } it { should contain_file('/usr/share/elasticsearch') .with(:owner => 'myesuser', :group => 'myesgroup') } it { should contain_file('/var/lib/elasticsearch') .with(:owner => 'myesuser', :group => 'myesgroup') } - it { should contain_file('/var/run/elasticsearch') - .with(:owner => 'myesuser') if facts[:os]['family'] == 'RedHat' } end describe 'setting jvm_options' do jvm_options = [ '-Xms16g', '-Xmx16g' ] let(:params) do default_params.merge( :jvm_options => jvm_options ) end jvm_options.each do |jvm_option| it { should contain_file_line("jvm_option_#{jvm_option}") .with( :ensure => 'present', :path => '/etc/elasticsearch/jvm.options', :line => jvm_option )} end end context 'with restart_on_change => true' do let(:params) do default_params.merge( :restart_on_change => true ) end describe 'should restart elasticsearch' do it { should contain_file('/etc/elasticsearch/elasticsearch.yml') .that_notifies('Service[elasticsearch]')} end describe 'setting jvm_options triggers restart' do let(:params) do super().merge( :jvm_options => ['-Xmx16g'] ) end it { should contain_file_line('jvm_option_-Xmx16g') .that_notifies('Service[elasticsearch]')} end end # This check helps catch dependency cycles. context 'create_resource' do # Helper for these tests def singular(s) case s when 'indices' 'index' when 'snapshot_repositories' 'snapshot_repository' else s[0..-2] end end { 'indices' => { 'test-index' => {} }, # 'instances' => { 'es-instance' => {} }, 'pipelines' => { 'testpipeline' => { 'content' => {} } }, 'plugins' => { 'head' => {} }, 'roles' => { 'elastic_role' => {} }, 'scripts' => { 'foo' => { 'source' => 'puppet:///path/to/foo.groovy' } }, 'snapshot_repositories' => { 'backup' => { 'location' => '/backups' } }, 'templates' => { 'foo' => { 'content' => {} } }, 'users' => { 'elastic' => { 'password' => 'foobar' } } }.each_pair do |deftype, params| describe deftype do let(:params) do default_params.merge( deftype => params ) end it { should compile } it { should send( "contain_elasticsearch__#{singular(deftype)}", params.keys.first ) } end end end describe 'oss' do let(:params) do default_params.merge(:oss => true) end it do should contain_package('elasticsearch').with( :name => 'elasticsearch-oss' ) end end end end end diff --git a/spec/classes/001_hiera_spec.rb b/spec/classes/001_hiera_spec.rb index b16df2c..e7ad80a 100644 --- a/spec/classes/001_hiera_spec.rb +++ b/spec/classes/001_hiera_spec.rb @@ -1,213 +1,213 @@ require 'spec_helper' describe 'elasticsearch', :type => 'class' do default_params = { :config => { 'node.name' => 'foo' } } let(:params) do default_params.merge({}) end on_supported_os( :hardwaremodels => ['x86_64'], :supported_os => [ { 'operatingsystem' => 'CentOS', 'operatingsystemrelease' => ['7'] } ] ).each do |os, facts| context "on #{os}" do context 'hiera' do describe 'indices' do context 'single indices' do let(:facts) { facts.merge(:scenario => 'singleindex') } it { should contain_elasticsearch__index('baz') .with( :ensure => 'present', :settings => { 'index' => { 'number_of_shards' => 1 } } ) } it { should contain_elasticsearch_index('baz') } it { should contain_es_instance_conn_validator( 'baz-index-conn-validator' ) } end context 'no indices' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__index('baz') } end end context 'config' do let(:facts) { facts.merge(:scenario => 'singleinstance') } - it { should contain_augeas('init_defaults') } + it { should contain_augeas('/etc/sysconfig/elasticsearch') } it { should contain_file('/etc/elasticsearch/elasticsearch.yml') } it { should contain_datacat('/etc/elasticsearch/elasticsearch.yml') } it { should contain_datacat_fragment('main_config') } it { should contain_service('elasticsearch').with( :ensure => 'running', :enable => true ) } end # of config describe 'pipelines' do context 'single pipeline' do let(:facts) { facts.merge(:scenario => 'singlepipeline') } it { should contain_elasticsearch__pipeline('testpipeline') .with( :ensure => 'present', :content => { 'description' => 'Add the foo field', 'processors' => [ { 'set' => { 'field' => 'foo', 'value' => 'bar' } } ] } ) } it { should contain_elasticsearch_pipeline('testpipeline') } end context 'no pipelines' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__pipeline('testpipeline') } end end describe 'plugins' do context 'single plugin' do let(:facts) { facts.merge(:scenario => 'singleplugin') } it { should contain_elasticsearch__plugin('mobz/elasticsearch-head') .with( :ensure => 'present', :module_dir => 'head' ) } it { should contain_elasticsearch_plugin('mobz/elasticsearch-head') } end context 'no plugins' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__plugin( 'mobz/elasticsearch-head/1.0.0' ) } end end describe 'roles' do context 'single roles' do let(:facts) { facts.merge(:scenario => 'singlerole') } let(:params) do default_params end it { should contain_elasticsearch__role('admin') .with( :ensure => 'present', :privileges => { 'cluster' => 'monitor', 'indices' => { '*' => 'all' } }, :mappings => [ 'cn=users,dc=example,dc=com' ] ) } it { should contain_elasticsearch_role('admin') } it { should contain_elasticsearch_role_mapping('admin') } end context 'no roles' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__role('admin') } end end describe 'scripts' do context 'single scripts' do let(:facts) { facts.merge(:scenario => 'singlescript') } it { should contain_elasticsearch__script('myscript') .with( :ensure => 'present', :source => 'puppet:///file/here' ) } it { should contain_file('/usr/share/elasticsearch/scripts/here') } end context 'no roles' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__script('myscript') } end end describe 'templates' do context 'single template' do let(:facts) { facts.merge(:scenario => 'singletemplate') } it { should contain_elasticsearch__template('foo') .with( :ensure => 'present', :content => { 'template' => 'foo-*', 'settings' => { 'index' => { 'number_of_replicas' => 0 } } } ) } it { should contain_elasticsearch_template('foo') } end context 'no templates' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__template('foo') } end end describe 'users' do context 'single users' do let(:facts) { facts.merge(:scenario => 'singleuser') } let(:params) do default_params end it { should contain_elasticsearch__user('elastic') .with( :ensure => 'present', :roles => ['admin'], :password => 'password' ) } it { should contain_elasticsearch_user('elastic') } end context 'no users' do let(:facts) { facts.merge(:scenario => '') } it { should_not contain_elasticsearch__user('elastic') } end end end end end end diff --git a/spec/defines/004_elasticsearch_plugin_spec.rb b/spec/defines/004_elasticsearch_plugin_spec.rb index 5d8cac6..48a99af 100644 --- a/spec/defines/004_elasticsearch_plugin_spec.rb +++ b/spec/defines/004_elasticsearch_plugin_spec.rb @@ -1,301 +1,301 @@ require 'spec_helper' describe 'elasticsearch::plugin', :type => 'define' do let(:title) { 'mobz/elasticsearch-head/1.0.0' } on_supported_os( :hardwaremodels => ['x86_64'], :supported_os => [ { 'operatingsystem' => 'CentOS', 'operatingsystemrelease' => ['6'] } ] ).each do |_os, facts| let(:facts) do facts.merge('scenario' => '', 'common' => '') end let(:pre_condition) do <<-EOS class { "elasticsearch": config => { "node" => { "name" => "test" } } } EOS end context 'default values' do context 'present' do let(:params) do { :ensure => 'present', :configdir => '/etc/elasticsearch' } end it { is_expected.to compile } end context 'absent' do let(:params) do { :ensure => 'absent' } end it { is_expected.to compile } end context 'configdir' do it { should contain_elasticsearch__plugin( 'mobz/elasticsearch-head/1.0.0' ).with_configdir('/etc/elasticsearch') } it { should contain_elasticsearch_plugin( 'mobz/elasticsearch-head/1.0.0' ).with_configdir('/etc/elasticsearch') } end end context 'with module_dir' do context 'add a plugin' do let(:params) do { :ensure => 'present', :module_dir => 'head' } end it { should contain_elasticsearch__plugin( 'mobz/elasticsearch-head/1.0.0' ) } it { should contain_elasticsearch_plugin( 'mobz/elasticsearch-head/1.0.0' ) } it { should contain_file( '/usr/share/elasticsearch/plugins/head' ).that_requires( 'Elasticsearch_plugin[mobz/elasticsearch-head/1.0.0]' ) } end context 'remove a plugin' do let(:params) do { :ensure => 'absent', :module_dir => 'head' } end it { should contain_elasticsearch__plugin( 'mobz/elasticsearch-head/1.0.0' ) } it { should contain_elasticsearch_plugin( 'mobz/elasticsearch-head/1.0.0' ).with( :ensure => 'absent' ) } it { should contain_file( '/usr/share/elasticsearch/plugins/head' ).that_requires( 'Elasticsearch_plugin[mobz/elasticsearch-head/1.0.0]' ) } end end context 'with url' do context 'add a plugin with full name' do let(:params) do { :ensure => 'present', :url => 'https://github.com/mobz/elasticsearch-head/archive/master.zip' } end it { should contain_elasticsearch__plugin('mobz/elasticsearch-head/1.0.0') } it { should contain_elasticsearch_plugin('mobz/elasticsearch-head/1.0.0').with(:ensure => 'present', :url => 'https://github.com/mobz/elasticsearch-head/archive/master.zip') } end end context 'offline plugin install' do let(:title) { 'head' } let(:params) do { :ensure => 'present', :source => 'puppet:///path/to/my/plugin.zip' } end it { should contain_elasticsearch__plugin('head') } it { should contain_file('/opt/elasticsearch/swdl/plugin.zip').with(:source => 'puppet:///path/to/my/plugin.zip', :before => 'Elasticsearch_plugin[head]') } it { should contain_elasticsearch_plugin('head').with(:ensure => 'present', :source => '/opt/elasticsearch/swdl/plugin.zip') } end describe 'service restarts' do let(:title) { 'head' } let(:params) do { :ensure => 'present', :module_dir => 'head' } end context 'restart_on_change set to false (default)' do let(:pre_condition) do <<-EOS class { "elasticsearch": } EOS end it { should_not contain_elasticsearch_plugin( 'head' ).that_notifies( 'Service[elasticsearch]' )} include_examples 'class', :sysv end context 'restart_on_change set to true' do let(:pre_condition) do <<-EOS class { "elasticsearch": restart_on_change => true, } EOS end it { should contain_elasticsearch_plugin( 'head' ).that_notifies( 'Service[elasticsearch]' )} - include_examples 'class', 'es-plugin', :sysv + include_examples('class') end context 'restart_plugin_change set to false (default)' do let(:pre_condition) do <<-EOS class { "elasticsearch": restart_plugin_change => false, } EOS end it { should_not contain_elasticsearch_plugin( 'head' ).that_notifies( 'Service[elasticsearch]' )} - include_examples 'class', 'es-plugin', :sysv + include_examples('class') end context 'restart_plugin_change set to true' do let(:pre_condition) do <<-EOS class { "elasticsearch": restart_plugin_change => true, } EOS end it { should contain_elasticsearch_plugin( 'head' ).that_notifies( 'Service[elasticsearch]' )} - include_examples 'class', 'es-plugin', :sysv + include_examples('class') end end describe 'proxy arguments' do let(:title) { 'head' } context 'unauthenticated' do context 'on define' do let(:params) do { :ensure => 'present', :proxy_host => 'es.local', :proxy_port => 8080 } end it { should contain_elasticsearch_plugin( 'head' ).with_proxy( 'http://es.local:8080' )} end context 'on main class' do let(:params) do { :ensure => 'present' } end let(:pre_condition) do <<-EOS class { 'elasticsearch': proxy_url => 'https://es.local:8080', } EOS end it { should contain_elasticsearch_plugin( 'head' ).with_proxy( 'https://es.local:8080' )} end end context 'authenticated' do context 'on define' do let(:params) do { :ensure => 'present', :proxy_host => 'es.local', :proxy_port => 8080, :proxy_username => 'elastic', :proxy_password => 'password' } end it { should contain_elasticsearch_plugin( 'head' ).with_proxy( 'http://elastic:password@es.local:8080' )} end context 'on main class' do let(:params) do { :ensure => 'present' } end let(:pre_condition) do <<-EOS class { 'elasticsearch': proxy_url => 'http://elastic:password@es.local:8080', } EOS end it { should contain_elasticsearch_plugin( 'head' ).with_proxy( 'http://elastic:password@es.local:8080' )} end end end describe 'collector ordering' do describe 'present' do let(:title) { 'head' } let(:pre_condition) do <<-EOS class { 'elasticsearch': } EOS end it { should contain_elasticsearch__plugin( 'head' ).that_requires( 'Class[elasticsearch::config]' )} - include_examples 'class', :sysv + include_examples('class') end end end end diff --git a/spec/fixtures/templates/post_6.0.json b/spec/fixtures/templates/6.x.json similarity index 99% rename from spec/fixtures/templates/post_6.0.json rename to spec/fixtures/templates/6.x.json index e118ec1..e1f9ba1 100644 --- a/spec/fixtures/templates/post_6.0.json +++ b/spec/fixtures/templates/6.x.json @@ -1,59 +1,59 @@ { "index_patterns": [ "logstash-*" ], "version": 123, "settings": { "index": { "refresh_interval": "5s", "analysis": { "analyzer": { "default": { "type": "standard", "stopwords": "_none_" } } } } }, "mappings": { "_doc": { "dynamic_templates": [ { "string_fields": { "match": "*", "match_mapping_type": "string", "mapping": { "type": "multi_field", "fields": { "{name}": { "type": "text", "index": "analyzed", "omit_norms": true }, "raw": { "type ": "text", "index": "not_analyzed", "ignore_above": 256 } } } } } ], "properties": { "@version": { "type": "text", "index": false }, "geoip": { "type": "object", "dynamic": true, "properties": { "location": { "type": "geo_point" } } } } } } -} +} \ No newline at end of file diff --git a/spec/fixtures/templates/post_8.0.json b/spec/fixtures/templates/7.x.json similarity index 74% copy from spec/fixtures/templates/post_8.0.json copy to spec/fixtures/templates/7.x.json index db8c484..54569e0 100644 --- a/spec/fixtures/templates/post_8.0.json +++ b/spec/fixtures/templates/7.x.json @@ -1,57 +1,42 @@ { "index_patterns": [ "logstash-*" ], "version": 123, "settings": { "index": { "refresh_interval": "5s", "analysis": { "analyzer": { "default": { "type": "standard", "stopwords": "_none_" } } } } }, "mappings": { "dynamic_templates": [ { "string_fields": { "match": "*", "match_mapping_type": "string", "mapping": { "type": "multi_field", "fields": { "{name}": { "type": "text", "index": "analyzed", "omit_norms": true }, "raw": { "type ": "text", "index": "not_analyzed", "ignore_above": 256 } } } } } - ], - "properties": { - "@version": { - "type": "text", - "index": false - }, - "geoip": { - "type": "object", - "dynamic": true, - "properties": { - "location": { - "type": "geo_point" - } - } - } - } + ] } -} +} \ No newline at end of file diff --git a/spec/fixtures/templates/post_8.0.json b/spec/fixtures/templates/post_8.0.json index db8c484..91cade1 100644 --- a/spec/fixtures/templates/post_8.0.json +++ b/spec/fixtures/templates/post_8.0.json @@ -1,57 +1,30 @@ { - "index_patterns": [ "logstash-*" ], - "version": 123, - "settings": { - "index": { - "refresh_interval": "5s", - "analysis": { - "analyzer": { - "default": { - "type": "standard", - "stopwords": "_none_" - } - } - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "string_fields": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "multi_field", - "fields": { - "{name}": { - "type": "text", - "index": "analyzed", - "omit_norms": true - }, - "raw": { - "type ": "text", - "index": "not_analyzed", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "@version": { - "type": "text", - "index": false + "index_patterns": ["te*", "bar*"], + "template": { + "settings": { + "number_of_shards": 1 + }, + "mappings": { + "_source": { + "enabled": false }, - "geoip": { - "type": "object", - "dynamic": true, - "properties": { - "location": { - "type": "geo_point" - } + "properties": { + "host_name": { + "type": "keyword" + }, + "created_at": { + "type": "date", + "format": "EEE MMM dd HH:mm:ss Z yyyy" } } + }, + "aliases": { + "mydata": { } } + }, + "priority": 10, + "version": 3, + "_meta": { + "description": "my custom" } -} +} \ No newline at end of file diff --git a/spec/fixtures/templates/pre_6.0.json b/spec/fixtures/templates/pre_6.0.json deleted file mode 100644 index 38a2379..0000000 --- a/spec/fixtures/templates/pre_6.0.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "template": "logstash-*", - "settings": { - "index": { - "refresh_interval": "5s", - "analysis": { - "analyzer": { - "default": { - "type": "standard", - "stopwords": "_none_" - } - } - } - } - }, - "mappings": { - "_default_": { - "_all": { - "enabled": true - }, - "dynamic_templates": [ - { - "string_fields": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "multi_field", - "fields": { - "{name}": { - "type": "string", - "index": "analyzed", - "omit_norms": true - }, - "raw": { - "type ": "string", - "index": "not_analyzed", - "ignore_above": 256 - } - } - } - } - } - ], - "properties": { - "@version": { - "type": "string", - "index": "not_analyzed" - }, - "geoip": { - "type": "object", - "dynamic": true, - "properties": { - "location": { - "type": "geo_point" - } - } - } - } - } - } -} diff --git a/spec/helpers/class_shared_examples.rb b/spec/helpers/class_shared_examples.rb index 701de97..c7d631a 100644 --- a/spec/helpers/class_shared_examples.rb +++ b/spec/helpers/class_shared_examples.rb @@ -1,19 +1,8 @@ -shared_examples 'class' do |init| +shared_examples 'class' do it { should compile.with_all_deps } - it { should contain_augeas('init_defaults') } + it { should contain_augeas('/etc/sysconfig/elasticsearch') } it { should contain_file('/etc/elasticsearch/elasticsearch.yml') } it { should contain_datacat('/etc/elasticsearch/elasticsearch.yml') } it { should contain_datacat_fragment('main_config') } it { should contain_service('elasticsearch') } - - case init - when :sysv - # it { should contain_elasticsearch__service__init(name) } - # it { should contain_elasticsearch_service_file("/etc/init.d/elasticsearch-#{name}") } - # it { should contain_file('/etc/init.d/elasticsearch') } - when :systemd - # it { should contain_elasticsearch__service__systemd(name) } - # it { should contain_file('/lib/systemd/system/elasticsearch.service') } - # it { should contain_exec('systemd_reload') } - end end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index e22289a..3f1c49b 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -1,255 +1,259 @@ require 'beaker-rspec' require 'beaker/puppet_install_helper' require 'securerandom' require 'thread' require 'infrataster/rspec' require 'rspec/retry' require 'vault' require 'simp/beaker_helpers' include Simp::BeakerHelpers require_relative 'spec_helper_tls' require_relative 'spec_utilities' require_relative '../lib/puppet_x/elastic/deep_to_i' require_relative '../lib/puppet_x/elastic/deep_to_s' # def f # RSpec.configuration.fact # end run_puppet_install_helper('agent') unless ENV['BEAKER_provision'] == 'no' RSpec.configure do |c| # General-purpose spec-global variables c.add_setting :v, :default => {} # Puppet debug logging v[:puppet_debug] = ENV['BEAKER_debug'] ? true : false unless ENV['snapshot_version'].nil? v[:snapshot_version] = ENV['snapshot_version'] v[:is_snapshot] = ENV['SNAPSHOT_TEST'] == 'true' end unless ENV['ELASTICSEARCH_VERSION'].nil? and v[:snapshot_version].nil? v[:elasticsearch_full_version] = ENV['ELASTICSEARCH_VERSION'] || v[:snapshot_version] v[:elasticsearch_major_version] = v[:elasticsearch_full_version].split('.').first.to_i v[:elasticsearch_package] = {} - v[:template] = if v[:elasticsearch_major_version] < 6 - JSON.load(File.new('spec/fixtures/templates/pre_6.0.json')) + v[:template] = if v[:elasticsearch_major_version] == 6 + JSON.load(File.new('spec/fixtures/templates/6.x.json')) elsif v[:elasticsearch_major_version] >= 8 JSON.load(File.new('spec/fixtures/templates/post_8.0.json')) else - JSON.load(File.new('spec/fixtures/templates/post_6.0.json')) + JSON.load(File.new('spec/fixtures/templates/7.x.json')) end v[:template] = Puppet_X::Elastic.deep_to_i(Puppet_X::Elastic.deep_to_s(v[:template])) v[:pipeline] = JSON.load(File.new('spec/fixtures/pipelines/example.json')) v[:elasticsearch_plugins] = Dir[ artifact("*#{v[:elasticsearch_full_version]}.zip", ['plugins']) ].map do |plugin| plugin_filename = File.basename(plugin) plugin_name = plugin_filename.match(/^(?.+)-#{v[:elasticsearch_full_version]}.zip/)[:name] [ plugin_name, { :path => plugin, :url => derive_plugin_urls_for(v[:elasticsearch_full_version], [plugin_name]).keys.first } ] end.to_h end v[:oss] = (not ENV['OSS_PACKAGE'].nil?) and ENV['OSS_PACKAGE'] == 'true' v[:cluster_name] = SecureRandom.hex(10) # rspec-retry c.display_try_failure_messages = true c.default_sleep_interval = 10 # General-case retry keyword for unstable tests c.around :each, :with_retries do |example| example.run_with_retry retry: 10 end # Helper hook for module cleanup c.after :context, :with_cleanup do apply_manifest <<-EOS class { 'elasticsearch': ensure => 'absent', manage_repo => true, oss => #{v[:oss]}, } file { '/usr/share/elasticsearch/plugin': ensure => 'absent', force => true, recurse => true, require => Class['elasticsearch'], } EOS end c.before :context, :with_certificates do @keystore_password = SecureRandom.hex @role = [*('a'..'z')].sample(8).join # Setup TLS cert placement @tls = gen_certs(2, '/tmp') create_remote_file hosts, @tls[:ca][:cert][:path], @tls[:ca][:cert][:pem] @tls[:clients].each do |node| node.each do |_type, params| create_remote_file hosts, params[:path], params[:pem] end end end c.before :context, :with_license do Vault.address = ENV['VAULT_ADDR'] - Vault.auth.approle ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID'] + if ENV['CI'] + Vault.auth.approle(ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID']) + else + Vault.auth.token(ENV['VAULT_TOKEN']) + end licenses = Vault.with_retries(Vault::HTTPConnectionError) do Vault.logical.read(ENV['VAULT_PATH']) end.data raise 'No license found!' unless licenses license = case v[:elasticsearch_major_version] - when 2 - licenses[:v2] - else + when 6 licenses[:v5] + else + licenses[:v7] end create_remote_file hosts, '/tmp/license.json', license v[:elasticsearch_license_path] = '/tmp/license.json' end c.after :context, :then_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end c.before :context, :first_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end # Provide a hook filter to spit out some ES logs if the example fails. c.after(:example, :logs_on_failure) do |example| if example.exception hosts.each do |host| on host, "find / -name '#{v[:cluster_name]}.log' | xargs cat || true" do |result| puts result.formatted_output end end end end end files_dir = ENV['files_dir'] || './spec/fixtures/artifacts' # General bootstrapping steps for each host hosts.each do |host| # # Set the host to 'aio' in order to adopt the puppet-agent style of # # installation, and configure paths/etc. # host[:type] = 'aio' # configure_defaults_on host, 'aio' if fact('os.family') == 'Suse' install_package host, '--force-resolution augeas-devel libxml2-devel ruby-devel' on host, 'gem install ruby-augeas --no-ri --no-rdoc' end v[:ext] = case fact('os.family') when 'Debian' 'deb' else 'rpm' end if v[:elasticsearch_package] v[:elasticsearch_package].merge!( derive_full_package_url( v[:elasticsearch_full_version], [v[:ext]] ).flat_map do |url, filename| [[:url, url], [:filename, filename], [:path, artifact(filename)]] end.to_h ) end Infrataster::Server.define(:docker) do |server| server.address = host[:ip] server.ssh = host[:ssh].tap { |s| s.delete :forward_agent } end Infrataster::Server.define(:container) do |server| server.address = host[:vm_ip] # this gets ignored anyway server.from = :docker end end RSpec.configure do |c| if v[:is_snapshot] c.before :suite do scp_to default, "#{files_dir}/elasticsearch-snapshot.#{v[:ext]}", "/tmp/elasticsearch-snapshot.#{v[:ext]}" v[:snapshot_package] = "file:/tmp/elasticsearch-snapshot.#{v[:ext]}" end end c.before :suite do # Install module and dependencies install_dev_puppet_module :ignore_list => [ 'junit' ] + Beaker::DSL::InstallUtils::ModuleUtils::PUPPET_MODULE_INSTALL_IGNORE hosts.each do |host| modules = %w[archive augeas_core datacat java java_ks stdlib elastic_stack] dist_module = { 'Debian' => ['apt'], 'Suse' => ['zypprepo'], 'RedHat' => %w[concat yumrepo_core] }[fact('os.family')] modules += dist_module unless dist_module.nil? modules.each do |mod| copy_module_to( host, :module_name => mod, :source => "spec/fixtures/modules/#{mod}" ) end on(host, 'mkdir -p etc/puppet/modules/another/files/') # Apt doesn't update package caches sometimes, ensure we're caught up. shell 'apt-get update' if fact('os.family') == 'Debian' end # Use the Java class once before the suite of tests unless shell('command -v java', :accept_all_exit_codes => true).exit_code.zero? java = case fact('os.name') when 'OpenSuSE' 'package => "java-1_8_0-openjdk-headless",' else '' end apply_manifest <<-MANIFEST class { "java" : distribution => "jre", #{java} } MANIFEST end end end # # Java 8 is only easy to manage on recent distros # def v5x_capable? # (fact('os.family') == 'RedHat' and \ # not (fact('os.name') == 'OracleLinux' and \ # f['os']['release']['major'] == '6')) or \ # f.dig 'os', 'distro', 'codename' == 'xenial' # end diff --git a/spec/spec_utilities.rb b/spec/spec_utilities.rb index 03568eb..489bd6a 100644 --- a/spec/spec_utilities.rb +++ b/spec/spec_utilities.rb @@ -1,132 +1,134 @@ require 'bcrypt' require 'open-uri' def to_agent_version(puppet_version) # REF: https://docs.puppet.com/puppet/latest/reference/about_agent.html { # Puppet => Agent '4.10.4' => '1.10.4', '4.10.3' => '1.10.3', '4.10.2' => '1.10.2', '4.10.1' => '1.10.1', '4.10.0' => '1.10.0', '4.9.4' => '1.9.3', '4.8.2' => '1.8.3', '4.7.1' => '1.7.2', '4.7.0' => '1.7.1', '4.6.2' => '1.6.2', '4.5.3' => '1.5.3', '4.4.2' => '1.4.2', '4.4.1' => '1.4.1', '4.4.0' => '1.4.0', '4.3.2' => '1.3.6', '4.3.1' => '1.3.2', '4.3.0' => '1.3.0', '4.2.3' => '1.2.7', '4.2.2' => '1.2.6', '4.2.1' => '1.2.2', '4.2.0' => '1.2.1', '4.1.0' => '1.1.1', '4.0.0' => '1.0.1' }[puppet_version] end def derive_artifact_urls_for(full_version, plugins = ['analysis-icu']) derive_full_package_url(full_version).merge( derive_plugin_urls_for(full_version, plugins) ) end def derive_full_package_url(full_version, extensions = %w[deb rpm]) extensions.map do |ext| - url = if full_version.start_with? '2' - "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-#{full_version}.#{ext}" - else + url = if full_version.start_with? '6' "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}.#{ext}" + elsif ext == 'deb' + "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}-amd64.#{ext}" + else + "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}-x86_64.#{ext}" end [url, File.basename(url)] end.to_h end def derive_plugin_urls_for(full_version, plugins = ['analysis-icu']) plugins.map do |plugin| - url = if full_version.start_with? '2' - "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/plugin/#{plugin}/#{full_version}/#{plugin}-#{full_version}.zip" - else - "https://artifacts.elastic.co/downloads/elasticsearch-plugins/#{plugin}/#{plugin}-#{full_version}.zip" - end + url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/#{plugin}/#{plugin}-#{full_version}.zip" [url, File.join('plugins', File.basename(url))] end.to_h end def artifact(file, fixture_path = []) File.join(%w[spec fixtures artifacts] + fixture_path + [File.basename(file)]) end def get(url, file_path) puts "Fetching #{url}..." found = false until found uri = URI.parse(url) conn = Net::HTTP.new(uri.host, uri.port) conn.use_ssl = true res = conn.get(uri.path) if res.header['location'] url = res.header['location'] else found = true end end File.open(file_path, 'w+') { |fh| fh.write res.body } end def fetch_archives(archives) archives.each do |url, orig_fp| fp = "spec/fixtures/artifacts/#{orig_fp}" if File.exist? fp if fp.end_with? 'tar.gz' and !system("tar -tzf #{fp} &>/dev/null") puts "Archive #{fp} corrupt, re-fetching..." File.delete fp else puts "Already retrieved intact archive #{fp}..." next end end get url, fp end end def pid_file if fact('operatingsystem') == 'Debian' \ and fact('lsbmajdistrelease').to_i <= 7 '/var/run/elasticsearch.pid' else '/var/run/elasticsearch/elasticsearch.pid' end end def vault_available? - %w[VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID VAULT_PATH].select do |var| - ENV[var].nil? - end.empty? + if ENV['CI'] + %w[VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID VAULT_PATH].select do |var| + ENV[var].nil? + end.empty? + else + true + end end def http_retry(url) retries ||= 0 open(url).read rescue retry if (retries += 1) < 3 end # Helper to store arbitrary testing setting values def v RSpec.configuration.v end def semver(version) Gem::Version.new version end def bcrypt(value) BCrypt::Password.create(value) end diff --git a/spec/unit/type/elasticsearch_template_spec.rb b/spec/unit/type/elasticsearch_template_spec.rb index 0d50347..abeabf0 100644 --- a/spec/unit/type/elasticsearch_template_spec.rb +++ b/spec/unit/type/elasticsearch_template_spec.rb @@ -1,134 +1,134 @@ require_relative '../../helpers/unit/type/elasticsearch_rest_shared_examples' describe Puppet::Type.type(:elasticsearch_template) do let(:resource_name) { 'test_template' } include_examples 'REST API types', 'template', :content describe 'template attribute validation' do it 'should have a source parameter' do expect(described_class.attrtype(:source)).to eq(:param) end describe 'content and source validation' do it 'should require either "content" or "source"' do expect do described_class.new( :name => resource_name, :ensure => :present ) end.to raise_error(Puppet::Error, /content.*or.*source.*required/) end it 'should fail with both defined' do expect do described_class.new( :name => resource_name, :content => {}, :source => 'puppet:///example.json' ) end.to raise_error(Puppet::Error, /simultaneous/) end it 'should parse source paths into the content property' do file_stub = 'foo' [ Puppet::FileServing::Metadata, Puppet::FileServing::Content ].each do |klass| allow(klass).to receive(:indirection) .and_return(Object) end allow(Object).to receive(:find) .and_return(file_stub) allow(file_stub).to receive(:content) .and_return('{"template":"foobar-*", "order": 1}') expect(described_class.new( :name => resource_name, :source => '/example.json' )[:content]).to include( 'template' => 'foobar-*', 'order' => 1 ) end it 'should qualify settings' do expect(described_class.new( :name => resource_name, :content => { 'settings' => { 'number_of_replicas' => '2', 'index' => { 'number_of_shards' => '3' } } } )[:content]).to eq( 'order' => 0, 'aliases' => {}, 'mappings' => {}, 'settings' => { 'index' => { 'number_of_replicas' => 2, 'number_of_shards' => 3 } } ) end it 'detects flat qualified index settings' do expect(described_class.new( :name => resource_name, :content => { 'settings' => { 'number_of_replicas' => '2', 'index.number_of_shards' => '3' } } )[:content]).to eq( 'order' => 0, 'aliases' => {}, 'mappings' => {}, 'settings' => { 'index' => { 'number_of_replicas' => 2, 'number_of_shards' => 3 } } ) end end end # of describing when validing values describe 'insync?' do # Although users can pass the type a hash structure with any sort of values # - string, integer, or other native datatype - the Elasticsearch API # normalizes all values to strings. In order to verify that the type does # not incorrectly detect changes when values may be in string form, we take # an example template and force all values to strings to mimic what # Elasticsearch does. it 'is idempotent' do def deep_stringify(obj) if obj.is_a? Array obj.map { |element| deep_stringify(element) } elsif obj.is_a? Hash obj.merge(obj) { |_key, val| deep_stringify(val) } elsif [true, false].include? obj obj else obj.to_s end end - json = JSON.parse(File.read('spec/fixtures/templates/post_6.0.json')) + json = JSON.parse(File.read('spec/fixtures/templates/6.x.json')) is_template = described_class.new( :name => resource_name, :ensure => 'present', :content => json ).property(:content) should_template = described_class.new( :name => resource_name, :ensure => 'present', :content => deep_stringify(json) ).property(:content).should expect(is_template.insync?(should_template)).to be_truthy end end end # of describe Puppet::Type