diff --git a/manifests/config.pp b/manifests/config.pp index 07afd92..86389e6 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,296 +1,287 @@ # This class exists to coordinate all configuration related actions, # functionality and logical units in a central place. # # It is not intended to be used directly by external resources like node # definitions or other modules. # # @example importing this class into other classes to use its functionality: # class { 'elasticsearch::config': } # # @author Richard Pijnenburg # @author Tyler Langlois # @author Gavin Williams # class elasticsearch::config { #### Configuration Exec { path => [ '/bin', '/usr/bin', '/usr/local/bin' ], cwd => '/', } if ( $elasticsearch::ensure == 'present' ) { file { $elasticsearch::homedir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user; $elasticsearch::configdir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => 'root', mode => '2750'; $elasticsearch::datadir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user; $elasticsearch::logdir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => '0750'; $elasticsearch::_plugindir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => 'o+Xr'; "${elasticsearch::homedir}/lib": ensure => 'directory', group => '0', owner => 'root', recurse => true; } if $elasticsearch::pid_dir { file { $elasticsearch::pid_dir: ensure => 'directory', group => undef, owner => $elasticsearch::elasticsearch_user, recurse => true, } if ($elasticsearch::service_provider == 'systemd') { $group = $elasticsearch::elasticsearch_group $user = $elasticsearch::elasticsearch_user $pid_dir = $elasticsearch::pid_dir file { '/usr/lib/tmpfiles.d/elasticsearch.conf': ensure => 'file', content => template("${module_name}/usr/lib/tmpfiles.d/elasticsearch.conf.erb"), group => '0', owner => 'root', } } } if $elasticsearch::defaults_location { augeas { "${elasticsearch::defaults_location}/elasticsearch": incl => "${elasticsearch::defaults_location}/elasticsearch", lens => 'Shellvars.lns', changes => [ 'rm CONF_FILE', 'rm CONF_DIR', 'rm ES_PATH_CONF', ], } file { "${elasticsearch::defaults_location}/elasticsearch": ensure => 'file', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => '0640'; } } if $::elasticsearch::security_plugin != undef and ($::elasticsearch::security_plugin in ['shield', 'x-pack']) { file { "${::elasticsearch::configdir}/${::elasticsearch::security_plugin}" : ensure => 'directory', owner => 'root', group => $elasticsearch::elasticsearch_group, mode => '0750', } } # Define logging config file for the in-use security plugin if $::elasticsearch::security_logging_content != undef or $::elasticsearch::security_logging_source != undef { if $::elasticsearch::security_plugin == undef or ! ($::elasticsearch::security_plugin in ['shield', 'x-pack']) { fail("\"${::elasticsearch::security_plugin}\" is not a valid security_plugin parameter value") } $_security_logging_file = $::elasticsearch::security_plugin ? { 'shield' => 'logging.yml', default => 'log4j2.properties' } file { "/etc/elasticsearch/${::elasticsearch::security_plugin}/${_security_logging_file}" : content => $::elasticsearch::security_logging_content, source => $::elasticsearch::security_logging_source, } } $init_defaults = merge( { 'MAX_OPEN_FILES' => '65535', }, $::elasticsearch::init_defaults ) # Setup init defaults if ($::elasticsearch::ensure == 'present') { # Defaults file, either from file source or from hash to augeas commands if ($::elasticsearch::init_defaults_file != undef) { file { "${::elasticsearch::defaults_location}/elasticsearch": ensure => $::elasticsearch::ensure, source => $::elasticsearch::init_defaults_file, owner => 'root', group => '0', mode => '0644', before => Service['elasticsearch'], notify => $::elasticsearch::_notify_service, } } else { augeas { 'init_defaults': incl => "${::elasticsearch::defaults_location}/elasticsearch", lens => 'Shellvars.lns', changes => template("${module_name}/etc/sysconfig/defaults.erb"), before => Service['elasticsearch'], notify => $::elasticsearch::_notify_service, } } } else { # absent file { "${elasticsearch::defaults_location}/elasticsearch": ensure => 'absent', subscribe => Service['elasticsearch'], } } # Generate config file $_config = deep_implode($elasticsearch::config) # Generate SSL config if $elasticsearch::ssl { if ($elasticsearch::keystore_password == undef) { fail('keystore_password required') } if ($elasticsearch::keystore_path == undef) { $_keystore_path = "${elasticsearch::configdir}/${elasticsearch::security_plugin}/${name}.ks" } else { $_keystore_path = $elasticsearch::keystore_path } - if $elasticsearch::security_plugin == 'shield' { - $_tls_config = { - 'shield.transport.ssl' => true, - 'shield.http.ssl' => true, - 'shield.ssl.keystore.path' => $_keystore_path, - 'shield.ssl.keystore.password' => $elasticsearch::keystore_password, - } - } elsif $elasticsearch::security_plugin == 'x-pack' { - $_tls_config = { - 'xpack.security.transport.ssl.enabled' => true, - 'xpack.security.http.ssl.enabled' => true, - 'xpack.ssl.keystore.path' => $_keystore_path, - 'xpack.ssl.keystore.password' => $elasticsearch::keystore_password, - } + $_tls_config = { + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.http.ssl.enabled' => true, + 'xpack.ssl.keystore.path' => $_keystore_path, + 'xpack.ssl.keystore.password' => $elasticsearch::keystore_password, } # Trust CA Certificate java_ks { 'elasticsearch_ca': ensure => 'latest', certificate => $elasticsearch::ca_certificate, target => $_keystore_path, password => $elasticsearch::keystore_password, trustcacerts => true, } # Load node certificate and private key java_ks { 'elasticsearch_node': ensure => 'latest', certificate => $elasticsearch::certificate, private_key => $elasticsearch::private_key, target => $_keystore_path, password => $elasticsearch::keystore_password, } } else { $_tls_config = {} } # Logging file or hash if ($::elasticsearch::logging_file != undef) { $_log4j_content = undef } else { if ($::elasticsearch::logging_template != undef ) { $_log4j_content = template($::elasticsearch::logging_template) } else { $_log4j_content = template("${module_name}/etc/elasticsearch/log4j2.properties.erb") } $_logging_source = undef } file { "${::elasticsearch::configdir}/log4j2.properties": ensure => file, content => $_log4j_content, source => $_logging_source, mode => '0644', notify => $::elasticsearch::_notify_service, require => Class['elasticsearch::package'], before => Class['elasticsearch::service'], } # Generate Elasticsearch config $_es_config = merge( $::elasticsearch::config, { 'path.data' => $::elasticsearch::datadir }, { 'path.logs' => $::elasticsearch::logdir }, $_tls_config ) datacat_fragment { 'main_config': target => "${::elasticsearch::configdir}/elasticsearch.yml", data => $_es_config, } datacat { "${::elasticsearch::configdir}/elasticsearch.yml": template => "${module_name}/etc/elasticsearch/elasticsearch.yml.erb", notify => $::elasticsearch::_notify_service, require => Class['elasticsearch::package'], owner => $::elasticsearch::elasticsearch_user, group => $::elasticsearch::elasticsearch_group, mode => '0440', } # Configure JVM options file { "${::elasticsearch::configdir}/jvm.options": content => template("${module_name}/etc/elasticsearch/jvm.options.erb"), group => $::elasticsearch::elasticsearch_group, notify => $::elasticsearch::_notify_service, owner => $::elasticsearch::elasticsearch_user, } if $::elasticsearch::system_key != undef { file { "${::elasticsearch::configdir}/${::elasticsearch::security_plugin}/system_key": ensure => 'file', source => $::elasticsearch::system_key, mode => '0400', require => File["${::elasticsearch::configdir}/${::elasticsearch::security_plugin}"], } } # Add secrets to keystore if $::elasticsearch::secrets != undef { elasticsearch_keystore { 'elasticsearch_secrets': configdir => $::elasticsearch::configdir, purge => $::elasticsearch::purge_secrets, settings => $::elasticsearch::secrets, notify => $::elaticsearch::_notify_service, } } } elsif ( $elasticsearch::ensure == 'absent' ) { file { $elasticsearch::_plugindir: ensure => 'absent', force => true, backup => false, } file { "${elasticsearch::configdir}/jvm.options": ensure => 'absent', } } } diff --git a/manifests/init.pp b/manifests/init.pp index 927da61..c0761e2 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,606 +1,618 @@ # Top-level Elasticsearch class which may manage installation of the # Elasticsearch package, package repository, and other # global options and parameters. # # @summary Manages the installation of Elasticsearch and related options. # # @example install Elasticsearch # class { 'elasticsearch': } # # @example removal and decommissioning # class { 'elasticsearch': # ensure => 'absent', # } # # @example install everything but disable service(s) afterwards # class { 'elasticsearch': # status => 'disabled', # } # # @param ensure # Controls if the managed resources shall be `present` or `absent`. # If set to `absent`, the managed software packages will be uninstalled, and # any traces of the packages will be purged as well as possible, possibly # including existing configuration files. # System modifications (if any) will be reverted as well as possible (e.g. # removal of created users, services, changed log settings, and so on). # This is a destructive parameter and should be used with care. # # @param api_basic_auth_password # Defines the default REST basic auth password for API authentication. # # @param api_basic_auth_username # Defines the default REST basic auth username for API authentication. # # @param api_ca_file # Path to a CA file which will be used to validate server certs when # communicating with the Elasticsearch API over HTTPS. # # @param api_ca_path # Path to a directory with CA files which will be used to validate server # certs when communicating with the Elasticsearch API over HTTPS. # # @param api_host # Default host to use when accessing Elasticsearch APIs. # # @param api_port # Default port to use when accessing Elasticsearch APIs. # # @param api_protocol # Default protocol to use when accessing Elasticsearch APIs. # # @param api_timeout # Default timeout (in seconds) to use when accessing Elasticsearch APIs. # # @param autoupgrade # If set to `true`, any managed package will be upgraded on each Puppet run # when the package provider is able to find a newer version than the present # one. The exact behavior is provider dependent (see # {package, "upgradeable"}[http://j.mp/xbxmNP] in the Puppet documentation). # # @param ca_certificate # Path to the trusted CA certificate to add to this node's java keystore. # # @param certificate # Path to the certificate for this node signed by the CA listed in # ca_certificate. # # @param config # Elasticsearch configuration hash. # # @param configdir # Directory containing the elasticsearch configuration. # Use this setting if your packages deviate from the norm (`/etc/elasticsearch`) # # @param configdir_recurselimit # Dictates how deeply the file copy recursion logic should descend when # copying files from the `configdir` to instance `configdir`s. # # @param daily_rolling_date_pattern # File pattern for the file appender log when file_rolling_type is 'dailyRollingFile'. # # @param datadir # Allows you to set the data directory of Elasticsearch. # # @param datadir_instance_directories # Control whether individual directories for instances will be created within # each instance's data directory. # # @param default_logging_level # Default logging level for Elasticsearch. # # @param defaults_location # Absolute path to directory containing init defaults file. # # @param deprecation_logging # Wheter to enable deprecation logging. If enabled, deprecation logs will be # saved to ${cluster.name}_deprecation.log in the elastic search log folder. # # @param deprecation_logging_level # Default deprecation logging level for Elasticsearch. # # @param download_tool # Command-line invocation with which to retrieve an optional package_url. # # @param download_tool_insecure # Command-line invocation with which to retrieve an optional package_url when # certificate verification should be ignored. # # @param download_tool_verify_certificates # Whether or not to verify SSL/TLS certificates when retrieving package files # using a download tool instead of a package management provider. # # @param elasticsearch_group # The group Elasticsearch should run as. This also sets file group # permissions. # # @param elasticsearch_user # The user Elasticsearch should run as. This also sets file ownership. # # @param file_rolling_type # Configuration for the file appender rotation. It can be 'dailyRollingFile', # 'rollingFile' or 'file'. The first rotates by name, the second one by size # or third don't rotate automatically. # # @param homedir # Directory where the elasticsearch installation's files are kept (plugins, etc.) # # @param indices # Define indices via a hash. This is mainly used with Hiera's auto binding. # # @param init_defaults # Defaults file content in hash representation. # # @param init_defaults_file # Defaults file as puppet resource. # # @param init_template # Service file as a template. # # @param jvm_options # Array of options to set in jvm_options. # +# @param keystore_password +# Password to encrypt this node's Java keystore. +# +# @param keystore_path +# Custom path to the java keystore file. This parameter is optional. +# # @param license # Optional Elasticsearch license in hash or string form. # # @param logdir # Directory that will be used for Elasticsearch logging. # # @param logging_config # Representation of information to be included in the logging.yml file. # # @param logging_file # Instead of a hash, you may supply a `puppet://` file source for the # logging.yml file. # # @param logging_level # Default logging level for Elasticsearch. # # @param logging_template # Use a custom logging template - just supply the relative path, i.e. # `$module/elasticsearch/logging.yml.erb` # # @param manage_repo # Enable repo management by enabling official Elastic repositories. # # @param oss # Whether to use the purely open source Elasticsearch package distribution. # # @param package_dir # Directory where packages are downloaded to. # # @param package_dl_timeout # For http, https, and ftp downloads, you may set how long the exec resource # may take. # # @param package_name # Name Of the package to install. # # @param package_provider # Method to install the packages, currently only `package` is supported. # # @param package_url # URL of the package to download. # This can be an http, https, or ftp resource for remote packages, or a # `puppet://` resource or `file:/` for local packages # # @param pid_dir # Directory where the elasticsearch process should write out its PID. # # @param pipelines # Define pipelines via a hash. This is mainly used with Hiera's auto binding. # # @param plugindir # Directory containing elasticsearch plugins. # Use this setting if your packages deviate from the norm (/usr/share/elasticsearch/plugins) # # @param plugins # Define plugins via a hash. This is mainly used with Hiera's auto binding. # +# @param private_key +# Path to the key associated with this node's certificate. +# # @param proxy_url # For http and https downloads, you may set a proxy server to use. By default, # no proxy is used. # Format: `proto://[user:pass@]server[:port]/` # # @param purge_configdir # Purge the config directory of any unmanaged files. # # @param purge_package_dir # Purge package directory on removal # # @param purge_secrets # Whether or not keys present in the keystore will be removed if they are not # present in the specified secrets hash. # # @param repo_stage # Use stdlib stage setup for managing the repo instead of relationship # ordering. # # @param restart_on_change # Determines if the application should be automatically restarted # whenever the configuration, package, or plugins change. Enabling this # setting will cause Elasticsearch to restart whenever there is cause to # re-read configuration files, load new plugins, or start the service using an # updated/changed executable. This may be undesireable in highly available # environments. If all other restart_* parameters are left unset, the value of # `restart_on_change` is used for all other restart_*_change defaults. # # @param restart_config_change # Determines if the application should be automatically restarted # whenever the configuration changes. This includes the Elasticsearch # configuration file, any service files, and defaults files. # Disabling automatic restarts on config changes may be desired in an # environment where you need to ensure restarts occur in a controlled/rolling # manner rather than during a Puppet run. # # @param restart_package_change # Determines if the application should be automatically restarted # whenever the package (or package version) for Elasticsearch changes. # Disabling automatic restarts on package changes may be desired in an # environment where you need to ensure restarts occur in a controlled/rolling # manner rather than during a Puppet run. # # @param restart_plugin_change # Determines if the application should be automatically restarted whenever # plugins are installed or removed. # Disabling automatic restarts on plugin changes may be desired in an # environment where you need to ensure restarts occur in a controlled/rolling # manner rather than during a Puppet run. # # @param roles # Define roles via a hash. This is mainly used with Hiera's auto binding. # # @param rolling_file_max_backup_index # Max number of logs to store whern file_rolling_type is 'rollingFile' # # @param rolling_file_max_file_size # Max log file size when file_rolling_type is 'rollingFile' # # @param scripts # Define scripts via a hash. This is mainly used with Hiera's auto binding. # # @param secrets # Optional default configuration hash of key/value pairs to store in the # Elasticsearch keystore file. If unset, the keystore is left unmanaged. # # @param security_logging_content # File content for shield/x-pack logging configuration file (will be placed # into logging.yml or log4j2.properties file as appropriate). # # @param security_logging_source # File source for shield/x-pack logging configuration file (will be placed # into logging.yml or log4j2.properties file as appropriate). # # @param service_name # Elasticsearch serviice name # # @param security_plugin # Which security plugin will be used to manage users, roles, and # certificates. # # @param service_provider # The service resource type provider to use when managing elasticsearch instances. # # @param snapshot_repositories # Define snapshot repositories via a hash. This is mainly used with Hiera's auto binding. # # @param ssl # Whether to manage TLS certificates for Shield. Requires the ca_certificate, # certificate, private_key and keystore_password parameters to be set. # # @param status # To define the status of the service. If set to `enabled`, the service will # be run and will be started at boot time. If set to `disabled`, the service # is stopped and will not be started at boot time. If set to `running`, the # service will be run but will not be started at boot time. You may use this # to start a service on the first Puppet run instead of the system startup. # If set to `unmanaged`, the service will not be started at boot time and Puppet # does not care whether the service is running or not. For example, this may # be useful if a cluster management software is used to decide when to start # the service plus assuring it is running on the desired node. # # @param system_key # Source for the Shield/x-pack system key. Valid values are any that are # supported for the file resource `source` parameter. # # @param systemd_service_path # Path to the directory in which to install systemd service units. # # @param templates # Define templates via a hash. This is mainly used with Hiera's auto binding. # # @param users # Define templates via a hash. This is mainly used with Hiera's auto binding. # # @param validate_tls # Enable TLS/SSL validation on API calls. # # @param version # To set the specific version you want to install. # # @author Richard Pijnenburg # @author Tyler Langlois # @author Gavin Williams # class elasticsearch ( Enum['absent', 'present'] $ensure, Optional[String] $api_basic_auth_password, Optional[String] $api_basic_auth_username, Optional[String] $api_ca_file, Optional[String] $api_ca_path, String $api_host, Integer[0, 65535] $api_port, Enum['http', 'https'] $api_protocol, Integer $api_timeout, Boolean $autoupgrade, Hash $config, Stdlib::Absolutepath $configdir, Integer $configdir_recurselimit, String $daily_rolling_date_pattern, Elasticsearch::Multipath $datadir, Boolean $datadir_instance_directories, Optional[Stdlib::Absolutepath] $defaults_location, Boolean $deprecation_logging, String $deprecation_logging_level, Optional[String] $download_tool, Optional[String] $download_tool_insecure, Boolean $download_tool_verify_certificates, String $elasticsearch_group, String $elasticsearch_user, Enum['dailyRollingFile', 'rollingFile', 'file'] $file_rolling_type, Stdlib::Absolutepath $homedir, Hash $indices, Hash $init_defaults, Optional[String] $init_defaults_file, String $init_template, Array[String] $jvm_options, Optional[Variant[String, Hash]] $license, Stdlib::Absolutepath $logdir, Hash $logging_config, Optional[String] $logging_file, String $logging_level, Optional[String] $logging_template, Boolean $manage_repo, Boolean $oss, Stdlib::Absolutepath $package_dir, Integer $package_dl_timeout, String $package_name, Enum['package'] $package_provider, Optional[String] $package_url, Optional[Stdlib::Absolutepath] $pid_dir, Hash $pipelines, Optional[Stdlib::Absolutepath] $plugindir, Hash $plugins, Optional[Stdlib::HTTPUrl] $proxy_url, Boolean $purge_configdir, Boolean $purge_package_dir, Boolean $purge_secrets, Variant[Boolean, String] $repo_stage, Boolean $restart_on_change, Hash $roles, Integer $rolling_file_max_backup_index, String $rolling_file_max_file_size, Hash $scripts, Optional[Hash] $secrets, Optional[String] $security_logging_content, Optional[String] $security_logging_source, Optional[Enum['shield', 'x-pack']] $security_plugin, String $service_name, Enum['init', 'openbsd', 'openrc', 'systemd'] $service_provider, Hash $snapshot_repositories, Boolean $ssl, Elasticsearch::Status $status, Optional[String] $system_key, Stdlib::Absolutepath $systemd_service_path, Hash $templates, Hash $users, Boolean $validate_tls, Variant[String, Boolean] $version, Optional[Stdlib::Absolutepath] $ca_certificate = undef, Optional[Stdlib::Absolutepath] $certificate = undef, String $default_logging_level = $logging_level, + Optional[String] $keystore_password = undef, + Optional[Stdlib::Absolutepath] $keystore_path = undef, + Optional[Stdlib::Absolutepath] $private_key = undef, Boolean $restart_config_change = $restart_on_change, Boolean $restart_package_change = $restart_on_change, Boolean $restart_plugin_change = $restart_on_change, ) { #### Validate parameters if ($package_url != undef and $version != false) { fail('Unable to set the version number when using package_url option.') } if ($version != false) { case $facts['os']['family'] { 'RedHat', 'Linux', 'Suse': { if ($version =~ /.+-\d/) { $pkg_version = $version } else { $pkg_version = "${version}-1" } } default: { $pkg_version = $version } } } # This value serves as an unchanging default for platforms as a default for # init scripts to fallback on. $_datadir_default = $facts['kernel'] ? { 'Linux' => '/var/lib/elasticsearch', 'OpenBSD' => '/var/elasticsearch/data', default => undef, } # The OSS package distribution's package appends `-oss` to the end of the # canonical package name. $_package_name = $oss ? { true => "${package_name}-oss", default => $package_name, } # Set the plugin path variable for use later in the module. if $plugindir == undef { $_plugindir = "${homedir}/plugins" } else { $_plugindir = $plugindir } # Can only enable SSL if security_plugin specified if $ssl or ($system_key != undef) { if $security_plugin == undef or ! ($security_plugin in ['shield', 'x-pack']) { fail("\"${security_plugin}\" is not a valid security_plugin parameter value") } } # Should we restart Elasticsearch on config change? $_notify_service = $elasticsearch::restart_config_change ? { true => Service[$elasticsearch::service_name], false => undef, } #### Manage actions contain elasticsearch::package contain elasticsearch::config contain elasticsearch::service create_resources('elasticsearch::index', $::elasticsearch::indices) create_resources('elasticsearch::pipeline', $::elasticsearch::pipelines) create_resources('elasticsearch::plugin', $::elasticsearch::plugins) create_resources('elasticsearch::role', $::elasticsearch::roles) create_resources('elasticsearch::script', $::elasticsearch::scripts) create_resources('elasticsearch::snapshot_repository', $::elasticsearch::snapshot_repositories) create_resources('elasticsearch::template', $::elasticsearch::templates) create_resources('elasticsearch::user', $::elasticsearch::users) if ($manage_repo == true) { if ($repo_stage == false) { # Use normal relationship ordering contain elastic_stack::repo Class['elastic_stack::repo'] -> Class['elasticsearch::package'] } else { # Use staging for ordering if !(defined(Stage[$repo_stage])) { stage { $repo_stage: before => Stage['main'] } } include elastic_stack::repo Class<|title == 'elastic_stack::repo'|>{ stage => $repo_stage, } } } if ($license != undef) { contain elasticsearch::license } #### Manage relationships # # Note that many of these overly verbose declarations work around # https://tickets.puppetlabs.com/browse/PUP-1410 # which means clean arrow order chaining won't work if someone, say, # doesn't declare any plugins. # # forgive me for what you're about to see if defined(Class['java']) { Class['java'] -> Class['elasticsearch::config'] } if $ensure == 'present' { # Installation, configuration and service Class['elasticsearch::package'] -> Class['elasticsearch::config'] ~> Class['elasticsearch::service'] # Top-level ordering bindings for resources. Class['elasticsearch::config'] -> Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> Elasticsearch::Plugin <| ensure == 'absent' |> -> Class['elasticsearch::config'] # Class['elasticsearch::config'] # -> Elasticsearch::User <| ensure == 'present' |> # Elasticsearch::User <| ensure == 'absent' |> # -> Class['elasticsearch::config'] # Class['elasticsearch::config'] # -> Elasticsearch::Role <| |> Class['elasticsearch::config'] -> Elasticsearch::Template <| |> Class['elasticsearch::config'] -> Elasticsearch::Pipeline <| |> Class['elasticsearch::config'] -> Elasticsearch::Index <| |> Class['elasticsearch::config'] -> Elasticsearch::Snapshot_repository <| |> } else { # Absent; remove configuration before the package. Class['elasticsearch::config'] -> Class['elasticsearch::package'] # Top-level ordering bindings for resources. Elasticsearch::Plugin <| |> -> Class['elasticsearch::config'] Elasticsearch::User <| |> -> Class['elasticsearch::config'] Elasticsearch::Role <| |> -> Class['elasticsearch::config'] Elasticsearch::Template <| |> -> Class['elasticsearch::config'] Elasticsearch::Pipeline <| |> -> Class['elasticsearch::config'] Elasticsearch::Index <| |> -> Class['elasticsearch::config'] Elasticsearch::Snapshot_repository <| |> -> Class['elasticsearch::config'] } # Install plugins before managing users/roles Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> -> Elasticsearch::User <| |> Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> -> Elasticsearch::Role <| |> # Remove plugins after managing users/roles Elasticsearch::User <| |> -> Elasticsearch::Plugin <| ensure == 'absent' |> Elasticsearch::Role <| |> -> Elasticsearch::Plugin <| ensure == 'absent' |> # Ensure roles are defined before managing users that reference roles Elasticsearch::Role <| |> -> Elasticsearch::User <| ensure == 'present' |> # Ensure users are removed before referenced roles are managed Elasticsearch::User <| ensure == 'absent' |> -> Elasticsearch::Role <| |> # Ensure users and roles are managed before calling out to REST resources Elasticsearch::Role <| |> -> Elasticsearch::Template <| |> Elasticsearch::User <| |> -> Elasticsearch::Template <| |> Elasticsearch::Role <| |> -> Elasticsearch::Pipeline <| |> Elasticsearch::User <| |> -> Elasticsearch::Pipeline <| |> Elasticsearch::Role <| |> -> Elasticsearch::Index <| |> Elasticsearch::User <| |> -> Elasticsearch::Index <| |> Elasticsearch::Role <| |> -> Elasticsearch::Snapshot_repository <| |> Elasticsearch::User <| |> -> Elasticsearch::Snapshot_repository <| |> # Ensure that any command-line based user changes are performed before the # file is modified Elasticsearch_user <| |> -> Elasticsearch_user_file <| |> } diff --git a/spec/acceptance/tests/acceptance_spec.rb b/spec/acceptance/tests/acceptance_spec.rb index c511149..cadae84 100644 --- a/spec/acceptance/tests/acceptance_spec.rb +++ b/spec/acceptance/tests/acceptance_spec.rb @@ -1,92 +1,92 @@ require 'spec_helper_acceptance' require 'helpers/acceptance/tests/basic_shared_examples.rb' require 'helpers/acceptance/tests/template_shared_examples.rb' require 'helpers/acceptance/tests/removal_shared_examples.rb' require 'helpers/acceptance/tests/pipeline_shared_examples.rb' require 'helpers/acceptance/tests/plugin_shared_examples.rb' require 'helpers/acceptance/tests/plugin_upgrade_shared_examples.rb' require 'helpers/acceptance/tests/snapshot_repository_shared_examples.rb' require 'helpers/acceptance/tests/datadir_shared_examples.rb' require 'helpers/acceptance/tests/package_url_shared_examples.rb' require 'helpers/acceptance/tests/hiera_shared_examples.rb' require 'helpers/acceptance/tests/usergroup_shared_examples.rb' require 'helpers/acceptance/tests/security_shared_examples.rb' describe "elasticsearch v#{v[:elasticsearch_full_version]} class" do es_config = { 'http.port' => 9200, 'node.name' => 'elasticsearch01' } let(:elastic_repo) { not v[:is_snapshot] } let(:manifest) do package = if not v[:is_snapshot] <<-MANIFEST # Hard version set here due to plugin incompatibilities. version => '#{v[:elasticsearch_full_version]}', MANIFEST else <<-MANIFEST manage_repo => false, package_url => '#{v[:snapshot_package]}', MANIFEST end <<-MANIFEST api_timeout => 60, config => { 'cluster.name' => '#{v[:cluster_name]}', 'http.bind_host' => '0.0.0.0', #{es_config.map { |k, v| " '#{k}' => '#{v}'," }.join("\n")} }, jvm_options => [ '-Xms128m', '-Xmx128m', ], oss => #{v[:oss]}, #{package} MANIFEST end context 'testing with' do describe 'simple config' do include_examples('basic acceptance tests', es_config) end include_examples('module removal', es_config) end include_examples('template operations', es_config, v[:template]) include_examples('pipeline operations', es_config, v[:pipeline]) include_examples('plugin acceptance tests', es_config, v[:elasticsearch_plugins]) unless v[:elasticsearch_plugins].empty? # # Only pre-5.x versions supported versions differing from core ES # if semver(v[:elasticsearch_full_version]) < semver('5.0.0') # include_examples( # 'plugin upgrade acceptance tests', # :name => 'kopf', # :initial => '2.0.1', # :upgraded => '2.1.2', # :repository => 'lmenezes/elasticsearch' # ) # end include_examples('snapshot repository acceptance tests') include_examples('datadir acceptance tests', es_config) # # Skip this for snapshot testing, as we only have package files anyway. # include_examples 'package_url acceptance tests' unless v[:is_snapshot] # include_examples 'hiera acceptance tests', v[:elasticsearch_plugins] # include_examples 'user/group acceptance tests' - # # Security-related tests (shield/x-pack). - # # - # # Skip OSS-only distributions since they do not bundle x-pack, and skip - # # snapshots since we they don't recognize prod licenses. - # include_examples 'security acceptance tests', instances unless v[:oss] or v[:is_snapshot] + # Security-related tests (shield/x-pack). + # + # Skip OSS-only distributions since they do not bundle x-pack, and skip + # snapshots since we they don't recognize prod licenses. + include_examples('security acceptance tests', es_config) unless v[:oss] or v[:is_snapshot] end diff --git a/spec/helpers/acceptance/tests/manifest_shared_examples.rb b/spec/helpers/acceptance/tests/manifest_shared_examples.rb index 3fdfaf0..eba3863 100644 --- a/spec/helpers/acceptance/tests/manifest_shared_examples.rb +++ b/spec/helpers/acceptance/tests/manifest_shared_examples.rb @@ -1,37 +1,38 @@ shared_examples 'manifest application' do |idempotency_check = true| - context '1-node manifest' do + context 'manifest' do let(:applied_manifest) do repo = if elastic_repo <<-MANIFEST class { 'elastic_stack::repo': oss => #{v[:oss]}, version => #{v[:elasticsearch_major_version]}, } MANIFEST else '' end <<-MANIFEST #{repo} class { 'elasticsearch' : #{manifest} #{defined?(manifest_class_parameters) && manifest_class_parameters} } #{defined?(extra_manifest) && extra_manifest} MANIFEST end it 'applies cleanly' do apply_manifest(applied_manifest, :catch_failures => true, :debug => v[:puppet_debug]) end + # binding.pry if idempotency_check it 'is idempotent', :logs_on_failure do apply_manifest(applied_manifest, :catch_changes => true, :debug => v[:puppet_debug]) end end end end diff --git a/spec/helpers/acceptance/tests/security_shared_examples.rb b/spec/helpers/acceptance/tests/security_shared_examples.rb index 8d2ad45..fcc2ecf 100644 --- a/spec/helpers/acceptance/tests/security_shared_examples.rb +++ b/spec/helpers/acceptance/tests/security_shared_examples.rb @@ -1,272 +1,245 @@ require 'json' require 'spec_utilities' require 'helpers/acceptance/tests/manifest_shared_examples' -shared_examples 'security plugin manifest' do |instances, credentials| +shared_examples 'security plugin manifest' do |es_config, credentials| let(:extra_manifest) do - instance_plugins = - <<-MANIFEST - Elasticsearch::Plugin { instances => #{instances.keys} } - MANIFEST - users = credentials.map do |username, meta| <<-USER #{meta[:changed] ? "notify { 'password change for #{username}' : } ~>" : ''} elasticsearch::user { '#{username}': password => '#{meta[:hash] ? meta[:hash] : meta[:plaintext]}', roles => #{meta[:roles].reduce({}) { |a, e| a.merge(e) }.keys}, } USER end.join("\n") roles = credentials.values.reduce({}) do |sum, user_metadata| # Collect all roles across users sum.merge user_metadata end[:roles].reduce({}) do |all_roles, role| all_roles.merge role end.reject do |_role, permissions| permissions.empty? end.map do |role, rights| <<-ROLE elasticsearch::role { '#{role}': privileges => #{rights} } ROLE end.join("\n") <<-MANIFEST #{security_plugins} - #{instance_plugins} - #{users} #{roles} - - #{ssl_params} MANIFEST end include_examples( 'manifest application', - instances, not(credentials.values.map { |p| p[:changed] }.any?) ) end -shared_examples 'secured request' do |test_desc, instances, path, http_test, expected, user = nil, pass = nil| - instances.each_value do |i| - describe port(i['config']['http.port']) do - it 'open', :with_retries do - should be_listening - end +shared_examples 'secured request' do |test_desc, es_config, path, http_test, expected, user = nil, pass = nil| + es_port = es_config['http.port'] + describe port(es_port) do + it 'open', :with_retries do + should be_listening end + end - describe server :container do - describe http( - "https://localhost:#{i['config']['http.port']}#{path}", - { - :ssl => { :verify => false } - }.merge((user and pass) ? { :basic_auth => [user, pass] } : {}) - ) do - it test_desc, :with_retries do - expect(http_test.call(response)).to eq(expected) - end + describe server :container do + describe http( + "https://localhost:#{es_port}#{path}", + { + :ssl => { :verify => false } + }.merge((user and pass) ? { :basic_auth => [user, pass] } : {}) + ) do + it test_desc, :with_retries do + expect(http_test.call(response)).to eq(expected) end end end end -shared_examples 'security acceptance tests' do |default_instances| +shared_examples 'security acceptance tests' do |es_config| describe 'security plugin operations', :if => vault_available?, :then_purge => true, :with_license => true, :with_certificates => true do superuser_role = v[:elasticsearch_major_version] > 2 ? 'superuser' : 'admin' rand_string = lambda { [*('a'..'z')].sample(8).join } admin_user = rand_string.call admin_password = rand_string.call admin = { admin_user => { :plaintext => admin_password, :roles => [{ superuser_role => [] }] } } let(:manifest_class_parameters) do <<-MANIFEST - license => file('#{v[:elasticsearch_license_path]}'), - restart_on_change => true, - security_plugin => '#{v[:elasticsearch_major_version] > 2 ? 'x-pack' : 'shield'}', api_basic_auth_password => '#{admin_password}', api_basic_auth_username => '#{admin_user}', api_ca_file => '#{@tls[:ca][:cert][:path]}', api_protocol => 'https', + ca_certificate => '#{@tls[:ca][:cert][:path]}', + certificate => '#{@tls[:clients].first[:cert][:path]}', + keystore_password => '#{@keystore_password}', + license => file('#{v[:elasticsearch_license_path]}'), + private_key => '#{@tls[:clients].first[:key][:path]}', + restart_on_change => true, + security_plugin => 'x-pack', + ssl => true, validate_tls => true, MANIFEST end let(:security_plugins) do - if v[:elasticsearch_major_version] <= 2 - <<-MANIFEST - elasticsearch::plugin { 'elasticsearch/license/latest' : } - elasticsearch::plugin { 'elasticsearch/shield/latest' : } - MANIFEST - elsif semver(v[:elasticsearch_full_version].split('-').first) < semver('6.3.0') + if semver(v[:elasticsearch_full_version].split('-').first) < semver('6.3.0') <<-MANIFEST elasticsearch::plugin { 'x-pack' : } MANIFEST else '' end end describe 'over tls' do user_one = rand_string.call user_two = rand_string.call user_one_pw = rand_string.call user_two_pw = rand_string.call - context "instance #{default_instances.first.first}" do - instance_name = default_instances.keys.first - instance = { instance_name => default_instances[instance_name].merge('ssl' => true) } - - let(:ssl_params) do - <<-MANIFEST - Elasticsearch::Instance['#{instance_name}'] { - ca_certificate => '#{@tls[:ca][:cert][:path]}', - certificate => '#{@tls[:clients].first[:cert][:path]}', - private_key => '#{@tls[:clients].first[:key][:path]}', - keystore_password => '#{@keystore_password}', - } - MANIFEST - end + describe 'user authentication' do + username_passwords = { + user_one => { :plaintext => user_one_pw, :roles => [{ superuser_role => [] }] }, + user_two => { :plaintext => user_two_pw, :roles => [{ superuser_role => [] }] } + }.merge(admin) + username_passwords[user_two][:hash] = bcrypt(username_passwords[user_two][:plaintext]) - describe 'user authentication' do - username_passwords = { - user_one => { :plaintext => user_one_pw, :roles => [{ superuser_role => [] }] }, - user_two => { :plaintext => user_two_pw, :roles => [{ superuser_role => [] }] } - }.merge(admin) - username_passwords[user_two][:hash] = bcrypt(username_passwords[user_two][:plaintext]) - - include_examples('security plugin manifest', instance, username_passwords) - include_examples( - 'secured request', 'denies unauthorized access', - instance, '/_cluster/health', - lambda { |r| r.status }, 401 - ) - include_examples( - 'secured request', "permits user #{user_one} access", - instance, '/_cluster/health', - lambda { |r| r.status }, 200, - user_one, user_one_pw - ) - include_examples( - 'secured request', "permits user #{user_two} access", - instance, '/_cluster/health', - lambda { |r| r.status }, 200, - user_two, user_two_pw - ) - end - - describe 'changing passwords' do - new_password = rand_string.call - username_passwords = { - user_one => { - :plaintext => new_password, - :changed => true, - :roles => [{ superuser_role => [] }] - } - } - - include_examples('security plugin manifest', instance, username_passwords) - include_examples( - 'secured request', 'denies unauthorized access', instance, '/_cluster/health', - lambda { |r| r.status }, 401 - ) - include_examples( - 'secured request', "permits user #{user_two} access with new password", - instance, '/_cluster/health', - lambda { |r| r.status }, 200, - user_one, new_password - ) - end + include_examples('security plugin manifest', es_config, username_passwords) + include_examples( + 'secured request', 'denies unauthorized access', + es_config, '/_cluster/health', + lambda { |r| r.status }, 401 + ) + include_examples( + 'secured request', "permits user #{user_one} access", + es_config, '/_cluster/health', + lambda { |r| r.status }, 200, + user_one, user_one_pw + ) + include_examples( + 'secured request', "permits user #{user_two} access", + es_config, '/_cluster/health', + lambda { |r| r.status }, 200, + user_two, user_two_pw + ) + end - describe 'roles' do - password = rand_string.call - username = rand_string.call - user = { - username => { - :plaintext => password, - :roles => [{ - rand_string.call => { - 'cluster' => [ - 'cluster:monitor/health' - ] - } - }] - } + describe 'changing passwords' do + new_password = rand_string.call + username_passwords = { + user_one => { + :plaintext => new_password, + :changed => true, + :roles => [{ superuser_role => [] }] } + } - include_examples('security plugin manifest', instance, user) - include_examples( - 'secured request', 'denies unauthorized access', - instance, '/_snapshot', - lambda { |r| r.status }, 403, - username, password - ) - include_examples( - 'secured request', 'permits authorized access', - instance, '/_cluster/health', - lambda { |r| r.status }, 200, - username, password - ) - end + include_examples('security plugin manifest', es_config, username_passwords) + include_examples( + 'secured request', 'denies unauthorized access', es_config, '/_cluster/health', + lambda { |r| r.status }, 401 + ) + include_examples( + 'secured request', "permits user #{user_two} access with new password", + es_config, '/_cluster/health', + lambda { |r| r.status }, 200, + user_one, new_password + ) end - describe 'with two instances' do - let(:ssl_params) do - @tls[:clients].each_with_index.map do |cert, i| - format(%( - Elasticsearch::Instance['es-%02d'] { - ca_certificate => '#{@tls[:ca][:cert][:path]}', - certificate => '#{cert[:cert][:path]}', - private_key => '#{cert[:key][:path]}', - keystore_password => '#{@keystore_password}', - } - ), i + 1) - end.join("\n") - end - - ssl_instances = default_instances.map do |instance, meta| - new_config = if v[:elasticsearch_major_version] > 2 - { 'xpack.ssl.verification_mode' => 'none' } - else - { 'shield.ssl.hostname_verification' => false } - end - [ - instance, - { - 'config' => meta['config'].merge(new_config).merge( - 'discovery.zen.minimum_master_nodes' => default_instances.keys.size - ), - 'ssl' => true - } - ] - end.to_h - - username = rand_string.call + describe 'roles' do password = rand_string.call - - include_examples( - 'security plugin manifest', - ssl_instances, + username = rand_string.call + user = { username => { :plaintext => password, - :roles => [{ superuser_role => [] }] + :roles => [{ + rand_string.call => { + 'cluster' => [ + 'cluster:monitor/health' + ] + } + }] } - ) + } + include_examples('security plugin manifest', es_config, user) include_examples( - 'secured request', 'clusters between two nodes', - ssl_instances, '/_nodes', - lambda { |r| JSON.parse(r.body)['nodes'].size }, 2, + 'secured request', 'denies unauthorized access', + es_config, '/_snapshot', + lambda { |r| r.status }, 403, + username, password + ) + include_examples( + 'secured request', 'permits authorized access', + es_config, '/_cluster/health', + lambda { |r| r.status }, 200, username, password ) end end + + # describe 'with two instances' do + # let(:ssl_params) do + # @tls[:clients].each_with_index.map do |cert, i| + # format(%( + # Elasticsearch::Instance['es-%02d'] { + # ca_certificate => '#{@tls[:ca][:cert][:path]}', + # certificate => '#{cert[:cert][:path]}', + # private_key => '#{cert[:key][:path]}', + # keystore_password => '#{@keystore_password}', + # } + # ), i + 1) + # end.join("\n") + # end + + # ssl_instances = default_instances.map do |instance, meta| + # new_config = if v[:elasticsearch_major_version] > 2 + # { 'xpack.ssl.verification_mode' => 'none' } + # else + # { 'shield.ssl.hostname_verification' => false } + # end + # [ + # instance, + # { + # 'config' => meta['config'].merge(new_config).merge( + # 'discovery.zen.minimum_master_nodes' => default_instances.keys.size + # ), + # 'ssl' => true + # } + # ] + # end.to_h + + # username = rand_string.call + # password = rand_string.call + + # include_examples( + # 'security plugin manifest', + # ssl_instances, + # username => { + # :plaintext => password, + # :roles => [{ superuser_role => [] }] + # } + # ) + + # include_examples( + # 'secured request', 'clusters between two nodes', + # ssl_instances, '/_nodes', + # lambda { |r| JSON.parse(r.body)['nodes'].size }, 2, + # username, password + # ) + # end end end