diff --git a/data/common.yaml b/data/common.yaml index bb17a21..1c6c29c 100644 --- a/data/common.yaml +++ b/data/common.yaml @@ -1,67 +1,72 @@ --- elasticsearch::ensure: present elasticsearch::api_basic_auth_password: ~ elasticsearch::api_basic_auth_username: ~ elasticsearch::api_ca_file: ~ elasticsearch::api_ca_path: ~ elasticsearch::api_host: localhost elasticsearch::api_port: 9200 elasticsearch::api_protocol: http elasticsearch::api_timeout: 10 elasticsearch::autoupgrade: false elasticsearch::config: {} elasticsearch::configdir: /etc/elasticsearch elasticsearch::configdir_recurselimit: 2 elasticsearch::daily_rolling_date_pattern: | "'.'yyyy-MM-dd" elasticsearch::datadir_instance_directories: true elasticsearch::default_logging_level: 'INFO' elasticsearch::defaults_location: ~ +elasticsearch::deprecation_logging: false +elasticsearch::deprecation_logging_level: 'DEBUG' elasticsearch::download_tool: ~ elasticsearch::download_tool_insecure: ~ elasticsearch::download_tool_verify_certificates: true elasticsearch::file_rolling_type: dailyRollingFile elasticsearch::indices: {} elasticsearch::init_defaults: {} elasticsearch::init_defaults_file: ~ elasticsearch::init_template: "%{module_name}/etc/init.d/elasticsearch.systemd.erb" elasticsearch::instances: {} elasticsearch::jvm_options: [] elasticsearch::license: ~ elasticsearch::logdir: /var/log/elasticsearch elasticsearch::logging_config: {} elasticsearch::logging_file: ~ +elasticsearch::logging_level: 'INFO' elasticsearch::logging_template: ~ elasticsearch::manage_repo: true elasticsearch::oss: false elasticsearch::package_dl_timeout: 600 elasticsearch::package_name: elasticsearch elasticsearch::package_provider: package elasticsearch::package_url: ~ elasticsearch::pid_dir: /var/run/elasticsearch elasticsearch::pipelines: {} elasticsearch::plugindir: ~ elasticsearch::plugins: {} elasticsearch::proxy_url: ~ elasticsearch::purge_configdir: false elasticsearch::purge_package_dir: false elasticsearch::purge_secrets: false elasticsearch::repo_stage: false elasticsearch::restart_on_change: false elasticsearch::roles: {} elasticsearch::rolling_file_max_backup_index: 1 elasticsearch::rolling_file_max_file_size: 10MB elasticsearch::scripts: {} elasticsearch::secrets: ~ elasticsearch::security_logging_content: ~ elasticsearch::security_logging_source: ~ elasticsearch::security_plugin: ~ +elasticsearch::service_name: elasticsearch elasticsearch::service_provider: systemd elasticsearch::snapshot_repositories: {} +elasticsearch::ssl: false elasticsearch::status: enabled elasticsearch::system_key: ~ elasticsearch::systemd_service_path: /lib/systemd/system elasticsearch::templates: {} elasticsearch::users: {} elasticsearch::validate_tls: true elasticsearch::version: false diff --git a/manifests/config.pp b/manifests/config.pp index aaae820..fd72a3a 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -1,175 +1,253 @@ # This class exists to coordinate all configuration related actions, # functionality and logical units in a central place. # # It is not intended to be used directly by external resources like node # definitions or other modules. # # @example importing this class into other classes to use its functionality: # class { 'elasticsearch::config': } # # @author Richard Pijnenburg # @author Tyler Langlois +# @author Gavin Williams # class elasticsearch::config { #### Configuration Exec { path => [ '/bin', '/usr/bin', '/usr/local/bin' ], cwd => '/', } if ( $elasticsearch::ensure == 'present' ) { file { + $elasticsearch::homedir: + ensure => 'directory', + group => $elasticsearch::elasticsearch_group, + owner => $elasticsearch::elasticsearch_user; $elasticsearch::configdir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => 'root', mode => '2750'; $elasticsearch::datadir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user; $elasticsearch::logdir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => '0750'; $elasticsearch::_plugindir: ensure => 'directory', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => 'o+Xr'; "${elasticsearch::homedir}/lib": ensure => 'directory', group => '0', owner => 'root', recurse => true; - $elasticsearch::homedir: - ensure => 'directory', - group => $elasticsearch::elasticsearch_group, - owner => $elasticsearch::elasticsearch_user; - "${elasticsearch::homedir}/templates_import": - ensure => 'directory', - group => $elasticsearch::elasticsearch_group, - owner => $elasticsearch::elasticsearch_user, - mode => '0755'; - "${elasticsearch::homedir}/scripts": - ensure => 'directory', - group => $elasticsearch::elasticsearch_group, - owner => $elasticsearch::elasticsearch_user, - mode => '0755'; - "${elasticsearch::configdir}/scripts": - ensure => 'directory', - source => "${elasticsearch::homedir}/scripts", - mode => '0755', - recurse => 'remote', - owner => $elasticsearch::elasticsearch_user, - group => $elasticsearch::elasticsearch_group; - '/etc/elasticsearch/elasticsearch.yml': - ensure => 'absent'; - '/etc/elasticsearch/jvm.options': - ensure => 'absent'; - '/etc/elasticsearch/logging.yml': - ensure => 'absent'; - '/etc/elasticsearch/log4j2.properties': - ensure => 'absent'; } if $elasticsearch::pid_dir { file { $elasticsearch::pid_dir: ensure => 'directory', group => undef, owner => $elasticsearch::elasticsearch_user, recurse => true, } if ($elasticsearch::service_provider == 'systemd') { $group = $elasticsearch::elasticsearch_group $user = $elasticsearch::elasticsearch_user $pid_dir = $elasticsearch::pid_dir file { '/usr/lib/tmpfiles.d/elasticsearch.conf': ensure => 'file', content => template("${module_name}/usr/lib/tmpfiles.d/elasticsearch.conf.erb"), group => '0', owner => 'root', } } } - if ($elasticsearch::service_provider == 'systemd') { - # Mask default unit (from package) - service { 'elasticsearch' : - ensure => false, - enable => 'mask', - provider => $elasticsearch::service_provider, - } - } else { - service { 'elasticsearch': - ensure => false, - enable => false, - } - } - if $elasticsearch::defaults_location { augeas { "${elasticsearch::defaults_location}/elasticsearch": incl => "${elasticsearch::defaults_location}/elasticsearch", lens => 'Shellvars.lns', changes => [ 'rm CONF_FILE', 'rm CONF_DIR', 'rm ES_PATH_CONF', ], } file { "${elasticsearch::defaults_location}/elasticsearch": ensure => 'file', group => $elasticsearch::elasticsearch_group, owner => $elasticsearch::elasticsearch_user, mode => '0640'; } } if $::elasticsearch::security_plugin != undef and ($::elasticsearch::security_plugin in ['shield', 'x-pack']) { file { "${::elasticsearch::configdir}/${::elasticsearch::security_plugin}" : ensure => 'directory', owner => 'root', group => $elasticsearch::elasticsearch_group, mode => '0750', } } # Define logging config file for the in-use security plugin if $::elasticsearch::security_logging_content != undef or $::elasticsearch::security_logging_source != undef { if $::elasticsearch::security_plugin == undef or ! ($::elasticsearch::security_plugin in ['shield', 'x-pack']) { fail("\"${::elasticsearch::security_plugin}\" is not a valid security_plugin parameter value") } $_security_logging_file = $::elasticsearch::security_plugin ? { 'shield' => 'logging.yml', default => 'log4j2.properties' } file { "/etc/elasticsearch/${::elasticsearch::security_plugin}/${_security_logging_file}" : content => $::elasticsearch::security_logging_content, source => $::elasticsearch::security_logging_source, } } + # Generate config file + $_config = deep_implode($elasticsearch::config) + + # Generate SSL config + if $elasticsearch::ssl { + if ($elasticsearch::keystore_password == undef) { + fail('keystore_password required') + } + + if ($elasticsearch::keystore_path == undef) { + $_keystore_path = "${elasticsearch::configdir}/${elasticsearch::security_plugin}/${name}.ks" + } else { + $_keystore_path = $elasticsearch::keystore_path + } + + if $elasticsearch::security_plugin == 'shield' { + $_tls_config = { + 'shield.transport.ssl' => true, + 'shield.http.ssl' => true, + 'shield.ssl.keystore.path' => $_keystore_path, + 'shield.ssl.keystore.password' => $elasticsearch::keystore_password, + } + } elsif $elasticsearch::security_plugin == 'x-pack' { + $_tls_config = { + 'xpack.security.transport.ssl.enabled' => true, + 'xpack.security.http.ssl.enabled' => true, + 'xpack.ssl.keystore.path' => $_keystore_path, + 'xpack.ssl.keystore.password' => $elasticsearch::keystore_password, + } + } + + # Trust CA Certificate + java_ks { 'elasticsearch_ca': + ensure => 'latest', + certificate => $elasticsearch::ca_certificate, + target => $_keystore_path, + password => $elasticsearch::keystore_password, + trustcacerts => true, + } + + # Load node certificate and private key + java_ks { 'elasticsearch_node': + ensure => 'latest', + certificate => $elasticsearch::certificate, + private_key => $elasticsearch::private_key, + target => $_keystore_path, + password => $elasticsearch::keystore_password, + } + } else { + $_tls_config = {} + } + + # Logging file or hash + if ($::elasticsearch::logging_file != undef) { + $_log4j_content = undef + } else { + if ($::elasticsearch::logging_template != undef ) { + $_log4j_content = template($::elasticsearch::logging_template) + } else { + $_log4j_content = template("${module_name}/etc/elasticsearch/log4j2.properties.erb") + } + $_logging_source = undef + } + file { + "${::elasticsearch::configdir}/log4j2.properties": + ensure => file, + content => $_log4j_content, + source => $_logging_source, + mode => '0644', + notify => $::elasticsearch::_notify_service, + require => Class['elasticsearch::package'], + before => Class['elasticsearch::service'] + } + + # Generate Elasticsearch config + $_es_config = merge( + $::elasticsearch::config, + { 'path.data' => $::elasticsearch::datadir }, + { 'path.logs' => $::elasticsearch::logdir }, + $_tls_config + ) + + datacat_fragment { "main_config_${name}": + target => "${::elasticsearch::configdir}/elasticsearch.yml", + data => $_es_config, + } + + datacat { "${::elasticsearch::configdir}/elasticsearch.yml": + template => "${module_name}/etc/elasticsearch/elasticsearch.yml.erb", + notify => $::elasticsearch::_notify_service, + require => Class['elasticsearch::package'], + owner => $::elasticsearch::elasticsearch_user, + group => $::elasticsearch::elasticsearch_group, + mode => '0440', + } + + if $::elasticsearch::system_key != undef { + file { "${::elasticsearch::configdir}/${::elasticsearch::security_plugin}/system_key": + ensure => 'file', + source => $::elasticsearch::system_key, + mode => '0400', + before => Elasticsearch::Service[$::elasticsearch::service_name], + require => File["${::elasticsearch::configdir}/${::elasticsearch::security_plugin}"], + } + } + + # Add secrets to keystore + if $::elasticsearch::secrets != undef { + elasticsearch_keystore { 'elasticsearch_secrets': + configdir => $::elasticsearch::configdir, + purge => $::elasticsearch::purge_secrets, + settings => $::elasticsearch::secrets, + notify => $::elaticsearch::_notify_service, + } + } + } elsif ( $elasticsearch::ensure == 'absent' ) { file { $elasticsearch::_plugindir: ensure => 'absent', force => true, backup => false, } file { "${elasticsearch::configdir}/jvm.options": ensure => 'absent', } } } diff --git a/manifests/init.pp b/manifests/init.pp index 0e4bd00..52ed330 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,597 +1,601 @@ # Top-level Elasticsearch class which may manage installation of the # Elasticsearch package, package repository, and other # global options and parameters. # # @summary Manages the installation of Elasticsearch and related options. # # @example install Elasticsearch # class { 'elasticsearch': } # # @example removal and decommissioning # class { 'elasticsearch': # ensure => 'absent', # } # # @example install everything but disable service(s) afterwards # class { 'elasticsearch': # status => 'disabled', # } # # @param ensure # Controls if the managed resources shall be `present` or `absent`. # If set to `absent`, the managed software packages will be uninstalled, and # any traces of the packages will be purged as well as possible, possibly # including existing configuration files. # System modifications (if any) will be reverted as well as possible (e.g. # removal of created users, services, changed log settings, and so on). # This is a destructive parameter and should be used with care. # # @param api_basic_auth_password # Defines the default REST basic auth password for API authentication. # # @param api_basic_auth_username # Defines the default REST basic auth username for API authentication. # # @param api_ca_file # Path to a CA file which will be used to validate server certs when # communicating with the Elasticsearch API over HTTPS. # # @param api_ca_path # Path to a directory with CA files which will be used to validate server # certs when communicating with the Elasticsearch API over HTTPS. # # @param api_host # Default host to use when accessing Elasticsearch APIs. # # @param api_port # Default port to use when accessing Elasticsearch APIs. # # @param api_protocol # Default protocol to use when accessing Elasticsearch APIs. # # @param api_timeout # Default timeout (in seconds) to use when accessing Elasticsearch APIs. # # @param autoupgrade # If set to `true`, any managed package will be upgraded on each Puppet run # when the package provider is able to find a newer version than the present # one. The exact behavior is provider dependent (see # {package, "upgradeable"}[http://j.mp/xbxmNP] in the Puppet documentation). # +# @param ca_certificate +# Path to the trusted CA certificate to add to this node's java keystore. +# +# @param certificate +# Path to the certificate for this node signed by the CA listed in +# ca_certificate. +# # @param config # Elasticsearch configuration hash. # # @param configdir # Directory containing the elasticsearch configuration. # Use this setting if your packages deviate from the norm (`/etc/elasticsearch`) # # @param configdir_recurselimit # Dictates how deeply the file copy recursion logic should descend when # copying files from the `configdir` to instance `configdir`s. # # @param daily_rolling_date_pattern # File pattern for the file appender log when file_rolling_type is 'dailyRollingFile'. # # @param datadir # Allows you to set the data directory of Elasticsearch. # # @param datadir_instance_directories # Control whether individual directories for instances will be created within # each instance's data directory. # # @param default_logging_level # Default logging level for Elasticsearch. # # @param defaults_location # Absolute path to directory containing init defaults file. # +# @param deprecation_logging +# Wheter to enable deprecation logging. If enabled, deprecation logs will be +# saved to ${cluster.name}_deprecation.log in the elastic search log folder. +# +# @param deprecation_logging_level +# Default deprecation logging level for Elasticsearch. +# # @param download_tool # Command-line invocation with which to retrieve an optional package_url. # # @param download_tool_insecure # Command-line invocation with which to retrieve an optional package_url when # certificate verification should be ignored. # # @param download_tool_verify_certificates # Whether or not to verify SSL/TLS certificates when retrieving package files # using a download tool instead of a package management provider. # # @param elasticsearch_group # The group Elasticsearch should run as. This also sets file group # permissions. # # @param elasticsearch_user # The user Elasticsearch should run as. This also sets file ownership. # # @param file_rolling_type # Configuration for the file appender rotation. It can be 'dailyRollingFile', # 'rollingFile' or 'file'. The first rotates by name, the second one by size # or third don't rotate automatically. # # @param homedir # Directory where the elasticsearch installation's files are kept (plugins, etc.) # # @param indices # Define indices via a hash. This is mainly used with Hiera's auto binding. # # @param init_defaults # Defaults file content in hash representation. # # @param init_defaults_file # Defaults file as puppet resource. # # @param init_template # Service file as a template. # -# @param instances -# Define instances via a hash. This is mainly used with Hiera's auto binding. -# # @param jvm_options # Array of options to set in jvm_options. # # @param license # Optional Elasticsearch license in hash or string form. # # @param logdir # Directory that will be used for Elasticsearch logging. # # @param logging_config # Representation of information to be included in the logging.yml file. # # @param logging_file # Instead of a hash, you may supply a `puppet://` file source for the # logging.yml file. # +# @param logging_level +# Default logging level for Elasticsearch. +# # @param logging_template # Use a custom logging template - just supply the relative path, i.e. # `$module/elasticsearch/logging.yml.erb` # # @param manage_repo # Enable repo management by enabling official Elastic repositories. # # @param oss # Whether to use the purely open source Elasticsearch package distribution. # # @param package_dir # Directory where packages are downloaded to. # # @param package_dl_timeout # For http, https, and ftp downloads, you may set how long the exec resource # may take. # # @param package_name # Name Of the package to install. # # @param package_provider # Method to install the packages, currently only `package` is supported. # # @param package_url # URL of the package to download. # This can be an http, https, or ftp resource for remote packages, or a # `puppet://` resource or `file:/` for local packages # # @param pid_dir # Directory where the elasticsearch process should write out its PID. # # @param pipelines # Define pipelines via a hash. This is mainly used with Hiera's auto binding. # # @param plugindir # Directory containing elasticsearch plugins. # Use this setting if your packages deviate from the norm (/usr/share/elasticsearch/plugins) # # @param plugins # Define plugins via a hash. This is mainly used with Hiera's auto binding. # # @param proxy_url # For http and https downloads, you may set a proxy server to use. By default, # no proxy is used. # Format: `proto://[user:pass@]server[:port]/` # # @param purge_configdir # Purge the config directory of any unmanaged files. # # @param purge_package_dir # Purge package directory on removal # # @param purge_secrets # Whether or not keys present in the keystore will be removed if they are not # present in the specified secrets hash. # # @param repo_stage # Use stdlib stage setup for managing the repo instead of relationship # ordering. # # @param restart_on_change # Determines if the application should be automatically restarted # whenever the configuration, package, or plugins change. Enabling this # setting will cause Elasticsearch to restart whenever there is cause to # re-read configuration files, load new plugins, or start the service using an # updated/changed executable. This may be undesireable in highly available # environments. If all other restart_* parameters are left unset, the value of # `restart_on_change` is used for all other restart_*_change defaults. # # @param restart_config_change # Determines if the application should be automatically restarted # whenever the configuration changes. This includes the Elasticsearch # configuration file, any service files, and defaults files. # Disabling automatic restarts on config changes may be desired in an # environment where you need to ensure restarts occur in a controlled/rolling # manner rather than during a Puppet run. # # @param restart_package_change # Determines if the application should be automatically restarted # whenever the package (or package version) for Elasticsearch changes. # Disabling automatic restarts on package changes may be desired in an # environment where you need to ensure restarts occur in a controlled/rolling # manner rather than during a Puppet run. # # @param restart_plugin_change # Determines if the application should be automatically restarted whenever # plugins are installed or removed. # Disabling automatic restarts on plugin changes may be desired in an # environment where you need to ensure restarts occur in a controlled/rolling # manner rather than during a Puppet run. # # @param roles # Define roles via a hash. This is mainly used with Hiera's auto binding. # # @param rolling_file_max_backup_index # Max number of logs to store whern file_rolling_type is 'rollingFile' # # @param rolling_file_max_file_size # Max log file size when file_rolling_type is 'rollingFile' # # @param scripts # Define scripts via a hash. This is mainly used with Hiera's auto binding. # # @param secrets # Optional default configuration hash of key/value pairs to store in the # Elasticsearch keystore file. If unset, the keystore is left unmanaged. # # @param security_logging_content # File content for shield/x-pack logging configuration file (will be placed # into logging.yml or log4j2.properties file as appropriate). # # @param security_logging_source # File source for shield/x-pack logging configuration file (will be placed # into logging.yml or log4j2.properties file as appropriate). # # @param security_plugin # Which security plugin will be used to manage users, roles, and # certificates. # # @param service_provider # The service resource type provider to use when managing elasticsearch instances. # # @param snapshot_repositories # Define snapshot repositories via a hash. This is mainly used with Hiera's auto binding. # +# @param ssl +# Whether to manage TLS certificates for Shield. Requires the ca_certificate, +# certificate, private_key and keystore_password parameters to be set. +# # @param status # To define the status of the service. If set to `enabled`, the service will # be run and will be started at boot time. If set to `disabled`, the service # is stopped and will not be started at boot time. If set to `running`, the # service will be run but will not be started at boot time. You may use this # to start a service on the first Puppet run instead of the system startup. # If set to `unmanaged`, the service will not be started at boot time and Puppet # does not care whether the service is running or not. For example, this may # be useful if a cluster management software is used to decide when to start # the service plus assuring it is running on the desired node. # # @param system_key # Source for the Shield/x-pack system key. Valid values are any that are # supported for the file resource `source` parameter. # # @param systemd_service_path # Path to the directory in which to install systemd service units. # # @param templates # Define templates via a hash. This is mainly used with Hiera's auto binding. # # @param users # Define templates via a hash. This is mainly used with Hiera's auto binding. # # @param validate_tls # Enable TLS/SSL validation on API calls. # # @param version # To set the specific version you want to install. # # @author Richard Pijnenburg # @author Tyler Langlois +# @author Gavin Williams # class elasticsearch ( Enum['absent', 'present'] $ensure, Optional[String] $api_basic_auth_password, Optional[String] $api_basic_auth_username, Optional[String] $api_ca_file, Optional[String] $api_ca_path, String $api_host, Integer[0, 65535] $api_port, Enum['http', 'https'] $api_protocol, Integer $api_timeout, Boolean $autoupgrade, Hash $config, Stdlib::Absolutepath $configdir, Integer $configdir_recurselimit, String $daily_rolling_date_pattern, Elasticsearch::Multipath $datadir, Boolean $datadir_instance_directories, - String $default_logging_level, Optional[Stdlib::Absolutepath] $defaults_location, + Boolean $deprecation_logging, + String $deprecation_logging_level, Optional[String] $download_tool, Optional[String] $download_tool_insecure, Boolean $download_tool_verify_certificates, String $elasticsearch_group, String $elasticsearch_user, Enum['dailyRollingFile', 'rollingFile', 'file'] $file_rolling_type, Stdlib::Absolutepath $homedir, Hash $indices, Hash $init_defaults, Optional[String] $init_defaults_file, String $init_template, - Hash $instances, Array[String] $jvm_options, Optional[Variant[String, Hash]] $license, Stdlib::Absolutepath $logdir, Hash $logging_config, Optional[String] $logging_file, + String $logging_level, Optional[String] $logging_template, Boolean $manage_repo, Boolean $oss, Stdlib::Absolutepath $package_dir, Integer $package_dl_timeout, String $package_name, Enum['package'] $package_provider, Optional[String] $package_url, Optional[Stdlib::Absolutepath] $pid_dir, Hash $pipelines, Optional[Stdlib::Absolutepath] $plugindir, Hash $plugins, Optional[Stdlib::HTTPUrl] $proxy_url, Boolean $purge_configdir, Boolean $purge_package_dir, Boolean $purge_secrets, Variant[Boolean, String] $repo_stage, Boolean $restart_on_change, Hash $roles, Integer $rolling_file_max_backup_index, String $rolling_file_max_file_size, Hash $scripts, Optional[Hash] $secrets, Optional[String] $security_logging_content, Optional[String] $security_logging_source, Optional[Enum['shield', 'x-pack']] $security_plugin, + String $service_name, Enum['init', 'openbsd', 'openrc', 'systemd'] $service_provider, Hash $snapshot_repositories, + Boolean $ssl, Elasticsearch::Status $status, Optional[String] $system_key, Stdlib::Absolutepath $systemd_service_path, Hash $templates, Hash $users, Boolean $validate_tls, Variant[String, Boolean] $version, - Boolean $restart_config_change = $restart_on_change, - Boolean $restart_package_change = $restart_on_change, - Boolean $restart_plugin_change = $restart_on_change, + Optional[Stdlib::Absolutepath] $ca_certificate = undef, + Optional[Stdlib::Absolutepath] $certificate = undef, + String $default_logging_level = $logging_level, + Boolean $restart_config_change = $restart_on_change, + Boolean $restart_package_change = $restart_on_change, + Boolean $restart_plugin_change = $restart_on_change, ) { #### Validate parameters if ($package_url != undef and $version != false) { fail('Unable to set the version number when using package_url option.') } if ($version != false) { case $facts['os']['family'] { 'RedHat', 'Linux', 'Suse': { if ($version =~ /.+-\d/) { $pkg_version = $version } else { $pkg_version = "${version}-1" } } default: { $pkg_version = $version } } } # This value serves as an unchanging default for platforms as a default for # init scripts to fallback on. $_datadir_default = $facts['kernel'] ? { 'Linux' => '/var/lib/elasticsearch', 'OpenBSD' => '/var/elasticsearch/data', default => undef, } # The OSS package distribution's package appends `-oss` to the end of the # canonical package name. $_package_name = $oss ? { true => "${package_name}-oss", default => $package_name, } # Set the plugin path variable for use later in the module. if $plugindir == undef { $_plugindir = "${homedir}/plugins" } else { $_plugindir = $plugindir } + # Can only enable SSL if security_plugin specified + if $ssl or ($system_key != undef) { + if $security_plugin == undef or ! ($security_plugin in ['shield', 'x-pack']) { + fail("\"${security_plugin}\" is not a valid security_plugin parameter value") + } + } + + # Should we restart Elasticsearch on config change? + $_notify_service = $elasticsearch::restart_config_change ? { + true => Service[$elasticsearch::service_name], + false => undef, + } + #### Manage actions contain elasticsearch::package contain elasticsearch::config + contain elasticsearch::service create_resources('elasticsearch::index', $::elasticsearch::indices) - create_resources('elasticsearch::instance', $::elasticsearch::instances) create_resources('elasticsearch::pipeline', $::elasticsearch::pipelines) create_resources('elasticsearch::plugin', $::elasticsearch::plugins) create_resources('elasticsearch::role', $::elasticsearch::roles) create_resources('elasticsearch::script', $::elasticsearch::scripts) create_resources('elasticsearch::snapshot_repository', $::elasticsearch::snapshot_repositories) create_resources('elasticsearch::template', $::elasticsearch::templates) create_resources('elasticsearch::user', $::elasticsearch::users) if ($manage_repo == true) { if ($repo_stage == false) { # Use normal relationship ordering contain elastic_stack::repo Class['elastic_stack::repo'] -> Class['elasticsearch::package'] } else { # Use staging for ordering if !(defined(Stage[$repo_stage])) { stage { $repo_stage: before => Stage['main'] } } include elastic_stack::repo Class<|title == 'elastic_stack::repo'|>{ stage => $repo_stage, } } } if ($license != undef) { contain elasticsearch::license } #### Manage relationships # # Note that many of these overly verbose declarations work around # https://tickets.puppetlabs.com/browse/PUP-1410 # which means clean arrow order chaining won't work if someone, say, # doesn't declare any plugins. # # forgive me for what you're about to see if defined(Class['java']) { Class['java'] -> Class['elasticsearch::config'] } if $ensure == 'present' { - # Installation and configuration + # Installation, configuration and service Class['elasticsearch::package'] -> Class['elasticsearch::config'] + ~> Class['elasticsearch::service'] # Top-level ordering bindings for resources. Class['elasticsearch::config'] -> Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> Elasticsearch::Plugin <| ensure == 'absent' |> -> Class['elasticsearch::config'] Class['elasticsearch::config'] - -> Elasticsearch::Instance <| |> - Class['elasticsearch::config'] -> Elasticsearch::User <| |> Class['elasticsearch::config'] -> Elasticsearch::Role <| |> Class['elasticsearch::config'] -> Elasticsearch::Template <| |> Class['elasticsearch::config'] -> Elasticsearch::Pipeline <| |> Class['elasticsearch::config'] -> Elasticsearch::Index <| |> Class['elasticsearch::config'] -> Elasticsearch::Snapshot_repository <| |> } else { # Absent; remove configuration before the package. Class['elasticsearch::config'] -> Class['elasticsearch::package'] # Top-level ordering bindings for resources. Elasticsearch::Plugin <| |> -> Class['elasticsearch::config'] - Elasticsearch::Instance <| |> - -> Class['elasticsearch::config'] Elasticsearch::User <| |> -> Class['elasticsearch::config'] Elasticsearch::Role <| |> -> Class['elasticsearch::config'] Elasticsearch::Template <| |> -> Class['elasticsearch::config'] Elasticsearch::Pipeline <| |> -> Class['elasticsearch::config'] Elasticsearch::Index <| |> -> Class['elasticsearch::config'] Elasticsearch::Snapshot_repository <| |> -> Class['elasticsearch::config'] } # Install plugins before managing instances or users/roles Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> - -> Elasticsearch::Instance <| |> - Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> -> Elasticsearch::User <| |> Elasticsearch::Plugin <| ensure == 'present' or ensure == 'installed' |> -> Elasticsearch::Role <| |> # Remove plugins after managing users/roles Elasticsearch::User <| |> -> Elasticsearch::Plugin <| ensure == 'absent' |> Elasticsearch::Role <| |> -> Elasticsearch::Plugin <| ensure == 'absent' |> # Ensure roles are defined before managing users that reference roles Elasticsearch::Role <| |> -> Elasticsearch::User <| ensure == 'present' |> # Ensure users are removed before referenced roles are managed Elasticsearch::User <| ensure == 'absent' |> -> Elasticsearch::Role <| |> # Ensure users and roles are managed before calling out to REST resources Elasticsearch::Role <| |> -> Elasticsearch::Template <| |> Elasticsearch::User <| |> -> Elasticsearch::Template <| |> Elasticsearch::Role <| |> -> Elasticsearch::Pipeline <| |> Elasticsearch::User <| |> -> Elasticsearch::Pipeline <| |> Elasticsearch::Role <| |> -> Elasticsearch::Index <| |> Elasticsearch::User <| |> -> Elasticsearch::Index <| |> Elasticsearch::Role <| |> -> Elasticsearch::Snapshot_repository <| |> Elasticsearch::User <| |> -> Elasticsearch::Snapshot_repository <| |> # Ensure that any command-line based user changes are performed before the # file is modified Elasticsearch_user <| |> -> Elasticsearch_user_file <| |> - - # Manage users/roles before instances (req'd to keep dir in sync) - Elasticsearch::Role <| |> - -> Elasticsearch::Instance <| |> - Elasticsearch::User <| |> - -> Elasticsearch::Instance <| |> - - # Ensure instances are started before managing REST resources - Elasticsearch::Instance <| ensure == 'present' |> - -> Elasticsearch::Template <| |> - Elasticsearch::Instance <| ensure == 'present' |> - -> Elasticsearch::Pipeline <| |> - Elasticsearch::Instance <| ensure == 'present' |> - -> Elasticsearch::Index <| |> - Elasticsearch::Instance <| ensure == 'present' |> - -> Elasticsearch::Snapshot_repository <| |> - # Ensure instances are stopped after managing REST resources - Elasticsearch::Template <| |> - -> Elasticsearch::Instance <| ensure == 'absent' |> - Elasticsearch::Pipeline <| |> - -> Elasticsearch::Instance <| ensure == 'absent' |> - Elasticsearch::Index <| |> - -> Elasticsearch::Instance <| ensure == 'absent' |> - Elasticsearch::Snapshot_repository <| |> - -> Elasticsearch::Instance <| ensure == 'absent' |> - - # Ensure scripts are installed before copying them to configuration directory - Elasticsearch::Script <| |> - -> File["${configdir}/scripts"] } diff --git a/manifests/service.pp b/manifests/service.pp index bf77308..29fe10f 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -1,93 +1,112 @@ # This class exists to coordinate all service management related actions, # functionality and logical units in a central place. # # *Note*: "service" is the Puppet term and type for background processes # in general and is used in a platform-independent way. E.g. "service" means # "daemon" in relation to Unix-like systems. # # @param ensure # Controls if the managed resources shall be `present` or `absent`. # If set to `absent`, the managed software packages will be uninstalled, and # any traces of the packages will be purged as well as possible, possibly # including existing configuration files. # System modifications (if any) will be reverted as well as possible (e.g. # removal of created users, services, changed log settings, and so on). # This is a destructive parameter and should be used with care. # # @param init_defaults # Defaults file content in hash representation # # @param init_defaults_file # Defaults file as puppet resource # # @param init_template # Service file as a template # # @param service_flags # Flags to pass to the service. # # @param status # Defines the status of the service. If set to `enabled`, the service is # started and will be enabled at boot time. If set to `disabled`, the # service is stopped and will not be started at boot time. If set to `running`, # the service is started but will not be enabled at boot time. You may use # this to start a service on the first Puppet run instead of the system startup. # If set to `unmanaged`, the service will not be started at boot time and Puppet # does not care whether the service is running or not. For example, this may # be useful if a cluster management software is used to decide when to start # the service plus assuring it is running on the desired node. # # @author Richard Pijnenburg # @author Tyler Langlois +# @author Gavin Williams # -define elasticsearch::service ( - Enum['absent', 'present'] $ensure = $elasticsearch::ensure, - Hash $init_defaults = {}, - Optional[String] $init_defaults_file = undef, - Optional[String] $init_template = undef, - Optional[String] $service_flags = undef, - Elasticsearch::Status $status = $elasticsearch::status, -) { +class elasticsearch::service { - case $elasticsearch::service_provider { + #### Service management - 'init': { - elasticsearch::service::init { $name: - ensure => $ensure, - status => $status, - init_defaults_file => $init_defaults_file, - init_defaults => $init_defaults, - init_template => $init_template, + if $::elasticsearch::ensure == 'present' { + + case $::elasticsearch::status { + # make sure service is currently running, start it on boot + 'enabled': { + $_service_ensure = 'running' + $_service_enable = true } - } - 'openbsd': { - elasticsearch::service::openbsd { $name: - ensure => $ensure, - status => $status, - init_template => $init_template, - service_flags => $service_flags, + # make sure service is currently stopped, do not start it on boot + 'disabled': { + $_service_ensure = 'stopped' + $_service_enable = false } - } - 'systemd': { - elasticsearch::service::systemd { $name: - ensure => $ensure, - status => $status, - init_defaults_file => $init_defaults_file, - init_defaults => $init_defaults, - init_template => $init_template, + # make sure service is currently running, do not start it on boot + 'running': { + $_service_ensure = 'running' + $_service_enable = false + } + # do not start service on boot, do not care whether currently running + # or not + 'unmanaged': { + $_service_ensure = undef + $_service_enable = false } + default: { } + } + } else { + # make sure the service is stopped and disabled (the removal itself will be + # done by package.pp) + $_service_ensure = 'stopped' + $_service_enable = false + } + + # TODO: Flesh this out + $_service_require = undef + + case $::elasticsearch::service_provider { + 'init': { + include elasticsearch::service::init + $_service_provider = undef + } + 'openbsd': { + include elasticsearch::service::openbsd + $_service_provider = undef } 'openrc': { - elasticsearch::service::openrc { $name: - ensure => $ensure, - status => $status, - init_defaults_file => $init_defaults_file, - init_defaults => $init_defaults, - init_template => $init_template, - } + include elasticsearch::service::openrc + $_service_provider = undef + } + 'systemd': { + # include elasticsearch::service::systemd + $_service_provider = 'systemd' } default: { - fail("Unknown service provider ${elasticsearch::service_provider}") + fail("Unknown service provider ${::elasticsearch::service_provider}") } } + + service { $::elasticsearch::service_name: + ensure => $_service_ensure, + enable => $_service_enable, + provider => $_service_provider, + require => $_service_require, + } } diff --git a/spec/acceptance/tests/acceptance_spec.rb b/spec/acceptance/tests/acceptance_spec.rb index 44abf7c..c6abb53 100644 --- a/spec/acceptance/tests/acceptance_spec.rb +++ b/spec/acceptance/tests/acceptance_spec.rb @@ -1,124 +1,92 @@ require 'spec_helper_acceptance' require 'helpers/acceptance/tests/basic_shared_examples.rb' require 'helpers/acceptance/tests/template_shared_examples.rb' require 'helpers/acceptance/tests/removal_shared_examples.rb' require 'helpers/acceptance/tests/pipeline_shared_examples.rb' require 'helpers/acceptance/tests/plugin_shared_examples.rb' require 'helpers/acceptance/tests/plugin_upgrade_shared_examples.rb' require 'helpers/acceptance/tests/snapshot_repository_shared_examples.rb' require 'helpers/acceptance/tests/datadir_shared_examples.rb' require 'helpers/acceptance/tests/package_url_shared_examples.rb' require 'helpers/acceptance/tests/hiera_shared_examples.rb' require 'helpers/acceptance/tests/usergroup_shared_examples.rb' require 'helpers/acceptance/tests/security_shared_examples.rb' describe "elasticsearch v#{v[:elasticsearch_full_version]} class" do - es_01 = { - 'es-01' => { - 'config' => { - 'http.port' => 9200, - 'node.name' => 'elasticsearch001' - } - } + es_config = { + 'http.port' => 9200, + 'node.name' => 'elasticsearch01' } - es_02 = { - 'es-02' => { - 'config' => { - 'http.port' => 9201, - 'node.name' => 'elasticsearch002' - } - } - } - instances = es_01.merge es_02 let(:elastic_repo) { not v[:is_snapshot] } let(:manifest) do package = if not v[:is_snapshot] <<-MANIFEST # Hard version set here due to plugin incompatibilities. version => '#{v[:elasticsearch_full_version]}', MANIFEST else <<-MANIFEST manage_repo => false, package_url => '#{v[:snapshot_package]}', MANIFEST end - heap = if v[:elasticsearch_major_version] > 2 - <<-MANIFEST - jvm_options => [ - '-Xms128m', - '-Xmx128m', - ], - MANIFEST - else - <<-MANIFEST - init_defaults => { - 'ES_HEAP_SIZE' => '128m', - }, - MANIFEST - end - <<-MANIFEST api_timeout => 60, config => { 'cluster.name' => '#{v[:cluster_name]}', 'http.bind_host' => '0.0.0.0', +#{es_config.map { |k, v| " '#{k}' => '#{v}'," }.join("\n")} }, + jvm_options => [ + '-Xms128m', + '-Xmx128m', + ], oss => #{v[:oss]}, #{package} - #{heap} MANIFEST end - context 'instance testing with' do - describe 'one' do - include_examples('basic acceptance tests', es_01) - end - - describe 'two' do - include_examples('basic acceptance tests', instances) + context 'testing with' do + describe 'simple config' do + include_examples('basic acceptance tests', es_config) end - describe 'one absent' do - include_examples('basic acceptance tests', es_01.merge('es-02' => {})) - end - - include_examples 'module removal', ['es-01'] + include_examples('module removal', es_config) end - include_examples('template operations', es_01, v[:template]) + # include_examples('template operations', es_config, v[:template]) - include_examples('pipeline operations', es_01, v[:pipeline]) if semver(v[:elasticsearch_full_version]) >= semver('5.0.0') + # include_examples('pipeline operations', es_01, v[:pipeline]) if semver(v[:elasticsearch_full_version]) >= semver('5.0.0') - include_examples('plugin acceptance tests', v[:elasticsearch_plugins]) unless v[:elasticsearch_plugins].empty? + # include_examples('plugin acceptance tests', v[:elasticsearch_plugins]) unless v[:elasticsearch_plugins].empty? - # Only pre-5.x versions supported versions differing from core ES - if semver(v[:elasticsearch_full_version]) < semver('5.0.0') - include_examples( - 'plugin upgrade acceptance tests', - :name => 'kopf', - :initial => '2.0.1', - :upgraded => '2.1.2', - :repository => 'lmenezes/elasticsearch' - ) - end + # # Only pre-5.x versions supported versions differing from core ES + # if semver(v[:elasticsearch_full_version]) < semver('5.0.0') + # include_examples( + # 'plugin upgrade acceptance tests', + # :name => 'kopf', + # :initial => '2.0.1', + # :upgraded => '2.1.2', + # :repository => 'lmenezes/elasticsearch' + # ) + # end - include_examples 'snapshot repository acceptance tests' + # include_examples 'snapshot repository acceptance tests' - include_examples 'datadir acceptance tests' + # include_examples 'datadir acceptance tests' - # Skip this for snapshot testing, as we only have package files anyway. - include_examples 'package_url acceptance tests' unless v[:is_snapshot] + # # Skip this for snapshot testing, as we only have package files anyway. + # include_examples 'package_url acceptance tests' unless v[:is_snapshot] - include_examples 'hiera acceptance tests', v[:elasticsearch_plugins] + # include_examples 'hiera acceptance tests', v[:elasticsearch_plugins] - include_examples 'user/group acceptance tests' + # include_examples 'user/group acceptance tests' - # Security-related tests (shield/x-pack). - # - # Skip OSS-only distributions since they do not bundle x-pack, and skip - # snapshots since we they don't recognize prod licenses. - include_examples 'security acceptance tests', instances unless v[:oss] or v[:is_snapshot] + # # Security-related tests (shield/x-pack). + # # + # # Skip OSS-only distributions since they do not bundle x-pack, and skip + # # snapshots since we they don't recognize prod licenses. + # include_examples 'security acceptance tests', instances unless v[:oss] or v[:is_snapshot] end diff --git a/spec/helpers/acceptance/tests/bad_manifest_shared_examples.rb b/spec/helpers/acceptance/tests/bad_manifest_shared_examples.rb index aadf579..eb2f80b 100644 --- a/spec/helpers/acceptance/tests/bad_manifest_shared_examples.rb +++ b/spec/helpers/acceptance/tests/bad_manifest_shared_examples.rb @@ -1,33 +1,18 @@ -shared_examples 'invalid manifest application' do |instances| - context "bad #{instances.count}-node manifest" do +shared_examples 'invalid manifest application' do + context "bad manifest" do let(:applied_manifest) do - instance_manifest = instances.map do |instance, meta| - config = meta.map { |k, v| "'#{k}' => '#{v}'," }.join(' ') - <<-MANIFEST - elasticsearch::instance { '#{instance}': - ensure => #{meta.empty? ? 'absent' : 'present'}, - config => { - #{config} - }, - #{defined?(manifest_instance_parameters) && manifest_instance_parameters} - } - MANIFEST - end.join("\n") - <<-MANIFEST class { 'elasticsearch' : #{manifest} #{defined?(manifest_class_parameters) && manifest_class_parameters} } - #{defined?(skip_instance_manifests) || instance_manifest} - #{defined?(extra_manifest) && extra_manifest} MANIFEST end it 'fails to apply' do apply_manifest applied_manifest, :expect_failures => true end end end diff --git a/spec/helpers/acceptance/tests/basic_shared_examples.rb b/spec/helpers/acceptance/tests/basic_shared_examples.rb index 500538d..c00a721 100644 --- a/spec/helpers/acceptance/tests/basic_shared_examples.rb +++ b/spec/helpers/acceptance/tests/basic_shared_examples.rb @@ -1,72 +1,65 @@ require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' -shared_examples 'basic acceptance tests' do |instances| - include_examples 'manifest application', instances +shared_examples 'basic acceptance tests' do | config | + include_examples 'manifest application' describe package("elasticsearch#{v[:oss] ? '-oss' : ''}") do it { should be_installed } end %w[ - /usr/share/elasticsearch/templates_import - /usr/share/elasticsearch/scripts + /etc/elasticsearch + /usr/share/elasticsearch + /var/lib/elasticsearch ].each do |dir| describe file(dir) do it { should be_directory } end end - instances.each do |instance, config| - describe "resources for instance #{instance}" do - describe service("elasticsearch-#{instance}") do - it { send(config.empty? ? :should_not : :should, be_enabled) } - it { send(config.empty? ? :should_not : :should, be_running) } - end + describe "resources for instance " do + describe service("elasticsearch") do + it { send(config.empty? ? :should_not : :should, be_enabled) } + it { send(config.empty? ? :should_not : :should, be_running) } + end - unless config.empty? - describe file(pid_for(instance)) do - it { should be_file } - its(:content) { should match(/[0-9]+/) } - end + unless config.empty? + describe file(pid_file()) do + it { should be_file } + its(:content) { should match(/[0-9]+/) } + end - describe file("/etc/elasticsearch/#{instance}/elasticsearch.yml") do - it { should be_file } - it { should contain "name: #{config['config']['node.name']}" } - it { should contain "/var/lib/elasticsearch/#{instance}" } - end + describe file("/etc/elasticsearch/elasticsearch.yml") do + it { should be_file } + it { should contain "name: #{config['node.name']}" } end + end - unless config.empty? - describe file("/var/lib/elasticsearch/#{instance}") do - it { should be_directory } + unless config.empty? + describe port(config['http.port']) do + it 'open', :with_retries do + should be_listening end + end - describe port(config['config']['http.port']) do - it 'open', :with_retries do - should be_listening + describe server :container do + describe http("http://localhost:#{config['http.port']}/_nodes/_local") do + it 'serves requests', :with_retries do + expect(response.status).to eq(200) end - end - - describe server :container do - describe http("http://localhost:#{config['config']['http.port']}/_nodes/_local") do - it 'serves requests', :with_retries do - expect(response.status).to eq(200) - end - it 'uses the default data path', :with_retries do - json = JSON.parse(response.body)['nodes'].values.first - expected = "/var/lib/elasticsearch/#{instance}" - expected = [expected] if v[:elasticsearch_major_version] > 2 - expect( - json['settings']['path'] - ).to include( - 'data' => expected - ) - end + it 'uses the default data path', :with_retries do + json = JSON.parse(response.body)['nodes'].values.first + data_dir = ['/var/lib/elasticsearch'] + expect( + json['settings']['path'] + ).to include( + 'data' => data_dir + ) end end end end end end diff --git a/spec/helpers/acceptance/tests/manifest_shared_examples.rb b/spec/helpers/acceptance/tests/manifest_shared_examples.rb index 8c01416..49c5597 100644 --- a/spec/helpers/acceptance/tests/manifest_shared_examples.rb +++ b/spec/helpers/acceptance/tests/manifest_shared_examples.rb @@ -1,49 +1,37 @@ -shared_examples 'manifest application' do |instances, idempotency_check = true| - context "#{instances.count}-node manifest" do +shared_examples 'manifest application' do |idempotency_check = true| + context '1-node manifest' do let(:applied_manifest) do - instance_manifest = instances.map do |instance, parameters| - <<-MANIFEST - elasticsearch::instance { '#{instance}': - ensure => #{parameters.empty? ? 'absent' : 'present'}, - #{parameters.map { |k, v| "#{k} => #{v}," }.join("\n")} - #{defined?(manifest_instance_parameters) && manifest_instance_parameters} - } - MANIFEST - end.join("\n") - repo = if elastic_repo <<-MANIFEST class { 'elastic_stack::repo': oss => #{v[:oss]}, version => #{v[:elasticsearch_major_version]}, } MANIFEST else '' end <<-MANIFEST #{repo} class { 'elasticsearch' : #{manifest} #{defined?(manifest_class_parameters) && manifest_class_parameters} } - #{defined?(skip_instance_manifests) || instance_manifest} - #{defined?(extra_manifest) && extra_manifest} MANIFEST end it 'applies cleanly' do - apply_manifest applied_manifest, :catch_failures => true + apply_manifest(applied_manifest, {:catch_failures => true, :debug => v[:puppet_debug]}) end if idempotency_check it 'is idempotent', :logs_on_failure do - apply_manifest applied_manifest, :catch_changes => true + apply_manifest(applied_manifest, {:catch_changes => true, :debug => v[:puppet_debug]}) end end end end diff --git a/spec/helpers/acceptance/tests/removal_shared_examples.rb b/spec/helpers/acceptance/tests/removal_shared_examples.rb index a173931..fa12824 100644 --- a/spec/helpers/acceptance/tests/removal_shared_examples.rb +++ b/spec/helpers/acceptance/tests/removal_shared_examples.rb @@ -1,35 +1,31 @@ -shared_examples 'module removal' do |instances| +shared_examples 'module removal' do |es_config| describe 'uninstalling' do let(:manifest) do - instance_resource = <<-RESOURCE - elasticsearch::instance { '%s' : - ensure => 'absent' - } - RESOURCE - <<-MANIFEST class { 'elasticsearch': ensure => 'absent', oss => #{v[:oss]} } - #{instances.map { |i| instance_resource % i }.join("\n")} MANIFEST end it 'should run successfully' do apply_manifest manifest, :catch_failures => true end it 'is idempotent' do apply_manifest manifest, :catch_changes => true end - instances.each do |instance| - describe file("/etc/elasticsearch/#{instance}") do - it { should_not be_directory } - end + describe service('elasticsearch') do + it { should_not be_enabled } + it { should_not be_running } + end - describe service(instance) do - it { should_not be_enabled } - it { should_not be_running } + unless es_config.empty? + describe port(es_config['http.port']) do + it 'closed' do + should_not be_listening + end end end + end end diff --git a/spec/helpers/acceptance/tests/security_shared_examples.rb b/spec/helpers/acceptance/tests/security_shared_examples.rb index a291410..8d2ad45 100644 --- a/spec/helpers/acceptance/tests/security_shared_examples.rb +++ b/spec/helpers/acceptance/tests/security_shared_examples.rb @@ -1,272 +1,272 @@ require 'json' require 'spec_utilities' require 'helpers/acceptance/tests/manifest_shared_examples' shared_examples 'security plugin manifest' do |instances, credentials| let(:extra_manifest) do instance_plugins = <<-MANIFEST Elasticsearch::Plugin { instances => #{instances.keys} } MANIFEST users = credentials.map do |username, meta| <<-USER #{meta[:changed] ? "notify { 'password change for #{username}' : } ~>" : ''} elasticsearch::user { '#{username}': password => '#{meta[:hash] ? meta[:hash] : meta[:plaintext]}', roles => #{meta[:roles].reduce({}) { |a, e| a.merge(e) }.keys}, } USER end.join("\n") roles = credentials.values.reduce({}) do |sum, user_metadata| # Collect all roles across users sum.merge user_metadata end[:roles].reduce({}) do |all_roles, role| all_roles.merge role end.reject do |_role, permissions| permissions.empty? end.map do |role, rights| <<-ROLE elasticsearch::role { '#{role}': privileges => #{rights} } ROLE end.join("\n") <<-MANIFEST #{security_plugins} #{instance_plugins} #{users} #{roles} #{ssl_params} MANIFEST end include_examples( 'manifest application', instances, not(credentials.values.map { |p| p[:changed] }.any?) ) end shared_examples 'secured request' do |test_desc, instances, path, http_test, expected, user = nil, pass = nil| instances.each_value do |i| describe port(i['config']['http.port']) do - it 'open', :with_generous_retries do + it 'open', :with_retries do should be_listening end end describe server :container do describe http( "https://localhost:#{i['config']['http.port']}#{path}", { :ssl => { :verify => false } }.merge((user and pass) ? { :basic_auth => [user, pass] } : {}) ) do it test_desc, :with_retries do expect(http_test.call(response)).to eq(expected) end end end end end shared_examples 'security acceptance tests' do |default_instances| describe 'security plugin operations', :if => vault_available?, :then_purge => true, :with_license => true, :with_certificates => true do superuser_role = v[:elasticsearch_major_version] > 2 ? 'superuser' : 'admin' rand_string = lambda { [*('a'..'z')].sample(8).join } admin_user = rand_string.call admin_password = rand_string.call admin = { admin_user => { :plaintext => admin_password, :roles => [{ superuser_role => [] }] } } let(:manifest_class_parameters) do <<-MANIFEST license => file('#{v[:elasticsearch_license_path]}'), restart_on_change => true, security_plugin => '#{v[:elasticsearch_major_version] > 2 ? 'x-pack' : 'shield'}', api_basic_auth_password => '#{admin_password}', api_basic_auth_username => '#{admin_user}', api_ca_file => '#{@tls[:ca][:cert][:path]}', api_protocol => 'https', validate_tls => true, MANIFEST end let(:security_plugins) do if v[:elasticsearch_major_version] <= 2 <<-MANIFEST elasticsearch::plugin { 'elasticsearch/license/latest' : } elasticsearch::plugin { 'elasticsearch/shield/latest' : } MANIFEST elsif semver(v[:elasticsearch_full_version].split('-').first) < semver('6.3.0') <<-MANIFEST elasticsearch::plugin { 'x-pack' : } MANIFEST else '' end end describe 'over tls' do user_one = rand_string.call user_two = rand_string.call user_one_pw = rand_string.call user_two_pw = rand_string.call context "instance #{default_instances.first.first}" do instance_name = default_instances.keys.first instance = { instance_name => default_instances[instance_name].merge('ssl' => true) } let(:ssl_params) do <<-MANIFEST Elasticsearch::Instance['#{instance_name}'] { ca_certificate => '#{@tls[:ca][:cert][:path]}', certificate => '#{@tls[:clients].first[:cert][:path]}', private_key => '#{@tls[:clients].first[:key][:path]}', keystore_password => '#{@keystore_password}', } MANIFEST end describe 'user authentication' do username_passwords = { user_one => { :plaintext => user_one_pw, :roles => [{ superuser_role => [] }] }, user_two => { :plaintext => user_two_pw, :roles => [{ superuser_role => [] }] } }.merge(admin) username_passwords[user_two][:hash] = bcrypt(username_passwords[user_two][:plaintext]) include_examples('security plugin manifest', instance, username_passwords) include_examples( 'secured request', 'denies unauthorized access', instance, '/_cluster/health', lambda { |r| r.status }, 401 ) include_examples( 'secured request', "permits user #{user_one} access", instance, '/_cluster/health', lambda { |r| r.status }, 200, user_one, user_one_pw ) include_examples( 'secured request', "permits user #{user_two} access", instance, '/_cluster/health', lambda { |r| r.status }, 200, user_two, user_two_pw ) end describe 'changing passwords' do new_password = rand_string.call username_passwords = { user_one => { :plaintext => new_password, :changed => true, :roles => [{ superuser_role => [] }] } } include_examples('security plugin manifest', instance, username_passwords) include_examples( 'secured request', 'denies unauthorized access', instance, '/_cluster/health', lambda { |r| r.status }, 401 ) include_examples( 'secured request', "permits user #{user_two} access with new password", instance, '/_cluster/health', lambda { |r| r.status }, 200, user_one, new_password ) end describe 'roles' do password = rand_string.call username = rand_string.call user = { username => { :plaintext => password, :roles => [{ rand_string.call => { 'cluster' => [ 'cluster:monitor/health' ] } }] } } include_examples('security plugin manifest', instance, user) include_examples( 'secured request', 'denies unauthorized access', instance, '/_snapshot', lambda { |r| r.status }, 403, username, password ) include_examples( 'secured request', 'permits authorized access', instance, '/_cluster/health', lambda { |r| r.status }, 200, username, password ) end end describe 'with two instances' do let(:ssl_params) do @tls[:clients].each_with_index.map do |cert, i| format(%( Elasticsearch::Instance['es-%02d'] { ca_certificate => '#{@tls[:ca][:cert][:path]}', certificate => '#{cert[:cert][:path]}', private_key => '#{cert[:key][:path]}', keystore_password => '#{@keystore_password}', } ), i + 1) end.join("\n") end ssl_instances = default_instances.map do |instance, meta| new_config = if v[:elasticsearch_major_version] > 2 { 'xpack.ssl.verification_mode' => 'none' } else { 'shield.ssl.hostname_verification' => false } end [ instance, { 'config' => meta['config'].merge(new_config).merge( 'discovery.zen.minimum_master_nodes' => default_instances.keys.size ), 'ssl' => true } ] end.to_h username = rand_string.call password = rand_string.call include_examples( 'security plugin manifest', ssl_instances, username => { :plaintext => password, :roles => [{ superuser_role => [] }] } ) include_examples( 'secured request', 'clusters between two nodes', ssl_instances, '/_nodes', lambda { |r| JSON.parse(r.body)['nodes'].size }, 2, username, password ) end end end end diff --git a/spec/helpers/acceptance/tests/template_shared_examples.rb b/spec/helpers/acceptance/tests/template_shared_examples.rb index fd9782d..8b92ae0 100644 --- a/spec/helpers/acceptance/tests/template_shared_examples.rb +++ b/spec/helpers/acceptance/tests/template_shared_examples.rb @@ -1,121 +1,111 @@ require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' require 'helpers/acceptance/tests/bad_manifest_shared_examples' # Describes how to apply a manifest with a template, verify it, and clean it up -shared_examples 'template application' do |instances, name, template, param| +shared_examples 'template application' do |es_config, name, template, param| context 'present' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{name}': ensure => 'present', #{param} } MANIFEST end - include_examples( - 'manifest application', - instances - ) + include_examples('manifest application') - include_examples 'template content', instances, template + include_examples 'template content', es_config, template end context 'absent' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{name}': ensure => absent, } MANIFEST end - include_examples( - 'manifest application', - instances - ) + include_examples('manifest application') end end # Verifies the content of a loaded index template. -shared_examples 'template content' do |instances, template| - instances.each_value do |i| - describe port(i['config']['http.port']) do - it 'open', :with_retries do - should be_listening - end +shared_examples 'template content' do |es_config, template| + elasticsearch_port = es_config['http.port'] + describe port(elasticsearch_port) do + it 'open', :with_retries do + should be_listening end + end - describe server :container do - describe http( - "http://localhost:#{i['config']['http.port']}/_template", - :params => { 'flat_settings' => 'false' } - ) do - it 'returns the installed template', :with_retries do - expect(JSON.parse(response.body).values) - .to include(include(template)) - end + describe server :container do + describe http( + "http://localhost:#{elasticsearch_port}/_template", + :params => { 'flat_settings' => 'false' } + ) do + it 'returns the installed template', :with_retries do + expect(JSON.parse(response.body).values) + .to include(include(template)) end end end end # Main entrypoint for template tests -shared_examples 'template operations' do |instances, template| +shared_examples 'template operations' do |es_config, template| describe 'template resources' do before :all do shell "mkdir -p #{default['distmoduledir']}/another/files" create_remote_file( default, "#{default['distmoduledir']}/another/files/good.json", JSON.dump(template) ) create_remote_file( default, "#{default['distmoduledir']}/another/files/bad.json", JSON.dump(template)[0..-5] ) end context 'configured through' do context '`source`' do include_examples( 'template application', - instances, + es_config, SecureRandom.hex(8), template, "source => 'puppet:///modules/another/good.json'" ) end context '`content`' do include_examples( 'template application', - instances, + es_config, SecureRandom.hex(8), template, "content => '#{JSON.dump(template)}'" ) end context 'bad json' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{SecureRandom.hex(8)}': ensure => 'present', file => 'puppet:///modules/another/bad.json' } MANIFEST end - include_examples( - 'invalid manifest application', - instances - ) + include_examples('invalid manifest application') end end end end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 3fe4fa0..25cc657 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -1,257 +1,256 @@ require 'beaker-rspec' require 'beaker/puppet_install_helper' require 'securerandom' require 'thread' require 'infrataster/rspec' require 'rspec/retry' require 'vault' require 'simp/beaker_helpers' include Simp::BeakerHelpers require_relative 'spec_helper_tls' require_relative 'spec_utilities' require_relative '../lib/puppet_x/elastic/deep_to_i' require_relative '../lib/puppet_x/elastic/deep_to_s' # def f # RSpec.configuration.fact # end run_puppet_install_helper('agent') unless ENV['BEAKER_provision'] == 'no' RSpec.configure do |c| # General-purpose spec-global variables c.add_setting :v, :default => {} + # Puppet debug logging + v[:puppet_debug] = ENV['BEAKER_debug'] ? true : false + unless ENV['snapshot_version'].nil? v[:snapshot_version] = ENV['snapshot_version'] v[:is_snapshot] = ENV['SNAPSHOT_TEST'] == 'true' end unless ENV['ELASTICSEARCH_VERSION'].nil? and v[:snapshot_version].nil? v[:elasticsearch_full_version] = ENV['ELASTICSEARCH_VERSION'] || v[:snapshot_version] v[:elasticsearch_major_version] = v[:elasticsearch_full_version].split('.').first.to_i v[:elasticsearch_package] = {} v[:template] = if v[:elasticsearch_major_version] < 6 JSON.load(File.new('spec/fixtures/templates/pre_6.0.json')) elsif v[:elasticsearch_major_version] >= 8 JSON.load(File.new('spec/fixtures/templates/post_8.0.json')) else JSON.load(File.new('spec/fixtures/templates/post_6.0.json')) end v[:template] = Puppet_X::Elastic.deep_to_i(Puppet_X::Elastic.deep_to_s(v[:template])) v[:pipeline] = JSON.load(File.new('spec/fixtures/pipelines/example.json')) v[:elasticsearch_plugins] = Dir[ artifact("*#{v[:elasticsearch_full_version]}.zip", ['plugins']) ].map do |plugin| plugin_filename = File.basename(plugin) plugin_name = plugin_filename.match(/^(?.+)-#{v[:elasticsearch_full_version]}.zip/)[:name] [ plugin_name, { :path => plugin, :url => derive_plugin_urls_for(v[:elasticsearch_full_version], [plugin_name]).keys.first } ] end.to_h end v[:oss] = (not ENV['OSS_PACKAGE'].nil?) and ENV['OSS_PACKAGE'] == 'true' v[:cluster_name] = SecureRandom.hex(10) # rspec-retry c.display_try_failure_messages = true c.default_sleep_interval = 10 # General-case retry keyword for unstable tests c.around :each, :with_retries do |example| - example.run_with_retry retry: 4 - end - # More forgiving retry config for really flaky tests - c.around :each, :with_generous_retries do |example| example.run_with_retry retry: 10 end # Helper hook for module cleanup c.after :context, :with_cleanup do apply_manifest <<-EOS class { 'elasticsearch': ensure => 'absent', manage_repo => true, oss => #{v[:oss]}, } elasticsearch::instance { 'es-01': ensure => 'absent' } file { '/usr/share/elasticsearch/plugin': ensure => 'absent', force => true, recurse => true, require => Class['elasticsearch'], } EOS end c.before :context, :with_certificates do @keystore_password = SecureRandom.hex @role = [*('a'..'z')].sample(8).join # Setup TLS cert placement @tls = gen_certs(2, '/tmp') create_remote_file hosts, @tls[:ca][:cert][:path], @tls[:ca][:cert][:pem] @tls[:clients].each do |node| node.each do |_type, params| create_remote_file hosts, params[:path], params[:pem] end end end c.before :context, :with_license do Vault.address = ENV['VAULT_ADDR'] Vault.auth.approle ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID'] licenses = Vault.with_retries(Vault::HTTPConnectionError) do Vault.logical.read(ENV['VAULT_PATH']) end.data raise 'No license found!' unless licenses license = case v[:elasticsearch_major_version] when 2 licenses[:v2] else licenses[:v5] end create_remote_file hosts, '/tmp/license.json', license v[:elasticsearch_license_path] = '/tmp/license.json' end c.after :context, :then_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end c.before :context, :first_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end # Provide a hook filter to spit out some ES logs if the example fails. c.after(:example, :logs_on_failure) do |example| if example.exception hosts.each do |host| on host, "find / -name '#{v[:cluster_name]}.log' | xargs cat || true" do |result| puts result.formatted_output end end end end end files_dir = ENV['files_dir'] || './spec/fixtures/artifacts' # General bootstrapping steps for each host hosts.each do |host| # # Set the host to 'aio' in order to adopt the puppet-agent style of # # installation, and configure paths/etc. # host[:type] = 'aio' # configure_defaults_on host, 'aio' if fact('os.family') == 'Suse' install_package host, '--force-resolution augeas-devel libxml2-devel ruby-devel' on host, 'gem install ruby-augeas --no-ri --no-rdoc' end v[:ext] = case fact('os.family') when 'Debian' 'deb' else 'rpm' end if v[:elasticsearch_package] v[:elasticsearch_package].merge!( derive_full_package_url( v[:elasticsearch_full_version], [v[:ext]] ).flat_map do |url, filename| [[:url, url], [:filename, filename], [:path, artifact(filename)]] end.to_h ) end Infrataster::Server.define(:docker) do |server| server.address = host[:ip] server.ssh = host[:ssh].tap { |s| s.delete :forward_agent } end Infrataster::Server.define(:container) do |server| server.address = host[:vm_ip] # this gets ignored anyway server.from = :docker end end RSpec.configure do |c| if v[:is_snapshot] c.before :suite do scp_to default, "#{files_dir}/elasticsearch-snapshot.#{v[:ext]}", "/tmp/elasticsearch-snapshot.#{v[:ext]}" v[:snapshot_package] = "file:/tmp/elasticsearch-snapshot.#{v[:ext]}" end end c.before :suite do # Install module and dependencies install_dev_puppet_module :ignore_list => [ 'junit' ] + Beaker::DSL::InstallUtils::ModuleUtils::PUPPET_MODULE_INSTALL_IGNORE hosts.each do |host| modules = %w[archive augeas_core datacat java java_ks stdlib elastic_stack] dist_module = { 'Debian' => ['apt'], 'Suse' => ['zypprepo'], 'RedHat' => %w[concat yumrepo_core] }[fact('os.family')] modules += dist_module unless dist_module.nil? modules.each do |mod| copy_module_to( host, :module_name => mod, :source => "spec/fixtures/modules/#{mod}" ) end on(host, 'mkdir -p etc/puppet/modules/another/files/') # Apt doesn't update package caches sometimes, ensure we're caught up. shell 'apt-get update' if fact('os.family') == 'Debian' end # Use the Java class once before the suite of tests unless shell('command -v java', :accept_all_exit_codes => true).exit_code.zero? java = case fact('os.name') when 'OpenSuSE' 'package => "java-1_8_0-openjdk-headless",' else '' end apply_manifest <<-MANIFEST class { "java" : distribution => "jre", #{java} } MANIFEST end end end # # Java 8 is only easy to manage on recent distros # def v5x_capable? # (fact('os.family') == 'RedHat' and \ # not (fact('os.name') == 'OracleLinux' and \ # f['os']['release']['major'] == '6')) or \ # f.dig 'os', 'distro', 'codename' == 'xenial' # end diff --git a/spec/spec_utilities.rb b/spec/spec_utilities.rb index 5ca112e..1e8035c 100644 --- a/spec/spec_utilities.rb +++ b/spec/spec_utilities.rb @@ -1,136 +1,149 @@ require 'bcrypt' require 'open-uri' def to_agent_version(puppet_version) # REF: https://docs.puppet.com/puppet/latest/reference/about_agent.html { # Puppet => Agent '4.10.4' => '1.10.4', '4.10.3' => '1.10.3', '4.10.2' => '1.10.2', '4.10.1' => '1.10.1', '4.10.0' => '1.10.0', '4.9.4' => '1.9.3', '4.8.2' => '1.8.3', '4.7.1' => '1.7.2', '4.7.0' => '1.7.1', '4.6.2' => '1.6.2', '4.5.3' => '1.5.3', '4.4.2' => '1.4.2', '4.4.1' => '1.4.1', '4.4.0' => '1.4.0', '4.3.2' => '1.3.6', '4.3.1' => '1.3.2', '4.3.0' => '1.3.0', '4.2.3' => '1.2.7', '4.2.2' => '1.2.6', '4.2.1' => '1.2.2', '4.2.0' => '1.2.1', '4.1.0' => '1.1.1', '4.0.0' => '1.0.1' }[puppet_version] end def derive_artifact_urls_for(full_version, plugins = ['analysis-icu']) derive_full_package_url(full_version).merge( derive_plugin_urls_for(full_version, plugins) ) end def derive_full_package_url(full_version, extensions = %w[deb rpm]) extensions.map do |ext| url = if full_version.start_with? '2' "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-#{full_version}.#{ext}" else "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}.#{ext}" end [url, File.basename(url)] end.to_h end def derive_plugin_urls_for(full_version, plugins = ['analysis-icu']) plugins.map do |plugin| url = if full_version.start_with? '2' "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/plugin/#{plugin}/#{full_version}/#{plugin}-#{full_version}.zip" else "https://artifacts.elastic.co/downloads/elasticsearch-plugins/#{plugin}/#{plugin}-#{full_version}.zip" end [url, File.join('plugins', File.basename(url))] end.to_h end def artifact(file, fixture_path = []) File.join(%w[spec fixtures artifacts] + fixture_path + [File.basename(file)]) end def get(url, file_path) puts "Fetching #{url}..." found = false until found uri = URI.parse(url) conn = Net::HTTP.new(uri.host, uri.port) conn.use_ssl = true res = conn.get(uri.path) if res.header['location'] url = res.header['location'] else found = true end end File.open(file_path, 'w+') { |fh| fh.write res.body } end def fetch_archives(archives) archives.each do |url, orig_fp| fp = "spec/fixtures/artifacts/#{orig_fp}" if File.exist? fp if fp.end_with? 'tar.gz' and !system("tar -tzf #{fp} &>/dev/null") puts "Archive #{fp} corrupt, re-fetching..." File.delete fp else puts "Already retrieved intact archive #{fp}..." next end end get url, fp end end +def pid_file + if fact('operatingsystem') == 'Ubuntu' \ + and Gem::Version.new(fact('operatingsystemrelease')) \ + < Gem::Version.new('15.04') + "/var/run/elasticsearch.pid" + elsif fact('operatingsystem') == 'Debian' \ + and fact('lsbmajdistrelease').to_i <= 7 + "/var/run/elasticsearch.pid" + else + "/var/run/elasticsearch/elasticsearch.pid" + end +end + def pid_for(instance) if fact('operatingsystem') == 'Ubuntu' \ and Gem::Version.new(fact('operatingsystemrelease')) \ < Gem::Version.new('15.04') "/var/run/elasticsearch-#{instance}.pid" elsif fact('operatingsystem') == 'Debian' \ and fact('lsbmajdistrelease').to_i <= 7 "/var/run/elasticsearch-#{instance}.pid" else "/var/run/elasticsearch/elasticsearch-#{instance}.pid" end end def vault_available? %w[VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID VAULT_PATH].select do |var| ENV[var].nil? end.empty? end def http_retry(url) retries ||= 0 open(url).read rescue retry if (retries += 1) < 3 end # Helper to store arbitrary testing setting values def v RSpec.configuration.v end def semver(version) Gem::Version.new version end def bcrypt(value) BCrypt::Password.create(value) end diff --git a/templates/etc/elasticsearch/log4j2.properties.erb b/templates/etc/elasticsearch/log4j2.properties.erb index 4e4d831..89ba143 100644 --- a/templates/etc/elasticsearch/log4j2.properties.erb +++ b/templates/etc/elasticsearch/log4j2.properties.erb @@ -1,99 +1,99 @@ -status = <%= @logging_level.downcase %> +status = <%= scope['elasticsearch::logging_level'].downcase %> # log action execution errors for easier debugging logger.action.name = org.elasticsearch.action logger.action.level = debug appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n -<%- if @file_rolling_type == 'file' -%> +<%- if scope['elasticsearch::file_rolling_type'] == 'file' -%> appender.fixed.type = File appender.fixed.name = fixed appender.fixed.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log appender.fixed.layout.type = PatternLayout appender.fixed.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n <%- else -%> appender.rolling.type = RollingFile appender.rolling.name = rolling appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log appender.rolling.layout.type = PatternLayout appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n -<%- if @file_rolling_type == 'dailyRollingFile' -%> +<%- if scope['elasticsearch::file_rolling_type'] == 'dailyRollingFile' -%> appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 appender.rolling.policies.time.modulate = true -<%- elsif @file_rolling_type == 'rollingFile' -%> +<%- elsif scope['elasticsearch::file_rolling_type'] == 'rollingFile' -%> appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%i.log.gz appender.rolling.policies.type = Policies appender.rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling.policies.size.size = <%= @rolling_file_max_file_size %> +appender.rolling.policies.size.size = <%= scope['elasticsearch::rolling_file_max_file_size'] %> appender.rolling.strategy.type = DefaultRolloverStrategy -appender.rolling.strategy.max = <%= @rolling_file_max_backup_index %> +appender.rolling.strategy.max = <%= scope['elasticsearch::rolling_file_max_backup_index'] %> <%- end -%> <%- end -%> -rootLogger.level = <%= @logging_level.downcase %> +rootLogger.level = <%= scope['elasticsearch::logging_level'].downcase %> rootLogger.appenderRef.console.ref = console -<%- if @file_rolling_type == 'file' -%> +<%- if scope['elasticsearch::file_rolling_type'] == 'file' -%> rootLogger.appenderRef.fixed.ref = fixed <%- else -%> rootLogger.appenderRef.rolling.ref = rolling <%- end -%> appender.deprecation_rolling.type = RollingFile appender.deprecation_rolling.name = deprecation_rolling appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log appender.deprecation_rolling.layout.type = PatternLayout appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz appender.deprecation_rolling.policies.type = Policies appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.deprecation_rolling.policies.size.size = 1GB appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 4 logger.deprecation.name = org.elasticsearch.deprecation -<%- if @deprecation_logging -%> -logger.deprecation.level = <%= @deprecation_logging_level.downcase %> +<%- if scope['elasticsearch::deprecation_logging'] -%> +logger.deprecation.level = <%= scope['elasitcsearch::deprecation_logging_level'].downcase %> <%- else -%> logger.deprecation.level = warn <%- end -%> logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling logger.deprecation.additivity = false appender.index_search_slowlog_rolling.type = RollingFile appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log appender.index_search_slowlog_rolling.layout.type = PatternLayout appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log appender.index_search_slowlog_rolling.policies.type = Policies appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.index_search_slowlog_rolling.policies.time.interval = 1 appender.index_search_slowlog_rolling.policies.time.modulate = true logger.index_search_slowlog_rolling.name = index.search.slowlog logger.index_search_slowlog_rolling.level = trace logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling logger.index_search_slowlog_rolling.additivity = false appender.index_indexing_slowlog_rolling.type = RollingFile appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log appender.index_indexing_slowlog_rolling.layout.type = PatternLayout appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log appender.index_indexing_slowlog_rolling.policies.type = Policies appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.index_indexing_slowlog_rolling.policies.time.interval = 1 appender.index_indexing_slowlog_rolling.policies.time.modulate = true logger.index_indexing_slowlog.name = index.indexing.slowlog.index logger.index_indexing_slowlog.level = trace logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling logger.index_indexing_slowlog.additivity = false diff --git a/templates/etc/elasticsearch/logging.yml.erb b/templates/etc/elasticsearch/logging.yml.erb deleted file mode 100644 index 795fa78..0000000 --- a/templates/etc/elasticsearch/logging.yml.erb +++ /dev/null @@ -1,71 +0,0 @@ -# This file is managed by Puppet, do not edit manually, your changes *will* be overwritten! -# -# Please see the source file for context and more information: -# -# https://github.com/elasticsearch/elasticsearch/blob/master/config/logging.yml -# - -es.logger.level: <%= @logging_level %> -rootLogger: <%= @logging_level %>, console, file - -# ----- Configuration set by Puppet --------------------------------------------- - -<% @logging_hash.sort.each do |key,value| %> -logger.<%= key %>: <%= value %> -<% end %> - -<% if @deprecation_logging -%> -logger.deprecation: <%= @deprecation_logging_level %>, deprecation_log_file -<% end -%> - -# ------------------------------------------------------------------------------- - -additivity: - index.search.slowlog: false - index.indexing.slowlog: false - -appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - file: - type: <%= @file_rolling_type %> - file: ${path.logs}/${cluster.name}.log - <%- if @file_rolling_type == 'dailyRollingFile' -%> - datePattern: <%= @daily_rolling_date_pattern %> - <%- elsif @file_rolling_type == 'rollingFile' -%> - maxBackupIndex: <%= @rolling_file_max_backup_index %> - maxFileSize: <%= @rolling_file_max_file_size %> - <%- end -%> - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_search_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_search_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_indexing_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - -<% if @deprecation_logging -%> - deprecation_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_deprecation.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" -<% end %>