diff --git a/.sync.yml b/.sync.yml index ab7e62b..431745e 100644 --- a/.sync.yml +++ b/.sync.yml @@ -1,14 +1,14 @@ --- Gemfile: optional: ':test': - gem: bcrypt - gem: webmock ':system_tests': - gem: bcrypt - gem: rspec-retry - - gem: infrataster + - gem: simp-beaker-helpers spec/spec_helper.rb: hiera_config: "'spec/fixtures/hiera/hiera.yaml'" spec/spec_helper_acceptance.rb: unmanaged: false diff --git a/Gemfile b/Gemfile index fdfb1dd..8edfc7b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,39 +1,39 @@ # Managed by modulesync - DO NOT EDIT # https://voxpupuli.org/docs/updating-files-managed-with-modulesync/ source ENV['GEM_SOURCE'] || "https://rubygems.org" group :test do gem 'voxpupuli-test', '~> 5.0', :require => false gem 'coveralls', :require => false gem 'simplecov-console', :require => false gem 'puppet_metadata', '~> 1.0', :require => false gem 'bcrypt', :require => false gem 'webmock', :require => false end group :development do gem 'guard-rake', :require => false gem 'overcommit', '>= 0.39.1', :require => false end group :system_tests do gem 'voxpupuli-acceptance', '~> 1.0', :require => false gem 'bcrypt', :require => false gem 'rspec-retry', :require => false - gem 'infrataster', :require => false + gem 'simp-beaker-helpers', :require => false end group :release do gem 'github_changelog_generator', '>= 1.16.1', :require => false if RUBY_VERSION >= '2.5' gem 'voxpupuli-release', '>= 1.2.0', :require => false gem 'puppet-strings', '>= 2.2', :require => false end gem 'rake', :require => false gem 'facter', ENV['FACTER_GEM_VERSION'], :require => false, :groups => [:test] puppetversion = ENV['PUPPET_VERSION'] || '>= 6.0' gem 'puppet', puppetversion, :require => false, :groups => [:test] # vim: syntax=ruby diff --git a/lib/puppet/provider/elastic_rest.rb b/lib/puppet/provider/elastic_rest.rb index f320b0e..2a56ed3 100644 --- a/lib/puppet/provider/elastic_rest.rb +++ b/lib/puppet/provider/elastic_rest.rb @@ -1,286 +1,286 @@ # frozen_string_literal: true require 'json' require 'net/http' require 'openssl' # Parent class encapsulating general-use functions for children REST-based # providers. class Puppet::Provider::ElasticREST < Puppet::Provider class << self attr_accessor :api_discovery_uri, :api_resource_style, :api_uri, :discrete_resource_creation, :metadata, :metadata_pipeline, :query_string end # Fetch arbitrary metadata for the class from an instance object. # # @return String def metadata self.class.metadata end # Retrieve the class query_string variable # # @return String def query_string self.class.query_string end # Perform a REST API request against the indicated endpoint. # # @return Net::HTTPResponse def self.rest(http, req, timeout = 10, username = nil, password = nil, validate_tls: true) if username && password req.basic_auth username, password elsif username || password Puppet.warning( 'username and password must both be defined, skipping basic auth' ) end req['Accept'] = 'application/json' http.read_timeout = timeout http.open_timeout = timeout http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless validate_tls begin http.request req rescue EOFError => e # Because the provider attempts a best guess at API access, we # only fail when HTTP operations fail for mutating methods. unless %w[GET OPTIONS HEAD].include? req.method raise Puppet::Error, "Received '#{e}' from the Elasticsearch API. Are your API settings correct?" end end end # Helper to format a remote URL request for Elasticsearch which takes into # account path ordering, et cetera. def self.format_uri(resource_path, property_flush = {}) return api_uri if resource_path.nil? || api_resource_style == :bare if discrete_resource_creation && !property_flush[:ensure].nil? resource_path else case api_resource_style when :prefix "#{resource_path}/#{api_uri}" else "#{api_uri}/#{resource_path}" end end end # Fetch Elasticsearch API objects. Accepts a variety of argument functions # dictating how to connect to the Elasticsearch API. # # @return Array # an array of Hashes representing the found API objects, whether they be # templates, pipelines, et cetera. def self.api_objects(protocol = 'http', host = 'localhost', port = 9200, timeout = 10, username = nil, password = nil, ca_file = nil, ca_path = nil, validate_tls: true) uri = URI("#{protocol}://#{host}:#{port}/#{format_uri(api_discovery_uri)}") http = Net::HTTP.new uri.host, uri.port req = Net::HTTP::Get.new uri.request_uri http.use_ssl = uri.scheme == 'https' [[ca_file, :ca_file=], [ca_path, :ca_path=]].each do |arg, method| http.send method, arg if arg && http.respond_to?(method) end response = rest http, req, timeout, username, password, validate_tls: validate_tls results = [] results = process_body(response.body) if response.respond_to?(:code) && response.code.to_i == 200 results end # Process the JSON response body def self.process_body(body) JSON.parse(body).map do |object_name, api_object| { :name => object_name, :ensure => :present, metadata => process_metadata(api_object), :provider => name } end end # Passes API objects through arbitrary Procs/lambdas in order to postprocess # API responses. def self.process_metadata(raw_metadata) if metadata_pipeline.is_a?(Array) && !metadata_pipeline.empty? metadata_pipeline.reduce(raw_metadata) do |md, processor| processor.call md end else raw_metadata end end # Fetch an array of provider objects from the Elasticsearch API. def self.instances api_objects.map { |resource| new resource } end # Unlike a typical #prefetch, which just ties discovered #instances to the # correct resources, we need to quantify all the ways the resources in the # catalog know about Elasticsearch API access and use those settings to # fetch any templates we can before associating resources and providers. def self.prefetch(resources) # Get all relevant API access methods from the resources we know about res = resources.map do |_, resource| p = resource.parameters [ p[:protocol].value, p[:host].value, p[:port].value, p[:timeout].value, (p.key?(:username) ? p[:username].value : nil), (p.key?(:password) ? p[:password].value : nil), (p.key?(:ca_file) ? p[:ca_file].value : nil), (p.key?(:ca_path) ? p[:ca_path].value : nil), - p[:validate_tls].value, + { validate_tls: p[:validate_tls].value }, ] # Deduplicate identical settings, and fetch templates end.uniq res = res.map do |api| api_objects(*api) # Flatten and deduplicate the array, instantiate providers, and do the # typical association dance end res.flatten.uniq.map { |resource| new resource }.each do |prov| if (resource = resources[prov.name]) resource.provider = prov end end end def initialize(value = {}) super(value) @property_flush = {} end # Generate a request body def generate_body JSON.generate( if metadata != :content && @property_flush[:ensure] == :present { metadata.to_s => resource[metadata] } else resource[metadata] end ) end # Call Elasticsearch's REST API to appropriately PUT/DELETE/or otherwise # update any managed API objects. def flush Puppet.debug('Got to flush') uri = URI( format( '%s://%s:%d/%s', resource[:protocol], resource[:host], resource[:port], self.class.format_uri(resource[:name], @property_flush) ) ) uri.query = URI.encode_www_form query_string if query_string Puppet.debug("Generated URI = #{uri.inspect}") case @property_flush[:ensure] when :absent req = Net::HTTP::Delete.new uri.request_uri else req = Net::HTTP::Put.new uri.request_uri req.body = generate_body Puppet.debug("Generated body looks like: #{req.body.inspect}") # As of Elasticsearch 6.x, required when requesting with a payload (so we # set it always to be safe) req['Content-Type'] = 'application/json' if req['Content-Type'].nil? end http = Net::HTTP.new uri.host, uri.port http.use_ssl = uri.scheme == 'https' %i[ca_file ca_path].each do |arg| http.send "#{arg}=".to_sym, resource[arg] if !resource[arg].nil? && http.respond_to?(arg) end response = self.class.rest( http, req, resource[:timeout], resource[:username], resource[:password], validate_tls: resource[:validate_tls] ) # Attempt to return useful error output unless response.code.to_i == 200 Puppet.debug("Non-OK reponse: Body = #{response.body.inspect}") json = JSON.parse(response.body) err_msg = if json.key? 'error' if json['error'].is_a?(Hash) \ && json['error'].key?('root_cause') # Newer versions have useful output json['error']['root_cause'].first['reason'] else # Otherwise fallback to old-style error messages json['error'] end else # As a last resort, return the response error code "HTTP #{response.code}" end raise Puppet::Error, "Elasticsearch API responded with: #{err_msg}" end @property_hash = self.class.api_objects( resource[:protocol], resource[:host], resource[:port], resource[:timeout], resource[:username], resource[:password], resource[:ca_file], resource[:ca_path], validate_tls: resource[:validate_tls] ).find do |t| t[:name] == resource[:name] end end # Set this provider's `:ensure` property to `:present`. def create @property_flush[:ensure] = :present end def exists? @property_hash[:ensure] == :present end # Set this provider's `:ensure` property to `:absent`. def destroy @property_flush[:ensure] = :absent end end diff --git a/spec/helpers/acceptance/tests/basic_shared_examples.rb b/spec/helpers/acceptance/tests/basic_shared_examples.rb index 5d8ea76..cda6f92 100644 --- a/spec/helpers/acceptance/tests/basic_shared_examples.rb +++ b/spec/helpers/acceptance/tests/basic_shared_examples.rb @@ -1,71 +1,71 @@ # frozen_string_literal: true require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' shared_examples 'basic acceptance tests' do |es_config| include_examples('manifest application') describe package("elasticsearch#{v[:oss] ? '-oss' : ''}") do it { expect(subject).to be_installed. with_version(v[:elasticsearch_full_version]) } end %w[ /etc/elasticsearch /usr/share/elasticsearch /var/lib/elasticsearch ].each do |dir| describe file(dir) do it { is_expected.to be_directory } end end describe 'resources' do describe service('elasticsearch') do it { send(es_config.empty? ? :should_not : :should, be_enabled) } it { send(es_config.empty? ? :should_not : :should, be_running) } end unless es_config.empty? describe file(pid_file) do it { is_expected.to be_file } its(:content) { is_expected.to match(%r{[0-9]+}) } end describe file('/etc/elasticsearch/elasticsearch.yml') do it { is_expected.to be_file } it { is_expected.to contain "name: #{es_config['node.name']}" } end end unless es_config.empty? es_port = es_config['http.port'] describe port(es_port) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http("http://localhost:#{es_port}/_nodes/_local") do - it 'serves requests', :with_retries do - expect(response.status).to eq(200) - end + describe "http://localhost:#{es_port}/_nodes/_local" do + subject { shell("curl http://localhost:#{es_port}/_nodes/_local") } - it 'uses the default data path', :with_retries do - json = JSON.parse(response.body)['nodes'].values.first - data_dir = ['/var/lib/elasticsearch'] - expect( - json['settings']['path'] - ).to include( - 'data' => data_dir - ) - end + it 'serves requests', :with_retries do + expect(subject.exit_code).to eq(0) + end + + it 'uses the default data path', :with_retries do + json = JSON.parse(subject.stdout)['nodes'].values.first + data_dir = ['/var/lib/elasticsearch'] + expect( + json['settings']['path'] + ).to include( + 'data' => data_dir + ) end end end end end diff --git a/spec/helpers/acceptance/tests/datadir_shared_examples.rb b/spec/helpers/acceptance/tests/datadir_shared_examples.rb index efe3914..dec5765 100644 --- a/spec/helpers/acceptance/tests/datadir_shared_examples.rb +++ b/spec/helpers/acceptance/tests/datadir_shared_examples.rb @@ -1,77 +1,75 @@ # frozen_string_literal: true require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' shared_examples 'datadir directory validation' do |es_config, datapaths| include_examples('manifest application') describe file('/etc/elasticsearch/elasticsearch.yml') do it { is_expected.to be_file } datapaths.each do |datapath| it { is_expected.to contain datapath } end end datapaths.each do |datapath| describe file(datapath) do it { is_expected.to be_directory } end end es_port = es_config['http.port'] describe port(es_port) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http( - "http://localhost:#{es_port}/_nodes/_local" - ) do - it 'uses a custom data path' do - json = JSON.parse(response.body)['nodes'].values.first - expect( - json['settings']['path']['data'] - ).to(datapaths.one? && v[:elasticsearch_major_version] <= 2 ? eq(datapaths.first) : contain_exactly(*datapaths)) - end + describe "http://localhost:#{es_port}/_nodes/_local" do + subject { shell("curl http://localhost:#{es_port}/_nodes/_local") } + + it 'uses a custom data path' do + json = JSON.parse(subject.stdout)['nodes'].values.first + expect( + json['settings']['path']['data'] + ).to(datapaths.one? && v[:elasticsearch_major_version] <= 2 ? eq(datapaths.first) : contain_exactly(*datapaths)) end end end shared_examples 'datadir acceptance tests' do |es_config| describe 'elasticsearch::datadir' do let(:manifest_class_parameters) { 'restart_on_change => true' } context 'single path', :with_cleanup do let(:manifest_class_parameters) do <<-MANIFEST datadir => '/var/lib/elasticsearch-data', restart_on_change => true, MANIFEST end include_examples('datadir directory validation', es_config, ['/var/lib/elasticsearch-data']) end context 'multiple paths', :with_cleanup do let(:manifest_class_parameters) do <<-MANIFEST datadir => [ '/var/lib/elasticsearch-01', '/var/lib/elasticsearch-02' ], restart_on_change => true, MANIFEST end include_examples('datadir directory validation', es_config, ['/var/lib/elasticsearch-01', '/var/lib/elasticsearch-02']) end end end diff --git a/spec/helpers/acceptance/tests/pipeline_shared_examples.rb b/spec/helpers/acceptance/tests/pipeline_shared_examples.rb index 3256d7e..a12ad10 100644 --- a/spec/helpers/acceptance/tests/pipeline_shared_examples.rb +++ b/spec/helpers/acceptance/tests/pipeline_shared_examples.rb @@ -1,59 +1,57 @@ # frozen_string_literal: true require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' require 'helpers/acceptance/tests/bad_manifest_shared_examples' shared_examples 'pipeline operations' do |es_config, pipeline| describe 'pipeline resources' do let(:pipeline_name) { 'foo' } context 'present' do let(:extra_manifest) do <<-MANIFEST elasticsearch::pipeline { '#{pipeline_name}': ensure => 'present', content => #{pipeline} } MANIFEST end include_examples('manifest application') include_examples('pipeline content', es_config, pipeline) end context 'absent' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{pipeline_name}': ensure => absent, } MANIFEST end include_examples('manifest application') end end end # Verifies the content of a loaded index template. shared_examples 'pipeline content' do |es_config, pipeline| elasticsearch_port = es_config['http.port'] describe port(elasticsearch_port) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http( - "http://localhost:#{elasticsearch_port}/_ingest/pipeline" - ) do - it 'returns the configured pipelines', :with_retries do - expect(JSON.parse(response.body).values). - to include(include(pipeline)) - end + describe "http://localhost:#{elasticsearch_port}/_ingest/pipeline" do + subject { shell("curl http://localhost:#{elasticsearch_port}/_ingest/pipeline") } + + it 'returns the configured pipelines', :with_retries do + expect(JSON.parse(subject.stdout).values). + to include(include(pipeline)) end end end diff --git a/spec/helpers/acceptance/tests/plugin_api_shared_examples.rb b/spec/helpers/acceptance/tests/plugin_api_shared_examples.rb index baf52fa..1c020ed 100644 --- a/spec/helpers/acceptance/tests/plugin_api_shared_examples.rb +++ b/spec/helpers/acceptance/tests/plugin_api_shared_examples.rb @@ -1,23 +1,21 @@ # frozen_string_literal: true require 'json' shared_examples 'plugin API response' do |es_config, desc, val| describe port(es_config['http.port']) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http( - "http://localhost:#{es_config['http.port']}/_cluster/stats" - ) do - it desc, :with_retries do - expect( - JSON.parse(response.body)['nodes']['plugins'] - ).to include(include(val)) - end + describe "http://localhost:#{es_config['http.port']}/_cluster/stats" do + subject { shell("curl http://localhost:#{es_config['http.port']}/_cluster/stats") } + + it desc, :with_retries do + expect( + JSON.parse(subject.stdout)['nodes']['plugins'] + ).to include(include(val)) end end end diff --git a/spec/helpers/acceptance/tests/security_shared_examples.rb b/spec/helpers/acceptance/tests/security_shared_examples.rb index 84c8217..ba880c9 100644 --- a/spec/helpers/acceptance/tests/security_shared_examples.rb +++ b/spec/helpers/acceptance/tests/security_shared_examples.rb @@ -1,186 +1,181 @@ # frozen_string_literal: true require 'json' require 'spec_utilities' require 'helpers/acceptance/tests/manifest_shared_examples' shared_examples 'security plugin manifest' do |credentials| let(:extra_manifest) do users = credentials.map do |username, meta| <<-USER #{meta[:changed] ? "notify { 'password change for #{username}' : } ~>" : ''} elasticsearch::user { '#{username}': password => '#{meta[:hash] || meta[:plaintext]}', roles => #{meta[:roles].reduce({}) { |acc, elem| acc.merge(elem) }.keys}, } USER end.join("\n") roles = credentials.values.reduce({}) do |sum, user_metadata| # Collect all roles across users sum.merge user_metadata end[:roles] roles = roles.reduce({}) do |all_roles, role| all_roles.merge role end roles = roles.reject do |_role, permissions| permissions.empty? end roles = roles.map do |role, rights| <<-ROLE elasticsearch::role { '#{role}': privileges => #{rights} } ROLE end roles = roles.join("\n") <<-MANIFEST #{users} #{roles} MANIFEST end include_examples( 'manifest application', credentials.values.map { |p| p[:changed] }.none? ) end shared_examples 'secured request' do |test_desc, es_config, path, http_test, expected, user = nil, pass = nil| es_port = es_config['http.port'] describe port(es_port) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http( - "https://localhost:#{es_port}#{path}", - { - ssl: { verify: false } - }.merge(user && pass ? { basic_auth: [user, pass] } : {}) - ) do - it test_desc, :with_retries do - expect(http_test.call(response)).to eq(expected) - end + describe "https://localhost:#{es_port}#{path}" do + subject { shell("curl -k -u #{user}:#{pass} https://localhost:#{es_port}#{path}") } + + it test_desc, :with_retries do + expect(http_test.call(subject.stdout)).to eq(expected) end end end shared_examples 'security acceptance tests' do |es_config| describe 'security plugin operations', if: vault_available?, then_purge: true, with_license: true, with_certificates: true do rand_string = -> { [*('a'..'z')].sample(8).join } admin_user = rand_string.call admin_password = rand_string.call admin = { admin_user => { plaintext: admin_password, roles: [{ 'superuser' => [] }] } } let(:manifest_class_parameters) do <<-MANIFEST api_basic_auth_password => '#{admin_password}', api_basic_auth_username => '#{admin_user}', api_ca_file => '#{tls[:ca][:cert][:path]}', api_protocol => 'https', ca_certificate => '#{tls[:ca][:cert][:path]}', certificate => '#{tls[:clients].first[:cert][:path]}', keystore_password => '#{keystore_password}', license => file('#{v[:elasticsearch_license_path]}'), private_key => '#{tls[:clients].first[:key][:path]}', restart_on_change => true, ssl => true, validate_tls => true, MANIFEST end describe 'over tls' do user_one = rand_string.call user_two = rand_string.call user_one_pw = rand_string.call user_two_pw = rand_string.call describe 'user authentication' do username_passwords = { user_one => { plaintext: user_one_pw, roles: [{ 'superuser' => [] }] }, user_two => { plaintext: user_two_pw, roles: [{ 'superuser' => [] }] } }.merge(admin) username_passwords[user_two][:hash] = bcrypt(username_passwords[user_two][:plaintext]) include_examples('security plugin manifest', username_passwords) include_examples( 'secured request', 'denies unauthorized access', es_config, '/_cluster/health', ->(r) { r.status }, 401 ) include_examples( 'secured request', "permits user #{user_one} access", es_config, '/_cluster/health', ->(r) { r.status }, 200, user_one, user_one_pw ) include_examples( 'secured request', "permits user #{user_two} access", es_config, '/_cluster/health', ->(r) { r.status }, 200, user_two, user_two_pw ) end describe 'changing passwords' do new_password = rand_string.call username_passwords = { user_one => { plaintext: new_password, changed: true, roles: [{ 'superuser' => [] }] } } include_examples('security plugin manifest', username_passwords) include_examples( 'secured request', 'denies unauthorized access', es_config, '/_cluster/health', ->(r) { r.status }, 401 ) include_examples( 'secured request', "permits user #{user_two} access with new password", es_config, '/_cluster/health', ->(r) { r.status }, 200, user_one, new_password ) end describe 'roles' do password = rand_string.call username = rand_string.call user = { username => { plaintext: password, roles: [{ rand_string.call => { 'cluster' => [ 'cluster:monitor/health' ] } }] } } include_examples('security plugin manifest', user) include_examples( 'secured request', 'denies unauthorized access', es_config, '/_snapshot', ->(r) { r.status }, 403, username, password ) include_examples( 'secured request', 'permits authorized access', es_config, '/_cluster/health', ->(r) { r.status }, 200, username, password ) end end end end diff --git a/spec/helpers/acceptance/tests/snapshot_repository_shared_examples.rb b/spec/helpers/acceptance/tests/snapshot_repository_shared_examples.rb index b9af542..1dbbd73 100644 --- a/spec/helpers/acceptance/tests/snapshot_repository_shared_examples.rb +++ b/spec/helpers/acceptance/tests/snapshot_repository_shared_examples.rb @@ -1,83 +1,81 @@ # frozen_string_literal: true require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' # Main entrypoint for snapshot tests shared_examples 'snapshot repository acceptance tests' do describe 'elasticsearch::snapshot_repository', :with_cleanup do es_config = { 'http.port' => 9200, 'node.name' => 'elasticsearchSnapshot01', 'path.repo' => '/var/lib/elasticsearch' } # Override the manifest in order to populate 'path.repo' let(:manifest) do package = if v[:is_snapshot] <<-MANIFEST manage_repo => false, package_url => '#{v[:snapshot_package]}', MANIFEST else <<-MANIFEST # Hard version set here due to plugin incompatibilities. version => '#{v[:elasticsearch_full_version]}', MANIFEST end <<-MANIFEST api_timeout => 60, config => { 'cluster.name' => '#{v[:cluster_name]}', 'http.bind_host' => '0.0.0.0', #{es_config.map { |k, v| " '#{k}' => '#{v}'," }.join("\n")} }, jvm_options => [ '-Xms128m', '-Xmx128m', ], oss => #{v[:oss]}, #{package} MANIFEST end let(:manifest_class_parameters) { 'restart_on_change => true' } let(:extra_manifest) do <<-MANIFEST elasticsearch::snapshot_repository { 'backup': ensure => 'present', api_timeout => 60, location => '/var/lib/elasticsearch/backup', max_restore_rate => '20mb', max_snapshot_rate => '80mb', } MANIFEST end include_examples('manifest application', es_config) es_port = es_config['http.port'] describe port(es_port) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http( - "http://localhost:#{es_port}/_snapshot/backup" - ) do - it 'returns the snapshot repository', :with_retries do - expect(JSON.parse(response.body)['backup']). - to include('settings' => a_hash_including( - 'location' => '/var/lib/elasticsearch/backup', - 'max_restore_rate' => '20mb', - 'max_snapshot_rate' => '80mb' - )) - end + describe "http://localhost:#{es_port}/_snapshot/backup" do + subject { shell("curl http://localhost:#{es_port}/_snapshot/backup") } + + it 'returns the snapshot repository', :with_retries do + expect(JSON.parse(subject.stdout)['backup']). + to include('settings' => a_hash_including( + 'location' => '/var/lib/elasticsearch/backup', + 'max_restore_rate' => '20mb', + 'max_snapshot_rate' => '80mb' + )) end end end end diff --git a/spec/helpers/acceptance/tests/template_shared_examples.rb b/spec/helpers/acceptance/tests/template_shared_examples.rb index e0ae9c6..d6c5a52 100644 --- a/spec/helpers/acceptance/tests/template_shared_examples.rb +++ b/spec/helpers/acceptance/tests/template_shared_examples.rb @@ -1,113 +1,110 @@ # frozen_string_literal: true require 'json' require 'helpers/acceptance/tests/manifest_shared_examples' require 'helpers/acceptance/tests/bad_manifest_shared_examples' # Describes how to apply a manifest with a template, verify it, and clean it up shared_examples 'template application' do |es_config, name, template, param| context 'present' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{name}': ensure => 'present', #{param} } MANIFEST end include_examples('manifest application') include_examples('template content', es_config, template) end context 'absent' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{name}': ensure => absent, } MANIFEST end include_examples('manifest application') end end # Verifies the content of a loaded index template. shared_examples 'template content' do |es_config, template| elasticsearch_port = es_config['http.port'] describe port(elasticsearch_port) do it 'open', :with_retries do expect(subject).to be_listening end end - describe server :container do - describe http( - "http://localhost:#{elasticsearch_port}/_template", - params: { 'flat_settings' => 'false' } - ) do - it 'returns the installed template', :with_retries do - expect(JSON.parse(response.body).values). - to include(include(template)) - end + describe "http://localhost:#{elasticsearch_port}/_template" do + subject { shell("curl http://localhost:#{elasticsearch_port}/_template") } + + it 'returns the installed template', :with_retries do + expect(JSON.parse(subject.stdout).values). + to include(include(template)) end end end # Main entrypoint for template tests shared_examples 'template operations' do |es_config, template| describe 'template resources' do before :all do # rubocop:disable RSpec/BeforeAfterAll shell "mkdir -p #{default['distmoduledir']}/another/files" create_remote_file( default, "#{default['distmoduledir']}/another/files/good.json", JSON.dump(template) ) create_remote_file( default, "#{default['distmoduledir']}/another/files/bad.json", JSON.dump(template)[0..-5] ) end context 'configured through' do context '`source`' do include_examples( 'template application', es_config, SecureRandom.hex(8), template, "source => 'puppet:///modules/another/good.json'" ) end context '`content`' do include_examples( 'template application', es_config, SecureRandom.hex(8), template, "content => '#{JSON.dump(template)}'" ) end context 'bad json' do let(:extra_manifest) do <<-MANIFEST elasticsearch::template { '#{SecureRandom.hex(8)}': ensure => 'present', file => 'puppet:///modules/another/bad.json' } MANIFEST end include_examples('invalid manifest application') end end end end diff --git a/spec/support/acceptance/elastic.rb b/spec/support/acceptance/elastic.rb index b12f969..bb46bfe 100644 --- a/spec/support/acceptance/elastic.rb +++ b/spec/support/acceptance/elastic.rb @@ -1,221 +1,220 @@ # frozen_string_literal: true -require 'infrataster/rspec' require 'securerandom' require 'rspec/retry' +require 'simp/beaker_helpers' +include Simp::BeakerHelpers # rubocop:disable Style/MixinUsage + require_relative '../../spec_helper_tls' require_relative '../../spec_utilities' require_relative '../../../lib/puppet_x/elastic/deep_to_i' require_relative '../../../lib/puppet_x/elastic/deep_to_s' # def f # RSpec.configuration.fact # end +# FIXME: This value should better not be hardcoded +ENV['ELASTICSEARCH_VERSION'] = '7.10.1' +ENV.delete('BEAKER_debug') + run_puppet_install_helper('agent') unless ENV['BEAKER_provision'] == 'no' RSpec.configure do |c| # General-purpose spec-global variables c.add_setting :v, default: {} # Puppet debug logging v[:puppet_debug] = ENV['BEAKER_debug'] ? true : false unless ENV['snapshot_version'].nil? v[:snapshot_version] = ENV['snapshot_version'] v[:is_snapshot] = ENV['SNAPSHOT_TEST'] == 'true' end unless ENV['ELASTICSEARCH_VERSION'].nil? && v[:snapshot_version].nil? v[:elasticsearch_full_version] = ENV['ELASTICSEARCH_VERSION'] || v[:snapshot_version] v[:elasticsearch_major_version] = v[:elasticsearch_full_version].split('.').first.to_i v[:elasticsearch_package] = {} v[:template] = if v[:elasticsearch_major_version] == 6 JSON.parse(File.read('spec/fixtures/templates/6.x.json')) elsif v[:elasticsearch_major_version] >= 8 JSON.parse(File.read('spec/fixtures/templates/post_8.0.json')) else JSON.parse(File.read('spec/fixtures/templates/7.x.json')) end v[:template] = Puppet_X::Elastic.deep_to_i(Puppet_X::Elastic.deep_to_s(v[:template])) v[:pipeline] = JSON.parse(File.read('spec/fixtures/pipelines/example.json')) end v[:elasticsearch_plugins] = Dir[ artifact("*#{v[:elasticsearch_full_version]}.zip", ['plugins']) ].map do |plugin| plugin_filename = File.basename(plugin) plugin_name = plugin_filename.match(%r{^(?.+)-#{v[:elasticsearch_full_version]}.zip})[:name] [ plugin_name, { path: plugin, url: derive_plugin_urls_for(v[:elasticsearch_full_version], [plugin_name]).keys.first, }, ] end.to_h v[:oss] = !ENV['OSS_PACKAGE'].nil? and ENV['OSS_PACKAGE'] == 'true' v[:cluster_name] = SecureRandom.hex(10) # rspec-retry c.display_try_failure_messages = true c.default_sleep_interval = 10 # General-case retry keyword for unstable tests c.around :each, :with_retries do |example| example.run_with_retry retry: 10 end # Helper hook for module cleanup c.after :context, :with_cleanup do apply_manifest <<-MANIFEST class { 'elasticsearch': ensure => 'absent', manage_repo => true, oss => #{v[:oss]}, } file { '/usr/share/elasticsearch/plugin': ensure => 'absent', force => true, recurse => true, require => Class['elasticsearch'], } MANIFEST end c.before :context, :with_certificates do @keystore_password = SecureRandom.hex @role = [*('a'..'z')].sample(8).join # Setup TLS cert placement @tls = gen_certs(2, '/tmp') create_remote_file hosts, @tls[:ca][:cert][:path], @tls[:ca][:cert][:pem] @tls[:clients].each do |node| node.each do |_type, params| create_remote_file hosts, params[:path], params[:pem] end end end c.before :context, :with_license do Vault.address = ENV['VAULT_ADDR'] if ENV['CI'] Vault.auth.approle(ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID']) else Vault.auth.token(ENV['VAULT_TOKEN']) end licenses = Vault.with_retries(Vault::HTTPConnectionError) do Vault.logical.read(ENV['VAULT_PATH']) end.data raise 'No license found!' unless licenses # license = case v[:elasticsearch_major_version] # when 6 # licenses[:v5] # else # licenses[:v7] # end license = licenses[:v7] create_remote_file hosts, '/tmp/license.json', license v[:elasticsearch_license_path] = '/tmp/license.json' end c.after :context, :then_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end c.before :context, :first_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end # Provide a hook filter to spit out some ES logs if the example fails. c.after(:example, :logs_on_failure) do |example| if example.exception hosts.each do |host| on host, "find / -name '#{v[:cluster_name]}.log' | xargs cat || true" do |result| puts result.formatted_output end end end end end files_dir = ENV['files_dir'] || './spec/fixtures/artifacts' # General bootstrapping steps for each host hosts.each do |host| # # Set the host to 'aio' in order to adopt the puppet-agent style of # # installation, and configure paths/etc. # host[:type] = 'aio' # configure_defaults_on host, 'aio' if fact('os.family') == 'Suse' install_package host, '--force-resolution augeas-devel libxml2-devel ruby-devel' on host, 'gem install ruby-augeas --no-ri --no-rdoc' end v[:ext] = case fact('os.family') when 'Debian' 'deb' else 'rpm' end v[:elasticsearch_package]&.merge!( derive_full_package_url( v[:elasticsearch_full_version], [v[:ext]] ).flat_map do |url, filename| [[:url, url], [:filename, filename], [:path, artifact(filename)]] end.to_h ) - - Infrataster::Server.define(:docker) do |server| - server.address = host[:ip] - server.ssh = host[:ssh].tap { |s| s.delete :forward_agent } - end - Infrataster::Server.define(:container) do |server| - server.address = host[:vm_ip] # this gets ignored anyway - server.from = :docker - end end RSpec.configure do |c| if v[:is_snapshot] c.before :suite do scp_to default, "#{files_dir}/elasticsearch-snapshot.#{v[:ext]}", "/tmp/elasticsearch-snapshot.#{v[:ext]}" v[:snapshot_package] = "file:/tmp/elasticsearch-snapshot.#{v[:ext]}" end end c.before :suite do + fetch_archives(derive_artifact_urls_for(ENV['ELASTICSEARCH_VERSION'])) + # Use the Java class once before the suite of tests unless shell('command -v java', accept_all_exit_codes: true).exit_code.zero? java = case fact('os.name') when 'OpenSuSE' 'package => "java-1_8_0-openjdk-headless",' else '' end apply_manifest <<-MANIFEST class { "java" : distribution => "jdk", #{java} } MANIFEST end end end # # Java 8 is only easy to manage on recent distros # def v5x_capable? # (fact('os.family') == 'RedHat' and \ # not (fact('os.name') == 'OracleLinux' and \ # f['os']['release']['major'] == '6')) or \ # f.dig 'os', 'distro', 'codename' == 'xenial' # end