diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 7a11b60..32d2a02 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -1,255 +1,259 @@ require 'beaker-rspec' require 'beaker/puppet_install_helper' require 'securerandom' require 'thread' require 'infrataster/rspec' require 'rspec/retry' require 'vault' require 'simp/beaker_helpers' include Simp::BeakerHelpers require_relative 'spec_helper_tls' require_relative 'spec_utilities' require_relative '../lib/puppet_x/elastic/deep_to_i' require_relative '../lib/puppet_x/elastic/deep_to_s' # def f # RSpec.configuration.fact # end run_puppet_install_helper('agent') unless ENV['BEAKER_provision'] == 'no' RSpec.configure do |c| # General-purpose spec-global variables c.add_setting :v, :default => {} # Puppet debug logging v[:puppet_debug] = ENV['BEAKER_debug'] ? true : false unless ENV['snapshot_version'].nil? v[:snapshot_version] = ENV['snapshot_version'] v[:is_snapshot] = ENV['SNAPSHOT_TEST'] == 'true' end unless ENV['ELASTICSEARCH_VERSION'].nil? and v[:snapshot_version].nil? v[:elasticsearch_full_version] = ENV['ELASTICSEARCH_VERSION'] || v[:snapshot_version] v[:elasticsearch_major_version] = v[:elasticsearch_full_version].split('.').first.to_i v[:elasticsearch_package] = {} v[:template] = if v[:elasticsearch_major_version] == 6 JSON.load(File.new('spec/fixtures/templates/6.x.json')) elsif v[:elasticsearch_major_version] >= 8 JSON.load(File.new('spec/fixtures/templates/post_8.0.json')) else JSON.load(File.new('spec/fixtures/templates/7.x.json')) end v[:template] = Puppet_X::Elastic.deep_to_i(Puppet_X::Elastic.deep_to_s(v[:template])) v[:pipeline] = JSON.load(File.new('spec/fixtures/pipelines/example.json')) v[:elasticsearch_plugins] = Dir[ artifact("*#{v[:elasticsearch_full_version]}.zip", ['plugins']) ].map do |plugin| plugin_filename = File.basename(plugin) plugin_name = plugin_filename.match(/^(?.+)-#{v[:elasticsearch_full_version]}.zip/)[:name] [ plugin_name, { :path => plugin, :url => derive_plugin_urls_for(v[:elasticsearch_full_version], [plugin_name]).keys.first } ] end.to_h end v[:oss] = (not ENV['OSS_PACKAGE'].nil?) and ENV['OSS_PACKAGE'] == 'true' v[:cluster_name] = SecureRandom.hex(10) # rspec-retry c.display_try_failure_messages = true c.default_sleep_interval = 10 # General-case retry keyword for unstable tests c.around :each, :with_retries do |example| example.run_with_retry retry: 10 end # Helper hook for module cleanup c.after :context, :with_cleanup do apply_manifest <<-EOS class { 'elasticsearch': ensure => 'absent', manage_repo => true, oss => #{v[:oss]}, } file { '/usr/share/elasticsearch/plugin': ensure => 'absent', force => true, recurse => true, require => Class['elasticsearch'], } EOS end c.before :context, :with_certificates do @keystore_password = SecureRandom.hex @role = [*('a'..'z')].sample(8).join # Setup TLS cert placement @tls = gen_certs(2, '/tmp') create_remote_file hosts, @tls[:ca][:cert][:path], @tls[:ca][:cert][:pem] @tls[:clients].each do |node| node.each do |_type, params| create_remote_file hosts, params[:path], params[:pem] end end end c.before :context, :with_license do Vault.address = ENV['VAULT_ADDR'] - Vault.auth.approle ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID'] + if ENV['CI'] + Vault.auth.approle(ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID']) + else + Vault.auth.token(ENV['VAULT_TOKEN']) + end licenses = Vault.with_retries(Vault::HTTPConnectionError) do Vault.logical.read(ENV['VAULT_PATH']) end.data raise 'No license found!' unless licenses license = case v[:elasticsearch_major_version] when 2 licenses[:v2] else licenses[:v5] end create_remote_file hosts, '/tmp/license.json', license v[:elasticsearch_license_path] = '/tmp/license.json' end c.after :context, :then_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end c.before :context, :first_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end # Provide a hook filter to spit out some ES logs if the example fails. c.after(:example, :logs_on_failure) do |example| if example.exception hosts.each do |host| on host, "find / -name '#{v[:cluster_name]}.log' | xargs cat || true" do |result| puts result.formatted_output end end end end end files_dir = ENV['files_dir'] || './spec/fixtures/artifacts' # General bootstrapping steps for each host hosts.each do |host| # # Set the host to 'aio' in order to adopt the puppet-agent style of # # installation, and configure paths/etc. # host[:type] = 'aio' # configure_defaults_on host, 'aio' if fact('os.family') == 'Suse' install_package host, '--force-resolution augeas-devel libxml2-devel ruby-devel' on host, 'gem install ruby-augeas --no-ri --no-rdoc' end v[:ext] = case fact('os.family') when 'Debian' 'deb' else 'rpm' end if v[:elasticsearch_package] v[:elasticsearch_package].merge!( derive_full_package_url( v[:elasticsearch_full_version], [v[:ext]] ).flat_map do |url, filename| [[:url, url], [:filename, filename], [:path, artifact(filename)]] end.to_h ) end Infrataster::Server.define(:docker) do |server| server.address = host[:ip] server.ssh = host[:ssh].tap { |s| s.delete :forward_agent } end Infrataster::Server.define(:container) do |server| server.address = host[:vm_ip] # this gets ignored anyway server.from = :docker end end RSpec.configure do |c| if v[:is_snapshot] c.before :suite do scp_to default, "#{files_dir}/elasticsearch-snapshot.#{v[:ext]}", "/tmp/elasticsearch-snapshot.#{v[:ext]}" v[:snapshot_package] = "file:/tmp/elasticsearch-snapshot.#{v[:ext]}" end end c.before :suite do # Install module and dependencies install_dev_puppet_module :ignore_list => [ 'junit' ] + Beaker::DSL::InstallUtils::ModuleUtils::PUPPET_MODULE_INSTALL_IGNORE hosts.each do |host| modules = %w[archive augeas_core datacat java java_ks stdlib elastic_stack] dist_module = { 'Debian' => ['apt'], 'Suse' => ['zypprepo'], 'RedHat' => %w[concat yumrepo_core] }[fact('os.family')] modules += dist_module unless dist_module.nil? modules.each do |mod| copy_module_to( host, :module_name => mod, :source => "spec/fixtures/modules/#{mod}" ) end on(host, 'mkdir -p etc/puppet/modules/another/files/') # Apt doesn't update package caches sometimes, ensure we're caught up. shell 'apt-get update' if fact('os.family') == 'Debian' end # Use the Java class once before the suite of tests unless shell('command -v java', :accept_all_exit_codes => true).exit_code.zero? java = case fact('os.name') when 'OpenSuSE' 'package => "java-1_8_0-openjdk-headless",' else '' end apply_manifest <<-MANIFEST class { "java" : distribution => "jre", #{java} } MANIFEST end end end # # Java 8 is only easy to manage on recent distros # def v5x_capable? # (fact('os.family') == 'RedHat' and \ # not (fact('os.name') == 'OracleLinux' and \ # f['os']['release']['major'] == '6')) or \ # f.dig 'os', 'distro', 'codename' == 'xenial' # end diff --git a/spec/spec_utilities.rb b/spec/spec_utilities.rb index b135dca..489bd6a 100644 --- a/spec/spec_utilities.rb +++ b/spec/spec_utilities.rb @@ -1,130 +1,134 @@ require 'bcrypt' require 'open-uri' def to_agent_version(puppet_version) # REF: https://docs.puppet.com/puppet/latest/reference/about_agent.html { # Puppet => Agent '4.10.4' => '1.10.4', '4.10.3' => '1.10.3', '4.10.2' => '1.10.2', '4.10.1' => '1.10.1', '4.10.0' => '1.10.0', '4.9.4' => '1.9.3', '4.8.2' => '1.8.3', '4.7.1' => '1.7.2', '4.7.0' => '1.7.1', '4.6.2' => '1.6.2', '4.5.3' => '1.5.3', '4.4.2' => '1.4.2', '4.4.1' => '1.4.1', '4.4.0' => '1.4.0', '4.3.2' => '1.3.6', '4.3.1' => '1.3.2', '4.3.0' => '1.3.0', '4.2.3' => '1.2.7', '4.2.2' => '1.2.6', '4.2.1' => '1.2.2', '4.2.0' => '1.2.1', '4.1.0' => '1.1.1', '4.0.0' => '1.0.1' }[puppet_version] end def derive_artifact_urls_for(full_version, plugins = ['analysis-icu']) derive_full_package_url(full_version).merge( derive_plugin_urls_for(full_version, plugins) ) end def derive_full_package_url(full_version, extensions = %w[deb rpm]) extensions.map do |ext| url = if full_version.start_with? '6' "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}.#{ext}" elsif ext == 'deb' "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}-amd64.#{ext}" else "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-#{full_version}-x86_64.#{ext}" end [url, File.basename(url)] end.to_h end def derive_plugin_urls_for(full_version, plugins = ['analysis-icu']) plugins.map do |plugin| url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/#{plugin}/#{plugin}-#{full_version}.zip" [url, File.join('plugins', File.basename(url))] end.to_h end def artifact(file, fixture_path = []) File.join(%w[spec fixtures artifacts] + fixture_path + [File.basename(file)]) end def get(url, file_path) puts "Fetching #{url}..." found = false until found uri = URI.parse(url) conn = Net::HTTP.new(uri.host, uri.port) conn.use_ssl = true res = conn.get(uri.path) if res.header['location'] url = res.header['location'] else found = true end end File.open(file_path, 'w+') { |fh| fh.write res.body } end def fetch_archives(archives) archives.each do |url, orig_fp| fp = "spec/fixtures/artifacts/#{orig_fp}" if File.exist? fp if fp.end_with? 'tar.gz' and !system("tar -tzf #{fp} &>/dev/null") puts "Archive #{fp} corrupt, re-fetching..." File.delete fp else puts "Already retrieved intact archive #{fp}..." next end end get url, fp end end def pid_file if fact('operatingsystem') == 'Debian' \ and fact('lsbmajdistrelease').to_i <= 7 '/var/run/elasticsearch.pid' else '/var/run/elasticsearch/elasticsearch.pid' end end def vault_available? - %w[VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID VAULT_PATH].select do |var| - ENV[var].nil? - end.empty? + if ENV['CI'] + %w[VAULT_ADDR VAULT_APPROLE_ROLE_ID VAULT_APPROLE_SECRET_ID VAULT_PATH].select do |var| + ENV[var].nil? + end.empty? + else + true + end end def http_retry(url) retries ||= 0 open(url).read rescue retry if (retries += 1) < 3 end # Helper to store arbitrary testing setting values def v RSpec.configuration.v end def semver(version) Gem::Version.new version end def bcrypt(value) BCrypt::Password.create(value) end