diff --git a/.travis.yml b/.travis.yml index 8e03da6..bd410f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,150 +1,126 @@ sudo: required group: deprecated-2017Q4 services: - docker cache: bundler: true directories: - spec/fixtures/artifacts - spec/fixtures/modules language: ruby script: travis_retry bundle exec rake $TASK jobs: allow_failures: - env: - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - TASK=beaker:centos-6-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-6-x64:snapshot - env: - TASK=beaker:centos-7-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-7-x64:snapshot - env: - TASK=beaker:debian-8-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-8-x64:snapshot - env: - TASK=beaker:debian-9-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-9-x64:snapshot include: - stage: intake env: - TASK=intake - - PUPPET_VERSION='~> 4.10.0' - - env: - - TASK=intake - env: - TASK=intake - PUPPET_VERSION='~> 5.0' - - env: - - TASK=intake - - PUPPET_VERSION='~> 6.0' - stage: acceptance env: - TASK=beaker:centos-6-x64:acceptance - env: - TASK=beaker:centos-6-x64:acceptance[5.6.9] - - env: - - TASK=beaker:centos-6-x64:acceptance[2.4.6] - env: - TASK=beaker:centos-7-x64:acceptance - env: - TASK=beaker:centos-7-x64:acceptance[5.6.9] - - env: - - TASK=beaker:centos-7-x64:acceptance[2.4.6] - env: - TASK=beaker:amazonlinux-1-x64:acceptance - env: - TASK=beaker:amazonlinux-1-x64:acceptance[5.6.9] - - env: - - TASK=beaker:amazonlinux-1-x64:acceptance[2.4.6] - env: - TASK=beaker:oracle-6-x64:acceptance - env: - TASK=beaker:oracle-6-x64:acceptance[5.6.9] - - env: - - TASK=beaker:oracle-6-x64:acceptance[2.4.6] - env: - TASK=beaker:oracle-7-x64:acceptance - env: - TASK=beaker:oracle-7-x64:acceptance[5.6.9] - - env: - - TASK=beaker:oracle-7-x64:acceptance[2.4.6] - env: - TASK=beaker:debian-8-x64:acceptance - env: - TASK=beaker:debian-8-x64:acceptance[5.6.9] - - env: - - TASK=beaker:debian-8-x64:acceptance[2.4.6] - env: - TASK=beaker:debian-9-x64:acceptance - env: - TASK=beaker:debian-9-x64:acceptance[5.6.9] - - env: - - TASK=beaker:debian-9-x64:acceptance[2.4.6] - env: - TASK=beaker:ubuntu-server-1404-x64:acceptance - env: - TASK=beaker:ubuntu-server-1404-x64:acceptance[5.6.9] - - env: - - TASK=beaker:ubuntu-server-1404-x64:acceptance[2.4.6] - env: - TASK=beaker:ubuntu-server-1604-x64:acceptance - env: - TASK=beaker:ubuntu-server-1604-x64:acceptance[5.6.9] - - env: - - TASK=beaker:ubuntu-server-1604-x64:acceptance[2.4.6] - stage: snapshots env: - TASK=beaker:ubuntu-server-1404-x64:snapshot env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - TASK=beaker:centos-6-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-6-x64:snapshot - env: - TASK=beaker:centos-7-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-7-x64:snapshot - env: - TASK=beaker:debian-8-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-8-x64:snapshot - env: - TASK=beaker:debian-9-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-9-x64:snapshot notifications: slack: secure: T1FO+ttrJNH+bXmNR6349qcttG68Qr1xmMqVVRnUr7+129GQO5174Z8MFC8ck0qOCZGHO7GCNO5seNFflrjF/5EKbdkmVnqhf9gVa9kN7I4psMzxJX9bp29xJA6m3wA4VqCosDKVFSfilDZujAblWT+KDHZLjP8sEWEnHPvCjf69S2XDQEWUoxZan5V9IJQas4XR+hMdIZTA3ChVrEyqRfeehAZImbAr/LH8zChZaTdHZQY7p2rN3+qVNi3+GISV9fNPpOCynnX/ACbdUaRt3+1etxGGaQMPzGmTejN3VlMw4OZRXImb6HQ2rXE+fNCASXiKiwylxTbriQsS0dFv4skxH03YlYM8pqaBpeIOwzf4n45tTzdAQZJMC5cOb+RvwS7qkAwuaVlVxiiA+MWRG/UcFpWS+iNn4KEKxbpBjYP8X1JIP9DlHLME7DNMM2pePv9X6ZjY6eDhVM1gbKi77dXOo5y2Sp0ru8QkLpIKFVXS01O+x7oDHHv2Osvih0jNMgM66Byso3KJYJ6EJ0D2/3Q9ZNpVM4CMuIY5pBQfXf691zqkBHI6JUnU6VMw97cH0k6Gq0ypZoW5trXdnRC5aEg4jKKid84zKmAeTpj/iMuagyb/a5msJstIVboynRtfDHR0J8WWhfSU2wzqKAb6L66iyRe62Fe8OGzLhk2+KNU= env: global: - secure: WFFcjwBIRBG2zyk4c8Ugq0tgI1YaH/+s5eV9h3i2kR1ggobT+nrNqn3hCOkmPtwGYPBNjVj6yp+7qy//MRe8AS2eo1XuMD/P4MYcDGmZiMnqPhz1UsLltGTYlh3y6jl9DJvNujFBQMnAu/ey2g/iWrcHdtl2qninvN3wOrXi2Bs= - secure: bvBaKoV5wBj2eQb4Zx8E2NaBDsMOyuHczRByVLNX5YqeuRWL9kcsUYzAUshFpd2GFa4tzfnSLKCp0+h3T4Uei5e8CjV5dx0VFmijXoZif0OJplRaJ+S3dJSluTV04NoE4u6l5Pg6kkFTMnAaApKVB4je2nSlgvrm/tuavhd9i0M= - secure: akshyW92CqV3Wt+rzQ3ScxIG55ILEaiwQ011rNF1kCXTds5HrHOGy++4VEidaTpems8OQH2+hCLK5r/7FXXgRQEV/TRYRGhp/y9mwqdioyDQ1D0yA3f42NWGNDGg2yOTTbhqQFJg394LDMiLmnevoiajEVIH+Ksr5bV/cIJc4Tc= diff --git a/Gemfile b/Gemfile index fc7ad61..4adb566 100644 --- a/Gemfile +++ b/Gemfile @@ -1,54 +1,54 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org' group :test do - gem 'puppet', (ENV['PUPPET_VERSION'] || '~> 4.10'), :require => false + gem 'puppet', (ENV['PUPPET_VERSION'] || '~> 6.0'), :require => false gem 'metadata-json-lint' gem 'specinfra', '~> 2.60' gem 'xmlrpc' gem 'ci_reporter_rspec' gem 'facter' gem 'pry' gem 'puppet-lint' gem 'puppet-strings' gem 'puppet-syntax' gem 'puppetlabs_spec_helper', '>= 2.7.0' gem 'rake' gem 'rspec', '~> 3.0' gem 'rspec-puppet', '~> 2.6' gem 'rspec-puppet-facts' gem 'rspec-puppet-utils' gem 'rspec-retry' # Required to test against Ruby 1.9 gem 'rubocop', '~> 0.41.2' gem 'rubysl-securerandom' gem 'webmock' # Extra Puppet-lint gems gem 'puppet-lint-appends-check', :git => 'https://github.com/voxpupuli/puppet-lint-appends-check', :ref => '07be8ce22d69353db055820b60bb77fe020238a6', :require => false gem 'puppet-lint-empty_string-check', :require => false gem 'puppet-lint-file_ensure-check', :require => false gem 'puppet-lint-leading_zero-check', :require => false gem 'puppet-lint-param-docs', :require => false gem 'puppet-lint-trailing_comma-check', :require => false gem 'puppet-lint-undef_in_function-check', :require => false gem 'puppet-lint-unquoted_string-check', :require => false gem 'puppet-lint-version_comparison-check', :require => false end group :development do gem 'puppet-blacksmith' end group :system_tests do gem 'bcrypt' gem 'beaker', '~> 3.7' gem 'beaker-rspec', '~> 6.0' gem 'docker-api', '~> 1.0' gem 'infrataster' gem 'vault' end diff --git a/Rakefile b/Rakefile index 797ee02..f453bd5 100644 --- a/Rakefile +++ b/Rakefile @@ -1,217 +1,217 @@ require 'digest/sha1' require 'rubygems' require 'puppetlabs_spec_helper/rake_tasks' require 'puppet_blacksmith/rake_tasks' require 'net/http' require 'uri' require 'fileutils' require 'rspec/core/rake_task' require 'puppet-strings' require 'puppet-strings/tasks' require 'yaml' require 'json' require_relative 'spec/spec_utilities' ENV['VAULT_APPROLE_ROLE_ID'] ||= '48adc137-3270-fc4a-ae65-1306919d4bb0' oss_package = ENV['OSS_PACKAGE'] and ENV['OSS_PACKAGE'] == 'true' # Workaround for certain rspec/beaker versions module TempFixForRakeLastComment def last_comment last_description end end Rake::Application.send :include, TempFixForRakeLastComment exclude_paths = [ 'pkg/**/*', 'vendor/**/*', 'spec/**/*' ] require 'puppet-lint/tasks/puppet-lint' require 'puppet-syntax/tasks/puppet-syntax' PuppetSyntax.exclude_paths = exclude_paths PuppetSyntax.future_parser = true if ENV['FUTURE_PARSER'] == 'true' %w[ 80chars class_inherits_from_params_class class_parameter_defaults single_quote_string_with_variable ].each do |check| PuppetLint.configuration.send("disable_#{check}") end PuppetLint.configuration.ignore_paths = exclude_paths PuppetLint.configuration.log_format = \ '%{path}:%{line}:%{check}:%{KIND}:%{message}' # Append custom cleanup tasks to :clean task :clean => [ :'artifact:clean', :spec_clean ] desc 'remove outdated module fixtures' task :spec_prune do mods = 'spec/fixtures/modules' fixtures = YAML.load_file '.fixtures.yml' fixtures['fixtures']['forge_modules'].each do |mod, params| next unless params.is_a? Hash \ and params.key? 'ref' \ and File.exist? "#{mods}/#{mod}" metadata = JSON.parse(File.read("#{mods}/#{mod}/metadata.json")) FileUtils.rm_rf "#{mods}/#{mod}" unless metadata['version'] == params['ref'] end end task :spec_prep => [:spec_prune] RSpec::Core::RakeTask.new(:spec_verbose) do |t| t.pattern = 'spec/{classes,defines,unit,functions,templates}/**/*_spec.rb' t.rspec_opts = [ '--format documentation', '--require "ci/reporter/rspec"', '--format CI::Reporter::RSpecFormatter', '--color' ] end task :spec_verbose => :spec_prep RSpec::Core::RakeTask.new(:spec_puppet) do |t| t.pattern = 'spec/{classes,defines,functions,templates,unit/facter}/**/*_spec.rb' t.rspec_opts = ['--color'] end task :spec_puppet => :spec_prep RSpec::Core::RakeTask.new(:spec_unit) do |t| t.pattern = 'spec/unit/{type,provider}/**/*_spec.rb' t.rspec_opts = ['--color'] end task :spec_unit => :spec_prep task :beaker => [:spec_prep] desc 'Run all linting/unit tests.' task :intake => [ :syntax, :rubocop, :lint, :validate, :spec_unit, :spec_puppet ] # Plumbing for snapshot tests desc 'Run the snapshot tests' RSpec::Core::RakeTask.new('beaker:snapshot', [:filter]) do |task, args| task.rspec_opts = ['--color'] task.pattern = 'spec/acceptance/tests/acceptance_spec.rb' task.rspec_opts = [] task.rspec_opts << '--format documentation' if ENV['CI'].nil? task.rspec_opts << "--example '#{args[:filter]}'" if args[:filter] ENV['SNAPSHOT_TEST'] = 'true' if Rake::Task.task_defined? 'artifact:snapshot:not_found' puts 'No snapshot artifacts found, skipping snapshot tests.' exit(0) end end beaker_node_sets.each do |node| desc "Run the snapshot tests against the #{node} nodeset" task "beaker:#{node}:snapshot", [:filter] => %w[ spec_prep artifact:snapshot:deb artifact:snapshot:rpm ] do |_task, args| ENV['BEAKER_set'] = node Rake::Task['beaker:snapshot'].reenable Rake::Task['beaker:snapshot'].invoke args[:filter] end desc "Run acceptance tests against #{node}" RSpec::Core::RakeTask.new( "beaker:#{node}:acceptance", [:version, :filter] => [:spec_prep] ) do |task, args| ENV['BEAKER_set'] = node - args.with_defaults(:version => '6.2.3', :filter => nil) + args.with_defaults(:version => '6.8.6', :filter => nil) task.pattern = 'spec/acceptance/tests/acceptance_spec.rb' task.rspec_opts = [] task.rspec_opts << '--format documentation' if ENV['CI'].nil? task.rspec_opts << "--example '#{args[:filter]}'" if args[:filter] ENV['ELASTICSEARCH_VERSION'] ||= args[:version] Rake::Task['artifact:fetch'].invoke(ENV['ELASTICSEARCH_VERSION']) end end namespace :artifact do desc 'Fetch specific installation artifacts' task :fetch, [:version] do |_t, args| fetch_archives( derive_artifact_urls_for(args[:version]) ) end namespace :snapshot do snapshot_version = JSON.parse(http_retry('https://artifacts-api.elastic.co/v1/versions'))['versions'].reject do |version| version.include? 'alpha' end.last ENV['snapshot_version'] = snapshot_version downloads = JSON.parse(http_retry("https://artifacts-api.elastic.co/v1/search/#{snapshot_version}/elasticsearch"))['packages'].select do |pkg, _| pkg =~ /(?:deb|rpm)/ and (oss_package ? pkg =~ /oss/ : pkg !~ /oss/) end.map do |package, urls| [package.split('.').last, urls] end.to_h # We end up with something like: # { # 'rpm' => {'url' => 'https://...', 'sha_url' => 'https://...'}, # 'deb' => {'url' => 'https://...', 'sha_url' => 'https://...'} # } # Note that checksums are currently broken on the Elastic unified release # side; once they start working we can verify them. if downloads.empty? puts 'No snapshot release available; skipping snapshot download' %w[deb rpm].each { |ext| task ext } task 'not_found' else # Download snapshot files downloads.each_pair do |extension, urls| filename = artifact urls['url'] checksum = artifact urls['sha_url'] link = artifact "elasticsearch-snapshot.#{extension}" FileUtils.rm link if File.exist? link task extension => link file link => filename do unless File.exist?(link) and File.symlink?(link) \ and File.readlink(link) == filename File.delete link if File.exist? link File.symlink File.basename(filename), link end end # file filename => checksum do file filename do get urls['url'], filename end task checksum do File.delete checksum if File.exist? checksum get urls['sha_url'], checksum end end end end desc 'Purge fetched artifacts' task :clean do FileUtils.rm_rf(Dir.glob('spec/fixtures/artifacts/*')) end end diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb index 63d60b6..6303656 100644 --- a/spec/spec_helper_acceptance.rb +++ b/spec/spec_helper_acceptance.rb @@ -1,283 +1,283 @@ require 'beaker-rspec' require 'securerandom' require 'thread' require 'infrataster/rspec' require 'rspec/retry' require 'vault' require_relative 'spec_helper_tls' require_relative 'spec_utilities' require_relative '../lib/puppet_x/elastic/deep_to_i' require_relative '../lib/puppet_x/elastic/deep_to_s' def f RSpec.configuration.fact end RSpec.configure do |c| # General-purpose spec-global variables c.add_setting :v, :default => {} unless ENV['snapshot_version'].nil? v[:snapshot_version] = ENV['snapshot_version'] v[:is_snapshot] = ENV['SNAPSHOT_TEST'] == 'true' end unless ENV['ELASTICSEARCH_VERSION'].nil? and v[:snapshot_version].nil? v[:elasticsearch_full_version] = ENV['ELASTICSEARCH_VERSION'] || v[:snapshot_version] v[:elasticsearch_major_version] = v[:elasticsearch_full_version].split('.').first.to_i v[:elasticsearch_package] = {} v[:template] = if v[:elasticsearch_major_version] < 6 JSON.load(File.new('spec/fixtures/templates/pre_6.0.json')) elsif v[:elasticsearch_major_version] >= 8 JSON.load(File.new('spec/fixtures/templates/post_8.0.json')) else JSON.load(File.new('spec/fixtures/templates/post_6.0.json')) end v[:template] = Puppet_X::Elastic.deep_to_i(Puppet_X::Elastic.deep_to_s(v[:template])) v[:pipeline] = JSON.load(File.new('spec/fixtures/pipelines/example.json')) v[:elasticsearch_plugins] = Dir[ artifact("*#{v[:elasticsearch_full_version]}.zip", ['plugins']) ].map do |plugin| plugin_filename = File.basename(plugin) plugin_name = plugin_filename.match(/^(?.+)-#{v[:elasticsearch_full_version]}.zip/)[:name] [ plugin_name, { :path => plugin, :url => derive_plugin_urls_for(v[:elasticsearch_full_version], [plugin_name]).keys.first } ] end.to_h end v[:oss] = (not ENV['OSS_PACKAGE'].nil?) and ENV['OSS_PACKAGE'] == 'true' v[:cluster_name] = SecureRandom.hex(10) # rspec-retry c.display_try_failure_messages = true - c.default_sleep_interval = 5 + c.default_sleep_interval = 10 # General-case retry keyword for unstable tests c.around :each, :with_retries do |example| example.run_with_retry retry: 4 end # More forgiving retry config for really flaky tests c.around :each, :with_generous_retries do |example| example.run_with_retry retry: 10 end # Helper hook for module cleanup c.after :context, :with_cleanup do apply_manifest <<-EOS class { 'elasticsearch': ensure => 'absent', manage_repo => true, oss => #{v[:oss]}, } elasticsearch::instance { 'es-01': ensure => 'absent' } file { '/usr/share/elasticsearch/plugin': ensure => 'absent', force => true, recurse => true, require => Class['elasticsearch'], } EOS end c.before :context, :with_certificates do @keystore_password = SecureRandom.hex @role = [*('a'..'z')].sample(8).join # Setup TLS cert placement @tls = gen_certs(2, '/tmp') create_remote_file hosts, @tls[:ca][:cert][:path], @tls[:ca][:cert][:pem] @tls[:clients].each do |node| node.each do |_type, params| create_remote_file hosts, params[:path], params[:pem] end end end c.before :context, :with_license do Vault.address = ENV['VAULT_ADDR'] Vault.auth.approle ENV['VAULT_APPROLE_ROLE_ID'], ENV['VAULT_APPROLE_SECRET_ID'] licenses = Vault.with_retries(Vault::HTTPConnectionError) do Vault.logical.read(ENV['VAULT_PATH']) end.data raise 'No license found!' unless licenses license = case v[:elasticsearch_major_version] when 2 licenses[:v2] else licenses[:v5] end create_remote_file hosts, '/tmp/license.json', license v[:elasticsearch_license_path] = '/tmp/license.json' end c.after :context, :then_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end c.before :context, :first_purge do shell 'rm -rf {/usr/share,/etc,/var/lib}/elasticsearch*' end # Provide a hook filter to spit out some ES logs if the example fails. c.after(:example, :logs_on_failure) do |example| if example.exception hosts.each do |host| on host, "find / -name '#{v[:cluster_name]}.log' | xargs cat || true" do |result| puts result.formatted_output end end end end end files_dir = ENV['files_dir'] || './spec/fixtures/artifacts' # General bootstrapping steps for each host hosts.each do |host| # Set the host to 'aio' in order to adopt the puppet-agent style of # installation, and configure paths/etc. host[:type] = 'aio' configure_defaults_on host, 'aio' # Install Puppet # # We spawn a thread to print dots periodically while installing puppet to # avoid inactivity timeouts in Travis. Don't judge me. progress = Thread.new do print 'Installing puppet..' print '.' while sleep 5 end case host.name when /debian-9/ # A few special cases need to be installed from gems (if the distro is # very new and has no puppet repo package or has no upstream packages). install_puppet_from_gem( host, version: Gem.loaded_specs['puppet'].version ) else # Otherwise, just use the all-in-one agent package. install_puppet_agent_on( host, puppet_agent_version: to_agent_version(Gem.loaded_specs['puppet'].version) ) end # Quit the print thread and include some debugging. progress.exit puts "done. Installed version #{shell('puppet --version').output}" RSpec.configure do |c| c.add_setting :fact, :default => JSON.parse(fact('', '-j')) end if f['os']['family'] == 'Suse' install_package host, '--force-resolution augeas-devel libxml2-devel ruby-devel' on host, 'gem install ruby-augeas --no-ri --no-rdoc' end v[:ext] = case f['os']['family'] when 'Debian' 'deb' else 'rpm' end if v[:elasticsearch_package] v[:elasticsearch_package].merge!( derive_full_package_url( v[:elasticsearch_full_version], [v[:ext]] ).flat_map do |url, filename| [[:url, url], [:filename, filename], [:path, artifact(filename)]] end.to_h ) end Infrataster::Server.define(:docker) do |server| server.address = host[:ip] server.ssh = host[:ssh].tap { |s| s.delete :forward_agent } end Infrataster::Server.define(:container) do |server| server.address = host[:vm_ip] # this gets ignored anyway server.from = :docker end end RSpec.configure do |c| if v[:is_snapshot] c.before :suite do scp_to default, "#{files_dir}/elasticsearch-snapshot.#{v[:ext]}", "/tmp/elasticsearch-snapshot.#{v[:ext]}" v[:snapshot_package] = "file:/tmp/elasticsearch-snapshot.#{v[:ext]}" end end c.before :suite do # Install module and dependencies install_dev_puppet_module :ignore_list => [ 'junit' ] + Beaker::DSL::InstallUtils::ModuleUtils::PUPPET_MODULE_INSTALL_IGNORE hosts.each do |host| modules = %w[archive augeas_core datacat java java_ks stdlib elastic_stack] dist_module = { 'Debian' => ['apt'], 'Suse' => ['zypprepo'], 'RedHat' => %w[concat yumrepo_core] }[f['os']['family']] modules += dist_module unless dist_module.nil? modules.each do |mod| copy_module_to( host, :module_name => mod, :source => "spec/fixtures/modules/#{mod}" ) end on(host, 'mkdir -p etc/puppet/modules/another/files/') # Apt doesn't update package caches sometimes, ensure we're caught up. shell 'apt-get update' if f['os']['family'] == 'Debian' end # Use the Java class once before the suite of tests unless shell('command -v java', :accept_all_exit_codes => true).exit_code.zero? java = case f['os']['name'] when 'OpenSuSE' 'package => "java-1_8_0-openjdk-headless",' else '' end apply_manifest <<-MANIFEST class { "java" : distribution => "jre", #{java} } MANIFEST end end end # Java 8 is only easy to manage on recent distros def v5x_capable? (f['os']['family'] == 'RedHat' and \ not (f['os']['name'] == 'OracleLinux' and \ f['os']['release']['major'] == '6')) or \ f.dig 'os', 'distro', 'codename' == 'xenial' end