diff --git a/.travis.yml b/.travis.yml index 664d430..1c87507 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,174 +1,174 @@ sudo: required # group: deprecated-2017Q4 services: - docker cache: bundler: true directories: - spec/fixtures/artifacts - spec/fixtures/modules language: ruby script: travis_retry bundle exec rake $TASK jobs: allow_failures: - env: - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - TASK=beaker:centos-6-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-6-x64:snapshot - env: - TASK=beaker:centos-7-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-7-x64:snapshot - env: - TASK=beaker:debian-8-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-8-x64:snapshot - env: - TASK=beaker:debian-9-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-9-x64:snapshot include: - stage: intake env: - TASK=intake - env: - TASK=intake - PUPPET_VERSION='~> 5.0' - stage: acceptance env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:amazonlinux-1-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:amazonlinux-1-x64:acceptance[7.8.0] + - TASK=beaker:amazonlinux-1-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:amazonlinux-2-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:amazonlinux-2-x64:acceptance[7.8.0] + - TASK=beaker:amazonlinux-2-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:centos-6-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:centos-6-x64:acceptance[7.8.0] + - TASK=beaker:centos-6-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:centos-7-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:centos-7-x64:acceptance[7.8.0] + - TASK=beaker:centos-7-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:centos-8-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:centos-8-x64:acceptance[7.8.0] + - TASK=beaker:centos-8-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:oracle-6-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:oracle-6-x64:acceptance[7.8.0] + - TASK=beaker:oracle-6-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:oracle-7-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:oracle-7-x64:acceptance[7.8.0] + - TASK=beaker:oracle-7-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:debian-8-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:debian-8-x64:acceptance[7.8.0] + - TASK=beaker:debian-8-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:debian-9-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:debian-9-x64:acceptance[7.8.0] + - TASK=beaker:debian-9-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:debian-10-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:debian-10-x64:acceptance[7.8.0] + - TASK=beaker:debian-10-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:ubuntu-server-1404-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:ubuntu-server-1404-x64:acceptance[7.8.0] + - TASK=beaker:ubuntu-server-1404-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:ubuntu-server-1604-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:ubuntu-server-1604-x64:acceptance[7.8.0] + - TASK=beaker:ubuntu-server-1604-x64:acceptance[6.8.12] - env: - BEAKER_PUPPET_COLLECTION=puppet6 - TASK=beaker:ubuntu-server-1804-x64:acceptance - env: - BEAKER_PUPPET_COLLECTION=puppet6 - - TASK=beaker:ubuntu-server-1804-x64:acceptance[7.8.0] + - TASK=beaker:ubuntu-server-1804-x64:acceptance[6.8.12] # - env: # - BEAKER_PUPPET_COLLECTION=puppet6 # - TASK=beaker:ubuntu-server-2004-x64:acceptance # - env: # - BEAKER_PUPPET_COLLECTION=puppet6 - # - TASK=beaker:ubuntu-server-2004-x64:acceptance[7.8.0] + # - TASK=beaker:ubuntu-server-2004-x64:acceptance[6.8.12] - stage: snapshots env: - TASK=beaker:ubuntu-server-1404-x64:snapshot env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1404-x64:snapshot - env: - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:ubuntu-server-1604-x64:snapshot - env: - TASK=beaker:centos-6-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-6-x64:snapshot - env: - TASK=beaker:centos-7-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:centos-7-x64:snapshot - env: - TASK=beaker:debian-8-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-8-x64:snapshot - env: - TASK=beaker:debian-9-x64:snapshot - env: - OSS_PACKAGE=true - TASK=beaker:debian-9-x64:snapshot notifications: slack: secure: T1FO+ttrJNH+bXmNR6349qcttG68Qr1xmMqVVRnUr7+129GQO5174Z8MFC8ck0qOCZGHO7GCNO5seNFflrjF/5EKbdkmVnqhf9gVa9kN7I4psMzxJX9bp29xJA6m3wA4VqCosDKVFSfilDZujAblWT+KDHZLjP8sEWEnHPvCjf69S2XDQEWUoxZan5V9IJQas4XR+hMdIZTA3ChVrEyqRfeehAZImbAr/LH8zChZaTdHZQY7p2rN3+qVNi3+GISV9fNPpOCynnX/ACbdUaRt3+1etxGGaQMPzGmTejN3VlMw4OZRXImb6HQ2rXE+fNCASXiKiwylxTbriQsS0dFv4skxH03YlYM8pqaBpeIOwzf4n45tTzdAQZJMC5cOb+RvwS7qkAwuaVlVxiiA+MWRG/UcFpWS+iNn4KEKxbpBjYP8X1JIP9DlHLME7DNMM2pePv9X6ZjY6eDhVM1gbKi77dXOo5y2Sp0ru8QkLpIKFVXS01O+x7oDHHv2Osvih0jNMgM66Byso3KJYJ6EJ0D2/3Q9ZNpVM4CMuIY5pBQfXf691zqkBHI6JUnU6VMw97cH0k6Gq0ypZoW5trXdnRC5aEg4jKKid84zKmAeTpj/iMuagyb/a5msJstIVboynRtfDHR0J8WWhfSU2wzqKAb6L66iyRe62Fe8OGzLhk2+KNU= env: global: - secure: WFFcjwBIRBG2zyk4c8Ugq0tgI1YaH/+s5eV9h3i2kR1ggobT+nrNqn3hCOkmPtwGYPBNjVj6yp+7qy//MRe8AS2eo1XuMD/P4MYcDGmZiMnqPhz1UsLltGTYlh3y6jl9DJvNujFBQMnAu/ey2g/iWrcHdtl2qninvN3wOrXi2Bs= - secure: bvBaKoV5wBj2eQb4Zx8E2NaBDsMOyuHczRByVLNX5YqeuRWL9kcsUYzAUshFpd2GFa4tzfnSLKCp0+h3T4Uei5e8CjV5dx0VFmijXoZif0OJplRaJ+S3dJSluTV04NoE4u6l5Pg6kkFTMnAaApKVB4je2nSlgvrm/tuavhd9i0M= - secure: akshyW92CqV3Wt+rzQ3ScxIG55ILEaiwQ011rNF1kCXTds5HrHOGy++4VEidaTpems8OQH2+hCLK5r/7FXXgRQEV/TRYRGhp/y9mwqdioyDQ1D0yA3f42NWGNDGg2yOTTbhqQFJg394LDMiLmnevoiajEVIH+Ksr5bV/cIJc4Tc= diff --git a/Rakefile b/Rakefile index cc9b964..715ae18 100644 --- a/Rakefile +++ b/Rakefile @@ -1,217 +1,219 @@ require 'digest/sha1' require 'rubygems' require 'puppetlabs_spec_helper/rake_tasks' require 'puppet_blacksmith/rake_tasks' require 'net/http' require 'uri' require 'fileutils' require 'rspec/core/rake_task' require 'puppet-strings' require 'puppet-strings/tasks' require 'yaml' require 'json' require_relative 'spec/spec_utilities' ENV['VAULT_APPROLE_ROLE_ID'] ||= '48adc137-3270-fc4a-ae65-1306919d4bb0' oss_package = ENV['OSS_PACKAGE'] and ENV['OSS_PACKAGE'] == 'true' +elasticsearch_default_version = '7.10.1' + # Workaround for certain rspec/beaker versions module TempFixForRakeLastComment def last_comment last_description end end Rake::Application.send :include, TempFixForRakeLastComment exclude_paths = [ 'pkg/**/*', 'vendor/**/*', 'spec/**/*' ] require 'puppet-lint/tasks/puppet-lint' require 'puppet-syntax/tasks/puppet-syntax' PuppetSyntax.exclude_paths = exclude_paths PuppetSyntax.future_parser = true if ENV['FUTURE_PARSER'] == 'true' %w[ 80chars class_inherits_from_params_class class_parameter_defaults single_quote_string_with_variable ].each do |check| PuppetLint.configuration.send("disable_#{check}") end PuppetLint.configuration.ignore_paths = exclude_paths PuppetLint.configuration.log_format = \ '%{path}:%{line}:%{check}:%{KIND}:%{message}' # Append custom cleanup tasks to :clean task :clean => [ :'artifact:clean', :spec_clean ] desc 'remove outdated module fixtures' task :spec_prune do mods = 'spec/fixtures/modules' fixtures = YAML.load_file '.fixtures.yml' fixtures['fixtures']['forge_modules'].each do |mod, params| next unless params.is_a? Hash \ and params.key? 'ref' \ and File.exist? "#{mods}/#{mod}" metadata = JSON.parse(File.read("#{mods}/#{mod}/metadata.json")) FileUtils.rm_rf "#{mods}/#{mod}" unless metadata['version'] == params['ref'] end end task :spec_prep => [:spec_prune] RSpec::Core::RakeTask.new(:spec_verbose) do |t| t.pattern = 'spec/{classes,defines,unit,functions,templates}/**/*_spec.rb' t.rspec_opts = [ '--format documentation', '--require "ci/reporter/rspec"', '--format CI::Reporter::RSpecFormatter', '--color' ] end task :spec_verbose => :spec_prep RSpec::Core::RakeTask.new(:spec_puppet) do |t| t.pattern = 'spec/{classes,defines,functions,templates,unit/facter}/**/*_spec.rb' t.rspec_opts = ['--color'] end task :spec_puppet => :spec_prep RSpec::Core::RakeTask.new(:spec_unit) do |t| t.pattern = 'spec/unit/{type,provider}/**/*_spec.rb' t.rspec_opts = ['--color'] end task :spec_unit => :spec_prep task :beaker => [:spec_prep] desc 'Run all linting/unit tests.' task :intake => [ :syntax, :rubocop, :lint, :validate, :spec_unit, :spec_puppet ] # Plumbing for snapshot tests desc 'Run the snapshot tests' RSpec::Core::RakeTask.new('beaker:snapshot', [:filter]) do |task, args| task.rspec_opts = ['--color'] task.pattern = 'spec/acceptance/tests/acceptance_spec.rb' task.rspec_opts = [] task.rspec_opts << '--format documentation' if ENV['CI'].nil? task.rspec_opts << "--example '#{args[:filter]}'" if args[:filter] ENV['SNAPSHOT_TEST'] = 'true' if Rake::Task.task_defined? 'artifact:snapshot:not_found' puts 'No snapshot artifacts found, skipping snapshot tests.' exit(0) end end beaker_node_sets.each do |node| desc "Run the snapshot tests against the #{node} nodeset" task "beaker:#{node}:snapshot", [:filter] => %w[ spec_prep artifact:snapshot:deb artifact:snapshot:rpm ] do |_task, args| ENV['BEAKER_set'] = node Rake::Task['beaker:snapshot'].reenable Rake::Task['beaker:snapshot'].invoke args[:filter] end desc "Run acceptance tests against #{node}" RSpec::Core::RakeTask.new( "beaker:#{node}:acceptance", [:version, :filter] => [:spec_prep] ) do |task, args| ENV['BEAKER_set'] = node - args.with_defaults(:version => '6.8.6', :filter => nil) + args.with_defaults(:version => elasticsearch_default_version, :filter => nil) task.pattern = 'spec/acceptance/tests/acceptance_spec.rb' task.rspec_opts = [] task.rspec_opts << '--format documentation' task.rspec_opts << "--example '#{args[:filter]}'" if args[:filter] ENV['ELASTICSEARCH_VERSION'] ||= args[:version] Rake::Task['artifact:fetch'].invoke(ENV['ELASTICSEARCH_VERSION']) end end namespace :artifact do desc 'Fetch specific installation artifacts' task :fetch, [:version] do |_t, args| fetch_archives( derive_artifact_urls_for(args[:version]) ) end namespace :snapshot do snapshot_version = JSON.parse(http_retry('https://artifacts-api.elastic.co/v1/versions'))['versions'].reject do |version| version.include? 'alpha' end.last ENV['snapshot_version'] = snapshot_version downloads = JSON.parse(http_retry("https://artifacts-api.elastic.co/v1/search/#{snapshot_version}/elasticsearch"))['packages'].select do |pkg, _| pkg =~ /(?:deb|rpm)/ and (oss_package ? pkg =~ /oss/ : pkg !~ /oss/) end.map do |package, urls| [package.split('.').last, urls] end.to_h # We end up with something like: # { # 'rpm' => {'url' => 'https://...', 'sha_url' => 'https://...'}, # 'deb' => {'url' => 'https://...', 'sha_url' => 'https://...'} # } # Note that checksums are currently broken on the Elastic unified release # side; once they start working we can verify them. if downloads.empty? puts 'No snapshot release available; skipping snapshot download' %w[deb rpm].each { |ext| task ext } task 'not_found' else # Download snapshot files downloads.each_pair do |extension, urls| filename = artifact urls['url'] checksum = artifact urls['sha_url'] link = artifact "elasticsearch-snapshot.#{extension}" FileUtils.rm link if File.exist? link task extension => link file link => filename do unless File.exist?(link) and File.symlink?(link) \ and File.readlink(link) == filename File.delete link if File.exist? link File.symlink File.basename(filename), link end end # file filename => checksum do file filename do get urls['url'], filename end task checksum do File.delete checksum if File.exist? checksum get urls['sha_url'], checksum end end end end desc 'Purge fetched artifacts' task :clean do FileUtils.rm_rf(Dir.glob('spec/fixtures/artifacts/*')) end end