diff --git a/lib/puppet/type/archive.rb b/lib/puppet/type/archive.rb index b3c85fb..b6285c6 100644 --- a/lib/puppet/type/archive.rb +++ b/lib/puppet/type/archive.rb @@ -1,284 +1,284 @@ require 'pathname' require 'uri' require 'puppet/util' require 'puppet/parameter/boolean' Puppet::Type.newtype(:archive) do @doc = 'Manage archive file download, extraction, and cleanup.' ensurable do desc 'whether archive file should be present/absent (default: present)' newvalue(:present) do provider.create end newvalue(:absent) do provider.destroy end defaultto(:present) # The following changes allows us to notify if the resource is being replaced def is_to_s(value) # rubocop:disable Style/PredicateName return "(#{resource[:checksum_type]})#{provider.archive_checksum}" if provider.archive_checksum super end def should_to_s(value) return "(#{resource[:checksum_type]})#{resource[:checksum]}" if provider.archive_checksum super end def change_to_s(currentvalue, newvalue) if currentvalue == :absent || currentvalue.nil? extract = resource[:extract] == :true ? "and extracted in #{resource[:extract_path]}" : '' cleanup = resource[:cleanup] == :true ? 'with cleanup' : 'without cleanup' if provider.archive_checksum "replace archive: #{provider.archive_filepath} from #{is_to_s(currentvalue)} to #{should_to_s(newvalue)}" else "download archive from #{resource[:source]} to #{provider.archive_filepath} #{extract} #{cleanup}" end elsif newvalue == :absent "remove archive: #{provider.archive_filepath} " else super end rescue StandardError super end end newparam(:path, namevar: true) do desc 'namevar, archive file fully qualified file path.' validate do |value| unless Puppet::Util.absolute_path? value raise ArgumentError, "archive path must be absolute: #{value}" end end end newparam(:filename) do desc 'archive file name (derived from path).' end newparam(:extract) do desc 'whether archive will be extracted after download (true|false).' newvalues(:true, :false) defaultto(:false) end newparam(:extract_path) do desc 'target folder path to extract archive.' validate do |value| unless Puppet::Util.absolute_path? value raise ArgumentError, "archive extract_path must be absolute: #{value}" end end end newparam(:target) do desc 'target folder path to extract archive. (this parameter is for camptocamp/archive compatibility)' validate do |value| unless Puppet::Util.absolute_path? value raise ArgumentError, "archive extract_path must be absolute: #{value}" end end munge do |val| resource[:extract_path] = val end end newparam(:extract_command) do desc "custom extraction command ('tar xvf example.tar.gz'), also support sprintf format ('tar xvf %s') which will be processed with the filename: sprintf('tar xvf %s', filename)" end newparam(:temp_dir) do desc 'Specify an alternative temporary directory to use for copying files, if unset then the operating system default will be used.' validate do |value| unless Puppet::Util.absolute_path?(value) raise ArgumentError, "Invalid temp_dir #{value}" end end end newparam(:extract_flags) do desc "custom extraction options, this replaces the default flags. A string such as 'xvf' for a tar file would replace the default xf flag. A hash is useful when custom flags are needed for different platforms. {'tar' => 'xzf', '7z' => 'x -aot'}." defaultto(:undef) end newproperty(:creates) do desc 'if file/directory exists, will not download/extract archive.' def should_to_s(value) "extracting in #{resource[:extract_path]} to create #{value}" end end newparam(:cleanup) do desc 'whether archive file will be removed after extraction (true|false).' newvalues(:true, :false) defaultto(:true) end newparam(:source) do desc 'archive file source, supports puppet|http|https|ftp|file|s3 uri.' validate do |value| unless value =~ URI.regexp(%w[puppet http https ftp file s3]) || Puppet::Util.absolute_path?(value) raise ArgumentError, "invalid source url: #{value}" end end end newparam(:url) do desc 'archive file source, supports http|https|ftp|file uri. (for camptocamp/archive compatibility)' validate do |value| unless value =~ URI.regexp(%w[http https file ftp]) raise ArgumentError, "invalid source url: #{value}" end end munge do |val| resource[:source] = val end end newparam(:cookie) do desc 'archive file download cookie.' end newparam(:checksum) do desc 'archive file checksum (match checksum_type).' newvalues(%r{\b[0-9a-f]{5,128}\b}, :true, :false, :undef, nil, '') munge do |val| if val.nil? || val.empty? || val == :undef :false - elsif %i[true false].include? val + elsif [:true, :false].include? val resource[:checksum_verify] = val else val end end end newparam(:digest_string) do desc 'archive file checksum (match checksum_type) (this parameter is for camptocamp/archive compatibility).' newvalues(%r{\b[0-9a-f]{5,128}\b}) munge do |val| if !val.nil? && !val.empty? resource[:checksum] = val else val end end end newparam(:checksum_url) do desc 'archive file checksum source (instead of specifying checksum)' end newparam(:digest_url) do desc 'archive file checksum source (instead of specifying checksum) (this parameter is for camptocamp/archive compatibility)' munge do |val| resource[:checksum_url] = val end end newparam(:checksum_type) do desc 'archive file checksum type (none|md5|sha1|sha2|sha256|sha384|sha512).' newvalues(:none, :md5, :sha1, :sha2, :sha256, :sha384, :sha512) defaultto(:none) end newparam(:digest_type) do desc 'archive file checksum type (none|md5|sha1|sha2|sha256|sha384|sha512) (this parameter is camptocamp/archive compatibility).' newvalues(:none, :md5, :sha1, :sha2, :sha256, :sha384, :sha512) munge do |val| resource[:checksum_type] = val end end newparam(:checksum_verify) do desc 'whether checksum wil be verified (true|false).' newvalues(:true, :false) defaultto(:true) end newparam(:username) do desc 'username to download source file.' end newparam(:password) do desc 'password to download source file.' end newparam(:user) do desc 'extract command user (using this option will configure the archive file permission to 0644 so the user can read the file).' end newparam(:group) do desc 'extract command group (using this option will configure the archive file permisison to 0644 so the user can read the file).' end newparam(:proxy_type) do desc 'proxy type (none|ftp|http|https)' newvalues(:none, :ftp, :http, :https) end newparam(:proxy_server) do desc 'proxy address to use when accessing source' end newparam(:allow_insecure, boolean: true, parent: Puppet::Parameter::Boolean) do desc 'ignore HTTPS certificate errors' defaultto :false end newparam(:download_options) do desc 'provider download options (affects curl, wget, and only s3 downloads for ruby provider)' validate do |val| unless val.is_a?(::String) || val.is_a?(::Array) raise ArgumentError, "download_options should be String or Array: #{val}" end end munge do |val| case val when ::String [val] else val end end end autorequire(:file) do [ Pathname.new(self[:path]).parent.to_s, self[:extract_path], '/root/.aws/config', '/root/.aws/credentials' ].compact end autorequire(:exec) do ['install_aws_cli'] end validate do filepath = Pathname.new(self[:path]) self[:filename] = filepath.basename.to_s if !self[:source].nil? && !self[:url].nil? && self[:source] != self[:url] raise ArgumentError, "invalid parameter: url (#{self[:url]}) and source (#{self[:source]}) are mutually exclusive." end if !self[:checksum_url].nil? && !self[:digest_url].nil? && self[:checksum_url] != self[:digest_url] raise ArgumentError, "invalid parameter: checksum_url (#{self[:checksum_url]}) and digest_url (#{self[:digest_url]}) are mutually exclusive." end if self[:proxy_server] self[:proxy_type] ||= URI(self[:proxy_server]).scheme.to_sym else self[:proxy_type] = :none end end end diff --git a/lib/puppet_x/bodeco/util.rb b/lib/puppet_x/bodeco/util.rb index 90cca6c..b7da192 100644 --- a/lib/puppet_x/bodeco/util.rb +++ b/lib/puppet_x/bodeco/util.rb @@ -1,180 +1,180 @@ module PuppetX module Bodeco module Util def self.download(url, filepath, options = {}) uri = URI(url) @connection = PuppetX::Bodeco.const_get(uri.scheme.upcase).new("#{uri.scheme}://#{uri.host}:#{uri.port}", options) @connection.download(uri, filepath) end def self.content(url, options = {}) uri = URI(url) @connection = PuppetX::Bodeco.const_get(uri.scheme.upcase).new("#{uri.scheme}://#{uri.host}:#{uri.port}", options) @connection.content(uri) end # # This allows you to use a puppet syntax for a file and return its content. # # @example # puppet_download 'puppet:///modules/my_module_name/my_file.dat # # @param [String] url this is the puppet url of the file to be fetched # @param [String] filepath this is path of the file to create # # @raise [ArgumentError] when the file doesn't exist # def self.puppet_download(url, filepath) # Somehow there is no consistent way to determine what terminus to use. So we switch to a # trial and error method. First we start withe the default. And if it doesn't work, we try the # other ones status = load_file_with_any_terminus(url) raise ArgumentError, "Could not retrieve information from environment #{Puppet['environment']} source(s) #{url}'" unless status File.open(filepath, 'w') { |file| file.write(status.content) } end # @private # rubocop:disable HandleExceptions def self.load_file_with_any_terminus(url) - termini_to_try = %i[file_server rest] + termini_to_try = [:file_server, :rest] termini_to_try.each do |terminus| with_terminus(terminus) do begin content = Puppet::FileServing::Content.indirection.find(url) rescue SocketError, Timeout::Error, Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH, Errno::ETIMEDOUT # rescue any network error end return content if content end end nil end # rubocop:enable HandleExceptions def self.with_terminus(terminus) old_terminus = Puppet[:default_file_terminus] Puppet[:default_file_terminus] = terminus value = yield Puppet[:default_file_terminus] = old_terminus value end end class HTTP require 'net/http' FOLLOW_LIMIT = 5 URI_UNSAFE = %r{[^\-_.!~*'()a-zA-Z\d;\/?:@&=+$,\[\]%]} def initialize(_url, options) @username = options[:username] @password = options[:password] @cookie = options[:cookie] @insecure = options[:insecure] if options[:proxy_server] uri = URI(options[:proxy_server]) unless uri.scheme uri = URI("#{options[:proxy_type]}://#{options[:proxy_server]}") end @proxy_addr = uri.hostname @proxy_port = uri.port end ENV['SSL_CERT_FILE'] = File.expand_path(File.join(__FILE__, '..', 'cacert.pem')) if Facter.value(:osfamily) == 'windows' && !ENV.key?('SSL_CERT_FILE') end def generate_request(uri) header = @cookie && { 'Cookie' => @cookie } request = Net::HTTP::Get.new(uri.request_uri, header) request.basic_auth(@username, @password) if @username && @password request end def follow_redirect(uri, option = { limit: FOLLOW_LIMIT }, &block) http_opts = if uri.scheme == 'https' { use_ssl: true, verify_mode: (@insecure ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER) } else { use_ssl: false } end Net::HTTP.start(uri.host, uri.port, @proxy_addr, @proxy_port, http_opts) do |http| http.request(generate_request(uri)) do |response| case response when Net::HTTPSuccess yield response when Net::HTTPRedirection limit = option[:limit] - 1 raise Puppet::Error, "Redirect limit exceeded, last url: #{uri}" if limit < 0 location = safe_escape(response['location']) new_uri = URI(location) new_uri = URI(uri.to_s + location) if new_uri.relative? follow_redirect(new_uri, limit: limit, &block) else raise Puppet::Error, "HTTP Error Code #{response.code}\nURL: #{uri}\nContent:\n#{response.body}" end end end end def download(uri, file_path, option = { limit: FOLLOW_LIMIT }) follow_redirect(uri, option) do |response| File.open file_path, 'wb' do |io| response.read_body do |chunk| io.write chunk end end end end def content(uri, option = { limit: FOLLOW_LIMIT }) follow_redirect(uri, option) do |response| return response.body end end def safe_escape(uri) uri.to_s.gsub(URI_UNSAFE) do |match| '%' + match.unpack('H2' * match.bytesize).join('%').upcase end end end class HTTPS < HTTP end class FTP require 'net/ftp' def initialize(url, options) uri = URI(url) username = options[:username] password = options[:password] proxy_server = options[:proxy_server] proxy_type = options[:proxy_type] ENV["#{proxy_type}_proxy"] = proxy_server @ftp = Net::FTP.new @ftp.connect(uri.host, uri.port) if username @ftp.login(username, password) else @ftp.login end end def download(uri, file_path) @ftp.getbinaryfile(uri.path, file_path) end end class FILE def initialize(_url, _options) end def download(uri, file_path) FileUtils.copy(uri.path, file_path) end end end end diff --git a/spec/unit/puppet/type/archive_spec.rb b/spec/unit/puppet/type/archive_spec.rb index fa8517b..a6806eb 100644 --- a/spec/unit/puppet/type/archive_spec.rb +++ b/spec/unit/puppet/type/archive_spec.rb @@ -1,161 +1,161 @@ require 'spec_helper' require 'puppet' describe Puppet::Type.type(:archive) do let(:resource) do Puppet::Type.type(:archive).new( path: '/tmp/example.zip', source: 'http://home.lan/example.zip' ) end context 'resource defaults' do it { expect(resource[:path]).to eq '/tmp/example.zip' } it { expect(resource[:name]).to eq '/tmp/example.zip' } it { expect(resource[:filename]).to eq 'example.zip' } it { expect(resource[:extract]).to eq :false } it { expect(resource[:cleanup]).to eq :true } it { expect(resource[:checksum_type]).to eq :none } it { expect(resource[:digest_type]).to eq nil } it { expect(resource[:checksum_verify]).to eq :true } it { expect(resource[:extract_flags]).to eq :undef } it { expect(resource[:allow_insecure]).to eq false } it { expect(resource[:download_options]).to eq nil } it { expect(resource[:temp_dir]).to eq nil } end it 'verify resource[:path] is absolute filepath' do expect do resource[:path] = 'relative/file' end.to raise_error(Puppet::Error, %r{archive path must be absolute: }) end it 'verify resource[:temp_dir] is absolute filetemp_dir' do expect do resource[:temp_dir] = 'relative/file' end.to raise_error(Puppet::Error, %r{Invalid temp_dir}) end describe 'on posix', if: Puppet.features.posix? do it 'accepts valid resource[:source]' do expect do resource[:source] = 'http://home.lan/example.zip' resource[:source] = 'https://home.lan/example.zip' resource[:source] = 'ftp://home.lan/example.zip' resource[:source] = 's3://home.lan/example.zip' resource[:source] = '/tmp/example.zip' end.not_to raise_error end %w[ afp://home.lan/example.zip \tmp D:/example.zip ].each do |s| it 'rejects invalid resource[:source]' do expect do resource[:source] = s end.to raise_error(Puppet::Error, %r{invalid source url: }) end end end describe 'on windows', if: Puppet.features.microsoft_windows? do it 'accepts valid windows resource[:source]' do expect do resource[:source] = 'D:/example.zip' end.not_to raise_error end %w[ /tmp/example.zip \Z: ].each do |s| it 'rejects invalid windows resource[:source]' do expect do resource[:source] = s end.to raise_error(Puppet::Error, %r{invalid source url: }) end end end %w[ 557e2ebb67b35d1fddff18090b6bc26b 557e2ebb67b35d1fddff18090b6bc26557e2ebb67b35d1fddff18090b6bc26bb ].each do |cs| it 'accepts valid resource[:checksum]' do expect do resource[:checksum] = cs end.not_to raise_error end end %w[ z57e2ebb67b35d1fddff18090b6bc26b 557e ].each do |cs| it 'rejects bad checksum' do expect do resource[:checksum] = cs end.to raise_error(Puppet::Error, %r{Invalid value}) end end it 'accepts valid resource[:checksum_type]' do expect do - %i[none md5 sha1 sha2 sha256 sha384 sha512].each do |type| + [:none, :md5, :sha1, :sha2, :sha256, :sha384, :sha512].each do |type| resource[:checksum_type] = type end end.not_to raise_error end it 'rejects invalid resource[:checksum_type]' do expect do resource[:checksum_type] = :crc32 end.to raise_error(Puppet::Error, %r{Invalid value}) end it 'verify resource[:allow_insecure] is valid' do expect do - %i[true false yes no].each do |type| + [:true, :false, :yes, :no].each do |type| resource[:allow_insecure] = type end end.not_to raise_error end it 'verify resource[:download_options] is valid' do expect do ['--tlsv1', ['--region', 'eu-central-1']].each do |type| resource[:download_options] = type end end.not_to raise_error end describe 'archive autorequire' do let(:file_resource) { Puppet::Type.type(:file).new(name: '/tmp') } let(:archive_resource) do described_class.new( path: '/tmp/example.zip', source: 'http://home.lan/example.zip' ) end let(:auto_req) do catalog = Puppet::Resource::Catalog.new catalog.add_resource file_resource catalog.add_resource archive_resource archive_resource.autorequire end it 'creates relationship' do expect(auto_req.size).to be 1 end it 'links to archive resource' do expect(auto_req[0].target).to eql archive_resource end it 'autorequires parent directory' do expect(auto_req[0].source).to eql file_resource end end end