diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -2509,6 +2509,9 @@ systemd_journal::logstash_hosts: - 'logstash.internal.softwareheritage.org:5044' +filebeat::logstash_hosts: + - 'logstash.internal.softwareheritage.org:5044' + memcached::server::bind: 127.0.0.1 memcached::server::port: 11211 memcached::server::max_memory: '5%' diff --git a/site-modules/profile/files/filebeat/00_header.yml b/site-modules/profile/files/filebeat/00_header.yml new file mode 100644 --- /dev/null +++ b/site-modules/profile/files/filebeat/00_header.yml @@ -0,0 +1,8 @@ +################################################ +# /!\ DON'T CHANGE THIS FILE /!\ # +# this file is generated and managed by puppet # +################################################ + +############################# Filebeat ###################################### + +filebeat: diff --git a/site-modules/profile/files/filebeat/05_inputs.yml b/site-modules/profile/files/filebeat/05_inputs.yml new file mode 100644 --- /dev/null +++ b/site-modules/profile/files/filebeat/05_inputs.yml @@ -0,0 +1 @@ + inputs : diff --git a/site-modules/profile/files/filebeat/15_output.yml b/site-modules/profile/files/filebeat/15_output.yml new file mode 100644 --- /dev/null +++ b/site-modules/profile/files/filebeat/15_output.yml @@ -0,0 +1,3 @@ +############################# Output ########################################## + +output: diff --git a/site-modules/profile/manifests/filebeat.pp b/site-modules/profile/manifests/filebeat.pp --- a/site-modules/profile/manifests/filebeat.pp +++ b/site-modules/profile/manifests/filebeat.pp @@ -1,6 +1,12 @@ # Filebeat apache log shipper profile -class profile::filebeat { +class profile::filebeat( + $config_directory = '/etc/filebeat', + $config_file = 'filebeat.yml', +) { + $config_path = "${config_directory}/${config_file}" + $logstash_hosts = lookup('filebeat::logstash_hosts') + include ::profile::elastic::apt_config $version = lookup('elastic::elk_version') @@ -19,4 +25,31 @@ ensure => running, enable => true, } + + concat { $config_path: + require => Package["filebeat"], + notify => Service["filebeat"], + } + + concat::fragment { 'filebeat_header': + target => $config_path, + source => 'puppet:///modules/profile/filebeat/00_header.yml', + order => '00', + } + concat::fragment { 'filebeat_inputs' : + target => $config_path, + source => 'puppet:///modules/profile/filebeat/05_inputs.yml', + order => '05' + } + concat::fragment { 'filebeat_output' : + target => $config_path, + source => 'puppet:///modules/profile/filebeat/15_output.yml', + order => '15' + } + + concat::fragment { 'filebeat_logstash_output' : + target => $config_path, + content => template('profile/filebeat/20_output_logstash.yml.erb'), + order => '20', + } } diff --git a/site-modules/profile/manifests/filebeat/log_input.pp b/site-modules/profile/manifests/filebeat/log_input.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/filebeat/log_input.pp @@ -0,0 +1,11 @@ +define profile::filebeat::log_input( + Array[String] $paths, + Array $fields = [], +) { + concat::fragment { "filebeat_input_${name}" : + target => "${profile::filebeat::config_directory}/${profile::filebeat::config_file}", + content => template('profile/filebeat/10_log_input.yml.erb'), + order => '10', + require => Class['profile::filebeat'], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -282,4 +282,9 @@ vhost_name => $vhost_name, }, } + + profile::filebeat::log_input { 'webapp-non-ssl-access': + paths => [ '/var/log/apache2/archive.softwareheritage.org_non-ssl_access.log' ], + fields => [ "apache_log_type: access_log" ], + } } diff --git a/site-modules/profile/templates/filebeat/10_log_input.yml.erb b/site-modules/profile/templates/filebeat/10_log_input.yml.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/filebeat/10_log_input.yml.erb @@ -0,0 +1,9 @@ + - type: log + paths: +<% scope.lookupvar("paths").flatten.each do |path| -%> + - <%= path %> +<% end -%> + fields: +<% scope.lookupvar("fields").flatten.each do |field| -%> + <%= field %> +<% end %> diff --git a/site-modules/profile/templates/filebeat/20_output_logstash.yml.erb b/site-modules/profile/templates/filebeat/20_output_logstash.yml.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/filebeat/20_output_logstash.yml.erb @@ -0,0 +1,3 @@ + ### Logstash as output + logstash: + hosts: [ <% scope.lookupvar("logstash_hosts").flatten.each do |host| -%>"<%= host %>", <%end %>]