diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -2509,6 +2509,9 @@ systemd_journal::logstash_hosts: - 'logstash.internal.softwareheritage.org:5044' +filebeat::logstash_hosts: + - 'logstash.internal.softwareheritage.org:5044' + memcached::server::bind: 127.0.0.1 memcached::server::port: 11211 memcached::server::max_memory: '5%' diff --git a/site-modules/profile/manifests/filebeat.pp b/site-modules/profile/manifests/filebeat.pp --- a/site-modules/profile/manifests/filebeat.pp +++ b/site-modules/profile/manifests/filebeat.pp @@ -1,6 +1,12 @@ # Filebeat apache log shipper profile -class profile::filebeat { +class profile::filebeat( + $config_directory = '/etc/filebeat', + $config_file = 'filebeat.yml', +) { + $config_path = "${config_directory}/${config_file}" + $logstash_hosts = lookup('filebeat::logstash_hosts') + include ::profile::elastic::apt_config $version = lookup('elastic::elk_version') @@ -19,4 +25,16 @@ ensure => running, enable => true, } + + file { "${config_directory}/inputs.d": + ensure => directory, + require => Package['filebeat'], + } + + file { $config_path : + ensure => present, + content => template('profile/filebeat/filebeat.yml.erb'), + require => [Package['filebeat']], + notify => Service['filebeat'], + } } diff --git a/site-modules/profile/manifests/filebeat/log_input.pp b/site-modules/profile/manifests/filebeat/log_input.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/filebeat/log_input.pp @@ -0,0 +1,17 @@ +define profile::filebeat::log_input( + Array[String] $paths, + Array $fields = [], +) { + + file { "filebeat_input_${name}" : + ensure => present, + path => "${profile::filebeat::config_directory}/inputs.d/${name}.yml", + content => template('profile/filebeat/log_input.yml.erb'), + require => [ + File["${profile::filebeat::config_directory}/inputs.d"], + Package["filebeat"], + ], + notify => Service["filebeat"], + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp --- a/site-modules/profile/manifests/swh/deploy/deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/deposit.pp @@ -267,4 +267,9 @@ tag => 'icinga2::exported', } + profile::filebeat::log_input { 'deposit-non-ssl-access': + paths => [ '/var/log/apache2/deposit.softwareheritage.org_non-ssl_access.log' ], + fields => [ "apache_log_type: access_log" ], + } + } diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -282,4 +282,9 @@ vhost_name => $vhost_name, }, } + + profile::filebeat::log_input { 'webapp-non-ssl-access': + paths => [ '/var/log/apache2/archive.softwareheritage.org_non-ssl_access.log' ], + fields => [ "apache_log_type: access_log" ], + } } diff --git a/site-modules/profile/templates/filebeat/filebeat.yml.erb b/site-modules/profile/templates/filebeat/filebeat.yml.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/filebeat/filebeat.yml.erb @@ -0,0 +1,11 @@ +################################################ +# /!\ DON'T CHANGE THIS FILE /!\ # +# this file is generated and managed by puppet # +################################################ + +filebeat.config.inputs: + enabled: true + path: inputs.d/*.yml + +output.logstash: + hosts: [ <% scope.lookupvar("logstash_hosts").flatten.each do |host| -%>"<%= host %>", <%end %>] diff --git a/site-modules/profile/templates/filebeat/log_input.yml.erb b/site-modules/profile/templates/filebeat/log_input.yml.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/filebeat/log_input.yml.erb @@ -0,0 +1,9 @@ +- type: log + paths: +<% scope.lookupvar("paths").flatten.each do |path| -%> + - <%= path %> +<% end -%> + fields: +<% scope.lookupvar("fields").flatten.each do |field| -%> + <%= field %> +<% end %> \ No newline at end of file