diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -2505,8 +2505,18 @@ icinga2::exported_checks::filename: "/etc/icinga2/zones.d/%{hiera('icinga2::parent_zone')}/exported-checks.conf" -systemd_journal::logstash_hosts: - - 'logstash.internal.softwareheritage.org:5044' +logstash_hosts: + - logstash.internal.softwareheritage.org:5044 + +systemd_journal::logstash_hosts: "%{alias('logstash_hosts')}" + +filebeat::config: + filebeat.config.inputs: + enabled: true + path: inputs.d/*.yml + output: + logstash: + hosts: "%{alias('logstash_hosts')}" memcached::server::bind: 127.0.0.1 memcached::server::port: 11211 diff --git a/site-modules/profile/manifests/filebeat.pp b/site-modules/profile/manifests/filebeat.pp --- a/site-modules/profile/manifests/filebeat.pp +++ b/site-modules/profile/manifests/filebeat.pp @@ -1,6 +1,10 @@ # Filebeat apache log shipper profile class profile::filebeat { + $config_directory = '/etc/filebeat' + $config_file = 'filebeat.yml' + $config_path = "${config_directory}/${config_file}" + include ::profile::elastic::apt_config $version = lookup('elastic::elk_version') @@ -19,4 +23,25 @@ ensure => running, enable => true, } + + file { "${config_directory}/inputs.d": + ensure => directory, + purge => true, + owner => 'root', + group => 'root', + mode => '0755', + require => Package['filebeat'], + } + + $filebeat_config = lookup('filebeat::config') + + file { $config_path : + ensure => present, + content => inline_yaml($filebeat_config), + owner => 'root', + group => 'root', + mode => '0644', + require => [Package['filebeat']], + notify => Service['filebeat'], + } } diff --git a/site-modules/profile/manifests/filebeat/log_input.pp b/site-modules/profile/manifests/filebeat/log_input.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/filebeat/log_input.pp @@ -0,0 +1,27 @@ +# Configure a log input for filebeat +define profile::filebeat::log_input( + Array[String] $paths, + Hash[String,String] $fields = {}, +) { + + $input_configuration = [{ + 'type' => 'log', + 'paths' => $paths, + 'fields' => $fields, + }] + + file { "filebeat_input_${name}" : + ensure => present, + path => "${profile::filebeat::config_directory}/inputs.d/${name}.yml", + content => inline_yaml($input_configuration), + owner => 'root', + group => 'root', + mode => '0644', + require => [ + File["${profile::filebeat::config_directory}/inputs.d"], + Package['filebeat'], + ], + notify => Service['filebeat'], + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp --- a/site-modules/profile/manifests/swh/deploy/deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/deposit.pp @@ -267,4 +267,10 @@ tag => 'icinga2::exported', } + include profile::filebeat + profile::filebeat::log_input { 'deposit-non-ssl-access': + paths => [ '/var/log/apache2/deposit.softwareheritage.org_non-ssl_access.log' ], + fields => { 'apache_log_type' => 'access_log', }, + } + } diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -282,4 +282,10 @@ vhost_name => $vhost_name, }, } + + include profile::filebeat + profile::filebeat::log_input { 'webapp-non-ssl-access': + paths => [ '/var/log/apache2/archive.softwareheritage.org_non-ssl_access.log' ], + fields => { 'apache_log_type' => 'access_log' }, + } }