diff --git a/site-modules/profile/manifests/logstash.pp b/site-modules/profile/manifests/logstash.pp index efa9b8f7..6b312d8b 100644 --- a/site-modules/profile/manifests/logstash.pp +++ b/site-modules/profile/manifests/logstash.pp @@ -1,24 +1,44 @@ class profile::logstash { package { 'openjdk-8-jre-headless': ensure => 'present', } $keyid = lookup('elastic::apt_config::keyid') $key = lookup('elastic::apt_config::key') apt::source { 'elastic-6.x': location => 'https://artifacts.elastic.co/packages/6.x/apt', release => 'stable', repos => 'main', key => { id => $keyid, content => $key, }, } package { 'logstash': ensure => 'installed', } + file { '/etc/logstash/conf.d/input.conf': + ensure => 'file', + content => template('profile/logstash/input.conf.erb'), + } + + file { '/etc/logstash/conf.d/output.conf': + ensure => 'file', + content => template('profile/logstash/output.conf.erb'), + } + + file { '/etc/logstash/conf.d/filter.conf': + ensure => 'file', + content => template('profile/logstash/filter.conf.erb'), + } + + service { 'logstash': + ensure => running, + enable => true, + } + } diff --git a/site-modules/profile/templates/logstash/filter.conf.erb b/site-modules/profile/templates/logstash/filter.conf.erb new file mode 100644 index 00000000..8b21dfc3 --- /dev/null +++ b/site-modules/profile/templates/logstash/filter.conf.erb @@ -0,0 +1,10 @@ +filter { + if [fields][apache_log_type] == "access_log" { + grok { + match => { "message" => "%{COMBINEDAPACHELOG}" } + } + date { + match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ] + } + } +} diff --git a/site-modules/profile/templates/logstash/input.conf.erb b/site-modules/profile/templates/logstash/input.conf.erb new file mode 100644 index 00000000..434066ac --- /dev/null +++ b/site-modules/profile/templates/logstash/input.conf.erb @@ -0,0 +1,6 @@ +input { + beats { + host => "<%= @ipaddress %>" + port => 5044 + } +} diff --git a/site-modules/profile/templates/logstash/output.conf.erb b/site-modules/profile/templates/logstash/output.conf.erb new file mode 100644 index 00000000..0438ac78 --- /dev/null +++ b/site-modules/profile/templates/logstash/output.conf.erb @@ -0,0 +1,27 @@ +output { + if [fields][apache_log_type] == "access_log" { + elasticsearch { + hosts => [ + "esnode1.internal.softwareheritage.org:9200", + "esnode2.internal.softwareheritage.org:9200" + ] + index => "apache_logs-%{+YYYY.MM.dd}" + } + } else if "swh-worker@" in [systemd_unit] { + elasticsearch { + hosts => [ + "esnode1.internal.softwareheritage.org:9200", + "esnode2.internal.softwareheritage.org:9200" + ] + index => "swh_workers-%{+YYYY.MM.dd}" + } + } else { + elasticsearch { + hosts => [ + "esnode1.internal.softwareheritage.org:9200", + "esnode2.internal.softwareheritage.org:9200" + ] + index => "systemlogs-%{+YYYY.MM.dd}" + } + } +}