diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -692,6 +692,9 @@ apache::http_port: 80 apache::https_port: 443 +apache::log_formats: + combined_with_duration: "%a %l %u %t \\\"%r\\\" %>s %b \\\"%{literal('%')}{Referer}i\\\" \\\"%{literal('%')}{User-Agent}i\\\" %{literal('%')}{ms}T" + # Hitch TLS proxy configuration hitch::frontend: "[*]:10443" hitch::proxy_support: false @@ -1956,6 +1959,7 @@ swh::deploy::webapp::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" swh::deploy::webapp::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" swh::deploy::webapp::vhost::hsts_header: "%{hiera('apache::hsts_header')}" +swh::deploy::webapp::vhost::access_log_format: combined_with_duration swh::deploy::webapp::config::es_workers_index_url: http://esnode1.internal.softwareheritage.org:9200/swh_workers-* swh::deploy::webapp::production_db_dir: /var/lib/swh diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -30,6 +30,7 @@ } $vhost_aliases = delete($vhosts, $vhost_name) + $vhost_access_log_format = lookup('swh::deploy::webapp::vhost::access_log_format') $vhost_port = lookup('apache::http_port') $vhost_docroot = "/var/www/${vhost_name}" $vhost_basic_auth_file = "${conf_directory}/http_auth" @@ -76,7 +77,7 @@ mode => '0770', } - file {"$conf_log_dir/swh-web.log": + file {"${conf_log_dir}/swh-web.log": ensure => present, owner => $user, group => $group, @@ -118,9 +119,9 @@ mode => '0664', } - $sentry_dsn = lookup("swh::deploy::webapp::sentry_dsn", Optional[String], 'first', undef) - $sentry_environment = lookup("swh::deploy::webapp::sentry_environment", Optional[String], 'first', undef) - $sentry_swh_package = lookup("swh::deploy::webapp::sentry_swh_package", Optional[String], 'first', undef) + $sentry_dsn = lookup('swh::deploy::webapp::sentry_dsn', Optional[String], 'first', undef) + $sentry_environment = lookup('swh::deploy::webapp::sentry_environment', Optional[String], 'first', undef) + $sentry_swh_package = lookup('swh::deploy::webapp::sentry_swh_package', Optional[String], 'first', undef) ::gunicorn::instance {'swh-webapp': ensure => enabled, @@ -129,12 +130,12 @@ executable => 'django.core.wsgi:get_wsgi_application()', config_base_module => 'swh.web.gunicorn_config', settings => { - bind => $backend_listen_address, - workers => $backend_workers, - worker_class => 'sync', - timeout => $backend_http_timeout, - graceful_timeout => $backend_reload_mercy, - keepalive => $backend_http_keepalive, + bind => $backend_listen_address, + workers => $backend_workers, + worker_class => 'sync', + timeout => $backend_http_timeout, + graceful_timeout => $backend_reload_mercy, + keepalive => $backend_http_keepalive, }, environment => { 'DJANGO_SETTINGS_MODULE' => 'swh.web.settings.production', @@ -185,6 +186,7 @@ require => [ File[$vhost_basic_auth_file], ], + access_log_format => $vhost_access_log_format, } if $endpoint_directories { @@ -204,9 +206,9 @@ include ::profile::swh::deploy::webapp::icinga_checks profile::prometheus::export_scrape_config {"swh-webapp_${fqdn}": - job => "swh-webapp", + job => 'swh-webapp', target => "${vhost_name}:${vhost_ssl_port}", - scheme => "https", + scheme => 'https', metrics_path => '/metrics/prometheus', labels => { vhost_name => $vhost_name, @@ -214,8 +216,16 @@ } include profile::filebeat - profile::filebeat::log_input { 'webapp-non-ssl-access': - paths => [ '/var/log/apache2/archive.softwareheritage.org_non-ssl_access.log' ], - fields => { 'apache_log_type' => 'access_log' }, + # To remove when cleanup is done + file {'/etc/filebeat/inputs.d/webapp-non-ssl-access.yml': + ensure => absent, + } + profile::filebeat::log_input { "${vhost_name}-non-ssl-access": + paths => [ "/var/log/apache2/${vhost_name}_non-ssl_access.log" ], + fields => { + 'apache_log_type' => 'access_log', + 'environment' => $environment, + 'vhost' => $vhost_name, + }, } } diff --git a/site-modules/profile/templates/logstash/filter.conf.erb b/site-modules/profile/templates/logstash/filter.conf.erb --- a/site-modules/profile/templates/logstash/filter.conf.erb +++ b/site-modules/profile/templates/logstash/filter.conf.erb @@ -1,7 +1,7 @@ filter { if [fields][apache_log_type] == "access_log" { grok { - match => { "message" => "%{COMBINEDAPACHELOG}" } + match => { "message" => "%{COMBINEDAPACHELOG}( %{NUMBER:duration_ms})?" } } date { match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ] @@ -10,6 +10,10 @@ add_field => { "[@metadata][target_index]" => "apache_logs-%{+YYYY.MM.dd}" } + convert => { + "response" => "integer" + "duration_ms" => "integer" + } } } else if "swh-worker@" in [systemd_unit] { mutate {