diff --git a/files/gunicorn.service b/files/gunicorn.service new file mode 100644 index 0000000..512251b --- /dev/null +++ b/files/gunicorn.service @@ -0,0 +1,13 @@ +# File managed by puppet (module swh-gunicorn), changes will be lost + +[Unit] +Description=All gunicorn services + +[Service] +Type=oneshot +ExecStart=/bin/true +ExecReload=/bin/true +RemainAfterExit=on + +[Install] +WantedBy=multi-user.target diff --git a/files/gunicorn.tmpfiles b/files/gunicorn.tmpfiles new file mode 100644 index 0000000..01f9667 --- /dev/null +++ b/files/gunicorn.tmpfiles @@ -0,0 +1,3 @@ +# Managed by puppet (class gunicorn). Modifications will be lost. + +d /run/gunicorn 0755 root root - - diff --git a/files/logconfig.ini b/files/logconfig.ini new file mode 100644 index 0000000..a0da6c2 --- /dev/null +++ b/files/logconfig.ini @@ -0,0 +1,40 @@ +[loggers] +keys=root, gunicorn.error, gunicorn.access + +[handlers] +keys=console, journal + +[formatters] +keys=generic + +[logger_root] +level=INFO +handlers=console,journal + +[logger_gunicorn.error] +level=INFO +propagate=0 +handlers=journal +qualname=gunicorn.error + +[logger_gunicorn.access] +level=INFO +propagate=0 +handlers=journal +qualname=gunicorn.access + +[handler_console] +class=StreamHandler +formatter=generic +args=(sys.stdout, ) + +[handler_journal] +class=swh.core.logger.JournalHandler +formatter=generic +args=() + +[formatter_generic] +format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s +datefmt=%Y-%m-%d %H:%M:%S +class=logging.Formatter + diff --git a/manifests/apt_config.pp b/manifests/apt_config.pp new file mode 100644 index 0000000..c3cbb89 --- /dev/null +++ b/manifests/apt_config.pp @@ -0,0 +1,20 @@ +# APT Configuration for gunicorn +class gunicorn::apt_config { + if $::lsbdistcodename == 'jessie' { + $pinned_packages = [ + 'gunicorn', + 'gunicorn-examples', + 'gunicorn3', + 'python', + 'python3', + ] + + ::apt::pin {'gunicorn': + explanation => 'Pin gunicorn and dependencies to backports', + codename => 'jessie-backports', + packages => $pinned_packages, + priority => 990, + } + } + +} diff --git a/manifests/init.pp b/manifests/init.pp index 3b56779..9babb78 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,48 +1,62 @@ -# Class: gunicorn -# =========================== -# -# Full description of class gunicorn here. -# -# Parameters -# ---------- -# -# Document parameters here. -# -# * `sample parameter` -# Explanation of what this parameter affects and what it defaults to. -# e.g. "Specify one or more upstream ntp servers as an array." -# -# Variables -# ---------- -# -# Here you should define a list of variables that this module would require. -# -# * `sample variable` -# Explanation of how this variable affects the function of this class and if -# it has a default. e.g. "The parameter enc_ntp_servers must be set by the -# External Node Classifier as a comma separated list of hostnames." (Note, -# global variables should be avoided in favor of class parameters as -# of Puppet 2.6.) -# -# Examples -# -------- -# -# @example -# class { 'gunicorn': -# servers => [ 'pool.ntp.org', 'ntp.local.company.com' ], -# } -# -# Authors -# ------- -# -# Author Name -# -# Copyright -# --------- -# -# Copyright 2017 Your name here, unless otherwise noted. -# class gunicorn { + $service_name = 'gunicorn' + $packages = ['gunicorn3'] + include ::systemd + include gunicorn::apt_config + package {$packages: + ensure => installed, + } + + service {$service_name: + ensure => running, + enable => true, + require => [ + Package[$packages], + Exec['systemd-daemon-reload'] + ], + } + + file {'/etc/systemd/system/gunicorn.service': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/gunicorn/gunicorn.service', + notify => Exec['systemd-daemon-reload'], + } + + file {['/etc/gunicorn', '/etc/gunicorn/instances']: + ensure => directory, + owner => 'root', + group => 'root', + mode => '0644', + recurse => true, + purge => true, + } + + file {'/etc/tmpfiles.d/gunicorn.conf': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/gunicorn/gunicorn.tmpfiles', + notify => Exec['systemd-tmpfiles-update-gunicorn'], + } + + file {'/etc/gunicorn/logconfig.ini': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/gunicorn/logconfig.ini', + } + + exec {'systemd-tmpfiles-update-gunicorn': + path => '/sbin:/usr/sbin:/bin:/usr/bin', + command => 'systemd-tmpfiles --update /etc/tmpfiles.d/gunicorn.conf', + refreshonly => true, + require => File['/etc/tmpfiles.d/gunicorn.conf'] + } } diff --git a/manifests/instance.pp b/manifests/instance.pp new file mode 100644 index 0000000..5185ed1 --- /dev/null +++ b/manifests/instance.pp @@ -0,0 +1,150 @@ +# == Defined Type: gunicorn::instance +# +# gunicorn::instance defines an instance of gunicorn +# +# === Parameters +# +# [*ensure*] +# Whether the instance should be enabled, present (but disabled) or absent. +# +# [*executable*] +# The wsgi callable to pass to gunicorn +# +# [*settings*] +# a hash of settings for the given site. +# +# === Examples +# +# gunicorn::instance {'foo': +# ensure => enabled, +# executable => 'foo.wsgi:app' +# user => 'foouser', +# group => 'foogroup', +# config_mode => 0644 +# settings => { +# plugin => 'python3', +# protocol => $uwsgi_protocol, +# socket => $uwsgi_listen_address, +# workers => $uwsgi_workers, +# max_requests => $uwsgi_max_requests, +# max_requests_delta => $uwsgi_max_requests_delta, +# worker_reload_mercy => $uwsgi_reload_mercy, +# reload_mercy => $uwsgi_reload_mercy, +# uid => $user, +# gid => $user, +# module => 'swh.storage.api.server', +# callable => 'run_from_webserver', +# } +# } +# +# === Authors +# +# Nicolas Dandrimont +# +# === Copyright +# +# Copyright 2017 The Software Heritage developers +# +define gunicorn::instance ( + $executable, + $user = 'root', + $group = 'root', + $ensure = 'enabled', + $config_mode = '0644', + $working_dir = undef, + $log_only_errors = true, + $settings = {} +) { + $config_file = "/etc/gunicorn/instances/${name}.cfg" + $service_name = "gunicorn-${name}" + $service_file = "/etc/systemd/system/${service_name}.service" + $tmpfiles_file = "/etc/tmpfiles.d/${service_name}.conf" + $runtime_dir = "/run/gunicorn/${name}" + + if $working_dir { + $working_dir_override = $working_dir + } else { + $working_dir_override = $runtime_dir + } + + if $log_only_errors { + $log_only_errors_str = 'True' + } else { + $log_only_errors_str = 'False' + } + + case $ensure { + default: { err("Unknown value ensure => ${ensure}.") } + 'enabled', 'present': { + + # Uses variables: + # - $settings + # - $name + # - $log_only_errors_str + file {$config_file: + ensure => present, + owner => $user, + group => $group, + mode => $config_mode, + content => template('gunicorn/gunicorn-instance.cfg.erb'), + notify => Service[$service_name], + } + + # Uses variables: + # - $config_file + # - $executable + # - $group + # - $name + # - $runtime_dir + # - $user + # - $working_dir_override + file {$service_file: + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + content => template('gunicorn/gunicorn-instance.cfg.erb'), + notify => Exec['systemd-daemon-reload'], + } + + # Uses variables: + # - $group + # - $name + # - $runtime_dir + # - $user + file {$tmpfiles_file: + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + content => template('gunicorn/gunicorn-instance.tmpfiles.erb'), + notify => Exec["systemd-tmpfiles-update-${service_name}"], + } + + exec {"systemd-tmpfiles-update-${service_name}": + path => '/sbin:/usr/sbin:/bin:/usr/bin', + command => "systemd-tmpfiles --update ${tmpfiles_file}", + refreshonly => true, + require => File[$tmpfiles_file] + } + + $service_enable = $ensure ? { + 'enabled' => true, + 'present' => undef, + } + + service {$service_name: + ensure => 'running', + enable => $service_enable, + restart => "/bin/systemctl reload ${service_name}.service", + } + } + + 'absent': { + file {[$tmpfiles_file, $service_file]: + ensure => 'absent' + } + } + } +} + diff --git a/templates/gunicorn-instance.cfg.erb b/templates/gunicorn-instance.cfg.erb new file mode 100644 index 0000000..1344a4c --- /dev/null +++ b/templates/gunicorn-instance.cfg.erb @@ -0,0 +1,37 @@ +# Gunicorn instance configuration. +# Managed by puppet (class gunicorn::instance <%= @name %>); changes will be lost + +import traceback +import gunicorn.glogging + +class Logger(gunicorn.glogging.Logger): + log_only_errors = <%= @log_only_errors_str %> + + def access(self, resp, req, environ, request_time): + """ See http://httpd.apache.org/docs/2.0/logs.html#combined + for format details + """ + + if not (self.cfg.accesslog or self.cfg.logconfig or self.cfg.syslog): + return + + # wrap atoms: + # - make sure atoms will be test case insensitively + # - if atom doesn't exist replace it by '-' + atoms = self.atoms(resp, req, environ, request_time) + safe_atoms = self.atoms_wrapper_class(atoms) + + try: + if self.log_only_errors and str(atoms['s']) == '200': + return + self.access_log.info(self.cfg.access_log_format % safe_atoms, extra={'swh_atoms': atoms}) + except: + self.exception('Failed processing access log entry') + +logger_class = Logger +logconfig = 'logconfig.ini' + +# custom settings +<% @settings.each do |key, value| -%> +<%= key %> = <%= value.to_json %> +<% end -%> diff --git a/templates/gunicorn-instance.service.erb b/templates/gunicorn-instance.service.erb new file mode 100644 index 0000000..d37fd7f --- /dev/null +++ b/templates/gunicorn-instance.service.erb @@ -0,0 +1,21 @@ +# File managed by puppet (class gunicorn::instance <%= @name %>), changes will be lost + +[Unit] +Description=Gunicorn instance <%= @name %> +ConditionPathExists=/etc/gunicorn/instances/<%= @name %>.cfg +PartOf=gunicorn.service +ReloadPropagatedFrom=gunicorn.service +Before=gunicorn.service + +[Service] +User=<%= @user %> +Group=<%= @group %> +PIDFile=<%= @runtime_dir %>/pidfile +RuntimeDirectory=<%= @runtime_dir %> +WorkingDirectory=<%= @working_dir_override %> +ExecStart=/usr/bin/gunicorn3 -p <%= @runtime_dir %>/pidfile -c <%= @config_file %> <%= @executable %> +ExecStop=/bin/kill -TERM $MAINPID +ExecReload=/bin/kill -HUP $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/templates/gunicorn-instance.tmpfiles.erb b/templates/gunicorn-instance.tmpfiles.erb new file mode 100644 index 0000000..71022e4 --- /dev/null +++ b/templates/gunicorn-instance.tmpfiles.erb @@ -0,0 +1,3 @@ +# Managed by puppet (class gunicorn::instance <%= @name %>). Modifications will be lost. + +d <%= @runtime_dir %> 0755 <%= @user %> <%= @group %> - -