diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..0ad22b51 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,78 @@ +# Software Heritage Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as Software +Heritage contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, regardless +of age, body size, disability, ethnicity, sex characteristics, gender identity +and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at `conduct@softwareheritage.org`. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an +incident. Further details of specific enforcement policies may be posted +separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/Puppetfile b/Puppetfile index c27750f0..b1e2309a 100644 --- a/Puppetfile +++ b/Puppetfile @@ -1,181 +1,193 @@ mod 'dar', :git => 'https://forge.softwareheritage.org/source/puppet-swh-dar', :branch => :control_branch, :default_branch => 'master' mod 'gunicorn', :git => 'https://forge.softwareheritage.org/source/puppet-swh-gunicorn', :branch => :control_branch, :default_branch => 'master' mod 'mediawiki', :git => 'https://forge.softwareheritage.org/source/puppet-swh-mediawiki', :branch => :control_branch, :default_branch => 'master' mod 'postfix', :git => 'https://forge.softwareheritage.org/source/puppet-swh-postfix', :branch => :control_branch, :default_branch => 'master' mod 'uwsgi', :git => 'https://forge.softwareheritage.org/source/puppet-swh-uwsgi', :branch => :control_branch, :default_branch => 'master' mod 'apt', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apt', - :tag => '4.5.1' + :tag => 'v7.0.1' mod 'archive', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-archive', :tag => 'v2.3.0' mod 'bind', :git => 'https://forge.softwareheritage.org/source/puppet-inkblot-bind', :ref => '7.3.1' mod 'apache', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apache', :tag => '3.0.0' mod 'ceph', :git => 'https://forge.softwareheritage.org/source/puppet-openstack-ceph', :ref => 'master' mod 'concat', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-concat', :tag => '3.0.0' mod 'cups', :git => 'https://forge.softwareheritage.org/source/puppet-mosen-cups', :ref => 'master' mod 'datacat', :git => 'https://forge.softwareheritage.org/source/puppet-richardc-datacat', :ref => '0.6.2' mod 'debconf', :git => 'https://forge.softwareheritage.org/source/puppet-stm-debconf', :ref => 'v2.1.0' mod 'debnet', :git => 'https://forge.softwareheritage.org/source/puppet-trepasi-debnet', :ref => 'v1.5.2' mod 'elasticsearch', :git => 'https://forge.softwareheritage.org/source/puppet-elastic-elasticsearch', :ref => '6.2.2' mod 'extlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-extlib', :tag => 'v2.0.1' mod 'grafana', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-grafana', - :tag => 'v4.2.0' + :tag => 'v6.0.0' mod 'hitch', :git => 'https://forge.softwareheritage.org/source/puppet-ssm-hitch', :ref => 'feature/additional-config' mod 'icinga2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icinga2', :tag => 'v1.3.5' mod 'icingaweb2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icingaweb2', :tag => 'v2.1.0' mod 'inifile', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-inifile', :ref => '2.2.0' mod 'java', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java', :tag => '2.4.0' mod 'kafka', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-kafka', - :ref => 'v5.0.0' + :ref => 'v5.3.0' + +mod 'letsencrypt', + :git => 'https://forge.softwareheritage.org/source/puppet-puppet-letsencrypt', + :ref => 'v4.0.0' mod 'locales', :git => 'https://forge.softwareheritage.org/source/puppet-saz-locales', :ref => 'v2.5.0' mod 'munin', :git => 'https://forge.softwareheritage.org/source/puppet-ssm-munin', :ref => '0.1.0' mod 'mysql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-mysql', :ref => '5.3.0' mod 'nginx', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-nginx', :ref => 'v0.11.0' mod 'ntp', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-ntp', :ref => '6.4.1' mod 'php', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-php', :ref => 'v5.3.0' mod 'postgresql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-postgresql', :ref => '5.3.0' +mod 'pgbouncer', + :git => 'https://forge.softwareheritage.org/source/puppet-covermymeds-pgbouncer', + :ref => '9ec0d8a1255bbb309c2ff38f229167209cad496b' + mod 'puppet', :git => 'https://forge.softwareheritage.org/source/puppet-theforeman-puppet', :tag => '8.2.0' mod 'puppetdb', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-puppetdb', :ref => '6.0.2' mod 'memcached', :git => 'https://forge.softwareheritage.org/source/puppet-saz-memcached', :ref => 'v3.1.0' +mod 'rabbitmq', + :git => 'https://forge.softwareheritage.org/source/puppet-puppet-rabbitmq.git', + :ref => 'v9.0.0' + mod 'resolv_conf', :git => 'https://forge.softwareheritage.org/source/puppet-saz-resolv_conf', :ref => 'v3.3.0' mod 'ssh', :git => 'https://forge.softwareheritage.org/source/puppet-saz-ssh', :ref => 'v3.0.1' mod 'stdlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-stdlib', :ref => '4.25.0' mod 'sudo', :git => 'https://forge.softwareheritage.org/source/puppet-saz-sudo', :ref => 'v5.0.0' mod 'systemd', :git => 'https://forge.softwareheritage.org/source/puppet-camptocamp-systemd', :ref => '1.1.1' mod 'timezone', :git => 'https://forge.softwareheritage.org/source/puppet-saz-timezone', :ref => 'v4.1.1' mod 'unattended_upgrades', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-unattended_upgrades', :ref => 'v3.1.0' mod 'varnish', :git => 'https://forge.softwareheritage.org/source/puppet-claranet-varnish', :ref => '5.0.0' mod 'vcsrepo', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-vcsrepo', :ref => '2.3.0' mod 'zookeeper', :git => 'https://forge.softwareheritage.org/source/puppet-deric-zookeeper', :ref => 'v0.7.7' diff --git a/data/defaults.yaml b/data/defaults.yaml index d427595e..6953f68b 100644 --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1,3226 +1,2970 @@ --- +### See also defaults_security.yaml for public key/cert fingerprint blocks +### + dns::local_cache: true dns::nameservers: - 127.0.0.1 dns::search_domains: - internal.softwareheritage.org - softwareheritage.org # dns::forward_zones per-location. No default value # dns::forwarders per-location. No Default value # ntp::servers per-location. Default value: ntp::servers: - 0.debian.pool.ntp.org - 1.debian.pool.ntp.org - 2.debian.pool.ntp.org - 3.debian.pool.ntp.org sudo::configs: {} # smtp::relayhost is per-location. Default value: smtp::relayhost: '[pergamon.internal.softwareheritage.org]' smtp::mydestination: - "%{::fqdn}" smtp::mynetworks: - 127.0.0.0/8 - "[::ffff:127.0.0.0]/104" - "[::1]/128" smtp::relay_destinations: [] smtp::virtual_aliases: [] smtp::mail_aliases: - user: anlambert aliases: - antoine.lambert33@gmail.com - user: ardumont aliases: - antoine.romain.dumont@gmail.com - user: ddouard aliases: - david.douard@sdfa3.org - user: fiendish aliases: - patcherton.fixesthings@gmail.com - user: ftigeot aliases: - ftswh@wolfpond.org - user: olasd aliases: - nicolas+swhinfra@dandrimont.eu - user: morane aliases: - morane.gg@gmail.com - user: postgres aliases: - root - user: rdicosmo aliases: - roberto@dicosmo.org - user: root aliases: - olasd - zack - ardumont - ftigeot - user: seirl aliases: - antoine.pietri1@gmail.com - user: swhstorage aliases: - root - user: swhworker aliases: - zack - olasd - ardumont - user: swhdeposit aliases: - ardumont - user: zack aliases: - zack@upsilon.cc - user: vlorentz aliases: - valentin.lorentz@inria.fr + - user: haltode + aliases: + - haltode@gmail.com locales::default_locale: C.UTF-8 locales::installed_locales: - C.UTF-8 UTF-8 - en_US.UTF-8 UTF-8 - fr_FR.UTF-8 UTF-8 - it_IT.UTF-8 UTF-8 timezone: Etc/UTC packages: - acl - etckeeper - git - htop - ipython3 - molly-guard - moreutils - ncdu - nfs-common - python3 - ruby-filesystem - strace - tmux - vim - zsh packages::desktop: - autojump - chromium - curl - emacs - ethtool - gnome - i3 - ii - libx11-dev - mosh - myrepos - net-tools - ruby-dev - rxvt-unicode-256color - screen - scrot - tcpdump - tree - vim-nox - weechat - weechat-scripts packages::devel: - arcanist - elpa-magit - git-email - gitg - gitk - ltrace - perl-doc packages::devel::debian: - devscripts - dpkg-dev - reprepro - sbuild packages::devel::python: - graphviz - make - python3-arrow - python3-azure-storage - python3-blinker - python3-celery - python3-cffi - python3-click - python3-dateutil - python3-dev - python3-dulwich - python3-flake8 - python3-flask - python3-flask-api - python3-flask-limiter - python3-flask-testing - python3-libcloud - python3-msgpack - python3-nose - python3-psycopg2 - python3-pygit2 - python3-requests - python3-retrying - python3-sphinx - python3-subvertpy - python3-vcversioner - python3-venv - python3-wheel packages::devel::broker: - rabbitmq-server packages::devel::postgres: - apgdiff - barman - check-postgres - libpq-dev - postgresql - postgresql-autodoc - postgresql-client - postgresql-contrib - postgresql-doc - postgresql-plpython3-9.6 users: root: uid: 0 full_name: shell: /bin/bash groups: [] authorized_keys: root@louvre: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQDMLEWHlUQldlvZs5rg0y42lRNAfOhD+6pmO8a73DzpJWHTqvAlfteLpU78IPjSacB4dO5ish1E/1RX/HC+Bt8p2v4RBqbCnVLx2w+Hx4ahWu6qbeTVmTz+U++1SQrHnL08fSlhT0OekCw0lRZM2sQq21FZi6+vul97Ecikag4Xaw6Qfumylu94pM3t05uzTUlKk1+6VMCjhT8dlSe8VS8OirVQpE/OqYtTMAWtQaMXGHPCsqDdYRAKzkJ8GjH7ydZmX5VCRyqS0RvPKAlcJfLCs5HBtv0u5rbeGtiHhuzhj/j3YgS/6NJOC2mUfcetcDOMPLnhkKpnF0vUAzTsJ7aR + root@banco: + type: ssh-ed25519 + key: AAAAC3NzaC1lZDI1NTE5AAAAIDcljv9eR52wJsu9yYan6/riIQw70lQuyz+Qt0XpGXMs zack: uid: 1000 full_name: Stefano Zacchiroli shell: /usr/bin/zsh groups: - adm - swhdev - swhstorage - swhscheduler - swhdeploy - sudo - gitorious - swhteam authorized_keys: zack-software-heritage: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQDU0O8tkUqtQCelLEatOGfGpx1sIwHPSMA+7OdXoZjZG5pT9Sfgf3ITsNgo1iYWge5bpH/TKhhvf20B05fa8cCEE5ULaD+xdV9eTIvBEaCiP36HH33WNl/UV8T8klTG2sqBXUgLMJuinfGkuRJ977ndm7mjNwzl3Ghf6JwKfpHrvob4GLc0hm54yzcnNEzQZLcdxmOCWdwTINKnL+W/DDM8NR3vNF6T5+xaiLJzsS0IGcTubklugD3m05qbswS/uACWys3FzRM8tttw/0wCRrC9SCSKoDLonab5y3Ld6vCj1k12J2RAHSqJYwVCm70JRPWZcmU67Udi6kbqkJMftp04K0pplu8V7RLPrpwLyH4sPx7Kkhslvxqj0rerLPOkoDkqneFgxNoMcxN5ayod7fBJAq5jQUmGozeTtgPLKybnxRDhsYpkEH9paZroQ3CqDsA0dptOpedVpcQUSbiLMaYd8kgCPkVIdKANnTGGXDcTfWv21IvFx6sKm1kld2Me3ExVMq7JFcmXutF/IQom9F4vj/xd/7Lt4KmqZKyiAq4n5iaPIRUbZvmwd2D6umOHpMGlqKwtsiWRUYnAVvhRfuSZmgrGgliYiYr+vU2xeWe+XXQhP9vt3eItmdSp/8/+a2lqaIE9slE75hEI2n8in7DeSn6QhFDbyUKwZz5OwK7QVw== olasd: uid: 1001 full_name: Nicolas Dandrimont shell: /bin/bash groups: - adm - swhdev - swhstorage - swhscheduler - swhdeploy - sudo - gitorious - swhteam authorized_keys: nicolasd@darboux: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQDZ1TCpfzrvxLhEMhxjbxqPDCwY0nazIr1cyIbhGD2bUdAbZqVMdNtr7MeDnlLIKrIPJWuvltauvLNkYU0iLc1jMntdBCBM3hgXjmTyDtc8XvXseeBp5tDqccYNR/cnDUuweNcL5tfeu5kzaAg3DFi5Dsncs5hQK5KQ8CPKWcacPjEk4ir9gdFrtKG1rZmg/wi7YbfxrJYWzb171hdV13gSgyXdsG5UAFsNyxsKSztulcLKxvbmDgYbzytr38FK2udRk7WuqPbtEAW1zV4yrBXBSB/uw8EAMi+wwvLTwyUcEl4u0CTlhREljUx8LhYrsQUCrBcmoPAmlnLCD5Q9XrGH ardumont: uid: 1003 full_name: Antoine R. Dumont shell: /usr/bin/zsh groups: - adm - swhdev - swhstorage - swhscheduler - swhdeploy - sudo - gitorious - swhteam authorized_keys: eniotna.t@gmail.com: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQDZarzgHrzUYspvrgSI6fszrALo92BDys7QOkJgUfZa9t9m4g7dUANNtwBiqIbqijAQPmB1zKgG6QTZC5rJkRy6KqXCW/+Qeedw/FWIbuI7jOD5WxnglbEQgvPkkB8kf1xIF7icRfWcQmK2je/3sFd9yS4/+jftNMPPXkBCxYm74onMenyllA1akA8FLyujLu6MNA1D8iLLXvz6pBDTT4GZ5/bm3vSE6Go8Xbuyu4SCtYZSHaHC2lXZ6Hhi6dbli4d3OwkUWz+YhFGaEra5Fx45Iig4UCL6kXPkvL/oSc9KGerpT//Xj9qz1K7p/IrBS8+eA4X69bHYYV0UZKDADZSn ardumont@louvre: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQC0Xj8nwGWTb6VGFNIrlhVTLX6VFTlvpirjdgOTOz8riRxBTS9ra35g3cz8zfDl0iVyE455GXzxlm33w/uu3DX0jQOIzkcoEBRw+T33EK89lo6tCCd9xQrteWCTNR1ZBFloHSnYk2m7kw9kyrisziyAdULsCrXmMd3BH1oJyEpISA+sv/dtVpIOWdEQmkbLmdHl2uEdjBLjqb3BtAp2oJZMmppE5YjAx0Aa1+7uSnURf7NnwMx+0wTDMdfqn8z4wqI8eQny+B+bqLH9kY++52FfMVALuErGh5+75/vtd2xzRQamjKsBlTGjFFbMRagZiVNLDX2wtdudhNmnQDIKA+rH swhworker: uid: 1004 full_name: SWH Worker Acccount shell: /bin/bash groups: - swhdeploy - gitorious swhstorage: uid: 1005 full_name: SWH Storage Account shell: /bin/bash groups: - swhdeploy - swhstorage swhwebapp: uid: 1006 full_name: SWH Web App Account shell: /bin/bash groups: [] swhbackup: uid: 1007 full_name: SWH Backup Account shell: /bin/bash groups: [] rdicosmo: uid: 1008 full_name: Roberto Di Cosmo shell: /bin/bash groups: - swhteam authorized_keys: dicosmo@voyager: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQC5aS/3Cps2Ru9EW+nIF9Z9o6/xq1thwtCgpIjSPgcrm2BVisj6xbD5OOapS3U6BpLKjWZG8sMGBCsJJ3S1cP0s2I+xHFToqCcbfOxIe/tq/UgTtxGJ0+TfUKNoD+QJjIKnjyC+HVEQm5bSm8mJv0vptj4On8yNopytSGuLcFHHnMB2t+IOkHnTW7n3emhh3SZKAcpI1h7WvPqsqBobMFDMeqvGeHaH2AM2OSoUi7AY+MmcVL0Je6QtJqpz60QI5dvaM4AsobC12AZSJKXnuqQTY6nJy4r9jPRK8RUqo5PuAAsNtlxf5xA4s1LrDR5PxBDpYz47Pq2LHtI9Hgf/SFB3IqZeBKqquMI1xThRBwP307/vOtTiwJr4ZKcpOH+SbU7Tnde4n8siM719QZM8VITtrbwm/VBiEwvhGC/23npX4S55W7Et/l9gmeP3Q+lSw50vBuQhBSn7BzedPM1CqbTN/zqM8TCDUtPVIo+6b2s5ao/Vcq9vBXm5bP0xZeNsqsCl05zpCShudKpT6AlMGAaRTd6NUHHsf4D1JjNx3v42R3vQr6OgHELVMGECuyPs3zWHOS/P6AdD0yJTSOMaklRh2HGN8uj0+aQ7RhnrkYqRfhN+6UkrTANuxdb44AGdLmBAKIYglVrAJe+DEji/LzJdZ22baAWg4ar/WikpFJtxkw== swhteamannex: uid: 1009 full_name: SWH Team Git Annex Account shell: /bin/bash groups: - swhteam authorized_keys: swhteamannex@louvre: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQDL/Ef9kktq/QkJ0lohan8ObQ3o7hMf7EOQPjO+u7UhIMjBNufJnaftQRGIA6N1/wEsDvxxNNz75/oJECJHgZs2OOTJJPsKfYeybmSBocSa/sn6IKK7/b/qlwHJlSGWPGVgbtfP0KexlSAKAmKZuJyqjES5igTLV5w4wTjvUUen9QyefuUehnCX3MJhTpoyixp7icXE80aNMaCPwHZppKb/28sNlPX3MbSONGM45wSFRXNuj0mAAjrgojkhAqFNnI9oKNAt9mDcw1hV0n86VvrDhEbMCJP/z58ecn376BgyXQ8zNUPIr2g0SrSPUNjfxZHfJ0XYpF7624wOMOmZE3fsQzZh+FeMF0IFRPvcG34RaelV9dXiy+/q45oqwbMF464gFSbyt++4jpgqHdsARM4zR//jBhyLvFXR+GaKC3hFENno5W5Raff4XE5rzN/q9jVJBNfvfuEPWrapyM3A/ePeuK3SyNJwyIx+bOEQXsRdxEWKszTeJO2SLPWtCrKrC+G4/HktQSQOj5S9a+N6HoKD8E889eBEYoeZGPIuzMot4cuUlyPt3P99z4oRIaeC6XwUCvZCD2DaTAkQWQMsmOn+soaeZ1zBHbsCBbV0mBMRx7K4Vjs62vhSelryQAXW+cBgd6+f5XBjOnNhHQhsNsDfYP4Kmztn58faQV2TzGG5ow== swhscheduler: uid: 1010 full_name: SWH Scheduler Account shell: /bin/bash groups: - swhscheduler jbertran: uid: 2001 full_name: Jordi Bertran de Balanda shell: /bin/false password: "!" qcampos: uid: 2002 full_name: Quentin Campos shell: /bin/false password: "!" gitorious: uid: 5000 full_name: Gitorious System User shell: /bin/false groups: - gitorious fiendish: uid: 1011 full_name: Avi Kelman shell: /bin/bash groups: - swhdev - swhstorage - swhteam authorized_keys: avi@Temperance: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAAEAQCi9FEAQZ+WvvuVKjm5EJRgW1xwKfQwndBtZhJ8qqFVpHCl25tQW4ndfHVdr1BJmXVbAKlfsZs/hYBQq6JbTaxHZKGyajNZfuXOQ71CR2pn0XytpcMyt8l893BLOriTLhKoKXNwzjdCY7LmhNWDqR0GHm8C1d6fI7oYG+0PiTQr8cJ8wYvdVH5qrhzKMLJmGIbSOkQxCFYUjpXLk53fsAVyUMTn5SV9CTfMLCQ4nh2IXbbOVC3sIShNINzhNhkRTbHMHmo4Ujmn2jC6qtk3lawCTmrXNn7/mRgfrSkt5fSisSFM1xZ17wBG+zxQbnK00zcTHfDdxBmmsES5lWIndM4CqUwGgBtGGvrHsNS12XTsmvM4FA2DSiiKVLhTTQnG4B0CUcZFJn+dLDk+9IRN2uDbOY7Sg6653cJWYAeZq4YXtmhccCgtz82I5CtPYTMMZJqlHjvcGl9euafKPC196MPcWPXGNsidP/g3BZNncZE/4RgnbQChqiS4juXLB3KtIH6C50g1GZq/azrvZFlzU0Z9kzK99Vw0xPFIfi5wobWEq0JS31BHsd+NIVgm/riZIheb3hi3jyS2+Awm00aLwjEtS+x31xn99WZg2fRp8xpbbiZwMn5QlxarDWV6afySlWRe+iUdRtM2PGMBwSosQcrvKe/rOD27c/Sa5dWPvrrVsKBEATpFcy5XnNEcPxAbYGx0IY4CulVe0EYVH8jSoRhUQn4O41N8St3cmJPFGBfUP5v7/QdXhR8m+6yZKlaFTzdz0ZEcmc/TaCBxxBywuNPiLHZtIBJVuUPGAZC0eidmYK7iva2/a4lhuDXA5mVYm6lQzjdv+mUGqjkEzQ23LCMg3tA+6ewTZwJzoUTTXosgLqVhJ4KchJTbGYdvG7IRQmmyTzRwHkqJej2e+xgdPPY2BFBl3WA+dEle6gMR7FB54ubIrCdJfb5hwVaGlcK3s4KpXapAjHP7jndsPKVSPP8Ax0fqPiCOw3wTIyQGV+ZfLf7PjqpTAQn7a1QA6CUG47p9HqofEqrrQEDyC9vRan7NeD5IVQq7dpgGwx4zEDnR1m2hfuoXzDRM4Vd+uNhC0WotkGJatKZeaoHx8P9VE8zhir2Py71AxellhfvGuzxPJ4XNDRjAenMmSbVURyLFpt6iA2bFeolCCb7qVodKORvquFRQEXTlkAxRj7OiMDzniRfZbtGER9zkXGp9+Sw3gz+igxND5KAzJgt2ubpChF1KyxKEu1QCeZ3n+1pIdeTSZT0rFSnZNYdb/UuIlKgICqxU0sutXIZqY1k5QSFWVNjA60QuWLMDW5CrhouGdFEBCksKDKiJinhcHXiDx93LJYzG2CVPTuW+gka0ZyfS18pN morane: uid: 1012 full_name: Morane Otilia Gruenpeter shell: /bin/bash groups: - swhdev - swhstorage - swhteam authorized_keys: morane.gg@gmail.com: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQDm8kH1pP+4ENKmpkTCkL2ashxxnOFVndGrcvfX05lV1hOo2NdItpdoR9txIgFEs3d7v73mtH4nWciUyaK7FIByrtvsR2TIhdVgEcb0Xai8viV+sDMTndpiNlWNilbfxm0K70tgpG4BeSWRJy8cPxnCR9CWoB2Vo9Df7lDKz1LXDgfY4VLJd69ahf1DPFUDjpWIEQdPFX2ZyGUYM+0yPXIoyYW/qreDt1JkYZXXVbRAV8j44/TVgTRYJLgYb9ThW6WzlGM1S4uP7GQdAuROCcspqW3ahV/UmV4Z9SM6S34NN182KvM0Ve7uxAPQz+IdWOgZTK0pvd+hfjHKbLSTA6I3 seirl: uid: 1013 full_name: Antoine Pietri shell: /usr/bin/zsh groups: - swhdev - swhstorage - swhteam - swhdeploy authorized_keys: seirl: type: ssh-ed25519 key: AAAAC3NzaC1lZDI1NTE5AAAAILiua8eEg+nU0XSbYPTgnOMftzvpbN+u7v5jDabeO/0E ssushant: uid: 1014 full_name: Sushant shell: /bin/bash groups: - swhdev - swhstorage - swhteam authorized_keys: sushant.mukhija0@gmail.com: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQCvWoRsznkpBZpgGd9qjbW8Vt6wrUSKw/0fm+HzQDEUsiajG9Igy9VJcMiU6CwFmXvsRjY5Sm4z7aZLpCn0d73ixmNWHy60UG1mnN+BVdfzxEZFfBiLw/PBaKgTpa00yMQbvszXfAnlo1ttnadCdCYWBBBYCx25HEp7MJyORFbbVkqukmp1rJN1o4zUKX7jkrnMHH/vaNdH+dBJAQP0rP/o6LR6GNeWX4hW8/0VgdVqiqqdHpy8OvB9tXKCwY1YYmr6lecOS20msak4UDA8uvufPc37KtxVcvbOstMTkmkSZ4RBxruNfF/GK/pUfg2e6j6B9JkNX4D22C/8NJWydMNxGsn3HMe6bu2QcQf+QgfqqzHZzcwQitHPF9vHMLg0TJLZNM1lAFSJeJkls4cVW/fko8pGrYouRv2ktcRAdugmfw/3txDuVJXOjjPdbLrI3+8TPYQvvPMBodgoxbDxKduO4qY+j8PmqdjjMWayDkasy2nIVwbJHQlmz687BvIRLv80t3RqeYpkjUBK6vld5RDgiZN+HdtlyLL49M5BuD7Gu+iXCziCmJjV1LfY3H29oJsx1SICnbuSjXpIRO+XnMTkgNuCOHtjnbErQnGq+tb7l1UDVw5xWqbeWcVIq5/vmPZRnIZyMvAAuq8bMPaew5oywAs55esdJ6oVgO6w4DX1OQ== sushant.mukhija0@gmail.com-2: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQC4oLTkZZscbdiMZnh/p8VhfEyRKapqRT7L24kTxxL9Y9QWj3DzCQ7tj8SK/0Y+O0NcrZtXZS/Mx/f6qAjuZwp6Vzo3hLcB016gd5+6suFwz5wfwLFlwF6zgRFc9SqlKuVQdfc0aUvkjReUUalvqXioUqAWi633ZCMGmvjI3c7paIaeGtmjJ2TUE/Qm7x/XxURwk02Y+t7yNhGRGeaGBTKW/OZmIkkaFiSA2syHVRB+fdynPcilYhzaKKWs+6OjG4osKGVJaSef02dSSYHXI4D0HcLgVjExDCt2GGp0Q8e3fFo954btoXLpmij+fOJH1DpHKfUFyOkAnGVOz1wM3Y97 anlambert: uid: 1015 full_name: Antoine Lambert shell: /bin/bash groups: - swhdev - swhstorage - swhteam - swhdeploy - swhwebapp authorized_keys: antoine.lambert@inria.fr: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQDLWPcZnSUszEedMa39dT3ZCHpRod3NTs6WT4OfMMRVahrhTtWYdSiNGy8U3kEQveTZvMrb9WLtLPB3K8o7Xrf8WCI8iTOl9eb9DVjE9XL+zS0ZAcEmoZ5YH8e3gEDoDm8ZrMxF+V5XSlvhNi6kbWzJdqhXu++bJHHqGrKUHeTCQCfpYYMrsnvhPjtxe+90BK7e+IGm1Ha8LZMCCmOtz0XggxD8d2mFBaP2p8v9xsM48KfwFvsRMb3TZIaO/+NcsRSTe7wfFAR1pb14pi5LZAHeb2tpWfVH2vQGaE7Rej+Ycf4UOeaRmFGpimw7u7fugvDvKfZ/vs7w7Qs2RtxNdqJf9JM+vvi78OQbloufot1Tz2r19aDbhM9nsCn+Uo3rNfkmD+UcSMKrRJCMEXVBbaY/bgzs7XoqCJ8ODE2U/dF3NtHBZr+CB52iilUtemXy+Xwqw4TSs/r9vW7/XueTdb0Yp/cUs5uLCqCwlMpGS5okorpdJextp5gRuN6EMlUo6PffRiz5T0CqKm1xJu0NeT0EaacAXoGTDQaS4pIQGglqWfAOmjej9dM8gxAF6rgrx70uJt6Hy18tvzdB5iwJ4F2LUjcZhFnrxjUDzhjPoDBiRtPNgEKrCc30OHsveqXwMPo3v/d3np1Vpkum0JEwmp83q92P5T2rbf+wiruxZhhtww== grouss: uid: 1016 full_name: Guillaume Rousseau shell: /bin/bash groups: - swhteam authorized_keys: guillaume.rousseau@univ-paris-diderot.fr: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Akcdxrod/MFcHg53dCf7iZY/ph9MR0tWU08pjMDfU04j1nAgmHmvumYbxBtFNnd0nu4A9YY4nT79273PCE3c6ba3zSGS9DBYhrASGDqHBECrgEREM3YPXpA2NI0FKEZ878Ic3CQlYaOmRoe/QkFpm2j8CMoG4VdKp0EcvV1RCTgWqJY1P4KC30CJUg+OdGRaaqHEoSskjstU5yjbZCC9M90Hz0xO+MsMl/xKdcDmvwbLDMtp/3SKDQeyN4Q7Uu/zZwoZ8FmgEU4Xp7nKN3yCiEB9rqMkP/lLY71hTPHn/GiZnPo4rWL13w3unuI3X0GDpqxPxjt0LZN4xQEGEn+1 ftigeot: uid: 1017 full_name: Francois Tigeot shell: /usr/bin/zsh groups: - adm - swhteam - sudo authorized_keys: ftswh@wolfpond.org: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQDXpL8nJw6kUgXSxxO1GrScPWYA7wa1ng7VwF0l1StAVavRGcYpYHdahCYrdIz4QF6fY+HSsUoTwyMHLsmU8tHOK6Urx+EAnUB0t6x0v3wfuwKslAK0hHSgZYGIEkFd4qWX2guHwekIILrSIj+TnlodFMUw5ulCrcGIj8Ss2YZlxXkw8opFLNA22cmrYVwHryv0yfYFejnpBlM9WLA8R8ZKc5Du9Enx5fG0YIFjPH4aaU20W77b+aIQrJTIostANwhSn2TnbhubFklqM314LfgUpVC7+rIe5L5Vk3iY0FHhYN4cBi1DK7NVrw1gwfuBt2Hpjnv/4FRazmn7LM+F0eLdYWzIFVv68vrCtOSSxvUeJtjDYWLWPn/sBJQ8/5yOP3vhxAJAgIvvHnWuGIPVLK9UO6lFGmhaxPP01I8pX/aE7AoRNqFFUXxnkSkSbjLNEGpDSfGVmxnM1fYlqkZzabPFEAbkm2iUKpAEcJWugLMNCGR5Vy2Lspf7DjviNelLvuIOGBeNhY99ecqFLRej0oTf46HHpsaZEWx8zdg7BvAjkCmII8YtZyqYD7tW8P3yEFPkBRhpu+85FwUe+335ZiTPnlR37+QHz4Ax4vBPB3/3q9u+JTXnKqkm3kfiLUcPu1VrwkLBXKr56fnw4+ckUNTVrnkt8/Czz9RghnCM4cD/3w== swhdeposit: uid: 1018 full_name: SWH Deposit App Account shell: /bin/bash groups: - swhscheduler swhvault: uid: 1019 full_name: SWH Vault Account shell: /bin/bash groups: - swhdeploy - swhstorage - swhvault ddouard: uid: 1020 full_name: David Douard shell: /bin/bash groups: - swhdev - swhteam + - swhscheduler authorized_keys: david.douard@sdfa3.org: type: ssh-rsa key: AAAAB3NzaC1yc2EAAAADAQABAAACAQCoON7De2Bx03owpZfzbOyucZTmyQdm7F+LP4D4H9EyOFxtyMpjH2S9Ve/JvMoFIWGQQlXSkYzRv63Z0BzPLKD2NsYgomcjOLdw1Baxnv8VOH+Q01g4B3cabcP2LMVjerHt/KRkY3E6dnKLQGE5UiER/taQ7KazAwvu89nUd4BJsV43rJ3X3DtFEfH3lR4ZEIgFyPUkVemQAjBhueFmN3w8debOdr7t9cBpnYvYKzLQN+G/kQVFc+fgs+fFOtOv+Az9kTXChfLs5pKPBm+MuGxz4gS3fPiAjY9cN6vGzr7ZNkCRUSUjJ10Hlm7Gf2EN8f+k6iSR4CPeixDcZ+scbCg4dCORqTsliSQzUORIJED9fbUR6bBjF4rRwm5GvnXx5ZTToWDJu0PSHYOkomqffp30wqvAvs6gLb+bG1daYsOLp+wYru3q09J9zUAA8vNXoWYaERFxgwsmsf57t8+JevUuePJGUC45asHjQh/ON1H5PDXtULmeD1GKkjqyaS7SBNbpOWgQb21l3pwhLet3Mq3TJmxVqzGMDnYvQMUCkiPdZq2pDplzfpDpOKLaDg8q82rR5+/tAfB4P2Z9RCOqnMLRcQk9AluTyO1D472Mkp+v5VA4di0eTWZ0tuzwYJEft0OVo+QOVTslCGsyGiEUoOcHzkrdgsT5uQziyAfgTMSuiw== vlorentz: uid: 1021 full_name: Valentin Lorentz shell: /usr/bin/zsh groups: - swhdev - swhteam authorized_keys: valentin.lorentz@inria.fr: type: ssh-ed25519 key: AAAAC3NzaC1lZDI1NTE5AAAAILsRMQjrrfUjX1ka9e6YlyMyDvTC+qk5a21Fp9yXYI7p + vlorentz@softwareheritage.org: + type: ssh-ed25519 + key: AAAAC3NzaC1lZDI1NTE5AAAAIIjJoY4XBTTNsxLVF/sUKBI4WGR2AIiR9qfMdspnsRfJ + haltode: + uid: 1022 + full_name: Thibault Allancon + shell: /usr/bin/zsh + groups: + - swhdev + - swhteam + authorized_keys: + haltode@gmail.com: + type: ssh-ed25519 + key: AAAAC3NzaC1lZDI1NTE5AAAAIORGwY56PpvgwMWqDei718PPriV6U7LL5JMPJWS7zTcg groups: adm: gid: 4 # assigned from base-files zack: gid: 1000 olasd: gid: 1001 ardumont: gid: 1003 ddouard: gid: 1020 swhworker: gid: 1004 swhdev: gid: 1002 swhstorage: gid: 1005 swhdeploy: gid: 1006 swhbackup: gid: 1007 swhwebapp: gid: 1008 swhteam: gid: 1009 swhscheduler: gid: 1010 sudo: gid: 27 # assigned from base-files gitorious: gid: 5000 swhdeposit: gid: 1018 swhvault: gid: 1019 +gunicorn::statsd::host: 127.0.0.1:8125 + munin::node::allow: - 192.168.100.20 munin::node::network: "%{lookup('internal_network')}" munin::node::plugins::enable: - apt - postfix_mailvolume - postfix_mailqueue munin::node::plugins::disable: - apt_all - df_inode - entropy - exim_mailstats - exim_mailqueue - interrupts - irqstats - netstat - nfs4_client - nfsd4 - open_files - open_inodes - proc_pri - vmstat munin::master::hostname: munin.internal.softwareheritage.org munin::plugins::rabbitmq::messages_warn: 18000000 munin::plugins::rabbitmq::messages_crit: 20000000 munin::plugins::rabbitmq::queue_memory_warn: 1073741824 # 1GB munin::plugins::rabbitmq::queue_memory_crit: 2147483648 # 2GB -rabbitmq::monitoring::user: swhdev -# rabbitmq::monitoring::password in private data +rabbitmq::enable::guest: false +# following password key in private data +# - rabbitmq::monitoring::password +# - swh::deploy::worker::task_broker::password +# - swh::deploy::scheduler::task_broker::password +rabbitmq::server::users: + # monitoring user + - name: swhdev + is_admin: true + password: "%{hiera('rabbitmq::monitoring::password')}" + tags: [] + - name: swhconsumer + is_admin: false + password: "%{hiera('swh::deploy::worker::task_broker::password')}" + tags: [] + - name: swhproducer + is_admin: false + password: "%{hiera('swh::deploy::scheduler::task_broker::password')}" + tags: + - management puppet::master::hostname: pergamon.internal.softwareheritage.org puppet::master::puppetdb: pergamon.internal.softwareheritage.org strict_transport_security::max_age: 15768000 # Those variables get picked up by 'include ::php::fpm::daemon' php::fpm::daemon::log_owner: www-data php::fpm::daemon::log_group: adm php::fpm::daemon::log_dir_mode: '0750' # Those variables get picked up by 'include ::apache' apache::server_tokens: 'Prod' apache::server_signature: 'Off' apache::trace_enable: 'Off' # Those variables get picked up by 'include ::apache::mod::passenger' apache::mod::passenger::passenger_root: /usr/lib/ruby/vendor_ruby/phusion_passenger/locations.ini # Those variables need to be set manually in the SSL vhosts. apache::ssl_protocol: all -SSLv2 -SSLv3 apache::ssl_honorcipherorder: 'On' apache::ssl_cipher: EDH+CAMELLIA:EDH+aRSA:EECDH+aRSA+AESGCM:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH:+CAMELLIA256:+AES256:+CAMELLIA128:+AES128:+SSLv3:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!DSS:!RC4:!SEED:!ECDSA:CAMELLIA256-SHA:AES256-SHA:CAMELLIA128-SHA:AES128-SHA apache::hsts_header: "set Strict-Transport-Security \"max-age=%{hiera('strict_transport_security::max_age')}\"" # Those variables need to be set manually for all vhosts apache::http_port: 80 apache::https_port: 443 # Hitch TLS proxy configuration hitch::frontend: "[*]:10443" hitch::proxy_support: false hitch::http2_support: false # Varnish configuration varnish::http_port: 10080 varnish::proxy_port: 6081 varnish::http2_support: false varnish::listen: - ":%{hiera('varnish::http_port')}" - "[::1]:%{hiera('varnish::proxy_port')},PROXY" varnish::backend_http_port: "%{hiera('apache::http_port')}" varnish::admin_listen: 127.0.0.1 varnish::admin_port: 6082 varnish::storage_type: malloc varnish::storage_size: 256m varnish::storage_file: /var/lib/varnish/varnish_storage.bin # varnish::secret in private-data -ssl: - star_softwareheritage_org: - certificate: | - -----BEGIN CERTIFICATE----- - MIIIXzCCB0egAwIBAgIQDPHPysLFk3NJoSYi+mieNDANBgkqhkiG9w0BAQsFADBk - MQswCQYDVQQGEwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJ - QW1zdGVyZGFtMQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wg - Q0EgMzAeFw0xODA5MjcwMDAwMDBaFw0yMDEyMzAwMDAwMDBaMIGQMQswCQYDVQQG - EwJGUjEVMBMGA1UEBxMMUm9jcXVlbmNvdXJ0MUkwRwYDVQQKE0BJbnN0aXR1dCBO - YXRpb25hbCBkZSBSZWNoZXJjaGUgZW4gSW5mb3JtYXRpcXVlIGV0IGVuIEF1dG9t - YXRpcXVlMR8wHQYDVQQDDBYqLnNvZnR3YXJlaGVyaXRhZ2Uub3JnMIICIjANBgkq - hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA5CWwZR0vKg7KRxYon/cqja0mpVjXAKMV - KyUQFvTRoY/fD07t51Z5nC0pBi4VxjVJKFN8UO6fX0kJhy4Lp+ErsOPw80YcCMq1 - 86W9Vo6DG2kqjHp0TnsUm+wu2J8F9fQR/HfANx7rflSs2CzBvf2nr0u4HevnCzk+ - zxky+j+YS9l2Xx5Svz/DTbVpHn162Sf+03g908dsLs2nq3IJbJnzkNkSWpizq/aE - Ud/vocQEK66tKN5NLfHezljDQBbK8cySUaX6BFjPRmDcw25rKDgClMIGbYA7P7Hl - fswcrsxNxAtwa9ZDTnPW5zRp9aeVcm5NT57vzKZGGsRt+aEzeq+IvcIR9RCQOgIP - G3Dzjva0eunppslXOz8hiZd86Omo38yfFi6xJS3qH6nMSItdKa+4yxKCMVlJkx00 - WtsovY7kfYFpzJfd9Mk7jTBdmD5WRcBiatFGNgaqXstuVHS1CGK2qs5+iOiaXOdn - 6glIJgN/dKcgA6nvBIYDguvgYU59HxS56Dq9PqdpFy1B6Qghu2Zj4P5iJG+Gl+fP - aK0WG5ISsdsr6QGLcJnD3Zjk5F68vo/on1xm3Zk+v7mHlps2BLgxUiP3D5vTpUgY - L0xqKfbjDr65R0s7CxZZmXdOUsHgo7P99BKUYiM3Tw9G4AOkFJPLP2OrbLckdmmO - K4Iwxhbj4H8CAwEAAaOCA94wggPaMB8GA1UdIwQYMBaAFGf9iCAUJ5jHCdIlGbvp - URFjdVBiMB0GA1UdDgQWBBRf7/Wq5kiF/QnNu0Bxe2q6bZjeTDA3BgNVHREEMDAu - ghYqLnNvZnR3YXJlaGVyaXRhZ2Uub3JnghRzb2Z0d2FyZWhlcml0YWdlLm9yZzAO - BgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMGsG - A1UdHwRkMGIwL6AtoCuGKWh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9URVJFTkFT - U0xDQTMuY3JsMC+gLaArhilodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vVEVSRU5B - U1NMQ0EzLmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsGAQUFBwIB - FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjBuBggrBgEF - BQcBAQRiMGAwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTA4 - BggrBgEFBQcwAoYsaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL1RFUkVOQVNT - TENBMy5jcnQwDAYDVR0TAQH/BAIwADCCAfUGCisGAQQB1nkCBAIEggHlBIIB4QHf - AHYApLkJkLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BAAAAFmG53a8gAABAMA - RzBFAiEA4FsESbszMapP/1928EYdqTnGxwDpAnjibblmY4Qb4nkCIF/dTJoCbDnO - 0LSLS+P5bc9cAkgGAyXBwpvB8+ZJszctAHYAh3W/51l8+IxDmV+9827/Vo1HVjb/ - SrVgwbTq/16ggw8AAAFmG53bzwAABAMARzBFAiEAlpn5W7rsxL6urq8S3tMw9Hr+ - cZaWAg7dp6kolPecYMECIDFDDBElmoHXCzW35kQXzGxgwrj/ns1nrbc7T5PzjcKw - AHUAu9nfvB+KcbWTlCOXqpJ7RzhXlQqrUugakJZkNo4e0YUAAAFmG53b9AAABAMA - RjBEAiA8F7P2NjU/AXfguds8cWDRqJsuVpF3qrzVZ4hnlTBeuQIgFohcpNsjkcY3 - hsgDI6j2VMuk5TVLd/2ehdtaT8lUX/kAdgBvU3asMfAxGdiZAKRRFf93FRwR2QLB - ACkGjbIImjfZEwAAAWYbndzFAAAEAwBHMEUCIEwt+EUHkqPBraADN4jCGF6tfzp6 - LcAIOWt4m/8OMM4pAiEA8m3u/w219RAamjShsaguGbQfxRZGcQcVc0++zXilApYw - DQYJKoZIhvcNAQELBQADggEBAKr3KUfjEWsIk1umodHo8Cck49TgioNy6e8w9ajW - 9Sp3FB/MRskWtL3NU64CecMJlvgJrZogUv+eZlquLCnTiKQdmTY0xYVmt+igT9ca - o907aEc2puEHDm8KsCvwLM6nbVPR03nEaj/P6aSM42eordzcWyQY5+uy69r1yfus - gVwJjm9aq+EX7KoK+HDE/fF8BOuHxoWzuTxbKTGjD7lEwMi+MdFGwhX/WEOu3Um8 - MZzX4bRVAgtJNbbrBRYQ1D2gSA+VAmXEBJhYV5UboNWgGE6EwMwTOCTyhpAjHZdy - 3yFaxgRObjDQEO/LMG+/iDKMUV1T2onVux89LEpaEzeJcnA= - -----END CERTIFICATE----- - ca_bundle: | - -----BEGIN CERTIFICATE----- - MIIE+zCCA+OgAwIBAgIQCHC8xa8/25Wakctq7u/kZTANBgkqhkiG9w0BAQsFADBl - MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 - d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv - b3QgQ0EwHhcNMTQxMTE4MTIwMDAwWhcNMjQxMTE4MTIwMDAwWjBkMQswCQYDVQQG - EwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJQW1zdGVyZGFt - MQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wgQ0EgMzCCASIw - DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMV2Dw/ZQyk7bG3RR63eEL8jwnio - Snc18SNb4EweQefCMQC9iDdFdd25AhCAHo/tZCMERaegOTuBTc9jP8JJ/yKeiLDS - lrlcinQfkioq8hLIt2hUtVhBgUBoBhpPhSn7tU08D08/QJYbzqjMXjX/ZJj1dd10 - VAWgNhEEEiRVY++Udy538RV27tOkWUUhn6i+0SftCuirOMo/h9Ha8Y+5Cx9E5+Ct - 85XCFk3shKM6ktTPxn3mvcsaQE+zVLHzj28NHuO+SaNW5Ae8jafOHbBbV1bRxBz8 - mGXRzUYvkZS/RYVJ+G1ShxwCVgEnFqtyLvRx5GG1IKD6JmlqCvGrn223zyUCAwEA - AaOCAaYwggGiMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMHkG - CCsGAQUFBwEBBG0wazAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu - Y29tMEMGCCsGAQUFBzAChjdodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln - aUNlcnRBc3N1cmVkSURSb290Q0EuY3J0MIGBBgNVHR8EejB4MDqgOKA2hjRodHRw - Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3Js - MDqgOKA2hjRodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVk - SURSb290Q0EuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxo - dHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBRn/YggFCeYxwnS - JRm76VERY3VQYjAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq - hkiG9w0BAQsFAAOCAQEAqSg1esR71tonHqyYzyc2TxEydHTmQN0dzfJodzWvs4xd - xgS/FfQjZ4u5b5cE60adws3J0aSugS7JurHogNAcyTnBVnZZbJx946nw09E02DxJ - WYsamM6/xvLYMDX/6W9doK867mZTrqqMaci+mqege9iCSzMTyAfzd9fzZM2eY/lC - J1OuEDOJcjcV8b73HjWizsMt8tey5gvHacDlH198aZt+ziYaM0TDuncFO7pdP0GJ - +hY77gRuW6xWS++McPJKe1e9GW6LNgdUJi2GCZQfXzer8CM/jyxflp5HcahE3qm5 - hS+1NGClXwmgmkMd1L8tRNaN2v11y18WoA5hwnA9Ng== - -----END CERTIFICATE----- - star_internal_softwareheritage_org: - certificate: | - -----BEGIN CERTIFICATE----- - MIIGsDCCBZigAwIBAgIQD1wiwe+Cg9VcwL5QrB+UqzANBgkqhkiG9w0BAQsFADBk - MQswCQYDVQQGEwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJ - QW1zdGVyZGFtMQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wg - Q0EgMzAeFw0xNTEwMTUwMDAwMDBaFw0xODEwMTkxMjAwMDBaMIHIMQswCQYDVQQG - EwJGUjERMA8GA1UECBMIWXZlbGluZXMxFTATBgNVBAcTDFJvY3F1ZW5jb3VydDFJ - MEcGA1UEChNASW5zdGl0dXQgTmF0aW9uYWwgZGUgUmVjaGVyY2hlIGVuIEluZm9y - bWF0aXF1ZSBldCBlbiBBdXRvbWF0aXF1ZTEaMBgGA1UECxMRU29mdHdhcmUgSGVy - aXRhZ2UxKDAmBgNVBAMMHyouaW50ZXJuYWwuc29mdHdhcmVoZXJpdGFnZS5vcmcw - ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDxz6BsVNPDk1fSAltRCwiU - 0iZtd+10WKJL+tgPmv/s3+WgIqj5+hQ/iZsAc7Y45yQo4yU+PsDq+BIFR2yt5rtk - B0Xz4sNHAo585IEvWvf3wAULf6GQ17o2XRxN5IZfNgLfRy6jQjeIbiFmO5M2g6To - Fl+MMAuK0+u9u6az41eBl1etTe7QjGaL+B45kfLyLeBB4rhusEQSPRTrGrdrgdEt - di9jedDjbMkV2B197D7CZHqKRR3B+yuRwgd/t/OqIBpN0M//kPE6AJzpLN2B4Z6L - HGSeyprQleFkS3d7hAlZal/9di8C+38bAGmTW8dQLGwFeYTHN6hBuMYm0y4Yk32W - 423rPZuguA3j2sIxOuIExKmlMDzuGociy3npfjTvWCi/6ESo0hkImnrWKUZ0eyFH - h52DsPC6ePhou4qh+KdwObxJmYgS69YhHgrsfZokbbWy/hj6N6dAFmTYJQ9hcrpr - x2CHHAmbd7J8gTInMDHlUMJ1vcL11coRpmfevMiaK4szZwzRfI+xsevVeHRusxpC - ErrtW+T+9rSfFVIROjXirD4uok8R3UTMpwNOhPQtKTHN/8v9Cvp5BzY0XNnRiVFc - lL/4YezG+YO6r4GieWEJyLyB7R/JRJbSuCJLzzqZek4twESiV7mtKe2P1clUF1z+ - O1+pLFUaRSSgpGxypDgIXQIDAQABo4IB9zCCAfMwHwYDVR0jBBgwFoAUZ/2IIBQn - mMcJ0iUZu+lREWN1UGIwHQYDVR0OBBYEFEZAY+NxWfpgt+qo+0pdRQ5uTsPeMEkG - A1UdEQRCMECCHyouaW50ZXJuYWwuc29mdHdhcmVoZXJpdGFnZS5vcmeCHWludGVy - bmFsLnNvZnR3YXJlaGVyaXRhZ2Uub3JnMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUE - FjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwawYDVR0fBGQwYjAvoC2gK4YpaHR0cDov - L2NybDMuZGlnaWNlcnQuY29tL1RFUkVOQVNTTENBMy5jcmwwL6AtoCuGKWh0dHA6 - Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9URVJFTkFTU0xDQTMuY3JsMEwGA1UdIARFMEMw - NwYJYIZIAYb9bAEBMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LmRpZ2ljZXJ0 - LmNvbS9DUFMwCAYGZ4EMAQICMG4GCCsGAQUFBwEBBGIwYDAkBggrBgEFBQcwAYYY - aHR0cDovL29jc3AuZGlnaWNlcnQuY29tMDgGCCsGAQUFBzAChixodHRwOi8vY2Fj - ZXJ0cy5kaWdpY2VydC5jb20vVEVSRU5BU1NMQ0EzLmNydDAMBgNVHRMBAf8EAjAA - MA0GCSqGSIb3DQEBCwUAA4IBAQBt7fVm/+sNa5VkhdxfiSSAqrqrjrNPI2izU3Bs - fNrN+o7buLGTmA83WJaJsX74kdegqsmT5J3qBO/RDNpvIJqeGBQdDO2L4tRCHxSZ - mJ0o7GKFfWY01hF6J6jSHzOehpL/UQ37U1Kh3l7GLBkOhPubV9wkG5jMLlrYNCAd - gY59W84hH9QCV+oO44F4Q3bdwTcXrZuQ3tpyqqgukmrCm0TcK7pOa5FWZ4r6ZIWe - RNpor8OyVqmMj8U2NX2478LsyE4Ut6NrtdOQHnVVFgPlsuiQgTUfqgL5XCMQw4vz - 5vNH/vho0zTFoJjB68mos1xazWNYqC+QmKXcsFWeodct9l3F - -----END CERTIFICATE----- - ca_bundle: | - -----BEGIN CERTIFICATE----- - MIIE+zCCA+OgAwIBAgIQCHC8xa8/25Wakctq7u/kZTANBgkqhkiG9w0BAQsFADBl - MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 - d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv - b3QgQ0EwHhcNMTQxMTE4MTIwMDAwWhcNMjQxMTE4MTIwMDAwWjBkMQswCQYDVQQG - EwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJQW1zdGVyZGFt - MQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wgQ0EgMzCCASIw - DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMV2Dw/ZQyk7bG3RR63eEL8jwnio - Snc18SNb4EweQefCMQC9iDdFdd25AhCAHo/tZCMERaegOTuBTc9jP8JJ/yKeiLDS - lrlcinQfkioq8hLIt2hUtVhBgUBoBhpPhSn7tU08D08/QJYbzqjMXjX/ZJj1dd10 - VAWgNhEEEiRVY++Udy538RV27tOkWUUhn6i+0SftCuirOMo/h9Ha8Y+5Cx9E5+Ct - 85XCFk3shKM6ktTPxn3mvcsaQE+zVLHzj28NHuO+SaNW5Ae8jafOHbBbV1bRxBz8 - mGXRzUYvkZS/RYVJ+G1ShxwCVgEnFqtyLvRx5GG1IKD6JmlqCvGrn223zyUCAwEA - AaOCAaYwggGiMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMHkG - CCsGAQUFBwEBBG0wazAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu - Y29tMEMGCCsGAQUFBzAChjdodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln - aUNlcnRBc3N1cmVkSURSb290Q0EuY3J0MIGBBgNVHR8EejB4MDqgOKA2hjRodHRw - Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3Js - MDqgOKA2hjRodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVk - SURSb290Q0EuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxo - dHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBRn/YggFCeYxwnS - JRm76VERY3VQYjAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq - hkiG9w0BAQsFAAOCAQEAqSg1esR71tonHqyYzyc2TxEydHTmQN0dzfJodzWvs4xd - xgS/FfQjZ4u5b5cE60adws3J0aSugS7JurHogNAcyTnBVnZZbJx946nw09E02DxJ - WYsamM6/xvLYMDX/6W9doK867mZTrqqMaci+mqege9iCSzMTyAfzd9fzZM2eY/lC - J1OuEDOJcjcV8b73HjWizsMt8tey5gvHacDlH198aZt+ziYaM0TDuncFO7pdP0GJ - +hY77gRuW6xWS++McPJKe1e9GW6LNgdUJi2GCZQfXzer8CM/jyxflp5HcahE3qm5 - hS+1NGClXwmgmkMd1L8tRNaN2v11y18WoA5hwnA9Ng== - -----END CERTIFICATE----- - stats_export_softwareheritage_org: - certificate: | - -----BEGIN CERTIFICATE----- - MIIGtDCCBZygAwIBAgIQBdh1rlqVNqqDtC3ToOG6zjANBgkqhkiG9w0BAQ0FADBk - MQswCQYDVQQGEwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJ - QW1zdGVyZGFtMQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wg - Q0EgMzAeFw0xNjA2MjAwMDAwMDBaFw0xOTA2MjUxMjAwMDBaMIHKMQswCQYDVQQG - EwJGUjERMA8GA1UECBMIWXZlbGluZXMxFTATBgNVBAcTDFJvY3F1ZW5jb3VydDFJ - MEcGA1UEChNASW5zdGl0dXQgTmF0aW9uYWwgZGUgUmVjaGVyY2hlIGVuIEluZm9y - bWF0aXF1ZSBldCBlbiBBdXRvbWF0aXF1ZTEaMBgGA1UECxMRU29mdHdhcmUgSGVy - aXRhZ2UxKjAoBgNVBAMTIXN0YXRzLmV4cG9ydC5zb2Z0d2FyZWhlcml0YWdlLm9y - ZzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOzz90M9R26tzSSGdU6w - Hy4xXP9mInZvR4JWa4TYgSIosvC+N6pZpV1PM3ZWR3RlJ6Mn5yyQ3uNJQnuHeqEX - X3Yq6xrloylgoE/bW+6rStv+MhCka7zkcbpZ900RMK6Uun7dlBxqW6+Y0e9Z0NZY - 7RW+R4w1MqcUR0kJ6pwM+bYIXtUkgpUi6aXLg7LhNoufkxcYnBDZe3GTYNeFdEUN - /NjbqxVcLRO7DNTszwpLoT+6Dg0lmbu2/ZROJg0c+YYzIvWGiSCxkP0jlra6W8EO - j9aXWp/7+lvX0qWsLNYxfnOb5QDQVDqbuIZuqjsFFgXgueS6cEJRWhgZaAeVeZif - HidHLWdpGHlQgqG+EwE3iaLJPOrGQqtNUwk6DEJpyhQCu9Fc78irolk9eyccgfQr - 7YEuxN2yukFwutQy7QP1/6CWtCovwwNw0/l1vmbFd2hcOmyq/DzOXeBkAHo8e5+t - EG+nN5Mk5vUI/OkQLO24/6IHge3uN2zXcDaqwAgQ/06TVCbdCrwzDFuhpt27EHDp - PvpD/751z7axqrIemXM5lDXgnG7QbQHN74P1n+g3fkVljBuWF2L1mfXBDRogslPI - VvYHql3QLIkHEvx2gTq/O2piVt3awlQqqXJmnURUMgf8acYfMc2QhFNmRIeorS1W - z6LXHohQRiVvVhmjlVpPa7mXAgMBAAGjggH5MIIB9TAfBgNVHSMEGDAWgBRn/Ygg - FCeYxwnSJRm76VERY3VQYjAdBgNVHQ4EFgQUb7Uzk+Q4+Zg/alPyn1AodsDoSbMw - SwYDVR0RBEQwQoIhc3RhdHMuZXhwb3J0LnNvZnR3YXJlaGVyaXRhZ2Uub3Jngh1w - ZXJnYW1vbi5zb2Z0d2FyZWhlcml0YWdlLm9yZzAOBgNVHQ8BAf8EBAMCBaAwHQYD - VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMGsGA1UdHwRkMGIwL6AtoCuGKWh0 - dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9URVJFTkFTU0xDQTMuY3JsMC+gLaArhilo - dHRwOi8vY3JsNC5kaWdpY2VydC5jb20vVEVSRU5BU1NMQ0EzLmNybDBMBgNVHSAE - RTBDMDcGCWCGSAGG/WwBATAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdp - Y2VydC5jb20vQ1BTMAgGBmeBDAECAjBuBggrBgEFBQcBAQRiMGAwJAYIKwYBBQUH - MAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTA4BggrBgEFBQcwAoYsaHR0cDov - L2NhY2VydHMuZGlnaWNlcnQuY29tL1RFUkVOQVNTTENBMy5jcnQwDAYDVR0TAQH/ - BAIwADANBgkqhkiG9w0BAQ0FAAOCAQEAUjG+ZmlzgZwiXp5xJqdbG8EZ9Dh7+utY - ZXkWlr2VsDzGNt9dLcht0FxfjozOoizpIdldLFkjW2OkNIQhChBpiLQ6gFn79B9/ - iNSZONQUpI1sqCLaOnOLbTHza0zi3Is+MBKjwxBVAcUERVjJbu5YSEm/Dle5IpUw - S8K5A7iwFIBQywLOySvz9P1c+MFqMLackEmlVM+vnF6axqQtMgOhscM06GW5bZdy - UnamhLXOIr+/Z9B+voB080qZZUn8DqFxP7og8au9IYKP5zLqTxTKayYX3qSXl+QT - NpzEGsZavMjzG7fNmQ8bTnEimThnqLbLAcgqWMmy/LBFRvabwHan4w== - -----END CERTIFICATE----- - ca_bundle: | - -----BEGIN CERTIFICATE----- - MIIE+zCCA+OgAwIBAgIQCHC8xa8/25Wakctq7u/kZTANBgkqhkiG9w0BAQsFADBl - MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 - d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv - b3QgQ0EwHhcNMTQxMTE4MTIwMDAwWhcNMjQxMTE4MTIwMDAwWjBkMQswCQYDVQQG - EwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJQW1zdGVyZGFt - MQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wgQ0EgMzCCASIw - DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMV2Dw/ZQyk7bG3RR63eEL8jwnio - Snc18SNb4EweQefCMQC9iDdFdd25AhCAHo/tZCMERaegOTuBTc9jP8JJ/yKeiLDS - lrlcinQfkioq8hLIt2hUtVhBgUBoBhpPhSn7tU08D08/QJYbzqjMXjX/ZJj1dd10 - VAWgNhEEEiRVY++Udy538RV27tOkWUUhn6i+0SftCuirOMo/h9Ha8Y+5Cx9E5+Ct - 85XCFk3shKM6ktTPxn3mvcsaQE+zVLHzj28NHuO+SaNW5Ae8jafOHbBbV1bRxBz8 - mGXRzUYvkZS/RYVJ+G1ShxwCVgEnFqtyLvRx5GG1IKD6JmlqCvGrn223zyUCAwEA - AaOCAaYwggGiMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMHkG - CCsGAQUFBwEBBG0wazAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu - Y29tMEMGCCsGAQUFBzAChjdodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln - aUNlcnRBc3N1cmVkSURSb290Q0EuY3J0MIGBBgNVHR8EejB4MDqgOKA2hjRodHRw - Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3Js - MDqgOKA2hjRodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVk - SURSb290Q0EuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxo - dHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBRn/YggFCeYxwnS - JRm76VERY3VQYjAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq - hkiG9w0BAQsFAAOCAQEAqSg1esR71tonHqyYzyc2TxEydHTmQN0dzfJodzWvs4xd - xgS/FfQjZ4u5b5cE60adws3J0aSugS7JurHogNAcyTnBVnZZbJx946nw09E02DxJ - WYsamM6/xvLYMDX/6W9doK867mZTrqqMaci+mqege9iCSzMTyAfzd9fzZM2eY/lC - J1OuEDOJcjcV8b73HjWizsMt8tey5gvHacDlH198aZt+ziYaM0TDuncFO7pdP0GJ - +hY77gRuW6xWS++McPJKe1e9GW6LNgdUJi2GCZQfXzer8CM/jyxflp5HcahE3qm5 - hS+1NGClXwmgmkMd1L8tRNaN2v11y18WoA5hwnA9Ng== - -----END CERTIFICATE----- +letsencrypt::account_email: sysop+letsencrypt@softwareheritage.org +letsencrypt::server: https://acme-v02.api.letsencrypt.org/directory + +letsencrypt::gandi_livedns_hook::config: + gandi_api: https://dns.api.gandi.net/api/v5/ + zones: + softwareheritage.org: + api_key: "%{alias('gandi::softwareheritage_org::api_key')}" + sharing_id: "%{alias('gandi::softwareheritage_org::sharing_id')}" + +letsencrypt::gandi_paas_hook::config: + gandi_xmlrpc: https://rpc.gandi.net/xmlrpc/ + zone_keys: + softwareheritage.org: "%{alias('gandi::softwareheritage_org::xmlrpc_key')}" + +letsencrypt::certificates::exported_directory: "%{::puppet_vardir}/letsencrypt_exports" +letsencrypt::certificates::directory: /etc/ssl/certs/letsencrypt +letsencrypt::certificates: + stats_export: + domains: + - stats.export.softwareheritage.org + - pergamon.softwareheritage.org + www-dev: + domains: + - www-dev.softwareheritage.org + deploy_hook: gandi_paas + www: + domains: + - softwareheritage.org + - www.softwareheritage.org + deploy_hook: gandi_paas + gandi-redirects: + domains: + - softwareheritage.org + - sponsors.softwareheritage.org + - sponsorship.softwareheritage.org + - testimonials.softwareheritage.org + deploy_hook: gandi_paas + bind::update_key: local-update bind::zones: internal.softwareheritage.org: domain: internal.softwareheritage.org 100.168.192.in-addr.arpa: domain: 100.168.192.in-addr.arpa 101.168.192.in-addr.arpa: domain: 101.168.192.in-addr.arpa + internal.staging.swh.network: + domain: internal.staging.swh.network + 128.168.192.in-addr.arpa: + domain: 128.168.192.in-addr.arpa 200.168.192.in-addr.arpa: domain: 200.168.192.in-addr.arpa 201.168.192.in-addr.arpa: domain: 201.168.192.in-addr.arpa 202.168.192.in-addr.arpa: domain: 202.168.192.in-addr.arpa 203.168.192.in-addr.arpa: domain: 203.168.192.in-addr.arpa 204.168.192.in-addr.arpa: domain: 204.168.192.in-addr.arpa 205.168.192.in-addr.arpa: domain: 205.168.192.in-addr.arpa 206.168.192.in-addr.arpa: domain: 206.168.192.in-addr.arpa 207.168.192.in-addr.arpa: domain: 207.168.192.in-addr.arpa # Defaults for secondary bind server bind::zones::type: slave bind::zones::masters: - 192.168.100.29 bind::zones::allow_transfers: - 192.168.100.0/24 - 192.168.101.0/24 - 192.168.200.22 bind::zones::default_data: zone_type: "%{alias('bind::zones::type')}" dynamic: true masters: "%{alias('bind::zones::masters')}" transfer_source: '' allow_updates: [] update_policies: '' allow_transfers: "%{alias('bind::zones::allow_transfers')}" dnssec: false key_directory: '' ns_notify: true also_notify: '' allow_notify: '' forwarders: '' forward: '' source: '' ns_records: - pergamon.internal.softwareheritage.org. - ns0.euwest.azure.internal.softwareheritage.org. bind::resource_records: archive/CNAME: type: CNAME record: archive.internal.softwareheritage.org data: moma.internal.softwareheritage.org. db/CNAME: type: CNAME record: db.internal.softwareheritage.org - data: prado.internal.softwareheritage.org. + data: belvedere.internal.softwareheritage.org. debian/CNAME: type: CNAME record: debian.internal.softwareheritage.org data: pergamon.internal.softwareheritage.org. backup/CNAME: type: CNAME record: backup.internal.softwareheritage.org data: banco.internal.softwareheritage.org. banco/A: record: banco.internal.softwareheritage.org data: 192.168.100.18 beaubourg/A: record: beaubourg.internal.softwareheritage.org data: 192.168.100.32 icinga/CNAME: type: CNAME record: icinga.internal.softwareheritage.org data: pergamon.internal.softwareheritage.org. faitout/CNAME: type: CNAME record: faitout.internal.softwareheritage.org data: prado.internal.softwareheritage.org. logstash/CNAME: type: CNAME record: logstash.internal.softwareheritage.org data: logstash0.internal.softwareheritage.org. logstash0/A: record: logstash0.internal.softwareheritage.org data: 192.168.100.19 munin/CNAME: type: CNAME record: munin.internal.softwareheritage.org data: munin0.internal.softwareheritage.org. munin0/A: record: munin0.internal.softwareheritage.org data: 192.168.100.20 kibana/CNAME: type: CNAME record: kibana.internal.softwareheritage.org data: banco.internal.softwareheritage.org. kibana0/A: record: kibana0.internal.softwareheritage.org data: 192.168.100.50 rabbitmq/CNAME: type: CNAME record: rabbitmq.internal.softwareheritage.org data: saatchi.internal.softwareheritage.org. esnode1/A: record: esnode1.internal.softwareheritage.org data: 192.168.100.61 esnode2/A: record: esnode2.internal.softwareheritage.org data: 192.168.100.62 esnode3/A: record: esnode3.internal.softwareheritage.org data: 192.168.100.63 # VPN hosts zack/A: record: zack.internal.softwareheritage.org data: 192.168.101.6 olasd/A: record: olasd.internal.softwareheritage.org data: 192.168.101.10 ardumont/A: record: ardumont.internal.softwareheritage.org data: 192.168.101.14 ardumont-desktop/A: record: ardumont-desktop.internal.softwareheritage.org data: 192.168.101.158 rdicosmo/A: record: rdicosmo.internal.softwareheritage.org data: 192.168.101.38 petitpalais/A: record: petitpalais.internal.softwareheritage.org data: 192.168.101.154 grand-palais/A: record: grand-palais.internal.softwareheritage.org data: 192.168.101.62 grandpalais/CNAME: type: CNAME record: grandpalais.internal.softwareheritage.org data: grand-palais.internal.softwareheritage.org. giverny/A: type: A record: giverny.internal.softwareheritage.org data: 192.168.101.118 orangeriedev/A: type: A record: orangeriedev.internal.softwareheritage.org data: 192.168.101.130 orangerie/A: type: A record: orangerie.internal.softwareheritage.org data: 192.168.101.142 ddouard-desktop/A: record: ddouard-desktop.internal.softwareheritage.org data: 192.168.101.162 + vlorentz-desktop/A: + record: vlorentz-desktop.internal.softwareheritage.org + data: 192.168.101.166 + gateway/A: + record: gateway.internal.staging.swh.network + data: 192.168.128.1 + storage0/A: + record: storage0.internal.staging.swh.network + data: 192.168.128.2 + db0/A: + record: db0.internal.staging.swh.network + data: 192.168.128.3 + scheduler0/A: + record: scheduler0.internal.staging.swh.network + data: 192.168.128.4 + worker0/A: + record: worker0.internal.staging.swh.network + data: 192.168.128.5 + worker1/A: + record: worker1.internal.staging.swh.network + data: 192.168.128.6 + deposit/A: + record: deposit.internal.staging.swh.network + data: 192.168.128.7 + webapp/A: + record: webapp.internal.staging.swh.network + data: 192.168.128.8 + vault/A: + record: vault.internal.staging.swh.network + data: 192.168.128.9 bind::resource_records::default_data: type: A bind::clients: - 192.168.100.0/24 - 192.168.101.0/24 - 192.168.200.0/21 - 127.0.0.0/8 - '::1/128' bind::autogenerate: - 192.168.100.0/24 - 192.168.200.0/21 dar::backup::storage: /srv/backups dar::backup::num_backups: 1 dar::backup::base: / dar::backup::select: [] # empty list = full backup dar::backup::exclude: - dev - proc - run - srv/backups + - srv/db-backups - srv/elasticsearch - srv/remote-backups - srv/softwareheritage/objects - srv/softwareheritage/postgres - srv/softwareheritage/scratch - srv/softwareheritage/scratch.2TB - srv/storage - sys - tmp - var/cache - var/lib/mysql - var/log/journal - var/run - var/tmp dar::backup::options: - -zbzip2 -dar::cron::hour: 0 -dar::cron::minute: fqdn_rand -dar::cron::month: '*' -dar::cron::monthday: '*' -dar::cron::weekday: '*' +dar::cron: + hour: 0 + minute: fqdn_rand dar_server::backup::storage: /srv/remote-backups -dar_server::cron::hour: '0-4' -dar_server::cron::minute: '*/10' -dar_server::cron::month: '*' -dar_server::cron::monthday: '*' -dar_server::cron::weekday: '*' +dar_server::central_host: uffizi.softwareheritage.org +dar_server::cron: + hour: '0-4' + minute: '*/10' phabricator::basepath: /srv/phabricator phabricator::user: phabricator phabricator::vcs_user: git phabricator::notification::client_host: 127.0.0.1 phabricator::notification::client_port: 22280 phabricator::notification::listen: "%{hiera('phabricator::notification::client_host')}:%{hiera('phabricator::notification::client_port')}" phabricator::mysql::database_prefix: phabricator phabricator::mysql::username: phabricator phabricator::mysql::conf::max_allowed_packet: 33554432 phabricator::mysql::conf::sql_mode: STRICT_ALL_TABLES phabricator::mysql::conf::ft_stopword_file: "%{hiera('phabricator::basepath')}/phabricator/resources/sql/stopwords.txt" phabricator::mysql::conf::ft_min_word_len: 3 phabricator::mysql::conf::ft_boolean_syntax: "' |-><()~*:\"\"&^'" phabricator::mysql::conf::innodb_buffer_pool_size: 4G phabricator::mysql::conf::innodb_file_per_table: TRUE phabricator::mysql::conf::innodb_flush_method: O_DIRECT phabricator::mysql::conf::innodb_log_file_size: 1G +phabricator::mysql::conf::max_connections: 16384 phabricator::php::fpm_listen: 127.0.0.1:9001 phabricator::php::max_file_size: 128M phabricator::php::opcache_validate_timestamps: 0 phabricator::vhost::name: forge.softwareheritage.org phabricator::vhost::docroot: "%{hiera('phabricator::basepath')}/phabricator/webroot" phabricator::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" phabricator::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" phabricator::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" phabricator::vhost::hsts_header: "%{hiera('apache::hsts_header')}" mediawiki::php::fpm_listen: 127.0.0.1:9002 mediawiki::vhosts: intranet.softwareheritage.org: swh_logo: /images/9/99/Swh-intranet-logo.png mysql: username: mw_intranet dbname: mediawiki_intranet aliases: [] site_name: Software Heritage Intranet wiki.softwareheritage.org: swh_logo: /images/b/b2/Swh-logo.png mysql: username: mw_public dbname: mediawiki_public aliases: [] site_name: Software Heritage Wiki mediawiki::vhost::docroot: /var/lib/mediawiki mediawiki::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" mediawiki::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" mediawiki::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" mediawiki::vhost::hsts_header: "%{hiera('apache::hsts_header')}" annex::basepath: /srv/softwareheritage/annex annex::vhost::name: annex.softwareheritage.org annex::vhost::docroot: "%{hiera('annex::basepath')}/webroot" annex::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" annex::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" annex::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" annex::vhost::hsts_header: "%{hiera('apache::hsts_header')}" docs::basepath: /srv/softwareheritage/docs docs::vhost::name: docs.softwareheritage.org docs::vhost::docroot: "%{hiera('docs::basepath')}/webroot" -docs::vhost::docroot_group: "swhdev" -docs::vhost::docroot_mode: "2775" +docs::vhost::docroot_owner: "jenkins-push-docs" +docs::vhost::docroot_group: "www-data" +docs::vhost::docroot_mode: "2755" docs::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" docs::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" docs::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" docs::vhost::hsts_header: "%{hiera('apache::hsts_header')}" ssh::port: 22 ssh::permitrootlogin: without-password swh::base_directory: /srv/softwareheritage swh::conf_directory: /etc/softwareheritage swh::log_directory: /var/log/softwareheritage swh::global_conf::file: "%{hiera('swh::conf_directory')}/global.ini" swh::global_conf::contents: | # Managed by puppet (class profile::swh) - modifications will be overwritten [main] log_db = swh::apt_config::swh_repository::hostname: debian.softwareheritage.org swh::apt_config::swh_repository: "https://%{hiera('swh::apt_config::swh_repository::hostname')}/" swh::apt_config::enable_non_free: false +swh::apt_config::backported_packages: + stretch: + # For swh.scheduler + - python3-msgpack + # T1609 + - python3-urllib3 + - python3-requests + - python3-chardet + - python3-idna + debian_repository::basepath: "%{hiera('swh::base_directory')}/repository" +debian_repository::owner: swhdebianrepo +debian_repository::owner::homedir: /home/swhdebianrepo +debian_repository::group: swhdev +debian_repository::mode: "02775" + +debian_repository::ssh_authorized_keys: + nicolasd@darboux: + type: ssh-rsa + key: AAAAB3NzaC1yc2EAAAADAQABAAABAQDZ1TCpfzrvxLhEMhxjbxqPDCwY0nazIr1cyIbhGD2bUdAbZqVMdNtr7MeDnlLIKrIPJWuvltauvLNkYU0iLc1jMntdBCBM3hgXjmTyDtc8XvXseeBp5tDqccYNR/cnDUuweNcL5tfeu5kzaAg3DFi5Dsncs5hQK5KQ8CPKWcacPjEk4ir9gdFrtKG1rZmg/wi7YbfxrJYWzb171hdV13gSgyXdsG5UAFsNyxsKSztulcLKxvbmDgYbzytr38FK2udRk7WuqPbtEAW1zV4yrBXBSB/uw8EAMi+wwvLTwyUcEl4u0CTlhREljUx8LhYrsQUCrBcmoPAmlnLCD5Q9XrGH + jenkins@thyssen: + type: ssh-rsa + key: AAAAB3NzaC1yc2EAAAADAQABAAABAQCrfYnl8v4QK1ClkPMHO4WiPqgLVoOGpOPFUvg3WehMo8xMQ9e/EeZddQn96mhHkbbC5HCWEVK1VwafpIeadaMHnypdGhpapncYPpoKItxmf1IwVtlt/h8OYai5pTMCgkuOHjhnQdO20Amr9WMkoRZ/K7v/GijIZ6svvgWiYKfDnu0s1ziFYIT5rEA5hL9SqNJTlKdy2H68/7mmTii9NpBsGWQYDOjcrwELNOI5EUgQSOzmeKxecPkABfh/dezp6jmrv/2x7bm7LT46d+rnVDqVRiUrLVnLhrZCmZDxXfbEmftTdAoK8U/wjLreanRxKOc7arYRyKu0RbAaejPejzgR + +debian_repository::gpg_keys: + # olasd + - 791F12396630DD71FD364375B8E5087766475AAF + # zack + - 4900707DDC5C07F2DECB02839C31503C6D866396 + # ardumont + - BF00203D741AC9D546A8BE0752E2E9840D10C3B8 + # anlambert + - 91FAF3F5CDE011E4FDF4CBF2D026E5C2F802586D + # seirl + - 225CD9E3FA9374BDF6E057042F8984858B1A9945 + # vlorentz + - 379043E3DF96D3237E6782AC0E082B40E4376B1E + # ddouard + - 7DC7325EF1A6226AB6C3D7E32388A3BF6F0A6938 + # jenkins-debian1 + - 1F4BDC445E30C7066324D7B3D7D3329147AE3148 + debian_repository::vhost::name: "%{hiera('swh::apt_config::swh_repository::hostname')}" debian_repository::vhost::aliases: - debian.internal.softwareheritage.org debian_repository::vhost::docroot: "%{hiera('debian_repository::basepath')}" -debian_repository::vhost::docroot_owner: root -debian_repository::vhost::docroot_group: swhdev -debian_repository::vhost::docroot_mode: "02775" debian_repository::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" debian_repository::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" debian_repository::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" debian_repository::vhost::hsts_header: "%{hiera('apache::hsts_header')}" swh::apt_config::debian_mirror::hostname: deb.debian.org swh::apt_config::debian_mirror: "http://%{hiera('swh::apt_config::debian_mirror::hostname')}/debian/" swh::apt_config::debian_security_mirror::hostname: "%{hiera('swh::apt_config::debian_mirror::hostname')}" swh::apt_config::debian_security_mirror: "http://%{hiera('swh::apt_config::debian_mirror::hostname')}/debian-security/" swh::apt_config::azure_repository::hostname: debian-archive.trafficmanager.net swh::apt_config::azure_repository: "http://%{hiera('swh::apt_config::azure_repository::hostname')}/debian-azure/" swh::apt_config::unattended_upgrades: true swh::apt_config::unattended_upgraes::origins: - "o=Debian,n=%{::lsbdistcodename}" # main Debian archive - "o=Debian,n=%{::lsbdistcodename}-updates" # stable-updates (ex-volatile) - "o=Debian,n=%{::lsbdistcodename},l=Debian-Security" # security updates - "o=debian icinga-%{::lsbdistcodename},n=icinga-%{::lsbdistcodename}" # Icinga2 repository - "o=Debian Azure,n=%{::lsbdistcodename}" # Debian Azure - "o=Proxmox,n=%{::lsbdistcodename}" # Proxmox repository - "o=packages.sury.org" # PHP backports (tate) + + + +##################################################################################################### +# Remote service configurations + +# Default ports swh::remote_service::storage::port: 5002 swh::remote_service::objstorage::port: 5003 swh::remote_service::webapp::port: 5004 swh::remote_service::vault::port: 5005 swh::remote_service::deposit::port: 5006 swh::remote_service::indexer::port: 5007 swh::remote_service::scheduler::port: 5008 + +# Default backend services. Override in specific sites if needed. Configurations +# are split between read-only (the default) and writable storages. In most cases +# overrides should only happen for read-only services. + +swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" +swh::remote_service::objstorage::config::writable: "%{alias('swh::remote_service::objstorage::config::uffizi')}" +swh::remote_service::objstorage::config_as_dict: + banco: "%{alias('swh::remote_service::objstorage::config::banco')}" + uffizi: "%{alias('swh::remote_service::objstorage::config::uffizi')}" + azure: "%{alias('swh::remote_service::objstorage::config::azure')}" + +swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::uffizi')}" +swh::remote_service::storage::config::writable: "%{alias('swh::remote_service::storage::config::uffizi')}" + +swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::uffizi')}" +swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::uffizi')}" + +swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::saatchi')}" +swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::saatchi')}" + +swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::azure')}" +swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::azure')}" + +# Objstorage backend configurations swh::remote_service::objstorage::config::azure: &swh_objstorage_config_azure cls: azure-prefixed args: accounts: "0": account_name: 0euwestswh api_secret_key: "%{hiera('swh::azure::credentials::0euwestswh')}" container_name: contents "1": account_name: 1euwestswh api_secret_key: "%{hiera('swh::azure::credentials::1euwestswh')}" container_name: contents "2": account_name: 2euwestswh api_secret_key: "%{hiera('swh::azure::credentials::2euwestswh')}" container_name: contents "3": account_name: 3euwestswh api_secret_key: "%{hiera('swh::azure::credentials::3euwestswh')}" container_name: contents "4": account_name: 4euwestswh api_secret_key: "%{hiera('swh::azure::credentials::4euwestswh')}" container_name: contents "5": account_name: 5euwestswh api_secret_key: "%{hiera('swh::azure::credentials::5euwestswh')}" container_name: contents "6": account_name: 6euwestswh api_secret_key: "%{hiera('swh::azure::credentials::6euwestswh')}" container_name: contents "7": account_name: 7euwestswh api_secret_key: "%{hiera('swh::azure::credentials::7euwestswh')}" container_name: contents "8": account_name: 8euwestswh api_secret_key: "%{hiera('swh::azure::credentials::8euwestswh')}" container_name: contents "9": account_name: 9euwestswh api_secret_key: "%{hiera('swh::azure::credentials::9euwestswh')}" container_name: contents "a": account_name: aeuwestswh api_secret_key: "%{hiera('swh::azure::credentials::aeuwestswh')}" container_name: contents "b": account_name: beuwestswh api_secret_key: "%{hiera('swh::azure::credentials::beuwestswh')}" container_name: contents "c": account_name: ceuwestswh api_secret_key: "%{hiera('swh::azure::credentials::ceuwestswh')}" container_name: contents "d": account_name: deuwestswh api_secret_key: "%{hiera('swh::azure::credentials::deuwestswh')}" container_name: contents "e": account_name: eeuwestswh api_secret_key: "%{hiera('swh::azure::credentials::eeuwestswh')}" container_name: contents "f": account_name: feuwestswh api_secret_key: "%{hiera('swh::azure::credentials::feuwestswh')}" container_name: contents swh::remote_service::objstorage::config::azure::readonly: cls: filtered args: storage_conf: "%{alias('swh::remote_service::objstorage::config::azure')}" filters_conf: - type: readonly swh::remote_service::objstorage::config::uffizi: &swh_objstorage_config_uffizi cls: remote args: url: "http://uffizi.internal.softwareheritage.org:%{hiera('swh::remote_service::objstorage::port')}/" swh::remote_service::objstorage::config::uffizi::readonly: cls: filtered args: storage_conf: "%{alias('swh::remote_service::objstorage::config::uffizi')}" filters_conf: - type: readonly swh::remote_service::objstorage::config::banco: &swh_objstorage_config_banco cls: remote args: url: "http://banco.internal.softwareheritage.org:%{hiera('swh::remote_service::objstorage::port')}/" swh::remote_service::objstorage::config::banco::readonly: cls: filtered args: storage_conf: "%{alias('swh::remote_service::objstorage::config::banco')}" filters_conf: - type: readonly -swh::remote_service::objstorage::config::azure_readonly_with_fallback: +swh::remote_service::objstorage::config::azure_readonly_with_fallback: &swh_azure_readonly_with_fallback cls: multiplexer args: objstorages: - "%{alias('swh::remote_service::objstorage::config::azure::readonly')}" - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" - "%{alias('swh::remote_service::objstorage::config::uffizi::readonly')}" swh::remote_service::objstorage::config::localhost: cls: remote args: url: "http://127.0.0.1:%{hiera('swh::remote_service::objstorage::port')}/" -swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::uffizi')}" -swh::remote_service::objstorage::config_as_dict: - banco: "%{alias('swh::remote_service::objstorage::config::banco')}" - uffizi: "%{alias('swh::remote_service::objstorage::config::uffizi')}" - azure: "%{alias('swh::remote_service::objstorage::config::azure')}" +# Storage backend configurations swh::remote_service::storage::config::uffizi: cls: remote args: url: "http://uffizi.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config::azure: cls: remote args: url: "http://storage0.euwest.azure.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config::localhost: cls: remote args: url: "http://localhost:%{hiera('swh::remote_service::storage::port')}/" -swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::uffizi')}" -swh::indexer::storage::config::uffizi: &swh_indexer_storage_config_uffizi +# Indexer backend configurations +swh::remote_service::indexer::config::uffizi: cls: remote args: url: "http://uffizi.internal.softwareheritage.org:%{hiera('swh::remote_service::indexer::port')}/" -swh::indexer::storage::config::azure: +swh::remote_service::indexer::config::azure: cls: remote args: url: "http://storage0.euwest.azure.internal.softwareheritage.org:%{hiera('swh::remote_service::indexer::port')}/" -swh::remote_service::scheduler::config::saatchi: &swh_scheduler_remote_config_saatchi + +# Scheduler backend configurations +swh::remote_service::scheduler::config::saatchi: cls: remote args: url: "http://saatchi.internal.softwareheritage.org:%{hiera('swh::remote_service::scheduler::port')}/" + +# Vault backend configurations +swh::remote_service::vault::config::azure: + cls: remote + args: + url: "http://vangogh.euwest.azure.internal.softwareheritage.org:%{hiera('swh::remote_service::vault::port')}/" + +# End remote service configurations +##################################################################################################### + +swh::deploy::db::pgbouncer::port: 5432 +swh::deploy::db::main::port: 5433 +swh::deploy::db::secondary::port: 5434 +swh::deploy::db::hdd::port: 5435 + +swh::deploy::db::pgbouncer::user::login: postgres + +pgbouncer::config_params: + logfile: /var/log/postgresql/pgbouncer.log + pidfile: /var/run/postgresql/pgbouncer.pid + unix_socket_dir: /var/run/postgresql + client_tls_sslmode: allow + client_tls_ca_file: /etc/ssl/certs/ssl-cert-snakeoil.pem + client_tls_key_file: /etc/ssl/private/ssl-cert-snakeoil.key + client_tls_cert_file: /etc/ssl/certs/ssl-cert-snakeoil.pem + server_tls_sslmode: allow + listen_port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + listen_addr: + - 127.0.0.1 + - "%{hiera('pgbouncer::listen_addr')}" + auth_type: "hba" + auth_file: /etc/pgbouncer/userlist.txt + auth_hba_file: "%{hiera('pgbouncer::auth_hba_file')}" + admin_users: + - "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + - olasd + pool_mode: session + server_reset_query: DISCARD ALL + max_client_conn: 2000 + default_pool_size: 2000 + max_db_connections: 2000 + max_user_connections: 2000 + log_connections: 0 + log_disconnections: 0 + +pgbouncer::user: postgres +pgbouncer::group: postgres + +# swh::deploy::db::pgbouncer::user::password in private data +pgbouncer::userlist: + - user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + password: "%{hiera('swh::deploy::db::pgbouncer::user::password')}" + +pgbouncer::databases: [] + swh::deploy::directory: "%{hiera('swh::conf_directory')}/deploy" swh::deploy::group: swhdeploy swh::deploy::public_key: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWrJX/uUss/EYZaTp2EIsZgg3ZSH8JcNZV5gBdNZ7EHcQcqxYUCqmwv9Ss3xT8n9kIrH6iz/vquqf84XR+keoZK3bsp50tMOY8LJWpcl/JK2XD6ovoJrHPu+iAroLkE59RdTa1Vz+jF67Q2UuG9f0nKwL4rnkeWTyuK/zAbyHyYKFQntkkwMr5/YTU8sjl/4aNF/2Ww8hitdi2GORlCjav2bB0wyPBA2e8sMt8Hp9O4TIWg/RD6vPX+ZvuFaB/Lw/Hv21622QGTHoZiO92/8/W9/t24il6SU4z96ZGfXqdUZkpPYKBGwyIkZkS4dN6jb4CcRlyXTObphyu3dAlABRt swhworker@worker01' swh::deploy::storage::conf_directory: "%{hiera('swh::conf_directory')}/storage" swh::deploy::storage::conf_file: "%{hiera('swh::deploy::storage::conf_directory')}/storage.yml" swh::deploy::storage::user: swhstorage swh::deploy::storage::group: swhstorage -swh::deploy::storage::db::host: db -swh::deploy::storage::db::port: 5432 +swh::deploy::storage::db::host: db.internal.softwareheritage.org +swh::deploy::storage::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" swh::deploy::storage::db::user: swhstorage swh::deploy::storage::db::dbname: softwareheritage swh::deploy::storage::directory: "%{hiera('swh::base_directory')}/objects" swh::deploy::storage::backend::listen::host: 127.0.0.1 swh::deploy::storage::backend::listen::port: "%{alias('swh::remote_service::storage::port')}" swh::deploy::storage::backend::workers: 4 swh::deploy::storage::backend::reload_mercy: 3600 swh::deploy::storage::backend::http_keepalive: 5 swh::deploy::storage::backend::http_timeout: 3600 swh::deploy::storage::backend::max_requests: 10000 swh::deploy::storage::backend::max_requests_jitter: 1000 swh::deploy::storage::backend::server_names: - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" swh::deploy::storage::config: storage: cls: local args: db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" swh::deploy::indexer::storage::conf_file: "%{hiera('swh::deploy::storage::conf_directory')}/indexer.yml" swh::deploy::indexer::storage::user: swhstorage swh::deploy::indexer::storage::group: swhstorage -swh::deploy::indexer::storage::db::host: somerset -swh::deploy::indexer::storage::db::port: 5434 +swh::deploy::indexer::storage::db::host: somerset.internal.softwareheritage.org +swh::deploy::indexer::storage::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" swh::deploy::indexer::storage::db::user: swhstorage swh::deploy::indexer::storage::db::dbname: softwareheritage-indexer swh::deploy::indexer::storage::backend::listen::host: 127.0.0.1 swh::deploy::indexer::storage::backend::listen::port: "%{alias('swh::remote_service::indexer::port')}" swh::deploy::indexer::storage::backend::workers: 4 swh::deploy::indexer::storage::backend::reload_mercy: 3600 swh::deploy::indexer::storage::backend::http_keepalive: 5 swh::deploy::indexer::storage::backend::http_timeout: 3600 swh::deploy::indexer::storage::backend::max_requests: 10000 swh::deploy::indexer::storage::backend::max_requests_jitter: 1000 swh::deploy::indexer::storage::backend::server_names: - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" swh::deploy::indexer::storage::config: - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" - -swh::deploy::storage_listener::conf_directory: "%{hiera('swh::conf_directory')}/storage" -swh::deploy::storage_listener::conf_file: "%{hiera('swh::deploy::storage_listener::conf_directory')}/listener.ini" -swh::deploy::storage_listener::user: swhstorage -swh::deploy::storage_listener::group: swhstorage -swh::deploy::storage_listener::database: host=db user=guest dbname=softwareheritage -swh::deploy::storage_listener::topic_prefix: swh.tmp_journal.new -swh::deploy::storage_listener::kafka_brokers: - - getty.internal.softwareheritage.org -swh::deploy::storage_listener::poll_timeout: 10 + indexer_storage: + cls: local + args: + db: "host=%{hiera('swh::deploy::indexer::storage::db::host')} port=%{hiera('swh::deploy::indexer::storage::db::port')} user=%{hiera('swh::deploy::indexer::storage::db::user')} dbname=%{hiera('swh::deploy::indexer::storage::db::dbname')} password=%{hiera('swh::deploy::indexer::storage::db::password')}" + +swh::deploy::vault::cache: "%{hiera('swh::base_directory')}/vault_cache" +# Default cache (orangerie/orangeriedev) is a pathslicing objstorage +swh::deploy::vault::config::cache: + cls: pathslicing + args: + root: "%{hiera('swh::deploy::vault::cache')}" + slicing: "0:1/1:5" swh::deploy::vault::conf_directory: "%{hiera('swh::conf_directory')}/vault" swh::deploy::vault::conf_file: "%{hiera('swh::deploy::vault::conf_directory')}/server.yml" swh::deploy::vault::user: swhvault swh::deploy::vault::group: swhvault -swh::deploy::vault::db::host: orangerie -swh::deploy::vault::db::port: 5432 -swh::deploy::vault::db::user: swhvault -swh::deploy::vault::db::dbname: swhvault -swh::deploy::vault::cache: "%{hiera('swh::base_directory')}/vault_cache" +swh::deploy::vault::db::host: db.internal.softwareheritage.org +swh::deploy::vault::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::vault::db::user: swh-vault +swh::deploy::vault::db::dbname: swh-vault swh::deploy::vault::backend::listen::host: 127.0.0.1 swh::deploy::vault::backend::listen::port: "%{alias('swh::remote_service::vault::port')}" swh::deploy::vault::backend::workers: 4 swh::deploy::vault::backend::reload_mercy: 3600 swh::deploy::vault::backend::http_keepalive: 5 swh::deploy::vault::backend::http_timeout: 3600 swh::deploy::vault::backend::max_requests: 10000 swh::deploy::vault::backend::max_requests_jitter: 1000 swh::deploy::vault::backend::server_names: - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" swh::deploy::vault::config: storage: "%{alias('swh::remote_service::storage::config')}" - cache: - cls: pathslicing + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + cache: "%{alias('swh::deploy::vault::config::cache')}" + vault: + cls: local args: - root: "%{hiera('swh::deploy::vault::cache')}" - slicing: "0:1/1:5" - db: "host=%{hiera('swh::deploy::vault::db::host')} port=%{hiera('swh::deploy::vault::db::port')} user=%{hiera('swh::deploy::vault::db::user')} dbname=%{hiera('swh::deploy::vault::db::dbname')} password=%{hiera('swh::deploy::vault::db::password')}" - scheduling_db: "%{hiera('swh::deploy::scheduler::database')}" - + db: "host=%{hiera('swh::deploy::vault::db::host')} port=%{hiera('swh::deploy::vault::db::port')} user=%{hiera('swh::deploy::vault::db::user')} dbname=%{hiera('swh::deploy::vault::db::dbname')} password=%{hiera('swh::deploy::vault::db::password')}" swh::deploy::journal::conf_directory: "%{hiera('swh::conf_directory')}/journal" -swh::deploy::journal_publisher::conf_file: "%{hiera('swh::deploy::journal::conf_directory')}/publisher.yml" -swh::deploy::journal_publisher::user: swhstorage -swh::deploy::journal_publisher::group: swhstorage -swh::deploy::journal_publisher::config: - brokers: - - getty.internal.softwareheritage.org - temporary_prefix: swh.tmp_journal.new - final_prefix: swh.journal.objects - consumer_id: swh.journal.publisher - publisher_id: swh.journal.publisher - object_types: - - content - - revision - - release - max_messages: 1000 - storage: "%{alias('swh::remote_service::storage::config')}" +swh::deploy::journal::brokers: + - esnode1.internal.softwareheritage.org + - esnode2.internal.softwareheritage.org + - esnode3.internal.softwareheritage.org + +swh::deploy::journal::prefix: swh.journal.objects swh::deploy::journal_simple_checker_producer::conf_file: "%{hiera('swh::deploy::journal::conf_directory')}/checker.yml" swh::deploy::journal_simple_checker_producer::user: swhstorage swh::deploy::journal_simple_checker_producer::group: swhstorage swh::deploy::journal_simple_checker_producer::config: - brokers: - - getty.internal.softwareheritage.org + brokers: "%{alias('swh::deploy::journal::brokers')}" temporary_prefix: swh.tmp_journal.new storage_dbconn: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" object_types: - content - directory - revision - release - origin - origin_visit -swh::deploy::archiver_content_updater::conf_file: "%{hiera('swh::deploy::worker::swh_storage_archiver::conf_directory')}/content_updater.yml" -swh::deploy::archiver_content_updater::user: swhstorage -swh::deploy::archiver_content_updater::group: swhstorage -swh::deploy::archiver_content_updater::config: - brokers: - - getty.internal.softwareheritage.org - auto_offset_reset: earliest - topic_prefix: swh.journal.objects - consumer_id: swh.journal.archiver.content.updater - object_types: - - content - archiver_storage: - cls: db - args: - dbconn: "host=%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" - sources_present: - - uffizi - swh::deploy::objstorage::conf_directory: "%{hiera('swh::conf_directory')}/objstorage" swh::deploy::objstorage::conf_file: "%{hiera('swh::deploy::objstorage::conf_directory')}/server.yml" swh::deploy::objstorage::user: "%{hiera('swh::deploy::storage::user')}" swh::deploy::objstorage::group: "%{hiera('swh::deploy::storage::group')}" swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" swh::deploy::objstorage::slicing: 0:2/2:4/4:6 swh::deploy::objstorage::config: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::objstorage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" - max_client_size: 1073741824 # 1 GiB + objstorage: + cls: pathslicing + args: + root: "%{hiera('swh::deploy::objstorage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + client_max_size: 1073741824 # 1 GiB swh::deploy::objstorage::backend::listen::host: 127.0.0.1 swh::deploy::objstorage::backend::listen::port: "%{alias('swh::remote_service::objstorage::port')}" swh::deploy::objstorage::backend::workers: 4 swh::deploy::objstorage::backend::reload_mercy: 3600 swh::deploy::objstorage::backend::http_workers: 1 swh::deploy::objstorage::backend::http_keepalive: 5 swh::deploy::objstorage::backend::http_timeout: 3600 swh::deploy::objstorage::backend::max_requests: 0 swh::deploy::objstorage::backend::max_requests_jitter: 0 swh::deploy::objstorage::backend::server_names: - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" swh::deploy::deposit::vhost::name: deposit.softwareheritage.org swh::deploy::deposit::vhost::aliases: [] swh::deploy::deposit::vhost::docroot: "/var/www/%{hiera('swh::deploy::deposit::vhost::name')}" swh::deploy::deposit::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" swh::deploy::deposit::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" swh::deploy::deposit::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" swh::deploy::deposit::locked_endpoints: - /1/private/[^/]+/[^/]+/[^/]+ - /1/private/deposits/ -swh::deploy::deposit::conf_directory: "%{hiera('swh::conf_directory')}/deposit" -swh::deploy::deposit::swh_conf_file: "%{hiera('swh::deploy::deposit::conf_directory')}/server.yml" -swh::deploy::deposit::settings_private_data_file: "%{hiera('swh::deploy::deposit::conf_directory')}/private.yml" +swh::deploy::deposit::config_directory: "%{hiera('swh::conf_directory')}/deposit" +swh::deploy::deposit::config_file: "%{hiera('swh::deploy::deposit::config_directory')}/server.yml" swh::deploy::deposit::user: swhdeposit swh::deploy::deposit::group: swhdeposit swh::deploy::deposit::media_root_directory: /srv/storage/space/swh-deposit/uploads/ -# test configuration +swh::deploy::deposit::db::host: db.internal.softwareheritage.org +swh::deploy::deposit::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::deposit::db::dbname: softwareheritage-deposit +swh::deploy::deposit::db::dbuser: swhstorage +swh::deploy::deposit::config::allowed_hosts: + - deposit.internal.softwareheritage.org + +# swh::deploy::deposit::db::password: in private data +# swh::deploy::deposit::runtime_secret_key in private data swh::deploy::deposit::config: max_upload_size: 209715200 tool: name: 'swh-deposit' version: '0.0.1' configuration: sword_version: 2 - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - -# swh::deploy::deposit::runtime_secret_key in private data -swh::deploy::deposit::settings_private_data: - secret_key: "%{hiera('swh::deploy::deposit::runtime_secret_key')}" - db: - host: db - port: 5432 - name: softwareheritage-deposit - user: swhstorage - password: "%{hiera('swh::deploy::storage::db::password')}" - media_root: "%{hiera('swh::deploy::deposit::media_root_directory')}" - -swh::deploy::worker::swh_loader_deposit::swh_conf_dir: "%{hiera('swh::conf_directory')}/loader" -swh::deploy::worker::swh_loader_deposit::swh_conf_file: "%{hiera('swh::deploy::worker::swh_loader_deposit::swh_conf_dir')}/deposit.yml" -swh::deploy::worker::swh_loader_deposit::concurrency: 2 -swh::deploy::worker::swh_loader_deposit::private_tmp: true -swh::deploy::worker::swh_loader_deposit::loglevel: info -swh::deploy::worker::swh_loader_deposit::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_deposit::config: - storage: "%{alias('swh::remote_service::storage::config')}" + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + allowed_hosts: "%{alias('swh::deploy::deposit::config::allowed_hosts')}" + private: + secret_key: "%{hiera('swh::deploy::deposit::runtime_secret_key')}" + db: + host: "%{hiera('swh::deploy::deposit::db::host')}" + port: "%{hiera('swh::deploy::deposit::db::port')}" + name: "%{hiera('swh::deploy::deposit::db::dbname')}" + user: "%{hiera('swh::deploy::deposit::db::dbuser')}" + password: "%{hiera('swh::deploy::deposit::db::password')}" + media_root: "%{hiera('swh::deploy::deposit::media_root_directory')}" + +swh::deploy::worker::loader_deposit::config_file: "%{hiera('swh::conf_directory')}/loader_deposit.yml" +swh::deploy::worker::loader_deposit::concurrency: 1 +swh::deploy::worker::loader_deposit::private_tmp: true +swh::deploy::worker::loader_deposit::loglevel: info +# deposit_basic_auth_swhworker_{username|password} in private_data +swh::deploy::worker::loader_deposit::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" extraction_dir: /tmp/swh.loader.deposit/ + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.deposit.loader.tasks + task_queues: + - swh.deposit.loader.tasks.LoadDepositArchiveTsk + url: https://deposit.softwareheritage.org + auth: + username: "%{hiera('deposit_basic_auth_swhworker_username')}" + password: "%{hiera('deposit_basic_auth_swhworker_password')}" -swh::deploy::deposit::client::swh_conf_file: "%{hiera('swh::deploy::deposit::conf_directory')}/client.yml" +swh::deploy::worker::checker_deposit::config_file: "%{hiera('swh::conf_directory')}/checker_deposit.yml" +swh::deploy::worker::checker_deposit::concurrency: 1 +swh::deploy::worker::checker_deposit::private_tmp: true +swh::deploy::worker::checker_deposit::loglevel: info # deposit_basic_auth_swhworker_{username|password} in private_data -swh::deploy::deposit::client::settings_private_data: +swh::deploy::worker::checker_deposit::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + extraction_dir: /tmp/swh.checker.deposit/ + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.deposit.loader.tasks + task_queues: + - swh.deposit.loader.tasks.ChecksDepositTsk url: https://deposit.softwareheritage.org auth: username: "%{hiera('deposit_basic_auth_swhworker_username')}" password: "%{hiera('deposit_basic_auth_swhworker_password')}" swh::deploy::deposit::backend::listen::host: 127.0.0.1 swh::deploy::deposit::backend::listen::port: "%{alias('swh::remote_service::deposit::port')}" swh::deploy::deposit::backend::workers: 8 swh::deploy::deposit::backend::reload_mercy: 3600 swh::deploy::deposit::backend::http_keepalive: 5 swh::deploy::deposit::backend::http_timeout: 3600 swh::deploy::objstorage_log_checker::conf_directory: "%{hiera('swh::deploy::objstorage::conf_directory')}" swh::deploy::objstorage_log_checker::conf_file: "%{hiera('swh::deploy::objstorage_log_checker::conf_directory')}/log_checker.yml" swh::deploy::objstorage_log_checker::user: "%{hiera('swh::deploy::objstorage::user')}" swh::deploy::objstorage_log_checker::group: "%{hiera('swh::deploy::objstorage::group')}" swh::deploy::objstorage_log_checker:config: storage: cls: pathslicing args: root: "%{hiera('swh::deploy::objstorage::directory')}" slicing: "%{hiera('swh::deploy::objstorage::slicing')}" batch_size: 1000 log_tag: objstorage.checker.log swh::deploy::objstorage_repair_checker::conf_directory: "%{hiera('swh::deploy::objstorage::conf_directory')}" swh::deploy::objstorage_repair_checker::conf_file: "%{hiera('swh::deploy::objstorage_repair_checker::conf_directory')}/repair_checker.yml" swh::deploy::objstorage_repair_checker::user: "%{hiera('swh::deploy::objstorage::user')}" swh::deploy::objstorage_repair_checker::group: "%{hiera('swh::deploy::objstorage::group')}" swh::deploy::objstorage_repair_checker::config: storage: cls: pathslicing args: root: "%{hiera('swh::deploy::objstorage::directory')}" slicing: "%{hiera('swh::deploy::objstorage::slicing')}" batch_size: 1000 log_tag: objstorage.checker.repair backup_storages: "%{alias('swh::remote_service::objstorage::config_as_dict')}" -swh::deploy::objstorage_archive_notifier_checker::conf_directory: "%{hiera('swh::deploy::objstorage::conf_directory')}" -swh::deploy::objstorage_archive_notifier_checker::conf_file: "%{hiera('swh::deploy::objstorage_archive_notifier_checker::conf_directory')}/archive_notifier_checker.yml" -swh::deploy::objstorage_archive_notifier_checker::user: "%{hiera('swh::deploy::objstorage::user')}" -swh::deploy::objstorage_archive_notifier_checker::group: "%{hiera('swh::deploy::objstorage::group')}" -swh::deploy::objstorage_archive_notifier_checker::log_tag: objstorage.checker.archive_notifier -swh::deploy::objstorage_archive_notifier_checker::storage_name: this-must-be-overriden-on-a-host-basis -swh::deploy::objstorage_archive_notifier_checker::db::host: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')}" -swh::deploy::objstorage_archive_notifier_checker::db::dbname: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')}" -swh::deploy::objstorage_archive_notifier_checker::db::user: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')}" -swh::deploy::objstorage_archive_notifier_checker::db::password: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" -swh::deploy::objstorage_archive_notifier_checker::config: - storage: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::objstorage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" - batch_size: 1000 - log_tag: objstorage.checker.archive_notifier - storage_name: this-must-be-overriden-on-a-per-host-basis - archiver_storage: - cls: db - args: - dbconn: "host=%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" - +swh::deploy::webapp::backported_packages: + stretch: + - python3-django + - python-django-common +swh::deploy::deposit::backported_packages: "%{alias('swh::deploy::webapp::backported_packages')}" swh::deploy::webapp::conf_directory: "%{hiera('swh::conf_directory')}/web" swh::deploy::webapp::conf_file: "%{hiera('swh::deploy::webapp::conf_directory')}/web.yml" swh::deploy::webapp::user: swhwebapp swh::deploy::webapp::group: swhwebapp swh::deploy::webapp::conf::log_dir: "%{hiera('swh::log_directory')}/webapp" swh::deploy::webapp::backend::listen::host: 127.0.0.1 swh::deploy::webapp::backend::listen::port: "%{alias('swh::remote_service::webapp::port')}" swh::deploy::webapp::backend::workers: 32 swh::deploy::webapp::backend::http_keepalive: 5 swh::deploy::webapp::backend::http_timeout: 3600 swh::deploy::webapp::backend::reload_mercy: 3600 swh::deploy::webapp::vhost::docroot: "/var/www/%{hiera('swh::deploy::webapp::vhost::name')}" swh::deploy::webapp::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" swh::deploy::webapp::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" swh::deploy::webapp::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" swh::deploy::webapp::vhost::hsts_header: "%{hiera('apache::hsts_header')}" swh::deploy::webapp::config::allowed_hosts: - archive.softwareheritage.org + - base.softwareheritage.org + - archive.internal.softwareheritage.org swh::deploy::webapp::production_db_dir: /var/lib/swh swh::deploy::webapp::production_db: "%{hiera('swh::deploy::webapp::production_db_dir')}/web.sqlite3" -swh::deploy::webapp::indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" # in private data: -# swh::deploy::webapp::config::grecaptcha::private_key # deposit_basic_auth_swhworker_username # deposit_basic_auth_swhworker_password swh::deploy::webapp::config: - storage: - cls: remote - args: - url: "http://127.0.0.1:%{hiera('swh::remote_service::storage::port')}/" - vault: "http://orangerie.internal.softwareheritage.org:%{hiera('swh::remote_service::vault::port')}/" - indexer_storage: "%{alias('swh::deploy::webapp::indexer_storage')}" + storage: "%{alias('swh::remote_service::storage::config')}" + vault: "%{alias('swh::remote_service::vault::config::writable')}" + indexer_storage: "%{alias('swh::remote_service::indexer::config')}" + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" log_dir: "%{hiera('swh::deploy::webapp::conf::log_dir')}" secret_key: "%{hiera('swh::deploy::webapp::conf::secret_key')}" content_display_max_size: 1048576 throttling: cache_uri: "%{hiera('memcached::server::bind')}:%{hiera('memcached::server::port')}" scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 128.93.193.29 + - 131.107.174.0/24 + # OpenAIRE + - 213.135.60.145 + - 213.135.60.146 + # DINSIC + - 37.187.137.47 + swh_api_origin_visit_latest: + # This endpoint gets called a lot (by default, up to 70 times + # per origin search), so it deserves a much higher rate-limit + # than the rest of the API. + limiter_rate: + default: 700/m swh_vault_cooking: limiter_rate: default: 120/h GET: 60/m exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 128.93.193.29 + - 131.107.174.0/24 + # OpenAIRE + - 213.135.60.145 + - 213.135.60.146 swh_save_origin: limiter_rate: default: 120/h POST: 10/h exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 128.93.193.29 + - 131.107.174.0/24 + # OpenAIRE + - 213.135.60.145 + - 213.135.60.146 allowed_hosts: "%{alias('swh::deploy::webapp::config::allowed_hosts')}" - scheduler: - cls: remote - args: - url: "http://saatchi.internal.softwareheritage.org:%{hiera('swh::remote_service::scheduler::port')}/" - grecaptcha: - validation_url: https://www.google.com/recaptcha/api/siteverify - site_key: 6Le0l2wUAAAAAHgtL_aJmIfnow8pFB62sAoG8B7m - private_key: "%{hiera('swh::deploy::webapp::config::grecaptcha::private_key')}" production_db: "%{hiera('swh::deploy::webapp::production_db')}" deposit: private_api_url: https://deposit.softwareheritage.org/1/private/ private_api_user: "%{hiera('deposit_basic_auth_swhworker_username')}" private_api_password: "%{hiera('deposit_basic_auth_swhworker_password')}" swh::deploy::webapp::locked_endpoints: - /api/1/content/[^/]+/symbol/ - /api/1/entity/ - /api/1/provenance/ -swh::deploy::scheduler::conf_file: "%{hiera('swh::conf_directory')}/scheduler.ini" +# local configuration for the scheduler +swh::deploy::scheduler::config::local: &swh_scheduler_local_config + scheduler: + cls: local + args: + db: "host=%{hiera('swh::deploy::scheduler::db::host')} port=%{hiera('swh::deploy::scheduler::db::port')} dbname=%{hiera('swh::deploy::scheduler::db::dbname')} user=%{hiera('swh::deploy::scheduler::db::user')} password=%{hiera('swh::deploy::scheduler::db::password')}" + +swh::deploy::scheduler::conf_file: "%{hiera('swh::conf_directory')}/scheduler.yml" swh::deploy::scheduler::user: swhscheduler swh::deploy::scheduler::group: swhscheduler -swh::deploy::scheduler::db::host: db +swh::deploy::scheduler::db::host: db.internal.softwareheritage.org +swh::deploy::scheduler::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" swh::deploy::scheduler::db::dbname: softwareheritage-scheduler swh::deploy::scheduler::db::user: swhscheduler # swh::deploy::scheduler::db::password in private data -swh::deploy::scheduler::database: "host=%{hiera('swh::deploy::scheduler::db::host')} dbname=%{hiera('swh::deploy::scheduler::db::dbname')} user=%{hiera('swh::deploy::scheduler::db::user')} password=%{hiera('swh::deploy::scheduler::db::password')}" # swh::deploy::scheduler::task_broker::password in private data swh::deploy::scheduler::task_broker: "amqp://swhproducer:%{hiera('swh::deploy::scheduler::task_broker::password')}@rabbitmq:5672//" +swh::deploy::scheduler::listener::log_level: INFO +swh::deploy::scheduler::runner::log_level: INFO +swh::deploy::scheduler::config: + <<: *swh_scheduler_local_config + celery: + task_broker: "%{alias('swh::deploy::scheduler::task_broker')}" + swh::deploy::scheduler::task_packages: - python3-swh.lister - python3-swh.loader.debian - python3-swh.loader.dir - python3-swh.loader.git - python3-swh.loader.mercurial - python3-swh.loader.pypi - python3-swh.loader.svn - python3-swh.loader.tar + - python3-swh.loader.npm - python3-swh.deposit.loader - python3-swh.indexer - python3-swh.vault swh::deploy::scheduler::backported_packages: jessie: - python3-sqlalchemy swh::deploy::scheduler::task_modules: + - swh.lister.bitbucket.tasks + - swh.lister.cgit.tasks - swh.lister.debian.tasks - swh.lister.github.tasks - swh.lister.gitlab.tasks + - swh.lister.npm.tasks - swh.lister.pypi.tasks - swh.loader.debian.tasks - swh.loader.dir.tasks - swh.loader.git.tasks - swh.loader.mercurial.tasks - swh.loader.pypi.tasks - swh.loader.svn.tasks - swh.loader.tar.tasks - swh.deposit.loader.tasks - swh.indexer.tasks - swh.vault.cooking_tasks swh::deploy::scheduler::remote::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::remote::conf_file: "%{hiera('swh::deploy::scheduler::remote::conf_dir')}/scheduler.yml" swh::deploy::scheduler::remote::user: swhscheduler swh::deploy::scheduler::remote::group: swhscheduler swh::deploy::scheduler::remote::backend::listen::host: 127.0.0.1 swh::deploy::scheduler::remote::backend::listen::port: "%{alias('swh::remote_service::scheduler::port')}" swh::deploy::scheduler::remote::backend::workers: 16 swh::deploy::scheduler::remote::backend::reload_mercy: 3600 swh::deploy::scheduler::remote::backend::http_keepalive: 5 swh::deploy::scheduler::remote::backend::http_timeout: 3600 swh::deploy::scheduler::remote::backend::max_requests: 10000 swh::deploy::scheduler::remote::backend::max_requests_jitter: 1000 swh::deploy::scheduler::remote::backend::server_names: - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" -swh::deploy::scheduler::remote::config: - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" +swh::deploy::scheduler::remote::config: "%{alias('swh::deploy::scheduler::config::local')}" swh::deploy::scheduler::updater::backend::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::updater::backend::conf_file: "%{hiera('swh::deploy::scheduler::updater::backend::conf_dir')}/scheduler-updater.yml" swh::deploy::scheduler::updater::backend::user: swhscheduler swh::deploy::scheduler::updater::backend::group: swhscheduler swh::deploy::scheduler::updater::backend::db::host: db.internal.softwareheritage.org -swh::deploy::scheduler::updater::backend::db::port: 5432 +swh::deploy::scheduler::updater::backend::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" swh::deploy::scheduler::updater::backend::db::dbname: softwareheritage-scheduler-updater swh::deploy::scheduler::updater::backend::db::user: swhscheduler swh::deploy::scheduler::updater::backend::config: - scheduling_updater_db: "host=%{hiera('swh::deploy::scheduler::updater::backend::db::host')} dbname=%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')} user=%{hiera('swh::deploy::scheduler::updater::backend::db::user')} password=%{hiera('swh::deploy::scheduler::updater::backend::db::password')}" + scheduling_updater_db: "host=%{hiera('swh::deploy::scheduler::updater::backend::db::host')} port=%{hiera('swh::deploy::scheduler::updater::backend::db::port')} dbname=%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')} user=%{hiera('swh::deploy::scheduler::updater::backend::db::user')} password=%{hiera('swh::deploy::scheduler::updater::backend::db::password')}" cache_read_limit: 1000 swh::deploy::scheduler::updater::consumer::ghtorrent::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::updater::consumer::ghtorrent::conf_file: "%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::conf_dir')}/ghtorrent.yml" swh::deploy::scheduler::updater::consumer::user: swhscheduler swh::deploy::scheduler::updater::consumer::group: swhscheduler swh::deploy::scheduler::updater::consumer::ghtorrent::user: streamer swh::deploy::scheduler::updater::consumer::ghtorrent::port: 2765 swh::deploy::scheduler::updater::consumer::ghtorrent::config: conn: url: "amqp://%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::user')}:%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::password')}@localhost:%{hiera('swh::deploy::scheduler::updater::consumer::ghtorrent::port')}" exchange_name: ght-streams routing_key: evt.*.insert queue_name: swh_queue debug: no batch_cache_write: 1000 rabbitmq_prefetch_read: 1000 swh::deploy::scheduler::updater::writer::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::updater::writer::conf_file: "%{hiera('swh::deploy::scheduler::updater::writer::conf_dir')}/scheduler-updater-writer.yml" swh::deploy::scheduler::updater::writer::user: swhscheduler swh::deploy::scheduler::updater::writer::group: swhscheduler # systemd-analyze calendar "*-*-* *:00:00" swh::deploy::scheduler::updater::writer::timer_period: "00/4:00" swh::deploy::scheduler::updater::writer::config: - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" scheduler_updater: scheduling_updater_db: "host=%{hiera('swh::deploy::scheduler::updater::backend::db::host')} dbname=%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')} user=%{hiera('swh::deploy::scheduler::updater::backend::db::user')} password=%{hiera('swh::deploy::scheduler::updater::backend::db::password')}" cache_read_limit: 1000 pause: 10 verbose: no swh::deploy::scheduler::archive::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::archive::conf_file: "%{hiera('swh::deploy::scheduler::archive::conf_dir')}/elastic.yml" swh::deploy::scheduler::archive::user: "%{hiera('swh::deploy::scheduler::user')}" swh::deploy::scheduler::archive::config: + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" storage_nodes: - host: esnode2.internal.softwareheritage.org port: 9200 - host: esnode3.internal.softwareheritage.org port: 9200 - host: esnode1.internal.softwareheritage.org port: 9200 client_options: sniff_on_start: false sniff_on_connection_fail: True http_compress: false -swh::deploy::loader_debian::db::host: db.internal.softwareheritage.org -swh::deploy::loader_debian::db::port: 5432 -swh::deploy::loader_debian::db::user: lister-debian -# in private data: swh::deploy::loader_debian::db::password -swh::deploy::loader_debian::db::dbname: lister-debian +# Main lister configuration +swh::deploy::worker::lister::db::user: swh-lister +swh::deploy::worker::lister::db::name: swh-lister +swh::deploy::worker::lister::db::host: db.internal.softwareheritage.org +swh::deploy::worker::lister::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +# swh::deploy::worker::lister::db::password in private data + +swh::deploy::worker::loader_debian::db::host: db.internal.softwareheritage.org +swh::deploy::worker::loader_debian::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +# in private data: swh::deploy::worker::loader_debian::db::password +swh::deploy::worker::loader_debian::db::user: "%{alias('swh::deploy::worker::lister::db::user')}" +swh::deploy::worker::loader_debian::db::dbname: "%{alias('swh::deploy::worker::lister::db::name')}" # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@rabbitmq:5672//" swh::deploy::worker::instances: - - swh_loader_debian - - swh_loader_git - - swh_lister_debian - - swh_lister_github - -swh::deploy::worker::swh_loader_git::concurrency: 2 -swh::deploy::worker::swh_loader_git::loglevel: info -swh::deploy::worker::swh_loader_git::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_git::config: - storage: "%{alias('swh::remote_service::storage::config')}" + - loader_debian + - loader_git + - lister + +swh::deploy::worker::loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" +swh::deploy::worker::loader_git::concurrency: 1 +swh::deploy::worker::loader_git::loglevel: info +swh::deploy::worker::loader_git::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" save_data: true save_data_path: /srv/storage/space/data/sharded_packfiles directory_packet_size: 100 - -swh::deploy::worker::swh_loader_debian::concurrency: 1 -swh::deploy::worker::swh_loader_debian::loglevel: info -swh::deploy::worker::swh_loader_debian::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_debian::config: - storage: "%{alias('swh::remote_service::storage::config')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.git.tasks + task_queues: + - swh.loader.git.tasks.UpdateGitRepository + - swh_loader_git + # loader-git-disk + - swh.loader.git.tasks.LoadDiskGitRepository + - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository + +swh::deploy::worker::loader_debian::config_file: "%{hiera('swh::conf_directory')}/loader_debian.yml" +swh::deploy::worker::loader_debian::concurrency: 1 +swh::deploy::worker::loader_debian::loglevel: info +swh::deploy::worker::loader_debian::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" directory_packet_size: 100 - lister_db_url: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" - -swh::deploy::worker::swh_lister_debian::concurrency: 1 -swh::deploy::worker::swh_lister_debian::loglevel: info -swh::deploy::worker::swh_lister_debian::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_lister_debian::config: - storage: "%{alias('swh::remote_service::storage::config')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - lister_db_url: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" - -swh::deploy::worker::swh_loader_git_disk::concurrency: 2 -swh::deploy::worker::swh_loader_git_disk::loglevel: info -swh::deploy::worker::swh_loader_git_disk::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_git_disk::config: - storage: "%{alias('swh::remote_service::storage::config')}" - directory_packet_size: 100 - -swh::deploy::worker::swh_loader_mercurial::concurrency: 1 -swh::deploy::worker::swh_loader_mercurial::private_tmp: true -swh::deploy::worker::swh_loader_mercurial::loglevel: info -swh::deploy::worker::swh_loader_mercurial::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_mercurial::config: - storage: "%{alias('swh::remote_service::storage::config')}" + lister_db_url: "postgresql://%{hiera('swh::deploy::worker::loader_debian::db::user')}:%{hiera('swh::deploy::worker::loader_debian::db::password')}@%{hiera('swh::deploy::worker::loader_debian::db::host')}:%{hiera('swh::deploy::worker::loader_debian::db::port')}/%{hiera('swh::deploy::worker::loader_debian::db::dbname')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.debian.tasks + task_queues: + - swh.loader.debian.tasks.LoadDebianPackage + - swh_loader_debian + +swh::deploy::worker::lister::config_file: "%{hiera('swh::conf_directory')}/lister.yml" +swh::deploy::worker::lister::concurrency: 5 +swh::deploy::worker::lister::loglevel: warning +swh::deploy::worker::lister::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + lister: + cls: local + args: + db: "postgresql://%{hiera('swh::deploy::worker::lister::db::user')}:%{hiera('swh::deploy::worker::lister::db::password')}@%{hiera('swh::deploy::worker::lister::db::host')}:%{hiera('swh::deploy::worker::lister::db::port')}/%{hiera('swh::deploy::worker::lister::db::name')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.lister.bitbucket.tasks + - swh.lister.cgit.tasks + - swh.lister.debian.tasks + - swh.lister.github.tasks + - swh.lister.gitlab.tasks + - swh.lister.gnu.tasks + - swh.lister.npm.tasks + - swh.lister.phabricator.tasks + - swh.lister.pypi.tasks + task_queues: + - swh.lister.bitbucket.tasks.IncrementalBitBucketLister + - swh.lister.bitbucket.tasks.FullBitBucketRelister + - swh.lister.cgit.tasks.CGitListerTask + - swh.lister.debian.tasks.DebianListerTask + - swh.lister.github.tasks.IncrementalGitHubLister + - swh.lister.github.tasks.RangeGitHubLister + - swh.lister.github.tasks.FullGitHubRelister + - swh.lister.gitlab.tasks.IncrementalGitLabLister + - swh.lister.gitlab.tasks.RangeGitLabLister + - swh.lister.gitlab.tasks.FullGitLabRelister + - swh.lister.gnu.tasks.GNUListerTask + - swh.lister.npm.tasks.NpmListerTask + - swh.lister.phabricator.tasks.FullPhabricatorLister + - swh.lister.pypi.tasks.PyPIListerTask + credentials: "%{alias('swh::deploy::worker::lister::config::credentials')}" + +swh::deploy::worker::loader_mercurial::config_file: "%{hiera('swh::conf_directory')}/loader_mercurial.yml" +swh::deploy::worker::loader_mercurial::concurrency: 1 +swh::deploy::worker::loader_mercurial::private_tmp: true +swh::deploy::worker::loader_mercurial::loglevel: info +swh::deploy::worker::loader_mercurial::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" reduce_effort: False - -swh::deploy::worker::swh_loader_pypi::concurrency: 1 -swh::deploy::worker::swh_loader_pypi::private_tmp: true -swh::deploy::worker::swh_loader_pypi::loglevel: info -swh::deploy::worker::swh_loader_pypi::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_pypi::config: - storage: "%{alias('swh::remote_service::storage::config')}" - -swh::deploy::worker::swh_loader_svn::concurrency: 1 -swh::deploy::worker::swh_loader_svn::private_tmp: true -swh::deploy::worker::swh_loader_svn::limit_no_file: 8192 -swh::deploy::worker::swh_loader_svn::loglevel: info + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.mercurial.tasks + task_queues: + - swh.loader.mercurial.tasks.LoadMercurial + - swh.loader.mercurial.tasks.LoadArchiveMercurial + - swh_loader_mercurial + - swh_loader_mercurial_archive + +swh::deploy::worker::loader_pypi::config_file: "%{hiera('swh::conf_directory')}/loader_pypi.yml" +swh::deploy::worker::loader_pypi::concurrency: 1 +swh::deploy::worker::loader_pypi::private_tmp: true +swh::deploy::worker::loader_pypi::loglevel: info +swh::deploy::worker::loader_pypi::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.pypi.tasks + task_queues: + - swh.loader.pypi.tasks.LoadPyPI + - swh_loader_pypi + +swh::deploy::worker::loader_npm::config_file: "%{hiera('swh::conf_directory')}/loader_npm.yml" +swh::deploy::worker::loader_npm::concurrency: 1 +swh::deploy::worker::loader_npm::private_tmp: true +swh::deploy::worker::loader_npm::loglevel: info +swh::deploy::worker::loader_npm::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.npm.tasks + task_queues: + - swh.loader.npm.tasks.LoadNpm + +swh::deploy::worker::loader_svn::config_file: "%{hiera('swh::conf_directory')}/loader_svn.yml" +swh::deploy::worker::loader_svn::concurrency: 1 +swh::deploy::worker::loader_svn::private_tmp: true +swh::deploy::worker::loader_svn::limit_no_file: 8192 +swh::deploy::worker::loader_svn::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_loader_svn::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_loader_svn::config: - storage: "%{alias('swh::remote_service::storage::config')}" - -swh::deploy::worker::swh_lister_github::concurrency: 1 -swh::deploy::worker::swh_lister_github::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::swh_lister_github::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_lister_github::db::host: db -swh::deploy::worker::swh_lister_github::db::dbname: lister-github -swh::deploy::worker::swh_lister_github::db::user: lister-github -#swh::deploy::worker::swh_lister_github::db::password in private data -swh::deploy::worker::swh_lister_github::config: - storage: "%{alias('swh::remote_service::storage::config')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - lister_db_url: "postgresql://%{hiera('swh::deploy::worker::swh_lister_github::db::user')}:%{hiera('swh::deploy::worker::swh_lister_github::db::password')}@%{hiera('swh::deploy::worker::swh_lister_github::db::host')}/%{hiera('swh::deploy::worker::swh_lister_github::db::dbname')}" - # credentials in private data - -swh::deploy::worker::swh_lister_gitlab::concurrency: 1 -swh::deploy::worker::swh_lister_gitlab::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::swh_lister_gitlab::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_lister_gitlab::db::host: db -swh::deploy::worker::swh_lister_gitlab::db::dbname: lister-gitlab -swh::deploy::worker::swh_lister_gitlab::db::user: lister-gitlab -#swh::deploy::worker::swh_lister_gitlab::db::password in private data -swh::deploy::worker::swh_lister_gitlab::config: - storage: "%{alias('swh::remote_service::storage::config')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - lister_db_url: "postgresql://%{hiera('swh::deploy::worker::swh_lister_gitlab::db::user')}:%{hiera('swh::deploy::worker::swh_lister_gitlab::db::password')}@%{hiera('swh::deploy::worker::swh_lister_gitlab::db::host')}/%{hiera('swh::deploy::worker::swh_lister_gitlab::db::dbname')}" - # credentials in private data - -swh::deploy::worker::swh_lister_pypi::concurrency: 1 -swh::deploy::worker::swh_lister_pypi::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::swh_lister_pypi::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_lister_pypi::db::host: db -swh::deploy::worker::swh_lister_pypi::db::dbname: lister-pypi -swh::deploy::worker::swh_lister_pypi::db::user: lister-pypi -#swh::deploy::worker::swh_lister_pypi::db::password in private data -swh::deploy::worker::swh_lister_pypi::config: - storage: "%{alias('swh::remote_service::storage::config')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" - lister_db_url: "postgresql://%{hiera('swh::deploy::worker::swh_lister_pypi::db::user')}:%{hiera('swh::deploy::worker::swh_lister_pypi::db::password')}@%{hiera('swh::deploy::worker::swh_lister_pypi::db::host')}/%{hiera('swh::deploy::worker::swh_lister_pypi::db::dbname')}" - # credentials in private data - -swh::deploy::worker::swh_storage_archiver::conf_directory: "%{hiera('swh::conf_directory')}/archiver" -swh::deploy::worker::swh_storage_archiver::conf_file: "%{hiera('swh::deploy::worker::swh_storage_archiver::conf_directory')}/worker.yml" -swh::deploy::worker::swh_storage_archiver::archival_max_age: 3600 -swh::deploy::worker::swh_storage_archiver::max_queue_length: 100000 -swh::deploy::worker::swh_storage_archiver::retention_policy: 3 -swh::deploy::worker::swh_storage_archiver::batch_max_size: 5 -swh::deploy::worker::swh_storage_archiver::db::host: db -swh::deploy::worker::swh_storage_archiver::db::dbname: softwareheritage-archiver -swh::deploy::worker::swh_storage_archiver::db::user: "%{hiera('swh::deploy::storage::db::user')}" -swh::deploy::worker::swh_storage_archiver::db::password: "%{hiera('swh::deploy::storage::db::password')}" -swh::deploy::worker::swh_storage_archiver::concurrency: 10 -swh::deploy::worker::swh_storage_archiver::max_tasks_per_child: 500 -swh::deploy::worker::swh_storage_archiver::loglevel: info +swh::deploy::worker::loader_svn::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.svn.tasks + task_queues: + - swh.loader.svn.tasks.LoadSvnRepository + - swh.loader.svn.tasks.MountAndLoadSvnRepository + - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository + - swh_loader_svn + - swh_loader_svn_mount_and_load + + +swh::deploy::base_indexer::config_directory: "%{hiera('swh::conf_directory')}/indexer" + +swh::deploy::indexer_journal_client::config_file: "journal_client.yml" +swh::deploy::indexer_journal_client::user: swhstorage +swh::deploy::indexer_journal_client::group: swhstorage +swh::deploy::indexer_journal_client::config: + journal: + brokers: "%{alias('swh::deploy::journal::brokers')}" + group_id: swh.indexer.journal_client + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + +swh::deploy::worker::indexer_content_mimetype::config_file: "%{hiera('swh::conf_directory')}/indexer_content_mimetype.yml" +swh::deploy::worker::indexer_content_mimetype::concurrency: 1 +swh::deploy::worker::indexer_content_mimetype::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_storage_archiver::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_storage_archiver::config: - archival_max_age: "%{hiera('swh::deploy::worker::swh_storage_archiver::archival_max_age')}" - retention_policy: "%{hiera('swh::deploy::worker::swh_storage_archiver::retention_policy')}" - batch_max_size: "%{hiera('swh::deploy::worker::swh_storage_archiver::batch_max_size')}" - archiver_storage: - cls: db - args: - dbconn: "host=%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" - storages: - - host: uffizi - <<: *swh_objstorage_config_uffizi - - host: banco - <<: *swh_objstorage_config_banco - - host: azure - <<: *swh_objstorage_config_azure - sources: - - uffizi - - banco - max_queue_length: "%{hiera('swh::deploy::worker::swh_storage_archiver::max_queue_length')}" - -swh::deploy::worker::swh_indexer::orchestrator::concurrency: 2 -swh::deploy::worker::swh_indexer::orchestrator::loglevel: info -swh::deploy::worker::swh_indexer::orchestrator::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::orchestrator::config: - indexers: - mimetype: - batch_size: 100 - check_presence: false - -swh::deploy::worker::swh_indexer::orchestrator_text::concurrency: 2 -swh::deploy::worker::swh_indexer::orchestrator_text::loglevel: info -swh::deploy::worker::swh_indexer::orchestrator_text::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::orchestrator_text::config: - indexers: - fossology_license: - batch_size: 100 - check_presence: false - -swh::deploy::worker::swh_indexer::base::concurrency: 2 -swh::deploy::worker::swh_indexer::base::loglevel: info -# Contains a password: in private data -swh::deploy::worker::swh_indexer::base::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::base::config: - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" - storage: "%{alias('swh::remote_service::storage::config')}" +swh::deploy::worker::indexer_content_mimetype::config: + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" + storage: "%{alias('swh::remote_service::storage::config')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.indexer.tasks + task_queues: + - swh.indexer.tasks.ContentMimetype + - swh.indexer.tasks.ContentRangeMimetype + - swh_indexer_content_mimetype_range + tools: + name: file + version: 1:5.30-1+deb9u1 + configuration: + type: library + debian-package: python3-magic + write_batch_size: 1000 -swh::deploy::worker::swh_indexer::mimetype::concurrency: 6 -swh::deploy::worker::swh_indexer::mimetype::loglevel: "%{hiera('swh::deploy::worker::swh_indexer::base::loglevel')}" +swh::deploy::worker::indexer_origin_intrinsic_metadata::config_file: "%{hiera('swh::conf_directory')}/indexer_origin_intrinsic_metadata.yml" +swh::deploy::worker::indexer_origin_intrinsic_metadata::concurrency: 1 +swh::deploy::worker::indexer_origin_intrinsic_metadata::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::mimetype::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::mimetype::config: - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" +swh::deploy::worker::indexer_origin_intrinsic_metadata::config: + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" - destination_queue: swh.indexer.tasks.SWHOrchestratorTextContentsTask - rescheduling_task: swh.indexer.tasks.SWHContentMimetypeTask + storage: "%{alias('swh::remote_service::storage::config')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.indexer.tasks + task_queues: + - swh.indexer.tasks.OriginMetadata + tools: + name: swh-metadata-detector + version: 0.0.2 + configuration: {} -swh::deploy::worker::swh_indexer::rehash::concurrency: 5 -swh::deploy::worker::swh_indexer::rehash::loglevel: "%{hiera('swh::deploy::worker::swh_indexer::base::loglevel')}" +swh::deploy::worker::indexer_rehash::config_file: "rehash.yml" +swh::deploy::worker::indexer_rehash::concurrency: 5 +swh::deploy::worker::indexer_rehash::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::rehash::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::rehash::config: - # Needs write access: explicitly configured to use uffizi - storage: "%{alias('swh::remote_service::storage::config::uffizi')}" +swh::deploy::worker::indexer_rehash::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" compute_checksums: - blake2s256 - rescheduling_task: swh.indexer.tasks.SWHRecomputeChecksumsTask batch_size_retrieve_content: 10000 batch_size_update: 5000 -swh::deploy::worker::swh_indexer::fossology_license::concurrency: 7 -swh::deploy::worker::swh_indexer::fossology_license::loglevel: "%{hiera('swh::deploy::worker::swh_indexer::base::loglevel')}" +swh::deploy::worker::indexer_fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml" +swh::deploy::worker::indexer_fossology_license::concurrency: 1 +swh::deploy::worker::indexer_fossology_license::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::fossology_license::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::fossology_license::config: - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" +swh::deploy::worker::indexer_fossology_license::config: + indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" - rescheduling_task: swh.indexer.tasks.SWHContentFossologyLicenseTask + storage: "%{alias('swh::remote_service::storage::config')}" workdir: /tmp/swh/indexer.fossology.license/ tools: name: 'nomos' version: '3.1.0rc2-31-ga2cbb8c' configuration: command_line: "nomossa " - -swh::deploy::worker::swh_indexer::language::concurrency: 3 -swh::deploy::worker::swh_indexer::language::loglevel: "%{hiera('swh::deploy::worker::swh_indexer::base::loglevel')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.indexer.tasks + task_queues: + - swh.indexer.tasks.ContentFossologyLicense + - swh.indexer.tasks.ContentRangeFossologyLicense + - swh_indexer_content_fossology_license_range + write_batch_size: 1000 + +swh::deploy::worker::indexer_content_ctags::config_file: "%{hiera('swh::conf_directory')}/indexer_content_ctags.yml" +swh::deploy::worker::indexer_content_ctags::concurrency: 2 +swh::deploy::worker::indexer_content_ctags::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::language::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_indexer::language::config: - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" - objstorage: "%{alias('swh::remote_service::objstorage::config')}" - rescheduling_task: swh.indexer.tasks.SWHContentLanguageTask - tools: - name: 'pygments' - version: '2.0.1+dfsg-1.1+deb8u1' - configuration: - type: "library" - debian-package: "python3-pygments" - max_content_size: 10240 - -swh::deploy::worker::swh_indexer::ctags::concurrency: 2 -swh::deploy::worker::swh_indexer::ctags::loglevel: "%{hiera('swh::deploy::worker::swh_indexer::base::loglevel')}" -# Contains a password: in private data -swh::deploy::worker::swh_indexer::ctags::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" # objstorage configuration from swh::azure_objstorage::config is merged in the manifest -swh::deploy::worker::swh_indexer::ctags::config: - indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" +swh::deploy::worker::indexer_content_ctags::config: + indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" workdir: /tmp/swh/indexer.ctags/ tools: name: 'universal-ctags' - version: '~git7859817b' + version: '0+git20181215-2' configuration: command_line: "ctags --fields=+lnz --sort=no --links=no --output-format=json " + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.indexer.tasks + task_queues: + - swh.indexer.tasks.Ctags + languages: abap: '' abnf: '' actionscript: '' actionscript-3: '' ada: Ada adl: '' agda: '' alloy: '' ambienttalk: '' antlr: '' antlr-with-actionscript-target: '' antlr-with-c#-target: '' antlr-with-cpp-target: '' antlr-with-java-target: '' antlr-with-objectivec-target: '' antlr-with-perl-target: '' antlr-with-python-target: '' antlr-with-ruby-target: '' apacheconf: '' apl: '' applescript: '' arduino: '' aspectj: '' aspx-cs: '' aspx-vb: '' asymptote: '' autohotkey: '' autoit: '' awk: Awk base-makefile: Make bash: Sh bash-session: Sh batchfile: DosBatch bbcode: '' bc: '' befunge: '' blitzbasic: Basic blitzmax: '' bnf: '' boo: '' boogie: '' brainfuck: '' bro: '' bugs: '' c: C c#: C# c++: C++ c-objdump: asm ca65-assembler: asm cadl: '' camkes: '' cbm-basic-v2: '' ceylon: Java cfengine3: '' cfstatement: '' chaiscript: '' chapel: '' cheetah: '' cirru: '' clay: '' clojure: Clojure clojurescript: Clojure cmake: Make cobol: Cobol cobolfree: Cobol coffeescript: CoffeeScript coldfusion-cfc: HTML coldfusion-html: HTML common-lisp: Lisp component-pascal: Pascal coq: '' cpp-objdump: Asm cpsa: '' crmsh: Sh croc: '' cryptol: '' csound-document: '' csound-orchestra: '' csound-score: '' css: CSS css+django/jinja: CSS css+genshi-text: CSS css+lasso: CSS css+mako: CSS css+mozpreproc: CSS css+myghty: CSS css+php: CSS css+ruby: CSS css+smarty: CSS cuda: '' cypher: '' cython: Python d: D d-objdump: Asm darcs-patch: Diff dart: '' debian-control-file: '' debian-sourcelist: '' delphi: '' dg: '' diff: Diff django/jinja: Python docker: Iniconf dtd: '' duel: '' dylan: '' dylan-session: '' dylanlid: '' earl-grey: '' easytrieve: '' ebnf: '' ec: '' ecl: '' eiffel: Eiffel elixir: '' elixir-iex-session: '' elm: '' emacslisp: Lisp embedded-ragel: '' erb: Ruby erlang: Erlang erlang-erl-session: Erlang evoque: '' ezhil: '' factor: '' fancy: '' fantom: '' felix: '' fish: '' fortran: Fortran fortranfixed: Fortran nfoxpro: '' fsharp: Ocaml gap: '' gas: '' genshi: '' genshi-text: '' gettext-catalog: '' gherkin: '' glsl: '' gnuplot: '' go: Go golo: '' gooddata-cl: '' gosu: '' gosu-template: '' groff: '' groovy: '' haml: '' handlebars: '' haskell: '' haxe: '' hexdump: '' html: HTML html+cheetah: HTML html+django/jinja: HTML html+evoque: HTML html+genshi: HTML html+handlebars: HTML html+lasso: HTML html+mako: HTML html+myghty: HTML html+php: HTML html+smarty: HTML html+twig: HTML html+velocity: HTML http: '' hxml: '' hy: Lisp hybris: '' idl: '' idris: '' igor: '' inform-6: '' inform-6-template: '' inform-7: '' ini: Iniconf io: '' ioke: '' irc-logs: '' isabelle: '' j: '' jade: '' jags: '' jasmin: '' java: Java java-server-page: Java javascript: JavaScript javascript+cheetah: JavaScript javascript+django/jinja: JavaScript javascript+genshi-text: JavaScript javascript+lasso: JavaScript javascript+mako: JavaScript javascript+mozpreproc: JavaScript javascript+myghty: JavaScript javascript+php: JavaScript javascript+ruby: JavaScript javascript+smarty: JavaScript jcl: '' json: JSON json-ld: JSON julia: '' julia-console: '' kal: '' kconfig: '' koka: '' kotlin: '' lasso: '' lean: '' lesscss: CSS lighttpd-configuration-file: Iniconf limbo: '' liquid: '' literate-agda: '' literate-cryptol: '' literate-haskell: '' literate-idris: '' livescript: '' llvm: '' logos: '' logtalk: '' lsl: '' lua: Lua makefile: Make mako: '' maql: '' mask: '' mason: '' mathematica: MatLab matlab: MatLab matlab-session: MatLab minid: '' modelica: '' modula-2: '' moinmoin/trac-wiki-markup: '' monkey: '' moocode: '' moonscript: '' mozhashpreproc: '' mozpercentpreproc: '' mql: '' mscgen: '' msdos-session: '' mupad: '' mxml: '' myghty: '' mysql: SQL nasm: Asm nemerle: '' nesc: '' newlisp: Lisp newspeak: '' nginx-configuration-file: '' nimrod: '' nit: '' nix: '' nsis: '' numpy: '' objdump: Asm objdump-nasm: Asm objective-c: ObjectiveC objective-c++: Objective-C objective-j: '' ocaml: Ocaml octave: '' odin: '' ooc: '' opa: '' openedge-abl: '' pacmanconf: '' pan: '' parasail: '' pawn: '' perl: Perl perl6: Perl6 php: PHP pig: '' pike: '' pkgconfig: '' pl/pgsql: SQL postgresql-console-(psql): '' postgresql-sql-dialect: SQL postscript: '' povray: '' powershell: '' powershell-session: '' praat: '' prolog: '' properties: Iniconf protocol-buffer: Protobuf puppet: '' pypy-log: '' python: Python python-3: Python python-3.0-traceback: Python python-console-session: Python python-traceback: Python qbasic: '' qml: '' qvto: '' racket: LISP ragel: '' ragel-in-c-host: '' ragel-in-cpp-host: '' ragel-in-d-host: '' ragel-in-java-host: '' ragel-in-objective-c-host: '' ragel-in-ruby-host: '' raw-token-data: '' rconsole: '' rd: '' rebol: '' red: '' redcode: '' reg: '' resourcebundle: '' restructuredtext: reStructuredText rexx: REXX rhtml: '' roboconf-graph: '' roboconf-instances: '' robotframework: '' rpmspec: RpmSpec rql: '' rsl: '' ruby: Ruby ruby-irb-session: Sh rust: Rust s: '' sass: '' scala: Java scalate-server-page: '' scaml: SML scheme: Lisp scilab: '' scss: '' shen: '' slim: '' smali: '' smalltalk: '' smarty: '' snobol: '' sourcepawn: '' sparql: '' sql: SQL sqlite3con: SQL squidconf: '' stan: '' standard-ml: SML supercollider: '' swift: '' swig: '' systemverilog: SystemVerilog tads-3: '' tap: '' tcl: '' tcsh: Sh tcsh-session: Sh tea: '' termcap: '' terminfo: '' terraform: '' tex: Tex text-only: '' thrift: '' todotxt: '' trafficscript: '' treetop: '' turtle: '' twig: '' typescript: '' urbiscript: '' vala: '' vb.net: Basic vctreestatus: '' velocity: '' verilog: Verilog vgl: '' vhdl: VHDL viml: Vim x10: '' xml: '' xml+cheetah: '' xml+django/jinja: '' xml+evoque: '' xml+lasso: '' xml+mako: '' xml+myghty: '' xml+php: '' xml+ruby: '' xml+smarty: '' xml+velocity: '' xquery: '' xslt: XSLT xtend: '' xul+mozpreproc: '' yaml: '' yaml+jinja: '' zephir: Zephir unknown: '' -swh::deploy::worker::swh_vault_cooker::concurrency: 20 -swh::deploy::worker::swh_vault_cooker::loglevel: info -swh::deploy::worker::swh_vault_cooker::task_broker: "%{hiera('swh::deploy::worker::task_broker')}" -swh::deploy::worker::swh_vault_cooker::conf_file: "%{hiera('swh::conf_directory')}/vault/cooker.yml" -swh::deploy::worker::swh_vault_cooker::config: +swh::deploy::worker::vault_cooker::config_file: "%{hiera('swh::conf_directory')}/vault_cooker.yml" +swh::deploy::worker::vault_cooker::concurrency: 20 +swh::deploy::worker::vault_cooker::loglevel: info +swh::deploy::worker::vault_cooker::conf_file: "%{hiera('swh::conf_directory')}/vault/cooker.yml" +swh::deploy::worker::vault_cooker::config: storage: "%{alias('swh::remote_service::storage::config')}" - vault_url: "http://orangerie:%{hiera('swh::remote_service::vault::port')}/" + vault: "%{alias('swh::remote_service::vault::config::writable')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.vault.cooking_tasks + task_queues: + - swh.vault.cooking_tasks.SWHCookingTask + - swh.vault.cooking_tasks.SWHBatchCookingTask desktop::printers: MFP_C: uri: lpd://print.paris.inria.fr/MFP_C-pro description: Impression couleur location: Partout ppd: "%{hiera('desktop::printers::ppd_dir')}/MFP_Paris.ppd" ppd_options: ColorType: Color MFP: uri: lpd://print.paris.inria.fr/MFP-pro description: Impression Noir et Blanc location: Partout ppd: "%{hiera('desktop::printers::ppd_dir')}/MFP_Paris.ppd" ppd_options: ColorType: Mono desktop::printers::default: MFP desktop::printers::ppd_dir: /usr/share/ppd/softwareheritage desktop::printers::cups_usernames: ardumont: andumont morane: mgruenpe olasd: ndandrim seirl: apietri zack: zacchiro ssushant: ssushant zookeeper::servers: 1: getty.internal.softwareheritage.org + 11: esnode1.internal.softwareheritage.org + 12: esnode2.internal.softwareheritage.org + 13: esnode3.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 -kafka::version: '1.0.1' -kafka::scala_version: '2.11' +kafka::version: '2.2.0' +kafka::scala_version: '2.12' kafka::mirror_url: https://mirrors.ircam.fr/pub/apache/ +kafka::logdirs: + - /srv/kafka/logdir kafka::broker_config: - broker.id: "%{alias('kafka::broker_id')}" - log.dirs: /srv/kafka/logdir + log.dirs: "%{alias('kafka::logdirs')}" kafka::zookeeper::chroot: '/kafka/softwareheritage' kafka::brokers: getty.internal.softwareheritage.org: id: 1 - + esnode1.internal.softwareheritage.org: + id: 11 + esnode2.internal.softwareheritage.org: + id: 12 + esnode3.internal.softwareheritage.org: + id: 13 + +# Real exported files from munin +stats_export::export_path: "/var/www/stats.export.softwareheritage.org" +stats_export::export_file: "%{hiera('stats_export::export_path')}/history_counters.json" + +# Exposed through the following host's apache venv stats_export::vhost::name: stats.export.softwareheritage.org stats_export::vhost::docroot: "/var/www/%{hiera('stats_export::vhost::name')}" stats_export::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" stats_export::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" stats_export::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" stats_export::vhost::hsts_header: "%{hiera('apache::hsts_header')}" -postgresql::apt_config::pgdg::mirror: 'http://apt.postgresql.org/pub/repos/apt/' -postgresql::apt_config::pgdg::keyid: B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 -postgresql::apt_config::pgdg::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1 - - mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja - UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V - G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4 - bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi - c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC - IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh - hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U - A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3 - RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj - Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2 - AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB - tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQI9BBMBCAAnAhsDBQsJCAcD - BRUKCQgLBRYCAwEAAh4BAheABQJS6RUZBQkOhCctAAoJEH/MfUaszEz4zmQP/2ad - HtuaXL5Xu3C3NGLha/aQb9iSJC8z5vN55HMCpsWlmslCBuEr+qR+oZvPkvwh0Io/ - 8hQl/qN54DMNifRwVL2n2eG52yNERie9BrAMK2kNFZZCH4OxlMN0876BmDuNq2U6 - 7vUtCv+pxT+g9R1LvlPgLCTjS3m+qMqUICJ310BMT2cpYlJx3YqXouFkdWBVurI0 - pGU/+QtydcJALz5eZbzlbYSPWbOm2ZSS2cLrCsVNFDOAbYLtUn955yXB5s4rIscE - vTzBxPgID1iBknnPzdu2tCpk07yJleiupxI1yXstCtvhGCbiAbGFDaKzhgcAxSIX - 0ZPahpaYLdCkcoLlfgD+ar4K8veSK2LazrhO99O0onRG0p7zuXszXphO4E/WdbTO - yDD35qCqYeAX6TaB+2l4kIdVqPgoXT/doWVLUK2NjZtd3JpMWI0OGYDFn2DAvgwP - xqKEoGTOYuoWKssnwLlA/ZMETegak27gFAKfoQlmHjeA/PLC2KRYd6Wg2DSifhn+ - 2MouoE4XFfeekVBQx98rOQ5NLwy/TYlsHXm1n0RW86ETN3chj/PPWjsi80t5oepx - 82azRoVu95LJUkHpPLYyqwfueoVzp2+B2hJU2Rg7w+cJq64TfeJG8hrc93MnSKIb - zTvXfdPtvYdHhhA2LYu4+5mh5ASlAMJXD7zIOZt2iEYEEBEIAAYFAk6XSO4ACgkQ - xa93SlhRC1qmjwCg9U7U+XN7Gc/dhY/eymJqmzUGT/gAn0guvoX75Y+BsZlI6dWn - qaFU6N8HiQIcBBABCAAGBQJOl0kLAAoJEExaa6sS0qeuBfEP/3AnLrcKx+dFKERX - o4NBCGWr+i1CnowupKS3rm2xLbmiB969szG5TxnOIvnjECqPz6skK3HkV3jTZaju - v3sR6M2ItpnrncWuiLnYcCSDp9TEMpCWzTEgtrBlKdVuTNTeRGILeIcvqoZX5w+u - i0eBvvbeRbHEyUsvOEnYjrqoAjqUJj5FUZtR1+V9fnZp8zDgpOSxx0LomnFdKnhj - uyXAQlRCA6/roVNR9ruRjxTR5ubteZ9ubTsVYr2/eMYOjQ46LhAgR+3Alblu/WHB - MR/9F9//RuOa43R5Sjx9TiFCYol+Ozk8XRt3QGweEH51YkSYY3oRbHBb2Fkql6N6 - YFqlLBL7/aiWnNmRDEs/cdpo9HpFsbjOv4RlsSXQfvvfOayHpT5nO1UQFzoyMVpJ - 615zwmQDJT5Qy7uvr2eQYRV9AXt8t/H+xjQsRZCc5YVmeAo91qIzI/tA2gtXik49 - 6yeziZbfUvcZzuzjjxFExss4DSAwMgorvBeIbiz2k2qXukbqcTjB2XqAlZasd6Ll - nLXpQdqDV3McYkP/MvttWh3w+J/woiBcA7yEI5e3YJk97uS6+ssbqLEd0CcdT+qz - +Waw0z/ZIU99Lfh2Qm77OT6vr//Zulw5ovjZVO2boRIcve7S97gQ4KC+G/+QaRS+ - VPZ67j5UMxqtT/Y4+NHcQGgwF/1iiQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYC - AwEAAh4BAheABQJQeSssBQkDwxbfAAoJEH/MfUaszEz4bgkP/0AI0UgDgkNNqplA - IpE/pkwem2jgGpJGKurh2xDu6j2ZL+BPzPhzyCeMHZwTXkkI373TXGQQP8dIa+RD - HAZ3iijw4+ISdKWpziEUJjUk04UMPTlN+dYJt2EHLQDD0VLtX0yQC/wLmVEH/REp - oclbVjZR/+ehwX2IxOIlXmkZJDSycl975FnSUjMAvyzty8P9DN0fIrQ7Ju+BfMOM - TnUkOdp0kRUYez7pxbURJfkM0NxAP1geACI91aISBpFg3zxQs1d3MmUIhJ4wHvYB - uaR7Fx1FkLAxWddre/OCYJBsjucE9uqc04rgKVjN5P/VfqNxyUoB+YZ+8Lk4t03p - RBcD9XzcyOYlFLWXbcWxTn1jJ2QMqRIWi5lzZIOMw5B+OK9LLPX0dAwIFGr9WtuV - J2zp+D4CBEMtn4Byh8EaQsttHeqAkpZoMlrEeNBDz2L7RquPQNmiuom15nb7xU/k - 7PGfqtkpBaaGBV9tJkdp7BdH27dZXx+uT+uHbpMXkRrXliHjWpAw+NGwADh/Pjmq - ExlQSdgAiXy1TTOdzxKH7WrwMFGDK0fddKr8GH3f+Oq4eOoNRa6/UhTCmBPbryCS - IA7EAd0Aae9YaLlOB+eTORg/F1EWLPm34kKSRtae3gfHuY2cdUmoDVnOF8C9hc0P - bL65G4NWPt+fW7lIj+0+kF19s2PviQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYC - AwEAAh4BAheABQJRKm2VBQkINsBBAAoJEH/MfUaszEz4RTEP/1sQHyjHaUiAPaCA - v8jw/3SaWP/g8qLjpY6ROjLnDMvwKwRAoxUwcIv4/TWDOMpwJN+CJIbjXsXNYvf9 - OX+UTOvq4iwi4ADrAAw2xw+Jomc6EsYla+hkN2FzGzhpXfZFfUsuphjY3FKL+4hX - H+R8ucNwIz3yrkfc17MMn8yFNWFzm4omU9/JeeaafwUoLxlULL2zY7H3+QmxCl0u - 6t8VvlszdEFhemLHzVYRY0Ro/ISrR78CnANNsMIy3i11U5uvdeWVCoWV1BXNLzOD - 4+BIDbMB/Do8PQCWiliSGZi8lvmj/sKbumMFQonMQWOfQswTtqTyQ3yhUM1LaxK5 - PYq13rggi3rA8oq8SYb/KNCQL5pzACji4TRVK0kNpvtxJxe84X8+9IB1vhBvF/Ji - /xDd/3VDNPY+k1a47cON0S8Qc8DA3mq4hRfcgvuWy7ZxoMY7AfSJOhleb9+PzRBB - n9agYgMxZg1RUWZazQ5KuoJqbxpwOYVFja/stItNS4xsmi0lh2I4MNlBEDqnFLUx - SvTDc22c3uJlWhzBM/f2jH19uUeqm4jaggob3iJvJmK+Q7Ns3WcfhuWwCnc1+58d - iFAMRUCRBPeFS0qd56QGk1r97B6+3UfLUslCfaaA8IMOFvQSHJwDO87xWGyxeRTY - IIP9up4xwgje9LB7fMxsSkCDTHOk - =s3DI - -----END PGP PUBLIC KEY BLOCK----- - -postgresql::apt_config::pglogical::mirror: 'http://packages.2ndquadrant.com/pglogical/apt/' -postgresql::apt_config::pglogical::keyid: 855AF5C7B897656417FA73D65D941908AA7A6805 -postgresql::apt_config::pglogical::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQENBFUQs34BCADT3W0Nnf0E0/2LGU4Afzg4dAViJqqhOLBwuWUS6Np+ZqCK6beY - E46KS/CowwBmca9stm5oJbcpz2WCNxnb6OGlgxQnYEaVl4z6bDRDJ3BdiIbcjdYM - cutJupmZbrlvfnf30AcQXc6RCZ/x/53CDuCXyrO7djkBkoolJFQbzfy0HcRnNRVp - x3LrsTkTaajDwsenRjq5NJDAnVaMvq+Jmsl0pJoIbiPNIUA2oNKptINlZWIvS7Ha - LYT6sLZxWwuUYhbrpb2EiF8Gx3D4WYf6Wm2h6GLYZkKIagfTsvN/tRPw+5SvIMAd - yVeVWbu77k0i8Om+RQ6wA1FOtZbAVfp8/YxrABEBAAG0P0JEUiBBcHQgU2lnbmlu - ZyBLZXkgZm9yIDJuZFF1YWRyYW50IDxwYWNrYWdlcnNAMm5kcXVhZHJhbnQuY29t - PokBPQQTAQoAJwIbAwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCWNNnkQUJB4Ub - EQAKCRBdlBkIqnpoBR6ACACAaDLPvfknZT7+LrumSIuTM0VqJDPsJykCwWhvjoDb - k2EfLYcqMxVQKEKQtGueSN06DSlfARQmAcoNLw0J5OG6Aw+vv53NmIusvDkTVI28 - TbsdwoWffpY5dcN39KovFsVsIgGROsBTsW1cL0MaeSuwMobaRGorGFKfOwH+xvMc - wjG0wzPoBP85nMulEmqXqPNjRq++NdWnutmc3uGTr8C0dViTlG5F/lYLQ7scsPDi - zBdDg+YPhehX0hG6RUtQsdYMUbzL5Bd51NgmI7gw5Xk61e3N2yDlSNMfdaY+G6tX - jP20hXMNuCRSc1ICNu3hKg/VDxVwYsSBRLJZJ/tlHTtiuQENBFUQs34BCAC8FxMX - v+UBEnLk1qcayhSsECnatkAoDDaKjuUWc7mPBhcwo6C3buFbPtHRKV88uAYy8MtY - 291+zMeO+ZVD6bOVnmXvs/HD6Hn9XIq8DX4LBNu1kKycxQ8BghJCpb5qQgDaxgYI - tWMI66xdaHlPyIhyCZGuv9Dx6T4gR6PUdRLy9kVR1V603B0LHhBbYjSLb6aAmc8w - o7uJJuToN6L/FxKM2asV9L7uacmaq5JhY658LFBexhaTQuih6JYFW3DHxtW527+S - lwXEPtpDrQIvE5hJqrDQX22hF/oiS+gnIE2T22aCIr6+Cwdxys/l104qjcS5YV2T - DujsmHOsvCCL7/QFABEBAAGJASUEGAEKAA8CGwwFAljTZ5oFCQeFGxoACgkQXZQZ - CKp6aAX1+wf/dDQi/3/mv0kGg/5B8NazRIYQ01pmTqW5Czot+3iorssa6fycsLBi - Pd30EQF++26x2hFqxEpvCyBuN1YoU1XlSJJacv80Um8jzdRY10mLGrttcnNFE8TU - xeEXa1rOnoHx45rt5qY8IyOODFkhbHfyBzcxeNtFtvkqT/ZRubj0lzC3Yjvm2xOc - yfIBhvAX93diiaFkg/BKhSkCxognvJnb6KLBwPZ6f6czDvcICvuxWnaT8jfK1610 - YwvZiujRAX8bedODRO+AkokOus7TLPgDx69e9i1SkFjlaaYag/FSLEDsCzKDqim8 - 2lBkSxP5ZrLrfAkji+bmAWtpWifTMgeP0Q== - =8MHy - -----END PGP PUBLIC KEY BLOCK----- - -icinga2::apt_config::mirror: 'http://packages.icinga.com/debian' -icinga2::apt_config::keyid: F51A91A5EE001AA5D77D53C4C6E319C334410682 -icinga2::apt_config::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v2.0.19 (GNU/Linux) - - mQGiBFKHzk4RBACSHMIFTtfw4ZsNKAA03Gf5t7ovsKWnS7kcMYleAidypqhOmkGg - 0petiYsMPYT+MOepCJFGNzwQwJhZrdLUxxMSWay4Xj0ArgpD9vbvU+gj8Tb02l+x - SqNGP8jXMV5UnK4gZsrYGLUPvx47uNNYRIRJAGOPYTvohhnFJiG402dzlwCg4u5I - 1RdFplkp9JM6vNM9VBIAmcED/2jr7UQGsPs8YOiPkskGHLh/zXgO8SvcNAxCLgbp - BjGcF4Iso/A2TAI/2KGJW6kBW/Paf722ltU6s/6mutdXJppgNAz5nfpEt4uZKZyu - oSWf77179B2B/Wl1BsX/Oc3chscAgQb2pD/qPF/VYRJU+hvdQkq1zfi6cVsxyREV - k+IwA/46nXh51CQxE29ayuy1BoIOxezvuXFUXZ8rP6aCh4KaiN9AJoy7pBieCzsq - d7rPEeGIzBjI+yhEu8p92W6KWzL0xduWfYg9I7a2GTk8CaLX2OCLuwnKd7RVDyyZ - yzRjWs0T5U7SRAWspLStYxMdKert9lLyQiRHtLwmlgBPqa0gh7Q+SWNpbmdhIE9w - ZW4gU291cmNlIE1vbml0b3JpbmcgKEJ1aWxkIHNlcnZlcikgPGluZm9AaWNpbmdh - Lm9yZz6IYAQTEQIAIAUCUofOTgIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJ - EMbjGcM0QQaCgSQAnRjXdbsyqziqhmxfAKffNJYuMPwdAKCS/IRCVyQzApFBtIBQ - 1xuoym/4C7kCDQRSh85OEAgAvPwjlURCi8z6+7i60no4n16dNcSzd6AT8Kizpv2r - 9BmNBff/GNYGnHyob/DMtmO2esEuVG8w62rO9m1wzzXzjbtmtU7NZ1Tg+C+reU2I - GNVu3SYtEVK/UTJHAhLcgry9yD99610tYPN2Fx33Efse94mXOreBfCvDsmFGSc7j - GVNCWXpMR3jTYyGj1igYd5ztOzG63D8gPyOucTTl+RWN/G9EoGBv6sWqk5eCd1Fs - JlWyQX4BJn3YsCZx3uj1DWL0dAl2zqcn6m1M4oj1ozW47MqM/efKOcV6VvCs9SL8 - F/NFvZcH4LKzeupCQ5jEONqcTlVlnLlIqId95Z4DI4AV9wADBQf/S6sKA4oH49tD - Yb5xAfUyEp5ben05TzUJbXs0Z7hfRQzy9+vQbWGamWLgg3QRUVPx1e4IT+W5vEm5 - dggNTMEwlLMI7izCPDcD32B5oxNVxlfj428KGllYWCFj+edY+xKTvw/PHnn+drKs - LE65Gwx4BPHm9EqWHIBX6aPzbgbJZZ06f6jWVBi/N7e/5n8lkxXqS23DBKemapyu - S1i56sH7mQSMaRZP/iiOroAJemPNxv1IQkykxw2woWMmTLKLMCD/i+4DxejE50tK - dxaOLTc4HDCsattw/RVJO6fwE414IXHMv330z4HKWJevMQ+CmQGfswvCwgeBP9n8 - PItLjBQAXIhJBBgRAgAJBQJSh85OAhsMAAoJEMbjGcM0QQaCzpAAmwUNoRyySf9p - 5G3/2UD1PMueIwOtAKDVVDXEq5LJPVg4iafNu0SRMwgP0Q== - =icbY - -----END PGP PUBLIC KEY BLOCK----- - icinga2::role: agent icinga2::master::zonename: master icinga2::master::db::username: icinga2 # icinga2::master::db::password in private data icinga2::master::db::database: icinga2 icinga2::icingaweb2::db::username: icingaweb2 # icinga2::icingaweb2::db::password in private data icinga2::icingaweb2::db::database: icingaweb2 icinga2::icingaweb2::protected_customvars: - "*pw*" - "*pass*" - community - http_auth_pair icinga2::icingaweb2::vhost::name: icinga.softwareheritage.org icinga2::icingaweb2::vhost::aliases: - icinga.internal.softwareheritage.org icinga2::icingaweb2::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" icinga2::icingaweb2::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" icinga2::icingaweb2::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" icinga2::icingaweb2::vhost::hsts_header: "%{hiera('apache::hsts_header')}" icinga2::parent_zone: master icinga2::parent_endpoints: pergamon.softwareheritage.org: host: 192.168.100.29 icinga2::network: "%{lookup('internal_network')}" icinga2::features: - checker - mainlog icinga2::service_configuration: load: default: load_wload1: 30 load_wload5: 28 load_wload15: 26 load_cload1: 50 load_cload5: 45 load_cload15: 40 high: load_wload1: 140 load_wload5: 120 load_wload15: 100 load_cload1: 240 load_cload5: 220 load_cload15: 200 icinga2::host::vars: os: Linux cores: "%{::processorcount}" virtual_machine: "%{::is_virtual}" distro: "%{::operatingsystem}" disks: 'disk /': disk_partitions: '/' icinga2::apiusers: root: # password in private data permissions: - '*' systemd_journal::logstash_hosts: - 'logstash.internal.softwareheritage.org:5044' memcached::server::bind: 127.0.0.1 memcached::server::port: 11211 memcached::server::max_memory: '5%' mountpoints: /srv/storage/space: device: uffizi:/srv/storage/space fstype: nfs options: - rw - soft - intr - rsize=8192 - wsize=8192 - noauto - x-systemd.automount - x-systemd.device-timeout=10 /srv/softwareheritage/objects: device: uffizi:/srv/softwareheritage/objects fstype: nfs options: - rw - soft - intr - rsize=8192 - wsize=8192 - noauto - x-systemd.automount - x-systemd.device-timeout=10 ceph::release: luminous ceph::fsid: b3e34018-388e-499b-9579-d1c0d57e8c09 # needs to match the values of $::hostname on the ceph monitors ceph::mon_initial_members: - ceph-mon1 ceph::mon_host: - 192.168.100.170 ceph::keys: admin: secret: "%{hiera('ceph::secrets::admin')}" cap_mds: allow cap_mgr: allow * cap_mon: allow * cap_osd: allow * bootstrap-osd: secret: "%{hiera('ceph::secrets::bootstrap_osd')}" cap_mon: allow profile bootstrap-osd proxmox-rbd: secret: "%{hiera('ceph::secrets::proxmox_rbd')}" cap_mon: profile rbd cap_osd: profile rbd pool=rbd swh-contents: secret: "%{hiera('ceph::secrets::swh_contents')}" cap_mon: allow r cap_osd: allow r pool=swh_contents swh-contents-rw: secret: "%{hiera('ceph::secrets::swh_contents_rw')}" cap_mon: allow r cap_osd: allow rw pool=swh_contents swh-contents-test: secret: "%{hiera('ceph::secrets::swh_contents_test')}" cap_mon: allow r cap_osd: allow r pool=swh_contents_test swh-contents-test-rw: secret: "%{hiera('ceph::secrets::swh_contents_test_rw')}" cap_mon: allow r cap_osd: allow rw pool=swh_contents_test ceph::default_client_keyring: /etc/softwareheritage/ceph-keyring ceph::client_keyrings: '/etc/softwareheritage/ceph-keyring': owner: root group: swhdev mode: '0644' keys: - swh-contents - swh-contents-test swh::deploy::objstorage::ceph::keyring: "%{alias('ceph::default_client_keyring')}" swh::deploy::objstorage::ceph::pool_name: swh_contents swh::deploy::objstorage::ceph::rados_id: swh-contents swh::deploy::objstorage::ceph::config: cls: rados args: pool_name: "%{alias('swh::deploy::objstorage::ceph::pool_name')}" rados_id: "%{alias('swh::deploy::objstorage::ceph::rados_id')}" ceph_config: keyring: "%{alias('swh::deploy::objstorage::ceph::keyring')}" nginx::package_name: nginx-light nginx::accept_mutex: 'off' nginx::names_hash_bucket_size: 128 nginx::names_hash_max_size: 1024 nginx::worker_processes: "%{::processorcount}" -php::version: '7.1' -php::apt_config::mirror: https://packages.sury.org/php/ -php::apt_config::keyid: DF3D585DB8F0EB658690A554AC0E47584A7A714D -php::apt_config::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFQO7UYBEAC7nEcPmukF6D7kDOf1tbIT/aXbiI0yqj4d8PPR9cHEDJrm/IwI - jWGVtlv5k66g2BVymqXSVgDhvJVg+9XH8yEO0dX1Ho8nfO4ftxX0K2LryXZvSJGd - 5QNauYsQAukOTnma6vlLiEEIrKUCuD7D7emvtWTpJLJQFNpMzymbjnjvRrLzUgZL - ez7TqglY1DZ+vhvoO3v6bx5I4p8UEnP4jViaLpr6f7qX+gFBZVW4iv80XaGJXJ0c - smbnUUvWD46GpfLEr/7H0peukljxqy0UC7zII4I4vnJMlRSMxmwb/n+HMhiDCPF7 - NUaoRFs51hQVa+2Q9xb7usKw0kUMwOBKHTfcqp7qMosKhiVGTApmo1GGpaqpqfCy - gs9I86wLmtT0k73H5FY2+n2uwEgJ8+c7j83gVeDeFRosLF+8HAwTwfpf4SatWKXl - QudTGvK5ppTFwIx62rqWX2MdJ07xmAs4Mkxntk0gU2heDDGSy6j8UcJ42yQoo/Vu - C3i1l8+VyfniTpNdoRzLbd0MFu+kClUt1aMiRNrw6Z1Sx0hhVug3ZjTCPOQm0UOK - yWrorqZC88hnc8XU9P35VOhdQEC0pdSdWPRYhodb3oAb3D6qmTAYP7GIAsupJ4dk - eGD58jlai8j8ScoG95d0o5UzKxXOCiKKyV97tsL27J2ayjjwXO8HSRSCSQARAQAB - tEFDWi5OSUMgTGFicyBBcmNoaXZlIEF1dG9tYXRpYyBTaWduaW5nIEtleSA8ZnRw - bWFzdGVyQGxhYnMubmljLmN6PokCNwQTAQoAIQUCVA7tRgIbAwULCQgHAwUVCgkI - CwUWAwIBAAIeAQIXgAAKCRCsDkdYSnpxTQSLD/9l1suU3IrnM6czffvit9WEjtHi - 0DgLQwVSqFmz2IQgiKEG3YnGF+2Xn7SHq/uPRgpwBJuU/CkufioZg2JFOWIIO/Sr - o1DYWB0y9BHv9Ilif17AzZfjHedKdxOlu0Wn/Xjs2T5tWhAkjnMpoARm0ah5cgQX - QDzO3t8QIXydGG+kdennR7FUgSxBKJ4l1Ubd01hVpYP9fkTpYNyKkb5IY/CY8Xqi - DJuh3tYg+8u0v2oT5Jhs7/6VvZiJqxWsDWYZB+v9aB8fuhicK3+FPVmU6Cb09J6g - XFQoaMGfe4hSBaXIRRZfMlb9zeJzia9ViIDoCVrtwt69xyF2keKKph2Gs2pvcKjJ - HhKqB9sBebzwewONGellcYBoQ/66pLhl/vT4MsVob3cZPfFbwQfkqDHZEP9ZIrp+ - CEmsongKVO66XqtUKJYPaAVdH4S8POF8tYG/8ak5OBqyozstR6keSFmpIAEFBKD9 - nf4VRmxWkJUE7Pkyg1IPUF3HQrzNTPAAbzzuRPJq1T1TQY8QvWnqFYX7Y2mNnfl4 - aRD2kw/XT4Q55lIPGjhSO9fS+kPKh1qx/muaKUtPBK8HOFhBrzdojmid7+0MBur9 - S0uUNghDYPt7YEoFhlZIb31U1zpLYL6Dn2yxFYuq++PTGxOPMECh3cbgfwSbNfbn - wbgwIefP50Qc07X5ELkCDQRUDu1GARAAxs2Zfd8zgrBrUSjKARxhvzvuA+yAmnqq - Vysv7aAJ5n/Zx7VVuljN11+6kk9QmWAyDWiUybrSKbdFKyTQv/EyJzH4VwWkygA1 - +eToKvfQqBqs4Orrek8rw8Ty5DaT5qogoDsh9Am4Za0hc9urmkI0IMcxBpe7MWsv - Jo2DnuQn0tbrPWIlvTDTOjOZ5RLoB7btJ03u7fJB8UqH8uTH55ySI1B7UFtxth5D - key1sakvFWA+WKAewwzGIFb9QgZb2UFw9VBcgXifdNvyfZzmBfpw5tjMWw2dauGX - bddh9/gL4Q8J3Rklk/j/E9efwyt0f/6cuPDmYBElWhPHT31WelNTRQeS3TyJVutG - yZkZ0b7lC4lY1tHpy56CYFgdEyjF+ePYaYSMxY5HkLojFIPrvpHGRY4b4SbO6zJH - yvqj7JWnBEJqsoWt0/8sXImVUF8UvjNRSlQqn815myjtgLJkRbTf7bnzFu7OGDTT - TvJql69Y8go30/Wf2oEdXR1P/P0B3saU5V5Au1A1Y7Cy+d5v3orlE9yrROFHDFlC - TuFAXnIHjw5fivqjO0Ls2KdpDoE79HyVfJr/G6hXBjdRHau+6L/F6+3VJ195AUeN - KW/UOLdLfz+F8whQB+bpRUtJ2M/au0FJRJijMy6kfhP/0q6zhE2cC3p7sUMu7i1e - FV7Lhxy3/kEAEQEAAYkCHwQYAQoACQUCVA7tRgIbDAAKCRCsDkdYSnpxTbJPEACM - iN/xxw8UZuwRKIL9ZARNgJSMTPwprU/ul/54ATvwbCO+HIn/JQefh3IEal1BZYOB - ntoRkvwxygbbJZFJ8A2iBQsC78Cx0mcczCfsVy+/aenE/g+DAinMmTAselbK+sl4 - Er1Vymhu+w3ZTOi2dx58MR+kYM1or1Rffm+1jBNCIPZokJGQvFr8E5HqxaQE6wXp - SqCHtCshiC2GBGpLhZ9FS2vWVhshr5TA4KIS1Rke22bYDu44br2mgdRMESDFwLhg - mFtSFoVJxY3ugEZmV491cVcWPHEMkeQJGDx3xKVacEd9n+4L6aTPEMq2rI35oRxU - a1FQZLX8kUJGAZO0MbZO6PpSk6rLeu+sSE7kGYyNN3g93rdgayx0hTxiYlyOGOJX - 85jGcpCpvp+qedaVSavQbHTJwXoGYOCL+WNwH6Xrr9tHJho30Qd+9qjyeNhGwekZ - FJy14s8L7m5nvSxRIB8QJ5Csaj6s9wK7iesywZfaYyqDDRAkSE44dQUuVQViAoMG - 9qluxcR71yID59xHfLv9ho8cLg0QI2JfvENGouSnPsgJGYa/AYzFv+H5vIR1m0n6 - NoWefRc6e9w/Q6IvWG5se3rh1sgJTajKoSca0EGQX+EbL5wLdFmO82gptQwCAjMd - IwsvfZGLfuzTOahvuEMccjo5L7K4HpCDHuFPjfE0rA== - =0VaR - -----END PGP PUBLIC KEY BLOCK----- - prometheus::server::defaults_config: web: enable_admin_api: true storage: tsdb: retention: '1y' prometheus::server::config::global: scrape_interval: 1m scrape_timeout: 45s prometheus::server::listen_network: "%{lookup('internal_network')}" prometheus::server::listen_port: 9090 prometheus::server::certname: pergamon.softwareheritage.org prometheus::node::listen_network: "%{lookup('internal_network')}" prometheus::node::listen_port: 9100 +prometheus::node::textfile_directory: /var/lib/prometheus/node-exporter prometheus::node::defaults_config: collector: diskstats: ignored_devices: "^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d+$" filesystem: ignored_mount_points: "^/(sys|proc|dev|run|srv/softwareheritage/objects)($|/)" systemd: true logind: true loadavg: true ntp: true netstat: true netdev: ignored_devices: "^lo$" textfile: - directory: /var/lib/prometheus/node-exporter + directory: "%{lookup('prometheus::node::textfile_directory')}" + +prometheus::node::scripts::directory: /var/lib/prometheus/node-exporter-scripts +prometheus::node::scripts: + puppet-classes: + mode: cron + cron: + user: root + specification: + minute: fqdn_rand + apt: + mode: cron + cron: + user: root + specification: + minute: fqdn_rand + +prometheus::statsd::listen_network: "%{lookup('internal_network')}" +prometheus::statsd::listen_port: 9102 +prometheus::statsd::defaults_config: {} +prometheus::statsd::statsd_listen_tcp: 127.0.0.1:8125 +prometheus::statsd::statsd_listen_udp: 127.0.0.1:8125 +prometheus::statsd::mapping: + defaults: + timer_type: histogram + buckets: + - .005 + - .01 + - .025 + - .05 + - .1 + - .25 + - .5 + - .75 + - 1 + - 2 + - 5 + - 10 + - 15 + - 30 + - 45 + - 60 + - 120 + - 300 + - 600 + - 900 + - 1800 + - 2700 + - 3600 + - 7200 prometheus::sql::listen_network: "%{lookup('internal_network')}" prometheus::sql::listen_port: 9237 +prometheus::sql::config_snippets: + - activity + - queries + - replication + - wal + +prometheus::jmx::version: 0.11.0 -# extra sql exporter configuration per host -# (cf. somerset.internal.softwareheritage.org.yml) -# prometheus::sql::exporter::extra_config: [] +prometheus::kafka::listen_network: "%{lookup('internal_network')}" +prometheus::kafka::listen_port: 7071 grafana::db::database: grafana grafana::db::username: grafana # grafana::db::password in private-data grafana::backend::port: 3000 grafana::vhost::name: grafana.softwareheritage.org grafana::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" grafana::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" grafana::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" grafana::vhost::hsts_header: "%{hiera('apache::hsts_header')}" grafana::config: app_mode: production server: root_url: "https://%{lookup('grafana::vhost::name')}/" http_port: "%{alias('grafana::backend::port')}" users: allow_sign_up: false auth.anonymous: enabled: true org_name: Software Heritage org_role: Viewer + smtp: + enabled: true + skip_verify: true + from_address: grafana@softwareheritage.org grafana::objects::organizations: - name: Software Heritage id: 1 grafana::objects::users: [] grafana::objects::datasources: - name: Prometheus (Pergamon) url: "http://pergamon.internal.softwareheritage.org:%{hiera('prometheus::server::listen_port')}" type: prometheus organization: 1 access_mode: proxy is_default: true +java::distribution: jre + jenkins::backend::url: http://thyssen.internal.softwareheritage.org:8080/ jenkins::vhost::name: jenkins.softwareheritage.org jenkins::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" jenkins::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" jenkins::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" jenkins::vhost::hsts_header: "%{hiera('apache::hsts_header')}" -jenkins::apt_config::mirror: https://pkg.jenkins.io/debian-stable -jenkins::apt_config::keyid: 150FDE3F7787E7D11EF4E12A9B7D32F2D50582E6 -jenkins::apt_config::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1 - - mQGiBEmFQG0RBACXScOxb6BTV6rQE/tcJopAEWsdvmE0jNIRWjDDzB7HovX6Anrq - n7+Vq4spAReSFbBVaYiiOx2cGDymj2dyx2i9NAI/9/cQXJOU+RPdDzHVlO1Edksp - 5rKn0cGPWY5sLxRf8s/tO5oyKgwCVgTaB5a8gBHaoGms3nNC4YYf+lqlpwCgjbti - 3u1iMIx6Rs+dG0+xw1oi5FUD/2tLJMx7vCUQHhPRupeYFPoD8vWpcbGb5nHfHi4U - 8/x4qZspAIwvXtGw0UBHildGpqe9onp22Syadn/7JgMWhHoFw5Ke/rTMlxREL7pa - TiXuagD2G84tjJ66oJP1FigslJzrnG61y85V7THL61OFqDg6IOP4onbsdqHby4VD - zZj9A/9uQxIn5250AGLNpARStAcNPJNJbHOQuv0iF3vnG8uO7/oscB0TYb8/juxr - hs9GdSN0U0BxENR+8KWy5lttpqLMKlKRknQYy34UstQiyFgAQ9Epncu9uIbVDgWt - y7utnqXN033EyYkcWx5EhLAgHkC7wSzeSWABV3JSXN7CeeOif7QiS29oc3VrZSBL - YXdhZ3VjaGkgPGtrQGtvaHN1a2Uub3JnPohjBBMRAgAjAhsDBgsJCAcDAgQVAggD - BBYCAwECHgECF4AFAko/7vYCGQEACgkQm30y8tUFguabhgCgi54IQR4rpJZ/uUHe - ZB879zUWTQwAniQDBO+Zly7Fsvm0Mcvqvl02UzxCiGAEExECACAFAkmFQG0CGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCbfTLy1QWC5qtXAJ9hPRisOhkexWXJ - nXQMl9cOTvm4LgCdGint1TONoZ2I4JtOiFzOmeP3ju3RzcvNyQEQAAEBAAAAAAAA - AAAAAAAA/9j/4AAQSkZJRgABAQEAYABgAAD/4QBgRXhpZgAASUkqAAgAAAAEADEB - AgAZAAAAPgAAABBRAQABAAAAAUOQABFRBAABAAAAEgsAABJRBAABAAAAEgsAAAAA - AABNYWNyb21lZGlhIEZpcmV3b3JrcyA0LjAAAP/bAEMACAYGBwYFCAcHBwkJCAoM - FA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0 - Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy - MjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAK4AlgMBIgACEQEDEQH/xAAfAAAB - BQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0B - AgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygp - KjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImK - kpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj - 5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJ - Cgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGh - scEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZ - WmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1 - tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEA - AhEDEQA/APcBI/8Afb86XzH/AL7fnUYpwqRknmN/fP50u9v7x/OmCgUASb2/vH86 - Xe394/nTBS0AP3t/eP50u4+p/OmUopgO3H1NO3H1NR5xThQA7cfWlyfU0ylFMQ/J - 9aXPvTKdQAuaM0lLQAtJmiigAzRSdqKAKApwpopc1mUOpRSUopgKKWkFLQAueKzr - zXbCwk2Tzxq3cFwK8v8Aih8V30aaTQ9DKtegYnuTyIvZR3b+VfP1/q17fzvLc3Ms - sjHJZ2JJNGr2HZdT6j8U/FbR/DcKsM3VxLkpGh6AetcI37Ql4Zcx6LAYx2aUgmvD - 1ju7obgJHA7nmmmG4TqjDHtS+ZXL1sfVPhT4yeH/ABFNHaXYbS71zhVnYGNz6B+n - 4HFejK2RmvhJJSDiTj6ivYvht8XptE8rSPEEklxpxwkFyTue39j6p+op3a3Javsf - RuacDVaC4juIUmhkWSKRQyspyGB7ipgasgfmlpoNLmgBaKSigBaKM0UAUBS0lKKz - KFFLSUooAdWR4o1qLw/4bvtSmZVEMRK57t2H51rCvJPj7etD4WsbQMQJ7jkDuFBN - D2GlqfP13dS3k89zM5eaZy7sTySTWvovhw3JWWdcqeQtUNGsWvtQRMfIvJr0u0t1 - hjUKOnpXFi8Q6a5Y7npYLDqfvyILXQolRVWMdOwp1x4cjYH5QPwrftQcDippFavM - UpvW569ktLHnOp+FFaNiijcOlcfcW8tlN5UgI+tezXEeSeM5rmtf0OK/tSVUCVOV - Irsw+KlF8s9jhxWFjNc0dzpfgh49MV1/wimozExyndYOx+6/eP6HqPcEd697Vq+I - baWbTb+G5hJWe3lWVCDj5lOf6V9naTqUeraRZ6jEMR3UKTKM9NwzivXj2PDmrM1A - 1PqBTUoNMlDqKSloAKKOpopAUacKbS1mWOFKKbS0xC14p+0Gw+z6Ihb+ORsfgK9r - rxT9oO3X7Ho1zn5vMePHrxn+lJjW55t4QgZbOe7CbmJ2IPU10sltriIDaSW7ORlg - 44HsKz/BCbtFyBysjVdvo9bcTNDMyEFfKCEDdzzknpx04NeVUles9vme3Rjairdu - hoaXqOqwt5Wo2cSjoHRuv4VuTXKCAuBzjoa5myW9SKJLmVpH25lLEEBs9sVuTgGw - BGN3f3rOU7SaOqEW43Me7l1a8l225SCL+9tyajfT7lHS4SdmkH+sVujj+lQakuo3 - ELC0uGjkBwqh9qlceuM5z/L3q1p9nfwyqzzs8WxQVkOTuxycjsT2q7+7e6MXH3mr - M898QWgtNbmVeEcbwK+l/hdK7/DXQjI+4iAgH0AY4FfO/jWMx6+oxx5QP619B/Cx - Wj+G2i7twzExww7bzj8K9bDO8UeJitJv1O5U1Mp4qshqdTW7RzpklLmmg0tSULmi - kopAU6WkFFZlDqWm0tMQteX/ABe8MXPiBLCSN1SODcq5H8bY5+mB+teoVi+KbQ3e - gXAU4dPnB9MVFS/I+Xc0pNKa5tjw/wAJ2L6fpbWsw2zRzOsg9wa6RIlk6Diszy5L - a5kYksJTuyfWrUN2xbArxpyUpczPoKS5VyiXKQwHoBk/mamID2AIFZ89w6SlvKSV - ugDNjFK2p3It/L8uIAc//WpRhd3RtKaSs2WLNIpQeAcGrjosYIFZVvcPLIr7Fibo - Qpzmp5rp/N24prTQmT0uYOv6LDrWt2avIIkSJjI3qMjAHuTmveNEsU0rRbGwjPyW - 0CRr9AK8k0y0S81yMMAzllQL3xnnAr2cdfavXwLbT8jwcwsmrbssoamU8VXQ1Otd - jOBEoNOBqMGnA1BY6ikHNFAypS0lLWRQtFFApgLTJoknheKQZRwVYe1OopiPO/GP - hq202xgu7RX+VishZs9a4pmaMtsGSRkAV7Xq9gupaXPasPvr8v17V4jKHt7qS3k4 - kjYqa8vF0lCSaWh6uDrOSab1KAuLia9a2CJCQu7zLhgoI9q2f+Ecv2h877XZbTuB - Ikz0x/jVK4RZVAdckDg1QfEY8kW6EeoYgH6jOKwi0z0emkrfK5LcyXNpex2YEVyz - ruEkD5Cj1NX1Lbt0hyVHP1qpbxiFCyqN5HYYAq/pcH2/WbSyLcSyAMfbqaduaSij - KpJRTdz03w3p0dpo1m7RL57JvLFRuG7nr16YrdWolAHAGB2qVa+hjFRioo+YlJyk - 5MnSp1NQpUopMESCnA+tMFOBqS0Oz6UUlFIZWopKXNZFi0UlFMQuaM0maM0wOU8Z - /ELRfA8UQ1Ayz3kw3RWkABcrnG4k8KPr17CvIbjWR4lSXXbW2Nv5srHyS+4gA9Cc - DNYfxfl+1fEbVCsm8xFI+T0wo4/CrHg9kt9OFm88TyffwrA43DOPw71y4xfuk13O - zBfxGn2NWDU4ZFXLbXHDKamN7a7cfLn3qCWyt2nKyxAj3FLJo9hFGH8sNu5HJrzo - 2PTbkupHPqcafLHlnPCqKu6VqMfhy4h1nUEkdIDvdIwC2MYwM455rMW502wlzLLD - Cq+p5P4dax9e8S2N5aSWtuXcOMFsYH61vSpzlNOKMKs4qLUme6+EvHWk+MRcLp6X - EUtuAzxzqAcHjIwTmuqQ185/CTXo9J8XRW0iqsF+v2bcxxtbOVOfcjH419EqcHBr - 3FqeDJWZbQ1KDVeNqmBqWCJRTs1GDTgakseKKQc0UgK1LTaq6lqljo9g99qV3Fa2 - qfellbAz6DuT7DmsjQuU15FiiaWR1SNBlndgFUe5PSvGfEfx02s8HhzTwR0F3eDr - 7rGP/Zj+FeU674u1zxE5bVtUuLlc5ETNiNfogwo/KrUWFj37xF8YfC+hiSK1mfVb - tePLtf8AVg+8h4/LNeSa/wDGHxRrcjpb3Q0u3OcRWZ2nHu5+Y/p9K89Z9x5ppOM8 - 1SihXHTTyO7NIzO7MWZmOSxPUk+tQrKyNuUkEdwcGnFs8EVGV9Kom5YGoXqtuW7n - B9fMNPOrag67Wvbgr6eYap4OelA5qeSPYrnl3Jg7McsxJ9SakTrzUCg+1SgqgyTm - rJLkbjII6e9dfp/xR8VaciLFqjTxxAKI7pFkBHuTz+tcL5xI9AeAKcpGSSe1Az37 - wx8adPv3S3122FjKeBPES8R+o6r+tepWl7b3tulxazxTwvyskbBlP4ivjASAnA4r - Z0DxVrHh2787TL+WDP3kzlG+qng0XFyo+wlfIp4NeN+FfjbaXs0dp4gt1tGPH2uH - Jjz/ALS9R9RmvWra6huoEnt5o5oXGUkjYMrD2IpE2aLgoqMOMUUWC5ka/rVv4e0K - 71W5G6O3QsEBwXboFH1OK+WPE3irVfE2pNeapcM7ZPlxA4jhX+6i9h+p71698dNZ - +z6Np+ko3zXMpmkH+yvA/U/pXgcz7k9x/KogtDR6DXmJ71EXOKYTzSE5qybi7uaU - mmd6UcimITPNKDmmnrQKAJM8Ck3egpuaQUAPBJ6k4ozknjimk9qB0oGO3E04NUYp - aQEu/wBqXOFAPeohyQKV25NMCdJDng103hjxnq/hm7WTTrp1jJy8LHMb/Vf8muU+ - 6g9TThIUGB1Pf0osNM+wPCnie18U6HHqNspjbOyaInJjcdR7jnINFeY/APUUJ1jS - pZVQER3K7jjn7rf+y0U1YiWj0OW+NmoG68dvbhsrawIgHoTyf515qzbth9eDXQ+P - NQOo+NNUus5DzED6Dj+lc0DnI9DmohsXLcaTQOaG6n60CqJEpVpM0A80ADDmkpzd - RSUALRRRQACiijvQAtFJRmgY9B3po5b605DhGNN70CHu2CT+ApEwX5+ppG5AP1pM - 4GB1PWmBraZez2rvJBM8TMMEocHFFVLViFOKKm1y0xb9zNI0pJLFiT+PNUlPz5NW - Jm+/9RVYjGPenYlisMufrSE05vu5qOgQtA60dqB1oAe3QU2nN0plAC0tJSjrQAlL - miigAptL0pO9AEi8RfU0mM8560H/AFaikzx+NMBxx0H40zOeaU8KffikHSgCxC+y - LPqaKYeAq+gooHc//9mIYAQTEQIAIAUCSj/3IAIbAwYLCQgHAwIEFQIIAwQWAgMB - Ah4BAheAAAoJEJt9MvLVBYLmt2sAnRUJQoS4J/5+LW+Iy3tUYMTsR8aLAJ9gp9qD - YbGfdcFG+HeSbh/PEwrqbLQzS29oc3VrZSBLYXdhZ3VjaGkgPGtvaHN1a2Uua2F3 - YWd1Y2hpQGNsb3VkYmVlcy5jb20+iGIEExECACIFAk0GnroCGwMGCwkIBwMCBhUI - AgkKCwQWAgMBAh4BAheAAAoJEJt9MvLVBYLmfugAnRb1qac6CqRaNUhHbzd1m/5S - niNzAJ9NJUC2Fjk7uEyvQ5bDJ+hAFbkQVLQpS29oc3VrZSBLYXdhZ3VjaGkgPGtv - aHN1a2VAY2xvdWRiZWVzLmNvbT6IYgQTEQIAIgUCVh045AIbAwYLCQgHAwIGFQgC - CQoLBBYCAwECHgECF4AACgkQm30y8tUFguZVLgCdElQ2ydLBp33/9SFyVEz3cFMk - 0DkAn2qWsQlPT549lAqeSnkhCOcGJAx0tCxLb2hzdWtlIEthd2FndWNoaSA8a2th - d2FndWNoaUBjbG91ZGJlZXMuY29tPohiBBMRAgAiBQJWHTjzAhsDBgsJCAcDAgYV - CAIJCgsEFgIDAQIeAQIXgAAKCRCbfTLy1QWC5sMTAKCA5kH0uH0x0HoTuxjrU740 - pU/53gCfaFWE6s7nBFMkJ3RyxjtZBGnY2Jm5Ag0ESYVAbRAIAOoBdaCKKzjKL3qi - zdBmYrnzT2iONNOeUgKBvO2tPnlwxVMMFz1Kd7JFCULRxL4zXPgOjqWPzWw0l0mI - E+pNhgDX57FMW+znMLE8icM/eG+pfEdM/XjZc3WF3O3ndHuyafw7TDI75EIFRvjh - 702S6y8F3lQ/cl7jj2GelcnhY7dxUwWbiCHGzsRGWkCLk1MSxVV0zx2odtkm2TyB - vN0AcfTJuIBeZbIsUZkO64qIUCSqb9aV53uJ3o35w/HXTt3AFyXA/HN8RgoSonVg - MMegOXJ/HjTXbLXnd7mwbJqH8g8Fiussx8b5aaLCvmcJfS2bA5zK6S4T3iFvMkJf - bAF1tYsAAwYIALOXdy4ziUa3/CvmWIziCi1elkCilj4SdssgG44cVddHsefICBJP - WMf8BRtp+8+PIOESQUPJQ/Xhe0c0gCqw3VSm7Jhsz3Rsw8BZcnGtrMyxIX5O/nIj - EeLLhxzWmOiocDaTCogYeZPFjM485LX1lZAC16+hMTqkIBGmFjR3OmxwJZpcaz9m - o0CGMv3pYthXU6hS372ZOc5yzpW7FrGnbA3ZLkMrVL2B0jFYRzzAxQ+JB7wJiTQ7 - JJ05EhuUyzdsaoMWgzkdwEBk/ViVeK08fachG/QO05AYxA4KSpRaZC5ABSApX5g7 - zqU7hLsSFMRP8Y+xBvo/t5+b8KzzBur/DIiISQQYEQIACQUCSYVAbQIbDAAKCRCb - fTLy1QWC5raYAJ4k0FbiycMLg7OMpTpBPfzr8YD2ywCfe8vNLCfw3XG/kyKFYavm - RXO9oTa5Ag0EWBjgRgEQALze0WQartDG4x1DaOpqKLAol9pfxSX+O88Nafw9dDdV - v80CD7Q66p6X5o1TOOqEAqsI/dUFzDoZzW/EBN5TVKdNhV55WsIbvFJnJ9ccQ1yk - fCYVQAH/eCIdM8dujAOZLjKSapz/wBdFbbOffvz7GLmsjn1wCruZfIOcaIcfaUfY - QWsafzwU9VsRLSDrbwpylQJkvblfeb+ohQ/AYlVJmD1HcKF81AajgxbTUDCBxslY - 4kL6FmqqfLJDWXyg0aG7UEbP3ye7/61qrsKR0g84BHYgkLzQkdgsAGAMo3HvQzss - BAqhZy2QSWKZCe6OQuIEzL01oTWJOWJYAoak9pSkjuFDsRbFRHC4YiaCIvwFHA8C - 3nCaa/jAXQ/NrBFyc1TsrDdxiXi6cEgER9WichpQaD/NCKGGHbEzzHow1Ni+pABq - 1leoVAfAEw8OwRYEftfoAQ5O8VdWe754xK2I5wFWjGKM0IHruEqnRgbWXL9Vy6Cv - NTrQIoJbVuO/kQWH4jZ63TzsBnxHzdnRSuCNGXnuneIju8+wr33y+r914cNziCHm - Tt0UsyTcf7xfzVB++obS0sCyklDIy+1EEzLePkUYl7Ebkst5tKgbVRNyH1niKRwX - xoyowmIRznO79l46u9JMdlt9VO9oo+yR9DqMgNqUnc9Z+rt8EyUam87838FfF+OF - ABEBAAGJAmgEGBECAAkFAlgY4EYCGwICKQkQm30y8tUFgubBXSAEGQECAAYFAlgY - 4EYACgkQlHo/RMJzQlXPTg//UpZd7vx0wNm6dPSUc9Agw5tQU5oCR4BUaDOBFDfb - nKPNa8JQPVdH6lrt1Zaqc9Uka+l1eVK8SZiujohr3bCyal+5ParAdVbTt08pvh5d - 3YllLIKKad82Qy6WsUlAQmUpba+Fn5naXdd8WDN03J7LVOqYCQUWZu65r5oqmv8B - eh+vcZO5ozEt/Huy+ruCsdb0WavbgI5+Pj6sKJtKBo5WwZzbDpbPUEUd3/T5zFbJ - G/XDk77qfBP4DKC96tphzGp6EaEtrZ9Qto8AisCYGvhDptYqXqZm4J1mJj/SI+4C - /1kVY0EEf4ySLy4/8f91h/jzcEliQNnmNZWgUTmP/nyUS+iLqUa4NmhdO45NYBfJ - PZyviHsFxJhYppiPt32n5FpGrXM8fWaQsA+aKOL2D+AWeC8W/pPmDurLbYA1yRk7 - T7E1llz4wDf53CumQGtT4gKwmUdGbwp0TNZKggv+/6auOMoBVjvWCRM0erxR+fAL - FKruuoXjQ69I2bTiZfoSHtDxqa+YMnNqqFOZdyJsH13Fx/Ma3k0EVI4uOuX5RoJ8 - BN3SAkBSiZu/yRf9XF/ikKvrb3YcaPaUgRPVP3EweJJx98whWxPmgSbv/GvQCQa7 - GyvwvqvWuiw+kgl4RlCGvL354zQwSoD+li+ZgnuhzRlSnj962O2cobvY+UzW1fiO - vTrGzQCgg7/WrciTjK8wtd8e/E26mU1agOMAniYHo/aFmpsSFfNp4n419EI+mCXU - =fBn8 - -----END PGP PUBLIC KEY BLOCK----- - -docker::apt_config::mirror: https://download.docker.com/linux/debian -docker::apt_config::keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 -docker::apt_config::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - - mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth - lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh - 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq - L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 - UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N - cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht - ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo - vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD - G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ - XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj - q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB - tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 - BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO - v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd - tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk - jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m - 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P - XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc - FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 - g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm - ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh - 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 - G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW - FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB - EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF - M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx - Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu - w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk - z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 - eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb - VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa - 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X - zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ - pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 - ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ - BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY - 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp - YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI - mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES - KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 - JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ - cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 - 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 - U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z - VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f - irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk - SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz - QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W - 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw - 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe - dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y - Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR - H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh - /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ - M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S - xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O - jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG - YT90qFF93M3v01BbxP+EIY2/9tiIPbrd - =0YYh - -----END PGP PUBLIC KEY BLOCK----- - -elastic::apt_config::keyid: 46095ACC8548582C1A2699A9D27D666CD88E42B4 -elastic::apt_config::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v2.0.14 (GNU/Linux) - - mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD - A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9 - CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ - j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd - 1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD - 2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg - KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy - Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC - F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75 - nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/ - 7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm - TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe - 8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/ - eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl - zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT - RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+ - 1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+ - Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt - KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww - EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0 - c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J - TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j - 6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7 - vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM - cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/ - qPDlGRlOgVTd9xUfHFkzB52c70E= - =92oX - -----END PGP PUBLIC KEY BLOCK----- - -hwraid_levert::apt_config::keyid: 0073C11919A641464163F7116005210E23B3D3B4 -hwraid_levert::apt_config::key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.12 (GNU/Linux) - - mQENBFHwGLoBCADGXHFostxbz4UzGFYtmox4pvyN1gMhq2KCuQ6f+FESa4HTd9L6 - XVhXWPCad3cdxBIls+41+AdZTWxWMu7DUdy8nMU1Ikfw6JeHcSx97G5BdxBVMjK4 - iMGfPdLfDgWf4BQ2h0dnTEWobt31WaqgNiNjNrKktqbymmF94pwYkwL53ydIA4zl - 8ZQRZooFigkS9WdoKjh30Pv/SWakILSLcSQFHK0dvSkeGd1NxT9dMNPAXXqLom4+ - 7kCc0s04sS+0DwW16b0Hpb46mtsR9kzOnrE/Smj24uOGzNZen0oCc2Y7bfZlyaN+ - RlTkWEze7lemc4Byup/QWkhT0Er8F8uxexy5ABEBAAG0PEhXUmFpZCAoaHR0cDov - L2h3cmFpZC5sZS12ZXJ0Lm5ldCkgPHJvb3RAaHdyYWlkLmxlLXZlcnQubmV0PokB - OAQTAQIAIgUCUfAYugIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQYAUh - DiOz07Rc4Af+N3dEZZHzLNVTjQ0+fCyeg8/flWOkR8DhP10cyoJhSHFTZRdXVshn - kP4VmmUycVeURh76DmrIRe/9Oyca6aGXccRMqvq+HMgBPVwD5qNhcJPIuzqEvmlO - 6UIeW2ydil/v1pWu740fGntyFRQcsfqjReVPXw9K588F7MDMyL+31vLm6aorLSzR - hvLhOmGisTs0wg2Oz9f4muauRy6cpQPw/Zi/P/F4WkQYscbHrSbhszj6OIg/vftR - UbZ7QB26/+40B0ag4JzLpmj3scFxf/WdUl5LXazqhsbkurk7huV41BNKXi1+BS3c - x6pFzWEHpiuG1j7U/nScGzEQpsMlUW9D+rkBDQRR8Bi6AQgAuhH1H0VLwcROI/5n - 9yTxSbTIZbyhUan3raAbit3pgo0zLagfUtp3vULVnm5ISqQcYFGLZoE1MUkmjGOL - 38W0lsIiZTaKOKXxBbLlPhhrvlXnNWAG/S1wnq7K+DV179KCTkUzaLRDbHvv999j - 9odBRtAkiTnCfHTMCN4AhydEejNxtlzJo4E5FecH4reimLI5euUdTltgCjixrbsa - KbQftYpSMdXnLy2+00QZoXu0U/h4WZcMhOSEEiyGP9BY6m5G76n03HIeQ6eALDFu - ryAgO+SB9rBrm/VN0kR/TZq0iA3uzLHC7zCw2aImipkr+rIuJOku0wH9MyowBbia - bQtnCQARAQABiQEfBBgBAgAJBQJR8Bi6AhsMAAoJEGAFIQ4js9O0d5YH/3fNQgsC - LvD0g2wdoksv5bG9CUOi9Bs0JHqI0LhXmPvMsbDojZ+zZle7KWNfK2227mWhmoG1 - WLujJSmTtxhEO1fXIdYjlDfk2uLJKuFi2wQX9n8dFDUmKY3CUJgeVZof1uQ/5C3D - O06CcuOtf2d/+iijuW112aV1q1hoQqw71ojTET0iIV6lD/0i1eEBSSe1Ohb9yTGR - VxTVrB78zU9hih4/Oq8wJT/Fv25aO1MDSc26CXAg0JA6IWvKal3BSPNhtz4L4FIg - lXleArf9oJqxDO3TsV5zcLyxsIuRuxyP0+AKdSQUqv0dFi4Jf79OmvOmgwydhHjY - +f7quLbwiiDmPbU= - =Yv6D - -----END PGP PUBLIC KEY BLOCK----- +jenkins::agent::jar_url: "https://%{hiera('jenkins::vhost::name')}/jnlpJars/agent.jar" +jenkins::agent::jnlp::url: "%{hiera('jenkins::backend::url')}computer/%{::swh_hostname.internal_fqdn}/slave-agent.jnlp" +# jenkins::agent::jnlp::token in private_data + +weekly_report_bot::user: nobody +weekly_report_bot::cron: + minute: 0 + hour: 12 + weekday: fri + +swh::postgres::service::users: + - root + - zack + - ardumont + +swh::postgres::service::dbs: + - alias: swh + name: "%{hiera('swh::deploy::storage::db::dbname')}" + host: "%{hiera('swh::deploy::storage::db::host')}" + user: "%{hiera('swh::deploy::storage::db::user')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::db::storage::password')}" + - alias: swh-deposit + name: "%{hiera('swh::deploy::deposit::db::dbname')}" + host: "%{hiera('swh::deploy::deposit::db::host')}" + user: "%{hiera('swh::deploy::deposit::db::dbuser')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::deposit::db::password')}" + - alias: swh-scheduler + name: "%{hiera('swh::deploy::scheduler::db::dbname')}" + host: "%{hiera('swh::deploy::scheduler::db::host')}" + user: "%{hiera('swh::deploy::scheduler::db::user')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::db::scheduler::password')}" + - alias: swh-vault + name: "%{hiera('swh::deploy::vault::db::dbname')}" + host: "%{hiera('swh::deploy::vault::db::host')}" + user: "%{hiera('swh::deploy::vault::db::user')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::vault::db::password')}" + - alias: swh-lister + name: "%{hiera('swh::deploy::worker::lister::db::name')}" + host: "%{hiera('swh::deploy::worker::lister::db::host')}" + user: "%{hiera('swh::deploy::worker::lister::db::name')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::worker::lister::db::password')}" + - alias: swh-replica + name: "%{hiera('swh::deploy::storage::db::dbname')}" + host: somerset.internal.softwareheritage.org + user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::db::storage::password')}" + - alias: swh-indexer + name: "%{hiera('swh::deploy::indexer::storage::db::dbname')}" + host: "%{hiera('swh::deploy::indexer::storage::db::host')}" + user: "%{hiera('swh::deploy::indexer::storage::db::user')}" + port: "%{hiera('swh::deploy::db::pgbouncer::port')}" + passwd: "%{hiera('swh::deploy::db::indexer::password')}" + - alias: staging-swh + name: swh + host: db0.internal.staging.swh.network + user: swh + port: 5432 + passwd: "%{hiera('swh::deploy::db::storage::password')}" + - alias: staging-swh-indexer + name: swh-indexer + host: db0.internal.staging.swh.network + user: swh-indexer + port: 5432 + passwd: "%{hiera('swh::deploy::db::indexer::password')}" + - alias: staging-swh-scheduler + name: swh-scheduler + host: db0.internal.staging.swh.network + user: swh-scheduler + port: 5432 + passwd: "%{hiera('swh::deploy::db::scheduler::password')}" + - alias: staging-swh-deposit + name: swh-deposit + host: deposit.internal.staging.swh.network + user: swh-deposit + port: 5432 + passwd: "%{hiera('swh::deploy::db::deposit::password')}" + diff --git a/data/defaults_security.yaml b/data/defaults_security.yaml new file mode 100644 index 00000000..da3429d0 --- /dev/null +++ b/data/defaults_security.yaml @@ -0,0 +1,659 @@ +--- +# configuration file to solely store certificate/gpg public key/cert blocks. +# For default configuration, see defaults.yaml + +ssl: + star_softwareheritage_org: + certificate: | + -----BEGIN CERTIFICATE----- + MIIIXzCCB0egAwIBAgIQDPHPysLFk3NJoSYi+mieNDANBgkqhkiG9w0BAQsFADBk + MQswCQYDVQQGEwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJ + QW1zdGVyZGFtMQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wg + Q0EgMzAeFw0xODA5MjcwMDAwMDBaFw0yMDEyMzAwMDAwMDBaMIGQMQswCQYDVQQG + EwJGUjEVMBMGA1UEBxMMUm9jcXVlbmNvdXJ0MUkwRwYDVQQKE0BJbnN0aXR1dCBO + YXRpb25hbCBkZSBSZWNoZXJjaGUgZW4gSW5mb3JtYXRpcXVlIGV0IGVuIEF1dG9t + YXRpcXVlMR8wHQYDVQQDDBYqLnNvZnR3YXJlaGVyaXRhZ2Uub3JnMIICIjANBgkq + hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA5CWwZR0vKg7KRxYon/cqja0mpVjXAKMV + KyUQFvTRoY/fD07t51Z5nC0pBi4VxjVJKFN8UO6fX0kJhy4Lp+ErsOPw80YcCMq1 + 86W9Vo6DG2kqjHp0TnsUm+wu2J8F9fQR/HfANx7rflSs2CzBvf2nr0u4HevnCzk+ + zxky+j+YS9l2Xx5Svz/DTbVpHn162Sf+03g908dsLs2nq3IJbJnzkNkSWpizq/aE + Ud/vocQEK66tKN5NLfHezljDQBbK8cySUaX6BFjPRmDcw25rKDgClMIGbYA7P7Hl + fswcrsxNxAtwa9ZDTnPW5zRp9aeVcm5NT57vzKZGGsRt+aEzeq+IvcIR9RCQOgIP + G3Dzjva0eunppslXOz8hiZd86Omo38yfFi6xJS3qH6nMSItdKa+4yxKCMVlJkx00 + WtsovY7kfYFpzJfd9Mk7jTBdmD5WRcBiatFGNgaqXstuVHS1CGK2qs5+iOiaXOdn + 6glIJgN/dKcgA6nvBIYDguvgYU59HxS56Dq9PqdpFy1B6Qghu2Zj4P5iJG+Gl+fP + aK0WG5ISsdsr6QGLcJnD3Zjk5F68vo/on1xm3Zk+v7mHlps2BLgxUiP3D5vTpUgY + L0xqKfbjDr65R0s7CxZZmXdOUsHgo7P99BKUYiM3Tw9G4AOkFJPLP2OrbLckdmmO + K4Iwxhbj4H8CAwEAAaOCA94wggPaMB8GA1UdIwQYMBaAFGf9iCAUJ5jHCdIlGbvp + URFjdVBiMB0GA1UdDgQWBBRf7/Wq5kiF/QnNu0Bxe2q6bZjeTDA3BgNVHREEMDAu + ghYqLnNvZnR3YXJlaGVyaXRhZ2Uub3JnghRzb2Z0d2FyZWhlcml0YWdlLm9yZzAO + BgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMGsG + A1UdHwRkMGIwL6AtoCuGKWh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9URVJFTkFT + U0xDQTMuY3JsMC+gLaArhilodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vVEVSRU5B + U1NMQ0EzLmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsGAQUFBwIB + FhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjBuBggrBgEF + BQcBAQRiMGAwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTA4 + BggrBgEFBQcwAoYsaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL1RFUkVOQVNT + TENBMy5jcnQwDAYDVR0TAQH/BAIwADCCAfUGCisGAQQB1nkCBAIEggHlBIIB4QHf + AHYApLkJkLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BAAAAFmG53a8gAABAMA + RzBFAiEA4FsESbszMapP/1928EYdqTnGxwDpAnjibblmY4Qb4nkCIF/dTJoCbDnO + 0LSLS+P5bc9cAkgGAyXBwpvB8+ZJszctAHYAh3W/51l8+IxDmV+9827/Vo1HVjb/ + SrVgwbTq/16ggw8AAAFmG53bzwAABAMARzBFAiEAlpn5W7rsxL6urq8S3tMw9Hr+ + cZaWAg7dp6kolPecYMECIDFDDBElmoHXCzW35kQXzGxgwrj/ns1nrbc7T5PzjcKw + AHUAu9nfvB+KcbWTlCOXqpJ7RzhXlQqrUugakJZkNo4e0YUAAAFmG53b9AAABAMA + RjBEAiA8F7P2NjU/AXfguds8cWDRqJsuVpF3qrzVZ4hnlTBeuQIgFohcpNsjkcY3 + hsgDI6j2VMuk5TVLd/2ehdtaT8lUX/kAdgBvU3asMfAxGdiZAKRRFf93FRwR2QLB + ACkGjbIImjfZEwAAAWYbndzFAAAEAwBHMEUCIEwt+EUHkqPBraADN4jCGF6tfzp6 + LcAIOWt4m/8OMM4pAiEA8m3u/w219RAamjShsaguGbQfxRZGcQcVc0++zXilApYw + DQYJKoZIhvcNAQELBQADggEBAKr3KUfjEWsIk1umodHo8Cck49TgioNy6e8w9ajW + 9Sp3FB/MRskWtL3NU64CecMJlvgJrZogUv+eZlquLCnTiKQdmTY0xYVmt+igT9ca + o907aEc2puEHDm8KsCvwLM6nbVPR03nEaj/P6aSM42eordzcWyQY5+uy69r1yfus + gVwJjm9aq+EX7KoK+HDE/fF8BOuHxoWzuTxbKTGjD7lEwMi+MdFGwhX/WEOu3Um8 + MZzX4bRVAgtJNbbrBRYQ1D2gSA+VAmXEBJhYV5UboNWgGE6EwMwTOCTyhpAjHZdy + 3yFaxgRObjDQEO/LMG+/iDKMUV1T2onVux89LEpaEzeJcnA= + -----END CERTIFICATE----- + ca_bundle: | + -----BEGIN CERTIFICATE----- + MIIE+zCCA+OgAwIBAgIQCHC8xa8/25Wakctq7u/kZTANBgkqhkiG9w0BAQsFADBl + MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 + d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv + b3QgQ0EwHhcNMTQxMTE4MTIwMDAwWhcNMjQxMTE4MTIwMDAwWjBkMQswCQYDVQQG + EwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJQW1zdGVyZGFt + MQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wgQ0EgMzCCASIw + DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMV2Dw/ZQyk7bG3RR63eEL8jwnio + Snc18SNb4EweQefCMQC9iDdFdd25AhCAHo/tZCMERaegOTuBTc9jP8JJ/yKeiLDS + lrlcinQfkioq8hLIt2hUtVhBgUBoBhpPhSn7tU08D08/QJYbzqjMXjX/ZJj1dd10 + VAWgNhEEEiRVY++Udy538RV27tOkWUUhn6i+0SftCuirOMo/h9Ha8Y+5Cx9E5+Ct + 85XCFk3shKM6ktTPxn3mvcsaQE+zVLHzj28NHuO+SaNW5Ae8jafOHbBbV1bRxBz8 + mGXRzUYvkZS/RYVJ+G1ShxwCVgEnFqtyLvRx5GG1IKD6JmlqCvGrn223zyUCAwEA + AaOCAaYwggGiMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMHkG + CCsGAQUFBwEBBG0wazAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu + Y29tMEMGCCsGAQUFBzAChjdodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln + aUNlcnRBc3N1cmVkSURSb290Q0EuY3J0MIGBBgNVHR8EejB4MDqgOKA2hjRodHRw + Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3Js + MDqgOKA2hjRodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVk + SURSb290Q0EuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxo + dHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBRn/YggFCeYxwnS + JRm76VERY3VQYjAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq + hkiG9w0BAQsFAAOCAQEAqSg1esR71tonHqyYzyc2TxEydHTmQN0dzfJodzWvs4xd + xgS/FfQjZ4u5b5cE60adws3J0aSugS7JurHogNAcyTnBVnZZbJx946nw09E02DxJ + WYsamM6/xvLYMDX/6W9doK867mZTrqqMaci+mqege9iCSzMTyAfzd9fzZM2eY/lC + J1OuEDOJcjcV8b73HjWizsMt8tey5gvHacDlH198aZt+ziYaM0TDuncFO7pdP0GJ + +hY77gRuW6xWS++McPJKe1e9GW6LNgdUJi2GCZQfXzer8CM/jyxflp5HcahE3qm5 + hS+1NGClXwmgmkMd1L8tRNaN2v11y18WoA5hwnA9Ng== + -----END CERTIFICATE----- + star_internal_softwareheritage_org: + certificate: | + -----BEGIN CERTIFICATE----- + MIIGsDCCBZigAwIBAgIQD1wiwe+Cg9VcwL5QrB+UqzANBgkqhkiG9w0BAQsFADBk + MQswCQYDVQQGEwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJ + QW1zdGVyZGFtMQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wg + Q0EgMzAeFw0xNTEwMTUwMDAwMDBaFw0xODEwMTkxMjAwMDBaMIHIMQswCQYDVQQG + EwJGUjERMA8GA1UECBMIWXZlbGluZXMxFTATBgNVBAcTDFJvY3F1ZW5jb3VydDFJ + MEcGA1UEChNASW5zdGl0dXQgTmF0aW9uYWwgZGUgUmVjaGVyY2hlIGVuIEluZm9y + bWF0aXF1ZSBldCBlbiBBdXRvbWF0aXF1ZTEaMBgGA1UECxMRU29mdHdhcmUgSGVy + aXRhZ2UxKDAmBgNVBAMMHyouaW50ZXJuYWwuc29mdHdhcmVoZXJpdGFnZS5vcmcw + ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDxz6BsVNPDk1fSAltRCwiU + 0iZtd+10WKJL+tgPmv/s3+WgIqj5+hQ/iZsAc7Y45yQo4yU+PsDq+BIFR2yt5rtk + B0Xz4sNHAo585IEvWvf3wAULf6GQ17o2XRxN5IZfNgLfRy6jQjeIbiFmO5M2g6To + Fl+MMAuK0+u9u6az41eBl1etTe7QjGaL+B45kfLyLeBB4rhusEQSPRTrGrdrgdEt + di9jedDjbMkV2B197D7CZHqKRR3B+yuRwgd/t/OqIBpN0M//kPE6AJzpLN2B4Z6L + HGSeyprQleFkS3d7hAlZal/9di8C+38bAGmTW8dQLGwFeYTHN6hBuMYm0y4Yk32W + 423rPZuguA3j2sIxOuIExKmlMDzuGociy3npfjTvWCi/6ESo0hkImnrWKUZ0eyFH + h52DsPC6ePhou4qh+KdwObxJmYgS69YhHgrsfZokbbWy/hj6N6dAFmTYJQ9hcrpr + x2CHHAmbd7J8gTInMDHlUMJ1vcL11coRpmfevMiaK4szZwzRfI+xsevVeHRusxpC + ErrtW+T+9rSfFVIROjXirD4uok8R3UTMpwNOhPQtKTHN/8v9Cvp5BzY0XNnRiVFc + lL/4YezG+YO6r4GieWEJyLyB7R/JRJbSuCJLzzqZek4twESiV7mtKe2P1clUF1z+ + O1+pLFUaRSSgpGxypDgIXQIDAQABo4IB9zCCAfMwHwYDVR0jBBgwFoAUZ/2IIBQn + mMcJ0iUZu+lREWN1UGIwHQYDVR0OBBYEFEZAY+NxWfpgt+qo+0pdRQ5uTsPeMEkG + A1UdEQRCMECCHyouaW50ZXJuYWwuc29mdHdhcmVoZXJpdGFnZS5vcmeCHWludGVy + bmFsLnNvZnR3YXJlaGVyaXRhZ2Uub3JnMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUE + FjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwawYDVR0fBGQwYjAvoC2gK4YpaHR0cDov + L2NybDMuZGlnaWNlcnQuY29tL1RFUkVOQVNTTENBMy5jcmwwL6AtoCuGKWh0dHA6 + Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9URVJFTkFTU0xDQTMuY3JsMEwGA1UdIARFMEMw + NwYJYIZIAYb9bAEBMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LmRpZ2ljZXJ0 + LmNvbS9DUFMwCAYGZ4EMAQICMG4GCCsGAQUFBwEBBGIwYDAkBggrBgEFBQcwAYYY + aHR0cDovL29jc3AuZGlnaWNlcnQuY29tMDgGCCsGAQUFBzAChixodHRwOi8vY2Fj + ZXJ0cy5kaWdpY2VydC5jb20vVEVSRU5BU1NMQ0EzLmNydDAMBgNVHRMBAf8EAjAA + MA0GCSqGSIb3DQEBCwUAA4IBAQBt7fVm/+sNa5VkhdxfiSSAqrqrjrNPI2izU3Bs + fNrN+o7buLGTmA83WJaJsX74kdegqsmT5J3qBO/RDNpvIJqeGBQdDO2L4tRCHxSZ + mJ0o7GKFfWY01hF6J6jSHzOehpL/UQ37U1Kh3l7GLBkOhPubV9wkG5jMLlrYNCAd + gY59W84hH9QCV+oO44F4Q3bdwTcXrZuQ3tpyqqgukmrCm0TcK7pOa5FWZ4r6ZIWe + RNpor8OyVqmMj8U2NX2478LsyE4Ut6NrtdOQHnVVFgPlsuiQgTUfqgL5XCMQw4vz + 5vNH/vho0zTFoJjB68mos1xazWNYqC+QmKXcsFWeodct9l3F + -----END CERTIFICATE----- + ca_bundle: | + -----BEGIN CERTIFICATE----- + MIIE+zCCA+OgAwIBAgIQCHC8xa8/25Wakctq7u/kZTANBgkqhkiG9w0BAQsFADBl + MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 + d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv + b3QgQ0EwHhcNMTQxMTE4MTIwMDAwWhcNMjQxMTE4MTIwMDAwWjBkMQswCQYDVQQG + EwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJQW1zdGVyZGFt + MQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wgQ0EgMzCCASIw + DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMV2Dw/ZQyk7bG3RR63eEL8jwnio + Snc18SNb4EweQefCMQC9iDdFdd25AhCAHo/tZCMERaegOTuBTc9jP8JJ/yKeiLDS + lrlcinQfkioq8hLIt2hUtVhBgUBoBhpPhSn7tU08D08/QJYbzqjMXjX/ZJj1dd10 + VAWgNhEEEiRVY++Udy538RV27tOkWUUhn6i+0SftCuirOMo/h9Ha8Y+5Cx9E5+Ct + 85XCFk3shKM6ktTPxn3mvcsaQE+zVLHzj28NHuO+SaNW5Ae8jafOHbBbV1bRxBz8 + mGXRzUYvkZS/RYVJ+G1ShxwCVgEnFqtyLvRx5GG1IKD6JmlqCvGrn223zyUCAwEA + AaOCAaYwggGiMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMHkG + CCsGAQUFBwEBBG0wazAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu + Y29tMEMGCCsGAQUFBzAChjdodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln + aUNlcnRBc3N1cmVkSURSb290Q0EuY3J0MIGBBgNVHR8EejB4MDqgOKA2hjRodHRw + Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3Js + MDqgOKA2hjRodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVk + SURSb290Q0EuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxo + dHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBRn/YggFCeYxwnS + JRm76VERY3VQYjAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq + hkiG9w0BAQsFAAOCAQEAqSg1esR71tonHqyYzyc2TxEydHTmQN0dzfJodzWvs4xd + xgS/FfQjZ4u5b5cE60adws3J0aSugS7JurHogNAcyTnBVnZZbJx946nw09E02DxJ + WYsamM6/xvLYMDX/6W9doK867mZTrqqMaci+mqege9iCSzMTyAfzd9fzZM2eY/lC + J1OuEDOJcjcV8b73HjWizsMt8tey5gvHacDlH198aZt+ziYaM0TDuncFO7pdP0GJ + +hY77gRuW6xWS++McPJKe1e9GW6LNgdUJi2GCZQfXzer8CM/jyxflp5HcahE3qm5 + hS+1NGClXwmgmkMd1L8tRNaN2v11y18WoA5hwnA9Ng== + -----END CERTIFICATE----- + stats_export_softwareheritage_org: + certificate: | + -----BEGIN CERTIFICATE----- + MIIGtDCCBZygAwIBAgIQBdh1rlqVNqqDtC3ToOG6zjANBgkqhkiG9w0BAQ0FADBk + MQswCQYDVQQGEwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJ + QW1zdGVyZGFtMQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wg + Q0EgMzAeFw0xNjA2MjAwMDAwMDBaFw0xOTA2MjUxMjAwMDBaMIHKMQswCQYDVQQG + EwJGUjERMA8GA1UECBMIWXZlbGluZXMxFTATBgNVBAcTDFJvY3F1ZW5jb3VydDFJ + MEcGA1UEChNASW5zdGl0dXQgTmF0aW9uYWwgZGUgUmVjaGVyY2hlIGVuIEluZm9y + bWF0aXF1ZSBldCBlbiBBdXRvbWF0aXF1ZTEaMBgGA1UECxMRU29mdHdhcmUgSGVy + aXRhZ2UxKjAoBgNVBAMTIXN0YXRzLmV4cG9ydC5zb2Z0d2FyZWhlcml0YWdlLm9y + ZzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOzz90M9R26tzSSGdU6w + Hy4xXP9mInZvR4JWa4TYgSIosvC+N6pZpV1PM3ZWR3RlJ6Mn5yyQ3uNJQnuHeqEX + X3Yq6xrloylgoE/bW+6rStv+MhCka7zkcbpZ900RMK6Uun7dlBxqW6+Y0e9Z0NZY + 7RW+R4w1MqcUR0kJ6pwM+bYIXtUkgpUi6aXLg7LhNoufkxcYnBDZe3GTYNeFdEUN + /NjbqxVcLRO7DNTszwpLoT+6Dg0lmbu2/ZROJg0c+YYzIvWGiSCxkP0jlra6W8EO + j9aXWp/7+lvX0qWsLNYxfnOb5QDQVDqbuIZuqjsFFgXgueS6cEJRWhgZaAeVeZif + HidHLWdpGHlQgqG+EwE3iaLJPOrGQqtNUwk6DEJpyhQCu9Fc78irolk9eyccgfQr + 7YEuxN2yukFwutQy7QP1/6CWtCovwwNw0/l1vmbFd2hcOmyq/DzOXeBkAHo8e5+t + EG+nN5Mk5vUI/OkQLO24/6IHge3uN2zXcDaqwAgQ/06TVCbdCrwzDFuhpt27EHDp + PvpD/751z7axqrIemXM5lDXgnG7QbQHN74P1n+g3fkVljBuWF2L1mfXBDRogslPI + VvYHql3QLIkHEvx2gTq/O2piVt3awlQqqXJmnURUMgf8acYfMc2QhFNmRIeorS1W + z6LXHohQRiVvVhmjlVpPa7mXAgMBAAGjggH5MIIB9TAfBgNVHSMEGDAWgBRn/Ygg + FCeYxwnSJRm76VERY3VQYjAdBgNVHQ4EFgQUb7Uzk+Q4+Zg/alPyn1AodsDoSbMw + SwYDVR0RBEQwQoIhc3RhdHMuZXhwb3J0LnNvZnR3YXJlaGVyaXRhZ2Uub3Jngh1w + ZXJnYW1vbi5zb2Z0d2FyZWhlcml0YWdlLm9yZzAOBgNVHQ8BAf8EBAMCBaAwHQYD + VR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMGsGA1UdHwRkMGIwL6AtoCuGKWh0 + dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9URVJFTkFTU0xDQTMuY3JsMC+gLaArhilo + dHRwOi8vY3JsNC5kaWdpY2VydC5jb20vVEVSRU5BU1NMQ0EzLmNybDBMBgNVHSAE + RTBDMDcGCWCGSAGG/WwBATAqMCgGCCsGAQUFBwIBFhxodHRwczovL3d3dy5kaWdp + Y2VydC5jb20vQ1BTMAgGBmeBDAECAjBuBggrBgEFBQcBAQRiMGAwJAYIKwYBBQUH + MAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTA4BggrBgEFBQcwAoYsaHR0cDov + L2NhY2VydHMuZGlnaWNlcnQuY29tL1RFUkVOQVNTTENBMy5jcnQwDAYDVR0TAQH/ + BAIwADANBgkqhkiG9w0BAQ0FAAOCAQEAUjG+ZmlzgZwiXp5xJqdbG8EZ9Dh7+utY + ZXkWlr2VsDzGNt9dLcht0FxfjozOoizpIdldLFkjW2OkNIQhChBpiLQ6gFn79B9/ + iNSZONQUpI1sqCLaOnOLbTHza0zi3Is+MBKjwxBVAcUERVjJbu5YSEm/Dle5IpUw + S8K5A7iwFIBQywLOySvz9P1c+MFqMLackEmlVM+vnF6axqQtMgOhscM06GW5bZdy + UnamhLXOIr+/Z9B+voB080qZZUn8DqFxP7og8au9IYKP5zLqTxTKayYX3qSXl+QT + NpzEGsZavMjzG7fNmQ8bTnEimThnqLbLAcgqWMmy/LBFRvabwHan4w== + -----END CERTIFICATE----- + ca_bundle: | + -----BEGIN CERTIFICATE----- + MIIE+zCCA+OgAwIBAgIQCHC8xa8/25Wakctq7u/kZTANBgkqhkiG9w0BAQsFADBl + MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 + d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv + b3QgQ0EwHhcNMTQxMTE4MTIwMDAwWhcNMjQxMTE4MTIwMDAwWjBkMQswCQYDVQQG + EwJOTDEWMBQGA1UECBMNTm9vcmQtSG9sbGFuZDESMBAGA1UEBxMJQW1zdGVyZGFt + MQ8wDQYDVQQKEwZURVJFTkExGDAWBgNVBAMTD1RFUkVOQSBTU0wgQ0EgMzCCASIw + DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMV2Dw/ZQyk7bG3RR63eEL8jwnio + Snc18SNb4EweQefCMQC9iDdFdd25AhCAHo/tZCMERaegOTuBTc9jP8JJ/yKeiLDS + lrlcinQfkioq8hLIt2hUtVhBgUBoBhpPhSn7tU08D08/QJYbzqjMXjX/ZJj1dd10 + VAWgNhEEEiRVY++Udy538RV27tOkWUUhn6i+0SftCuirOMo/h9Ha8Y+5Cx9E5+Ct + 85XCFk3shKM6ktTPxn3mvcsaQE+zVLHzj28NHuO+SaNW5Ae8jafOHbBbV1bRxBz8 + mGXRzUYvkZS/RYVJ+G1ShxwCVgEnFqtyLvRx5GG1IKD6JmlqCvGrn223zyUCAwEA + AaOCAaYwggGiMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMHkG + CCsGAQUFBwEBBG0wazAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGlnaWNlcnQu + Y29tMEMGCCsGAQUFBzAChjdodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5jb20vRGln + aUNlcnRBc3N1cmVkSURSb290Q0EuY3J0MIGBBgNVHR8EejB4MDqgOKA2hjRodHRw + Oi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVkSURSb290Q0EuY3Js + MDqgOKA2hjRodHRwOi8vY3JsNC5kaWdpY2VydC5jb20vRGlnaUNlcnRBc3N1cmVk + SURSb290Q0EuY3JsMD0GA1UdIAQ2MDQwMgYEVR0gADAqMCgGCCsGAQUFBwIBFhxo + dHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMB0GA1UdDgQWBBRn/YggFCeYxwnS + JRm76VERY3VQYjAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYunpyGd823IDzANBgkq + hkiG9w0BAQsFAAOCAQEAqSg1esR71tonHqyYzyc2TxEydHTmQN0dzfJodzWvs4xd + xgS/FfQjZ4u5b5cE60adws3J0aSugS7JurHogNAcyTnBVnZZbJx946nw09E02DxJ + WYsamM6/xvLYMDX/6W9doK867mZTrqqMaci+mqege9iCSzMTyAfzd9fzZM2eY/lC + J1OuEDOJcjcV8b73HjWizsMt8tey5gvHacDlH198aZt+ziYaM0TDuncFO7pdP0GJ + +hY77gRuW6xWS++McPJKe1e9GW6LNgdUJi2GCZQfXzer8CM/jyxflp5HcahE3qm5 + hS+1NGClXwmgmkMd1L8tRNaN2v11y18WoA5hwnA9Ng== + -----END CERTIFICATE----- + +postgresql::apt_config::pgdg::mirror: 'http://apt.postgresql.org/pub/repos/apt/' +postgresql::apt_config::pgdg::keyid: B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 +postgresql::apt_config::pgdg::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja + UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V + G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4 + bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi + c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC + IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh + hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U + A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3 + RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj + Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2 + AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB + tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQI9BBMBCAAnAhsDBQsJCAcD + BRUKCQgLBRYCAwEAAh4BAheABQJS6RUZBQkOhCctAAoJEH/MfUaszEz4zmQP/2ad + HtuaXL5Xu3C3NGLha/aQb9iSJC8z5vN55HMCpsWlmslCBuEr+qR+oZvPkvwh0Io/ + 8hQl/qN54DMNifRwVL2n2eG52yNERie9BrAMK2kNFZZCH4OxlMN0876BmDuNq2U6 + 7vUtCv+pxT+g9R1LvlPgLCTjS3m+qMqUICJ310BMT2cpYlJx3YqXouFkdWBVurI0 + pGU/+QtydcJALz5eZbzlbYSPWbOm2ZSS2cLrCsVNFDOAbYLtUn955yXB5s4rIscE + vTzBxPgID1iBknnPzdu2tCpk07yJleiupxI1yXstCtvhGCbiAbGFDaKzhgcAxSIX + 0ZPahpaYLdCkcoLlfgD+ar4K8veSK2LazrhO99O0onRG0p7zuXszXphO4E/WdbTO + yDD35qCqYeAX6TaB+2l4kIdVqPgoXT/doWVLUK2NjZtd3JpMWI0OGYDFn2DAvgwP + xqKEoGTOYuoWKssnwLlA/ZMETegak27gFAKfoQlmHjeA/PLC2KRYd6Wg2DSifhn+ + 2MouoE4XFfeekVBQx98rOQ5NLwy/TYlsHXm1n0RW86ETN3chj/PPWjsi80t5oepx + 82azRoVu95LJUkHpPLYyqwfueoVzp2+B2hJU2Rg7w+cJq64TfeJG8hrc93MnSKIb + zTvXfdPtvYdHhhA2LYu4+5mh5ASlAMJXD7zIOZt2iEYEEBEIAAYFAk6XSO4ACgkQ + xa93SlhRC1qmjwCg9U7U+XN7Gc/dhY/eymJqmzUGT/gAn0guvoX75Y+BsZlI6dWn + qaFU6N8HiQIcBBABCAAGBQJOl0kLAAoJEExaa6sS0qeuBfEP/3AnLrcKx+dFKERX + o4NBCGWr+i1CnowupKS3rm2xLbmiB969szG5TxnOIvnjECqPz6skK3HkV3jTZaju + v3sR6M2ItpnrncWuiLnYcCSDp9TEMpCWzTEgtrBlKdVuTNTeRGILeIcvqoZX5w+u + i0eBvvbeRbHEyUsvOEnYjrqoAjqUJj5FUZtR1+V9fnZp8zDgpOSxx0LomnFdKnhj + uyXAQlRCA6/roVNR9ruRjxTR5ubteZ9ubTsVYr2/eMYOjQ46LhAgR+3Alblu/WHB + MR/9F9//RuOa43R5Sjx9TiFCYol+Ozk8XRt3QGweEH51YkSYY3oRbHBb2Fkql6N6 + YFqlLBL7/aiWnNmRDEs/cdpo9HpFsbjOv4RlsSXQfvvfOayHpT5nO1UQFzoyMVpJ + 615zwmQDJT5Qy7uvr2eQYRV9AXt8t/H+xjQsRZCc5YVmeAo91qIzI/tA2gtXik49 + 6yeziZbfUvcZzuzjjxFExss4DSAwMgorvBeIbiz2k2qXukbqcTjB2XqAlZasd6Ll + nLXpQdqDV3McYkP/MvttWh3w+J/woiBcA7yEI5e3YJk97uS6+ssbqLEd0CcdT+qz + +Waw0z/ZIU99Lfh2Qm77OT6vr//Zulw5ovjZVO2boRIcve7S97gQ4KC+G/+QaRS+ + VPZ67j5UMxqtT/Y4+NHcQGgwF/1iiQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYC + AwEAAh4BAheABQJQeSssBQkDwxbfAAoJEH/MfUaszEz4bgkP/0AI0UgDgkNNqplA + IpE/pkwem2jgGpJGKurh2xDu6j2ZL+BPzPhzyCeMHZwTXkkI373TXGQQP8dIa+RD + HAZ3iijw4+ISdKWpziEUJjUk04UMPTlN+dYJt2EHLQDD0VLtX0yQC/wLmVEH/REp + oclbVjZR/+ehwX2IxOIlXmkZJDSycl975FnSUjMAvyzty8P9DN0fIrQ7Ju+BfMOM + TnUkOdp0kRUYez7pxbURJfkM0NxAP1geACI91aISBpFg3zxQs1d3MmUIhJ4wHvYB + uaR7Fx1FkLAxWddre/OCYJBsjucE9uqc04rgKVjN5P/VfqNxyUoB+YZ+8Lk4t03p + RBcD9XzcyOYlFLWXbcWxTn1jJ2QMqRIWi5lzZIOMw5B+OK9LLPX0dAwIFGr9WtuV + J2zp+D4CBEMtn4Byh8EaQsttHeqAkpZoMlrEeNBDz2L7RquPQNmiuom15nb7xU/k + 7PGfqtkpBaaGBV9tJkdp7BdH27dZXx+uT+uHbpMXkRrXliHjWpAw+NGwADh/Pjmq + ExlQSdgAiXy1TTOdzxKH7WrwMFGDK0fddKr8GH3f+Oq4eOoNRa6/UhTCmBPbryCS + IA7EAd0Aae9YaLlOB+eTORg/F1EWLPm34kKSRtae3gfHuY2cdUmoDVnOF8C9hc0P + bL65G4NWPt+fW7lIj+0+kF19s2PviQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYC + AwEAAh4BAheABQJRKm2VBQkINsBBAAoJEH/MfUaszEz4RTEP/1sQHyjHaUiAPaCA + v8jw/3SaWP/g8qLjpY6ROjLnDMvwKwRAoxUwcIv4/TWDOMpwJN+CJIbjXsXNYvf9 + OX+UTOvq4iwi4ADrAAw2xw+Jomc6EsYla+hkN2FzGzhpXfZFfUsuphjY3FKL+4hX + H+R8ucNwIz3yrkfc17MMn8yFNWFzm4omU9/JeeaafwUoLxlULL2zY7H3+QmxCl0u + 6t8VvlszdEFhemLHzVYRY0Ro/ISrR78CnANNsMIy3i11U5uvdeWVCoWV1BXNLzOD + 4+BIDbMB/Do8PQCWiliSGZi8lvmj/sKbumMFQonMQWOfQswTtqTyQ3yhUM1LaxK5 + PYq13rggi3rA8oq8SYb/KNCQL5pzACji4TRVK0kNpvtxJxe84X8+9IB1vhBvF/Ji + /xDd/3VDNPY+k1a47cON0S8Qc8DA3mq4hRfcgvuWy7ZxoMY7AfSJOhleb9+PzRBB + n9agYgMxZg1RUWZazQ5KuoJqbxpwOYVFja/stItNS4xsmi0lh2I4MNlBEDqnFLUx + SvTDc22c3uJlWhzBM/f2jH19uUeqm4jaggob3iJvJmK+Q7Ns3WcfhuWwCnc1+58d + iFAMRUCRBPeFS0qd56QGk1r97B6+3UfLUslCfaaA8IMOFvQSHJwDO87xWGyxeRTY + IIP9up4xwgje9LB7fMxsSkCDTHOk + =s3DI + -----END PGP PUBLIC KEY BLOCK----- + +icinga2::apt_config::mirror: 'http://packages.icinga.com/debian' +icinga2::apt_config::keyid: F51A91A5EE001AA5D77D53C4C6E319C334410682 +icinga2::apt_config::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v2.0.19 (GNU/Linux) + + mQGiBFKHzk4RBACSHMIFTtfw4ZsNKAA03Gf5t7ovsKWnS7kcMYleAidypqhOmkGg + 0petiYsMPYT+MOepCJFGNzwQwJhZrdLUxxMSWay4Xj0ArgpD9vbvU+gj8Tb02l+x + SqNGP8jXMV5UnK4gZsrYGLUPvx47uNNYRIRJAGOPYTvohhnFJiG402dzlwCg4u5I + 1RdFplkp9JM6vNM9VBIAmcED/2jr7UQGsPs8YOiPkskGHLh/zXgO8SvcNAxCLgbp + BjGcF4Iso/A2TAI/2KGJW6kBW/Paf722ltU6s/6mutdXJppgNAz5nfpEt4uZKZyu + oSWf77179B2B/Wl1BsX/Oc3chscAgQb2pD/qPF/VYRJU+hvdQkq1zfi6cVsxyREV + k+IwA/46nXh51CQxE29ayuy1BoIOxezvuXFUXZ8rP6aCh4KaiN9AJoy7pBieCzsq + d7rPEeGIzBjI+yhEu8p92W6KWzL0xduWfYg9I7a2GTk8CaLX2OCLuwnKd7RVDyyZ + yzRjWs0T5U7SRAWspLStYxMdKert9lLyQiRHtLwmlgBPqa0gh7Q+SWNpbmdhIE9w + ZW4gU291cmNlIE1vbml0b3JpbmcgKEJ1aWxkIHNlcnZlcikgPGluZm9AaWNpbmdh + Lm9yZz6IYAQTEQIAIAUCUofOTgIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJ + EMbjGcM0QQaCgSQAnRjXdbsyqziqhmxfAKffNJYuMPwdAKCS/IRCVyQzApFBtIBQ + 1xuoym/4C7kCDQRSh85OEAgAvPwjlURCi8z6+7i60no4n16dNcSzd6AT8Kizpv2r + 9BmNBff/GNYGnHyob/DMtmO2esEuVG8w62rO9m1wzzXzjbtmtU7NZ1Tg+C+reU2I + GNVu3SYtEVK/UTJHAhLcgry9yD99610tYPN2Fx33Efse94mXOreBfCvDsmFGSc7j + GVNCWXpMR3jTYyGj1igYd5ztOzG63D8gPyOucTTl+RWN/G9EoGBv6sWqk5eCd1Fs + JlWyQX4BJn3YsCZx3uj1DWL0dAl2zqcn6m1M4oj1ozW47MqM/efKOcV6VvCs9SL8 + F/NFvZcH4LKzeupCQ5jEONqcTlVlnLlIqId95Z4DI4AV9wADBQf/S6sKA4oH49tD + Yb5xAfUyEp5ben05TzUJbXs0Z7hfRQzy9+vQbWGamWLgg3QRUVPx1e4IT+W5vEm5 + dggNTMEwlLMI7izCPDcD32B5oxNVxlfj428KGllYWCFj+edY+xKTvw/PHnn+drKs + LE65Gwx4BPHm9EqWHIBX6aPzbgbJZZ06f6jWVBi/N7e/5n8lkxXqS23DBKemapyu + S1i56sH7mQSMaRZP/iiOroAJemPNxv1IQkykxw2woWMmTLKLMCD/i+4DxejE50tK + dxaOLTc4HDCsattw/RVJO6fwE414IXHMv330z4HKWJevMQ+CmQGfswvCwgeBP9n8 + PItLjBQAXIhJBBgRAgAJBQJSh85OAhsMAAoJEMbjGcM0QQaCzpAAmwUNoRyySf9p + 5G3/2UD1PMueIwOtAKDVVDXEq5LJPVg4iafNu0SRMwgP0Q== + =icbY + -----END PGP PUBLIC KEY BLOCK----- + +php::version: '7.1' +php::apt_config::mirror: https://packages.sury.org/php/ +php::apt_config::keyid: 15058500A0235D97F5D10063B188E2B695BD4743 +php::apt_config::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQGNBFyPb58BDADTDlJLrGJktWDaUT0tFohjFxy/lL2GcVYp4zB981MWIDC0aIQZ + ERfUZRaq/ov/LG3F0UhkvouCNrnXiFaKRCeNG52pQM0P/p3gmIOoPO4/jF0o3SK1 + Aapf/NaKTh3EgeYYCnVKuxdXGqyu1JT4qfztsmUGmODzxVr+/YJLP54jrCUgI3lj + 4zEeTBDexQvnlVUF59U1/ipMq4iWqqth8/aMsoZl3Ztfcc87jBFbJIoeQMhZtNZk + Ik7L15aYIZXWY2byBy6LB42HPm9DwM99l2eY4EXGfAq/UQeYbDGonibBqrDURggH + rkLfG7ZfoexF67/9S2s6VYfS4npWVfw2SEPTfSBdibElbGncd+p9Wb6SovqapCPl + crkLgPhBAz/R9M7E/G3zedmiEhsV78pBF3bup+nQVvBVtV/NucN5N6LkAclT4O3F + flGZa1/mJcpgjVapT6duY0POXczfS6ts55x2BE0UfYtXfRnVnHtu2+j8kqYG3N1G + sfVnzRkwtTWBMxMAEQEAAbQxREVCLlNVUlkuT1JHIEF1dG9tYXRpYyBTaWduaW5n + IEtleSA8ZGViQHN1cnkub3JnPokB1AQTAQoAPhYhBBUFhQCgI12X9dEAY7GI4raV + vUdDBQJcj2+fAhsDBQkDwmcABQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJELGI + 4raVvUdDROUMANJjLVGk6TLYZKyRc8HZXyMRmw55nCQXsy8DHen6H7MuZHaxV9sf + 1tF1cQwPnv3HFlg7nZBFszyReW7s3LOcPuNXS90Sk5o7WLqVMkE+t46iNNGuIt3s + WxPdYqY4ueCnUHHEFDKhlwlJnNh3+yVNci4nV+6SlzoasEjy5P82+pviop3viSlA + 8lgXdOYERRqZ1wh0Vip5gxcNIdm+TqWINI9+7T/87GfZzn84Zlvd0GnfjT6aN6RC + sIADIOqJUW/TPzvdytwlXc4oZvqk47P0YePS6VFd+TrIRHHsmYxwTVPNjCT7eMz6 + AfEtPbTZHWI9oOWU+tiAogeR7k1yORknf+HcLZ1RnfDxyWgEh/p9eWuBeNCgLVSF + tcYCJvRU8OFwRPz8B7rs3/tjrabrkV8iVaEeiCZkqPpaqZF6QaY2Z2YmZEtFsUMb + rVwLbfBj4+3CSY90rlRolZDJjea6oXtjWXPXANSJ5gJZI9qFbX0WBXJ7dilrfUfH + wrsqkH97072OTbkBjQRcj2+fAQwA4McaM/y2XQSHlJBSYR7yqZtHX/kZ8g9pnViq + kCEADz8XKCroEzvY1gaWtR6obtjaq8pF0g4KtAC65/gIOtsHvWg3OclrODPkXN+x + OM1LpXZGV6kwk+LXOrybtPhVZe3FtvDMW0MVZeHYi+soZ4tTQHkKjZUPAXZs3ZoZ + rWfE5ft447sCxzX+jxDwwlckkKqZ9sHYD0TV8Y5av3RsxiWBt+coch8jvw+1mDZ0 + zBjMO8ZRD8PuvP9UTKCNOIm0mW9A2cUfpkk/uAwo5hCnw4iljS81/KKGM/scwc5K + x6G3WWoAb8kajt0VFG/wYN2qjfjdhXtdu3ZxYtDdjA2UGGRbgkCsr+gRCnSTiuwv + LzCVZCz9WNzZjUMg6LFP2IrHned4Kdy4KjJo+g/weKJoxfKokZ/9vUYpw5OYx3UE + SUk3yHDN9r/JC4RJJ2tE2qkeggJ892RJGxUK/Lw3/7jIQKalO3Qx2zYUqnCYMC9g + PhQGH+F9kwSpGVwb0DKFT6gR9Pt3ABEBAAGJAbwEGAEKACYWIQQVBYUAoCNdl/XR + AGOxiOK2lb1HQwUCXI9vnwIbDAUJA8JnAAAKCRCxiOK2lb1HQ9pWDACel7BTcpj9 + qYJOkEx9eAVH1LmdP5oHAJSe4fYVIc5Tn6X0J1pdju+UJvtUS6WQu7pfcsQ7IkbK + XVygaw/YPqk/tE/vJVUaCGlzAaVHO7y5ZM0cdDquOgmo/RZe2WNVFhVEPILjz+qc + mX1OVeTElCu4iR/5CD0ocWZmdFvMRitKL+Nt5/pS8yXRfQlXaUxerLvx27QhwpHX + HFGNgbI8Tj/yCUCButZvDKUc+5o9WCYHvt/bf6VcNY6P585J0+CF1vAyfWQ9KCyC + F5CplURsNPMKAdkPo7vKRq6qviJpeWjJzXPkADfVnfhW9GMna8bPi86NJo1PkGcb + 1IInxGmNRmnCqtqyPFd+AFGk2OFuEqeDw7GvjB5zD5u1vIdH3WE0Q+D0xxLfaxJ7 + 1qmhuTPRnpWqfmdA2tnuFOZ+Th4HRfbyA4IoJr3U8FuFXJnJdoBjh6+pJacI9UYr + IcNyOoi1I35gHvFoKJ7Ut9deNdKMeN3bAR+1JShVtdROzgGtPj/bMFg= + =74SA + -----END PGP PUBLIC KEY BLOCK----- + +# APT repository settings + +jenkins::apt_config::mirror: https://pkg.jenkins.io/debian-stable +jenkins::apt_config::keyid: 150FDE3F7787E7D11EF4E12A9B7D32F2D50582E6 +jenkins::apt_config::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQGiBEmFQG0RBACXScOxb6BTV6rQE/tcJopAEWsdvmE0jNIRWjDDzB7HovX6Anrq + n7+Vq4spAReSFbBVaYiiOx2cGDymj2dyx2i9NAI/9/cQXJOU+RPdDzHVlO1Edksp + 5rKn0cGPWY5sLxRf8s/tO5oyKgwCVgTaB5a8gBHaoGms3nNC4YYf+lqlpwCgjbti + 3u1iMIx6Rs+dG0+xw1oi5FUD/2tLJMx7vCUQHhPRupeYFPoD8vWpcbGb5nHfHi4U + 8/x4qZspAIwvXtGw0UBHildGpqe9onp22Syadn/7JgMWhHoFw5Ke/rTMlxREL7pa + TiXuagD2G84tjJ66oJP1FigslJzrnG61y85V7THL61OFqDg6IOP4onbsdqHby4VD + zZj9A/9uQxIn5250AGLNpARStAcNPJNJbHOQuv0iF3vnG8uO7/oscB0TYb8/juxr + hs9GdSN0U0BxENR+8KWy5lttpqLMKlKRknQYy34UstQiyFgAQ9Epncu9uIbVDgWt + y7utnqXN033EyYkcWx5EhLAgHkC7wSzeSWABV3JSXN7CeeOif7QiS29oc3VrZSBL + YXdhZ3VjaGkgPGtrQGtvaHN1a2Uub3JnPohjBBMRAgAjAhsDBgsJCAcDAgQVAggD + BBYCAwECHgECF4AFAko/7vYCGQEACgkQm30y8tUFguabhgCgi54IQR4rpJZ/uUHe + ZB879zUWTQwAniQDBO+Zly7Fsvm0Mcvqvl02UzxCiGAEExECACAFAkmFQG0CGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCbfTLy1QWC5qtXAJ9hPRisOhkexWXJ + nXQMl9cOTvm4LgCdGint1TONoZ2I4JtOiFzOmeP3ju3RzcvNyQEQAAEBAAAAAAAA + AAAAAAAA/9j/4AAQSkZJRgABAQEAYABgAAD/4QBgRXhpZgAASUkqAAgAAAAEADEB + AgAZAAAAPgAAABBRAQABAAAAAUOQABFRBAABAAAAEgsAABJRBAABAAAAEgsAAAAA + AABNYWNyb21lZGlhIEZpcmV3b3JrcyA0LjAAAP/bAEMACAYGBwYFCAcHBwkJCAoM + FA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0 + Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy + MjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAK4AlgMBIgACEQEDEQH/xAAfAAAB + BQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0B + AgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygp + KjQ1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImK + kpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj + 5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEBAQEBAQEBAAAAAAAAAQIDBAUGBwgJ + Cgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFRB2FxEyIygQgUQpGh + scEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVWV1hZ + WmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1 + tre4ubrCw8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEA + AhEDEQA/APcBI/8Afb86XzH/AL7fnUYpwqRknmN/fP50u9v7x/OmCgUASb2/vH86 + Xe394/nTBS0AP3t/eP50u4+p/OmUopgO3H1NO3H1NR5xThQA7cfWlyfU0ylFMQ/J + 9aXPvTKdQAuaM0lLQAtJmiigAzRSdqKAKApwpopc1mUOpRSUopgKKWkFLQAueKzr + zXbCwk2Tzxq3cFwK8v8Aih8V30aaTQ9DKtegYnuTyIvZR3b+VfP1/q17fzvLc3Ms + sjHJZ2JJNGr2HZdT6j8U/FbR/DcKsM3VxLkpGh6AetcI37Ql4Zcx6LAYx2aUgmvD + 1ju7obgJHA7nmmmG4TqjDHtS+ZXL1sfVPhT4yeH/ABFNHaXYbS71zhVnYGNz6B+n + 4HFejK2RmvhJJSDiTj6ivYvht8XptE8rSPEEklxpxwkFyTue39j6p+op3a3Javsf + RuacDVaC4juIUmhkWSKRQyspyGB7ipgasgfmlpoNLmgBaKSigBaKM0UAUBS0lKKz + KFFLSUooAdWR4o1qLw/4bvtSmZVEMRK57t2H51rCvJPj7etD4WsbQMQJ7jkDuFBN + D2GlqfP13dS3k89zM5eaZy7sTySTWvovhw3JWWdcqeQtUNGsWvtQRMfIvJr0u0t1 + hjUKOnpXFi8Q6a5Y7npYLDqfvyILXQolRVWMdOwp1x4cjYH5QPwrftQcDippFavM + UpvW569ktLHnOp+FFaNiijcOlcfcW8tlN5UgI+tezXEeSeM5rmtf0OK/tSVUCVOV + Irsw+KlF8s9jhxWFjNc0dzpfgh49MV1/wimozExyndYOx+6/eP6HqPcEd697Vq+I + baWbTb+G5hJWe3lWVCDj5lOf6V9naTqUeraRZ6jEMR3UKTKM9NwzivXj2PDmrM1A + 1PqBTUoNMlDqKSloAKKOpopAUacKbS1mWOFKKbS0xC14p+0Gw+z6Ihb+ORsfgK9r + rxT9oO3X7Ho1zn5vMePHrxn+lJjW55t4QgZbOe7CbmJ2IPU10sltriIDaSW7ORlg + 44HsKz/BCbtFyBysjVdvo9bcTNDMyEFfKCEDdzzknpx04NeVUles9vme3Rjairdu + hoaXqOqwt5Wo2cSjoHRuv4VuTXKCAuBzjoa5myW9SKJLmVpH25lLEEBs9sVuTgGw + BGN3f3rOU7SaOqEW43Me7l1a8l225SCL+9tyajfT7lHS4SdmkH+sVujj+lQakuo3 + ELC0uGjkBwqh9qlceuM5z/L3q1p9nfwyqzzs8WxQVkOTuxycjsT2q7+7e6MXH3mr + M898QWgtNbmVeEcbwK+l/hdK7/DXQjI+4iAgH0AY4FfO/jWMx6+oxx5QP619B/Cx + Wj+G2i7twzExww7bzj8K9bDO8UeJitJv1O5U1Mp4qshqdTW7RzpklLmmg0tSULmi + kopAU6WkFFZlDqWm0tMQteX/ABe8MXPiBLCSN1SODcq5H8bY5+mB+teoVi+KbQ3e + gXAU4dPnB9MVFS/I+Xc0pNKa5tjw/wAJ2L6fpbWsw2zRzOsg9wa6RIlk6Diszy5L + a5kYksJTuyfWrUN2xbArxpyUpczPoKS5VyiXKQwHoBk/mamID2AIFZ89w6SlvKSV + ugDNjFK2p3It/L8uIAc//WpRhd3RtKaSs2WLNIpQeAcGrjosYIFZVvcPLIr7Fibo + Qpzmp5rp/N24prTQmT0uYOv6LDrWt2avIIkSJjI3qMjAHuTmveNEsU0rRbGwjPyW + 0CRr9AK8k0y0S81yMMAzllQL3xnnAr2cdfavXwLbT8jwcwsmrbssoamU8VXQ1Otd + jOBEoNOBqMGnA1BY6ikHNFAypS0lLWRQtFFApgLTJoknheKQZRwVYe1OopiPO/GP + hq202xgu7RX+VishZs9a4pmaMtsGSRkAV7Xq9gupaXPasPvr8v17V4jKHt7qS3k4 + kjYqa8vF0lCSaWh6uDrOSab1KAuLia9a2CJCQu7zLhgoI9q2f+Ecv2h877XZbTuB + Ikz0x/jVK4RZVAdckDg1QfEY8kW6EeoYgH6jOKwi0z0emkrfK5LcyXNpex2YEVyz + ruEkD5Cj1NX1Lbt0hyVHP1qpbxiFCyqN5HYYAq/pcH2/WbSyLcSyAMfbqaduaSij + KpJRTdz03w3p0dpo1m7RL57JvLFRuG7nr16YrdWolAHAGB2qVa+hjFRioo+YlJyk + 5MnSp1NQpUopMESCnA+tMFOBqS0Oz6UUlFIZWopKXNZFi0UlFMQuaM0maM0wOU8Z + /ELRfA8UQ1Ayz3kw3RWkABcrnG4k8KPr17CvIbjWR4lSXXbW2Nv5srHyS+4gA9Cc + DNYfxfl+1fEbVCsm8xFI+T0wo4/CrHg9kt9OFm88TyffwrA43DOPw71y4xfuk13O + zBfxGn2NWDU4ZFXLbXHDKamN7a7cfLn3qCWyt2nKyxAj3FLJo9hFGH8sNu5HJrzo + 2PTbkupHPqcafLHlnPCqKu6VqMfhy4h1nUEkdIDvdIwC2MYwM455rMW502wlzLLD + Cq+p5P4dax9e8S2N5aSWtuXcOMFsYH61vSpzlNOKMKs4qLUme6+EvHWk+MRcLp6X + EUtuAzxzqAcHjIwTmuqQ185/CTXo9J8XRW0iqsF+v2bcxxtbOVOfcjH419EqcHBr + 3FqeDJWZbQ1KDVeNqmBqWCJRTs1GDTgakseKKQc0UgK1LTaq6lqljo9g99qV3Fa2 + qfellbAz6DuT7DmsjQuU15FiiaWR1SNBlndgFUe5PSvGfEfx02s8HhzTwR0F3eDr + 7rGP/Zj+FeU674u1zxE5bVtUuLlc5ETNiNfogwo/KrUWFj37xF8YfC+hiSK1mfVb + tePLtf8AVg+8h4/LNeSa/wDGHxRrcjpb3Q0u3OcRWZ2nHu5+Y/p9K89Z9x5ppOM8 + 1SihXHTTyO7NIzO7MWZmOSxPUk+tQrKyNuUkEdwcGnFs8EVGV9Kom5YGoXqtuW7n + B9fMNPOrag67Wvbgr6eYap4OelA5qeSPYrnl3Jg7McsxJ9SakTrzUCg+1SgqgyTm + rJLkbjII6e9dfp/xR8VaciLFqjTxxAKI7pFkBHuTz+tcL5xI9AeAKcpGSSe1Az37 + wx8adPv3S3122FjKeBPES8R+o6r+tepWl7b3tulxazxTwvyskbBlP4ivjASAnA4r + Z0DxVrHh2787TL+WDP3kzlG+qng0XFyo+wlfIp4NeN+FfjbaXs0dp4gt1tGPH2uH + Jjz/ALS9R9RmvWra6huoEnt5o5oXGUkjYMrD2IpE2aLgoqMOMUUWC5ka/rVv4e0K + 71W5G6O3QsEBwXboFH1OK+WPE3irVfE2pNeapcM7ZPlxA4jhX+6i9h+p71698dNZ + +z6Np+ko3zXMpmkH+yvA/U/pXgcz7k9x/KogtDR6DXmJ71EXOKYTzSE5qybi7uaU + mmd6UcimITPNKDmmnrQKAJM8Ck3egpuaQUAPBJ6k4ozknjimk9qB0oGO3E04NUYp + aQEu/wBqXOFAPeohyQKV25NMCdJDng103hjxnq/hm7WTTrp1jJy8LHMb/Vf8muU+ + 6g9TThIUGB1Pf0osNM+wPCnie18U6HHqNspjbOyaInJjcdR7jnINFeY/APUUJ1jS + pZVQER3K7jjn7rf+y0U1YiWj0OW+NmoG68dvbhsrawIgHoTyf515qzbth9eDXQ+P + NQOo+NNUus5DzED6Dj+lc0DnI9DmohsXLcaTQOaG6n60CqJEpVpM0A80ADDmkpzd + RSUALRRRQACiijvQAtFJRmgY9B3po5b605DhGNN70CHu2CT+ApEwX5+ppG5AP1pM + 4GB1PWmBraZez2rvJBM8TMMEocHFFVLViFOKKm1y0xb9zNI0pJLFiT+PNUlPz5NW + Jm+/9RVYjGPenYlisMufrSE05vu5qOgQtA60dqB1oAe3QU2nN0plAC0tJSjrQAlL + miigAptL0pO9AEi8RfU0mM8560H/AFaikzx+NMBxx0H40zOeaU8KffikHSgCxC+y + LPqaKYeAq+gooHc//9mIYAQTEQIAIAUCSj/3IAIbAwYLCQgHAwIEFQIIAwQWAgMB + Ah4BAheAAAoJEJt9MvLVBYLmt2sAnRUJQoS4J/5+LW+Iy3tUYMTsR8aLAJ9gp9qD + YbGfdcFG+HeSbh/PEwrqbLQzS29oc3VrZSBLYXdhZ3VjaGkgPGtvaHN1a2Uua2F3 + YWd1Y2hpQGNsb3VkYmVlcy5jb20+iGIEExECACIFAk0GnroCGwMGCwkIBwMCBhUI + AgkKCwQWAgMBAh4BAheAAAoJEJt9MvLVBYLmfugAnRb1qac6CqRaNUhHbzd1m/5S + niNzAJ9NJUC2Fjk7uEyvQ5bDJ+hAFbkQVLQpS29oc3VrZSBLYXdhZ3VjaGkgPGtv + aHN1a2VAY2xvdWRiZWVzLmNvbT6IYgQTEQIAIgUCVh045AIbAwYLCQgHAwIGFQgC + CQoLBBYCAwECHgECF4AACgkQm30y8tUFguZVLgCdElQ2ydLBp33/9SFyVEz3cFMk + 0DkAn2qWsQlPT549lAqeSnkhCOcGJAx0tCxLb2hzdWtlIEthd2FndWNoaSA8a2th + d2FndWNoaUBjbG91ZGJlZXMuY29tPohiBBMRAgAiBQJWHTjzAhsDBgsJCAcDAgYV + CAIJCgsEFgIDAQIeAQIXgAAKCRCbfTLy1QWC5sMTAKCA5kH0uH0x0HoTuxjrU740 + pU/53gCfaFWE6s7nBFMkJ3RyxjtZBGnY2Jm5Ag0ESYVAbRAIAOoBdaCKKzjKL3qi + zdBmYrnzT2iONNOeUgKBvO2tPnlwxVMMFz1Kd7JFCULRxL4zXPgOjqWPzWw0l0mI + E+pNhgDX57FMW+znMLE8icM/eG+pfEdM/XjZc3WF3O3ndHuyafw7TDI75EIFRvjh + 702S6y8F3lQ/cl7jj2GelcnhY7dxUwWbiCHGzsRGWkCLk1MSxVV0zx2odtkm2TyB + vN0AcfTJuIBeZbIsUZkO64qIUCSqb9aV53uJ3o35w/HXTt3AFyXA/HN8RgoSonVg + MMegOXJ/HjTXbLXnd7mwbJqH8g8Fiussx8b5aaLCvmcJfS2bA5zK6S4T3iFvMkJf + bAF1tYsAAwYIALOXdy4ziUa3/CvmWIziCi1elkCilj4SdssgG44cVddHsefICBJP + WMf8BRtp+8+PIOESQUPJQ/Xhe0c0gCqw3VSm7Jhsz3Rsw8BZcnGtrMyxIX5O/nIj + EeLLhxzWmOiocDaTCogYeZPFjM485LX1lZAC16+hMTqkIBGmFjR3OmxwJZpcaz9m + o0CGMv3pYthXU6hS372ZOc5yzpW7FrGnbA3ZLkMrVL2B0jFYRzzAxQ+JB7wJiTQ7 + JJ05EhuUyzdsaoMWgzkdwEBk/ViVeK08fachG/QO05AYxA4KSpRaZC5ABSApX5g7 + zqU7hLsSFMRP8Y+xBvo/t5+b8KzzBur/DIiISQQYEQIACQUCSYVAbQIbDAAKCRCb + fTLy1QWC5raYAJ4k0FbiycMLg7OMpTpBPfzr8YD2ywCfe8vNLCfw3XG/kyKFYavm + RXO9oTa5Ag0EWBjgRgEQALze0WQartDG4x1DaOpqKLAol9pfxSX+O88Nafw9dDdV + v80CD7Q66p6X5o1TOOqEAqsI/dUFzDoZzW/EBN5TVKdNhV55WsIbvFJnJ9ccQ1yk + fCYVQAH/eCIdM8dujAOZLjKSapz/wBdFbbOffvz7GLmsjn1wCruZfIOcaIcfaUfY + QWsafzwU9VsRLSDrbwpylQJkvblfeb+ohQ/AYlVJmD1HcKF81AajgxbTUDCBxslY + 4kL6FmqqfLJDWXyg0aG7UEbP3ye7/61qrsKR0g84BHYgkLzQkdgsAGAMo3HvQzss + BAqhZy2QSWKZCe6OQuIEzL01oTWJOWJYAoak9pSkjuFDsRbFRHC4YiaCIvwFHA8C + 3nCaa/jAXQ/NrBFyc1TsrDdxiXi6cEgER9WichpQaD/NCKGGHbEzzHow1Ni+pABq + 1leoVAfAEw8OwRYEftfoAQ5O8VdWe754xK2I5wFWjGKM0IHruEqnRgbWXL9Vy6Cv + NTrQIoJbVuO/kQWH4jZ63TzsBnxHzdnRSuCNGXnuneIju8+wr33y+r914cNziCHm + Tt0UsyTcf7xfzVB++obS0sCyklDIy+1EEzLePkUYl7Ebkst5tKgbVRNyH1niKRwX + xoyowmIRznO79l46u9JMdlt9VO9oo+yR9DqMgNqUnc9Z+rt8EyUam87838FfF+OF + ABEBAAGJAmgEGBECAAkFAlgY4EYCGwICKQkQm30y8tUFgubBXSAEGQECAAYFAlgY + 4EYACgkQlHo/RMJzQlXPTg//UpZd7vx0wNm6dPSUc9Agw5tQU5oCR4BUaDOBFDfb + nKPNa8JQPVdH6lrt1Zaqc9Uka+l1eVK8SZiujohr3bCyal+5ParAdVbTt08pvh5d + 3YllLIKKad82Qy6WsUlAQmUpba+Fn5naXdd8WDN03J7LVOqYCQUWZu65r5oqmv8B + eh+vcZO5ozEt/Huy+ruCsdb0WavbgI5+Pj6sKJtKBo5WwZzbDpbPUEUd3/T5zFbJ + G/XDk77qfBP4DKC96tphzGp6EaEtrZ9Qto8AisCYGvhDptYqXqZm4J1mJj/SI+4C + /1kVY0EEf4ySLy4/8f91h/jzcEliQNnmNZWgUTmP/nyUS+iLqUa4NmhdO45NYBfJ + PZyviHsFxJhYppiPt32n5FpGrXM8fWaQsA+aKOL2D+AWeC8W/pPmDurLbYA1yRk7 + T7E1llz4wDf53CumQGtT4gKwmUdGbwp0TNZKggv+/6auOMoBVjvWCRM0erxR+fAL + FKruuoXjQ69I2bTiZfoSHtDxqa+YMnNqqFOZdyJsH13Fx/Ma3k0EVI4uOuX5RoJ8 + BN3SAkBSiZu/yRf9XF/ikKvrb3YcaPaUgRPVP3EweJJx98whWxPmgSbv/GvQCQa7 + GyvwvqvWuiw+kgl4RlCGvL354zQwSoD+li+ZgnuhzRlSnj962O2cobvY+UzW1fiO + vTrGzQCgg7/WrciTjK8wtd8e/E26mU1agOMAniYHo/aFmpsSFfNp4n419EI+mCXU + =fBn8 + -----END PGP PUBLIC KEY BLOCK----- + +docker::apt_config::mirror: https://download.docker.com/linux/debian +docker::apt_config::keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 +docker::apt_config::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- + +elastic::apt_config::keyid: 46095ACC8548582C1A2699A9D27D666CD88E42B4 +elastic::apt_config::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v2.0.14 (GNU/Linux) + + mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD + A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9 + CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ + j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd + 1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD + 2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg + KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy + Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC + F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75 + nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/ + 7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm + TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe + 8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/ + eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl + zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT + RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+ + 1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+ + Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt + KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww + EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0 + c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J + TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j + 6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7 + vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM + cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/ + qPDlGRlOgVTd9xUfHFkzB52c70E= + =92oX + -----END PGP PUBLIC KEY BLOCK----- +elastic::elk_version: '6.4.2' + +hwraid_levert::apt_config::keyid: 0073C11919A641464163F7116005210E23B3D3B4 +hwraid_levert::apt_config::key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.12 (GNU/Linux) + + mQENBFHwGLoBCADGXHFostxbz4UzGFYtmox4pvyN1gMhq2KCuQ6f+FESa4HTd9L6 + XVhXWPCad3cdxBIls+41+AdZTWxWMu7DUdy8nMU1Ikfw6JeHcSx97G5BdxBVMjK4 + iMGfPdLfDgWf4BQ2h0dnTEWobt31WaqgNiNjNrKktqbymmF94pwYkwL53ydIA4zl + 8ZQRZooFigkS9WdoKjh30Pv/SWakILSLcSQFHK0dvSkeGd1NxT9dMNPAXXqLom4+ + 7kCc0s04sS+0DwW16b0Hpb46mtsR9kzOnrE/Smj24uOGzNZen0oCc2Y7bfZlyaN+ + RlTkWEze7lemc4Byup/QWkhT0Er8F8uxexy5ABEBAAG0PEhXUmFpZCAoaHR0cDov + L2h3cmFpZC5sZS12ZXJ0Lm5ldCkgPHJvb3RAaHdyYWlkLmxlLXZlcnQubmV0PokB + OAQTAQIAIgUCUfAYugIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQYAUh + DiOz07Rc4Af+N3dEZZHzLNVTjQ0+fCyeg8/flWOkR8DhP10cyoJhSHFTZRdXVshn + kP4VmmUycVeURh76DmrIRe/9Oyca6aGXccRMqvq+HMgBPVwD5qNhcJPIuzqEvmlO + 6UIeW2ydil/v1pWu740fGntyFRQcsfqjReVPXw9K588F7MDMyL+31vLm6aorLSzR + hvLhOmGisTs0wg2Oz9f4muauRy6cpQPw/Zi/P/F4WkQYscbHrSbhszj6OIg/vftR + UbZ7QB26/+40B0ag4JzLpmj3scFxf/WdUl5LXazqhsbkurk7huV41BNKXi1+BS3c + x6pFzWEHpiuG1j7U/nScGzEQpsMlUW9D+rkBDQRR8Bi6AQgAuhH1H0VLwcROI/5n + 9yTxSbTIZbyhUan3raAbit3pgo0zLagfUtp3vULVnm5ISqQcYFGLZoE1MUkmjGOL + 38W0lsIiZTaKOKXxBbLlPhhrvlXnNWAG/S1wnq7K+DV179KCTkUzaLRDbHvv999j + 9odBRtAkiTnCfHTMCN4AhydEejNxtlzJo4E5FecH4reimLI5euUdTltgCjixrbsa + KbQftYpSMdXnLy2+00QZoXu0U/h4WZcMhOSEEiyGP9BY6m5G76n03HIeQ6eALDFu + ryAgO+SB9rBrm/VN0kR/TZq0iA3uzLHC7zCw2aImipkr+rIuJOku0wH9MyowBbia + bQtnCQARAQABiQEfBBgBAgAJBQJR8Bi6AhsMAAoJEGAFIQ4js9O0d5YH/3fNQgsC + LvD0g2wdoksv5bG9CUOi9Bs0JHqI0LhXmPvMsbDojZ+zZle7KWNfK2227mWhmoG1 + WLujJSmTtxhEO1fXIdYjlDfk2uLJKuFi2wQX9n8dFDUmKY3CUJgeVZof1uQ/5C3D + O06CcuOtf2d/+iijuW112aV1q1hoQqw71ojTET0iIV6lD/0i1eEBSSe1Ohb9yTGR + VxTVrB78zU9hih4/Oq8wJT/Fv25aO1MDSc26CXAg0JA6IWvKal3BSPNhtz4L4FIg + lXleArf9oJqxDO3TsV5zcLyxsIuRuxyP0+AKdSQUqv0dFi4Jf79OmvOmgwydhHjY + +f7quLbwiiDmPbU= + =Yv6D + -----END PGP PUBLIC KEY BLOCK----- diff --git a/data/hostname/banco.softwareheritage.org.yaml b/data/hostname/banco.softwareheritage.org.yaml index d0831590..0b96a9d5 100644 --- a/data/hostname/banco.softwareheritage.org.yaml +++ b/data/hostname/banco.softwareheritage.org.yaml @@ -1,87 +1,88 @@ smtp::mynetworks: - 127.0.0.0/8 - "[::ffff:127.0.0.0]/104" - "[::1]/128" - 192.168.128.0/24 # storage array, for notification emails - 192.168.129.0/24 # storage array, for notification emails smtp::mail_aliases: - user: barman aliases: - root smtp::relayhost: '[pergamon.internal.softwareheritage.org]' dar::backup::exclude: - srv/barman + - srv/rsnapshot - srv/softwareheritage/annex swh::apt_config::enable_non_free: true packages: - intel-microcode # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 16 swh::deploy::objstorage::ceph::keyring: /etc/softwareheritage/objstorage/ceph-keyring swh::deploy::objstorage::ceph::rados_id: swh-contents-rw ceph::client_keyrings: '/etc/softwareheritage/objstorage/ceph-keyring': owner: root group: swhstorage mode: '0640' keys: - "%{alias('swh::deploy::objstorage::ceph::rados_id')}" icinga2::host::vars: disks: disk /srv/storage/0: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/1: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/2: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/3: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/4: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/5: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/6: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/7: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/8: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/9: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/a: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/b: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/c: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/d: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/e: disk_wfree: '6%' disk_cfree: '3%' disk /srv/storage/f: disk_wfree: '6%' disk_cfree: '3%' diff --git a/data/hostname/belvedere.internal.softwareheritage.org.yaml b/data/hostname/belvedere.internal.softwareheritage.org.yaml new file mode 100644 index 00000000..a0a52d18 --- /dev/null +++ b/data/hostname/belvedere.internal.softwareheritage.org.yaml @@ -0,0 +1,37 @@ +prometheus::sql::config_snippets: + - swh-scheduler + - swh-storage + +pgbouncer::auth_hba_file: /etc/postgresql/11/secondary/pg_hba.conf +pgbouncer::listen_addr: 192.168.100.210 +pgbouncer::databases: + # swh + - source_db: "%{hiera('swh::deploy::storage::db::dbname')}" + host: belvedere.internal.softwareheritage.org + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::main::port')}" + # swh-deposit + - source_db: "%{hiera('swh::deploy::deposit::db::dbname')}" + host: "%{hiera('swh::deploy::deposit::db::host')}" + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::secondary::port')}" + # swh-scheduler + - source_db: "%{hiera('swh::deploy::scheduler::db::dbname')}" + host: "%{hiera('swh::deploy::scheduler::db::host')}" + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::secondary::port')}" + # swh-scheduler-updater + - source_db: "%{hiera('swh::deploy::scheduler::updater::backend::db::dbname')}" + host: "%{hiera('swh::deploy::scheduler::updater::backend::db::host')}" + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::secondary::port')}" + # swh-vault + - source_db: "%{hiera('swh::deploy::vault::db::dbname')}" + host: "%{hiera('swh::deploy::vault::db::host')}" + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::secondary::port')}" + # swh-lister + - source_db: "%{hiera('swh::deploy::worker::lister::db::name')}" + host: "%{hiera('swh::deploy::worker::lister::db::host')}" + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::secondary::port')}" diff --git a/data/hostname/db0.internal.staging.swh.network.yaml b/data/hostname/db0.internal.staging.swh.network.yaml new file mode 100644 index 00000000..9a6a4eeb --- /dev/null +++ b/data/hostname/db0.internal.staging.swh.network.yaml @@ -0,0 +1,56 @@ +--- +networks: + default: + interface: eth0 + address: 192.168.128.3 + netmask: 255.255.255.0 + gateway: 192.168.128.1 + +swh::dbs: + storage: + name: swh + user: swh + indexer: + name: swh-indexer + user: swh-indexer + scheduler: + name: swh-scheduler + user: swh-scheduler + vault: + name: swh-vault + user: swh-vault + +postgres::server::port: 5433 +postgres::server::listen_addresses: + - localhost + - 192.168.128.3 +postgres::server::network_access: + - 192.168.100.0/24 + - 192.168.128.0/24 + +pgbouncer::auth_hba_file: /etc/postgresql/11/main/pg_hba.conf +pgbouncer::listen_addr: 192.168.128.3 +pgbouncer::databases: + - source_db: swh + host: db0.internal.staging.swh.network + auth_user: postgres + port: 5433 + alias: staging-swh + - source_db: swh-indexer + host: db0.internal.staging.swh.network + auth_user: postgres + port: 5433 + alias: staging-swh-indexer + - source_db: swh-scheduler + host: db0.internal.staging.swh.network + auth_user: postgres + port: 5433 + alias: staging-swh-scheduler + - source_db: swh-vault + host: db0.internal.staging.swh.network + auth_user: postgres + port: 5433 + alias: staging-swh-vault + +dar::backup::exclude: + - srv/softwareheritage/postgres diff --git a/data/hostname/deposit.internal.staging.swh.network.yaml b/data/hostname/deposit.internal.staging.swh.network.yaml new file mode 100644 index 00000000..b14e5967 --- /dev/null +++ b/data/hostname/deposit.internal.staging.swh.network.yaml @@ -0,0 +1,52 @@ +networks: + default: + interface: eth0 + address: 192.168.128.7 + netmask: 255.255.255.0 + gateway: 192.168.128.1 + +## db setup + +postgres::server::port: 5433 +postgres::server::listen_addresses: + - localhost + - 192.168.128.7 +postgres::server::network_access: + - 192.168.100.0/24 + - 192.168.128.0/24 + +# Dbs to create +swh::dbs: + deposit: + name: swh-deposit + user: swh-deposit + +pgbouncer::auth_hba_file: /etc/postgresql/11/main/pg_hba.conf +pgbouncer::listen_addr: 192.168.128.7 +pgbouncer::databases: + - source_db: swh-deposit + host: deposit.internal.staging.swh.network + auth_user: postgres + port: 5433 + alias: staging-swh-deposit + +## frontend + +hitch::frontend: "[*]:443" +hitch::proxy_support: true + +varnish::http_port: 80 +apache::http_port: 9080 + +# Disable default vhost on port 80 +apache::default_vhost: false + +## deposit + +swh::deploy::deposit::media_root_directory: /tmp/swh-deposit +swh::deploy::deposit::vhost::name: deposit.internal.staging.swh.network +swh::deploy::deposit::vhost::aliases: [ + deposit.staging.swh.network +] +swh::deploy::deposit::config::allowed_hosts: + - deposit.internal.staging.swh.network diff --git a/data/hostname/gateway.internal.staging.swh.network.yaml b/data/hostname/gateway.internal.staging.swh.network.yaml new file mode 100644 index 00000000..46417510 --- /dev/null +++ b/data/hostname/gateway.internal.staging.swh.network.yaml @@ -0,0 +1,17 @@ +--- +networks: + default: + interface: eth0 + address: 192.168.100.125 + netmask: 255.255.255.0 + gateway: 192.168.100.1 + ups: + - 'iptables -t nat -A POSTROUTING -s 192.168.128.0/24 -o eth0 -j MASQUERADE' + downs: + - 'iptables -t nat -F' + private: + interface: eth1 + address: 192.168.128.1 + netmask: 255.255.255.0 + ups: [] + downs: [] diff --git a/data/hostname/getty.internal.softwareheritage.org.yaml b/data/hostname/getty.internal.softwareheritage.org.yaml index 68225c5c..382b115b 100644 --- a/data/hostname/getty.internal.softwareheritage.org.yaml +++ b/data/hostname/getty.internal.softwareheritage.org.yaml @@ -1,4 +1,2 @@ dar::backup::exclude: - srv/kafka - -kafka::broker_id: 1 diff --git a/data/hostname/moma.softwareheritage.org.yaml b/data/hostname/moma.softwareheritage.org.yaml index 93848cd6..2523006c 100644 --- a/data/hostname/moma.softwareheritage.org.yaml +++ b/data/hostname/moma.softwareheritage.org.yaml @@ -1,37 +1,36 @@ networks: private: interface: eth1 address: 192.168.100.31 netmask: 255.255.255.0 gateway: 192.168.100.1 default: interface: eth0 address: 128.93.193.31 netmask: 255.255.255.0 gateway: 128.93.193.254 dar::backup::exclude: - var/lib/rabbitmq -swh::deploy::storage::db::host: somerset -swh::deploy::storage::db::port: 5433 - -swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" +swh::deploy::storage::db::host: somerset.internal.softwareheritage.org hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disabled as it seems to be flaky #hitch::http2_support: true #varnish::http2_support: true # Disable default vhost on port 80 apache::default_vhost: false swh::deploy::webapp::vhost::name: archive.softwareheritage.org swh::deploy::webapp::vhost::aliases: - base.softwareheritage.org - archive.internal.softwareheritage.org + +swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::localhost')}" diff --git a/data/hostname/orangerie.internal.softwareheritage.org.yaml b/data/hostname/orangerie.internal.softwareheritage.org.yaml index 2c0c46e1..54f00f2a 100644 --- a/data/hostname/orangerie.internal.softwareheritage.org.yaml +++ b/data/hostname/orangerie.internal.softwareheritage.org.yaml @@ -1,8 +1,5 @@ ssh::port: 7022 -# open objstorage api -swh::deploy::objstorage::backend::listen::host: 0.0.0.0 -swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::vault::cache')}" swh::deploy::vault::backend::listen::host: 0.0.0.0 diff --git a/data/hostname/orangeriedev.internal.softwareheritage.org.yaml b/data/hostname/orangeriedev.internal.softwareheritage.org.yaml index e6c53498..144afbea 100644 --- a/data/hostname/orangeriedev.internal.softwareheritage.org.yaml +++ b/data/hostname/orangeriedev.internal.softwareheritage.org.yaml @@ -1,17 +1,15 @@ ssh::port: 7022 -# open objstorage api -swh::deploy::objstorage::backend::listen::host: 0.0.0.0 -swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::vault::cache')}" swh::deploy::vault::db::host: orangeriedev swh::deploy::vault::backend::listen::host: 0.0.0.0 # Don't use a worker for now as this will conflict with the production workers # because they have the same queue names. # # swh::deploy::worker::instances: -# - swh_vault_cooker -# swh::deploy::worker::swh_vault_cooker::vault_url: "http://orangeriedev.internal.softwareheritage.org:5005/" +# - vault_cooker +# swh::deploy::worker::vault_cooker::vault_url: "http://orangeriedev.internal.softwareheritage.org:5005/" swh::deploy::worker::instances: [] +swh::deploy::vault::db::port: 5432 diff --git a/data/hostname/pergamon.softwareheritage.org.yaml b/data/hostname/pergamon.softwareheritage.org.yaml index 13d65a20..a8e68439 100644 --- a/data/hostname/pergamon.softwareheritage.org.yaml +++ b/data/hostname/pergamon.softwareheritage.org.yaml @@ -1,46 +1,78 @@ dns::local_cache: false # Overrides for primary bind server bind::zones::type: master smtp::relay_destinations: - destination: "%{hiera('phabricator::vhost::name')}" route: smtp:[tate.internal.softwareheritage.org] smtp::mynetworks: - 127.0.0.0/8 - "[::ffff:127.0.0.0]/104" - "[::1]/128" - 192.168.100.0/23 - 192.168.200.0/21 networks: private: interface: eth1 address: 192.168.100.29 netmask: 255.255.255.0 gateway: 192.168.100.1 + ups: + - "ip route add 192.168.101.0/24 via 192.168.100.1" + - "ip route add 192.168.200.0/21 via 192.168.100.1" + - "ip route add 192.168.128.0/24 via 192.168.100.125" + - "ip rule add from 192.168.100.29 table private" + - "ip route add 192.168.100.0/24 src 192.168.100.29 dev eth1 table private" + - "ip route add default via 192.168.100.1 dev eth1 table private" + - 'ip route flush cache' + downs: + - "ip route del default via 192.168.100.1 dev eth1 table private" + - "ip route del 192.168.100.0/24 src 192.168.100.29 dev eth1 table private" + - "ip rule del from 192.168.100.29 table private" + - "ip route del 192.168.128.0/24 via 192.168.100.125" + - "ip route del 192.168.200.0/24 via 192.168.100.1" + - "ip route del 192.168.101.0/24 via 192.168.100.1" + - 'ip route flush cache' + default: interface: eth0 address: 128.93.193.29 netmask: 255.255.255.0 gateway: 128.93.193.254 # Set apache MPM to prefork apache::mpm_module: prefork dar::backup::exclude: - srv/softwareheritage/annex/annexroot - - var/log + - var/lib/prometheus icinga2::role: master icinga2::features: - checker - mainlog - notification - statusdata - compatlog - command systemd_journal::role: collector + + +users: + jenkins-push-docs: + uid: 3000 + full_name: Jenkins Documentation Push user + shell: /bin/bash + authorized_keys: + jenkins-push-docs@thyssen: + type: ssh-rsa + key: AAAAB3NzaC1yc2EAAAADAQABAAACAQDWk4WLCNPdSthCdYSDGP7UEIzrBigy2q74ux6OjxSd7SxUKObbsVJeV1MLxIrK3DALQEGZuNUn8hPH6NnyuRBoY+5b7KJ1uV4UAZc4CctxLrrq81cS4x71wU6bzNlZH8DFZa5s0WFnskzg1X5KvyNDj/EdJ8a1TbL/wtj8dPw9odcCw82uqT9Dookvn+yAJ6Lld4MJYy03TfQGCufq2aRRbe/wwNTgL01g3FOuOpaXgmNGGyPpUae290M3+2/slqnHmTDTabnAwDFGfgdX3EIZ2janNJN1j9/5sqDmRQt/cpc5GXfZkpuEIm6+PBj2EWPpHuOUOCAJOQ8u/x2m+v1JJ7qtEmp4sKGqRddUROsWJD73z6XA/p3Xd+nfSrxgnFpW/38upttkqh1OVZshv0+8RijK9Ve5NTU2tIQXmFyYniHUJ8CYpDJug/0pOWNy8Jasqk8jt2Qm5mwR4q9v47PR413KAv+mr/VrhECJKbfoExf3djNYekXtYwD/L45dKg2ogFiZOHgzJSqtoUlIy6RY6ylo1/u3PZY+g3HpcDUYCjNigO8Wwc4ACYIS+DvPRYm0/6+rGl/GoHcgWV4sFKXZCkcGPikL/ECIB6i5AFBKArEYtijN86lhw+dKDEEjQHrURqGMkX2v2TQ37KRSVDY7YoC7Bn+aKWKPGvlR0l9nuw== + +groups: + jenkins-push-docs: + gid: 3000 diff --git a/data/hostname/prado.softwareheritage.org.yaml b/data/hostname/prado.softwareheritage.org.yaml deleted file mode 100644 index d37272dc..00000000 --- a/data/hostname/prado.softwareheritage.org.yaml +++ /dev/null @@ -1,33 +0,0 @@ -dar::backup::exclude: - - srv/softwareheritage/postgres - - srv/softwareheritage/postgres-hdd - -icinga2::host::vars: - load: high - -prometheus::sql::exporter::extra_config: - - name: "SoftwareHeritageScheduler" - interval: '1h' - connections: - - 'postgres://postgres@:5434/softwareheritage-scheduler?sslmode=disable' - queries: - - name: "swh_scheduler_oneshot" - help: "Software Heritage Scheduled Oneshot Tasks" - labels: - - "type_policy_status" - values: - - "count" - query: | - select type ||' '|| status as type_policy_status, count(*) as count - from task - where policy='oneshot' group by 1 order by 1 - - name: "swh_scheduler_recurring" - help: "Software Heritage Scheduled Recurring Tasks" - labels: - - "type_policy_status" - values: - - "count" - query: | - select type ||' '|| status as type_policy_status, count(*) as count - from task - where policy='recurring' group by 1 order by 1 diff --git a/data/hostname/saatchi.internal.softwareheritage.org.yaml b/data/hostname/saatchi.internal.softwareheritage.org.yaml index afa49a7b..25ccd359 100644 --- a/data/hostname/saatchi.internal.softwareheritage.org.yaml +++ b/data/hostname/saatchi.internal.softwareheritage.org.yaml @@ -1,10 +1,4 @@ dar::backup::exclude: - var/lib/rabbitmq swh::deploy::scheduler::remote::backend::listen::host: 0.0.0.0 - -swh::deploy::scheduler::remote::config: - scheduler: - cls: local - args: - scheduling_db: "%{hiera('swh::deploy::scheduler::database')}" diff --git a/data/hostname/scheduler0.internal.staging.swh.network.yaml b/data/hostname/scheduler0.internal.staging.swh.network.yaml new file mode 100644 index 00000000..d872af7f --- /dev/null +++ b/data/hostname/scheduler0.internal.staging.swh.network.yaml @@ -0,0 +1,23 @@ +networks: + default: + interface: eth0 + address: 192.168.128.4 + netmask: 255.255.255.0 + gateway: 192.168.128.1 + +dar::backup::exclude: + - var/lib/rabbitmq + +swh::postgres::service::users: + - swhscheduler + +rabbitmq::enable::guest: true + +swh::deploy::scheduler::remote::backend::listen::host: 0.0.0.0 +swh::deploy::scheduler::task_broker: "amqp://guest:guest@127.0.0.1:5672//" +swh::deploy::scheduler::config::local: &swh_scheduler_local_config + scheduler: + cls: local + args: + db: service=admin-staging-swh-scheduler + diff --git a/data/hostname/somerset.internal.softwareheritage.org.yaml b/data/hostname/somerset.internal.softwareheritage.org.yaml index b2534ba6..c3acbaba 100644 --- a/data/hostname/somerset.internal.softwareheritage.org.yaml +++ b/data/hostname/somerset.internal.softwareheritage.org.yaml @@ -1,26 +1,22 @@ dar::backup::exclude: - srv/softwareheritage/postgres # that's a mirror of the load on the host... icinga2::host::vars: load: high -prometheus::sql::exporter::extra_config: - - name: "SoftwareHeritageIndexerCounters" - interval: '1h' - connections: - - 'postgres://postgres@:5434/softwareheritage-indexer?sslmode=disable' - queries: - - name: "swh_indexer_count" - help: "Indexer count per type" - labels: - - "mimetype" - - "fossology_license" - values: - - "count_mimetype" - - "count_fossology_license" - query: | - select 'mimetype', - (select count(*) from content_mimetype) as count_mimetype, - 'fossology_license', - (select count(*) from content_fossology_license) as count_fossology_license; +prometheus::sql::config_snippets: + - swh-indexer + +pgbouncer::auth_hba_file: /etc/postgresql/11/replica/pg_hba.conf +pgbouncer::listen_addr: 192.168.100.103 +pgbouncer::databases: + # swh + - source_db: "%{hiera('swh::deploy::storage::db::dbname')}" + host: somerset.internal.softwareheritage.org + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::main::port')}" + - source_db: "%{hiera('swh::deploy::indexer::storage::db::dbname')}" + host: somerset.internal.softwareheritage.org + auth_user: "%{hiera('swh::deploy::db::pgbouncer::user::login')}" + port: "%{hiera('swh::deploy::db::secondary::port')}" diff --git a/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml index 0ae97ffc..6259966f 100644 --- a/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml @@ -1,24 +1,35 @@ swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 48 swh::deploy::storage::backend::max_requests: 1000 swh::deploy::storage::backend::max_requests_jitter: 100 swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 8 - swh::deploy::storage::db::host: dbreplica0.euwest.azure.internal.softwareheritage.org swh::deploy::storage::db::port: 5433 swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::db::host: dbreplica1.euwest.azure.internal.softwareheritage.org swh::deploy::indexer::storage::db::port: 5432 -swh::deploy::indexer::storage::config: - indexer_storage: + +swh::remote_service::objstorage::config::azure_readonly_with_fallback: &swh_azure_readonly_with_fallback + cls: multiplexer + args: + objstorages: + - "%{alias('swh::remote_service::objstorage::config::azure::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::uffizi::readonly')}" + +swh::deploy::objstorage::config: + objstorage: + <<: *swh_azure_readonly_with_fallback + +swh::deploy::storage::config: + storage: cls: local args: - db: "host=%{hiera('swh::deploy::indexer::storage::db::host')} port=%{hiera('swh::deploy::indexer::storage::db::port')} user=%{hiera('swh::deploy::indexer::storage::db::user')} dbname=%{hiera('swh::deploy::indexer::storage::db::dbname')} password=%{hiera('swh::deploy::indexer::storage::db::password')}" - -swh::deploy::objstorage::config: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" + db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" + objstorage: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" swh::deploy::worker::instances: [] diff --git a/data/hostname/storage0.internal.staging.swh.network.yaml b/data/hostname/storage0.internal.staging.swh.network.yaml new file mode 100644 index 00000000..ecdeac80 --- /dev/null +++ b/data/hostname/storage0.internal.staging.swh.network.yaml @@ -0,0 +1,42 @@ +networks: + default: + interface: eth0 + address: 192.168.128.2 + netmask: 255.255.255.0 + gateway: 192.168.128.1 + +swh::postgres::service::users: + - swhstorage + +# open objstorage api +swh::deploy::objstorage::backend::listen::host: 0.0.0.0 +swh::deploy::objstorage::backend::workers: 4 +swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" +swh::deploy::objstorage::slicing: 0:1/1:5 + +# Deploy the storage server as a public resource +swh::deploy::storage::backend::listen::host: 0.0.0.0 +swh::deploy::storage::backend::workers: 4 +swh::deploy::storage::backend::max_requests: 100 +swh::deploy::storage::backend::max_requests_jitter: 10 +swh::deploy::storage::config: + storage: + cls: local + args: + db: service=admin-staging-swh + objstorage: + cls: pathslicing + args: + root: "%{hiera('swh::deploy::storage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + +# Deploy the indexer storage server as a public resource +swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 +swh::deploy::indexer::storage::backend::workers: 4 +swh::deploy::indexer::storage::config: + indexer_storage: + cls: local + args: + db: service=admin-staging-swh-indexer + +nginx::worker_processes: 4 diff --git a/data/hostname/thyssen.internal.softwareheritage.org.yaml b/data/hostname/thyssen.internal.softwareheritage.org.yaml index 2047dd0e..9e0a9ef6 100644 --- a/data/hostname/thyssen.internal.softwareheritage.org.yaml +++ b/data/hostname/thyssen.internal.softwareheritage.org.yaml @@ -1,7 +1,10 @@ --- # Add ddouard as jenkins admin users: ddouard: groups: - sudo + +dar::backup::exclude: + - /var/lib/docker/ diff --git a/data/hostname/uffizi.softwareheritage.org.yaml b/data/hostname/uffizi.softwareheritage.org.yaml index cfd49c1d..dea97c57 100644 --- a/data/hostname/uffizi.softwareheritage.org.yaml +++ b/data/hostname/uffizi.softwareheritage.org.yaml @@ -1,106 +1,117 @@ dar::backup::exclude: + - annex + - data + - mnt - srv/softwareheritage/annex + - tank # Deploy the storage server as a public resource swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 64 swh::deploy::storage::backend::max_requests: 500 swh::deploy::storage::backend::max_requests_jitter: 50 swh::deploy::storage::config: storage: cls: local args: db: "host=%{hiera('swh::deploy::storage::db::host')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" objstorage: - cls: striping + cls: multiplexer args: objstorages: - cls: pathslicing args: root: "%{hiera('swh::deploy::storage::directory')}" slicing: "0:2/2:4/4:6" - - "%{alias('swh::remote_service::objstorage::config::banco')}" + - "%{alias('swh::remote_service::objstorage::config::azure')}" + journal_writer: + cls: kafka + args: + brokers: "%{alias('swh::deploy::journal::brokers')}" + prefix: "%{alias('swh::deploy::journal::prefix')}" + client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::backend::workers: 32 swh::deploy::indexer::storage::config: indexer_storage: cls: local args: db: "host=%{hiera('swh::deploy::indexer::storage::db::host')} port=%{hiera('swh::deploy::indexer::storage::db::port')} user=%{hiera('swh::deploy::indexer::storage::db::user')} dbname=%{hiera('swh::deploy::indexer::storage::db::dbname')} password=%{hiera('swh::deploy::indexer::storage::db::password')}" # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 16 swh::deploy::objstorage::ceph::keyring: /etc/softwareheritage/objstorage/ceph-keyring swh::deploy::objstorage::ceph::rados_id: swh-contents-rw ceph::client_keyrings: '/etc/softwareheritage/objstorage/ceph-keyring': owner: root group: swhstorage mode: '0640' keys: - "%{alias('swh::deploy::objstorage::ceph::rados_id')}" -swh::deploy::worker::instances: - - swh_storage_archiver - icinga2::host::vars: load: high disks: disk /srv/storage/0: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/1: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/2: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/3: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/4: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/5: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/6: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/7: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/8: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/9: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/a: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/b: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/c: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/d: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/e: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/f: disk_wfree: '5%' disk_cfree: '3%' disk /srv/storage/space/mirrors/gitorious.org/mnt: disk_wfree: '5%' disk_cfree: '3%' nginx::worker_processes: 16 + +swh::apt_config::enable_non_free: true +packages: + - intel-microcode diff --git a/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml new file mode 100644 index 00000000..800467ed --- /dev/null +++ b/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml @@ -0,0 +1,15 @@ +# open vault api +swh::deploy::vault::backend::listen::host: 0.0.0.0 + +# vault's cache backend is an azure objstorage +swh::deploy::vault::config::cache: + cls: azure + args: + account_name: "%{hiera('swh::azure::credentials::vaultstorage::account')}" + api_secret_key: "%{hiera('swh::azure::credentials::vaultstorage::key')}" + container_name: contents + +swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" + +# No workers +swh::deploy::worker::instances: [] diff --git a/data/hostname/vault.internal.staging.swh.network.yaml b/data/hostname/vault.internal.staging.swh.network.yaml new file mode 100644 index 00000000..adfd0f67 --- /dev/null +++ b/data/hostname/vault.internal.staging.swh.network.yaml @@ -0,0 +1,9 @@ +# open vault api +swh::deploy::vault::backend::listen::host: 0.0.0.0 + +swh::deploy::vault::backend::workers: 4 +swh::deploy::vault::backend::reload_mercy: 3600 +swh::deploy::vault::backend::http_keepalive: 5 +swh::deploy::vault::backend::http_timeout: 3600 +swh::deploy::vault::backend::max_requests: 10000 +swh::deploy::vault::backend::max_requests_jitter: 1000 diff --git a/data/hostname/webapp.internal.staging.swh.network.yaml b/data/hostname/webapp.internal.staging.swh.network.yaml new file mode 100644 index 00000000..d62ac6a7 --- /dev/null +++ b/data/hostname/webapp.internal.staging.swh.network.yaml @@ -0,0 +1,84 @@ +networks: + default: + interface: eth0 + address: 192.168.128.8 + netmask: 255.255.255.0 + gateway: 192.168.128.1 + +hitch::frontend: "[*]:443" +hitch::proxy_support: true + +varnish::http_port: 80 +apache::http_port: 9080 + +# Disable default vhost on port 80 +apache::default_vhost: false + +swh::deploy::webapp::vhost::name: webapp.internal.staging.swh.network +swh::deploy::webapp::vhost::aliases: + - webapp.staging.swh.network + - webapp.staging.softwareheritage.org + +swh::deploy::webapp::config::allowed_hosts: + - webapp.internal.staging.swh.network + - webapp.staging.swh.network + - webapp.staging.softwareheritage.org + +swh::deploy::webapp::backend::workers: 16 +swh::deploy::webapp::backend::http_keepalive: 5 +swh::deploy::webapp::backend::http_timeout: 3600 +swh::deploy::webapp::backend::reload_mercy: 3600 + +# in private data: +# deposit_basic_auth_swhworker_username +# deposit_basic_auth_swhworker_password +swh::deploy::webapp::config: + storage: "%{alias('swh::remote_service::storage::config')}" + vault: "%{alias('swh::remote_service::vault::config::writable')}" + indexer_storage: "%{alias('swh::remote_service::indexer::config')}" + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + log_dir: "%{hiera('swh::deploy::webapp::conf::log_dir')}" + secret_key: "%{hiera('swh::deploy::webapp::conf::secret_key')}" + content_display_max_size: 1048576 + throttling: + cache_uri: "%{hiera('memcached::server::bind')}:%{hiera('memcached::server::port')}" + scopes: + swh_api: + limiter_rate: + default: 120/h + exempted_networks: + - 127.0.0.0/8 + - 192.168.100.0/23 + - 129.168.128.0/24 + swh_api_origin_visit_latest: + # This endpoint gets called a lot (by default, up to 70 times + # per origin search), so it deserves a much higher rate-limit + # than the rest of the API. + limiter_rate: + default: 700/m + exempted_networks: + - 127.0.0.0/8 + - 192.168.100.0/23 + - 192.168.128.0/24 + swh_vault_cooking: + limiter_rate: + default: 120/h + GET: 60/m + exempted_networks: + - 127.0.0.0/8 + - 192.168.100.0/23 + - 192.168.128.0/24 + swh_save_origin: + limiter_rate: + default: 120/h + POST: 10/h + exempted_networks: + - 127.0.0.0/8 + - 192.168.100.0/23 + - 129.168.128.0/24 + allowed_hosts: "%{alias('swh::deploy::webapp::config::allowed_hosts')}" + production_db: "%{hiera('swh::deploy::webapp::production_db')}" + deposit: + private_api_url: https://deposit.internal.swh.staging/1/private/ + private_api_user: "%{hiera('deposit_basic_auth_swhworker_username')}" + private_api_password: "%{hiera('deposit_basic_auth_swhworker_password')}" diff --git a/data/hostname/webapp0.softwareheritage.org.yaml b/data/hostname/webapp0.softwareheritage.org.yaml index 1b0469cc..4cb09619 100644 --- a/data/hostname/webapp0.softwareheritage.org.yaml +++ b/data/hostname/webapp0.softwareheritage.org.yaml @@ -1,23 +1,15 @@ hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disable default vhost on port 80 apache::default_vhost: false swh::deploy::webapp::vhost::name: webapp0.softwareheritage.org swh::deploy::webapp::vhost::aliases: - webapp0.softwareheritage.org -swh::deploy::storage::config: - storage: "%{alias('swh::remote_service::storage::config::azure')}" - -swh::deploy::indexer::storage::config: - indexer_storage: "%{alias('swh::indexer::storage::config::azure')}" - swh::deploy::webapp::config::allowed_hosts: - webapp0.softwareheritage.org - -swh::deploy::webapp::indexer_storage: "%{alias('swh::indexer::storage::config::azure')}" diff --git a/data/hostname/worker0.internal.staging.swh.network.yaml b/data/hostname/worker0.internal.staging.swh.network.yaml new file mode 100644 index 00000000..00c851b8 --- /dev/null +++ b/data/hostname/worker0.internal.staging.swh.network.yaml @@ -0,0 +1,6 @@ +networks: + default: + interface: eth0 + address: 192.168.128.5 + netmask: 255.255.255.0 + gateway: 192.168.128.1 diff --git a/data/hostname/worker1.internal.staging.swh.network.yaml b/data/hostname/worker1.internal.staging.swh.network.yaml new file mode 100644 index 00000000..6551b050 --- /dev/null +++ b/data/hostname/worker1.internal.staging.swh.network.yaml @@ -0,0 +1,6 @@ +networks: + default: + interface: eth0 + address: 192.168.128.6 + netmask: 255.255.255.0 + gateway: 192.168.128.1 diff --git a/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml index 38ba9c7f..fabf315b 100644 --- a/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml @@ -1,3 +1,3 @@ --- swh::deploy::worker::instances: - - swh_vault_cooker + - vault_cooker diff --git a/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml index 38ba9c7f..fabf315b 100644 --- a/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml @@ -1,3 +1,3 @@ --- swh::deploy::worker::instances: - - swh_vault_cooker + - vault_cooker diff --git a/data/location/azure_euwest.yaml b/data/location/azure_euwest.yaml index c91a2d2f..8d6d3815 100644 --- a/data/location/azure_euwest.yaml +++ b/data/location/azure_euwest.yaml @@ -1,24 +1,25 @@ --- dns::forward_zones: 'internal.softwareheritage.org.': - 192.168.200.22 '100.168.192.in-addr.arpa.': - 192.168.200.22 '101.168.192.in-addr.arpa.': - 192.168.200.22 -dns::forwarders: - - 168.63.129.16 -dns::forwarder_insecure: true +dns::forwarders: [] +dns::forwarder_insecure: false internal_network: 192.168.200.0/21 swh::deploy::worker::instances: - - swh_indexer_rehash - - swh_indexer_mimetype - - swh_indexer_fossology_license - - swh_indexer_orchestrator - - swh_indexer_orchestrator_text + - indexer_content_mimetype + - indexer_fossology_license + - indexer_origin_intrinsic_metadata swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" -swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" +swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::azure')}" + +# Add extra needed package to run on azure +packages: + - python3-swh.objstorage.cloud diff --git a/data/location/sesi_rocquencourt.yaml b/data/location/sesi_rocquencourt.yaml index fbfdbf51..6ae0b2ae 100644 --- a/data/location/sesi_rocquencourt.yaml +++ b/data/location/sesi_rocquencourt.yaml @@ -1,34 +1,32 @@ --- dns::forward_zones: 'internal.softwareheritage.org.': - 192.168.100.29 '100.168.192.in-addr.arpa.': - 192.168.100.29 '101.168.192.in-addr.arpa.': - 192.168.100.29 dns::forwarders: - 193.51.196.130 - 193.51.196.131 dns::forwarder_insecure: true ntp::servers: - sesi-ntp1.inria.fr - sesi-ntp2.inria.fr internal_network: 192.168.100.0/24 smtp::relayhost: '[smtp.inria.fr]' swh::deploy::worker::instances: - - swh_lister_debian - - swh_lister_github - - swh_lister_gitlab - - swh_lister_pypi - - swh_loader_debian - - swh_loader_deposit - - swh_loader_git - - swh_loader_git_disk - - swh_loader_mercurial - - swh_loader_pypi - - swh_loader_svn + - checker_deposit + - lister + - loader_debian + - loader_deposit + - loader_git + - loader_mercurial + - loader_npm + - loader_pypi + - loader_svn diff --git a/data/location/sesi_rocquencourt_staging.yaml b/data/location/sesi_rocquencourt_staging.yaml new file mode 100644 index 00000000..45be4083 --- /dev/null +++ b/data/location/sesi_rocquencourt_staging.yaml @@ -0,0 +1,109 @@ +--- +dns::local_cache: false +dns::nameservers: + - 192.168.100.29 +dns::search_domains: + - internal.staging.swh.network + +dns::forward_zones: + 'internal.softwareheritage.org.': + - 192.168.100.29 + '100.168.192.in-addr.arpa.': + - 192.168.100.29 + '101.168.192.in-addr.arpa.': + - 192.168.100.29 + 'internal.staging.swh.network': + - 192.168.100.29 + '128.168.192.in-addr.arpa.': + - 192.168.100.29 + +dns::forwarders: + - 193.51.196.130 + - 193.51.196.131 +dns::forwarder_insecure: true + +ntp::servers: + - sesi-ntp1.inria.fr + - sesi-ntp2.inria.fr + +internal_network: 192.168.128.0/24 + +smtp::relayhost: '[smtp.inria.fr]' + +swh::deploy::storage::db::host: db0.internal.staging.swh.network +swh::deploy::storage::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::storage::db::user: swh +swh::deploy::storage::db::dbname: swh + +swh::deploy::indexer::storage::db::host: db0.internal.staging.swh.network +swh::deploy::indexer::storage::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::indexer::storage::db::user: swh-indexer +swh::deploy::indexer::storage::db::dbname: swh-indexer + +swh::deploy::scheduler::db::host: db0.internal.staging.swh.network +swh::deploy::scheduler::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::scheduler::db::dbname: swh-scheduler +swh::deploy::scheduler::db::user: swh-scheduler + +swh::deploy::deposit::db::host: deposit.internal.staging.swh.network +swh::deploy::deposit::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::deposit::db::dbuser: swh-deposit +swh::deploy::deposit::db::dbname: swh-deposit + +swh::deploy::vault::db::host: db0.internal.staging.swh.network +swh::deploy::vault::db::port: "%{alias('swh::deploy::db::pgbouncer::port')}" +swh::deploy::vault::db::user: swh-vault +swh::deploy::vault::db::dbname: swh-vault + +swh::deploy::worker::instances: + - loader_git + - vault_cooker + +#### Rabbitmq instance to use +# swh::deploy::worker::task_broker::password in private data +swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672//" + +#### Storage/Indexer/Vault/Scheduler services to use in staging area + +swh::remote_service::storage::config::storage0: + cls: remote + args: + url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" +swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" +swh::remote_service::storage::config::writable: "%{alias('swh::remote_service::storage::config::storage0')}" + +swh::remote_service::vault::config::vault0: + cls: remote + args: + url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" +swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" +swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" + +swh::remote_service::indexer::config::storage0: + cls: remote + args: + url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" +swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" +swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" + +swh::remote_service::scheduler::config::scheduler0: + cls: remote + args: + url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" + +swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" +swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" + +swh::deploy::worker::loader_git::config: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + save_data: false + directory_packet_size: 100 + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.git.tasks + task_queues: + - swh.loader.git.tasks.UpdateGitRepository + - swh.loader.git.tasks.LoadDiskGitRepository + - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository + diff --git a/data/location/unibo.yaml b/data/location/unibo.yaml index 855412bd..b4997ea5 100644 --- a/data/location/unibo.yaml +++ b/data/location/unibo.yaml @@ -1,13 +1,21 @@ --- +dns::forward_zones: + 'internal.softwareheritage.org.': + - 192.168.100.29 + '100.168.192.in-addr.arpa.': + - 192.168.100.29 + '101.168.192.in-addr.arpa.': + - 192.168.100.29 + dns::forwarders: - 137.204.25.71 - 137.204.25.213 - 137.204.25.77 dns::forwarder_insecure: true internal_network: 192.168.101.0/24 swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" dar::backup::exclude: - srv/softwareheritage diff --git a/hiera.yaml b/hiera.yaml index 4dc952bd..4bb336a5 100644 --- a/hiera.yaml +++ b/hiera.yaml @@ -1,16 +1,18 @@ --- version: 4 datadir: data hierarchy: - name: "private/hostname/%{trusted.certname}" backend: yaml - name: "private/location/%{::location}" backend: yaml - name: "private/defaults" backend: yaml - name: "hostname/%{trusted.certname}" backend: yaml - name: "location/%{::location}" backend: yaml + - name: defaults_security + backend: yaml - name: defaults backend: yaml diff --git a/manifests/site.pp b/manifests/site.pp index f9780190..dbd5d869 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,120 +1,157 @@ -node 'louvre.softwareheritage.org' { - include role::swh_hypervisor_master - include role::swh_lsi_storage_adapter +node 'louvre.internal.softwareheritage.org' { + include role::swh_server } -node 'beaubourg.softwareheritage.org', 'orsay.softwareheritage.org' { +node /^(orsay|beaubourg|hypervisor\d+)\.(internal\.)?softwareheritage\.org$/ +{ include role::swh_hypervisor - include role::swh_lsi_storage_adapter } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin + include profile::export_archive_counters } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_api - include role::swh_apache_log_archiver } node 'webapp0.softwareheritage.org' { - include role::swh_api_azure + include role::swh_base_api } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler } -node /^(prado|somerset).(internal.)?softwareheritage.org$/ { +node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database + include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup - include role::swh_lsi_storage_adapter + include role::postgresql_backup } -node - 'esnode1.internal.softwareheritage.org', - 'esnode2.internal.softwareheritage.org', - 'esnode3.internal.softwareheritage.org' -{ +node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } -node /^(unibo-test|orangeriedev).(internal.)?softwareheritage.org$/ { +node /^(unibo-test).(internal.)?softwareheritage.org$/ { include role::swh_vault_test } -node /^(unibo-prod|orangerie).(internal.)?softwareheritage.org$/ { +node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } -node /^(uffizi|storage\d+\.[^.]+\.azure).(internal.)?softwareheritage.org$/ { +node /^uffizi\.(internal\.)?softwareheritage\.org$/ { + include role::swh_storage_baremetal +} + +node /^storage\d+\.[^.]+\.azure\.internal\.softwareheritage\.org$/ { include role::swh_storage } node /^getty.(internal.)?softwareheritage.org$/ { include role::swh_eventlog } -node 'worker08.softwareheritage.org' { - include role::swh_worker_inria_miracle -} - node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } +node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { + include role::swh_ci_agent_debian +} + node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'munin0.internal.softwareheritage.org' { include role::swh_munin_master } -node - 'giverny.softwareheritage.org', - 'petit-palais.softwareheritage.org', - 'grand-palais.softwareheritage.org', - 'ddouard-desktop.internal.softwareheritage.org' { +node 'giverny.softwareheritage.org' { include role::swh_desktop } +node 'db0.internal.staging.swh.network' { + include role::swh_base_database + include profile::postgresql::server + include profile::pgbouncer + include ::profile::devel::postgres +} + +node 'scheduler0.internal.staging.swh.network' { + include role::swh_scheduler + include ::profile::devel::postgres +} + +node 'gateway.internal.staging.swh.network' { + include role::swh_gateway +} + +node 'storage0.internal.staging.swh.network' { + include role::swh_base_storage + include ::profile::devel::postgres +} + +node /^worker\d\.internal\.staging\.swh\.network$/ { + include role::swh_worker_inria +} + +node 'webapp.internal.staging.swh.network' { + include role::swh_base_api + include profile::network +} + +node 'deposit.internal.staging.swh.network' { + include role::swh_deposit + include profile::postgresql::server + include profile::pgbouncer + include ::profile::devel::postgres +} + +node 'vault.internal.staging.swh.network' { + include role::swh_vault +} + node default { - include role::swh_server + include role::swh_base include profile::puppet::agent } diff --git a/site-modules/profile/files/icinga2/plugins/check_newest_file_age b/site-modules/profile/files/icinga2/plugins/check_newest_file_age new file mode 100755 index 00000000..9aa2ab8f --- /dev/null +++ b/site-modules/profile/files/icinga2/plugins/check_newest_file_age @@ -0,0 +1,343 @@ +#! /bin/sh +# +# Newest file in a directory plugin for Nagios. +# Written by Chad Phillips (chad@apartmentlines.com) +# Last Modified: 2016-04-21 + +PROGPATH=`dirname $0` + +STATE_OK=0 +STATE_WARNING=1 +STATE_CRITICAL=2 +STATE_UNKNOWN=3 +STATE_DEPENDENT=4 + +print_usage() { + echo " +Usage: check_newest_file_age --dirs | -d [-w ] [-c ] [-W] [-C] [-t ] [-V] [--check-dirs] [--base-dir ] +Usage: check_newest_file_age --help | -h + +Description: + +This plugin pulls the most recently created file in each specified directory, +and checks it's created time against the current time. If the maximum age of +the file is exceeded, a warning/critical message is returned as appropriate. + +This is useful for examining backup directories for freshness. + +Tested to work on Linux/FreeBSD/OS X. + +The following arguments are accepted: + + --dirs | -d A space separated list of directories to examine. Each + directory will be checked for the newest created file in that + directory. + + -w (Optional) Generate a warning message if the last created + file is older than this value. Defaults to 26 hours. + + -c (Optional) Generate a critical message if the last created + file is older than this value. Defaults to 52 hours. + + -W (Optional) If set, a warning message will be returned if the + specified directory doesn't exist, or there are no checkable + files in the specified directory. + + -C (Optional) If set, a critical message will be returned if the + specified directory doesn't exist, or there are no checkable + files in the specified directory. + + -t (Optional) The time unit used for the -w and -c values. Must + be one of: seconds, minutes, hours, days. Defaults to hours. + + -V (Optional) Output verbose information about all checked + directories. Default is only to print verbose information + for directories with non-OK states. + + --check-dirs (Optional) If set, directories inside the specified directory + will also be checked for their creation time. Note that this + check is not recursive. Without this option, only real files + inside the specified directory will be checked. + + --base-dir (Optional) If set, this path will be prepended to all + checked directories. + + --help | -h Print this help and exit. + +Examples: + +Generate a warning if the newest file in /backups is more than 26 hours old, +and a critical if it's more than 52 hours old... + + check_newest_file_age -d \"/backups\" + +Generate a warning if the newest file in /backups/bill or /backups/dave is more +than one week old, or a critical if it's more than two weeks old... + + check_newest_file_age -d \"/backups/bill /backups/dave\" -w 7 -c 14 -t days + +Caveats: + +Although multiple directories can be specified, only one set of +warning/critical times can be supplied. + +Linux doesn't seem to have an easy way to check file/directory creation time, +so file/directory last modification time is used instead. +" +} + +print_help() { + print_usage + echo "Newest file in a directory plugin for Nagios." + echo "" +} + +# Sets the exit status for the plugin. This is done in such a way that the +# status can only go in one direction: OK -> WARNING -> CRITICAL. +set_exit_status() { + new_status=$1 + # Nothing needs to be done if the state is already critical, so exclude + # that case. + case $exitstatus + in + $STATE_WARNING) + # Only upgrade from warning to critical. + if [ "$new_status" = "$STATE_CRITICAL" ]; then + exitstatus=$new_status; + fi + ;; + $STATE_OK) + # Always update state if current state is OK. + exitstatus=$new_status; + ;; + esac +} + +# Make sure the correct number of command line +# arguments have been supplied +if [ $# -lt 1 ]; then + print_usage + exit $STATE_UNKNOWN +fi + +# Defaults. +exitstatus=$STATE_OK +warning=26 +critical=52 +time_unit=hours +verbose= +on_empty=$STATE_OK +check_dirs= +base_dir= + +# Grab the command line arguments. +while test -n "$1"; do + case "$1" in + --help) + print_help + exit $STATE_OK + ;; + -h) + print_help + exit $STATE_OK + ;; + --dirs) + dirs=$2 + shift + ;; + -d) + dirs=$2 + shift + ;; + -w) + warning=$2 + shift + ;; + -c) + critical=$2 + shift + ;; + -W) + on_empty=$STATE_WARNING + ;; + -C) + on_empty=$STATE_CRITICAL + ;; + -t) + time_unit=$2 + shift + ;; + -V) + verbose=1 + ;; + --check-dirs) + check_dirs=1 + ;; + --base-dir) + base_dir=$2 + shift + ;; + -x) + exitstatus=$2 + shift + ;; + --exitstatus) + exitstatus=$2 + shift + ;; + *) + echo "Unknown argument: $1" + print_usage + exit $STATE_UNKNOWN + ;; + esac + shift +done + +if [ ! "$dirs" ]; then + echo "No directories provided." + exit $STATE_UNKNOWN +fi + +if [ `echo "$warning" | grep [^0-9]` ] || [ ! "$warning" ]; then + echo "Warning value must be a number." + exit $STATE_UNKNOWN +fi + +if [ `echo "$critical" | grep [^0-9]` ] || [ ! "$critical" ]; then + echo "Critical value must be a number." + exit $STATE_UNKNOWN +fi + +if [ ! `echo "$time_unit" | grep "seconds\|minutes\|hours\|days"` ]; then + echo "Time unit must be one of: seconds, minutes, hours, days." + exit $STATE_UNKNOWN +fi + +if [ "$warning" -ge "$critical" ]; then + echo "Critical time must be greater than warning time." + exit $STATE_UNKNOWN +fi + +case $time_unit +in + days) + multiplier=86400; + abbreviation="days"; + ;; + hours) + multiplier=3600; + abbreviation="hrs"; + ;; + minutes) + multiplier=60; + abbreviation="mins"; + ;; + *) + multiplier=1 + abbreviation="secs"; + ;; +esac + +# Starting values. +DIR_COUNT=0 +OK_FILE_COUNT=0 +OUTPUT= +CURRENT_TIME=`date +%s` +OS_DISTRO=`uname -s` + +# Loop through each provided directory. +for dir in $dirs +do + check_file= + DIR_COUNT=$(($DIR_COUNT + 1)) + + # Check if dir exists. + full_path=${base_dir}${dir} + if [ -d "$full_path" ]; then + file_list=`ls -t $full_path` + # Cycle through files, looking for a checkable file. + for next_file in $file_list + do + next_filepath=$full_path/$next_file + if [ "$check_dirs" ]; then + # Check if it's a file or directory. + if [ -f "$next_filepath" ] || [ -d "$next_filepath" ]; then + check_file=1 + fi + else + # Check if it's a file. + if [ -f "$next_filepath" ]; then + check_file=1 + fi + fi + if [ "$check_file" ]; then + # stat doesn't work the same on Linux/Solaris vs. FreeBSD/Darwin, so + # make adjustments here. + if [ "$OS_DISTRO" = "Linux" ] || [ "$OS_DISTRO" = "SunOS" ]; then + st_ctime=`stat --printf=%Y ${next_filepath}` + else + eval $(stat -s ${next_filepath}) + fi + + FILE_AGE=$(($CURRENT_TIME - $st_ctime)) + FILE_AGE_UNITS=$(($FILE_AGE / $multiplier)) + MAX_WARN_AGE=$(($warning * $multiplier)) + MAX_CRIT_AGE=$(($critical * $multiplier)) + if [ $FILE_AGE -gt $MAX_CRIT_AGE ]; then + OUTPUT="$OUTPUT ${dir}: ${FILE_AGE_UNITS}${abbreviation}" + set_exit_status $STATE_CRITICAL + elif [ $FILE_AGE -gt $MAX_WARN_AGE ]; then + OUTPUT="$OUTPUT ${dir}: ${FILE_AGE_UNITS}${abbreviation}" + set_exit_status $STATE_WARNING + else + OK_FILE_COUNT=$(($OK_FILE_COUNT + 1)) + if [ "$verbose" ]; then + OUTPUT="$OUTPUT ${dir}: ${FILE_AGE_UNITS}${abbreviation}" + fi + fi + break + fi + done + # Check here to see if any files got tested in the directory. + if [ ! "$check_file" ]; then + set_exit_status $on_empty + OUTPUT="$OUTPUT ${dir}: No files" + # If empty is an OK state, then increment the ok file count. + if [ "$on_empty" = "$STATE_OK" ]; then + OK_FILE_COUNT=$(($OK_FILE_COUNT + 1)) + fi + fi + else + set_exit_status $on_empty + OUTPUT="$OUTPUT ${dir}: Does not exist" + fi +done + +case $exitstatus +in + $STATE_CRITICAL) + exit_message="CRITICAL"; + ;; + $STATE_WARNING) + exit_message="WARNING"; + ;; + $STATE_OK) + exit_message="OK"; + ;; + *) + exitstatus=$STATE_UNKNOWN; + exit_message="UNKNOWN"; + ;; +esac + +exit_message="${exit_message}: ${OK_FILE_COUNT}/${DIR_COUNT}" + +if [ "$OUTPUT" ]; then + exit_message="${exit_message} --${OUTPUT}" +fi + +echo "$exit_message" +exit $exitstatus + + diff --git a/site-modules/profile/files/kafka/jmx_exporter.yml b/site-modules/profile/files/kafka/jmx_exporter.yml new file mode 100644 index 00000000..b1e75b5c --- /dev/null +++ b/site-modules/profile/files/kafka/jmx_exporter.yml @@ -0,0 +1,88 @@ +# Fetched from https://github.com/prometheus/jmx_exporter example_configs/kafka-2_0_0.yml +# at commit 96d8a72e2bc71575b2fed576a9e8b194ffa9777d + +lowercaseOutputName: true + +rules: +# Special cases and very specific rules +- pattern : kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + topic: "$4" + partition: "$5" +- pattern : kafka.server<>Value + name: kafka_server_$1_$2 + type: GAUGE + labels: + clientId: "$3" + broker: "$4:$5" + +# Generic per-second counters with 0-2 key/value pairs +- pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" +- pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + labels: + "$4": "$5" +- pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_total + type: COUNTER + +# Generic gauges with 0-2 key/value pairs +- pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" +- pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" +- pattern: kafka.(\w+)<>Value + name: kafka_$1_$2_$3 + type: GAUGE + +# Emulate Prometheus 'Summary' metrics for the exported 'Histogram's. +# +# Note that these are missing the '_sum' metric! +- pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" + "$6": "$7" +- pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + "$6": "$7" + quantile: "0.$8" +- pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER + labels: + "$4": "$5" +- pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + "$4": "$5" + quantile: "0.$6" +- pattern: kafka.(\w+)<>Count + name: kafka_$1_$2_$3_count + type: COUNTER +- pattern: kafka.(\w+)<>(\d+)thPercentile + name: kafka_$1_$2_$3 + type: GAUGE + labels: + quantile: "0.$4" diff --git a/site-modules/profile/files/munin/stats_export/export-rrd b/site-modules/profile/files/munin/stats_export/export-rrd deleted file mode 100755 index 78c44c9e..00000000 --- a/site-modules/profile/files/munin/stats_export/export-rrd +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 - -"""Script to execute the export of softwareheritage's rrds data. - -""" - -import click -import json -import os -import subprocess - - -DIRPATH='/var/lib/munin/softwareheritage.org/' -FILENAME_PATTERN="prado.softwareheritage.org-softwareheritage_objects_softwareheritage-###-g.rrd" -# The data source used at rrd creation time -DS=42 - -ENTITIES=[ - "content", - "origin", - "revision", - # "directory_entry_dir", - # "directory_entry_file", - # "directory_entry_rev", - # "directory", - # "entity", - # "occurrence_history", - # "person", - # "project", - # "release", - # "revision_history", - # "skipped_content", - # "visit", -] - -def compute_cmd(dirpath, - start, - step=86400): - """Compute the command to execute to retrieve the needed data. - - Returns: - The command as string. - - """ - cmd = ['rrdtool', 'xport', '--json', '--start', str(start), '--end', 'now-1d', - '--step', str(step)] - for entity in ENTITIES: - filename = FILENAME_PATTERN.replace('###', entity) - filepath = os.path.join(dirpath, filename) - - if os.path.exists(filepath): - cmd.extend(['DEF:out-%s1=%s:%s:AVERAGE' % (entity, filepath, DS), - 'XPORT:out-%s1:%s' % (entity, entity)]) - - return cmd - - -def retrieve_json(cmd): - """Given the cmd command, execute and returns the right json format. - - Args: - cmd: the command to execute to retrieve the desired json. - - Returns: - The desired result as json string. - """ - cmdpipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) - data = b'' - while True: - line = cmdpipe.stdout.readline() - if not line: - break - # Hack: the json output is not well-formed... - line = line.replace(b'\'', b'"') - line = line.replace(b'about: ', b'"about": ') - line = line.replace(b'meta:', b'"meta": ') - data += line - - cmdpipe.stdout.close() - return json.loads(data.decode('utf-8')) - - -def prepare_data(data): - """Prepare the data with x,y coordinate. - - x is the time, y is the actual value. - """ - # javascript has a ratio of 1000... - step = data['meta']['step'] * 1000 # nb of milliseconds - start_ts = data['meta']['start'] * 1000 # starting ts - - legends = data['meta']['legend'] - - # The legends, something like - # ["content-avg", "content-min", "content-max", "directory_entry_dir-avg", ...] - r = {} - day_ts = start_ts - for day, values in enumerate(data['data']): - day_ts += step - for col, value in enumerate(values): - if value is None: - continue - legend_col = legends[col] - l = r.get(legend_col, []) - l.append((day_ts, value)) - r[legend_col] = l - - return r - - -@click.command() -@click.option('--dirpath', default=DIRPATH, help="Default path to look for rrd files.") -@click.option('--start', default=1434499200, help="Default starting timestamp") # Default to 2015-05-12T16:51:25Z -@click.option('--step', default=86400, help="Compute the data step (default to 86400).") -def main(dirpath, start, step): - - # Delegate the execution to the system - run_cmd = compute_cmd(dirpath, start, step) - data = retrieve_json(run_cmd) - - # Format data - data = prepare_data(data) - - print(json.dumps(data)) - - -if __name__ == '__main__': - main() diff --git a/site-modules/profile/files/prometheus/sql/README.md b/site-modules/profile/files/prometheus/sql/README.md new file mode 100644 index 00000000..4fa082b2 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/README.md @@ -0,0 +1,10 @@ +Elephant Shed is an open source project, developed and maintained by credativ. +The Elephant Shed itself is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html). + +The following files have been pulled from commit 8cf0db0b225623adec291019405dca8a00b11e1c of https://github.com/credativ/elephant-shed/ + + - update-prometheus-sql-exporter-config + - config/activity.yml + - config/queries.yml + - config/replication.yml + - config/wal.yml diff --git a/site-modules/profile/files/prometheus/sql/config/activity.yml b/site-modules/profile/files/prometheus/sql/config/activity.yml new file mode 100644 index 00000000..dcd56e01 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/activity.yml @@ -0,0 +1,88 @@ +# queries run once per cluster (via the 'postgres' database) + +- name: "running_queries" + help: "number of running queries" + scope: cluster + labels: + - "datname" + - "usename" + values: + - "count" + query: >- + SELECT datname::text, usename::text, COUNT(*)::float AS count + FROM pg_stat_activity + WHERE NOT datname ~ '^template(0|1)$' + GROUP BY datname, usename + +- name: "pg_stat_activity" + help: "running backends by database and state" + scope: cluster + min_version: 9.2 + max_version: 9.5 + labels: + - "datname" + - "state" + values: + - "count" + - "max_tx_duration" + query: >- + SELECT + pg_database.datname::text, + states.state::text, + COALESCE(count, 0) as count, + COALESCE(max_tx_duration, 0) as max_tx_duration + FROM + (VALUES ('active'), + ('waiting'), + ('idle'), + ('idle in transaction'), + ('idle in transaction (aborted)'), + ('fastpath function call'), + ('disabled')) AS states(state) + CROSS JOIN pg_database + LEFT JOIN + (SELECT + datname, + CASE WHEN waiting THEN 'waiting' ELSE state END AS state, + count(*) AS count, + MAX(EXTRACT(EPOCH FROM now() - xact_start))::float AS max_tx_duration + FROM pg_stat_activity + GROUP BY 1, 2) AS act + ON states.state = act.state AND pg_database.datname = act.datname + WHERE NOT pg_database.datname ~ '^template(0|1)$' + +- name: "pg_stat_activity" + help: "running backends by database and state" + scope: cluster + min_version: 9.6 + labels: + - "datname" + - "state" + values: + - "count" + - "max_tx_duration" + query: >- + SELECT + pg_database.datname::text, + states.state::text, + COALESCE(count, 0) as count, + COALESCE(max_tx_duration, 0) as max_tx_duration + FROM + (VALUES ('active'), + ('waiting'), + ('idle'), + ('idle in transaction'), + ('idle in transaction (aborted)'), + ('fastpath function call'), + ('disabled')) AS states(state) + CROSS JOIN pg_database + LEFT JOIN + (SELECT + datname, + CASE WHEN state = 'active' AND wait_event_type IS NOT NULL THEN 'waiting' ELSE state END AS state, + count(*) AS count, + MAX(EXTRACT(EPOCH FROM now() - xact_start))::float AS max_tx_duration + FROM pg_stat_activity + GROUP BY 1, 2) AS act + ON states.state = act.state AND pg_database.datname = act.datname + WHERE NOT pg_database.datname ~ '^template(0|1)$' diff --git a/site-modules/profile/files/prometheus/sql/config/queries.yml b/site-modules/profile/files/prometheus/sql/config/queries.yml new file mode 100644 index 00000000..3650d1e7 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/queries.yml @@ -0,0 +1,291 @@ +# queries run once per cluster (via the 'postgres' database) + +- name: "settings" + help: "PostgreSQL settings" + scope: cluster + labels: + - "settings" + values: + - "max_connections" + - "autovacuum_freeze_max_age" + - "superuser_reserved_connections" + - "max_wal_senders" + - "max_prepared_transactions" + query: >- + SELECT + current_setting('max_connections')::float AS max_connections, + current_setting('autovacuum_freeze_max_age')::float AS autovacuum_freeze_max_age, + current_setting('superuser_reserved_connections')::float AS superuser_reserved_connections, + current_setting('max_wal_senders')::float AS max_wal_senders, + current_setting('max_prepared_transactions')::float AS max_prepared_transactions + +- name: "pg_locks" + help: "locks held" + scope: cluster + labels: + - "datname" + - "mode" + values: + - "count" + query: >- + SELECT datname, t.mode, count(l.mode) FROM pg_database d CROSS JOIN + (VALUES ('AccessShareLock'), ('RowShareLock'), ('RowExclusiveLock'), ('ShareUpdateExclusiveLock'), ('ShareLock'), ('ShareRowExclusiveLock'), ('ExclusiveLock'), ('AccessExclusiveLock')) t(mode) + LEFT JOIN pg_locks l ON d.oid = l.database AND t.mode = l.mode AND l.pid <> pg_backend_pid() + WHERE datname !~ '^template(0|1)$' GROUP BY 1, 2 + +- name: "pg_stat_database" + help: "database statistics" + scope: cluster + min_version: 9.2 + type: "counter" + labels: + - "datname" + values: + - "numbackends:count" + - "xact_commit" + - "xact_rollback" + - "blks_read" + - "blks_hit" + - "tup_returned" + - "tup_fetched" + - "tup_inserted" + - "tup_updated" + - "tup_deleted" + - "conflicts" + - "temp_files" + - "temp_bytes" + - "deadlocks" + - "blk_read_time" + - "blk_write_time" + - "freeze_age" + - "dbsize" + query: >- + SELECT + s.datname::text, + numbackends::float, + xact_commit::float, + xact_rollback::float, + blks_read::float, + blks_hit::float, + tup_returned::float, + tup_fetched::float, + tup_inserted::float, + tup_updated::float, + tup_deleted::float, + conflicts::float, + temp_files::float, + temp_bytes::float, + deadlocks::float, + blk_read_time, + blk_write_time, + age(d.datfrozenxid) AS freeze_age, + pg_database_size(s.datname)::float AS dbsize + FROM pg_stat_database s + LEFT JOIN pg_database d ON d.datname = s.datname + WHERE NOT s.datname ~ '^template(0|1)$' + +- name: "pg_stat_statements" + help: "statement statistics" + scope: cluster + min_version: 9.2 + labels: + - "usename" + - "datname" + - "queryid" + - "query" + values: + - "calls" + - "total_time" + - "rows" + - "shared_blks_hit" + - "shared_blks_read" + - "shared_blks_dirtied" + - "shared_blks_writte" + - "local_blks_hit" + - "local_blks_read" + - "local_blks_dirtied" + - "local_blks_writte" + - "temp_blks_read" + - "temp_blks_written" + query: >- + WITH w_pg_stat_statements AS ( SELECT * FROM pg_stat_statements) + (SELECT + usename::text, + datname::text, + queryid::text, + substr(regexp_replace(query, E'[\\n\\r]+', ' ', 'g' ), 1, 1024) AS query, + calls, + total_time, + rows, + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written + FROM w_pg_stat_statements pss JOIN pg_database pd ON pss.dbid = pd.oid + JOIN pg_user pu ON pss.userid = pu.usesysid + ORDER BY pss.total_time DESC + LIMIT 25) + UNION + SELECT + usename::text, + datname::text, + queryid::text, + substr(regexp_replace(query, E'[\\n\\r]+', ' ', 'g' ), 1, 1024) AS query, + calls, + total_time, + rows, + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written + FROM w_pg_stat_statements pss2 JOIN pg_database pd2 ON pss2.dbid = pd2.oid + JOIN pg_user pu2 ON pss2.userid = pu2.usesysid + ORDER BY calls DESC + LIMIT 25 + +- name: "txid" + help: "current txid" + scope: cluster + values: + - "txid_current" + query: SELECT CASE WHEN pg_is_in_recovery() THEN 0 ELSE txid_current() END + +- name: "prepared_transactions" + help: "prepared transactions" + scope: cluster + labels: + - "datname" + values: + - "count" + query: >- + SELECT + datname::text, + COUNT(transaction) AS count + FROM pg_database d + LEFT JOIN pg_prepared_xacts x ON d.datname = x.database + WHERE NOT d.datname ~ '^template(0|1)$' + GROUP BY datname + +# queries run for each database (except template0/template1) + +- name: "pg_stat_user_tables" + help: "table statistics" + scope: database + labels: + - "datname" + - "schemaname" + - "relname" + values: + - "seq_scan" + - "seq_tup_read" + - "idx_scan" + - "idx_tup_fetch" + - "n_tup_ins" + - "n_tup_upd" + - "n_tup_del" + - "n_tup_hot_upd" + - "n_live_tup" + - "n_dead_tup" + - "vacuum_count" + - "autovacuum_count" + - "analyze_count" + - "autoanalyze_count" + query: >- + SELECT + current_database()::text AS datname, + COALESCE(schemaname::text, 'null') AS schemaname, + COALESCE(relname::text, 'null') AS relname, + COALESCE(seq_scan, 0)::float AS seq_scan, + COALESCE(seq_tup_read, 0)::float AS seq_tup_read, + COALESCE(idx_scan, 0)::float AS idx_scan, + COALESCE(idx_tup_fetch, 0)::float AS idx_tup_fetch, + COALESCE(n_tup_ins, 0)::float AS n_tup_ins, + COALESCE(n_tup_upd, 0)::float AS n_tup_upd, + COALESCE(n_tup_del, 0)::float AS n_tup_del, + COALESCE(n_tup_hot_upd, 0)::float AS n_tup_hot_upd, + COALESCE(n_live_tup, 0)::float AS n_live_tup, + COALESCE(n_dead_tup, 0)::float AS n_dead_tup, + COALESCE(vacuum_count, 0)::float AS vacuum_count, + COALESCE(autovacuum_count, 0)::float AS autovacuum_count, + COALESCE(analyze_count, 0)::float AS analyze_count, + COALESCE(autoanalyze_count, 0)::float AS autoanalyze_count + FROM pg_stat_user_tables FULL JOIN (VALUES(0)) filler(i) ON TRUE + +- name: "pg_statio_user_tables" + help: "IO statistics" + scope: database + labels: + - "datname" + - "schemaname" + - "relname" + values: + - "heap_blks_read" + - "heap_blks_hit" + - "idx_blks_read" + - "idx_blks_hit" + query: >- + SELECT + current_database()::text AS datname, + COALESCE(schemaname::text, 'null') AS schemaname, + COALESCE(relname::text, 'null') AS relname, + COALESCE(heap_blks_read::float, 0) AS heap_blks_read, + COALESCE(heap_blks_hit::float, 0) AS heap_blks_hit, + COALESCE(idx_blks_read::float, 0) AS idx_blks_read, + COALESCE(idx_blks_hit::float, 0) AS idx_blks_hit + FROM pg_statio_user_tables FULL JOIN (VALUES(0)) filler(i) ON TRUE + +- name: "BufferAccess" + help: "buffer access statistics" + scope: database + labels: + - "datname" + - "schemaname" + - "relname" + values: + - "reads" + query: >- + SELECT + current_database()::text AS datname, + COALESCE(schemaname::text, 'null') AS schemaname, + COALESCE(relname::text, 'null') AS relname, + SUM(COALESCE(heap_blks_read, 0) + + COALESCE(heap_blks_hit, 0) + + COALESCE(idx_blks_hit, 0) + + COALESCE(idx_blks_read, 0) + + COALESCE(toast_blks_hit, 0) + + COALESCE(toast_blks_read, 0) + + COALESCE(tidx_blks_hit, 0) + + COALESCE(tidx_blks_read, 0)) * 8192::bigint as reads + FROM pg_statio_user_tables FULL JOIN (VALUES(0)) filler(i) ON TRUE + GROUP BY 1, 2, 3 + +- name: "Maintenancecounters" + help: "table maintenance job counters" + scope: database + labels: + - "datname" + values: + - "vacuum_count" + - "autovacuum_count" + - "analyze_count" + - "autoanalyze_count" + query: >- + SELECT + current_database()::text AS datname, + COALESCE(SUM(vacuum_count), 0) vacuum_count, + COALESCE(SUM(autovacuum_count), 0) autovacuum_count, + COALESCE(SUM(analyze_count), 0) analyze_count, + COALESCE(SUM(autoanalyze_count), 0) autoanalyze_count + FROM pg_stat_user_tables diff --git a/site-modules/profile/files/prometheus/sql/config/replication.yml b/site-modules/profile/files/prometheus/sql/config/replication.yml new file mode 100644 index 00000000..1c84bceb --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/replication.yml @@ -0,0 +1,66 @@ +# queries run once per cluster (via the 'postgres' database) + +- name: "archive_ready" + help: "number of WAL files waiting to be archived" + scope: cluster + max_version: 9.6 + values: + - "archive_ready" + query: SELECT COUNT(*) AS archive_ready FROM pg_ls_dir('pg_xlog/archive_status') WHERE pg_ls_dir ~ '^[0-9a-fA-F]{24}\.ready$' + +- name: "archive_ready" + help: "number of WAL files waiting to be archived" + scope: cluster + min_version: 10 + values: + - "archive_ready" + query: SELECT COUNT(*) AS archive_ready FROM pg_ls_dir('pg_wal/archive_status') WHERE pg_ls_dir ~ '^[0-9a-fA-F]{24}\.ready$' + +- name: "pg_stat_archiver" + help: "archiver statistics" + scope: cluster + min_version: 9.4 + values: + - "archived_count" + - "last_archived_time" + - "failed_count" + - "last_failed_time" + - "stats_reset" + query: SELECT archived_count, failed_count FROM pg_stat_archiver + +- name: "pg_stat_replication" + help: "replication statistics" + scope: cluster + min_version: 9.2 + max_version: 9.6 + labels: + - "application_name" + values: + - "send_lag_bytes" + - "flush_lag_bytes" + - "replay_lag_bytes" + query: >- + SELECT + COALESCE(application_name, '')::text AS application_name, + COALESCE(pg_xlog_location_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_xlog_receive_location() ELSE pg_current_xlog_location() END, sent_location), 0) AS send_lag_bytes, + COALESCE(pg_xlog_location_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_xlog_receive_location() ELSE pg_current_xlog_location() END, flush_location), 0) AS flush_lag_bytes, + COALESCE(pg_xlog_location_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_xlog_receive_location() ELSE pg_current_xlog_location() END, replay_location), 0) AS replay_lag_bytes + FROM pg_stat_replication FULL JOIN (VALUES(0)) filler(i) ON TRUE + +- name: "pg_stat_replication" + help: "replication statistics" + scope: cluster + min_version: 10 + labels: + - "application_name" + values: + - "send_lag_bytes" + - "flush_lag_bytes" + - "replay_lag_bytes" + query: >- + SELECT + COALESCE(application_name, '')::text AS application_name, + COALESCE(pg_wal_lsn_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() ELSE pg_current_wal_lsn() END, sent_lsn), 0) AS send_lag_bytes, + COALESCE(pg_wal_lsn_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() ELSE pg_current_wal_lsn() END, flush_lsn), 0) AS flush_lag_bytes, + COALESCE(pg_wal_lsn_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() ELSE pg_current_wal_lsn() END, replay_lsn), 0) AS replay_lag_bytes + FROM pg_stat_replication FULL JOIN (VALUES(0)) filler(i) ON TRUE diff --git a/site-modules/profile/files/prometheus/sql/config/swh-indexer.yml b/site-modules/profile/files/prometheus/sql/config/swh-indexer.yml new file mode 100644 index 00000000..e5f9a9d4 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/swh-indexer.yml @@ -0,0 +1,28 @@ +- name: "swh_indexer_count" + help: "Indexer count per type" + scope: database + cluster: indexer + database: ^softwareheritage-indexer$ + interval: '1h' + labels: + - "mimetype" + - "fossology_license" + values: + - "count_mimetype" + - "count_fossology_license" + query: >- + select 'mimetype', + (select reltuples + from pg_class C + left join pg_namespace N on (N.oid = C.relnamespace) + where nspname not in ('pg_catalog', 'information_schema') and + relkind='r' and + relname='content_mimetype' + ) as count_mimetype, + 'fossology_license', + (select reltuples + from pg_class C + left join pg_namespace N on (N.oid = C.relnamespace) + where nspname not in ('pg_catalog', 'information_schema') and + relkind='r' and + relname='content_fossology_license') as count_fossology_license; diff --git a/site-modules/profile/files/prometheus/sql/config/swh-scheduler.yml b/site-modules/profile/files/prometheus/sql/config/swh-scheduler.yml new file mode 100644 index 00000000..b40abb79 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/swh-scheduler.yml @@ -0,0 +1,225 @@ +- name: "swh_scheduler_oneshot" + scope: database + cluster: secondary + database: ^softwareheritage-scheduler$ + interval: '1h' + help: "Software Heritage Scheduled Oneshot Tasks" + labels: + - "type_policy_status" + values: + - "count" + query: | + select type ||' '|| status as type_policy_status, count(*) as count + from task + where policy='oneshot' group by 1 order by 1 + +- name: "swh_scheduler_recurring" + scope: database + cluster: secondary + database: ^softwareheritage-scheduler$ + interval: '1h' + help: "Software Heritage Scheduled Recurring Tasks" + labels: + - "type_policy_status" + values: + - "count" + query: | + select type ||' '|| status as type_policy_status, count(*) as count + from task + where policy='recurring' group by 1 order by 1 + +- name: swh_scheduler_delay + scope: database + cluster: secondary + database: ^softwareheritage-scheduler$ + interval: '1h' + help: "Software Heritage Scheduler task delay spread. Positive delay for tasks whose execution is late" + query: | + with task_count_by_bucket as ( + -- get the count of tasks by delay bucket. Tasks are grouped by their + -- characteristics (type, status, policy, priority, current interval), + -- then by delay buckets that are 1 hour wide between -24 and +24 hours, + -- and 1 day wide outside of this range. + -- A positive delay means the task execution is late wrt scheduling. + select + "type", + status, + "policy", + priority, + current_interval, + ( + -- select the bucket widths + case when delay between - 24 * 3600 and 24 * 3600 then + (ceil(delay / 3600)::bigint) * 3600 + else + (ceil(delay / (24 * 3600))::bigint) * 24 * 3600 + end + ) as delay_bucket, + count(*) + from + task + join lateral ( + -- this is where the "positive = late" convention is set + select + extract(epoch from (now() - next_run)) as delay + ) as d on true + group by + "type", + status, + "policy", + priority, + current_interval, + delay_bucket + order by + "type", + status, + "policy", + priority, + current_interval, + delay_bucket + ), + delay_bounds as ( + -- get the minimum and maximum delay bucket for each task group. This will + -- let us generate all the buckets, even the empty ones in the next CTE. + select + "type", + status, + "policy", + priority, + current_interval, + min(delay_bucket) as min, + max(delay_bucket) as max + from + task_count_by_bucket + group by + "type", + status, + "policy", + priority, + current_interval + ), + task_buckets as ( + -- Generate all time buckets for all categories. + select + "type", + status, + "policy", + priority, + current_interval, + delay_bucket + from + delay_bounds + join lateral ( + -- 1 hour buckets + select + generate_series(- 23, 23) * 3600 as delay_bucket + union + -- 1 day buckets. The "- 1" is used to make sure we generate an empty + -- bucket as lowest delay bucket, so prometheus quantile calculations + -- stay accurate + select + generate_series(min / (24 * 3600) - 1, max / (24 * 3600)) * 24 * 3600 as delay_bucket + ) as buckets on true + ), + task_count_for_all_buckets as ( + -- This join merges the non-empty buckets (task_count_by_bucket) with + -- the full list of buckets (task_buckets). + -- The join clause can't use the "using (x, y, z)" syntax, as it uses + -- equality and priority and current_interval can be null. This also + -- forces us to label all the fields in the select. Ugh. + select + task_buckets."type", + task_buckets.status, + task_buckets."policy", + task_buckets.priority, + task_buckets.current_interval, + task_buckets.delay_bucket, + coalesce(count, 0) as count -- make sure empty buckets have a 0 count instead of null + from + task_buckets + left join task_count_by_bucket + on task_count_by_bucket."type" = task_buckets."type" + and task_count_by_bucket.status = task_buckets.status + and task_count_by_bucket. "policy" = task_buckets."policy" + and task_count_by_bucket.priority is not distinct from task_buckets.priority + and task_count_by_bucket.current_interval is not distinct from task_buckets.current_interval + and task_count_by_bucket.delay_bucket = task_buckets.delay_bucket + ), + cumulative_buckets as ( + -- Prometheus wants cumulative histograms: for each bucket, the value + -- needs to be the total of all measurements below the given value (this + -- allows downsampling by just throwing away some buckets). We use the + -- "sum over partition" window function to compute this. + -- Prometheus also expects a "+Inf" bucket for the total count. We + -- generate it with a null le value so we can sort it after the rest of + -- the buckets. + + -- cumulative data + select + "type", + status, + "policy", + priority, + current_interval, + delay_bucket as le, + sum(count) over ( + partition by + "type", + status, + "policy", + priority, + current_interval + order by + delay_bucket + ) + from + task_count_for_all_buckets + union all + -- +Inf data + select + "type", + status, + "policy", + priority, + current_interval, + null as le, + sum(count) + from + task_count_for_all_buckets + group by + "type", + status, + "policy", + priority, + current_interval + -- sorting of all buckets + order by + "type", + status, + "policy", + priority, + current_interval, + le asc NULLS last -- make sure +Inf ends up last + ) + -- The final query, which at this point just has to make sure that all + -- labels are text (or the SQL exporter croaks) + select + -- we retrieve the backend name here as that's what we have e.g. on the celery side + (select backend_name from task_type where cumulative_buckets."type" = task_type."type") as task, + status::text as status, + policy::text as policy, + coalesce(priority::text, '') as priority, + coalesce(current_interval::text, '') as current_interval, + coalesce(le::text, '+Inf') as le, + sum + from + cumulative_buckets + labels: + - task + - status + - policy + - priority + - current_interval + - le + values: + - sum diff --git a/site-modules/profile/files/prometheus/sql/config/swh-storage.yml b/site-modules/profile/files/prometheus/sql/config/swh-storage.yml new file mode 100644 index 00000000..8a9cf502 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/swh-storage.yml @@ -0,0 +1,11 @@ +- name: swh_archive_object_count + help: Software Heritage Archive object counters + scope: database + cluster: main + database: softwareheritage + labels: + - object_type + values: + - value + query: >- + select label as object_type, value from swh_stat_counters() diff --git a/site-modules/profile/files/prometheus/sql/config/wal.yml b/site-modules/profile/files/prometheus/sql/config/wal.yml new file mode 100644 index 00000000..dbb2dd91 --- /dev/null +++ b/site-modules/profile/files/prometheus/sql/config/wal.yml @@ -0,0 +1,45 @@ +# queries run once per cluster (via the 'postgres' database) + +- name: "checkpoints" + help: "requested and timed checkpoints" + scope: cluster + values: + - "timed" + - "requested" + query: >- + SELECT + pg_stat_get_bgwriter_timed_checkpoints() timed, + pg_stat_get_bgwriter_requested_checkpoints() requested + +- name: "LastCheckpointDistance" + help: "distance to the last checkpoint" + scope: cluster + version: 9.6 + values: + - "distance" + query: SELECT pg_xlog_location_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_xlog_receive_location() ELSE pg_current_xlog_location() END, checkpoint_location) "distance" FROM pg_control_checkpoint() + +- name: "LastCheckpointDistance" + help: "distance to the last checkpoint" + scope: cluster + min_version: 10 + values: + - "distance" + query: SELECT pg_wal_lsn_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() ELSE pg_current_wal_lsn() END, checkpoint_lsn) "distance" FROM pg_control_checkpoint() + +- name: "waldistance" + help: "amount of WAL written" + scope: cluster + min_version: 9.2 + max_version: 9.6 + values: + - "location" + query: SELECT pg_xlog_location_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_xlog_receive_location() ELSE pg_current_xlog_location() END, '0/0') "location" + +- name: "waldistance" + help: "amount of WAL written" + scope: cluster + min_version: 10 + values: + - "location" + query: SELECT pg_wal_lsn_diff(CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() ELSE pg_current_wal_lsn() END, '0/0') "location" diff --git a/site-modules/profile/files/prometheus/sql/update-prometheus-sql-exporter-config b/site-modules/profile/files/prometheus/sql/update-prometheus-sql-exporter-config old mode 100755 new mode 100644 index e1a51f58..83ef6528 --- a/site-modules/profile/files/prometheus/sql/update-prometheus-sql-exporter-config +++ b/site-modules/profile/files/prometheus/sql/update-prometheus-sql-exporter-config @@ -1,164 +1,108 @@ -#!/usr/bin/python3 -# -# Originally part of Credativ's Elephant Shed -# https://github.com/credativ/elephant-shed -# prometheus/sql_exporter/update-prometheus-sql-exporter-config -# Licensed under the GPLv3. -# -# Adapted to run on Python 3.5 - -import sys -import yaml -import subprocess -import traceback -from pkg_resources import parse_version - -""" -Returns a list of clusters (dict) containing the following attributes: - - name - - version - - port - - status - - owner - - databases (dict) - -Calls get_databases for each cluster. -""" -def get_clusters(min_version=None, max_version=None): - clusters = list() - - proc = subprocess.Popen(["pg_lsclusters"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True) - (out, err) = proc.communicate() - - out = out.rstrip("\n") - out_lines = out.splitlines() - - for i in range(1,len(out_lines)): - cluster = dict() - version, name, port, status, owner = out_lines[i].split()[:5] - cluster["version"] = version - cluster["name"] = name - cluster["port"] = port - cluster["status"] = status - cluster["owner"] = owner - cluster["databases"] = get_databases(cluster) - clusters.append(cluster) - - return clusters - -""" -Returns a list of databases for the given cluster. -""" -def get_databases(cluster): - databases = list() - command = "sudo -u %s psql -p %s -tXA -c \"SELECT datname FROM pg_database WHERE NOT datname ~ '^template(0|1)$';\"" - command = command % (cluster["owner"], cluster["port"]) - proc = subprocess.Popen([command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True) - (out, err) = proc.communicate() - out_lines = out.splitlines() - for line in out_lines: - databases.append(line.split("|")[0]) - - return databases - -""" -Returns the a config element (dict). -""" -def read_config_template(): - config_filename = "/etc/prometheus-sql-exporter.yml.in" - with open(config_filename, "r") as f: - yaml_conf = yaml.load(f) - - return yaml_conf - -""" -Writes the given config to file. -""" -def write_config(conf): - config_filename = "/etc/prometheus-sql-exporter.yml" - with open(config_filename, "w") as f: - yaml.dump(conf, f) - - return True - -""" -Takes a given config and appends connection strings to it. -""" -def append_conn_strings_to_template(conf): - clusters = get_clusters() - - for job in conf["jobs"]: - min_version = parse_version("0") - max_version = parse_version("99999999999") - used_clusters = list() - conn_strings = list() - - if "min_version" in job: - min_version = parse_version(job["min_version"]) - - if "max_version" in job: - max_version = parse_version(job["max_version"]) - - for cluster in clusters: - cluster_version = parse_version(cluster["version"]) - if cluster_version >= min_version and cluster_version <= max_version: - used_clusters.append(cluster) - - if job['name'].startswith('cluster'): - for cluster in used_clusters: - conn_strings.extend(build_conn_strings(cluster, False)) - elif job['name'].startswith('database'): - for cluster in used_clusters: - conn_strings.extend(build_conn_strings(cluster, True)) - else: - conn_strings.extend(job['connections']) - - job["connections"] = conn_strings - - return conf - -""" -Returns a list of connection strings for all clusters/databases. - -If per_db is True a list of all databases is returned. Otherwise a list of all -clusters (using template1) is returned. -""" -def build_conn_strings(cluster, per_db=True): - conn_strings = list() - - if per_db: - for db in cluster["databases"]: - conn_string = 'postgres://%s@:%s/%s?sslmode=disable' - conn_string = conn_string % (cluster["owner"], cluster["port"], db) - conn_strings.append(conn_string) - else: - conn_string = 'postgres://%s@:%s/postgres?sslmode=disable' - conn_string = conn_string % (cluster["owner"], cluster["port"]) - conn_strings.append(conn_string) - - return conn_strings - -if __name__ == "__main__": - clusters = get_clusters() - - try: - yaml_conf = read_config_template() - except: - sys.stderr.write("Could not read config template.\n") - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) - exit(1) - - try: - yaml_conf = append_conn_strings_to_template(yaml_conf) - except: - sys.stderr.write("Could not write new config.\n") - print('-'*60) - traceback.print_exc(file=sys.stdout) - print('-'*60) - exit(1) - - write_config(yaml_conf) - - exit(0) +#!/usr/bin/perl + +use strict; +use warnings; +use PgCommon; +use Storable qw(dclone); +use YAML; + +# from list of queries, remove all that do not satisfy the scope/version/cluster/database constraints +sub filter_queries($$$$;$) +{ + my ($queries, $scope, $version, $cluster, $database) = @_; + my @result; + foreach my $query (@$queries) { + $query->{scope} //= 'database'; + die "scope '$query->{scope}' must be either 'cluster' or 'database'" + unless ($query->{scope} =~ /^(cluster|database)$/); + next if $query->{scope} ne $scope; + next if $query->{min_version} and $version < $query->{min_version}; + next if $query->{max_version} and $version > $query->{max_version}; + next if $query->{version} and $version !~ /$query->{version}/; + next if $query->{cluster} and $cluster !~ /$query->{cluster}/; + next if $query->{database} and $database !~ /$query->{database}/; + + my $q = dclone($query); + delete $q->{scope}; delete $q->{min_version}; delete $q->{max_version}; + delete $q->{version}; delete $q->{cluster}; delete $q->{database}; + push @result, $q; + } + return \@result; +} + +# Generate jobs from the given queries. One job is created for all the queries +# with null interval, and one job is created per query with non-null interval. +sub push_queries_as_jobs($$$) +{ + my ($jobs, $queries, $job_settings) = @_; + my @instant; + foreach my $query (@$queries) { + if (!$query->{interval}) { + push @instant, $query; + } else { + # Generate separate job for query with non-null interval + my $job_settings_query = dclone($job_settings); + $job_settings_query->{interval} = $query->{interval}; + $job_settings_query->{name} .= '/' . $query->{name}; + my $q = dclone($query); + delete $q->{interval}; + $job_settings_query->{queries} = [$q]; + + push @$jobs, $job_settings_query; + } + } + + my $instant_job = dclone($job_settings); + $instant_job->{interval} = 0; + $instant_job->{queries} = \@instant; + push @$jobs, $instant_job; +} + +my $queries_directory = $ARGV[0] // die "No directory for *.yml query files specified"; +my $output_yaml = $ARGV[1] // die "No output yaml file specified"; + +# load all *.yml files from input directory and collect the contained query lists +my $queries; +foreach my $yml (glob "$queries_directory/*.yml") { + my $q; + eval { $q = YAML::LoadFile($yml); }; + die "Error loading $yml: $@" if ($@); + next if (ref($q) eq ''); # file is empty + die "$yml is not a yaml list:" . ref($q) unless (ref($q) eq 'ARRAY'); + push @$queries, @$q; +} + +# walk all clusters and databases and produce jobs for them +my $jobs = []; +foreach my $version (get_versions()) { + foreach my $cluster (get_version_clusters($version)) { + my %info = cluster_info($version, $cluster); + next unless $info{running}; # cluster is down, skip it + my $owner = (getpwuid $info{owneruid})[0] // die "Could not determine owner name of cluster $version $cluster"; + my $socket = get_cluster_socketdir($version, $cluster); + + # jobs for cluster-wide queries + push_queries_as_jobs $jobs, filter_queries($queries, 'cluster', $version, $cluster), { + connections => [ "postgres://$owner\@:$info{port}/postgres?sslmode=disable&host=$socket" ], + name => "$version/$cluster", + }; + + # jobs for per-database queries + my @cluster_databases = get_cluster_databases($version, $cluster); + foreach my $database (grep { $_ and $_ !~ /^template[01]$/ } @cluster_databases) { + push_queries_as_jobs $jobs, filter_queries($queries, 'database', $version, $cluster, $database), { + connections => [ "postgres://$owner\@:$info{port}/$database?sslmode=disable&host=$socket" ], + name => "$version/$cluster/$database", + }; + } + + # create pg_stat_statements in "postgres" database if missing + next if $info{recovery}; # cluster is in recovery mode, skip it + open PSQL, "|-", "su -c 'psql -q -h $socket -p $info{port} postgres' $owner"; + print PSQL "DO \$\$DECLARE ext name; BEGIN SELECT INTO ext extname FROM pg_extension WHERE extname = 'pg_stat_statements'; IF NOT FOUND THEN RAISE NOTICE 'Creating pg_stat_statements extension in cluster $version/$cluster'; CREATE EXTENSION pg_stat_statements; END IF; END\$\$ LANGUAGE plpgsql;\n"; + close PSQL; + } +} + +# write output yml +YAML::DumpFile($output_yaml, { jobs => $jobs }); diff --git a/site-modules/profile/files/stats_exporter/export-archive_counters.py b/site-modules/profile/files/stats_exporter/export-archive_counters.py new file mode 100755 index 00000000..06d0afce --- /dev/null +++ b/site-modules/profile/files/stats_exporter/export-archive_counters.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 + +import requests +import json +import time + +def clean_item(item): + """Javascript expects timestamps to be in milliseconds + and counter values as floats + + """ + timestamp = int(item[0]) + counter_value = item[1] + return [timestamp*1000, float(counter_value)] + + +def get_timestamp_history(label): + result = [] + rrd_data = [] + now = int(time.time()) + url = 'http://pergamon.internal.softwareheritage.org:9090/api/v1/query_range?' + url = url + 'query=sum(sql_swh_archive_object_count) by (object_type)' + url = url + '&start=1544543227&end=%s&step=12h' % now + + # We only want to process timevalues for Source files + if (label == "content"): + # Historical data has already been processed for javascript usage + # No need to clean it further + with open("/usr/local/share/swh-data/history-counters.munin.json", "r") as f: + rrd_data = json.load(f)[label] + + response = requests.get(url) + if response.ok: + data = response.json() + # In contrast, Prometheus-provided data has to be adapted to + # Javascript expectations + result = [clean_item(i) for i in + data['data']['result'][0]['values']] + return rrd_data + result + + +def main(): + result = {} + for label in ['content', 'origin', 'revision']: + result[label] = get_timestamp_history(label) + return result + + +if __name__ == '__main__': + r = main() + print(json.dumps(r)) diff --git a/site-modules/profile/files/stats_exporter/history-counters.munin.json b/site-modules/profile/files/stats_exporter/history-counters.munin.json new file mode 100644 index 00000000..517e2355 --- /dev/null +++ b/site-modules/profile/files/stats_exporter/history-counters.munin.json @@ -0,0 +1 @@ +{"content": [[1444348800000, 220307136.07], [1444435200000, 220307136.07], [1444521600000, 220307136.07], [1444608000000, 220307136.07], [1444694400000, 220307136.07], [1444780800000, 220307136.07], [1444867200000, 220307136.07], [1444953600000, 306574115.84], [1445040000000, 306574115.84], [1445126400000, 306574115.84], [1445212800000, 306574115.84], [1445299200000, 306574115.84], [1445385600000, 306574115.84], [1445472000000, 306574115.84], [1445558400000, 438815813.08], [1445644800000, 438815813.08], [1445731200000, 438815813.08], [1445817600000, 438815813.08], [1445904000000, 438815813.08], [1445990400000, 438815813.08], [1446076800000, 438815813.08], [1446163200000, 584557475.72], [1446249600000, 584557475.72], [1446336000000, 584557475.72], [1446422400000, 584557475.72], [1446508800000, 584557475.72], [1446595200000, 584557475.72], [1446681600000, 584557475.72], [1446768000000, 722269076.3], [1446854400000, 722269076.3], [1446940800000, 722269076.3], [1447027200000, 722269076.3], [1447113600000, 722269076.3], [1447200000000, 722269076.3], [1447286400000, 722269076.3], [1447372800000, 851212248.07], [1447459200000, 851212248.07], [1447545600000, 851212248.07], [1447632000000, 851212248.07], [1447718400000, 851212248.07], [1447804800000, 851212248.07], [1447891200000, 851212248.07], [1447977600000, 966222889.01], [1448064000000, 966222889.01], [1448150400000, 966222889.01], [1448236800000, 966222889.01], [1448323200000, 966222889.01], [1448409600000, 966222889.01], [1448496000000, 966222889.01], [1448582400000, 1075138884.8], [1448668800000, 1075138884.8], [1448755200000, 1075138884.8], [1448841600000, 1075138884.8], [1448928000000, 1075138884.8], [1449014400000, 1075138884.8], [1449100800000, 1075138884.8], [1449187200000, 1164211880.1], [1449273600000, 1164211880.1], [1449360000000, 1164211880.1], [1449446400000, 1164211880.1], [1449532800000, 1164211880.1], [1449619200000, 1164211880.1], [1449705600000, 1164211880.1], [1449792000000, 1226246110.8], [1449878400000, 1226246110.8], [1449964800000, 1226246110.8], [1450051200000, 1226246110.8], [1450137600000, 1226246110.8], [1450224000000, 1226246110.8], [1450310400000, 1226246110.8], [1450396800000, 1309871671.9], [1450483200000, 1309871671.9], [1450569600000, 1309871671.9], [1450656000000, 1309871671.9], [1450742400000, 1309871671.9], [1450828800000, 1309871671.9], [1450915200000, 1309871671.9], [1451001600000, 1411125400.7], [1451088000000, 1411125400.7], [1451174400000, 1411125400.7], [1451260800000, 1411125400.7], [1451347200000, 1411125400.7], [1451433600000, 1411125400.7], [1451520000000, 1411125400.7], [1451606400000, 1502309510.0], [1451692800000, 1502309510.0], [1451779200000, 1502309510.0], [1451865600000, 1502309510.0], [1451952000000, 1502309510.0], [1452038400000, 1502309510.0], [1452124800000, 1502309510.0], [1452211200000, 1584411348.0], [1452297600000, 1584411348.0], [1452384000000, 1584411348.0], [1452470400000, 1584411348.0], [1452556800000, 1584411348.0], [1452643200000, 1584411348.0], [1452729600000, 1584411348.0], [1452816000000, 1671332828.1], [1452902400000, 1671332828.1], [1452988800000, 1671332828.1], [1453075200000, 1671332828.1], [1453161600000, 1671332828.1], [1453248000000, 1671332828.1], [1453334400000, 1671332828.1], [1453420800000, 1687132778.0], [1453507200000, 1687132778.0], [1453593600000, 1687132778.0], [1453680000000, 1687132778.0], [1453766400000, 1687132778.0], [1453852800000, 1687132778.0], [1453939200000, 1687132778.0], [1454025600000, 1687132778.0], [1454112000000, 1687132778.0], [1454198400000, 1687132778.0], [1454284800000, 1687132778.0], [1454371200000, 1687132778.0], [1454457600000, 1687132778.0], [1454544000000, 1687132778.0], [1454630400000, 1687132778.0], [1454716800000, 1687132778.0], [1454803200000, 1687132778.0], [1454889600000, 1687132778.0], [1454976000000, 1687132778.0], [1455062400000, 1687132778.0], [1455148800000, 1687132778.0], [1455235200000, 1687132778.0], [1455321600000, 1687132778.0], [1455408000000, 1687132778.0], [1455494400000, 1687132778.0], [1455580800000, 1687132778.0], [1455667200000, 1687132778.0], [1455753600000, 1687132778.0], [1455840000000, 1688017903.9], [1455926400000, 1688017903.9], [1456012800000, 1688017903.9], [1456099200000, 1688017903.9], [1456185600000, 1688017903.9], [1456272000000, 1688017903.9], [1456358400000, 1688017903.9], [1456444800000, 1727368775.3], [1456531200000, 1727368775.3], [1456617600000, 1727368775.3], [1456704000000, 1727368775.3], [1456790400000, 1727368775.3], [1456876800000, 1727368775.3], [1456963200000, 1727368775.3], [1457049600000, 1800029760.6], [1457136000000, 1800029760.6], [1457222400000, 1800029760.6], [1457308800000, 1800029760.6], [1457395200000, 1800029760.6], [1457481600000, 1800029760.6], [1457568000000, 1800029760.6], [1457654400000, 1878494398.7], [1457740800000, 1878494398.7], [1457827200000, 1878494398.7], [1457913600000, 1878494398.7], [1458000000000, 1878494398.7], [1458086400000, 1878494398.7], [1458172800000, 1878494398.7], [1458259200000, 1970166583.0], [1458345600000, 1970166583.0], [1458432000000, 1970166583.0], [1458518400000, 1970166583.0], [1458604800000, 1970166583.0], [1458691200000, 1970166583.0], [1458777600000, 1970166583.0], [1458864000000, 2040921551.2], [1458950400000, 2040921551.2], [1459036800000, 2040921551.2], [1459123200000, 2040921551.2], [1459209600000, 2040921551.2], [1459296000000, 2040921551.2], [1459382400000, 2040921551.2], [1459468800000, 2089293765.3], [1459555200000, 2089293765.3], [1459641600000, 2089293765.3], [1459728000000, 2089293765.3], [1459814400000, 2089293765.3], [1459900800000, 2089293765.3], [1459987200000, 2089293765.3], [1460073600000, 2102191753.3], [1460160000000, 2102191753.3], [1460246400000, 2102191753.3], [1460332800000, 2102191753.3], [1460419200000, 2102191753.3], [1460505600000, 2102191753.3], [1460592000000, 2102191753.3], [1460678400000, 2168940947.8], [1460764800000, 2168940947.8], [1460851200000, 2168940947.8], [1460937600000, 2168940947.8], [1461024000000, 2168940947.8], [1461110400000, 2168940947.8], [1461196800000, 2168940947.8], [1461283200000, 2233968053.6], [1461369600000, 2233968053.6], [1461456000000, 2233968053.6], [1461542400000, 2233968053.6], [1461628800000, 2233968053.6], [1461715200000, 2233968053.6], [1461801600000, 2233968053.6], [1461888000000, 2269876784.6], [1461974400000, 2269876784.6], [1462060800000, 2269876784.6], [1462147200000, 2269876784.6], [1462233600000, 2269876784.6], [1462320000000, 2269876784.6], [1462406400000, 2269876784.6], [1462492800000, 2339681113.8], [1462579200000, 2339681113.8], [1462665600000, 2339681113.8], [1462752000000, 2339681113.8], [1462838400000, 2339681113.8], [1462924800000, 2339681113.8], [1463011200000, 2339681113.8], [1463097600000, 2424104827.5], [1463184000000, 2424104827.5], [1463270400000, 2424104827.5], [1463356800000, 2424104827.5], [1463443200000, 2424104827.5], [1463529600000, 2424104827.5], [1463616000000, 2424104827.5], [1463702400000, 2496199546.4], [1463788800000, 2496199546.4], [1463875200000, 2496199546.4], [1463961600000, 2496199546.4], [1464048000000, 2496199546.4], [1464134400000, 2496199546.4], [1464220800000, 2496199546.4], [1464307200000, 2520942090.1], [1464393600000, 2520942090.1], [1464480000000, 2520942090.1], [1464566400000, 2520942090.1], [1464652800000, 2520942090.1], [1464739200000, 2520942090.1], [1464825600000, 2520942090.1], [1464912000000, 2560552020.3], [1464998400000, 2560552020.3], [1465084800000, 2560552020.3], [1465171200000, 2560552020.3], [1465257600000, 2560552020.3], [1465344000000, 2560552020.3], [1465430400000, 2560552020.3], [1465516800000, 2636172692.2], [1465603200000, 2636172692.2], [1465689600000, 2636172692.2], [1465776000000, 2636172692.2], [1465862400000, 2636172692.2], [1465948800000, 2636172692.2], [1466035200000, 2636172692.2], [1466121600000, 2674180960.5], [1466208000000, 2674180960.5], [1466294400000, 2674180960.5], [1466380800000, 2674180960.5], [1466467200000, 2674180960.5], [1466553600000, 2674180960.5], [1466640000000, 2674180960.5], [1466726400000, 2674769407.0], [1466812800000, 2674769407.0], [1466899200000, 2674769407.0], [1466985600000, 2674769407.0], [1467072000000, 2674769407.0], [1467158400000, 2674769407.0], [1467244800000, 2674769407.0], [1467331200000, 2674769407.0], [1467417600000, 2674769407.0], [1467504000000, 2674769407.0], [1467590400000, 2674769407.0], [1467676800000, 2674769407.0], [1467763200000, 2674769407.0], [1467849600000, 2674769407.0], [1467936000000, 2674769407.0], [1468022400000, 2674769407.0], [1468108800000, 2674769407.0], [1468195200000, 2674769407.0], [1468281600000, 2674769407.0], [1468368000000, 2674769407.0], [1468454400000, 2674769407.0], [1468540800000, 2674769407.0], [1468627200000, 2674769407.0], [1468713600000, 2674769407.0], [1468800000000, 2674769407.0], [1468886400000, 2674769407.0], [1468972800000, 2674769407.0], [1469059200000, 2674769407.0], [1469145600000, 2677904750.2], [1469232000000, 2677904750.2], [1469318400000, 2677904750.2], [1469404800000, 2677904750.2], [1469491200000, 2677904750.2], [1469577600000, 2677904750.2], [1469664000000, 2677904750.2], [1469750400000, 2706687916.2], [1469836800000, 2706687916.2], [1469923200000, 2706687916.2], [1470009600000, 2706687916.2], [1470096000000, 2706687916.2], [1470182400000, 2706687916.2], [1470268800000, 2706687916.2], [1470355200000, 2736496505.8], [1470441600000, 2736496505.8], [1470528000000, 2736496505.8], [1470614400000, 2736496505.8], [1470700800000, 2736496505.8], [1470787200000, 2736496505.8], [1470873600000, 2736496505.8], [1470960000000, 2770247638.5], [1471046400000, 2770247638.5], [1471132800000, 2770247638.5], [1471219200000, 2770247638.5], [1471305600000, 2770247638.5], [1471392000000, 2770247638.5], [1471478400000, 2770247638.5], [1471564800000, 2831698788.5], [1471651200000, 2831698788.5], [1471737600000, 2831698788.5], [1471824000000, 2831698788.5], [1471910400000, 2831698788.5], [1471996800000, 2831698788.5], [1472083200000, 2831698788.5], [1472169600000, 2910289354.5], [1472256000000, 2910289354.5], [1472342400000, 2910289354.5], [1472428800000, 2910289354.5], [1472515200000, 2910289354.5], [1472601600000, 2910289354.5], [1472688000000, 2910289354.5], [1472774400000, 2937536525.1], [1472860800000, 2937536525.1], [1472947200000, 2937536525.1], [1473033600000, 2937536525.1], [1473120000000, 2937536525.1], [1473206400000, 2937536525.1], [1473292800000, 2937536525.1], [1473379200000, 2952919167.4], [1473465600000, 2952919167.4], [1473552000000, 2952919167.4], [1473638400000, 2952919167.4], [1473724800000, 2952919167.4], [1473811200000, 2952919167.4], [1473897600000, 2952919167.4], [1473984000000, 2989469788.8], [1474070400000, 2989469788.8], [1474156800000, 2989469788.8], [1474243200000, 2989469788.8], [1474329600000, 2989469788.8], [1474416000000, 2989469788.8], [1474502400000, 2989469788.8], [1474588800000, 3038660352.1], [1474675200000, 3038660352.1], [1474761600000, 3038660352.1], [1474848000000, 3038660352.1], [1474934400000, 3038660352.1], [1475020800000, 3038660352.1], [1475107200000, 3038660352.1], [1475193600000, 3060164062.1], [1475280000000, 3060164062.1], [1475366400000, 3060164062.1], [1475452800000, 3060164062.1], [1475539200000, 3060164062.1], [1475625600000, 3060164062.1], [1475712000000, 3060164062.1], [1475798400000, 3069791211.0], [1475884800000, 3069791211.0], [1475971200000, 3069791211.0], [1476057600000, 3069791211.0], [1476144000000, 3069791211.0], [1476230400000, 3069791211.0], [1476316800000, 3069791211.0], [1476403200000, 3077967605.3], [1476489600000, 3077967605.3], [1476576000000, 3077967605.3], [1476662400000, 3077967605.3], [1476748800000, 3077967605.3], [1476835200000, 3077967605.3], [1476921600000, 3077967605.3], [1477008000000, 3086911292.0], [1477094400000, 3086911292.0], [1477180800000, 3086911292.0], [1477267200000, 3086911292.0], [1477353600000, 3086911292.0], [1477440000000, 3086911292.0], [1477526400000, 3086911292.0], [1477612800000, 3093069133.6], [1477699200000, 3093069133.6], [1477785600000, 3093069133.6], [1477872000000, 3093069133.6], [1477958400000, 3093069133.6], [1478044800000, 3093069133.6], [1478131200000, 3093069133.6], [1478217600000, 3098286129.1], [1478304000000, 3098286129.1], [1478390400000, 3098286129.1], [1478476800000, 3098286129.1], [1478563200000, 3098286129.1], [1478649600000, 3098286129.1], [1478736000000, 3098286129.1], [1478822400000, 3100076588.9], [1478908800000, 3100076588.9], [1478995200000, 3100076588.9], [1479081600000, 3100076588.9], [1479168000000, 3100076588.9], [1479254400000, 3100076588.9], [1479340800000, 3100076588.9], [1479427200000, 3100447015.0], [1479513600000, 3100447015.0], [1479600000000, 3100447015.0], [1479686400000, 3100447015.0], [1479772800000, 3100447015.0], [1479859200000, 3100447015.0], [1479945600000, 3100447015.0], [1480032000000, 3100447015.0], [1480118400000, 3100447015.0], [1480204800000, 3100447015.0], [1480291200000, 3100447015.0], [1480377600000, 3100447015.0], [1480464000000, 3100447015.0], [1480550400000, 3100447015.0], [1480636800000, 3108188640.7], [1480723200000, 3108188640.7], [1480809600000, 3108188640.7], [1480896000000, 3108188640.7], [1480982400000, 3108188640.7], [1481068800000, 3108188640.7], [1481155200000, 3108188640.7], [1481241600000, 3117631896.9], [1481328000000, 3117631896.9], [1481414400000, 3117631896.9], [1481500800000, 3117631896.9], [1481587200000, 3117631896.9], [1481673600000, 3117631896.9], [1481760000000, 3117631896.9], [1481846400000, 3127016045.0], [1481932800000, 3127016045.0], [1482019200000, 3127016045.0], [1482105600000, 3127016045.0], [1482192000000, 3127016045.0], [1482278400000, 3127016045.0], [1482364800000, 3127016045.0], [1482451200000, 3134166808.4], [1482537600000, 3134166808.4], [1482624000000, 3134166808.4], [1482710400000, 3134166808.4], [1482796800000, 3134166808.4], [1482883200000, 3134166808.4], [1482969600000, 3134166808.4], [1483056000000, 3141230838.1], [1483142400000, 3141230838.1], [1483228800000, 3141230838.1], [1483315200000, 3141230838.1], [1483401600000, 3141230838.1], [1483488000000, 3141230838.1], [1483574400000, 3141230838.1], [1483660800000, 3144874246.9], [1483747200000, 3144874246.9], [1483833600000, 3144874246.9], [1483920000000, 3144874246.9], [1484006400000, 3144874246.9], [1484092800000, 3144874246.9], [1484179200000, 3144874246.9], [1484265600000, 3147703602.1], [1484352000000, 3147703602.1], [1484438400000, 3147703602.1], [1484524800000, 3147703602.1], [1484611200000, 3147703602.1], [1484697600000, 3147703602.1], [1484784000000, 3147703602.1], [1484870400000, 3153467614.7], [1484956800000, 3153467614.7], [1485043200000, 3153467614.7], [1485129600000, 3153467614.7], [1485216000000, 3153467614.7], [1485302400000, 3153467614.7], [1485388800000, 3153467614.7], [1485475200000, 3159890691.3], [1485561600000, 3159890691.3], [1485648000000, 3159890691.3], [1485734400000, 3159890691.3], [1485820800000, 3159890691.3], [1485907200000, 3159890691.3], [1485993600000, 3159890691.3], [1486080000000, 3164232521.8], [1486166400000, 3164232521.8], [1486252800000, 3164232521.8], [1486339200000, 3164232521.8], [1486425600000, 3164232521.8], [1486512000000, 3164232521.8], [1486598400000, 3164232521.8], [1486684800000, 3173695067.4], [1486771200000, 3173695067.4], [1486857600000, 3173695067.4], [1486944000000, 3173695067.4], [1487030400000, 3173695067.4], [1487116800000, 3173695067.4], [1487203200000, 3173695067.4], [1487289600000, 3182485132.5], [1487376000000, 3182485132.5], [1487462400000, 3182485132.5], [1487548800000, 3182485132.5], [1487635200000, 3182485132.5], [1487721600000, 3182485132.5], [1487808000000, 3182485132.5], [1487894400000, 3197962348.0], [1487980800000, 3197962348.0], [1488067200000, 3197962348.0], [1488153600000, 3197962348.0], [1488240000000, 3197962348.0], [1488326400000, 3197962348.0], [1488412800000, 3197962348.0], [1488499200000, 3205575651.9], [1488585600000, 3205575651.9], [1488672000000, 3205575651.9], [1488758400000, 3205575651.9], [1488844800000, 3205575651.9], [1488931200000, 3205575651.9], [1489017600000, 3205575651.9], [1489104000000, 3211867967.3], [1489190400000, 3211867967.3], [1489276800000, 3211867967.3], [1489363200000, 3211867967.3], [1489449600000, 3211867967.3], [1489536000000, 3211867967.3], [1489622400000, 3211867967.3], [1489708800000, 3216990646.7], [1489795200000, 3216990646.7], [1489881600000, 3216990646.7], [1489968000000, 3216990646.7], [1490054400000, 3216990646.7], [1490140800000, 3216990646.7], [1490227200000, 3216990646.7], [1490313600000, 3239711486.4], [1490400000000, 3239711486.4], [1490486400000, 3239711486.4], [1490572800000, 3239711486.4], [1490659200000, 3239711486.4], [1490745600000, 3239711486.4], [1490832000000, 3239711486.4], [1490918400000, 3257992944.4], [1491004800000, 3257992944.4], [1491091200000, 3257992944.4], [1491177600000, 3257992944.4], [1491264000000, 3257992944.4], [1491350400000, 3257992944.4], [1491436800000, 3257992944.4], [1491523200000, 3262986485.2], [1491609600000, 3262986485.2], [1491696000000, 3262986485.2], [1491782400000, 3262986485.2], [1491868800000, 3262986485.2], [1491955200000, 3262986485.2], [1492041600000, 3262986485.2], [1492128000000, 3264879286.0], [1492214400000, 3264879286.0], [1492300800000, 3264879286.0], [1492387200000, 3264879286.0], [1492473600000, 3264879286.0], [1492560000000, 3264879286.0], [1492646400000, 3264879286.0], [1492732800000, 3290480754.2], [1492819200000, 3290480754.2], [1492905600000, 3290480754.2], [1492992000000, 3290480754.2], [1493078400000, 3290480754.2], [1493164800000, 3290480754.2], [1493251200000, 3290480754.2], [1493337600000, 3310557876.0], [1493424000000, 3310557876.0], [1493510400000, 3310557876.0], [1493596800000, 3310557876.0], [1493683200000, 3310557876.0], [1493769600000, 3310557876.0], [1493856000000, 3310557876.0], [1493942400000, 3368250330.1], [1494028800000, 3368250330.1], [1494115200000, 3368250330.1], [1494201600000, 3368250330.1], [1494288000000, 3368250330.1], [1494374400000, 3368250330.1], [1494460800000, 3368250330.1], [1494547200000, 3399239580.9], [1494633600000, 3399239580.9], [1494720000000, 3399239580.9], [1494806400000, 3399239580.9], [1494892800000, 3399239580.9], [1494979200000, 3399239580.9], [1495065600000, 3399239580.9], [1495152000000, 3423518074.9], [1495238400000, 3423518074.9], [1495324800000, 3423518074.9], [1495411200000, 3423518074.9], [1495497600000, 3423518074.9], [1495584000000, 3423518074.9], [1495670400000, 3423518074.9], [1495756800000, 3433154462.7], [1495843200000, 3433154462.7], [1495929600000, 3433154462.7], [1496016000000, 3433154462.7], [1496102400000, 3433154462.7], [1496188800000, 3433154462.7], [1496275200000, 3433154462.7], [1496361600000, 3444769730.3], [1496448000000, 3444769730.3], [1496534400000, 3444769730.3], [1496620800000, 3444769730.3], [1496707200000, 3444769730.3], [1496793600000, 3444769730.3], [1496880000000, 3444769730.3], [1496966400000, 3454376161.3], [1497052800000, 3454376161.3], [1497139200000, 3454376161.3], [1497225600000, 3454376161.3], [1497312000000, 3454376161.3], [1497398400000, 3454376161.3], [1497484800000, 3454376161.3], [1497571200000, 3464912029.8], [1497657600000, 3464912029.8], [1497744000000, 3464912029.8], [1497830400000, 3464912029.8], [1497916800000, 3464912029.8], [1498003200000, 3464912029.8], [1498089600000, 3464912029.8], [1498176000000, 3474144788.4], [1498262400000, 3474144788.4], [1498348800000, 3474144788.4], [1498435200000, 3474144788.4], [1498521600000, 3474144788.4], [1498608000000, 3474144788.4], [1498694400000, 3474144788.4], [1498780800000, 3517324250.1], [1498867200000, 3517324250.1], [1498953600000, 3517324250.1], [1499040000000, 3517324250.1], [1499126400000, 3517324250.1], [1499212800000, 3517324250.1], [1499299200000, 3517324250.1], [1499385600000, 3533711969.5], [1499472000000, 3533711969.5], [1499558400000, 3533711969.5], [1499644800000, 3533711969.5], [1499731200000, 3533711969.5], [1499817600000, 3533711969.5], [1499904000000, 3533711969.5], [1499990400000, 3551552890.9], [1500076800000, 3551552890.9], [1500163200000, 3551552890.9], [1500249600000, 3551552890.9], [1500336000000, 3551552890.9], [1500422400000, 3551552890.9], [1500508800000, 3551552890.9], [1500595200000, 3559047989.8], [1500681600000, 3559047989.8], [1500768000000, 3559047989.8], [1500854400000, 3559047989.8], [1500940800000, 3559047989.8], [1501027200000, 3559047989.8], [1501113600000, 3559047989.8], [1501200000000, 3561680645.0], [1501286400000, 3561680645.0], [1501372800000, 3561680645.0], [1501459200000, 3561680645.0], [1501545600000, 3561680645.0], [1501632000000, 3561680645.0], [1501718400000, 3561680645.0], [1501804800000, 3565947018.0], [1501891200000, 3565947018.0], [1501977600000, 3565947018.0], [1502064000000, 3565947018.0], [1502150400000, 3565947018.0], [1502236800000, 3565947018.0], [1502323200000, 3565947018.0], [1502409600000, 3567510826.0], [1502496000000, 3567510826.0], [1502582400000, 3567510826.0], [1502668800000, 3567510826.0], [1502755200000, 3567510826.0], [1502841600000, 3567510826.0], [1502928000000, 3567510826.0], [1503014400000, 3567510826.0], [1503100800000, 3567510826.0], [1503187200000, 3567510826.0], [1503273600000, 3567510826.0], [1503360000000, 3567510826.0], [1503446400000, 3567510826.0], [1503532800000, 3567510826.0], [1503619200000, 3567510826.0], [1503705600000, 3567510826.0], [1503792000000, 3567510826.0], [1503878400000, 3567510826.0], [1503964800000, 3567510826.0], [1504051200000, 3567510826.0], [1504137600000, 3567510826.0], [1504224000000, 3572668452.7], [1504310400000, 3572668452.7], [1504396800000, 3572668452.7], [1504483200000, 3572668452.7], [1504569600000, 3572668452.7], [1504656000000, 3572668452.7], [1504742400000, 3572668452.7], [1504828800000, 3627132019.8], [1504915200000, 3627132019.8], [1505001600000, 3627132019.8], [1505088000000, 3627132019.8], [1505174400000, 3627132019.8], [1505260800000, 3627132019.8], [1505347200000, 3627132019.8], [1505433600000, 3683115614.5], [1505520000000, 3683115614.5], [1505606400000, 3683115614.5], [1505692800000, 3683115614.5], [1505779200000, 3683115614.5], [1505865600000, 3683115614.5], [1505952000000, 3683115614.5], [1506038400000, 3707356297.4], [1506124800000, 3707356297.4], [1506211200000, 3707356297.4], [1506297600000, 3707356297.4], [1506384000000, 3707356297.4], [1506470400000, 3707356297.4], [1506556800000, 3707356297.4], [1506643200000, 3747144522.5], [1506729600000, 3747144522.5], [1506816000000, 3747144522.5], [1506902400000, 3747144522.5], [1506988800000, 3747144522.5], [1507075200000, 3747144522.5], [1507161600000, 3747144522.5], [1507248000000, 3769623386.7], [1507334400000, 3769623386.7], [1507420800000, 3769623386.7], [1507507200000, 3769623386.7], [1507593600000, 3769623386.7], [1507680000000, 3769623386.7], [1507766400000, 3769623386.7], [1507852800000, 3824023756.3], [1507939200000, 3824023756.3], [1508025600000, 3824023756.3], [1508112000000, 3824023756.3], [1508198400000, 3824023756.3], [1508284800000, 3824023756.3], [1508371200000, 3824023756.3], [1508457600000, 3839165864.7], [1508544000000, 3839165864.7], [1508630400000, 3839165864.7], [1508716800000, 3839165864.7], [1508803200000, 3839165864.7], [1508889600000, 3839165864.7], [1508976000000, 3839165864.7], [1509062400000, 3839931553.0], [1509148800000, 3839931553.0], [1509235200000, 3839931553.0], [1509321600000, 3839931553.0], [1509408000000, 3839931553.0], [1509494400000, 3839931553.0], [1509580800000, 3839931553.0], [1509667200000, 3848888369.5], [1509753600000, 3848888369.5], [1509840000000, 3848888369.5], [1509926400000, 3848888369.5], [1510012800000, 3848888369.5], [1510099200000, 3848888369.5], [1510185600000, 3848888369.5], [1510272000000, 3852008408.6], [1510358400000, 3852008408.6], [1510444800000, 3852008408.6], [1510531200000, 3852008408.6], [1510617600000, 3852008408.6], [1510704000000, 3852008408.6], [1510790400000, 3852008408.6], [1510876800000, 3853152259.7], [1510963200000, 3853152259.7], [1511049600000, 3853152259.7], [1511136000000, 3853152259.7], [1511222400000, 3853152259.7], [1511308800000, 3853152259.7], [1511395200000, 3853152259.7], [1511481600000, 3854751135.1], [1511568000000, 3854751135.1], [1511654400000, 3854751135.1], [1511740800000, 3854751135.1], [1511827200000, 3854751135.1], [1511913600000, 3854751135.1], [1512000000000, 3854751135.1], [1512086400000, 3865332394.2], [1512172800000, 3865332394.2], [1512259200000, 3865332394.2], [1512345600000, 3865332394.2], [1512432000000, 3865332394.2], [1512518400000, 3865332394.2], [1512604800000, 3865332394.2], [1512691200000, 3913754882.4], [1512777600000, 3913754882.4], [1512864000000, 3913754882.4], [1512950400000, 3913754882.4], [1513036800000, 3913754882.4], [1513123200000, 3913754882.4], [1513209600000, 3913754882.4], [1513296000000, 3961599594.0], [1513382400000, 3961599594.0], [1513468800000, 3961599594.0], [1513555200000, 3961599594.0], [1513641600000, 3961599594.0], [1513728000000, 3961599594.0], [1513814400000, 3961599594.0], [1513900800000, 4006371914.5], [1513987200000, 4006371914.5], [1514073600000, 4006371914.5], [1514160000000, 4006371914.5], [1514246400000, 4006371914.5], [1514332800000, 4006371914.5], [1514419200000, 4006371914.5], [1514505600000, 4036966874.6], [1514592000000, 4036966874.6], [1514678400000, 4036966874.6], [1514764800000, 4036966874.6], [1514851200000, 4036966874.6], [1514937600000, 4036966874.6], [1515024000000, 4036966874.6], [1515110400000, 4094629146.6], [1515196800000, 4094629146.6], [1515283200000, 4094629146.6], [1515369600000, 4094629146.6], [1515456000000, 4094629146.6], [1515542400000, 4094629146.6], [1515628800000, 4094629146.6], [1515715200000, 4121388430.3], [1515801600000, 4121388430.3], [1515888000000, 4121388430.3], [1515974400000, 4121388430.3], [1516060800000, 4121388430.3], [1516147200000, 4121388430.3], [1516233600000, 4121388430.3], [1516320000000, 4130367685.4], [1516406400000, 4130367685.4], [1516492800000, 4130367685.4], [1516579200000, 4130367685.4], [1516665600000, 4130367685.4], [1516752000000, 4130367685.4], [1516838400000, 4130367685.4], [1516924800000, 4130492226.0], [1517011200000, 4130492226.0], [1517097600000, 4130492226.0], [1517184000000, 4130492226.0], [1517270400000, 4130492226.0], [1517356800000, 4130492226.0], [1517443200000, 4130492226.0], [1517529600000, 4130492226.0], [1517616000000, 4130492226.0], [1517702400000, 4130492226.0], [1517788800000, 4130492226.0], [1517875200000, 4130492226.0], [1517961600000, 4130492226.0], [1518048000000, 4130492226.0], [1518134400000, 4137425464.6], [1518220800000, 4137425464.6], [1518307200000, 4137425464.6], [1518393600000, 4137425464.6], [1518480000000, 4137425464.6], [1518566400000, 4137425464.6], [1518652800000, 4137425464.6], [1518739200000, 4163188852.0], [1518825600000, 4163188852.0], [1518912000000, 4163188852.0], [1518998400000, 4163188852.0], [1519084800000, 4163188852.0], [1519171200000, 4163188852.0], [1519257600000, 4163188852.0], [1519344000000, 4188009091.6], [1519430400000, 4188009091.6], [1519516800000, 4188009091.6], [1519603200000, 4188009091.6], [1519689600000, 4188009091.6], [1519776000000, 4188009091.6], [1519862400000, 4188009091.6], [1519948800000, 4201958584.4], [1520035200000, 4201958584.4], [1520121600000, 4201958584.4], [1520208000000, 4201958584.4], [1520294400000, 4201958584.4], [1520380800000, 4201958584.4], [1520467200000, 4201958584.4], [1520553600000, 4229810198.6], [1520640000000, 4229810198.6], [1520726400000, 4229810198.6], [1520812800000, 4229810198.6], [1520899200000, 4229810198.6], [1520985600000, 4229810198.6], [1521072000000, 4229810198.6], [1521158400000, 4250833200.9], [1521244800000, 4250833200.9], [1521331200000, 4250833200.9], [1521417600000, 4250833200.9], [1521504000000, 4250833200.9], [1521590400000, 4250833200.9], [1521676800000, 4250833200.9], [1521763200000, 4289586633.8], [1521849600000, 4289586633.8], [1521936000000, 4289586633.8], [1522022400000, 4289586633.8], [1522108800000, 4289586633.8], [1522195200000, 4289586633.8], [1522281600000, 4289586633.8], [1522368000000, 4340918633.5], [1522454400000, 4340918633.5], [1522540800000, 4340918633.5], [1522627200000, 4340918633.5], [1522713600000, 4340918633.5], [1522800000000, 4340918633.5], [1522886400000, 4340918633.5], [1522972800000, 4389524454.7], [1523059200000, 4389524454.7], [1523145600000, 4389524454.7], [1523232000000, 4389524454.7], [1523318400000, 4389524454.7], [1523404800000, 4389524454.7], [1523491200000, 4389524454.7], [1523577600000, 4415776600.0], [1523664000000, 4415776600.0], [1523750400000, 4415776600.0], [1523836800000, 4415776600.0], [1523923200000, 4415776600.0], [1524009600000, 4415776600.0], [1524096000000, 4415776600.0], [1524182400000, 4448152410.1], [1524268800000, 4448152410.1], [1524355200000, 4448152410.1], [1524441600000, 4448152410.1], [1524528000000, 4448152410.1], [1524614400000, 4448152410.1], [1524700800000, 4448152410.1], [1524787200000, 4481219472.7], [1524873600000, 4481219472.7], [1524960000000, 4481219472.7], [1525046400000, 4481219472.7], [1525132800000, 4481219472.7], [1525219200000, 4481219472.7], [1525305600000, 4481219472.7], [1525392000000, 4509558579.1], [1525478400000, 4509558579.1], [1525564800000, 4509558579.1], [1525651200000, 4509558579.1], [1525737600000, 4509558579.1], [1525824000000, 4509558579.1], [1525910400000, 4509558579.1], [1525996800000, 4515240524.1], [1526083200000, 4515240524.1], [1526169600000, 4515240524.1], [1526256000000, 4515240524.1], [1526342400000, 4515240524.1], [1526428800000, 4515240524.1], [1526515200000, 4515240524.1], [1526601600000, 4533993353.4], [1526688000000, 4533993353.4], [1526774400000, 4533993353.4], [1526860800000, 4533993353.4], [1526947200000, 4533993353.4], [1527033600000, 4533993353.4], [1527120000000, 4533993353.4], [1527206400000, 4536257204.0], [1527292800000, 4536257204.0], [1527379200000, 4536257204.0], [1527465600000, 4536257204.0], [1527552000000, 4536257204.0], [1527638400000, 4536257204.0], [1527724800000, 4536257204.0], [1527811200000, 4561189705.6], [1527897600000, 4561189705.6], [1527984000000, 4561189705.6], [1528070400000, 4561189705.6], [1528156800000, 4561189705.6], [1528243200000, 4561189705.6], [1528329600000, 4561189705.6], [1528416000000, 4579162535.4], [1528502400000, 4579162535.4], [1528588800000, 4579162535.4], [1528675200000, 4579162535.4], [1528761600000, 4579162535.4], [1528848000000, 4579162535.4], [1528934400000, 4579162535.4], [1529020800000, 4579199413.0], [1529107200000, 4579199413.0], [1529193600000, 4579199413.0], [1529280000000, 4579199413.0], [1529366400000, 4579199413.0], [1529452800000, 4579199413.0], [1529539200000, 4579199413.0], [1529625600000, 4579230272.4], [1529712000000, 4579230272.4], [1529798400000, 4579230272.4], [1529884800000, 4579230272.4], [1529971200000, 4579230272.4], [1530057600000, 4579230272.4], [1530144000000, 4579230272.4], [1530230400000, 4611610161.9], [1530316800000, 4611610161.9], [1530403200000, 4611610161.9], [1530489600000, 4611610161.9], [1530576000000, 4611610161.9], [1530662400000, 4611610161.9], [1530748800000, 4611610161.9], [1530835200000, 4668906585.7], [1530921600000, 4668906585.7], [1531008000000, 4668906585.7], [1531094400000, 4668906585.7], [1531180800000, 4668906585.7], [1531267200000, 4668906585.7], [1531353600000, 4668906585.7], [1531440000000, 4711454679.5], [1531526400000, 4711454679.5], [1531612800000, 4711454679.5], [1531699200000, 4711454679.5], [1531785600000, 4711454679.5], [1531872000000, 4711454679.5], [1531958400000, 4711454679.5], [1532044800000, 4747686979.8], [1532131200000, 4747686979.8], [1532217600000, 4747686979.8], [1532304000000, 4747686979.8], [1532390400000, 4747686979.8], [1532476800000, 4747686979.8], [1532563200000, 4747686979.8], [1532649600000, 4772156580.9], [1532736000000, 4772156580.9], [1532822400000, 4772156580.9], [1532908800000, 4772156580.9], [1532995200000, 4772156580.9], [1533081600000, 4772156580.9], [1533168000000, 4772156580.9], [1533254400000, 4780442196.0], [1533340800000, 4780442196.0], [1533427200000, 4780442196.0], [1533513600000, 4780442196.0], [1533600000000, 4780442196.0], [1533686400000, 4780442196.0], [1533772800000, 4780442196.0], [1533859200000, 4781805127.9], [1533945600000, 4781805127.9], [1534032000000, 4781805127.9], [1534118400000, 4781805127.9], [1534204800000, 4781805127.9], [1534291200000, 4781805127.9], [1534377600000, 4781805127.9], [1534464000000, 4782088322.8], [1534550400000, 4782088322.8], [1534636800000, 4782088322.8], [1534723200000, 4782088322.8], [1534809600000, 4782088322.8], [1534896000000, 4782088322.8], [1534982400000, 4782088322.8], [1535068800000, 4782131719.0], [1535155200000, 4782131719.0], [1535241600000, 4782131719.0], [1535328000000, 4782131719.0], [1535414400000, 4782131719.0], [1535500800000, 4782131719.0], [1535587200000, 4782131719.0], [1535673600000, 4782131719.0], [1535760000000, 4782131719.0], [1535846400000, 4782131719.0], [1535932800000, 4782131719.0], [1536019200000, 4782131719.0], [1536105600000, 4782131719.0], [1536192000000, 4782131719.0], [1536278400000, 4782131719.0], [1536364800000, 4782131719.0], [1536451200000, 4782131719.0], [1536537600000, 4782131719.0], [1536624000000, 4782131719.0], [1536710400000, 4782131719.0], [1536796800000, 4782131719.0], [1536883200000, 4880892606.3], [1536969600000, 4880892606.3], [1537056000000, 4880892606.3], [1537142400000, 4880892606.3], [1537228800000, 4880892606.3], [1537315200000, 4880892606.3], [1537401600000, 4880892606.3], [1537488000000, 4984423896.1], [1537574400000, 4984423896.1], [1537660800000, 4984423896.1], [1537747200000, 4984423896.1], [1537833600000, 4984423896.1], [1537920000000, 4984423896.1], [1538006400000, 4984423896.1], [1538092800000, 5009103270.4], [1538179200000, 5009103270.4], [1538265600000, 5009103270.4], [1538352000000, 5009103270.4], [1538438400000, 5009103270.4], [1538524800000, 5009103270.4], [1538611200000, 5009103270.4], [1538697600000, 5041689025.3], [1538784000000, 5041689025.3], [1538870400000, 5041689025.3], [1538956800000, 5041689025.3], [1539043200000, 5041689025.3], [1539129600000, 5041689025.3], [1539216000000, 5041689025.3], [1539302400000, 5097672356.1], [1539388800000, 5097672356.1], [1539475200000, 5097672356.1], [1539561600000, 5097672356.1], [1539648000000, 5097672356.1], [1539734400000, 5097672356.1], [1539820800000, 5097672356.1], [1539907200000, 5121956379.0], [1539993600000, 5121956379.0], [1540080000000, 5121956379.0], [1540166400000, 5121956379.0], [1540252800000, 5121956379.0], [1540339200000, 5121956379.0], [1540425600000, 5121956379.0], [1540512000000, 5180409982.3], [1540598400000, 5180409982.3], [1540684800000, 5180409982.3], [1540771200000, 5180409982.3], [1540857600000, 5180409982.3], [1540944000000, 5180409982.3], [1541030400000, 5180409982.3], [1541116800000, 5239346453.4], [1541203200000, 5239346453.4], [1541289600000, 5239346453.4], [1541376000000, 5239346453.4], [1541462400000, 5239346453.4], [1541548800000, 5239346453.4], [1541635200000, 5239346453.4], [1541721600000, 5273622660.8], [1541808000000, 5273622660.8], [1541894400000, 5273622660.8], [1541980800000, 5273622660.8], [1542067200000, 5273622660.8], [1542153600000, 5273622660.8], [1542240000000, 5273622660.8], [1542326400000, 5305475989.8], [1542412800000, 5305475989.8], [1542499200000, 5305475989.8], [1542585600000, 5305475989.8], [1542672000000, 5305475989.8], [1542758400000, 5305475989.8], [1542844800000, 5305475989.8], [1542931200000, 5326421652.1], [1543017600000, 5326421652.1], [1543104000000, 5326421652.1], [1543190400000, 5326421652.1], [1543276800000, 5326421652.1], [1543363200000, 5326421652.1], [1543449600000, 5326421652.1], [1543536000000, 5338819199.4], [1543622400000, 5338819199.4], [1543708800000, 5338819199.4], [1543795200000, 5338819199.4], [1543881600000, 5338819199.4], [1543968000000, 5338819199.4], [1544054400000, 5338819199.4], [1544140800000, 5362215171.7], [1544227200000, 5362215171.7], [1544313600000, 5362215171.7], [1544400000000, 5362215171.7], [1544486400000, 5362215171.7], [1544572800000, 5362215171.7], [1544659200000, 5362215171.7], [1544745600000, 5388943186.6], [1544832000000, 5388943186.6], [1544918400000, 5388943186.6], [1545004800000, 5388943186.6], [1545091200000, 5388943186.6], [1545177600000, 5388943186.6], [1545264000000, 5388943186.6], [1545350400000, 5429292655.6], [1545436800000, 5429292655.6], [1545523200000, 5429292655.6], [1545609600000, 5429292655.6], [1545696000000, 5429292655.6], [1545782400000, 5429292655.6], [1545868800000, 5429292655.6], [1545955200000, 5483835682.0], [1546041600000, 5483835682.0], [1546128000000, 5483835682.0], [1546214400000, 5483835682.0], [1546300800000, 5483835682.0], [1546387200000, 5483835682.0], [1546473600000, 5483835682.0], [1546560000000, 5527388245.6], [1546646400000, 5527388245.6], [1546732800000, 5527388245.6], [1546819200000, 5527388245.6], [1546905600000, 5527388245.6], [1546992000000, 5527388245.6], [1547078400000, 5527388245.6], [1547164800000, 5563602655.9], [1547251200000, 5563602655.9], [1547337600000, 5563602655.9], [1547424000000, 5563602655.9], [1547510400000, 5563602655.9], [1547596800000, 5563602655.9], [1547683200000, 5563602655.9], [1547769600000, 5592666279.0], [1547856000000, 5592666279.0], [1547942400000, 5592666279.0], [1548028800000, 5592666279.0], [1548115200000, 5592666279.0], [1548201600000, 5592666279.0], [1548288000000, 5592666279.0], [1548374400000, 5621174744.7], [1548460800000, 5621174744.7], [1548547200000, 5621174744.7], [1548633600000, 5621174744.7], [1548720000000, 5621174744.7], [1548806400000, 5621174744.7], [1548892800000, 5621174744.7], [1548979200000, 5655184653.4], [1549065600000, 5655184653.4], [1549152000000, 5655184653.4], [1549238400000, 5655184653.4], [1549324800000, 5655184653.4], [1549411200000, 5655184653.4], [1549497600000, 5655184653.4], [1549584000000, 5685682899.0], [1549670400000, 5685682899.0], [1549756800000, 5685682899.0], [1549843200000, 5685682899.0], [1549929600000, 5685682899.0], [1550016000000, 5685682899.0], [1550102400000, 5685682899.0], [1550188800000, 5685682899.0], [1550275200000, 5685682899.0], [1550361600000, 5685682899.0], [1550448000000, 5685682899.0], [1550534400000, 5685682899.0], [1550620800000, 5685682899.0], [1550707200000, 5685682899.0], [1550793600000, 5685682899.0], [1550880000000, 5685682899.0], [1550966400000, 5685682899.0], [1551052800000, 5685682899.0], [1551139200000, 5685682899.0], [1551225600000, 5685682899.0], [1551312000000, 5685682899.0], [1551398400000, 5685682899.0], [1551484800000, 5685682899.0], [1551571200000, 5685682899.0], [1551657600000, 5685682899.0], [1551744000000, 5685682899.0], [1551830400000, 5685682899.0], [1551916800000, 5685682899.0], [1552003200000, 5685682899.0], [1552089600000, 5685682899.0], [1552176000000, 5685682899.0], [1552262400000, 5685682899.0], [1552348800000, 5685682899.0], [1552435200000, 5685682899.0], [1552521600000, 5685682899.0], [1552608000000, 5685682899.0], [1552694400000, 5685682899.0], [1552780800000, 5685682899.0], [1552867200000, 5685682899.0], [1552953600000, 5685682899.0], [1553040000000, 5685682899.0], [1553126400000, 5685682899.0], [1553212800000, 5685682899.0], [1553299200000, 5685682899.0], [1553385600000, 5685682899.0], [1553472000000, 5685682899.0], [1553558400000, 5685682899.0], [1553644800000, 5685682899.0], [1553731200000, 5685682899.0], [1553817600000, 5812586422.1], [1553904000000, 5812586422.1], [1553990400000, 5812586422.1], [1554076800000, 5812586422.1], [1554163200000, 5812586422.1], [1554249600000, 5812586422.1], [1554336000000, 5812586422.1], [1554422400000, 5848103230.6], [1554508800000, 5848103230.6], [1554595200000, 5848103230.6], [1554681600000, 5848103230.6], [1554768000000, 5848103230.6], [1554854400000, 5848103230.6], [1554940800000, 5848103230.6], [1555027200000, 5867793950.8], [1555113600000, 5867793950.8], [1555200000000, 5867793950.8], [1555286400000, 5867793950.8], [1555372800000, 5867793950.8], [1555459200000, 5867793950.8], [1555545600000, 5867793950.8], [1555632000000, 5878495920.2], [1555718400000, 5878495920.2], [1555804800000, 5878495920.2], [1555891200000, 5878495920.2], [1555977600000, 5878495920.2], [1556064000000, 5878495920.2], [1556150400000, 5878495920.2], [1556236800000, 5902796526.1], [1556323200000, 5902796526.1], [1556409600000, 5902796526.1], [1556496000000, 5902796526.1], [1556582400000, 5902796526.1], [1556668800000, 5902796526.1], [1556755200000, 5902796526.1], [1556841600000, 5921197125.1], [1556928000000, 5921197125.1], [1557014400000, 5921197125.1], [1557100800000, 5921197125.1], [1557187200000, 5921197125.1], [1557273600000, 5921197125.1], [1557360000000, 5921197125.1], [1557446400000, 5924743567.0], [1557532800000, 5924743567.0], [1557619200000, 5924743567.0], [1557705600000, 5924743567.0], [1557792000000, 5924743567.0], [1557878400000, 5924743567.0], [1557964800000, 5924743567.0], [1558051200000, 5935206630.2], [1558137600000, 5935206630.2], [1558224000000, 5935206630.2], [1558310400000, 5935206630.2], [1558396800000, 5935206630.2], [1558483200000, 5935206630.2], [1558569600000, 5935206630.2], [1558656000000, 5952888617.7], [1558742400000, 5952888617.7], [1558828800000, 5952888617.7], [1558915200000, 5952888617.7], [1559001600000, 5952888617.7], [1559088000000, 5952888617.7], [1559174400000, 5952888617.7], [1559260800000, 5957273357.3], [1559347200000, 5957273357.3], [1559433600000, 5957273357.3], [1559520000000, 5957273357.3], [1559606400000, 5957273357.3], [1559692800000, 5957273357.3], [1559779200000, 5957273357.3], [1559865600000, 5960812924.0], [1559952000000, 5960812924.0], [1560038400000, 5960812924.0], [1560124800000, 5960812924.0], [1560211200000, 5960812924.0], [1560297600000, 5960812924.0], [1560384000000, 5960812924.0]], "origin": [[1521417600000, 83797299.781], [1521504000000, 83797299.781], [1521590400000, 83797299.781], [1521676800000, 83797299.781], [1521763200000, 83797826.562], [1521849600000, 83797826.562], [1521936000000, 83797826.562], [1522022400000, 83797826.562], [1522108800000, 83798437.909], [1522195200000, 83798437.909], [1522281600000, 83798437.909], [1522368000000, 83798437.909], [1522454400000, 83799151.255], [1522540800000, 83799151.255], [1522627200000, 83799151.255], [1522713600000, 83799151.255], [1522800000000, 83799701.744], [1522886400000, 83799701.744], [1522972800000, 83799701.744], [1523059200000, 83799701.744], [1523145600000, 83800104.171], [1523232000000, 83800104.171], [1523318400000, 83800104.171], [1523404800000, 83800104.171], [1523491200000, 83800517.545], [1523577600000, 83800517.545], [1523664000000, 83800517.545], [1523750400000, 83800517.545], [1523836800000, 83801525.886], [1523923200000, 83801525.886], [1524009600000, 83801525.886], [1524096000000, 83801525.886], [1524182400000, 83801554.576], [1524268800000, 83801554.576], [1524355200000, 83801554.576], [1524441600000, 83801554.576], [1524528000000, 83801601.006], [1524614400000, 83801601.006], [1524700800000, 83801601.006], [1524787200000, 83801601.006], [1524873600000, 83801632.16], [1524960000000, 83801632.16], [1525046400000, 83801632.16], [1525132800000, 83801632.16], [1525219200000, 83801648.614], [1525305600000, 83801648.614], [1525392000000, 83801648.614], [1525478400000, 83801648.614], [1525564800000, 83801679.332], [1525651200000, 83801679.332], [1525737600000, 83801679.332], [1525824000000, 83801679.332], [1525910400000, 83801698.351], [1525996800000, 83801698.351], [1526083200000, 83801698.351], [1526169600000, 83801698.351], [1526256000000, 83801733.787], [1526342400000, 83801733.787], [1526428800000, 83801733.787], [1526515200000, 83801733.787], [1526601600000, 83801785.367], [1526688000000, 83801785.367], [1526774400000, 83801785.367], [1526860800000, 83801785.367], [1526947200000, 83801797.0], [1527033600000, 83801797.0], [1527120000000, 83801797.0], [1527206400000, 83801797.0], [1527292800000, 83801797.0], [1527379200000, 83801797.0], [1527465600000, 83801797.0], [1527552000000, 83801797.0], [1527638400000, 83803487.095], [1527724800000, 83803487.095], [1527811200000, 83803487.095], [1527897600000, 83803487.095], [1527984000000, 83821424.973], [1528070400000, 83821424.973], [1528156800000, 83821424.973], [1528243200000, 83821424.973], [1528329600000, 83834738.035], [1528416000000, 83834738.035], [1528502400000, 83834738.035], [1528588800000, 83834738.035], [1528675200000, 83835219.682], [1528761600000, 83835219.682], [1528848000000, 83835219.682], [1528934400000, 83835219.682], [1529020800000, 83835273.283], [1529107200000, 83835273.283], [1529193600000, 83835273.283], [1529280000000, 83835273.283], [1529366400000, 83835274.0], [1529452800000, 83835274.0], [1529539200000, 83835274.0], [1529625600000, 83835274.0], [1529712000000, 83835274.0], [1529798400000, 83835274.0], [1529884800000, 83835274.0], [1529971200000, 83835274.0], [1530057600000, 83838069.081], [1530144000000, 83838069.081], [1530230400000, 83838069.081], [1530316800000, 83838069.081], [1530403200000, 83866757.094], [1530489600000, 83866757.094], [1530576000000, 83866757.094], [1530662400000, 83866757.094], [1530748800000, 83904339.598], [1530835200000, 83904339.598], [1530921600000, 83904339.598], [1531008000000, 83904339.598], [1531094400000, 83936703.098], [1531180800000, 83936703.098], [1531267200000, 83936703.098], [1531353600000, 83936703.098], [1531440000000, 83970967.575], [1531526400000, 83970967.575], [1531612800000, 83970967.575], [1531699200000, 83970967.575], [1531785600000, 84013267.917], [1531872000000, 84013267.917], [1531958400000, 84013267.917], [1532044800000, 84013267.917], [1532131200000, 84628660.056], [1532217600000, 84628660.056], [1532304000000, 84628660.056], [1532390400000, 84628660.056], [1532476800000, 84644921.608], [1532563200000, 84644921.608], [1532649600000, 84644921.608], [1532736000000, 84644921.608], [1532822400000, 84652962.277], [1532908800000, 84652962.277], [1532995200000, 84652962.277], [1533081600000, 84652962.277], [1533168000000, 84657684.12], [1533254400000, 84657684.12], [1533340800000, 84657684.12], [1533427200000, 84657684.12], [1533513600000, 84659249.42], [1533600000000, 84659249.42], [1533686400000, 84659249.42], [1533772800000, 84659249.42], [1533859200000, 84659714.893], [1533945600000, 84659714.893], [1534032000000, 84659714.893], [1534118400000, 84659714.893], [1534204800000, 84659811.537], [1534291200000, 84659811.537], [1534377600000, 84659811.537], [1534464000000, 84659811.537], [1534550400000, 84660341.048], [1534636800000, 84660341.048], [1534723200000, 84660341.048], [1534809600000, 84660341.048], [1534896000000, 84670682.784], [1534982400000, 84670682.784], [1535068800000, 84670682.784], [1535155200000, 84670682.784], [1535241600000, 84691813.483], [1535328000000, 84691813.483], [1535414400000, 84691813.483], [1535500800000, 84691813.483], [1535587200000, 84717961.459], [1535673600000, 84717961.459], [1535760000000, 84717961.459], [1535846400000, 84717961.459], [1535932800000, 84756987.747], [1536019200000, 84756987.747], [1536105600000, 84756987.747], [1536192000000, 84756987.747], [1536278400000, 84807156.696], [1536364800000, 84807156.696], [1536451200000, 84807156.696], [1536537600000, 84807156.696], [1536624000000, 84862113.899], [1536710400000, 84862113.899], [1536796800000, 84862113.899], [1536883200000, 84862113.899], [1536969600000, 84880824.665], [1537056000000, 84880824.665], [1537142400000, 84880824.665], [1537228800000, 84880824.665], [1537315200000, 84946509.399], [1537401600000, 84946509.399], [1537488000000, 84946509.399], [1537574400000, 84946509.399], [1537660800000, 85117928.077], [1537747200000, 85117928.077], [1537833600000, 85117928.077], [1537920000000, 85117928.077], [1538006400000, 85151877.448], [1538092800000, 85151877.448], [1538179200000, 85151877.448], [1538265600000, 85151877.448], [1538352000000, 85195274.03], [1538438400000, 85195274.03], [1538524800000, 85195274.03], [1538611200000, 85195274.03], [1538697600000, 85213868.535], [1538784000000, 85213868.535], [1538870400000, 85213868.535], [1538956800000, 85213868.535], [1539043200000, 85263763.555], [1539129600000, 85263763.555], [1539216000000, 85263763.555], [1539302400000, 85263763.555], [1539388800000, 85339873.564], [1539475200000, 85339873.564], [1539561600000, 85339873.564], [1539648000000, 85339873.564], [1539734400000, 85492951.692], [1539820800000, 85492951.692], [1539907200000, 85492951.692], [1539993600000, 85492951.692], [1540080000000, 85643831.24], [1540166400000, 85643831.24], [1540252800000, 85643831.24], [1540339200000, 85643831.24], [1540425600000, 85748002.197], [1540512000000, 85748002.197], [1540598400000, 85748002.197], [1540684800000, 85748002.197], [1540771200000, 85860877.327], [1540857600000, 85860877.327], [1540944000000, 85860877.327], [1541030400000, 85860877.327], [1541116800000, 85977347.845], [1541203200000, 85977347.845], [1541289600000, 85977347.845], [1541376000000, 85977347.845], [1541462400000, 86080902.281], [1541548800000, 86080902.281], [1541635200000, 86080902.281], [1541721600000, 86080902.281], [1541808000000, 86178273.357], [1541894400000, 86178273.357], [1541980800000, 86178273.357], [1542067200000, 86178273.357], [1542153600000, 86287227.035], [1542240000000, 86287227.035], [1542326400000, 86287227.035], [1542412800000, 86287227.035], [1542499200000, 86382677.247], [1542585600000, 86382677.247], [1542672000000, 86382677.247], [1542758400000, 86382677.247], [1542844800000, 86451289.953], [1542931200000, 86451289.953], [1543017600000, 86451289.953], [1543104000000, 86451289.953], [1543190400000, 86517425.006], [1543276800000, 86517425.006], [1543363200000, 86517425.006], [1543449600000, 86517425.006], [1543536000000, 86563100.171], [1543622400000, 86563100.171], [1543708800000, 86563100.171], [1543795200000, 86563100.171], [1543881600000, 86606209.243], [1543968000000, 86606209.243], [1544054400000, 86606209.243], [1544140800000, 86606209.243], [1544227200000, 86707918.215], [1544313600000, 86707918.215], [1544400000000, 86707918.215], [1544486400000, 86707918.215], [1544572800000, 86838767.03], [1544659200000, 86838767.03], [1544745600000, 86838767.03], [1544832000000, 86838767.03], [1544918400000, 86918211.635], [1545004800000, 86918211.635], [1545091200000, 86918211.635], [1545177600000, 86918211.635], [1545264000000, 87057779.888], [1545350400000, 87057779.888], [1545436800000, 87057779.888], [1545523200000, 87057779.888], [1545609600000, 87580814.427], [1545696000000, 87580814.427], [1545782400000, 87580814.427], [1545868800000, 87580814.427], [1545955200000, 88070465.494], [1546041600000, 88070465.494], [1546128000000, 88070465.494], [1546214400000, 88070465.494], [1546300800000, 88090183.39], [1546387200000, 88090183.39], [1546473600000, 88090183.39], [1546560000000, 88090183.39], [1546646400000, 88090648.861], [1546732800000, 88090648.861], [1546819200000, 88090648.861], [1546905600000, 88090648.861], [1546992000000, 88091069.223], [1547078400000, 88091069.223], [1547164800000, 88091069.223], [1547251200000, 88091069.223], [1547337600000, 88091773.443], [1547424000000, 88091773.443], [1547510400000, 88091773.443], [1547596800000, 88091773.443], [1547683200000, 88274454.771], [1547769600000, 88274454.771], [1547856000000, 88274454.771], [1547942400000, 88274454.771], [1548028800000, 88288393.804], [1548115200000, 88288393.804], [1548201600000, 88288393.804], [1548288000000, 88288393.804], [1548374400000, 88289573.711], [1548460800000, 88289573.711], [1548547200000, 88289573.711], [1548633600000, 88289573.711], [1548720000000, 88290192.882], [1548806400000, 88290192.882], [1548892800000, 88290192.882], [1548979200000, 88290192.882], [1549065600000, 88290600.116], [1549152000000, 88290600.116], [1549238400000, 88290600.116], [1549324800000, 88290600.116], [1549411200000, 88291347.162], [1549497600000, 88291347.162], [1549584000000, 88291347.162], [1549670400000, 88291347.162], [1549756800000, 88291473.0], [1549843200000, 88291473.0], [1549929600000, 88291473.0], [1550016000000, 88291473.0], [1550102400000, 88291736.571], [1550188800000, 88291736.571], [1550275200000, 88291736.571], [1550361600000, 88291736.571], [1550448000000, 88293238.46], [1550534400000, 88293238.46], [1550620800000, 88293238.46], [1550707200000, 88293238.46], [1550793600000, 88293899.684], [1550880000000, 88293899.684], [1550966400000, 88293899.684], [1551052800000, 88293899.684], [1551139200000, 88294309.49], [1551225600000, 88294309.49], [1551312000000, 88294309.49], [1551398400000, 88294309.49], [1551484800000, 88294318.782], [1551571200000, 88294318.782], [1551657600000, 88294318.782], [1551744000000, 88294318.782], [1551830400000, 88294320.0], [1551916800000, 88294320.0], [1552003200000, 88294320.0], [1552089600000, 88294320.0], [1552176000000, 88294334.673], [1552262400000, 88294334.673], [1552348800000, 88294334.673], [1552435200000, 88294334.673], [1552521600000, 88295190.409], [1552608000000, 88295190.409], [1552694400000, 88295190.409], [1552780800000, 88295190.409], [1552867200000, 88297546.969], [1552953600000, 88297546.969], [1553040000000, 88297546.969], [1553126400000, 88297546.969], [1553212800000, 88297582.024], [1553299200000, 88297582.024], [1553385600000, 88297582.024], [1553472000000, 88297582.024], [1553558400000, 88297586.0], [1553644800000, 88297586.0], [1553731200000, 88297586.0], [1553817600000, 88297586.0], [1553904000000, 88297600.556], [1553990400000, 88297600.556], [1554076800000, 88297600.556], [1554163200000, 88297600.556], [1554249600000, 88297612.232], [1554336000000, 88297612.232], [1554422400000, 88297612.232], [1554508800000, 88297612.232], [1554595200000, 88297677.569], [1554681600000, 88297677.569], [1554768000000, 88297677.569], [1554854400000, 88297677.569], [1554940800000, 88352782.448], [1555027200000, 88352782.448], [1555113600000, 88352782.448], [1555200000000, 88352782.448], [1555286400000, 89248845.516], [1555372800000, 89248845.516], [1555459200000, 89248845.516], [1555545600000, 89248845.516], [1555632000000, 89251175.1], [1555718400000, 89251175.1], [1555804800000, 89251175.1], [1555891200000, 89251175.1], [1555977600000, 89255516.567], [1556064000000, 89255516.567], [1556150400000, 89255516.567], [1556236800000, 89255516.567], [1556323200000, 89255527.731], [1556409600000, 89255527.731], [1556496000000, 89255527.731], [1556582400000, 89255527.731], [1556668800000, 89255530.0], [1556755200000, 89255530.0], [1556841600000, 89255530.0], [1556928000000, 89255530.0], [1557014400000, 89255542.221], [1557100800000, 89255542.221], [1557187200000, 89255542.221], [1557273600000, 89255542.221], [1557360000000, 89255546.616], [1557446400000, 89255546.616], [1557532800000, 89255546.616], [1557619200000, 89255546.616], [1557705600000, 89255550.912], [1557792000000, 89255550.912], [1557878400000, 89255550.912], [1557964800000, 89255550.912], [1558051200000, 89269730.223], [1558137600000, 89269730.223], [1558224000000, 89269730.223], [1558310400000, 89269730.223], [1558396800000, 89279199.924], [1558483200000, 89279199.924], [1558569600000, 89279199.924], [1558656000000, 89279199.924], [1558742400000, 89279203.564], [1558828800000, 89279203.564], [1558915200000, 89279203.564], [1559001600000, 89279203.564], [1559088000000, 89288593.455], [1559174400000, 89288593.455], [1559260800000, 89288593.455], [1559347200000, 89288593.455], [1559433600000, 89297983.604], [1559520000000, 89297983.604], [1559606400000, 89297983.604], [1559692800000, 89297983.604], [1559779200000, 89298656.272], [1559865600000, 89298656.272], [1559952000000, 89298656.272], [1560038400000, 89298656.272], [1560124800000, 89299093.404], [1560211200000, 89299093.404], [1560297600000, 89299093.404], [1560384000000, 89299093.404]], "revision": [[1521417600000, 974655669.1], [1521504000000, 974655669.1], [1521590400000, 974655669.1], [1521676800000, 974655669.1], [1521763200000, 979068871.16], [1521849600000, 979068871.16], [1521936000000, 979068871.16], [1522022400000, 979068871.16], [1522108800000, 984003431.12], [1522195200000, 984003431.12], [1522281600000, 984003431.12], [1522368000000, 984003431.12], [1522454400000, 989843918.99], [1522540800000, 989843918.99], [1522627200000, 989843918.99], [1522713600000, 989843918.99], [1522800000000, 995422166.31], [1522886400000, 995422166.31], [1522972800000, 995422166.31], [1523059200000, 995422166.31], [1523145600000, 1000047974.4], [1523232000000, 1000047974.4], [1523318400000, 1000047974.4], [1523404800000, 1000047974.4], [1523491200000, 1002152722.9], [1523577600000, 1002152722.9], [1523664000000, 1002152722.9], [1523750400000, 1002152722.9], [1523836800000, 1006717295.8], [1523923200000, 1006717295.8], [1524009600000, 1006717295.8], [1524096000000, 1006717295.8], [1524182400000, 1009661579.3], [1524268800000, 1009661579.3], [1524355200000, 1009661579.3], [1524441600000, 1009661579.3], [1524528000000, 1013687779.4], [1524614400000, 1013687779.4], [1524700800000, 1013687779.4], [1524787200000, 1013687779.4], [1524873600000, 1016412029.3], [1524960000000, 1016412029.3], [1525046400000, 1016412029.3], [1525132800000, 1016412029.3], [1525219200000, 1019473519.1], [1525305600000, 1019473519.1], [1525392000000, 1019473519.1], [1525478400000, 1019473519.1], [1525564800000, 1021088944.1], [1525651200000, 1021088944.1], [1525737600000, 1021088944.1], [1525824000000, 1021088944.1], [1525910400000, 1021317229.6], [1525996800000, 1021317229.6], [1526083200000, 1021317229.6], [1526169600000, 1021317229.6], [1526256000000, 1022034778.6], [1526342400000, 1022034778.6], [1526428800000, 1022034778.6], [1526515200000, 1022034778.6], [1526601600000, 1024739584.1], [1526688000000, 1024739584.1], [1526774400000, 1024739584.1], [1526860800000, 1024739584.1], [1526947200000, 1025931256.0], [1527033600000, 1025931256.0], [1527120000000, 1025931256.0], [1527206400000, 1025931256.0], [1527292800000, 1025931256.0], [1527379200000, 1025931256.0], [1527465600000, 1025931256.0], [1527552000000, 1025931256.0], [1527638400000, 1026977759.4], [1527724800000, 1026977759.4], [1527811200000, 1026977759.4], [1527897600000, 1026977759.4], [1527984000000, 1032764397.8], [1528070400000, 1032764397.8], [1528156800000, 1032764397.8], [1528243200000, 1032764397.8], [1528329600000, 1036902827.7], [1528416000000, 1036902827.7], [1528502400000, 1036902827.7], [1528588800000, 1036902827.7], [1528675200000, 1037016758.2], [1528761600000, 1037016758.2], [1528848000000, 1037016758.2], [1528934400000, 1037016758.2], [1529020800000, 1037021406.0], [1529107200000, 1037021406.0], [1529193600000, 1037021406.0], [1529280000000, 1037021406.0], [1529366400000, 1037021406.0], [1529452800000, 1037021406.0], [1529539200000, 1037021406.0], [1529625600000, 1037021406.0], [1529712000000, 1037021406.0], [1529798400000, 1037021406.0], [1529884800000, 1037021406.0], [1529971200000, 1037021406.0], [1530057600000, 1038749978.7], [1530144000000, 1038749978.7], [1530230400000, 1038749978.7], [1530316800000, 1038749978.7], [1530403200000, 1047539525.0], [1530489600000, 1047539525.0], [1530576000000, 1047539525.0], [1530662400000, 1047539525.0], [1530748800000, 1055962382.8], [1530835200000, 1055962382.8], [1530921600000, 1055962382.8], [1531008000000, 1055962382.8], [1531094400000, 1061655034.8], [1531180800000, 1061655034.8], [1531267200000, 1061655034.8], [1531353600000, 1061655034.8], [1531440000000, 1066834629.2], [1531526400000, 1066834629.2], [1531612800000, 1066834629.2], [1531699200000, 1066834629.2], [1531785600000, 1070995300.8], [1531872000000, 1070995300.8], [1531958400000, 1070995300.8], [1532044800000, 1070995300.8], [1532131200000, 1074487054.6], [1532217600000, 1074487054.6], [1532304000000, 1074487054.6], [1532390400000, 1074487054.6], [1532476800000, 1077431390.7], [1532563200000, 1077431390.7], [1532649600000, 1077431390.7], [1532736000000, 1077431390.7], [1532822400000, 1079516345.4], [1532908800000, 1079516345.4], [1532995200000, 1079516345.4], [1533081600000, 1079516345.4], [1533168000000, 1080576765.3], [1533254400000, 1080576765.3], [1533340800000, 1080576765.3], [1533427200000, 1080576765.3], [1533513600000, 1080814167.8], [1533600000000, 1080814167.8], [1533686400000, 1080814167.8], [1533772800000, 1080814167.8], [1533859200000, 1080885892.0], [1533945600000, 1080885892.0], [1534032000000, 1080885892.0], [1534118400000, 1080885892.0], [1534204800000, 1080887640.1], [1534291200000, 1080887640.1], [1534377600000, 1080887640.1], [1534464000000, 1080887640.1], [1534550400000, 1080997776.4], [1534636800000, 1080997776.4], [1534723200000, 1080997776.4], [1534809600000, 1080997776.4], [1534896000000, 1081002442.0], [1534982400000, 1081002442.0], [1535068800000, 1081002442.0], [1535155200000, 1081002442.0], [1535241600000, 1081002442.0], [1535328000000, 1081002442.0], [1535414400000, 1081002442.0], [1535500800000, 1081002442.0], [1535587200000, 1081002442.0], [1535673600000, 1081002442.0], [1535760000000, 1081002442.0], [1535846400000, 1081002442.0], [1535932800000, 1081002442.0], [1536019200000, 1081002442.0], [1536105600000, 1081002442.0], [1536192000000, 1081002442.0], [1536278400000, 1081002442.0], [1536364800000, 1081002442.0], [1536451200000, 1081002442.0], [1536537600000, 1081002442.0], [1536624000000, 1081002442.0], [1536710400000, 1081002442.0], [1536796800000, 1081002442.0], [1536883200000, 1081002442.0], [1536969600000, 1093795098.7], [1537056000000, 1093795098.7], [1537142400000, 1093795098.7], [1537228800000, 1093795098.7], [1537315200000, 1117918667.8], [1537401600000, 1117918667.8], [1537488000000, 1117918667.8], [1537574400000, 1117918667.8], [1537660800000, 1121752736.0], [1537747200000, 1121752736.0], [1537833600000, 1121752736.0], [1537920000000, 1121752736.0], [1538006400000, 1125354079.3], [1538092800000, 1125354079.3], [1538179200000, 1125354079.3], [1538265600000, 1125354079.3], [1538352000000, 1126348335.0], [1538438400000, 1126348335.0], [1538524800000, 1126348335.0], [1538611200000, 1126348335.0], [1538697600000, 1130523335.2], [1538784000000, 1130523335.2], [1538870400000, 1130523335.2], [1538956800000, 1130523335.2], [1539043200000, 1136959838.4], [1539129600000, 1136959838.4], [1539216000000, 1136959838.4], [1539302400000, 1136959838.4], [1539388800000, 1143758715.7], [1539475200000, 1143758715.7], [1539561600000, 1143758715.7], [1539648000000, 1143758715.7], [1539734400000, 1148672535.4], [1539820800000, 1148672535.4], [1539907200000, 1148672535.4], [1539993600000, 1148672535.4], [1540080000000, 1148956138.0], [1540166400000, 1148956138.0], [1540252800000, 1148956138.0], [1540339200000, 1148956138.0], [1540425600000, 1153110961.1], [1540512000000, 1153110961.1], [1540598400000, 1153110961.1], [1540684800000, 1153110961.1], [1540771200000, 1166338659.7], [1540857600000, 1166338659.7], [1540944000000, 1166338659.7], [1541030400000, 1166338659.7], [1541116800000, 1170959900.9], [1541203200000, 1170959900.9], [1541289600000, 1170959900.9], [1541376000000, 1170959900.9], [1541462400000, 1175457939.1], [1541548800000, 1175457939.1], [1541635200000, 1175457939.1], [1541721600000, 1175457939.1], [1541808000000, 1179142494.6], [1541894400000, 1179142494.6], [1541980800000, 1179142494.6], [1542067200000, 1179142494.6], [1542153600000, 1182889098.2], [1542240000000, 1182889098.2], [1542326400000, 1182889098.2], [1542412800000, 1182889098.2], [1542499200000, 1186042371.8], [1542585600000, 1186042371.8], [1542672000000, 1186042371.8], [1542758400000, 1186042371.8], [1542844800000, 1188232038.0], [1542931200000, 1188232038.0], [1543017600000, 1188232038.0], [1543104000000, 1188232038.0], [1543190400000, 1190172102.8], [1543276800000, 1190172102.8], [1543363200000, 1190172102.8], [1543449600000, 1190172102.8], [1543536000000, 1191514855.1], [1543622400000, 1191514855.1], [1543708800000, 1191514855.1], [1543795200000, 1191514855.1], [1543881600000, 1192528950.0], [1543968000000, 1192528950.0], [1544054400000, 1192528950.0], [1544140800000, 1192528950.0], [1544227200000, 1196204815.2], [1544313600000, 1196204815.2], [1544400000000, 1196204815.2], [1544486400000, 1196204815.2], [1544572800000, 1199706896.5], [1544659200000, 1199706896.5], [1544745600000, 1199706896.5], [1544832000000, 1199706896.5], [1544918400000, 1201747973.0], [1545004800000, 1201747973.0], [1545091200000, 1201747973.0], [1545177600000, 1201747973.0], [1545264000000, 1205300265.6], [1545350400000, 1205300265.6], [1545436800000, 1205300265.6], [1545523200000, 1205300265.6], [1545609600000, 1211843118.6], [1545696000000, 1211843118.6], [1545782400000, 1211843118.6], [1545868800000, 1211843118.6], [1545955200000, 1218258299.9], [1546041600000, 1218258299.9], [1546128000000, 1218258299.9], [1546214400000, 1218258299.9], [1546300800000, 1226545272.0], [1546387200000, 1226545272.0], [1546473600000, 1226545272.0], [1546560000000, 1226545272.0], [1546646400000, 1231953844.0], [1546732800000, 1231953844.0], [1546819200000, 1231953844.0], [1546905600000, 1231953844.0], [1546992000000, 1235916119.7], [1547078400000, 1235916119.7], [1547164800000, 1235916119.7], [1547251200000, 1235916119.7], [1547337600000, 1241771030.0], [1547424000000, 1241771030.0], [1547510400000, 1241771030.0], [1547596800000, 1241771030.0], [1547683200000, 1243705138.9], [1547769600000, 1243705138.9], [1547856000000, 1243705138.9], [1547942400000, 1243705138.9], [1548028800000, 1248389319.0], [1548115200000, 1248389319.0], [1548201600000, 1248389319.0], [1548288000000, 1248389319.0], [1548374400000, 1251349255.9], [1548460800000, 1251349255.9], [1548547200000, 1251349255.9], [1548633600000, 1251349255.9], [1548720000000, 1254135192.0], [1548806400000, 1254135192.0], [1548892800000, 1254135192.0], [1548979200000, 1254135192.0], [1549065600000, 1258175737.0], [1549152000000, 1258175737.0], [1549238400000, 1258175737.0], [1549324800000, 1258175737.0], [1549411200000, 1262321770.0], [1549497600000, 1262321770.0], [1549584000000, 1262321770.0], [1549670400000, 1262321770.0], [1549756800000, 1262321770.0], [1549843200000, 1262321770.0], [1549929600000, 1262321770.0], [1550016000000, 1262321770.0], [1550102400000, 1262321770.0], [1550188800000, 1262321770.0], [1550275200000, 1262321770.0], [1550361600000, 1262321770.0], [1550448000000, 1262321770.0], [1550534400000, 1262321770.0], [1550620800000, 1262321770.0], [1550707200000, 1262321770.0], [1550793600000, 1262321770.0], [1550880000000, 1262321770.0], [1550966400000, 1262321770.0], [1551052800000, 1262321770.0], [1551139200000, 1262321770.0], [1551225600000, 1262321770.0], [1551312000000, 1262321770.0], [1551398400000, 1262321770.0], [1551484800000, 1262321770.0], [1551571200000, 1262321770.0], [1551657600000, 1262321770.0], [1551744000000, 1262321770.0], [1551830400000, 1262321770.0], [1551916800000, 1262321770.0], [1552003200000, 1262321770.0], [1552089600000, 1262321770.0], [1552176000000, 1262321770.0], [1552262400000, 1262321770.0], [1552348800000, 1262321770.0], [1552435200000, 1262321770.0], [1552521600000, 1262321770.0], [1552608000000, 1262321770.0], [1552694400000, 1262321770.0], [1552780800000, 1262321770.0], [1552867200000, 1262321770.0], [1552953600000, 1262321770.0], [1553040000000, 1262321770.0], [1553126400000, 1262321770.0], [1553212800000, 1262321770.0], [1553299200000, 1262321770.0], [1553385600000, 1262321770.0], [1553472000000, 1262321770.0], [1553558400000, 1266214854.9], [1553644800000, 1266214854.9], [1553731200000, 1266214854.9], [1553817600000, 1266214854.9], [1553904000000, 1288250515.2], [1553990400000, 1288250515.2], [1554076800000, 1288250515.2], [1554163200000, 1288250515.2], [1554249600000, 1292164078.2], [1554336000000, 1292164078.2], [1554422400000, 1292164078.2], [1554508800000, 1292164078.2], [1554595200000, 1294373475.8], [1554681600000, 1294373475.8], [1554768000000, 1294373475.8], [1554854400000, 1294373475.8], [1554940800000, 1298059447.0], [1555027200000, 1298059447.0], [1555113600000, 1298059447.0], [1555200000000, 1298059447.0], [1555286400000, 1299703291.9], [1555372800000, 1299703291.9], [1555459200000, 1299703291.9], [1555545600000, 1299703291.9], [1555632000000, 1300369846.6], [1555718400000, 1300369846.6], [1555804800000, 1300369846.6], [1555891200000, 1300369846.6], [1555977600000, 1301968572.1], [1556064000000, 1301968572.1], [1556150400000, 1301968572.1], [1556236800000, 1301968572.1], [1556323200000, 1304644027.4], [1556409600000, 1304644027.4], [1556496000000, 1304644027.4], [1556582400000, 1304644027.4], [1556668800000, 1307819489.9], [1556755200000, 1307819489.9], [1556841600000, 1307819489.9], [1556928000000, 1307819489.9], [1557014400000, 1309261987.7], [1557100800000, 1309261987.7], [1557187200000, 1309261987.7], [1557273600000, 1309261987.7], [1557360000000, 1309791295.1], [1557446400000, 1309791295.1], [1557532800000, 1309791295.1], [1557619200000, 1309791295.1], [1557705600000, 1310406607.9], [1557792000000, 1310406607.9], [1557878400000, 1310406607.9], [1557964800000, 1310406607.9], [1558051200000, 1311780433.4], [1558137600000, 1311780433.4], [1558224000000, 1311780433.4], [1558310400000, 1311780433.4], [1558396800000, 1314737418.8], [1558483200000, 1314737418.8], [1558569600000, 1314737418.8], [1558656000000, 1314737418.8], [1558742400000, 1316714079.2], [1558828800000, 1316714079.2], [1558915200000, 1316714079.2], [1559001600000, 1316714079.2], [1559088000000, 1317277608.6], [1559174400000, 1317277608.6], [1559260800000, 1317277608.6], [1559347200000, 1317277608.6], [1559433600000, 1317718958.1], [1559520000000, 1317718958.1], [1559606400000, 1317718958.1], [1559692800000, 1317718958.1], [1559779200000, 1318481892.0], [1559865600000, 1318481892.0], [1559952000000, 1318481892.0], [1560038400000, 1318481892.0], [1560124800000, 1318481892.0], [1560211200000, 1318481892.0], [1560297600000, 1318481892.0], [1560384000000, 1318481892.0]]} diff --git a/site-modules/profile/files/swh/deploy/worker/swh-worker-generator b/site-modules/profile/files/swh/deploy/worker/swh-worker-generator deleted file mode 100755 index 2e0bd77c..00000000 --- a/site-modules/profile/files/swh/deploy/worker/swh-worker-generator +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# This systemd generator creates dependency symlinks that make all swh-worker -# instances in /etc/softwareheritage/worker/ be started/stopped/reloaded when -# swh-worker.service is started/stopped/reloaded. - -set -eu - -gendir="$1" -wantdir="$1/swh-worker.service.wants" -swhworkerservice="/etc/systemd/system/swh-worker@.service" - -mkdir -p "$wantdir" - -for conf in /etc/softwareheritage/worker/*.ini; do - test -e "$conf" || continue - base=$(basename $conf) - instancename=$(basename $base .ini) - ln -s "$swhworkerservice" "$wantdir/swh-worker@$instancename.service" -done - -exit 0 diff --git a/site-modules/profile/files/swh/deploy/worker/swh-worker@.service b/site-modules/profile/files/swh/deploy/worker/swh-worker@.service index ef55b70e..22baacd7 100644 --- a/site-modules/profile/files/swh/deploy/worker/swh-worker@.service +++ b/site-modules/profile/files/swh/deploy/worker/swh-worker@.service @@ -1,24 +1,25 @@ [Unit] Description=Software Heritage Worker (%i) After=network.target [Service] User=swhworker Group=swhworker Type=simple -Environment=SWH_WORKER_INSTANCE=%i +Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/%i.yml +Environment=SWH_LOG_TARGET=journal Environment=CONCURRENCY=10 Environment=MAX_TASKS_PER_CHILD=5 Environment=LOGLEVEL=info ExecStart=/usr/bin/python3 -m celery worker --app=swh.scheduler.celery_backend.config.app --pool=prefork --events --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat -n %i.%%h KillMode=process KillSignal=SIGTERM TimeoutStopSec=0 Restart=always RestartSec=10 [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/site-modules/profile/files/weekly_report_bot/weekly-report-bot b/site-modules/profile/files/weekly_report_bot/weekly-report-bot new file mode 100755 index 00000000..3b46b9b7 --- /dev/null +++ b/site-modules/profile/files/weekly_report_bot/weekly-report-bot @@ -0,0 +1,17 @@ +#!/bin/bash + +DEST=swh-team@inria.fr + +/usr/lib/sendmail -t < +To: $DEST +Subject: [Weekly Report] Week $(date +%G/%V) + +Beep boop, I'm a bot. + +Here's a thread for your weekly reports! + +Yours faithfully, +-- +Cron for $(whoami)@$(hostname) +EOF diff --git a/site-modules/profile/functions/cron/validate_field.pp b/site-modules/profile/functions/cron/validate_field.pp new file mode 100644 index 00000000..d0273fff --- /dev/null +++ b/site-modules/profile/functions/cron/validate_field.pp @@ -0,0 +1,173 @@ +function profile::cron::validate_field( + String $field, + Variant[Integer, String, Array[Variant[Integer, String]]] $value, + Optional[Array[String]] $valid_strings, + Optional[Tuple[Integer, Integer]] $int_range, + Optional[String] $seed = undef, + Boolean $arrays_valid = true, +) >> Tuple[Optional[String], Array[String]] { + + if $value =~ Array and !$arrays_valid { + return [undef, ["Cannot nest Arrays in value for ${field}"]] + } + + $_valid_strings = pick_default($valid_strings, []) + $_int_range = pick_default($int_range, [0, 0]) + + case $value { + Array: { + $ret = $value.map |$_value| { + profile::cron::validate_field( + $field, + $_value, + $valid_strings, + $int_range, + $seed, + false, + ) + } + + $_failed_values = $ret.filter |$_value| { + $_value[0] == undef + } + + if empty($_failed_values) { + return [$ret.map |$_value| {$_value[0]}.join(','), []] + } else { + return [undef, $_failed_values.map |$_value| { $_value[1] }.flatten] + } + } + + *($_valid_strings + ['*']): { + return [$value, []] + } + + 'fqdn_rand': { + [$_min, $_max] = $_int_range + return [$_min + fqdn_rand($_max - $_min + 1, "${seed}_${field}"), []] + } + + /^\d+$/: { + return profile::cron::validate_field( + $field, + Integer($value, 10), + $valid_strings, + $int_range, + $seed, + $arrays_valid, + ) + } + + /[ ,]/: { + return profile::cron::validate_field( + $field, + $value.split('[ ,]'), + $valid_strings, + $int_range, + $seed, + $arrays_valid, + ) + } + + /^([0-9a-z]+)-([0-9a-z]+)$/: { + $min_valid = profile::cron::validate_field( + $field, + $1, + $valid_strings, + $int_range, + $seed, + false, + ) + + $max_valid = profile::cron::validate_field( + $field, + $2, + $valid_strings, + $int_range, + $seed, + false, + ) + + $_errors = $min_valid[1] + $max_valid[1] + if empty($_errors) { + $_parsed_min = $min_valid[0] + $_parsed_max = $max_valid[0] + return ["${_parsed_min}-${_parsed_max}", []] + } else { + return [undef, $_errors] + } + } + + /^([0-9a-z]+)-([0-9a-z]+)\/(\d+)$/: { + $min_valid = profile::cron::validate_field( + $field, + $1, + $valid_strings, + $int_range, + $seed, + false, + ) + + $max_valid = profile::cron::validate_field( + $field, + $2, + $valid_strings, + $int_range, + $seed, + false, + ) + + $interval_valid = profile::cron::validate_field( + $field, + Integer($3, 10), + [], + $int_range, + $seed, + false, + ) + $_errors = $min_valid[1] + $max_valid[1] + $interval_valid[1] + if empty($_errors) { + $_parsed_min = $min_valid[0] + $_parsed_max = $max_valid[0] + $_parsed_interval = $interval_valid[0] + return ["${_parsed_min}-${_parsed_max}/${_parsed_interval}", []] + } else { + return [undef, $_errors] + } + } + + /^\*\/(\d+)$/: { + $interval_valid = profile::cron::validate_field( + $field, + Integer($1, 10), + [], + $int_range, + $seed, + false, + ) + if empty($interval_valid[1]) { + $_parsed_interval = $interval_valid[0] + return ["*/${_parsed_interval}", []] + } else { + return $interval_valid + } + } + + String: { + return [undef, ["Could not parse value ${value} for field ${field}"]] + } + + Integer: { + [$_min, $_max] = $_int_range + if $_min <= $value and $value <= $_max { + return [String($value), []] + } else { + return [undef, ["Value ${value} out of range ${_min}-${_max} for field ${field}"]] + } + } + + default: { + return [undef, ["Unknown type ${value} for field ${field}"]] + } + } +} diff --git a/site-modules/profile/functions/cron_rand.pp b/site-modules/profile/functions/cron_rand.pp new file mode 100644 index 00000000..3d4911a5 --- /dev/null +++ b/site-modules/profile/functions/cron_rand.pp @@ -0,0 +1,26 @@ +# Randomize a cron run +function profile::cron_rand ( + Hash[Enum['minute', 'hour', 'month', 'monthday', 'weekday'], Variant[Integer, String, Undef]] $data, + String $seed, +) { + $maxima = { + 'minute' => 60, + 'hour' => 24, + 'month' => 12, + 'monthday' => 31, + 'weekday' => 7, + } + + Hash( + map($maxima) |$key, $max| { + [ + $key, + case $data[$key] { + 'fqdn_rand': {fqdn_rand($max, "${seed}_${key}")} + undef: {'*'} + default: {$data[$key]} + }, + ] + } + ) +} diff --git a/site-modules/profile/functions/letsencrypt/certificate_paths.pp b/site-modules/profile/functions/letsencrypt/certificate_paths.pp new file mode 100644 index 00000000..d4e8a8af --- /dev/null +++ b/site-modules/profile/functions/letsencrypt/certificate_paths.pp @@ -0,0 +1,15 @@ +# Return the paths for the given letsencrypt certificate + +function profile::letsencrypt::certificate_paths( + String $basename, +) { + $certs_directory = lookup('letsencrypt::certificates::directory') + $basedir = "${certs_directory}/${basename}" + + Hash({ + cert => "${basedir}/cert.pem", + chain => "${basedir}/chain.pem", + fullchain => "${basedir}/fullchain.pem", + privkey => "${basedir}/privkey.pem", + }) +} diff --git a/site-modules/profile/lib/facter/mounts.rb b/site-modules/profile/lib/facter/mounts.rb index 56f9d730..df004dc9 100644 --- a/site-modules/profile/lib/facter/mounts.rb +++ b/site-modules/profile/lib/facter/mounts.rb @@ -1,61 +1,63 @@ begin Facter.add("mounts") do ignorefs = [ "NFS", "afs", "autofs", "binfmt_misc", + "bpf", "cgroup", + "cgroup2", "cifs", "coda", "configfs", "debugfs", "devfs", "devpts", "devtmpfs", "efivarfs", "ftpfs", "fuse", "fuse.gvfsd-fuse", "fuse.lxcfs", "fuse.snapshotfs", "fusectl", "hugetlbfs", "iso9660", "lustre_lite", "mfs", "mqueue", "ncpfs", "nfs", "nfs4", "nfsd", "proc", "pstore", "rpc_pipefs", "securityfs", "shfs", "smbfs", "sysfs", "tmpfs", "tracefs", "udf", "usbfs", ].uniq.sort.join(',') mountpoints = [] Facter::Util::Resolution.exec("findmnt --list --noheadings -o TARGET,SOURCE --invert --types #{ignorefs}").lines.each do |line| mountpoint, source = line.chomp.split # bind mounts if not source.end_with?(']') mountpoints << mountpoint end end setcode do mountpoints end end rescue Exception => _ end diff --git a/site-modules/profile/lib/facter/public_ipaddresses.rb b/site-modules/profile/lib/facter/public_ipaddresses.rb new file mode 100644 index 00000000..7430b6a4 --- /dev/null +++ b/site-modules/profile/lib/facter/public_ipaddresses.rb @@ -0,0 +1,46 @@ +require "ipaddr" + +excluded_ranges = [ + # RFC 1918 + '10.0.0.0/8', + '172.16.0.0/12', + '192.168.0.0/16', + # Loopback + '127.0.0.0/8', + '::1/128', + # RFC 6598 (CGNAT) + '100.64.0.0/10', + # Multicast + '224.0.0.0/4', + 'ff00::/8', + # Future use + '240.0.0.0/4', + # IPv6 ULA + 'fc00::/7', + # IPv6 LLA + 'fe80::/10', +].map { |x| IPAddr.new(x) } + +Facter.add(:public_ipaddresses) do + setcode do + addresses = [] + interfaces = Facter.value(:networking).fetch('interfaces', {}) + + interfaces.each do |iface, data| + [ + data.fetch('bindings', []), + data.fetch('bindings6', []), + ].flatten.each do |addr| + ip = addr['address'] + unless ip.nil? + addresses.push(ip) + end + end + end + + addresses.uniq.select do |addr| + ipaddress = IPAddr.new(addr) + not excluded_ranges.any? { |range| range.include?(ipaddress) } + end + end +end diff --git a/site-modules/profile/lib/facter/swh_hostname.rb b/site-modules/profile/lib/facter/swh_hostname.rb index 96f7a621..2e98aaaf 100644 --- a/site-modules/profile/lib/facter/swh_hostname.rb +++ b/site-modules/profile/lib/facter/swh_hostname.rb @@ -1,19 +1,29 @@ require 'puppet' require 'socket' -domain_name = ".internal.softwareheritage.org" +domain_names = [ + ".internal.softwareheritage.org", + ".internal.staging.swh.network" +] + +default_domain_name = domain_names[0] Facter.add(:swh_hostname) do setcode do retval = {} retval["fqdn"] = Facter.value(:fqdn).to_s - if retval["fqdn"].end_with?(domain_name) - retval["short"] = retval["fqdn"].chomp(domain_name) + + filtered_domain = domain_names.select { |domain_name| + retval["fqdn"].end_with?(domain_name) + }.first + + if filtered_domain + retval["short"] = retval["fqdn"].chomp(filtered_domain) retval["internal_fqdn"] = retval["fqdn"] else retval["short"] = Socket.gethostname - retval["internal_fqdn"] = retval["short"] + domain_name + retval["internal_fqdn"] = retval["short"] + default_domain_name end retval end end diff --git a/site-modules/profile/lib/puppet/provider/gpg_key/gnupg.rb b/site-modules/profile/lib/puppet/provider/gpg_key/gnupg.rb new file mode 100644 index 00000000..b7f18944 --- /dev/null +++ b/site-modules/profile/lib/puppet/provider/gpg_key/gnupg.rb @@ -0,0 +1,134 @@ +require 'tempfile' + +Puppet::Type.type(:gpg_key).provide(:gnupg) do + + def exists? + return !get_fingerprint().nil? + end + + def create + data = [] + data.push("Key-Type: " + @resource.value(:keytype).to_s) + data.push("Key-Length: " + @resource.value(:keylength).to_s) + data.push("Name-Real: " + @resource.value(:uid).to_s) + data.push("Expire-Date: " + @resource.value(:expire).to_s) + data.push("%commit") + + stdin_data = data.join("\n") + "\n" + self.debug "Generating key with data: \n#{stdin_data.strip}" + + passphrase = @resource.value(:passphrase) + exec_gnupg( + params: ['--generate-key'], + stdin_data: stdin_data, + raise_on_error: true, + passphrase: passphrase, + ) + end + + def destroy + fpr = get_fingerprint() + if !fpr.nil? + exec_gnupg( + params: ['--yes', '--delete-secret-and-public-key', fpr], + raise_on_error: true, + ) + end + end + + def get_fingerprint + ret = exec_gnupg( + params: ['--with-colons', '--list-secret-keys', @resource.value(:uid)], + ) + if ret[:exit_status] != 0 + self.debug "list-secret-keys exited with #{ret[:exit_status]}: #{ret[:stderr].strip}" + return nil + end + + ret[:stdout].each_line do |l| + l.chomp! + fields = l.split(':') + if fields[0] != 'fpr' + next + end + return fields[9] + end + end + + def get_public_key + output = nil + if fingerprint = get_fingerprint() + ret = exec_gnupg( + ['--export', '--export-options', 'export-minimal', '--armor', fingerprint], + raise_on_error => true, + ) + output = ret[:stdout] + end + return output + end + + private + def exec_gnupg(params:[], stdin_data:nil, raise_on_error:false, passphrase:'') + stdin_r, stdin_w = IO.pipe + stdout_r, stdout_w = IO.pipe + stderr_r, stderr_w = IO.pipe + pass_r, pass_w = IO.pipe + + if stdin_data + stdin_w.write(stdin_data) + end + stdin_w.close + + pass_w.write(passphrase + "\n") + pass_w.close + + child_pid = Kernel.fork do + passphrase_args = ['--passphrase-fd', pass_r.fileno.to_s] + + command = [ + 'gpg', '--batch', '--pinentry-mode', 'loopback' + ] + passphrase_args + params + + self.debug "running gnupg command: #{command}" + + Process.setsid + begin + Puppet::Util::SUIDManager.change_privileges(resource[:owner], nil, true) + # Clean environment + Puppet::Util::POSIX::USER_ENV_VARS.each { |name| ENV.delete(name) } + Puppet::Util::POSIX::LOCALE_ENV_VARS.each { |name| ENV.delete(name) } + ENV['LANG'] = 'C' + ENV['LC_ALL'] = 'C' + Puppet::Util.withenv({}) do + Kernel.exec( + *command, + :in => stdin_r, + :out => stdout_w, + :err => stderr_w, + pass_r.fileno => pass_r, + ) + end + rescue => detail + Puppet.log_exception(detail, "Could not execute command: #{detail}") + exit!(1) + end + end + stdin_r.close + stdout_w.close + stderr_w.close + + exit_status = Process.waitpid2(child_pid).last.exitstatus + output = stdout_r.read() + outerr = stderr_r.read() + + if raise_on_error and exit_status != 0 + raise Puppet::ExecutionFailure, "Execution of 'gpg #{params}' returned #{exit_status}: #{output.strip} (stderr: #{outerr.strip})" + end + + return { + :exit_status => exit_status, + :stdout => output, + :stderr => outerr, + } + end +end diff --git a/site-modules/profile/lib/puppet/type/gpg_key.rb b/site-modules/profile/lib/puppet/type/gpg_key.rb new file mode 100644 index 00000000..48e4bc52 --- /dev/null +++ b/site-modules/profile/lib/puppet/type/gpg_key.rb @@ -0,0 +1,89 @@ +Puppet::Type.newtype(:gpg_key) do + ensurable + @doc = "Creates and manages GPG keys" + + newparam(:uid, :namevar => true) do + desc 'The uid for the generated key.' + end + + newparam(:owner) do + desc 'The owner of the generated key' + validate do |owner| + if !Puppet.features.root? && resource.current_username() != owner + self.fail _("Only root can generate keys for other users") + end + end + end + + newparam(:keytype) do + desc 'Type of the generated primary key (default: RSA)' + newvalues(:RSA, :DSA) + defaultto :RSA + end + + newparam(:keylength) do + desc 'Length of the generated key in bits (default: 4096)' + defaultto 4096 + + munge do |value| + case value + when String + value = Integer(value) + end + + return value + end + end + + newparam(:expire) do + desc 'Expiry of the generated key' + defaultto '0' + end + + newparam(:passphrase) do + desc 'Passphrase for the generated key' + defaultto '' + end + + newproperty(:fingerprint) do + desc %q{A read-only state to check the key fingerprint.} + + def retrieve + current_value = :absent + if fingerprint = provider.get_fingerprint + current_value = fingerprint + end + current_value + end + + validate do |val| + fail 'fingerprint is read-only' + end + end + + newproperty(:public_key) do + desc %q{A read-only state to get the public key.} + + def retrieve + current_value = :absent + if public_key = provider.get_public_key + current_value = public_key + end + current_value + end + + validate do |val| + fail 'public_key is read-only' + end + end + + autorequire(:user) do + if owner = self[:owner] + owner + end + end + + def current_username + Etc.getpwuid(Process.uid).name + end +end diff --git a/site-modules/profile/manifests/annex_web.pp b/site-modules/profile/manifests/annex_web.pp index b10e51fb..4ce68772 100644 --- a/site-modules/profile/manifests/annex_web.pp +++ b/site-modules/profile/manifests/annex_web.pp @@ -1,123 +1,146 @@ # Deployment of web-facing public Git-annex class profile::annex_web { $annex_basepath = lookup('annex::basepath') $annex_vhost_name = lookup('annex::vhost::name') $annex_vhost_docroot = lookup('annex::vhost::docroot') $annex_vhost_basic_auth_file = "${annex_basepath}/http_auth" + $annex_vhost_provenance_basic_auth_file = "${annex_basepath}/http_auth_provenance" $annex_vhost_basic_auth_content = lookup('annex::vhost::basic_auth_content') + $annex_vhost_provenance_basic_auth_content = lookup('annex::vhost::provenance::basic_auth_content') $annex_vhost_ssl_protocol = lookup('annex::vhost::ssl_protocol') $annex_vhost_ssl_honorcipherorder = lookup('annex::vhost::ssl_honorcipherorder') $annex_vhost_ssl_cipher = lookup('annex::vhost::ssl_cipher') $annex_vhost_hsts_header = lookup('annex::vhost::hsts_header') include ::profile::ssl include ::profile::apache::common ::apache::vhost {"${annex_vhost_name}_non-ssl": servername => $annex_vhost_name, port => '80', docroot => $annex_vhost_docroot, redirect_status => 'permanent', redirect_dest => "https://${annex_vhost_name}/", } $ssl_cert_name = 'star_softwareheritage_org' $ssl_cert = $::profile::ssl::certificate_paths[$ssl_cert_name] $ssl_chain = $::profile::ssl::chain_paths[$ssl_cert_name] $ssl_key = $::profile::ssl::private_key_paths[$ssl_cert_name] ::apache::vhost {"${annex_vhost_name}_ssl": servername => $annex_vhost_name, port => '443', ssl => true, ssl_protocol => $annex_vhost_ssl_protocol, ssl_honorcipherorder => $annex_vhost_ssl_honorcipherorder, ssl_cipher => $annex_vhost_ssl_cipher, ssl_cert => $ssl_cert, ssl_chain => $ssl_chain, ssl_key => $ssl_key, headers => [$annex_vhost_hsts_header], docroot => $annex_vhost_docroot, directories => [{ 'path' => $annex_vhost_docroot, 'require' => 'all granted', 'options' => ['Indexes', 'FollowSymLinks', 'MultiViews'], + custom_fragment => 'IndexIgnore private provenance-index', }, { # hide (annex) .git directory 'path' => '.*/\.git/?$', 'provider' => 'directorymatch', 'require' => 'all denied', - }], + }, + { # 'basic' provenance-index authentication + 'path' => "$annex_vhost_docroot/provenance-index", + 'auth_type' => 'basic', + 'auth_name' => 'SWH - Password Required', + 'auth_user_file' => $annex_vhost_provenance_basic_auth_file, + 'auth_require' => 'valid-user', + 'index_options' => 'FancyIndexing', + custom_fragment => 'ReadmeName readme.txt', + }, + ], require => [ File[$ssl_cert], File[$ssl_chain], File[$ssl_key], ], } file {"${annex_vhost_docroot}/public": ensure => link, target => "../annexroot/public", require => File[$annex_vhost_docroot], } file {$annex_vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', + # FIXME: this seems wrong, should be double quote to expand the variable + # don't want to break existing behavior though content => '$annex_vhost_basic_auth_content', } + file {$annex_vhost_provenance_basic_auth_file: + ensure => present, + owner => 'root', + group => 'www-data', + mode => '0640', + content => "$annex_vhost_provenance_basic_auth_content", + } + $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"annex http redirect on ${::fqdn}": service_name => 'annex http redirect', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $annex_vhost_name, http_vhost => $annex_vhost_name, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"annex https on ${::fqdn}": service_name => 'annex https', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $annex_vhost_name, http_vhost => $annex_vhost_name, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"annex https certificate ${::fqdn}": service_name => 'annex https certificate', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $annex_vhost_name, http_vhost => $annex_vhost_name, http_ssl => true, http_sni => true, http_certificate => 60, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } diff --git a/site-modules/profile/manifests/cron.pp b/site-modules/profile/manifests/cron.pp new file mode 100644 index 00000000..317c3016 --- /dev/null +++ b/site-modules/profile/manifests/cron.pp @@ -0,0 +1,22 @@ +# Handle a /etc/puppet-cron.d directory with properly managed cron.d snippets + +class profile::cron { + $directory = '/etc/puppet-cron.d' + + file {$directory: + ensure => directory, + mode => '0755', + owner => 'root', + group => 'root', + recurse => true, + purge => true, + notify => Exec['clean-cron.d-symlinks'], + } + + exec {'clean-cron.d-symlinks': + path => ['/bin', '/usr/bin'], + command => 'find /etc/cron.d -type l ! -exec test -e {} \; -delete', + refreshonly => true, + require => File[$directory], + } +} diff --git a/site-modules/profile/manifests/cron/d.pp b/site-modules/profile/manifests/cron/d.pp new file mode 100644 index 00000000..5ba8d1d7 --- /dev/null +++ b/site-modules/profile/manifests/cron/d.pp @@ -0,0 +1,76 @@ +# Add a cron.d snippet to the /etc/puppet-cron.d directory + +define profile::cron::d( + String $command, + String $unique_tag = $title, + String $target = 'default', + Optional[Variant[Integer, String, Array[Variant[Integer, String]]]] $minute = undef, + Optional[Variant[Integer, String, Array[Variant[Integer, String]]]] $hour = undef, + Optional[Variant[Integer, String, Array[Variant[Integer, String]]]] $monthday = undef, + Optional[Variant[Integer, String, Array[Variant[Integer, String]]]] $month = undef, + Optional[Variant[Integer, String, Array[Variant[Integer, String]]]] $weekday = undef, + Optional[Enum['@reboot', '@yearly', '@annually', '@monthly', '@weekly', '@daily', '@midnight', '@hourly']] $special = undef, + String $user = 'root', + Optional[String] $random_seed = undef, +) { + include profile::cron + + $_params = { + 'minute' => $minute, + 'hour' => $hour, + 'monthday' => $monthday, + 'month' => $month, + 'weekday' => $weekday, + } + + $_int_limits = { + 'minute' => [0, 59], + 'hour' => [0, 23], + 'monthday' => [1, 31], + 'month' => [1, 12], + 'weekday' => [0, 7], + } + + $_str_values = { + 'month' => ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'], + 'weekday' => ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'], + } + + if $special != undef { + $_defined_params = $_params.filter |$field, $value| { $value == undef } + + unless empty($_defined_params) { + $_defined_fields = keys($_defined_params).each |$field| {"'${field}'"}.join(', ') + fail("profile::cron::d parameter 'special' is exclusive with ${_defined_fields}.") + } + } + + $_parsed_params = $_params.map |$field, $value| { + [$field] + profile::cron::validate_field( + $field, + pick_default($value, '*'), + $_str_values[$field], + $_int_limits[$field], + $random_seed, + ) + } + + $_parse_errors = $_parsed_params.filter |$value| { $value[1] == undef }.map |$value| { $value[2] }.flatten + unless empty($_parse_errors) { + $_str_parse_errors = $_parse_errors.join(', ') + fail("Parse errors in profile::cron::d: ${_str_parse_errors}") + } + + $_params_hash = $_parsed_params.map |$value| { $value[0,2] }.hash + + if !defined(Profile::Cron::File[$target]) { + profile::cron::file {$target:} + } + + concat_fragment {"profile::cron::${unique_tag}": + order => '10', + content => template('profile/cron/snippet.erb'), + tag => "profile::cron::${target}", + target => "profile::cron::${target}", + } +} diff --git a/site-modules/profile/manifests/cron/file.pp b/site-modules/profile/manifests/cron/file.pp new file mode 100644 index 00000000..2e92374e --- /dev/null +++ b/site-modules/profile/manifests/cron/file.pp @@ -0,0 +1,28 @@ +# Base definition of a /etc/puppet-cron.d + +define profile::cron::file ( + String $target = $title, +) { + include profile::cron + + $file = "${profile::cron::directory}/${target}" + concat_file {"profile::cron::${target}": + path => $file, + owner => 'root', + group => 'root', + mode => '0644', + tag => "profile::cron::${target}", + } + -> file {"/etc/cron.d/puppet-${target}": + ensure => 'link', + target => $file, + } + -> Exec['clean-cron.d-symlinks'] + + concat_fragment {"profile::cron::${target}::_header": + target => "profile::cron::${target}", + tag => "profile::cron::${target}", + order => '00', + content => "# Managed by puppet (module profile::cron), manual changes will be lost\n\n", + } +} diff --git a/site-modules/profile/manifests/dar/client.pp b/site-modules/profile/manifests/dar/client.pp index 484b7f14..43d88ff7 100644 --- a/site-modules/profile/manifests/dar/client.pp +++ b/site-modules/profile/manifests/dar/client.pp @@ -1,68 +1,49 @@ class profile::dar::client { include ::dar $dar_remote_hostname = $::swh_hostname['short'] $dar_backup_name = $::hostname - $hierahour = lookup('dar::cron::hour') - if $hierahour == 'fqdn_rand' { - $hour = fqdn_rand(24, 'backup_hour') - } else { - $hour = $hierahour - } - - $hieraminute = lookup('dar::cron::minute') - if $hieraminute == 'fqdn_rand' { - $minute = fqdn_rand(60, 'backup_minute') - } else { - $minute = $hieraminute - } - - $hieramonth = lookup('dar::cron::month') - if $hieramonth == 'fqdn_rand' { - $month = fqdn_rand(12, 'backup_month') - } else { - $month = $hieramonth - } - - $hieramonthday = lookup('dar::cron::monthday') - if $hieramonthday == 'fqdn_rand' { - $monthday = fqdn_rand(31, 'backup_monthday') - } else { - $monthday = $hieramonthday - } - - $hieraweekday = lookup('dar::cron::weekday') - if $hieraweekday == 'fqdn_rand' { - $weekday = fqdn_rand(31, 'backup_weekday') - } else { - $weekday = $hieraweekday - } + $backup_cron = profile::cron_rand(lookup('dar::cron', Hash), 'backup') dar::backup { $dar_backup_name: backup_storage => lookup('dar::backup::storage'), keep_backups => lookup('dar::backup::num_backups'), backup_base => lookup('dar::backup::base'), backup_selection => lookup('dar::backup::select'), backup_exclusion => lookup('dar::backup::exclude', Array, 'unique'), backup_options => lookup('dar::backup::options'), - hour => $hour, - minute => $minute, - month => $month, - monthday => $monthday, - weekday => $weekday, + * => $backup_cron, } + $server_cron = profile::cron_rand(lookup('dar_server::cron', Hash), 'backup_server') + + $dir_for_fetched_backups = lookup('dar_server::backup::storage') + $central_backup_host = lookup('dar_server::central_host') + # Export a remote backup to the backup server @@dar::remote_backup { "${dar_remote_hostname}.${dar_backup_name}": remote_backup_storage => lookup('dar::backup::storage'), remote_backup_host => $dar_remote_hostname, remote_backup_name => $dar_backup_name, - local_backup_storage => lookup('dar_server::backup::storage'), - hour => lookup('dar_server::cron::hour'), - minute => lookup('dar_server::cron::minute'), - month => lookup('dar_server::cron::month'), - monthday => lookup('dar_server::cron::monthday'), - weekday => lookup('dar_server::cron::weekday'), + local_backup_storage => $dir_for_fetched_backups, + * => $server_cron, + } + + # Export an icinga check to verify backup freshness + $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' + $checked_directory = "${dir_for_fetched_backups}/${dar_remote_hostname}" + + @@::icinga2::object::service {"backup freshness for ${dar_remote_hostname}": + service_name => 'backup freshness', + import => ['generic-service'], + host_name => $::fqdn, + command_endpoint => $central_backup_host, + check_command => 'check_newest_file_age', + vars => { + check_directory => $checked_directory, + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', } } diff --git a/site-modules/profile/manifests/debian_repository.pp b/site-modules/profile/manifests/debian_repository.pp index a241d1c1..ada06591 100644 --- a/site-modules/profile/manifests/debian_repository.pp +++ b/site-modules/profile/manifests/debian_repository.pp @@ -1,118 +1,200 @@ # Debian repository configuration class profile::debian_repository { $packages = ['reprepro'] package {$packages: ensure => installed, } - $repository_basepath = lookup('debian_repository::basepath') - - $repository_vhost_name = lookup('debian_repository::vhost::name') - $repository_vhost_aliases = lookup('debian_repository::vhost::aliases') - $repository_vhost_docroot = lookup('debian_repository::vhost::docroot') - $repository_vhost_docroot_owner = lookup('debian_repository::vhost::docroot_owner') - $repository_vhost_docroot_group = lookup('debian_repository::vhost::docroot_group') - $repository_vhost_docroot_mode = lookup('debian_repository::vhost::docroot_mode') - $repository_vhost_ssl_protocol = lookup('debian_repository::vhost::ssl_protocol') - $repository_vhost_ssl_honorcipherorder = lookup('debian_repository::vhost::ssl_honorcipherorder') - $repository_vhost_ssl_cipher = lookup('debian_repository::vhost::ssl_cipher') - $repository_vhost_hsts_header = lookup('debian_repository::vhost::hsts_header') + $basepath = lookup('debian_repository::basepath') + + $owner = lookup('debian_repository::owner') + $group = lookup('debian_repository::group') + $mode = lookup('debian_repository::mode') + + $owner_homedir = lookup('debian_repository::owner::homedir') + + user {$owner: + ensure => present, + system => true, + home => $owner_homedir, + } + -> file {$owner_homedir: + ensure => 'directory', + owner => $owner, + group => $owner, + mode => '0750', + } + -> file {"${owner_homedir}/.ssh": + ensure => 'directory', + owner => $owner, + group => $owner, + mode => '0700', + } + + $authorized_keys = lookup('debian_repository::ssh_authorized_keys', Hash) + each($authorized_keys) |$name, $key| { + ssh_authorized_key { "${owner} ${name}": + ensure => 'present', + user => $owner, + key => $key['key'], + type => $key['type'], + require => File["${owner_homedir}/.ssh"], + } + } + + file {$basepath: + ensure => 'directory', + owner => $owner, + group => $group, + mode => $mode, + } + + $incoming = "${basepath}/incoming" + + file {$incoming: + ensure => 'directory', + owner => $owner, + group => $group, + mode => $mode, + } + + $gpg_keys = lookup('debian_repository::gpg_keys', Array) + + $gpg_raw_command = 'gpg --batch --pinentry-mode loopback' + each($gpg_keys) |$keyid| { + exec {"debian repository gpg key ${keyid}": + path => ['/usr/bin'], + command => "${gpg_raw_command} --recv-keys ${keyid}", + user => $owner, + unless => "${gpg_raw_command} --list-keys ${keyid}", + } + + profile::cron::d {"debrepo-keyring-refresh-${keyid}": + target => 'debrepo-keyring-refresh', + user => $owner, + command => "chronic ${gpg_raw_command} --recv-keys ${keyid}", + random_seed => "debrepo-keyring-${keyid}", + minute => 'fqdn_rand', + hour => 'fqdn_rand', + } + } + + file {"$basepath/conf": + ensure => 'directory', + owner => $owner, + group => $group, + mode => $mode, + } + + file {"$basepath/conf/uploaders": + ensure => 'file', + owner => $owner, + group => $group, + mode => '0644', + content => template('profile/debian_repository/uploaders.erb') + } + + $vhost_name = lookup('debian_repository::vhost::name') + $vhost_aliases = lookup('debian_repository::vhost::aliases') + $vhost_docroot = lookup('debian_repository::vhost::docroot') + $vhost_ssl_protocol = lookup('debian_repository::vhost::ssl_protocol') + $vhost_ssl_honorcipherorder = lookup('debian_repository::vhost::ssl_honorcipherorder') + $vhost_ssl_cipher = lookup('debian_repository::vhost::ssl_cipher') + $vhost_hsts_header = lookup('debian_repository::vhost::hsts_header') include ::profile::ssl include ::profile::apache::common - ::apache::vhost {"${repository_vhost_name}_non-ssl": - servername => $repository_vhost_name, - serveraliases => $repository_vhost_aliases, + ::apache::vhost {"${vhost_name}_non-ssl": + servername => $vhost_name, + serveraliases => $vhost_aliases, port => '80', - docroot => $repository_vhost_docroot, - manage_docroot => false, # will be managed by the SSL resource + docroot => $vhost_docroot, + manage_docroot => false, redirect_status => 'permanent', - redirect_dest => "https://${repository_vhost_name}/", + redirect_dest => "https://${vhost_name}/", } $ssl_cert_name = 'star_softwareheritage_org' $ssl_cert = $::profile::ssl::certificate_paths[$ssl_cert_name] $ssl_chain = $::profile::ssl::chain_paths[$ssl_cert_name] $ssl_key = $::profile::ssl::private_key_paths[$ssl_cert_name] - ::apache::vhost {"${repository_vhost_name}_ssl": - servername => $repository_vhost_name, + ::apache::vhost {"${vhost_name}_ssl": + servername => $vhost_name, port => '443', ssl => true, - ssl_protocol => $repository_vhost_ssl_protocol, - ssl_honorcipherorder => $repository_vhost_ssl_honorcipherorder, - ssl_cipher => $repository_vhost_ssl_cipher, + ssl_protocol => $vhost_ssl_protocol, + ssl_honorcipherorder => $vhost_ssl_honorcipherorder, + ssl_cipher => $vhost_ssl_cipher, ssl_cert => $ssl_cert, ssl_chain => $ssl_chain, ssl_key => $ssl_key, - headers => [$repository_vhost_hsts_header], - docroot => $repository_vhost_docroot, - docroot_owner => $repository_vhost_docroot_owner, - docroot_group => $repository_vhost_docroot_group, - docroot_mode => $repository_vhost_docroot_mode, + headers => [$vhost_hsts_header], + docroot => $vhost_docroot, + manage_docroot => false, directories => [ { - path => $repository_vhost_docroot, + path => $vhost_docroot, require => 'all granted', options => ['Indexes', 'FollowSymLinks', 'MultiViews'], }, ], require => [ File[$ssl_cert], File[$ssl_chain], File[$ssl_key], ], } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"debian repository http redirect on ${::fqdn}": service_name => 'debian repository http redirect', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { - http_address => $repository_vhost_name, - http_vhost => $repository_vhost_name, + http_address => $vhost_name, + http_vhost => $vhost_name, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"debian repository https on ${::fqdn}": service_name => 'debian repository https', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { - http_address => $repository_vhost_name, - http_vhost => $repository_vhost_name, + http_address => $vhost_name, + http_vhost => $vhost_name, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"debian repository https certificate ${::fqdn}": service_name => 'debian repository https certificate', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { - http_address => $repository_vhost_name, - http_vhost => $repository_vhost_name, + http_address => $vhost_name, + http_vhost => $vhost_name, http_ssl => true, http_sni => true, http_certificate => 60, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } diff --git a/site-modules/profile/manifests/devel/postgres.pp b/site-modules/profile/manifests/devel/postgres.pp index cc7bac4e..b27d8afc 100644 --- a/site-modules/profile/manifests/devel/postgres.pp +++ b/site-modules/profile/manifests/devel/postgres.pp @@ -1,13 +1,36 @@ class profile::devel::postgres { $packages = lookup('packages::devel::postgres', Array, 'unique') package { $packages: ensure => present, } - file { '/etc/postgresql-common/pg_service.conf': - ensure => file, - content => template('profile/postgres/pg_service.conf.erb'), - require => Package[$packages], + $dbs = lookup('swh::postgres::service::dbs', Array, 'deep') + + # Explicitly install the configuration files per user's home + # TL;DR the pgpass must be readonly per user so we can't use the global one + $users = lookup('swh::postgres::service::users', Array, 'deep') + each ($users) | $user | { + if $user == 'root' { + $home = '/root' + } else { + $home = "/home/${user}" + } + + file {"${home}/.pg_service.conf": + ensure => file, + content => template('profile/postgres/pg_service.conf.erb'), + owner => $user, + group => $user, + mode => '0400', + } + file {"${home}/.pgpass": + ensure => file, + content => template('profile/postgres/pgpass.conf.erb'), + owner => $user, + group => $user, + mode => '0400', + } } + } diff --git a/site-modules/profile/manifests/docs_web.pp b/site-modules/profile/manifests/docs_web.pp index b931c78a..610284ea 100644 --- a/site-modules/profile/manifests/docs_web.pp +++ b/site-modules/profile/manifests/docs_web.pp @@ -1,106 +1,111 @@ # Deployment of web-facing static documentation class profile::docs_web { $docs_basepath = lookup('docs::basepath') $docs_vhost_name = lookup('docs::vhost::name') $docs_vhost_docroot = lookup('docs::vhost::docroot') + $docs_vhost_docroot_owner = lookup('docs::vhost::docroot_owner') $docs_vhost_docroot_group = lookup('docs::vhost::docroot_group') $docs_vhost_docroot_mode = lookup('docs::vhost::docroot_mode') $docs_vhost_ssl_protocol = lookup('docs::vhost::ssl_protocol') $docs_vhost_ssl_honorcipherorder = lookup('docs::vhost::ssl_honorcipherorder') $docs_vhost_ssl_cipher = lookup('docs::vhost::ssl_cipher') $docs_vhost_hsts_header = lookup('docs::vhost::hsts_header') include ::profile::ssl include ::profile::apache::common ::apache::vhost {"${docs_vhost_name}_non-ssl": servername => $docs_vhost_name, port => '80', docroot => $docs_vhost_docroot, manage_docroot => false, # will be managed by the SSL resource redirect_status => 'permanent', redirect_dest => "https://${docs_vhost_name}/", } $ssl_cert_name = 'star_softwareheritage_org' $ssl_cert = $::profile::ssl::certificate_paths[$ssl_cert_name] $ssl_chain = $::profile::ssl::chain_paths[$ssl_cert_name] $ssl_key = $::profile::ssl::private_key_paths[$ssl_cert_name] ::apache::vhost {"${docs_vhost_name}_ssl": servername => $docs_vhost_name, port => '443', ssl => true, ssl_protocol => $docs_vhost_ssl_protocol, ssl_honorcipherorder => $docs_vhost_ssl_honorcipherorder, ssl_cipher => $docs_vhost_ssl_cipher, ssl_cert => $ssl_cert, ssl_chain => $ssl_chain, ssl_key => $ssl_key, headers => [$docs_vhost_hsts_header], docroot => $docs_vhost_docroot, + docroot_owner => $docs_vhost_docroot_owner, docroot_group => $docs_vhost_docroot_group, docroot_mode => $docs_vhost_docroot_mode, directories => [{ 'path' => $docs_vhost_docroot, 'require' => 'all granted', 'options' => ['Indexes', 'FollowSymLinks', 'MultiViews'], }], require => [ File[$ssl_cert], File[$ssl_chain], File[$ssl_key], ], + rewrites => [ + { rewrite_rule => '^/?$ /devel/ [R,L]' }, + ], } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"docs http redirect on ${::fqdn}": service_name => 'docs http redirect', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $docs_vhost_name, http_vhost => $docs_vhost_name, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"docs https on ${::fqdn}": service_name => 'docs https', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $docs_vhost_name, http_vhost => $docs_vhost_name, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"docs https certificate ${::fqdn}": service_name => 'docs https certificate', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $docs_vhost_name, http_vhost => $docs_vhost_name, http_ssl => true, http_sni => true, http_certificate => 60, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } diff --git a/site-modules/profile/manifests/elasticsearch.pp b/site-modules/profile/manifests/elasticsearch.pp index fc3cfc5f..28a5fbb3 100644 --- a/site-modules/profile/manifests/elasticsearch.pp +++ b/site-modules/profile/manifests/elasticsearch.pp @@ -1,58 +1,59 @@ # Elasticsearch cluster node profile class profile::elasticsearch { user { 'elasticsearch': ensure => 'present', uid => '114', gid => '119', home => '/home/elasticsearch', shell => '/bin/false', } package { 'openjdk-8-jre-headless': ensure => 'present', } # Elasticsearch official package installation instructions: # https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html - $keyid = lookup('elastic::apt_config::keyid') - $key = lookup('elastic::apt_config::key') + $keyid = lookup('elastic::apt_config::keyid') + $key = lookup('elastic::apt_config::key') + $version = lookup('elastic::elk_version') apt::source { 'elastic-6.x': location => 'https://artifacts.elastic.co/packages/6.x/apt', release => 'stable', repos => 'main', key => { id => $keyid, content => $key, }, } package { 'elasticsearch': - ensure => '6.3.2', + ensure => $version, } apt::pin { 'elasticsearch': packages => 'elasticsearch elasticsearch-oss', - version => '6.3.2', + version => $version, priority => 1001, } # niofs increases I/O performance and node reliability file_line { 'elasticsearch niofs': ensure => present, line => 'index.store.type: niofs', path => '/etc/elasticsearch/elasticsearch.yml', } systemd::dropin_file { 'elasticsearch.conf': unit => 'elasticsearch.service', content => template('profile/swh/elasticsearch.conf.erb'), } service { 'elasticsearch': ensure => running, enable => true, } } diff --git a/site-modules/profile/manifests/export_archive_counters.pp b/site-modules/profile/manifests/export_archive_counters.pp new file mode 100644 index 00000000..74a8ee75 --- /dev/null +++ b/site-modules/profile/manifests/export_archive_counters.pp @@ -0,0 +1,43 @@ +# stats_export master class +class profile::export_archive_counters { + $export_path = lookup('stats_export::export_path') + $export_file = lookup('stats_export::export_file') + + $packages = ['python3-click'] + + package {$packages: + ensure => present, + } + + file {'/usr/local/bin/export-archive_counters.py': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/profile/stats_exporter/export-archive_counters.py', + require => Package[$packages], + } + + file {'/usr/local/share/swh-data/history-counters.munin.json': + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/stats_exporter/history-counters.munin.json', + } + + cron {'stats_export': + ensure => present, + user => 'www-data', + command => "/usr/local/bin/export-archive_counters.py > ${export_file}.tmp && /bin/mv ${export_file}.tmp ${export_file}", + hour => fqdn_rand(24, 'stats_export_hour'), + minute => fqdn_rand(60, 'stats_export_minute'), + month => '*', + monthday => '*', + weekday => '*', + require => [ + File['/usr/local/bin/export-archive_counters.py'], + File['/usr/local/share/swh-data/history-counters.munin.json'], + ], + } +} diff --git a/site-modules/profile/manifests/grafana.pp b/site-modules/profile/manifests/grafana.pp index 7aca07ec..759a4dbf 100644 --- a/site-modules/profile/manifests/grafana.pp +++ b/site-modules/profile/manifests/grafana.pp @@ -1,31 +1,30 @@ class profile::grafana { $db = lookup('grafana::db::database') $db_username = lookup('grafana::db::username') $db_password = lookup('grafana::db::password') $config = lookup('grafana::config') include ::postgresql::server ::postgresql::server::db {$db: user => $db_username, password => postgresql_password($db_username, $db_password), } class {'::grafana': install_method => 'repo', - version => 'latest', cfg => $config + { database => { type => 'postgres', host => '127.0.0.1:5432', name => $db, user => $db_username, password => $db_password } } } contain profile::grafana::vhost contain profile::grafana::objects } diff --git a/site-modules/profile/manifests/haveged.pp b/site-modules/profile/manifests/haveged.pp new file mode 100644 index 00000000..d26d7416 --- /dev/null +++ b/site-modules/profile/manifests/haveged.pp @@ -0,0 +1,12 @@ +# Setup haveged +class profile::haveged { + package {'haveged': + ensure => installed, + } + + service {'haveged': + ensure => running, + enable => true, + require => Package['haveged'], + } -> Gpg_Key <<| |>> +} diff --git a/site-modules/profile/manifests/icinga2/agent.pp b/site-modules/profile/manifests/icinga2/agent.pp index dbcbae71..c45e2e15 100644 --- a/site-modules/profile/manifests/icinga2/agent.pp +++ b/site-modules/profile/manifests/icinga2/agent.pp @@ -1,78 +1,78 @@ # Icinga2 agent configuration class profile::icinga2::agent { $features = lookup('icinga2::features') $icinga2_network = lookup('icinga2::network') $hiera_host_vars = lookup('icinga2::host::vars', Hash, 'deep') $parent_zone = lookup('icinga2::parent_zone') $parent_endpoints = lookup('icinga2::parent_endpoints') include profile::icinga2::objects::agent_checks $check_mounts = $::mounts.filter |$mount| { - $mount !~ /^\/srv\/containers/ + $mount !~ /^\/srv\/containers\// and $mount !~ /^\/var\/lib\/docker\/overlay2\// } $local_host_vars = { disks => hash(flatten( $check_mounts.map |$mount| { ["disk ${mount}", {disk_partitions => $mount}] }, )), plugins => keys($profile::icinga2::objects::agent_checks::plugins), } class {'::icinga2': confd => true, features => $features, } class { '::icinga2::feature::api': accept_config => true, accept_commands => true, zones => { 'ZoneName' => { endpoints => ['NodeName'], parent => $parent_zone, }, }, } create_resources('::icinga2::object::endpoint', $parent_endpoints) ::icinga2::object::zone {$parent_zone: endpoints => keys($parent_endpoints), } @@::icinga2::object::endpoint {$::fqdn: host => ip_for_network($icinga2_network), target => "/etc/icinga2/zones.d/${parent_zone}/${::fqdn}.conf", } @@::icinga2::object::zone {$::fqdn: endpoints => [$::fqdn], parent => $parent_zone, target => "/etc/icinga2/zones.d/${parent_zone}/${::fqdn}.conf", } @@::icinga2::object::host {$::fqdn: address => ip_for_network($icinga2_network), display_name => $::fqdn, check_command => 'hostalive', vars => deep_merge($local_host_vars, $hiera_host_vars), target => "/etc/icinga2/zones.d/${parent_zone}/${::fqdn}.conf", } icinga2::object::zone { 'global-templates': global => true, } file {['/etc/icinga2/conf.d']: ensure => directory, owner => 'nagios', group => 'nagios', mode => '0755', purge => true, recurse => true, tag => 'icinga2::config::file', } } diff --git a/site-modules/profile/manifests/icinga2/objects/agent_checks.pp b/site-modules/profile/manifests/icinga2/objects/agent_checks.pp index a39a263d..af4e2d9e 100644 --- a/site-modules/profile/manifests/icinga2/objects/agent_checks.pp +++ b/site-modules/profile/manifests/icinga2/objects/agent_checks.pp @@ -1,89 +1,110 @@ # Checks that need to be supported on icinga2 agents class profile::icinga2::objects::agent_checks { $plugins = { 'check_journal' => { arguments => { '-f' => { 'value' => '$journal_cursor_file$', 'set_if' => '$journal_cursor_file$', }, '-w' => '$journal_lag_warn$', '-c' => '$journal_lag_crit$', '-wn' => { 'value' => '$journal_lag_entries_warn$', 'set_if' => '$journal_lag_entries_warn$', }, '-cn' => { 'value' => '$journal_lag_entries_crit$', 'set_if' => '$journal_lag_entries_crit$', }, }, vars => { 'journal_lag_warn' => 1200, 'journal_lag_crit' => 3600, }, sudo => true, sudo_user => 'journalbeat', }, + 'check_newest_file_age' => { + arguments => { + '-d' => '$check_directory$', + '-w' => '$check_directory_warn_age$', + '-c' => '$check_directory_crit_age$', + '-W' => { + 'set_if' => '$check_directory_missing_warn$', + }, + '-C' => { + 'set_if' => '$check_directory_missing_crit$', + }, + }, + vars => { + 'check_directory_warn_age' => 26, + 'check_directory_crit_age' => 52, + 'check_directory_missing_warn' => false, + 'check_directory_missing_crit' => true, + }, + sudo => true, + sudo_user => 'root', + }, } $swh_plugin_dir = '/usr/lib/nagios/plugins/swh' $swh_plugin_configfile = '/etc/icinga2/conf.d/swh-plugins.conf' $packages = [ 'python3-nagiosplugin', 'python3-systemd', 'monitoring-plugins-basic', ] package {$packages: ensure => present, } file {$swh_plugin_dir: ensure => 'directory', owner => 'root', group => 'root', mode => '0755', recurse => true, purge => true, require => Package[$packages], } $plugins.each |$command, $plugin| { $command_path = "${swh_plugin_dir}/${command}" file {$command_path: ensure => present, owner => 'root', group => 'root', mode => '0755', source => "puppet:///modules/profile/icinga2/plugins/${command}", require => Package[$packages], } if $plugin['sudo'] { $sudo_user = $plugin['sudo_user'] $icinga_command = ['sudo', '-u', $sudo_user, $command_path] ::sudo::conf { "icinga-${command}": ensure => present, content => "nagios ALL=(${sudo_user}) NOPASSWD: ${command_path}", priority => 50, } } else { $icinga_command = [$command_path] ::sudo::conf { "icinga-${command}": ensure => absent, } } ::icinga2::object::checkcommand {$command: import => ['plugin-check-command'], command => $icinga_command, arguments => $plugin['arguments'], vars => $plugin['vars'], target => $swh_plugin_configfile, } } } diff --git a/site-modules/profile/manifests/icinga2/objects/static_checks.pp b/site-modules/profile/manifests/icinga2/objects/static_checks.pp index f89be704..3e037b29 100644 --- a/site-modules/profile/manifests/icinga2/objects/static_checks.pp +++ b/site-modules/profile/manifests/icinga2/objects/static_checks.pp @@ -1,45 +1,94 @@ # Static checks on the icinga master class profile::icinga2::objects::static_checks { $checks_file = '/etc/icinga2/conf.d/static-checks.conf' ::icinga2::object::host {'www.softwareheritage.org': import => ['generic-host'], check_command => 'hostalive4', address => 'www.softwareheritage.org', target => $checks_file, + vars => { + ping_wrta => 150, + ping_crta => 300, + }, + } + + ::icinga2::object::host {'softwareheritage.org': + import => ['generic-host'], + check_command => 'hostalive4', + address => 'softwareheritage.org', + target => $checks_file, + vars => { + ping_wrta => 150, + ping_crta => 300, + }, } ::icinga2::object::service {'Software Heritage Homepage': import => ['generic-service'], host_name => 'www.softwareheritage.org', check_command => 'http', target => $checks_file, vars => { http_vhost => 'www.softwareheritage.org', http_uri => '/', http_ssl => true, http_sni => true, http_string => 'Software Heritage', }, } + ::icinga2::object::service {'Software Heritage Homepage (redirect to www)': + import => ['generic-service'], + host_name => 'softwareheritage.org', + check_command => 'http', + target => $checks_file, + vars => { + http_vhost => 'softwareheritage.org', + http_uri => '/', + http_ssl => true, + http_sni => true, + }, + } + ::icinga2::object::host {'swh-logging-prod': check_command => 'hostalive4', address => '127.0.0.1', target => $checks_file, } ::icinga2::object::service {'swh-logging-prod cluster': host_name => 'swh-logging-prod', check_command => 'check_escluster', target => $checks_file, } ::icinga2::object::checkcommand {'check_escluster': import => ['plugin-check-command'], command => '/usr/lib/nagios/plugins/icinga_check_elasticsearch.sh', target => $checks_file, } + ::icinga2::object::host {'DNS resolvers': + check_command => 'hostalive4', + address => '127.0.0.1', + target => $checks_file, + } + + ::icinga2::object::service {'SOA': + host_name => 'DNS resolvers', + check_command => 'check_resolvers', + target => $checks_file, + } + + ::icinga2::object::checkcommand {'check_resolvers': + import => ['plugin-check-command'], + command => [ + '/usr/lib/nagios/plugins/dsa-nagios-checks_checks_dsa-check-soas.txt', + 'internal.softwareheritage.org', + ], + target => $checks_file, + } + } diff --git a/site-modules/profile/manifests/jenkins/agent.pp b/site-modules/profile/manifests/jenkins/agent.pp new file mode 100644 index 00000000..aa1a1894 --- /dev/null +++ b/site-modules/profile/manifests/jenkins/agent.pp @@ -0,0 +1,45 @@ +class profile::jenkins::agent { + include profile::jenkins::base + include ::java + + $jenkins_agent_jar_url = lookup('jenkins::agent::jar_url') + $jenkins_jnlp_url = lookup('jenkins::agent::jnlp::url') + $jenkins_jnlp_token = lookup('jenkins::agent::jnlp::token') + + $workdir = '/var/lib/jenkins/agent-workdir' + file {$workdir: + mode => '0700', + owner => 'jenkins', + group => 'jenkins', + } + + $jenkins_agent_jar = '/usr/share/jenkins/agent.jar' + file {$jenkins_agent_jar: + source => $jenkins_agent_jar_url, + mode => '0644', + owner => 'root', + group => 'root', + } + + $environment_file = '/etc/default/jenkins-agent' + file {$environment_file: + # Contains credentials + mode => '0600', + owner => 'root', + group => 'root', + content => template('profile/jenkins/agent/jenkins-agent.defaults.erb'), + notify => Service['jenkins-agent'], + } + + ::systemd::unit_file {'jenkins-agent.service': + ensure => present, + content => template('profile/jenkins/agent/jenkins-agent.service.erb'), + } -> service {'jenkins-agent': + ensure => running, + enable => true, + require => [ + File[$environment_file], + File[$workdir], + ], + } +} diff --git a/site-modules/profile/manifests/jenkins/worker.pp b/site-modules/profile/manifests/jenkins/agent/docker.pp similarity index 73% rename from site-modules/profile/manifests/jenkins/worker.pp rename to site-modules/profile/manifests/jenkins/agent/docker.pp index 141c56ae..234b28e4 100644 --- a/site-modules/profile/manifests/jenkins/worker.pp +++ b/site-modules/profile/manifests/jenkins/agent/docker.pp @@ -1,16 +1,15 @@ -class profile::jenkins::worker { +class profile::jenkins::agent::docker { include profile::docker - include profile::jenkins::service exec {'add jenkins user to docker group': path => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'], command => 'gpasswd -a jenkins docker', onlyif => 'getent passwd jenkins', unless => 'getent group docker | cut -d: -f4 | grep -qE \'(^|,)jenkins(,|$)\'', require => [ Package['docker-ce'], - Package['jenkins'], + User['jenkins'], ], - notify => Service['jenkins'], + tag => 'reload_jenkins', } } diff --git a/site-modules/profile/manifests/jenkins/agent/sbuild.pp b/site-modules/profile/manifests/jenkins/agent/sbuild.pp new file mode 100644 index 00000000..f7e834c7 --- /dev/null +++ b/site-modules/profile/manifests/jenkins/agent/sbuild.pp @@ -0,0 +1,68 @@ +class profile::jenkins::agent::sbuild { + include ::profile::haveged + + $packages = ['sbuild', 'build-essential', 'devscripts', 'git-buildpackage'] + + package {$packages: + ensure => installed, + } + + if $::lsbdistcodename == 'stretch' { + $pinned_packages = [ + 'devscripts', + 'git', + 'git-buildpackage', + 'git-man', + 'libsbuild-perl', + 'sbuild', + 'schroot', + ] + } + else { + $pinned_packages = undef + } + + if $pinned_packages { + ::apt::pin {'jenkins-sbuild': + explanation => 'Pin jenkins to backports', + codename => "${::lsbdistcodename}-backports", + packages => $pinned_packages, + priority => 990, + } + } else { + ::apt::pin {'jenkins-sbuild': + ensure => 'absent', + } + } + + file {'/usr/share/jenkins/debian-scripts': + ensure => 'directory', + owner => 'jenkins', + group => 'jenkins', + } + + exec {'add jenkins user to sbuild group': + path => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'], + command => 'gpasswd -a jenkins sbuild', + onlyif => 'getent passwd jenkins', + unless => 'getent group sbuild | cut -d: -f4 | grep -qE \'(^|,)jenkins(,|$)\'', + require => [ + Package['sbuild'], + User['jenkins'], + ], + tag => 'restart_jenkins', + } + + ::sudo::conf { 'jenkins-sbuild': + ensure => present, + content => 'jenkins ALL = NOPASSWD: ALL', + priority => 20, + } + + $key_uid = "Software Heritage autobuilder (on ${::swh_hostname['short']}) " + [$key] = (gpg_key {$key_uid: + ensure => present, + owner => 'jenkins', + expire => '365d', + }) +} diff --git a/site-modules/profile/manifests/jenkins/base.pp b/site-modules/profile/manifests/jenkins/base.pp new file mode 100644 index 00000000..39f3c42e --- /dev/null +++ b/site-modules/profile/manifests/jenkins/base.pp @@ -0,0 +1,31 @@ +class profile::jenkins::base { + group {'jenkins': + ensure => present, + system => true, + } + -> user {'jenkins': + ensure => present, + system => true, + gid => 'jenkins', + home => '/var/lib/jenkins', + } + -> file {'/var/lib/jenkins': + ensure => 'directory', + mode => '0755', + owner => 'jenkins', + group => 'jenkins', + } + -> file {'/var/lib/jenkins/.gitconfig': + ensure => present, + mode => '0644', + owner => 'jenkins', + group => 'jenkins', + content => template('profile/jenkins/gitconfig.erb') + } + file {'/usr/share/jenkins': + ensure => 'directory', + mode => '0755', + owner => 'root', + group => 'root', + } +} diff --git a/site-modules/profile/manifests/jenkins/service.pp b/site-modules/profile/manifests/jenkins/service.pp index c9b6a3b5..351989ee 100644 --- a/site-modules/profile/manifests/jenkins/service.pp +++ b/site-modules/profile/manifests/jenkins/service.pp @@ -1,12 +1,19 @@ class profile::jenkins::service { include profile::jenkins::apt_config + include profile::jenkins::base package {'jenkins': ensure => present, - require => Apt::Source['jenkins'], + require => [ + Apt::Source['jenkins'], + User['jenkins'], + Group['jenkins'], + ], } -> service {'jenkins': ensure => running, enable => true, } + + Exec <| tag == 'reload_jenkins' |> ~> Service['jenkins'] } diff --git a/site-modules/profile/manifests/kafka/broker.pp b/site-modules/profile/manifests/kafka/broker.pp index 290bb9e8..9c75da56 100644 --- a/site-modules/profile/manifests/kafka/broker.pp +++ b/site-modules/profile/manifests/kafka/broker.pp @@ -1,29 +1,80 @@ # Kafka broker profile class profile::kafka::broker { include ::profile::zookeeper class {'::kafka': mirror_url => lookup('kafka::mirror_url'), version => lookup('kafka::version'), scala_version => lookup('kafka::scala_version'), } $base_kafka_config = lookup('kafka::broker_config', Hash) $zookeeper_chroot = lookup('kafka::zookeeper::chroot') $zookeeper_servers = lookup('zookeeper::servers', Hash) $zookeeper_port = lookup('zookeeper::client_port', Integer) - $zookeeper_connect_string = join( - $zookeeper_servers.map |$id, $server| {"${server}:${zookeeper_port}${zookeeper_chroot}"}, + $zookeeper_server_string = join( + $zookeeper_servers.map |$id, $server| {"${server}:${zookeeper_port}"}, ',' ) + $zookeeper_connect_string = "${zookeeper_server_string}${zookeeper_chroot}" + + $broker_id = lookup('kafka::brokers', Hash)[$::swh_hostname['internal_fqdn']]['id'] + $kafka_config = $base_kafka_config + { 'zookeeper.connect' => $zookeeper_connect_string, + 'broker.id' => $broker_id + } + + $kafka_logdirs = lookup('kafka::logdirs', Array) + $kafka_logdirs.each |$logdir| { + file {$logdir: + ensure => directory, + owner => 'kafka', + group => 'kafka', + mode => '0750', + } -> Service['kafka'] + } + + include ::profile::prometheus::jmx + + $exporter = $::profile::prometheus::jmx::jar_path + + $exporter_network = lookup('prometheus::kafka::listen_network', Optional[String], 'first', undef) + $exporter_address = lookup('prometheus::kafka::listen_address', Optional[String], 'first', undef) + $actual_exporter_address = pick($exporter_address, ip_for_network($exporter_network)) + $exporter_port = lookup('prometheus::kafka::listen_port') + $target = "${actual_exporter_address}:${exporter_port}" + + $exporter_config = "${::profile::prometheus::jmx::base_directory}/kafka.yml" + + file {$exporter_config: + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/profile/kafka/jmx_exporter.yml', } class {'::kafka::broker': - config => $kafka_config, + config => $kafka_config, + opts => "-javaagent:${exporter}=${exporter_port}:${exporter_config}", + limit_nofile => '65536', + require => [ + File[$exporter], + File[$exporter_config], + ], + } + + ::systemd::dropin_file {"kafka/restart.conf": + ensure => present, + unit => "kafka.service", + filename => 'restart.conf', + content => "[Service]\nRestart=on-failure\nRestartSec=5\n", + } + + ::profile::prometheus::export_scrape_config {'kafka': + target => $target, } } diff --git a/site-modules/profile/manifests/kibana.pp b/site-modules/profile/manifests/kibana.pp index f723fb98..7d6503e2 100644 --- a/site-modules/profile/manifests/kibana.pp +++ b/site-modules/profile/manifests/kibana.pp @@ -1,35 +1,36 @@ class profile::kibana { package { 'openjdk-8-jre-headless': ensure => 'present', } $keyid = lookup('elastic::apt_config::keyid') $key = lookup('elastic::apt_config::key') + $version = lookup('elastic::elk_version') apt::source { 'elastic-6.x': location => 'https://artifacts.elastic.co/packages/6.x/apt', release => 'stable', repos => 'main', key => { id => $keyid, content => $key, }, } package { 'kibana': - ensure => '6.3.2', + ensure => $version, } apt::pin { 'kibana': packages => 'kibana', - version => '6.3.2', + version => $version, priority => 1001, } file { '/etc/kibana/kibana.yml': ensure => 'file', content => template('profile/kibana/kibana.yml.erb'), } } diff --git a/site-modules/profile/manifests/letsencrypt.pp b/site-modules/profile/manifests/letsencrypt.pp new file mode 100644 index 00000000..53e4fb01 --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt.pp @@ -0,0 +1,41 @@ +# Base configuration for Let's Encrypt + +class profile::letsencrypt { + include ::profile::letsencrypt::apt_config + include ::profile::letsencrypt::gandi_livedns_hook + + class {'letsencrypt': + config => { + email => lookup('letsencrypt::account_email'), + server => lookup('letsencrypt::server'), + } + } + + $certificates = lookup('letsencrypt::certificates', Hash) + + $certificates.each |$key, $settings| { + $domains = $settings['domains'] + + $deploy_hook = pick($settings['deploy_hook'], 'puppet_export') + + include "::profile::letsencrypt::${deploy_hook}_hook" + $deploy_hook_path = getvar("profile::letsencrypt::${deploy_hook}_hook::hook_path") + $deploy_hook_extra_opts = getvar("profile::letsencrypt::${deploy_hook}_hook::hook_extra_opts") + + File[$deploy_hook_path] + -> ::letsencrypt::certonly {$key: + * => deep_merge({ + domains => $domains, + custom_plugin => true, + additional_args => [ + '--authenticator manual', + '--preferred-challenges dns', + '--manual-public-ip-logging-ok', + "--manual-auth-hook '${::profile::letsencrypt::gandi_livedns_hook::hook_path} auth'", + "--manual-cleanup-hook '${::profile::letsencrypt::gandi_livedns_hook::hook_path} cleanup'", + "--deploy-hook '${deploy_hook_path}'", + ], + }, $deploy_hook_extra_opts) + } -> Profile::Letsencrypt::Certificate <| title == $key |> + } +} diff --git a/site-modules/profile/manifests/letsencrypt/apt_config.pp b/site-modules/profile/manifests/letsencrypt/apt_config.pp new file mode 100644 index 00000000..153f0c4a --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt/apt_config.pp @@ -0,0 +1,21 @@ +# apt configuration for Let's Encrypt + +class profile::letsencrypt::apt_config { + $pinned_packages = ['certbot'] + prefix([ + 'acme', 'asn1crypto', 'certbot', 'certbot-*', 'cryptography', 'idna', + 'josepy', 'parsedatetime', 'pbr', + ], 'python3-') + + if $::lsbdistcodename == 'stretch' { + ::apt::pin {'letsencrypt-backports': + explanation => 'Pin letsencrypt backports', + codename => "${::lsbdistcodename}-backports", + packages => $pinned_packages, + priority => 990, + } + } else { + ::apt::pin {'letsencrypt-backports': + ensure => absent, + } + } +} diff --git a/site-modules/profile/manifests/letsencrypt/certificate.pp b/site-modules/profile/manifests/letsencrypt/certificate.pp new file mode 100644 index 00000000..5c15f1c4 --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt/certificate.pp @@ -0,0 +1,41 @@ +# Retrieve the given certificate from the puppet master + +define profile::letsencrypt::certificate ( + String $basename = $title, + String $privkey_owner = 'root', + String $privkey_group = 'root', + Stdlib::Filemode $privkey_mode = '0600', +) { + include ::profile::letsencrypt::certificate_base + + $certs_directory = lookup('letsencrypt::certificates::directory') + + $basedir = "${certs_directory}/${basename}" + + file {$basedir: + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + + ['cert.pem', 'chain.pem', 'fullchain.pem'].each |$filename| { + file {"${basedir}/${filename}": + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + content => template("profile/letsencrypt/fetch_certificate_file.erb") + } + } + + ['privkey.pem'].each |$filename| { + file {"${basedir}/${filename}": + ensure => present, + owner => $privkey_owner, + group => $privkey_group, + mode => $privkey_mode, + content => template("profile/letsencrypt/fetch_certificate_file.erb") + } + } +} diff --git a/site-modules/profile/manifests/letsencrypt/certificate_base.pp b/site-modules/profile/manifests/letsencrypt/certificate_base.pp new file mode 100644 index 00000000..005d71d6 --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt/certificate_base.pp @@ -0,0 +1,14 @@ +# Base resources for imported letsencrypt certs + +class profile::letsencrypt::certificate_base { + $certs_directory = lookup('letsencrypt::certificates::directory') + + file {$certs_directory: + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + recurse => true, + purge => true, + } +} diff --git a/site-modules/profile/manifests/letsencrypt/gandi_livedns_hook.pp b/site-modules/profile/manifests/letsencrypt/gandi_livedns_hook.pp new file mode 100644 index 00000000..7dff2629 --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt/gandi_livedns_hook.pp @@ -0,0 +1,19 @@ +class profile::letsencrypt::gandi_livedns_hook { + $hook_path = '/usr/local/bin/letsencrypt_gandi_livedns' + $hook_configfile = '/etc/letsencrypt/gandi_livedns.yml' + $hook_config = lookup('letsencrypt::gandi_livedns_hook::config', Hash) + + file {$hook_path: + owner => 'root', + group => 'root', + mode => '0755', + content => template('profile/letsencrypt/letsencrypt_gandi_livedns.erb'), + } + + file {$hook_configfile: + owner => 'root', + group => 'root', + mode => '0600', + content => inline_yaml($hook_config), + } +} diff --git a/site-modules/profile/manifests/letsencrypt/gandi_paas_hook.pp b/site-modules/profile/manifests/letsencrypt/gandi_paas_hook.pp new file mode 100644 index 00000000..3e60386f --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt/gandi_paas_hook.pp @@ -0,0 +1,26 @@ +# Push certificates to Gandi PaaS + +class profile::letsencrypt::gandi_paas_hook { + # Gandi PaaS only supports keys up to 2048 bits + $hook_extra_opts = { + key_size => 2048, + } + + $hook_path = '/usr/local/bin/letsencrypt_gandi_paas' + $hook_configfile = '/etc/letsencrypt/gandi_paas.yml' + $hook_config = lookup('letsencrypt::gandi_paas_hook::config', Hash) + + file {$hook_path: + owner => 'root', + group => 'root', + mode => '0755', + content => template('profile/letsencrypt/letsencrypt_gandi_paas.erb'), + } + + file {$hook_configfile: + owner => 'root', + group => 'root', + mode => '0600', + content => inline_yaml($hook_config), + } +} diff --git a/site-modules/profile/manifests/letsencrypt/puppet_export_hook.pp b/site-modules/profile/manifests/letsencrypt/puppet_export_hook.pp new file mode 100644 index 00000000..8a3b5dfa --- /dev/null +++ b/site-modules/profile/manifests/letsencrypt/puppet_export_hook.pp @@ -0,0 +1,20 @@ +# Certbot deploy hook to copy certificates to a puppet-accessible path + +class profile::letsencrypt::puppet_export_hook { + $hook_extra_opts = {} + $hook_path = '/usr/local/bin/letsencrypt_puppet_export' + file {$hook_path: + owner => 'root', + group => 'root', + mode => '0755', + content => template('profile/letsencrypt/letsencrypt_puppet_export.erb'), + } + + $export_directory = lookup('letsencrypt::certificates::exported_directory') + file {$export_directory: + ensure => 'directory', + owner => 'puppet', + group => 'puppet', + mode => '0700', + } +} diff --git a/site-modules/profile/manifests/logstash.pp b/site-modules/profile/manifests/logstash.pp index 292e2ba2..22e58901 100644 --- a/site-modules/profile/manifests/logstash.pp +++ b/site-modules/profile/manifests/logstash.pp @@ -1,44 +1,51 @@ class profile::logstash { package { 'openjdk-8-jre-headless': ensure => 'present', } - $keyid = lookup('elastic::apt_config::keyid') - $key = lookup('elastic::apt_config::key') + $keyid = lookup('elastic::apt_config::keyid') + $key = lookup('elastic::apt_config::key') + $version = sprintf("1:%s-1", lookup('elastic::elk_version')) apt::source { 'elastic-6.x': location => 'https://artifacts.elastic.co/packages/6.x/apt', release => 'stable', repos => 'main', key => { id => $keyid, content => $key, }, } package { 'logstash': - ensure => '1:6.3.2-1', + ensure => $version, + } + + apt::pin { 'logstash': + packages => 'logstash', + version => $version, + priority => 1001, } file { '/etc/logstash/conf.d/input.conf': ensure => 'file', content => template('profile/logstash/input.conf.erb'), } file { '/etc/logstash/conf.d/output.conf': ensure => 'file', content => template('profile/logstash/output.conf.erb'), } file { '/etc/logstash/conf.d/filter.conf': ensure => 'file', content => template('profile/logstash/filter.conf.erb'), } service { 'logstash': ensure => running, enable => true, } } diff --git a/site-modules/profile/manifests/memcached.pp b/site-modules/profile/manifests/memcached.pp index 0c35c3cc..2cd05c76 100644 --- a/site-modules/profile/manifests/memcached.pp +++ b/site-modules/profile/manifests/memcached.pp @@ -1,12 +1,12 @@ # Install and configure local memcached server class profile::memcached { $memcached_bind = lookup('memcached::server::bind') $memcached_port = lookup('memcached::server::port') $memcached_memory = lookup('memcached::server::max_memory') class {'::memcached': listen_ip => $memcached_bind, tcp_port => $memcached_port, - max_memory => $memcached_max_memory, + max_memory => $memcached_memory, } } diff --git a/site-modules/profile/manifests/munin/master.pp b/site-modules/profile/manifests/munin/master.pp index f4a7b5eb..01040b20 100644 --- a/site-modules/profile/manifests/munin/master.pp +++ b/site-modules/profile/manifests/munin/master.pp @@ -1,51 +1,64 @@ # Munin master class class profile::munin::master { $master_hostname = lookup('munin::master::hostname') $master_hostname_domain = join(delete_at(split($master_hostname, '[.]'), 0), '.') $master_hostname_target = "${::hostname}.${master_hostname_domain}." class { '::munin::master': extra_config => ["cgiurl_graph http://$master_hostname"], } include ::profile::apache::common include ::apache::mod::rewrite include ::apache::mod::fcgid + $docroot = '/var/www/html' + + file {$docroot: + ensure => directory, + owner => 'www-data', + group => 'www-data', + mode => '0755', + } + apache::vhost { $master_hostname: port => 80, - docroot => '/var/www/html', + docroot => $docroot, rewrites => [ { comment => 'static resources', rewrite_rule => [ '^/favicon.ico /etc/munin/static/favicon.ico [L]', '^/static/(.*) /etc/munin/static/$1 [L]', ], }, { comment => 'HTML', rewrite_cond => [ '%{REQUEST_URI} .html$ [or]', '%{REQUEST_URI} =/', ], rewrite_rule => [ '^/(.*) /usr/lib/munin/cgi/munin-cgi-html/$1 [L]', ], }, { comment => 'Images', rewrite_rule => [ '^/munin-cgi/munin-cgi-graph/(.*) /usr/lib/munin/cgi/munin-cgi-graph/$1 [L]', '^/(.*) /usr/lib/munin/cgi/munin-cgi-graph/$1 [L]', ], }, ], directories => [ { 'path' => '/usr/lib/munin/cgi', 'options' => '+ExecCGI', - 'sethandler' => 'fcgid-script' }, + 'sethandler' => 'fcgid-script' + }, + { 'path' => "${export_path}", + 'options' => '+Indexes', # allow listing + } ], } } diff --git a/site-modules/profile/manifests/munin/stats_export.pp b/site-modules/profile/manifests/munin/stats_export.pp deleted file mode 100644 index 79a9d353..00000000 --- a/site-modules/profile/manifests/munin/stats_export.pp +++ /dev/null @@ -1,84 +0,0 @@ -# stats_export master class -class profile::munin::stats_export { - $vhost_name = lookup('stats_export::vhost::name') - $vhost_docroot = lookup('stats_export::vhost::docroot') - $vhost_ssl_protocol = lookup('stats_export::vhost::ssl_protocol') - $vhost_ssl_honorcipherorder = lookup('stats_export::vhost::ssl_honorcipherorder') - $vhost_ssl_cipher = lookup('stats_export::vhost::ssl_cipher') - $vhost_hsts_header = lookup('stats_export::vhost::hsts_header') - - $export_file = "${vhost_docroot}/history_counters.json" - - $packages = ['python3-click'] - - package {$packages: - ensure => present, - } - - file {'/usr/local/bin/export-rrd': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/profile/munin/stats_export/export-rrd', - require => Package[$packages], - } - - cron {'stats_export': - ensure => present, - user => 'www-data', - command => "/usr/local/bin/export-rrd > ${export_file}.tmp && /bin/mv ${export_file}.tmp ${export_file}", - hour => fqdn_rand(24, 'stats_export_hour'), - minute => fqdn_rand(60, 'stats_export_minute'), - month => '*', - monthday => '*', - weekday => '*', - require => [ - File['/usr/local/bin/export-rrd'], - File[$vhost_docroot], - ], - } - - file {$vhost_docroot: - ensure => directory, - owner => 'www-data', - group => 'www-data', - mode => '0755', - } - - include ::profile::apache::common - include ::profile::ssl - - ::apache::vhost {"${vhost_name}_non-ssl": - servername => $vhost_name, - port => '80', - docroot => $vhost_docroot, - redirect_status => 'permanent', - redirect_dest => "https://${vhost_name}/", - } - - $ssl_cert_name = 'stats_export_softwareheritage_org' - $ssl_cert = $::profile::ssl::certificate_paths[$ssl_cert_name] - $ssl_chain = $::profile::ssl::chain_paths[$ssl_cert_name] - $ssl_key = $::profile::ssl::private_key_paths[$ssl_cert_name] - - ::apache::vhost {"${vhost_name}_ssl": - servername => $vhost_name, - port => '443', - ssl => true, - ssl_protocol => $vhost_ssl_protocol, - ssl_honorcipherorder => $vhost_ssl_honorcipherorder, - ssl_cipher => $vhost_ssl_cipher, - ssl_cert => $ssl_cert, - ssl_chain => $ssl_chain, - ssl_key => $ssl_key, - headers => [$vhost_hsts_header], - docroot => $vhost_docroot, - require => [ - File[$ssl_cert], - File[$ssl_chain], - File[$ssl_key], - ], - } - -} diff --git a/site-modules/profile/manifests/network.pp b/site-modules/profile/manifests/network.pp index 70ec6b32..fa088047 100644 --- a/site-modules/profile/manifests/network.pp +++ b/site-modules/profile/manifests/network.pp @@ -1,47 +1,81 @@ # Network configuration for Software Heritage servers # # Supports one private and one public interface class profile::network { debnet::iface::loopback { 'lo': } + # The network description is expected to be a dict of key route_label + # (values: private, default) and value a dict describing the interface. + # The interface dict has the following possible keys: + # - interface: interface's name + # - address: ip address for the node + # - netmask: netmask + # - gateway: to use for the network + # - ups: Post instruction when the interface is up (should be set to [] when + # none) + # - downs: Post instructions to run when the interface is teared down (should + # be set to [] when none) + $interfaces = lookup('networks') each($interfaces) |$label, $data| { if $label == 'private' { file_line {'private route table': ensure => 'present', line => '42 private', path => '/etc/iproute2/rt_tables', } - $ups = [ - "ip route add 192.168.101.0/24 via ${data['gateway']}", - "ip route add 192.168.200.0/21 via ${data['gateway']}", - "ip rule add from ${data['address']} table private", - "ip route add default via ${data['gateway']} dev ${data['interface']} table private", - 'ip route flush cache', - ] - $downs = [ - "ip route del default via ${data['gateway']} dev ${data['interface']} table private", - "ip rule del from ${data['address']} table private", - "ip route del 192.168.200.0/24 via ${data['gateway']}", - "ip route del 192.168.101.0/24 via ${data['gateway']}", - 'ip route flush cache', - ] + + if $data['ups'] { + $ups = $data['ups'] + } else { + $ups = [ + "ip route add 192.168.101.0/24 via ${data['gateway']}", + "ip route add 192.168.200.0/21 via ${data['gateway']}", + "ip rule add from ${data['address']} table private", + "ip route add 192.168.100.0/24 src ${data['address']} dev ${data['interface']} table private", + "ip route add default via ${data['gateway']} dev ${data['interface']} table private", + 'ip route flush cache', + ] + } + + if $data['downs'] { + $downs = $data['downs'] + } else { + $downs = [ + "ip route del default via ${data['gateway']} dev ${data['interface']} table private", + "ip route del 192.168.100.0/24 src ${data['address']} dev ${data['interface']} table private", + "ip rule del from ${data['address']} table private", + "ip route del 192.168.200.0/24 via ${data['gateway']}", + "ip route del 192.168.101.0/24 via ${data['gateway']}", + 'ip route flush cache', + ] + } + $gateway = undef } else { - $ups = [] - $downs = [] + if $data['ups'] { + $ups = $data['ups'] + } else { + $ups = [] + } + + if $data['downs'] { + $downs = $data['downs'] + } else { + $downs = [] + } $gateway = $data['gateway'] } debnet::iface { $data['interface']: method => 'static', address => $data['address'], netmask => $data['netmask'], gateway => $gateway, ups => $ups, downs => $downs, } } } diff --git a/site-modules/profile/manifests/pgbouncer.pp b/site-modules/profile/manifests/pgbouncer.pp new file mode 100644 index 00000000..7a43a2b5 --- /dev/null +++ b/site-modules/profile/manifests/pgbouncer.pp @@ -0,0 +1,26 @@ +# Manage a pgbouncer server +class profile::pgbouncer { + $config_params = lookup('pgbouncer::config_params') + $userlist = lookup('pgbouncer::userlist') + $databases = lookup('pgbouncer::databases') + $pgbouncer_user = lookup('pgbouncer::user') + $pgbouncer_group = lookup('pgbouncer::group') + + # Need format manipulation (expected format in pgbouncer class is {key, + # value} with no nested data) + $listen_addr = join($config_params['listen_addr'], ',') + $admin_users = join($config_params['admin_users'], ',') + + $expected_config_params = merge($config_params, { + listen_addr => $listen_addr, + admin_users => $admin_users, + }) + + class {'::pgbouncer': + config_params => $expected_config_params, + userlist => $userlist, + databases => $databases, + user => $pgbouncer_user, + group => $pgbouncer_group, + } +} diff --git a/site-modules/profile/manifests/phabricator.pp b/site-modules/profile/manifests/phabricator.pp index dbb7c7b5..91ea614b 100644 --- a/site-modules/profile/manifests/phabricator.pp +++ b/site-modules/profile/manifests/phabricator.pp @@ -1,305 +1,321 @@ # Setup an instance of phabricator class profile::phabricator { $phabricator_basepath = lookup('phabricator::basepath') $phabricator_user = lookup('phabricator::user') $phabricator_vcs_user = lookup('phabricator::vcs_user') $phabricator_db_root_password = lookup('phabricator::mysql::root_password') $phabricator_db_basename = lookup('phabricator::mysql::database_prefix') $phabricator_db_user = lookup('phabricator::mysql::username') $phabricator_db_password = lookup('phabricator::mysql::password') $phabricator_db_max_allowed_packet = lookup('phabricator::mysql::conf::max_allowed_packet') $phabricator_db_sql_mode = lookup('phabricator::mysql::conf::sql_mode') $phabricator_db_ft_stopword_file = lookup('phabricator::mysql::conf::ft_stopword_file') $phabricator_db_ft_min_word_len = lookup('phabricator::mysql::conf::ft_min_word_len') $phabricator_db_ft_boolean_syntax = lookup('phabricator::mysql::conf::ft_boolean_syntax') $phabricator_db_innodb_buffer_pool_size = lookup('phabricator::mysql::conf::innodb_buffer_pool_size') $phabricator_db_innodb_file_per_table = lookup('phabricator::mysql::conf::innodb_file_per_table') $phabricator_db_innodb_flush_method = lookup('phabricator::mysql::conf::innodb_flush_method') $phabricator_db_innodb_log_file_size = lookup('phabricator::mysql::conf::innodb_log_file_size') + $phabricator_db_max_connections = lookup('phabricator::mysql::conf::max_connections') $phabricator_fpm_listen = lookup('phabricator::php::fpm_listen') $phabricator_max_size = lookup('phabricator::php::max_file_size') $phabricator_opcache_validate_timestamps = lookup('phabricator::php::opcache_validate_timestamps') $phabricator_notification_listen = lookup('phabricator::notification::listen') $phabricator_notification_client_host = lookup('phabricator::notification::client_host') $phabricator_notification_client_port = lookup('phabricator::notification::client_port') $phabricator_vhost_name = lookup('phabricator::vhost::name') $phabricator_vhost_docroot = lookup('phabricator::vhost::docroot') $phabricator_vhost_basic_auth_file = "${phabricator_basepath}/http_auth" $phabricator_vhost_basic_auth_content = lookup('phabricator::vhost::basic_auth_content') $phabricator_vhost_ssl_protocol = lookup('phabricator::vhost::ssl_protocol') $phabricator_vhost_ssl_honorcipherorder = lookup('phabricator::vhost::ssl_honorcipherorder') $phabricator_vhost_ssl_cipher = lookup('phabricator::vhost::ssl_cipher') $phabricator_vhost_hsts_header = lookup('phabricator::vhost::hsts_header') $homedirs = { $phabricator_user => $phabricator_basepath, $phabricator_vcs_user => "${phabricator_basepath}/vcshome", } $homedir_modes = { $phabricator_user => '0644', $phabricator_vcs_user => '0640', } each([$phabricator_user, $phabricator_vcs_user]) |$name| { user {$name: ensure => present, system => true, shell => '/bin/bash', home => $homedirs[$name], } file {$homedirs[$name]: ensure => directory, owner => $name, group => $name, mode => $homedir_modes[$name], } } ::sudo::conf {'phabricator-ssh': ensure => present, content => "${phabricator_vcs_user} ALL=(${phabricator_user}) SETENV: NOPASSWD: /usr/bin/git-upload-pack, /usr/bin/git-receive-pack, /usr/bin/hg", } ::sudo::conf {'phabricator-http': ensure => present, content => "www-data ALL=(${phabricator_user}) SETENV: NOPASSWD: /usr/bin/git-http-backend, /usr/bin/hg", require => File['/usr/bin/git-http-backend'], } file {'/usr/bin/git-http-backend': ensure => link, target => '/usr/lib/git-core/git-http-backend', } $phabricator_ssh_hook = '/usr/bin/phabricator-ssh-hook.sh' $phabricator_ssh_config = '/etc/ssh/ssh_config.phabricator' file {$phabricator_ssh_hook: ensure => present, owner => 'root', group => 'root', mode => '0755', content => template('profile/phabricator/phabricator-ssh-hook.sh.erb'), } file {$phabricator_ssh_config: ensure => present, owner => 'root', group => 'root', mode => '0600', content => template('profile/phabricator/sshd_config.phabricator.erb'), require => File[$phabricator_ssh_hook], } ::systemd::unit_file {'phabricator-sshd.service': ensure => present, content => template('profile/phabricator/phabricator-sshd.service.erb'), require => File[$phabricator_ssh_config], } ~> service {'phabricator-sshd': ensure => 'running', enable => true, require => [ File['/etc/systemd/system/phabricator-sshd.service'], ], } include ::mysql::client class {'::mysql::server': root_password => $phabricator_db_root_password, override_options => { mysqld => { max_allowed_packet => $phabricator_db_max_allowed_packet, sql_mode => $phabricator_db_sql_mode, ft_stopword_file => $phabricator_db_ft_stopword_file, ft_min_word_len => $phabricator_db_ft_min_word_len, ft_boolean_syntax => $phabricator_db_ft_boolean_syntax, innodb_buffer_pool_size => $phabricator_db_innodb_buffer_pool_size, innodb_file_per_table => $phabricator_db_innodb_file_per_table, innodb_flush_method => $phabricator_db_innodb_flush_method, innodb_log_file_size => $phabricator_db_innodb_log_file_size, + max_connections => $phabricator_db_max_connections, + local_infile => 0, } } } $mysql_username = "${phabricator_db_user}@localhost" $mysql_tables = "${phabricator_db_basename}_%.*" mysql_user {$mysql_username: ensure => present, password_hash => mysql_password($phabricator_db_password), } mysql_grant {"${mysql_username}/${mysql_tables}": user => $mysql_username, table => $mysql_tables, privileges => ['ALL'], require => Mysql_user[$mysql_username], } include ::profile::php ::php::fpm::pool {'phabricator': listen => $phabricator_fpm_listen, user => 'www-data', php_admin_value => { post_max_size => $phabricator_max_size, upload_max_filesize => $phabricator_max_size, 'opcache.validate_timestamps' => $phabricator_opcache_validate_timestamps, + 'mysqli.allow_local_infile' => 0, }, } ::php::extension {[ 'apcu', 'mailparse', ]: provider => 'apt', package_prefix => 'php-', } ::php::extension {[ 'curl', 'gd', ]: provider => 'apt', } include ::profile::ssl include ::profile::apache::common include ::apache::mod::proxy include ::profile::apache::mod_proxy_fcgi ::apache::mod {'proxy_wstunnel':} ::apache::vhost {"${phabricator_vhost_name}_non-ssl": servername => $phabricator_vhost_name, port => '80', docroot => $phabricator_vhost_docroot, docroot_owner => $phabricator_user, docroot_group => $phabricator_user, redirect_status => 'permanent', redirect_dest => "https://${phabricator_vhost_name}/", } $ssl_cert_name = 'star_softwareheritage_org' $ssl_cert = $::profile::ssl::certificate_paths[$ssl_cert_name] $ssl_chain = $::profile::ssl::chain_paths[$ssl_cert_name] $ssl_key = $::profile::ssl::private_key_paths[$ssl_cert_name] ::apache::vhost {"${phabricator_vhost_name}_ssl": servername => $phabricator_vhost_name, port => '443', ssl => true, ssl_protocol => $phabricator_vhost_ssl_protocol, ssl_honorcipherorder => $phabricator_vhost_ssl_honorcipherorder, ssl_cipher => $phabricator_vhost_ssl_cipher, ssl_cert => $ssl_cert, ssl_chain => $ssl_chain, ssl_key => $ssl_key, headers => [$phabricator_vhost_hsts_header], docroot => $phabricator_vhost_docroot, docroot_owner => $phabricator_user, docroot_group => $phabricator_user, rewrites => [ { rewrite_rule => '^/rsrc/(.*) - [L,QSA]' }, { rewrite_rule => '^/favicon.ico - [L,QSA]' }, { rewrite_rule => "^/ws/(.*)$ ws://${phabricator_notification_listen}/\$1 [L,P]" }, { rewrite_rule => "^(.*)$ fcgi://${phabricator_fpm_listen}${phabricator_vhost_docroot}/index.php?__path__=\$1 [B,L,P,QSA]" }, ], setenvif => [ "Authorization \"(.*)\" HTTP_AUTHORIZATION=\$1", ], require => [ File[$ssl_cert], File[$ssl_chain], File[$ssl_key], ], } file {$phabricator_vhost_basic_auth_file: ensure => absent, } # Uses: # $phabricator_basepath # $phabricator_user ::systemd::unit_file {'phabricator-phd.service': ensure => present, content => template('profile/phabricator/phabricator-phd.service.erb'), } ~> service {'phabricator-phd': ensure => 'running', enable => true, } # Uses: # $phabricator_basepath # $phabricator_user # $phabricator_notification_* ::systemd::unit_file {'phabricator-aphlict.service': ensure => present, content => template('profile/phabricator/phabricator-aphlict.service.erb'), } ~> service {'phabricator-aphlict': ensure => 'running', enable => true, } package {'python-pygments': ensure => installed, } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"phabricator http redirect on ${::fqdn}": service_name => 'phabricator http redirect', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $phabricator_vhost_name, http_vhost => $phabricator_vhost_name, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"phabricator https on ${::fqdn}": service_name => 'phabricator', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $phabricator_vhost_name, http_vhost => $phabricator_vhost_name, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"phabricator https certificate ${::fqdn}": service_name => 'phabricator https certificate', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $phabricator_vhost_name, http_vhost => $phabricator_vhost_name, http_ssl => true, http_sni => true, http_certificate => 60, }, target => $icinga_checks_file, tag => 'icinga2::exported', } + + each($::ssh) |$algo, $data| { + $real_algo = $algo ? { + 'ecdsa' => 'ecdsa-sha2-nistp256', + default => $algo, + } + @@sshkey {"phabricator-${::fqdn}-${real_algo}": + host_aliases => [$phabricator_vhost_name], + type => $real_algo, + key => $data['key'], + } + } } diff --git a/site-modules/profile/manifests/postgresql/apt_config.pp b/site-modules/profile/manifests/postgresql/apt_config.pp index 55f8b122..adad1f0a 100644 --- a/site-modules/profile/manifests/postgresql/apt_config.pp +++ b/site-modules/profile/manifests/postgresql/apt_config.pp @@ -1,29 +1,20 @@ # PostgreSQL APT configuration class profile::postgresql::apt_config { $pgdg_mirror = lookup('postgresql::apt_config::pgdg::mirror') $pgdg_keyid = lookup('postgresql::apt_config::pgdg::keyid') $pgdg_key = lookup('postgresql::apt_config::pgdg::key') - $pglogical_mirror = lookup('postgresql::apt_config::pglogical::mirror') - $pglogical_keyid = lookup('postgresql::apt_config::pglogical::keyid') - $pglogical_key = lookup('postgresql::apt_config::pglogical::key') ::apt::source {'pgdg': location => $pgdg_mirror, release => "${::lsbdistcodename}-pgdg", repos => 'main', key => { id => $pgdg_keyid, content => $pgdg_key, }, } ::apt::source {'pglogical': - location => $pglogical_mirror, - release => "${::lsbdistcodename}-2ndquadrant", - repos => 'main', - key => { - id => $pglogical_keyid, - content => $pglogical_key, - }, + ensure => 'absent', } } diff --git a/site-modules/profile/manifests/postgresql/backup.pp b/site-modules/profile/manifests/postgresql/backup.pp new file mode 100644 index 00000000..a67c8aaa --- /dev/null +++ b/site-modules/profile/manifests/postgresql/backup.pp @@ -0,0 +1,15 @@ +class profile::postgresql::backup { + include profile::postgresql::client + + $pg_backup_user = lookup('swh::deploy::db::pgbouncer::user::login') + $pg_backup_password = lookup('swh::deploy::db::pgbouncer::user::password') + + file { '/usr/local/bin/pg_cluster_backup.sh': + ensure => file, + owner => root, + group => staff, + mode => '0755', + content => template('profile/swh/pg_cluster_backup.sh.erb'), + } + +} diff --git a/site-modules/profile/manifests/postgresql/client.pp b/site-modules/profile/manifests/postgresql/client.pp new file mode 100644 index 00000000..87671fb1 --- /dev/null +++ b/site-modules/profile/manifests/postgresql/client.pp @@ -0,0 +1,7 @@ +class profile::postgresql::client { + include profile::postgresql::apt_config + + package { 'postgresql-client': + ensure => present, + } +} diff --git a/site-modules/profile/manifests/postgresql/server.pp b/site-modules/profile/manifests/postgresql/server.pp new file mode 100644 index 00000000..55ba135c --- /dev/null +++ b/site-modules/profile/manifests/postgresql/server.pp @@ -0,0 +1,50 @@ +class profile::postgresql::server { + class { 'postgresql::globals': + encoding => 'UTF-8', + locale => 'en_US.UTF-8', + manage_package_repo => true, + version => '11', + } + + $postgres_pass = lookup('swh::deploy::db::postgres::password') + $server_port = lookup('postgres::server::port') + $server_addresses = lookup('postgres::server::listen_addresses').join(',') + # allow access through credentials + $network_access = lookup('postgres::server::network_access').map | $nwk | { + "host all all ${nwk} md5" + } + + class { 'postgresql::server': + ip_mask_allow_all_users => '0.0.0.0/0', + ipv4acls => $network_access, + postgres_password => $postgres_pass, + port => $server_port, + listen_addresses => [$server_addresses], + } + + $guest = 'guest' + postgresql::server::role { $guest: + password_hash => postgresql_password($guest, 'guest'), + } + + $dbs = lookup('swh::dbs') + each($dbs) | $db_type, $db_config | { + # db_type in {storage, indexer, scheduler, etc...} + $db_pass = lookup("swh::deploy::db::${db_type}::password") + $db_name = $db_config['name'] + $db_user = $db_config['user'] + + postgresql::server::db { $db_name: + user => $db_user, + password => $db_pass, + owner => $db_user + } + + # guest user has read access on tables + postgresql::server::database_grant { $db_name: + privilege => 'connect', + db => $db_name, + role => $guest, + } + } +} diff --git a/site-modules/profile/manifests/prometheus/base.pp b/site-modules/profile/manifests/prometheus/base.pp new file mode 100644 index 00000000..8fb486bb --- /dev/null +++ b/site-modules/profile/manifests/prometheus/base.pp @@ -0,0 +1,11 @@ +# Base configuration for all prometheus exporters +class profile::prometheus::base { + include profile::prometheus::apt_config + + file {'/etc/prometheus': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0644', + } +} diff --git a/site-modules/profile/manifests/prometheus/jmx.pp b/site-modules/profile/manifests/prometheus/jmx.pp new file mode 100644 index 00000000..ba8e78db --- /dev/null +++ b/site-modules/profile/manifests/prometheus/jmx.pp @@ -0,0 +1,22 @@ +class profile::prometheus::jmx { + $base_directory = '/opt/prometheus-jmx-exporter' + $version = lookup('prometheus::jmx::version') + + $jar_name = "jmx_prometheus_javaagent-${version}.jar" + $upstream_url = "https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${version}/${jar_name}" + $jar_path = "${base_directory}/${jar_name}" + + file {$base_directory: + ensure => 'directory', + mode => '0644', + owner => 'root', + group => 'root', + } + + file {$jar_path: + ensure => 'present', + owner => 'root', + group => 'root', + source => $upstream_url, + } +} diff --git a/site-modules/profile/manifests/prometheus/node.pp b/site-modules/profile/manifests/prometheus/node.pp index c2f575ee..800d8fee 100644 --- a/site-modules/profile/manifests/prometheus/node.pp +++ b/site-modules/profile/manifests/prometheus/node.pp @@ -1,58 +1,96 @@ # Prometheus configuration for nodes class profile::prometheus::node { - include profile::prometheus::apt_config + include profile::prometheus::base $defaults_file = '/etc/default/prometheus-node-exporter' package {'prometheus-node-exporter': - ensure => latest, + ensure => present, notify => Service['prometheus-node-exporter'], } service {'prometheus-node-exporter': ensure => 'running', enable => true, require => [ Package['prometheus-node-exporter'], File[$defaults_file], ] } ::systemd::dropin_file {'prometheus-node-exporter/restart.conf': ensure => present, unit => 'prometheus-node-exporter.service', filename => 'restart.conf', content => "[Service]\nRestart=always\nRestartSec=5\n", } $lookup_defaults_config = lookup('prometheus::node::defaults_config', Hash) $listen_network = lookup('prometheus::node::listen_network', Optional[String], 'first', undef) $listen_address = lookup('prometheus::node::listen_address', Optional[String], 'first', undef) $actual_listen_address = pick($listen_address, ip_for_network($listen_network)) $listen_port = lookup('prometheus::node::listen_port') $target = "${actual_listen_address}:${listen_port}" $defaults_config = deep_merge( $lookup_defaults_config, { web => { listen_address => $target, }, } ) # Uses $defaults_config file {$defaults_file: ensure => 'present', owner => 'root', group => 'root', mode => '0644', content => template('profile/prometheus/node/prometheus-node-exporter.defaults.erb'), require => Package['prometheus-node-exporter'], notify => Service['prometheus-node-exporter'], } + $textfile_directory = lookup('prometheus::node::textfile_directory') + $scripts = lookup('prometheus::node::scripts', Hash, 'deep') + $scripts_directory = lookup('prometheus::node::scripts::directory') + + file {$scripts_directory: + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0700', + recurse => true, + purge => true, + require => Package['prometheus-node-exporter'], + } + + each($scripts) |$script, $data| { + file {"${scripts_directory}/${script}": + ensure => present, + owner => 'root', + group => 'root', + mode => '0700', + content => template("profile/prometheus/node/scripts/${script}.erb"), + } + if $data['mode'] == 'cron' { + cron {"prometheus-node-exporter-${script}": + ensure => absent, + user => $data['cron']['user'], + } + + profile::cron::d {"prometheus-node-exporter-${script}": + target => 'prometheus', + user => $data['cron']['user'], + command => "chronic ${scripts_directory}/${script}", + random_seed => "prometheus-node-exporter-${script}", + * => $data['cron']['specification'], + } + } + } + profile::prometheus::export_scrape_config {'node': target => $target, } } diff --git a/site-modules/profile/manifests/prometheus/server.pp b/site-modules/profile/manifests/prometheus/server.pp index 78142217..8032cdf8 100644 --- a/site-modules/profile/manifests/prometheus/server.pp +++ b/site-modules/profile/manifests/prometheus/server.pp @@ -1,109 +1,109 @@ # Configure the Prometheus server class profile::prometheus::server { - include profile::prometheus::apt_config + include profile::prometheus::base $config_dir = '/etc/prometheus' $config_file = "${config_dir}/prometheus.yml" $defaults_file = '/etc/default/prometheus' $scrape_configs_dirname = 'exported-configs' $scrape_configs_dir = "${config_dir}/${scrape_configs_dirname}" $global_config = lookup('prometheus::server::config::global', Hash) $rule_files = [] $scrape_configs = [] $remote_read = [] $remote_write = [] $alert_relabel_configs = [] $alertmanagers = [] $full_config = { global => $global_config, rule_files => $rule_files, scrape_configs => $scrape_configs + [ { job_name => 'exported', file_sd_configs => [ { files => [ "${scrape_configs_dirname}/*.yaml", ] }, ] }, ], alerting => { alert_relabel_configs => $alert_relabel_configs, alertmanagers => $alertmanagers, }, remote_read => $remote_read, remote_write => $remote_write, } $lookup_defaults_config = lookup('prometheus::server::defaults_config', Hash) $listen_network = lookup('prometheus::server::listen_network', Optional[String], 'first', undef) $listen_address = lookup('prometheus::server::listen_address', Optional[String], 'first', undef) $actual_listen_address = pick($listen_address, ip_for_network($listen_network)) $listen_port = lookup('prometheus::server::listen_port') $target = "${actual_listen_address}:${listen_port}" $defaults_config = deep_merge( $lookup_defaults_config, { web => { listen_address => $target, }, } ) profile::prometheus::export_scrape_config {'prometheus': target => $target, } package {'prometheus': - ensure => latest, + ensure => present, notify => Service['prometheus'], } service {'prometheus': ensure => 'running', enable => true, require => [ Package['prometheus'], File[$config_file], File[$defaults_file] ], } file {$config_file: ensure => 'present', owner => 'root', group => 'root', mode => '0644', require => Package['prometheus'], notify => Service['prometheus'], content => inline_yaml($full_config), } file {$scrape_configs_dir: ensure => 'directory', owner => 'root', group => 'root', mode => '0644', require => Package['prometheus'], recurse => true, purge => true, } # Uses $defaults_config file {$defaults_file: ensure => 'present', owner => 'root', group => 'root', mode => '0644', content => template('profile/prometheus/server/prometheus.defaults.erb'), require => Package['prometheus'], notify => Service['prometheus'], } Profile::Prometheus::Scrape_config <<| prometheus_server == $trusted['certname'] |>> } diff --git a/site-modules/profile/manifests/prometheus/sql.pp b/site-modules/profile/manifests/prometheus/sql.pp index 9cc99c03..7a9b64c2 100644 --- a/site-modules/profile/manifests/prometheus/sql.pp +++ b/site-modules/profile/manifests/prometheus/sql.pp @@ -1,96 +1,98 @@ # Deployment of prometheus SQL exporter class profile::prometheus::sql { + include profile::prometheus::base + $exporter_name = 'sql' $package_name = "prometheus-${exporter_name}-exporter" $service_name = $package_name $defaults_file = "/etc/default/${package_name}" - $config_file = "/etc/${package_name}.yml" - $config_template = "${config_file}.in" + $config_snippet_dir = "/etc/${package_name}" + $config_file = "/var/run/postgresql/${package_name}.yml" $config_updater = "/usr/bin/update-${package_name}-config" package {$package_name: - ensure => latest, + ensure => installed, } service {$service_name: ensure => 'running', enable => true, require => [ Package[$package_name], - Exec[$config_updater], ] } ::systemd::dropin_file {"${service_name}/restart.conf": ensure => present, unit => "${service_name}.service", filename => 'restart.conf', content => "[Service]\nRestart=always\nRestartSec=5\n", } - file {$config_updater: - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/profile/prometheus/sql/update-prometheus-sql-exporter-config', + ::systemd::dropin_file {"${service_name}/update_config.conf": + ensure => present, + unit => "${service_name}.service", + filename => 'update_config.conf', + content => template('profile/prometheus/sql/systemd/update_config.conf.erb'), + notify => Service[$service_name], } - # needed for the the configuration generation - # optional extra configuration per host - $extra_config = lookup('prometheus::sql::exporter::extra_config', Data, 'first', undef) + $update_deps = ['postgresql-client-common', 'libyaml-perl'] + ensure_packages( + $update_deps, { + ensure => present + }, + ) - file {$config_template: + file {$config_updater: ensure => present, owner => 'root', group => 'root', - mode => '0644', - content => template('profile/prometheus/sql/prometheus-sql-exporter.yml.in.erb'), - notify => Exec[$config_updater], + mode => '0755', + source => 'puppet:///modules/profile/prometheus/sql/update-prometheus-sql-exporter-config', + require => Package[$update_deps], + notify => Service[$service_name], } - $update_deps = ['python3-pkg-resources', 'python3-yaml'] - ensure_packages( - $update_deps, { - ensure => present - }, - ) + file {$config_snippet_dir: + ensure => directory, + owner => 'root', + group => 'root', + mode => '0644', + notify => Service[$service_name], + } + + $config_snippets = lookup('prometheus::sql::config_snippets', Array[String], 'unique') - exec {$config_updater: - refreshonly => true, - creates => $config_file, - require => [ - Package[$update_deps], - File[$config_template], - File[$config_updater], - ], - notify => Service[$service_name], + each($config_snippets) |$snippet| { + file {"${config_snippet_dir}/${snippet}.yml": + ensure => present, + owner => 'root', + group => 'root', + mode => '0644', + source => "puppet:///modules/profile/prometheus/sql/config/${snippet}.yml", + notify => Service[$service_name], + } } $listen_network = lookup('prometheus::sql::listen_network', Optional[String], 'first', undef) - $listen_address = lookup('prometheus::sql::listen_address', Optional[String], 'first', undef) - $actual_listen_address = pick($listen_address, ip_for_network($listen_network)) + $listen_ip = lookup('prometheus::sql::listen_address', Optional[String], 'first', undef) + $actual_listen_ip = pick($listen_ip, ip_for_network($listen_network)) $listen_port = lookup('prometheus::sql::listen_port') - $target = "${actual_listen_address}:${listen_port}" - - $defaults_config = { - web => { - listen_address => $target, - }, - } + $listen_address = "${actual_listen_ip}:${listen_port}" file {$defaults_file: ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/prometheus/sql/prometheus-sql-exporter.defaults.erb'), require => Package[$package_name], notify => Service[$service_name], } profile::prometheus::export_scrape_config {'sql': - target => $target, + target => $listen_address, } } diff --git a/site-modules/profile/manifests/prometheus/statsd.pp b/site-modules/profile/manifests/prometheus/statsd.pp new file mode 100644 index 00000000..90e317a5 --- /dev/null +++ b/site-modules/profile/manifests/prometheus/statsd.pp @@ -0,0 +1,80 @@ +# Prometheus configuration for statsd exporter +class profile::prometheus::statsd { + include profile::prometheus::base + + $defaults_file = '/etc/default/prometheus-statsd-exporter' + $mapping_config_file = '/etc/prometheus/statsd_exporter_mapping.yml' + + package {'prometheus-statsd-exporter': + ensure => present, + notify => Service['prometheus-statsd-exporter'], + } + + service {'prometheus-statsd-exporter': + ensure => 'running', + enable => true, + require => [ + Package['prometheus-statsd-exporter'], + File[$defaults_file], + ] + } + + ::systemd::dropin_file {'prometheus-statsd-exporter/restart.conf': + ensure => present, + unit => 'prometheus-statsd-exporter.service', + filename => 'restart.conf', + content => "[Service]\nRestart=always\nRestartSec=5\n", + } + + $lookup_defaults_config = lookup('prometheus::statsd::defaults_config', Hash) + $listen_network = lookup('prometheus::statsd::listen_network', Optional[String], 'first', undef) + $listen_address = lookup('prometheus::statsd::listen_address', Optional[String], 'first', undef) + $actual_listen_address = pick($listen_address, ip_for_network($listen_network)) + $prometheus_listen_port = lookup('prometheus::statsd::listen_port') + $target = "${actual_listen_address}:${prometheus_listen_port}" + + $listen_tcp = lookup('prometheus::statsd::statsd_listen_tcp') + $listen_udp = lookup('prometheus::statsd::statsd_listen_udp') + + $mapping_config = lookup('prometheus::statsd::mapping', Hash) + + + $defaults_config = deep_merge( + $lookup_defaults_config, + { + web => { + listen_address => $target, + }, + statsd => { + mapping_config => $mapping_config_file, + listen_tcp => $listen_tcp, + listen_udp => $listen_udp, + } + } + ) + + # Uses $defaults_config + file {$defaults_file: + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0644', + content => template('profile/prometheus/statsd/prometheus-statsd-exporter.defaults.erb'), + require => Package['prometheus-statsd-exporter'], + notify => Service['prometheus-statsd-exporter'], + } + + file {$mapping_config_file: + ensure => 'present', + owner => 'root', + group => 'root', + mode => '0644', + content => inline_yaml($mapping_config), + require => Package['prometheus-statsd-exporter'], + notify => Service['prometheus-statsd-exporter'], + } + + profile::prometheus::export_scrape_config {'statsd': + target => $target, + } +} diff --git a/site-modules/profile/manifests/puppet/master.pp b/site-modules/profile/manifests/puppet/master.pp index 74edf56d..5ab11f85 100644 --- a/site-modules/profile/manifests/puppet/master.pp +++ b/site-modules/profile/manifests/puppet/master.pp @@ -1,29 +1,37 @@ # Puppet master profile class profile::puppet::master { $puppetdb = lookup('puppet::master::puppetdb') include ::profile::puppet::base class { '::puppet': server => true, server_common_modules_path => '', server_environments => [], server_external_nodes => '', server_foreman => false, server_passenger => true, server_puppetdb_host => $puppetdb, server_reports => 'store,puppetdb', server_storeconfigs_backend => 'puppetdb', * => $::profile::puppet::base::agent_config, } file { '/usr/local/sbin/swh-puppet-master-deploy': ensure => 'file', owner => 'root', group => 'root', mode => '0755', content => template('profile/puppet/swh-puppet-master-deploy.sh.erb'), } + file { '/usr/local/sbin/swh-puppet-master-clean-certificate': + ensure => 'file', + owner => 'root', + group => 'root', + mode => '0755', + content => template('profile/puppet/swh-puppet-master-clean-certificate.sh.erb'), + } + } diff --git a/site-modules/profile/manifests/rabbitmq.pp b/site-modules/profile/manifests/rabbitmq.pp index a689f1cc..4204495e 100644 --- a/site-modules/profile/manifests/rabbitmq.pp +++ b/site-modules/profile/manifests/rabbitmq.pp @@ -1,34 +1,56 @@ class profile::rabbitmq { include ::profile::munin::plugins::rabbitmq - $rabbitmq_user = lookup('rabbitmq::monitoring::user') - $rabbitmq_password = lookup('rabbitmq::monitoring::password') + $rabbitmq_vhost = '/' + $rabbitmq_enable_guest = lookup('rabbitmq::enable::guest') - package {'rabbitmq-server': - ensure => installed - } + $users = lookup('rabbitmq::server::users') - service {'rabbitmq-server': - ensure => 'running', - enable => true, - require => Package['rabbitmq-server'], + class { 'rabbitmq': + service_manage => true, + port => 5672, + admin_enable => true, + node_ip_address => '0.0.0.0', + config_variables => { + vm_memory_high_watermark => 0.6, + }, + heartbeat => 0, + delete_guest_user => ! $rabbitmq_enable_guest, + } + -> rabbitmq_vhost { $rabbitmq_vhost: + provider => 'rabbitmqctl', } + each ( $users ) | $user | { + $username = $user['name'] + rabbitmq_user { $username: + admin => $user['is_admin'], + password => $user['password'], + tags => $user['tags'], + provider => 'rabbitmqctl', + } + -> rabbitmq_user_permissions { "${username}@${rabbitmq_vhost}": + configure_permission => '.*', + read_permission => '.*', + write_permission => '.*', + provider => 'rabbitmqctl', + } + } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"rabbitmq-server on ${::fqdn}": service_name => 'rabbitmq server', import => ['generic-service'], host_name => $::fqdn, check_command => 'rabbitmq_server', vars => { rabbitmq_port => 15672, rabbitmq_vhost => '/', rabbitmq_node => $::hostname, rabbitmq_user => $rabbitmq_user, rabbitmq_password => $rabbitmq_password, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } diff --git a/site-modules/profile/manifests/rsnapshot/master.pp b/site-modules/profile/manifests/rsnapshot/master.pp new file mode 100644 index 00000000..2c58b950 --- /dev/null +++ b/site-modules/profile/manifests/rsnapshot/master.pp @@ -0,0 +1,45 @@ +# Rsnapshot master host +class profile::rsnapshot::master { + + $backup_exclusion = lookup('dar::backup::exclude', Array, 'unique') + + file {'/etc/rsnapshot.conf': + content => template('profile/swh/rsnapshot.conf.erb'), + owner => 'root', + group => 'root', + mode => '0644', + } + + file {'/srv/rsnapshot': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0644', + } + + file { '/var/log/rsnapshot': + ensure => directory, + owner => $rsnapshot_user, + group => $rsnapshot_group, + mode => '0700', + } + + package { 'rsnapshot': + ensure => 'present', + } + + cron { 'rsnapshot_hourly': + command => '/usr/bin/rsnapshot hourly', + user => 'root', + hour => '*/4', + minute => '0', + } + + cron { 'rsnapshot_daily': + command => '/usr/bin/rsnapshot daily', + user => 'root', + hour => '22', + minute => '33', + } + +} diff --git a/site-modules/profile/manifests/ssh/server.pp b/site-modules/profile/manifests/ssh/server.pp index 0b3b524a..855f6a3d 100644 --- a/site-modules/profile/manifests/ssh/server.pp +++ b/site-modules/profile/manifests/ssh/server.pp @@ -1,49 +1,82 @@ # Configure the SSH server class profile::ssh::server { $sshd_port = lookup('ssh::port') $sshd_permitrootlogin = lookup('ssh::permitrootlogin') class { '::ssh::server': storeconfigs_enabled => false, options => { 'PermitRootLogin' => $sshd_permitrootlogin, 'Port' => $sshd_port, }, } $users = lookup('users', Hash, 'deep') each($users) |$name, $data| { if $name == 'root' { $home = '/root' } else { $home = "/home/${name}" } file { "${home}/.ssh": ensure => directory, owner => $name, group => $name, mode => '0600', require => [ User[$name], File[$home], ], } if $data['authorized_keys'] { each($data['authorized_keys']) |$nick, $key| { ssh_authorized_key { "${name} ${nick}": ensure => 'present', user => $name, key => $key['key'], type => $key['type'], require => File["${home}/.ssh"], } } } } + each($::ssh) |$algo, $data| { + $real_algo = $algo ? { + 'ecdsa' => 'ecdsa-sha2-nistp256', + default => $algo, + } + + $aliases = [ + values($::swh_hostname), + ip_for_network(lookup('internal_network')), + $::public_ipaddresses, + ] + .flatten + .unique + .filter |$x| { !!$x } # filter empty values + .map |$x| { + case $sshd_port { + 22: { + case $x { + /:/: { "[${x}]" } # bracket IPv6 addresses + default: { $x } + } + } + default: { "[${x}]:${sshd_port}" } # specify non-default ssh port + } + } + + @@sshkey {"ssh-${::fqdn}-${real_algo}": + host_aliases => $aliases, + type => $real_algo, + key => $data['key'], + } + } + Sshkey <<| |>> } diff --git a/site-modules/profile/manifests/stats_web.pp b/site-modules/profile/manifests/stats_web.pp new file mode 100644 index 00000000..084174e1 --- /dev/null +++ b/site-modules/profile/manifests/stats_web.pp @@ -0,0 +1,49 @@ +# Deployment of web-facing stats export (from munin) +class profile::stats_web { + $vhost_name = lookup('stats_export::vhost::name') + $vhost_docroot = lookup('stats_export::vhost::docroot') + $vhost_ssl_protocol = lookup('stats_export::vhost::ssl_protocol') + $vhost_ssl_honorcipherorder = lookup('stats_export::vhost::ssl_honorcipherorder') + $vhost_ssl_cipher = lookup('stats_export::vhost::ssl_cipher') + $vhost_hsts_header = lookup('stats_export::vhost::hsts_header') + + file {$vhost_docroot: + ensure => directory, + owner => 'www-data', + group => 'www-data', + mode => '0755', + } + + include ::profile::apache::common + + ::apache::vhost {"${vhost_name}_non-ssl": + servername => $vhost_name, + port => '80', + docroot => $vhost_docroot, + redirect_status => 'permanent', + redirect_dest => "https://${vhost_name}/", + } + + $ssl_cert_name = 'stats_export' + ::profile::letsencrypt::certificate {$ssl_cert_name:} + $cert_paths = ::profile::letsencrypt::certificate_paths($ssl_cert_name) + + ::apache::vhost {"${vhost_name}_ssl": + servername => $vhost_name, + port => '443', + ssl => true, + ssl_protocol => $vhost_ssl_protocol, + ssl_honorcipherorder => $vhost_ssl_honorcipherorder, + ssl_cipher => $vhost_ssl_cipher, + ssl_cert => $cert_paths['cert'], + ssl_chain => $cert_paths['chain'], + ssl_key => $cert_paths['privkey'], + headers => [$vhost_hsts_header], + docroot => $vhost_docroot, + require => [ + Profile::Letsencrypt::Certificate[$ssl_cert_name], + File[$ssl_chain], + File[$ssl_key], + ], + } +} diff --git a/site-modules/profile/manifests/swh/apt_config.pp b/site-modules/profile/manifests/swh/apt_config.pp index 4d1055a1..b882ac45 100644 --- a/site-modules/profile/manifests/swh/apt_config.pp +++ b/site-modules/profile/manifests/swh/apt_config.pp @@ -1,75 +1,72 @@ # Base class for Software Heritage-specific apt configuration class profile::swh::apt_config { $debian_mirror = lookup('swh::apt_config::debian_mirror') $debian_security_mirror = lookup('swh::apt_config::debian_security_mirror') $debian_enable_non_free = lookup('swh::apt_config::enable_non_free') class {'::apt': purge => { 'sources.list' => true, 'sources.list.d' => false, 'preferences' => true, 'preferences.d' => true, }, } package {'apt-transport-https': ensure => 'present', } if lookup('swh::apt_config::unattended_upgrades') { include profile::swh::apt_config::unattended_upgrades } $repos = $debian_enable_non_free ? { true => 'main contrib non-free', default => 'main', } + include profile::swh::apt_config::backports + ::apt::source {'debian': location => $debian_mirror, release => $::lsbdistcodename, repos => $repos, } - ::apt::source {'debian-updates': - location => $debian_mirror, - release => "${::lsbdistcodename}-updates", - repos => $repos, - } - - ::apt::source {'debian-security': - location => $debian_security_mirror, - release => "${::lsbdistcodename}/updates", - repos => $repos, - } - - if $::lsbdistcodename == 'stretch' { - class {'::apt::backports': - pin => 100, + if $::lsbdistcodename != 'sid' { + ::apt::source {'debian-updates': location => $debian_mirror, + release => "${::lsbdistcodename}-updates", + repos => $repos, + } + + ::apt::source {'debian-security': + location => $debian_security_mirror, + release => "${::lsbdistcodename}/updates", repos => $repos, } + } else { - ::apt::source {'backports': + ::apt::source {['debian-updates', 'debian-security']: ensure => absent, } } $swh_repository = lookup('swh::apt_config::swh_repository') $swh_release = $::lsbdistcodename ? { 'buster' => 'sid', default => "${::lsbdistcodename}-swh", } ::apt::source {'softwareheritage': comment => 'Software Heritage specific package repository', location => $swh_repository, release => $swh_release, repos => 'main', allow_unsigned => true, } Package['apt-transport-https'] -> Class['apt::update'] -> Package <| title != 'apt-transport-https' |> } diff --git a/site-modules/profile/manifests/swh/apt_config/backports.pp b/site-modules/profile/manifests/swh/apt_config/backports.pp new file mode 100644 index 00000000..08627009 --- /dev/null +++ b/site-modules/profile/manifests/swh/apt_config/backports.pp @@ -0,0 +1,30 @@ +# Configure apt pinning for packages we always want from backports +class profile::swh::apt_config::backports { + $backported_packages = lookup('swh::apt_config::backported_packages', Hash, 'deep') + $pinned_packages = $backported_packages[$::lsbdistcodename] + if $pinned_packages { + ::apt::pin {'swh-backported-packages': + explanation => 'Pin packages backports', + codename => "${::lsbdistcodename}-backports", + packages => $pinned_packages, + priority => 990, + } + } else { + ::apt::pin {'swh-backported-packages': + ensure => absent, + } + } + + if $::lsbdistcodename != 'sid' { + class {'::apt::backports': + pin => 100, + location => $profile::swh::apt_config::debian_mirror, + repos => $profile::swh::apt_config::repos, + } + } + else { + ::apt::source {['backports', 'debian-updates', 'debian-security']: + ensure => absent, + } + } +} diff --git a/site-modules/profile/manifests/swh/deploy/archiver.pp b/site-modules/profile/manifests/swh/deploy/archiver.pp deleted file mode 100644 index ff2a715a..00000000 --- a/site-modules/profile/manifests/swh/deploy/archiver.pp +++ /dev/null @@ -1,21 +0,0 @@ -# Archiver base configuration - -class profile::swh::deploy::archiver { - include ::profile::swh::deploy::objstorage_cloud - - $config_dir = lookup('swh::deploy::worker::swh_storage_archiver::conf_directory') - - file {$config_dir: - ensure => 'directory', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - } - - $packages = ['python3-swh.archiver'] - - package {$packages: - ensure => 'installed', - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp b/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp deleted file mode 100644 index a1035837..00000000 --- a/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment of the swh.storage.archiver.updater - -class profile::swh::deploy::archiver_content_updater { - include profile::swh::deploy::archiver - - $conf_file = lookup('swh::deploy::archiver_content_updater::conf_file') - $user = lookup('swh::deploy::archiver_content_updater::user') - $group = lookup('swh::deploy::archiver_content_updater::group') - - $content_updater_config = lookup('swh::deploy::archiver_content_updater::config') - - $service_name = 'swh-archiver-content-updater' - $unit_name = "${service_name}.service" - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @content_updater_config.to_yaml %>\n"), - notify => Service[$service_name], - } - - # Template uses variables - # - $user - # - $group - # - ::systemd::unit_file {$unit_name: - ensure => present, - content => template('profile/swh/deploy/archiver/swh-content-updater.service.erb'), - } ~> service {$service_name: - ensure => running, - enable => false, - require => File[$conf_file], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/base_indexer.pp b/site-modules/profile/manifests/swh/deploy/base_indexer.pp new file mode 100644 index 00000000..d9040190 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/base_indexer.pp @@ -0,0 +1,16 @@ +# Base class for indexer related manifests +class profile::swh::deploy::base_indexer { + $config_directory = lookup('swh::deploy::base_indexer::config_directory') + + file {$config_directory: + ensure => 'directory', + owner => 'swhworker', + group => 'swhworker', + mode => '0755', + } + + $packages = ['python3-swh.indexer'] + package {$packages: + ensure => 'present', + } +} diff --git a/site-modules/profile/manifests/swh/deploy/base_lister.pp b/site-modules/profile/manifests/swh/deploy/base_lister.pp deleted file mode 100644 index e4f8c2f9..00000000 --- a/site-modules/profile/manifests/swh/deploy/base_lister.pp +++ /dev/null @@ -1,20 +0,0 @@ -# deployment for the lister package -class profile::swh::deploy::base_lister { - $config_dir = '/etc/softwareheritage/lister' - - # Contains passwords - file {$config_dir: - ensure => 'directory', - owner => 'swhworker', - group => 'swhdev', - mode => '0644', - purge => true, - recurse => true, - } - - $packages = ['python3-swh.lister'] - - package {$packages: - ensure => latest, - } -} diff --git a/site-modules/profile/manifests/swh/deploy/base_loader_git.pp b/site-modules/profile/manifests/swh/deploy/base_loader_git.pp index 51258674..f9595758 100644 --- a/site-modules/profile/manifests/swh/deploy/base_loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/base_loader_git.pp @@ -1,12 +1,12 @@ # Git Loader base configuration class profile::swh::deploy::base_loader_git { include ::profile::swh::deploy::loader $packages = ['python3-swh.loader.git'] package {$packages: - ensure => 'latest', + ensure => 'present', } } diff --git a/site-modules/profile/manifests/swh/deploy/base_vault.pp b/site-modules/profile/manifests/swh/deploy/base_vault.pp index aaeb5ddb..f8c22a78 100644 --- a/site-modules/profile/manifests/swh/deploy/base_vault.pp +++ b/site-modules/profile/manifests/swh/deploy/base_vault.pp @@ -1,16 +1,17 @@ class profile::swh::deploy::base_vault { $conf_directory = lookup('swh::deploy::vault::conf_directory') + $group = lookup('swh::deploy::vault::group') file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } $packages = ['python3-swh.vault'] package {$packages: ensure => 'present', } } diff --git a/site-modules/profile/manifests/swh/deploy/deposit.pp b/site-modules/profile/manifests/swh/deploy/deposit.pp index 6e22abf5..8dc60d7f 100644 --- a/site-modules/profile/manifests/swh/deploy/deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/deposit.pp @@ -1,260 +1,251 @@ # Deployment of the swh.deposit server class profile::swh::deploy::deposit { - $conf_directory = lookup('swh::deploy::deposit::conf_directory') - - $swh_conf_file = lookup('swh::deploy::deposit::swh_conf_file') + $config_directory = lookup('swh::deploy::deposit::config_directory') + $config_file = lookup('swh::deploy::deposit::config_file') $user = lookup('swh::deploy::deposit::user') $group = lookup('swh::deploy::deposit::group') $swh_conf_raw = lookup('swh::deploy::deposit::config') $swh_packages = ['python3-swh.deposit'] $static_dir = '/usr/lib/python3/dist-packages/swh/deposit/static' - # private data file to read from swh.deposit.settings.production - $settings_private_data_file = lookup('swh::deploy::deposit::settings_private_data_file') - $settings_private_data = lookup('swh::deploy::deposit::settings_private_data') - $backend_listen_host = lookup('swh::deploy::deposit::backend::listen::host') $backend_listen_port = lookup('swh::deploy::deposit::backend::listen::port') $backend_listen_address = "${backend_listen_host}:${backend_listen_port}" $backend_workers = lookup('swh::deploy::deposit::backend::workers') $backend_http_keepalive = lookup('swh::deploy::deposit::backend::http_keepalive') $backend_http_timeout = lookup('swh::deploy::deposit::backend::http_timeout') $backend_reload_mercy = lookup('swh::deploy::deposit::backend::reload_mercy') $vhost_name = lookup('swh::deploy::deposit::vhost::name') $vhost_port = lookup('apache::http_port') $vhost_aliases = lookup('swh::deploy::deposit::vhost::aliases') $vhost_docroot = lookup('swh::deploy::deposit::vhost::docroot') - $vhost_basic_auth_file = "${conf_directory}/http_auth" + $vhost_basic_auth_file = "${config_directory}/http_auth" # swh::deploy::deposit::vhost::basic_auth_content in private $vhost_basic_auth_content = lookup('swh::deploy::deposit::vhost::basic_auth_content') $vhost_ssl_port = lookup('apache::https_port') $vhost_ssl_protocol = lookup('swh::deploy::deposit::vhost::ssl_protocol') $vhost_ssl_honorcipherorder = lookup('swh::deploy::deposit::vhost::ssl_honorcipherorder') $vhost_ssl_cipher = lookup('swh::deploy::deposit::vhost::ssl_cipher') $locked_endpoints = lookup('swh::deploy::deposit::locked_endpoints', Array, 'unique') $media_root_directory = lookup('swh::deploy::deposit::media_root_directory') include ::gunicorn - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - notify => Service['gunicorn-swh-deposit'], + # Install the necessary deps + ::profile::swh::deploy::install_web_deps { 'swh-deposit': + services => ['gunicorn-swh-deposit'], + backport_list => 'swh::deploy::deposit::backported_packages', + # FIXME: should be fixed in the deposit package + swh_packages => ['python3-django', 'python3-djangorestframework', 'python3-swh.deposit'], } - file {$conf_directory: + file {$config_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } # swh's configuration part (upload size, etc...) - file {$swh_conf_file: + file {$config_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @swh_conf_raw.to_yaml %>\n"), notify => Service['gunicorn-swh-deposit'], } file {$media_root_directory: ensure => directory, owner => $user, group => $group, mode => '2750', } - # swh's private configuration part (db, secret key, media_root) - file {$settings_private_data_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @settings_private_data.to_yaml %>\n"), - notify => Service['gunicorn-swh-deposit'], - } - ::gunicorn::instance {'swh-deposit': ensure => enabled, user => $user, group => $group, executable => 'swh.deposit.wsgi', + environment => { + 'SWH_CONFIG_FILENAME' => $config_file, + 'DJANGO_SETTINGS_MODULE' => 'swh.deposit.settings.production', + }, settings => { bind => $backend_listen_address, workers => $backend_workers, worker_class => 'sync', timeout => $backend_http_timeout, graceful_timeout => $backend_reload_mercy, keepalive => $backend_http_keepalive, } } $endpoint_directories = $locked_endpoints.map |$endpoint| { { path => "^${endpoint}", provider => 'locationmatch', auth_type => 'Basic', auth_name => 'Software Heritage Deposit', auth_user_file => $vhost_basic_auth_file, auth_require => 'valid-user', } } include ::profile::apache::common include ::apache::mod::proxy include ::apache::mod::headers ::apache::vhost {"${vhost_name}_non-ssl": servername => $vhost_name, serveraliases => $vhost_aliases, port => $vhost_port, docroot => $vhost_docroot, proxy_pass => [ { path => '/static', url => '!', }, { path => '/robots.txt', url => '!', }, { path => '/favicon.ico', url => '!', }, { path => '/', url => "http://${backend_listen_address}/", }, ], directories => [ { path => '/1', provider => 'location', allow => 'from all', satisfy => 'Any', headers => ['add Access-Control-Allow-Origin "*"'], }, { path => $static_dir, options => ['-Indexes'], }, ] + $endpoint_directories, aliases => [ { alias => '/static', path => $static_dir, }, { alias => '/robots.txt', path => "${static_dir}/robots.txt", }, ], require => [ File[$vhost_basic_auth_file], ] } $ssl_cert_name = 'star_softwareheritage_org' include ::profile::hitch realize(::Profile::Hitch::Ssl_cert[$ssl_cert_name]) include ::profile::varnish ::profile::varnish::vhost {$vhost_name: aliases => $vhost_aliases, hsts_max_age => lookup('strict_transport_security::max_age'), } file {$vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', content => $vhost_basic_auth_content, } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"swh-deposit api (localhost on ${::fqdn})": service_name => 'swh-deposit api (localhost)', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', command_endpoint => $::fqdn, vars => { http_address => '127.0.0.1', http_port => $backend_listen_port, http_uri => '/', http_string => 'The Software Heritage Deposit', }, target => $icinga_checks_file, tag => 'icinga2::exported', } if $backend_listen_host != '127.0.0.1' { @@::icinga2::object::service {"swh-deposit api (remote on ${::fqdn})": service_name => 'swh-deposit api (remote)', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_port => $backend_listen_port, http_uri => '/', http_string => 'The Software Heritage Deposit', }, target => $icinga_checks_file, tag => 'icinga2::exported', } } @@::icinga2::object::service {"swh-deposit http redirect on ${::fqdn}": service_name => 'swh deposit http redirect', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_port, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"swh-deposit https on ${::fqdn}": service_name => 'swh deposit', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"swh-deposit https certificate ${::fqdn}": service_name => 'swh deposit https certificate', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, http_sni => true, http_certificate => 60, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } diff --git a/site-modules/profile/manifests/swh/deploy/indexer.pp b/site-modules/profile/manifests/swh/deploy/indexer.pp index fb59160c..7735ebef 100644 --- a/site-modules/profile/manifests/swh/deploy/indexer.pp +++ b/site-modules/profile/manifests/swh/deploy/indexer.pp @@ -1,31 +1,4 @@ -# Base class for the indexer manifests +# Base class for the indexer worker manifests class profile::swh::deploy::indexer { - - include ::profile::swh::deploy::objstorage_cloud - - $config_directory = '/etc/softwareheritage/indexer' - $config_file = "${config_directory}/base.yml" - $config = lookup('swh::deploy::worker::swh_indexer::base::config') - - $packages = ['python3-swh.indexer'] - - file {$config_directory: - ensure => 'directory', - owner => 'swhworker', - group => 'swhworker', - mode => '0755', - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } - - package {$packages: - ensure => 'latest', - } + include ::profile::swh::deploy::base_indexer } diff --git a/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp b/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp new file mode 100644 index 00000000..bdc04100 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp @@ -0,0 +1,37 @@ +# Deployment of the swh.indexer.journal_client +class profile::swh::deploy::indexer_journal_client { + include ::profile::swh::deploy::base_indexer + include ::profile::swh::deploy::journal + + $config_file = lookup('swh::deploy::indexer_journal_client::config_file') + $config_directory = lookup('swh::deploy::base_indexer::config_directory') + $config_path = "${config_directory}/${config_file}" + $config = lookup('swh::deploy::indexer_journal_client::config') + + $user = lookup('swh::deploy::indexer_journal_client::user') + $group = lookup('swh::deploy::indexer_journal_client::group') + + $service_name = 'swh-indexer-journal-client' + $unit_name = "${service_name}.service" + + file {$config_path: + ensure => present, + owner => 'root', + group => 'swhdev', + mode => '0644', + content => inline_template("<%= @config.to_yaml %>\n"), + notify => Service[$service_name], + } + + # Template uses variables + # - $user + # - $group + # + ::systemd::unit_file {$unit_name: + ensure => present, + content => template("profile/swh/deploy/journal/${unit_name}.erb"), + } ~> service {$service_name: + ensure => running, + enable => true, + } +} diff --git a/site-modules/profile/manifests/swh/deploy/indexer_storage.pp b/site-modules/profile/manifests/swh/deploy/indexer_storage.pp index 50c38b8d..c718ec9b 100644 --- a/site-modules/profile/manifests/swh/deploy/indexer_storage.pp +++ b/site-modules/profile/manifests/swh/deploy/indexer_storage.pp @@ -1,14 +1,14 @@ # Deployment of the swh.indexer.storage.api.server class profile::swh::deploy::indexer_storage { include ::profile::swh::deploy::base_storage package {'python3-swh.indexer.storage': - ensure => 'latest', + ensure => 'present', } ~> ::profile::swh::deploy::rpc_server {'indexer-storage': config_key => 'indexer::storage', - executable => 'swh.indexer.storage.api.server:run_from_webserver', + executable => 'swh.indexer.storage.api.wsgi', worker => 'sync', http_check_string => 'SWH Indexer Storage API server', } } diff --git a/site-modules/profile/manifests/swh/deploy/install_web_deps.pp b/site-modules/profile/manifests/swh/deploy/install_web_deps.pp new file mode 100644 index 00000000..b53a6e7b --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/install_web_deps.pp @@ -0,0 +1,29 @@ +# Install web dependencies (eventually backporting some packages) +define profile::swh::deploy::install_web_deps ( + Array $services = [], + String $pin_name = $name, + String $backport_list = 'swh::deploy::webapp::backported_packages', + Array $swh_packages = ['python3-swh.web'], +) { + $task_backported_packages = lookup($backport_list) + $pinned_packages = $task_backported_packages[$::lsbdistcodename] + if $pinned_packages { + ::apt::pin {$pin_name: + explanation => "Pin ${pin_name} dependencies to backports", + codename => "${::lsbdistcodename}-backports", + packages => $pinned_packages, + priority => 990, + } + -> package {$swh_packages: + ensure => latest, + require => Apt::Source['softwareheritage'], + notify => Service[$services], + } + } else { + package {$swh_packages: + ensure => latest, + require => Apt::Source['softwareheritage'], + notify => Service[$services], + } + } +} diff --git a/site-modules/profile/manifests/swh/deploy/journal.pp b/site-modules/profile/manifests/swh/deploy/journal.pp index 338f947e..03c1679b 100644 --- a/site-modules/profile/manifests/swh/deploy/journal.pp +++ b/site-modules/profile/manifests/swh/deploy/journal.pp @@ -1,18 +1,19 @@ # Base Journal configuration class profile::swh::deploy::journal { $conf_directory = lookup('swh::deploy::journal::conf_directory') file {$conf_directory: ensure => 'directory', owner => 'swhworker', group => 'swhworker', mode => '0644', } - $package_name = 'python3-swh.journal' + $swh_packages = ['python3-swh.journal'] - package {$package_name: - ensure => latest, + package {$swh_packages: + ensure => present, + require => Apt::Source['softwareheritage'], } } diff --git a/site-modules/profile/manifests/swh/deploy/journal_publisher.pp b/site-modules/profile/manifests/swh/deploy/journal_publisher.pp deleted file mode 100644 index 8e61e211..00000000 --- a/site-modules/profile/manifests/swh/deploy/journal_publisher.pp +++ /dev/null @@ -1,35 +0,0 @@ -# Deployment of the swh.journal.publisher - -class profile::swh::deploy::journal_publisher { - include ::profile::swh::deploy::journal - - $conf_file = lookup('swh::deploy::journal_publisher::conf_file') - $user = lookup('swh::deploy::journal_publisher::user') - $group = lookup('swh::deploy::journal_publisher::group') - - $publisher_config = lookup('swh::deploy::journal_publisher::config') - - $service_name = 'swh-journal-publisher' - $unit_name = "${service_name}.service" - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @publisher_config.to_yaml %>\n"), - notify => Service[$service_name], - } - - # Template uses variables - # - $user - # - $group - # - ::systemd::unit_file {$unit_name: - ensure => present, - content => template('profile/swh/deploy/journal/swh-journal-publisher.service.erb'), - } ~> service {$service_name: - ensure => running, - enable => false, - } -} diff --git a/site-modules/profile/manifests/swh/deploy/journal_simple_checker_producer.pp b/site-modules/profile/manifests/swh/deploy/journal_simple_checker_producer.pp index 77966383..06b6b363 100644 --- a/site-modules/profile/manifests/swh/deploy/journal_simple_checker_producer.pp +++ b/site-modules/profile/manifests/swh/deploy/journal_simple_checker_producer.pp @@ -1,36 +1,36 @@ # Deployment of the swh.journal.checker class profile::swh::deploy::journal_simple_checker_producer { include ::profile::swh::deploy::journal $conf_file = lookup('swh::deploy::journal_simple_checker_producer::conf_file') $user = lookup('swh::deploy::journal_simple_checker_producer::user') $group = lookup('swh::deploy::journal_simple_checker_producer::group') $checker_config = lookup( 'swh::deploy::journal_simple_checker_producer::config') $service_name = 'swh-journal-simple-checker-producer' $unit_name = "${service_name}.service" file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @checker_config.to_yaml %>\n"), notify => Service[$service_name], } # Template uses variables # - $user # - $group # - ::systemd::unit {$unit_name: + ::systemd::unit_file {$unit_name: ensure => present, content => template('profile/swh/deploy/journal/swh-journal-simple-checker-producer.service.erb'), } ~> service {$service_name: ensure => running, enable => false, } } diff --git a/site-modules/profile/manifests/swh/deploy/objstorage.pp b/site-modules/profile/manifests/swh/deploy/objstorage.pp index e5daa43b..faa8885b 100644 --- a/site-modules/profile/manifests/swh/deploy/objstorage.pp +++ b/site-modules/profile/manifests/swh/deploy/objstorage.pp @@ -1,25 +1,39 @@ # Deployment of the swh.objstorage.api server class profile::swh::deploy::objstorage { $conf_directory = lookup('swh::deploy::objstorage::conf_directory') + $user = lookup('swh::deploy::objstorage::user') $group = lookup('swh::deploy::objstorage::group') $swh_packages = ['python3-swh.objstorage'] package {$swh_packages: - ensure => latest, + ensure => present, require => Apt::Source['softwareheritage'], } Package[$swh_packages] ~> Service['gunicorn-swh-objstorage'] file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0750', } ::profile::swh::deploy::rpc_server {'objstorage': - executable => 'swh.objstorage.api.server:app', + executable => 'swh.objstorage.api.wsgi', worker => 'async', } + + # special configuration for pathslicing + $objstorage_cfg = lookup('swh::deploy::objstorage::config', Hash)['objstorage'] + + if $objstorage_cfg['cls'] == 'pathslicing' { + $obj_directory = $objstorage_cfg['args']['root'] + file {$obj_directory: + ensure => directory, + owner => $user, + group => $group, + mode => '0750', + } + } } diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_archive_notifier_checker.pp b/site-modules/profile/manifests/swh/deploy/objstorage_archive_notifier_checker.pp deleted file mode 100644 index 753ef679..00000000 --- a/site-modules/profile/manifests/swh/deploy/objstorage_archive_notifier_checker.pp +++ /dev/null @@ -1,42 +0,0 @@ -# Deployment of the swh.objstorage.checker.ArchiveNotifierContentChecker - -class profile::swh::deploy::objstorage_archive_notifier_checker { - $conf_directory = lookup('swh::deploy::objstorage_archive_notifier_checker::conf_directory') - $conf_file = lookup('swh::deploy::objstorage_archive_notifier_checker::conf_file') - $user = lookup('swh::deploy::objstorage_archive_notifier_checker::user') - $group = lookup('swh::deploy::objstorage_archive_notifier_checker::group') - - # configuration file - $archive_notifier_config = lookup('swh::deploy::objstorage_archive_notifier_checker::config') - - $swh_packages = ['python3-swh.objstorage.checker'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @archive_notifier_config.to_yaml %>\n"), - } - - ::systemd::unit_file {'objstorage_archive_notifier_checker.service': - ensure => present, - content => template('profile/swh/deploy/storage/objstorage_archive_notifier_checker.service.erb'), - require => [ - File[$conf_file], - Package[$swh_packages], - ] - } -} diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_cloud.pp b/site-modules/profile/manifests/swh/deploy/objstorage_cloud.pp deleted file mode 100644 index c399f1f4..00000000 --- a/site-modules/profile/manifests/swh/deploy/objstorage_cloud.pp +++ /dev/null @@ -1,9 +0,0 @@ -# Deployment of the cloud objstorage - -class profile::swh::deploy::objstorage_cloud { - $objstorage_packages = ['python3-swh.objstorage.cloud'] - - package {$objstorage_packages: - ensure => installed, - } -} diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_log_checker.pp b/site-modules/profile/manifests/swh/deploy/objstorage_log_checker.pp deleted file mode 100644 index cd9345e0..00000000 --- a/site-modules/profile/manifests/swh/deploy/objstorage_log_checker.pp +++ /dev/null @@ -1,43 +0,0 @@ -# Deployment of the swh.objstorage.checker.LogContentChecker - -class profile::swh::deploy::objstorage_log_checker { - $conf_directory = lookup('swh::deploy::objstorage_log_checker::conf_directory') - $conf_file = lookup('swh::deploy::objstorage_log_checker::conf_file') - $user = lookup('swh::deploy::objstorage_log_checker::user') - $group = lookup('swh::deploy::objstorage_log_checker::group') - - # configuration file - $log_checker_config = lookup('swh::deploy::objstorage_log_checker::config') - - $swh_packages = ['python3-swh.objstorage.checker'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @log_checker_config.to_yaml %>\n"), - } - - ::systemd::unit_file {'objstorage_log_checker.service': - ensure => present, - content => template('profile/swh/deploy/storage/objstorage_log_checker.service.erb'), - require => [ - File[$conf_file], - Package[$swh_packages], - ] - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_repair_checker.pp b/site-modules/profile/manifests/swh/deploy/objstorage_repair_checker.pp deleted file mode 100644 index ef8e7537..00000000 --- a/site-modules/profile/manifests/swh/deploy/objstorage_repair_checker.pp +++ /dev/null @@ -1,42 +0,0 @@ -# Deployment of the swh.objstorage.checker.RepairContentChecker - -class profile::swh::deploy::objstorage_repair_checker { - $conf_directory = lookup('swh::deploy::objstorage_repair_checker::conf_directory') - $conf_file = lookup('swh::deploy::objstorage_repair_checker::conf_file') - $user = lookup('swh::deploy::objstorage_repair_checker::user') - $group = lookup('swh::deploy::objstorage_repair_checker::group') - - $repair_checker_config = lookup('swh::deploy::objstorage_repair_checker::config') - - $swh_packages = ['python3-swh.objstorage.checker'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @repair_checker_config.to_yaml %>\n"), - } - - ::systemd::unit_file {'objstorage_repair_checker.service': - ensure => present, - content => template('profile/swh/deploy/storage/objstorage_repair_checker.service.erb'), - require => [ - File[$conf_file], - Package[$swh_packages], - ] - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/rpc_server.pp b/site-modules/profile/manifests/swh/deploy/rpc_server.pp index 3d158168..f3ae5a7d 100644 --- a/site-modules/profile/manifests/swh/deploy/rpc_server.pp +++ b/site-modules/profile/manifests/swh/deploy/rpc_server.pp @@ -1,141 +1,151 @@ # Deploy an instance of a rpc service define profile::swh::deploy::rpc_server ( String $executable, String $instance_name = $name, String $config_key = $name, String $http_check_string = "SWH ${capitalize($name)} API server", Enum['sync', 'async'] $worker = 'sync', ) { include ::profile::nginx $conf_file = lookup("swh::deploy::${config_key}::conf_file") $user = lookup("swh::deploy::${config_key}::user") $group = lookup("swh::deploy::${config_key}::group") $service_name = "swh-${instance_name}" $gunicorn_service_name = "gunicorn-${service_name}" $gunicorn_unix_socket = "unix:/run/gunicorn/${service_name}/gunicorn.sock" $backend_listen_host = lookup("swh::deploy::${config_key}::backend::listen::host") $backend_listen_port = lookup("swh::deploy::${config_key}::backend::listen::port") $nginx_server_names = lookup("swh::deploy::${config_key}::backend::server_names") $backend_workers = lookup("swh::deploy::${config_key}::backend::workers") $backend_http_keepalive = lookup("swh::deploy::${config_key}::backend::http_keepalive") $backend_http_timeout = lookup("swh::deploy::${config_key}::backend::http_timeout") $backend_reload_mercy = lookup("swh::deploy::${config_key}::backend::reload_mercy") $backend_max_requests = lookup("swh::deploy::${config_key}::backend::max_requests") $backend_max_requests_jitter = lookup("swh::deploy::${config_key}::backend::max_requests_jitter") $instance_config = lookup("swh::deploy::${config_key}::config") + $gunicorn_statsd_host = lookup('gunicorn::statsd::host') + include ::gunicorn case $worker { 'sync': { $gunicorn_worker_class = 'sync' $nginx_proxy_buffering = 'on' } 'async': { $gunicorn_worker_class = 'aiohttp.worker.GunicornWebWorker' $nginx_proxy_buffering = 'off' } default: { fail("Worker class ${worker} is unsupported by this module.") } } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @instance_config.to_yaml %>\n"), notify => Service["gunicorn-swh-${instance_name}"], } ::nginx::resource::upstream {"swh-${instance_name}-gunicorn": upstream_fail_timeout => 0, members => [ $gunicorn_unix_socket, ], } # Default server on listen_port: return 444 for wrong domain name ::nginx::resource::server {"nginx-swh-${instance_name}-default": ensure => present, listen_ip => $backend_listen_host, listen_port => $backend_listen_port, listen_options => 'default_server', maintenance => true, maintenance_value => 'return 444', } # actual server ::nginx::resource::server {"nginx-swh-${instance_name}": ensure => present, listen_ip => $backend_listen_host, listen_port => $backend_listen_port, listen_options => 'deferred', server_name => $nginx_server_names, client_max_body_size => '4G', proxy => "http://swh-${instance_name}-gunicorn", proxy_buffering => $nginx_proxy_buffering, proxy_read_timeout => "${backend_http_timeout}s", format_log => "combined if=\$error_status", } ::gunicorn::instance {$service_name: - ensure => enabled, - user => $user, - group => $group, - executable => $executable, - settings => { + ensure => enabled, + user => $user, + group => $group, + executable => $executable, + environment => { + 'SWH_CONFIG_FILENAME' => $conf_file, + 'SWH_LOG_TARGET' => 'journal', + }, + settings => { bind => $gunicorn_unix_socket, workers => $backend_workers, worker_class => $gunicorn_worker_class, timeout => $backend_http_timeout, graceful_timeout => $backend_reload_mercy, keepalive => $backend_http_keepalive, max_requests => $backend_max_requests, max_requests_jitter => $backend_max_requests_jitter, + statsd_host => $gunicorn_statsd_host, + statsd_prefix => $service_name, }, } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"swh-${instance_name} api (localhost on ${::fqdn})": service_name => "swh-${instance_name} api (localhost)", import => ['generic-service'], host_name => $::fqdn, check_command => 'http', command_endpoint => $::fqdn, vars => { http_address => '127.0.0.1', http_vhost => '127.0.0.1', http_port => $backend_listen_port, http_uri => '/', + http_header => ['Accept: application/json'], http_string => $http_check_string, }, target => $icinga_checks_file, tag => 'icinga2::exported', } if $backend_listen_host != '127.0.0.1' { @@::icinga2::object::service {"swh-${instance_name} api (remote on ${::fqdn})": service_name => "swh-${instance_name} api (remote)", import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_vhost => $::swh_hostname['internal_fqdn'], http_port => $backend_listen_port, http_uri => '/', + http_header => ['Accept: application/json'], http_string => $http_check_string, }, target => $icinga_checks_file, tag => 'icinga2::exported', } } } diff --git a/site-modules/profile/manifests/swh/deploy/scheduler.pp b/site-modules/profile/manifests/swh/deploy/scheduler.pp index 997eebf3..e2f35074 100644 --- a/site-modules/profile/manifests/swh/deploy/scheduler.pp +++ b/site-modules/profile/manifests/swh/deploy/scheduler.pp @@ -1,161 +1,161 @@ # Deployment of swh-scheduler related utilities class profile::swh::deploy::scheduler { - $conf_file = lookup('swh::deploy::scheduler::conf_file') + $config_file = lookup('swh::deploy::scheduler::conf_file') $user = lookup('swh::deploy::scheduler::user') $group = lookup('swh::deploy::scheduler::group') - $database = lookup('swh::deploy::scheduler::database') + $config = lookup('swh::deploy::scheduler::config') + + $listener_log_level = lookup('swh::deploy::scheduler::listener::log_level') + $runner_log_level = lookup('swh::deploy::scheduler::runner::log_level') $task_broker = lookup('swh::deploy::scheduler::task_broker') $task_packages = lookup('swh::deploy::scheduler::task_packages') $task_modules = lookup('swh::deploy::scheduler::task_modules') $task_backported_packages = lookup('swh::deploy::scheduler::backported_packages') $listener_service_name = 'swh-scheduler-listener' $listener_unit_name = "${listener_service_name}.service" $listener_unit_template = "profile/swh/deploy/scheduler/${listener_service_name}.service.erb" $runner_service_name = 'swh-scheduler-runner' $runner_unit_name = "${runner_service_name}.service" $runner_unit_template = "profile/swh/deploy/scheduler/${runner_service_name}.service.erb" $worker_conf_file = '/etc/softwareheritage/worker.ini' $packages = ['python3-swh.scheduler'] $services = [$listener_service_name, $runner_service_name] $pinned_packages = $task_backported_packages[$::lsbdistcodename] if $pinned_packages { ::apt::pin {'swh-scheduler': explanation => 'Pin swh.scheduler dependencies to backports', codename => "${::lsbdistcodename}-backports", packages => $pinned_packages, priority => 990, } -> package {$task_packages: ensure => installed, notify => Service[$runner_service_name], } } else { package {$task_packages: ensure => installed, notify => Service[$runner_service_name], } } package {$packages: ensure => installed, notify => Service[$services], } - # Template uses variables - # - $database - # - file {$conf_file: + file {$config_file: ensure => present, owner => 'root', group => $group, mode => '0640', - content => template('profile/swh/deploy/scheduler/scheduler.ini.erb'), + content => inline_template("<%= @config.to_yaml %>\n"), notify => Service[$services], } # Template uses variables # - $task_broker # - $task_modules # file {$worker_conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => template('profile/swh/deploy/scheduler/worker.ini.erb'), notify => Service[$runner_service_name], } # Template uses variables # - $user # - $group # ::systemd::unit_file {$listener_unit_name: ensure => present, content => template($listener_unit_template), notify => Service[$listener_service_name], } # Template uses variables # - $user # - $group # ::systemd::unit_file {$runner_unit_name: ensure => present, content => template($runner_unit_template), notify => Service[$runner_service_name], } service {$runner_service_name: ensure => running, enable => true, require => [ Package[$packages], Package[$task_packages], - File[$conf_file], + File[$config_file], File[$worker_conf_file], Systemd::Unit_File[$runner_unit_name], ], } service {$listener_service_name: ensure => running, enable => true, require => [ Package[$packages], - File[$conf_file], + File[$config_file], File[$worker_conf_file], Systemd::Unit_File[$listener_unit_name], ], } # scheduler rpc server ::profile::swh::deploy::rpc_server {'scheduler': config_key => 'scheduler::remote', - executable => 'swh.scheduler.api.server:run_from_webserver', + executable => 'swh.scheduler.api.wsgi', } # task archival cron $archive_config_dir = lookup('swh::deploy::scheduler::archive::conf_dir') $archive_config_file = lookup('swh::deploy::scheduler::archive::conf_file') $archive_config = lookup('swh::deploy::scheduler::archive::config') file {$archive_config_dir: ensure => 'directory', owner => $user, group => 'swhscheduler', mode => '0644', } file {$archive_config_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @archive_config.to_yaml %>\n"), require => [ File[$archive_config_dir], ], } cron {'archive_completed_oneshot_and_disabled_recurring_tasks': ensure => present, user => $user, - command => "/usr/bin/python3 -m swh.scheduler.cli task archive", + command => "/usr/bin/swh scheduler --config-file ${archive_config_file} task archive", hour => '*', minute => fqdn_rand(60, 'archival_tasks_minute'), require => [ Package[$packages], File[$archive_config_file], ], } } diff --git a/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp b/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp index 0aae812f..fd63ac88 100644 --- a/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp +++ b/site-modules/profile/manifests/swh/deploy/scheduler_updater.pp @@ -1,32 +1,32 @@ # Deployment of swh-scheduler-updater related utilities class profile::swh::deploy::scheduler_updater { # Package and backend configuration $scheduler_updater_packages = ['python3-swh.scheduler.updater'] package {$scheduler_updater_packages: - ensure => latest, + ensure => present, } $backend_conf_dir = lookup('swh::deploy::scheduler::updater::backend::conf_dir') $backend_conf_file = lookup('swh::deploy::scheduler::updater::backend::conf_file') $backend_user = lookup('swh::deploy::scheduler::updater::backend::user') $backend_group = lookup('swh::deploy::scheduler::updater::backend::group') $backend_config = lookup('swh::deploy::scheduler::updater::backend::config') # file {$backend_conf_dir: # ensure => directory, # owner => 'root', # group => $backend_group, # mode => '0755', # } file {$backend_conf_file: ensure => present, owner => 'root', group => $backend_group, mode => '0640', content => inline_template("<%= @backend_config.to_yaml %>\n"), } } diff --git a/site-modules/profile/manifests/swh/deploy/storage.pp b/site-modules/profile/manifests/swh/deploy/storage.pp index 7e7f5ff8..c2d4504c 100644 --- a/site-modules/profile/manifests/swh/deploy/storage.pp +++ b/site-modules/profile/manifests/swh/deploy/storage.pp @@ -1,12 +1,21 @@ # Deployment of the swh.storage.api server class profile::swh::deploy::storage { include ::profile::swh::deploy::base_storage package {'python3-swh.storage': - ensure => 'latest', + ensure => 'present', } ~> ::profile::swh::deploy::rpc_server {'storage': - executable => 'swh.storage.api.server:run_from_webserver', - worker => 'sync', + executable => 'swh.storage.api.wsgi', + worker => 'sync', + http_check_string => 'Software Heritage storage server', + } + + $storage_config = lookup('swh::deploy::storage::config')['storage'] + + if ($storage_config['cls'] == 'local' + and $storage_config['args']['journal_writer'] + and $storage_config['args']['journal_writer']['cls'] == 'kafka') { + include ::profile::swh::deploy::journal } } diff --git a/site-modules/profile/manifests/swh/deploy/storage_listener.pp b/site-modules/profile/manifests/swh/deploy/storage_listener.pp deleted file mode 100644 index dc5e0250..00000000 --- a/site-modules/profile/manifests/swh/deploy/storage_listener.pp +++ /dev/null @@ -1,57 +0,0 @@ -# Deployment of the swh.storage.listener - -class profile::swh::deploy::storage_listener { - $conf_directory = lookup('swh::deploy::storage_listener::conf_directory') - $conf_file = lookup('swh::deploy::storage_listener::conf_file') - $user = lookup('swh::deploy::storage_listener::user') - $group = lookup('swh::deploy::storage_listener::group') - $database = lookup('swh::deploy::storage_listener::database') - $topic_prefix = lookup('swh::deploy::storage_listener::topic_prefix') - $kafka_brokers = lookup('swh::deploy::storage_listener::kafka_brokers', Array, 'unique') - $poll_timeout = lookup('swh::deploy::storage_listener::poll_timeout') - - $service_name = 'swh-storage-listener' - $unit_name = "${service_name}.service" - - package {'python3-swh.storage.listener': - ensure => latest, - notify => Service[$service_name], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - # Template uses variables - # - $database - # - $kafka_brokers - # - $topic_prefix - # - $poll_timeout - # - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - require => File[$conf_directory], - content => template('profile/swh/deploy/storage_listener/listener.ini.erb'), - notify => Service[$service_name], - } - - # Template uses variables - # - $user - # - $group - # - ::systemd::unit_file {$unit_name: - ensure => present, - content => template('profile/swh/deploy/storage_listener/swh-storage-listener.service.erb'), - require => Package['python3-swh.storage.listener'], - } ~> service {$service_name: - ensure => running, - enable => true, - require => File[$conf_file], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/vault.pp b/site-modules/profile/manifests/swh/deploy/vault.pp index ae5be6fd..d776182f 100644 --- a/site-modules/profile/manifests/swh/deploy/vault.pp +++ b/site-modules/profile/manifests/swh/deploy/vault.pp @@ -1,11 +1,20 @@ # Deployment of the swh.vault.api server class profile::swh::deploy::vault { include ::profile::swh::deploy::base_vault Package['python3-swh.vault'] ~> Service['gunicorn-swh-vault'] + $user = lookup('swh::deploy::vault::user') + $cache_directory = lookup('swh::deploy::vault::cache') + file {$cache_directory: + ensure => directory, + owner => $user, + group => 'swhdev', + mode => '0755', + } + ::profile::swh::deploy::rpc_server {'vault': - executable => 'swh.vault.api.server:make_app_from_configfile()', + executable => 'swh.vault.api.wsgi:app', worker => 'async', } } diff --git a/site-modules/profile/manifests/swh/deploy/webapp.pp b/site-modules/profile/manifests/swh/deploy/webapp.pp index 4a770ea8..8b942662 100644 --- a/site-modules/profile/manifests/swh/deploy/webapp.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp.pp @@ -1,238 +1,253 @@ # WebApp deployment class profile::swh::deploy::webapp { $conf_directory = lookup('swh::deploy::webapp::conf_directory') $conf_file = lookup('swh::deploy::webapp::conf_file') $user = lookup('swh::deploy::webapp::user') $group = lookup('swh::deploy::webapp::group') $webapp_config = lookup('swh::deploy::webapp::config') $conf_log_dir = lookup('swh::deploy::webapp::conf::log_dir') $backend_listen_host = lookup('swh::deploy::webapp::backend::listen::host') $backend_listen_port = lookup('swh::deploy::webapp::backend::listen::port') $backend_listen_address = "${backend_listen_host}:${backend_listen_port}" $backend_workers = lookup('swh::deploy::webapp::backend::workers') $backend_http_keepalive = lookup('swh::deploy::webapp::backend::http_keepalive') $backend_http_timeout = lookup('swh::deploy::webapp::backend::http_timeout') $backend_reload_mercy = lookup('swh::deploy::webapp::backend::reload_mercy') - $swh_packages = ['python3-swh.web'] $static_dir = '/usr/lib/python3/dist-packages/swh/web/static' $varnish_http_port = lookup('varnish::http_port') $vhost_name = lookup('swh::deploy::webapp::vhost::name') $vhost_port = lookup('apache::http_port') $vhost_aliases = lookup('swh::deploy::webapp::vhost::aliases') $vhost_docroot = lookup('swh::deploy::webapp::vhost::docroot') $vhost_basic_auth_file = "${conf_directory}/http_auth" $vhost_basic_auth_content = lookup('swh::deploy::webapp::vhost::basic_auth_content', String, 'first', '') $vhost_ssl_port = lookup('apache::https_port') $vhost_ssl_protocol = lookup('swh::deploy::webapp::vhost::ssl_protocol') $vhost_ssl_honorcipherorder = lookup('swh::deploy::webapp::vhost::ssl_honorcipherorder') $vhost_ssl_cipher = lookup('swh::deploy::webapp::vhost::ssl_cipher') $production_db_dir = lookup('swh::deploy::webapp::production_db_dir') + $production_db_file = lookup('swh::deploy::webapp::production_db') $locked_endpoints = lookup('swh::deploy::webapp::locked_endpoints', Array, 'unique') $endpoint_directories = $locked_endpoints.map |$endpoint| { { path => "^${endpoint}", provider => 'locationmatch', auth_type => 'Basic', auth_name => 'Software Heritage development', auth_user_file => $vhost_basic_auth_file, auth_require => 'valid-user', } } - include ::gunicorn - - $services = ['gunicorn-swh-webapp', 'gunicorn-swh-storage'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - notify => Service[$services], + # Install the necessary deps + ::profile::swh::deploy::install_web_deps { 'swh-web': + services => ['gunicorn-swh-webapp'], + backport_list => 'swh::deploy::webapp::backported_packages', + swh_packages => ['python3-swh.web'], } + include ::gunicorn + file {$conf_directory: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_log_dir: ensure => directory, owner => 'root', group => $group, mode => '0770', } + file {"$conf_log_dir/swh-web.log": + ensure => present, + owner => $user, + group => $group, + mode => '0770', + } + file {$vhost_docroot: ensure => directory, owner => 'root', group => $group, mode => '0755', } file {$conf_file: ensure => present, owner => 'root', group => $group, mode => '0640', content => inline_template("<%= @webapp_config.to_yaml %>\n"), notify => Service['gunicorn-swh-webapp'], } file {$production_db_dir: ensure => directory, - owner => 'swhwebapp', + owner => $user, group => $group, mode => '0755', } + file {$production_db_file: + ensure => present, + owner => $user, + group => $group, + mode => '0664', + } + ::gunicorn::instance {'swh-webapp': ensure => enabled, user => $user, group => $group, executable => 'swh.web.wsgi:application', settings => { bind => $backend_listen_address, workers => $backend_workers, worker_class => 'sync', timeout => $backend_http_timeout, graceful_timeout => $backend_reload_mercy, keepalive => $backend_http_keepalive, } } include ::profile::apache::common include ::apache::mod::proxy include ::apache::mod::headers ::apache::vhost {"${vhost_name}_non-ssl": - servername => $vhost_name, - serveraliases => $vhost_aliases, - port => $vhost_port, - docroot => $vhost_docroot, - proxy_pass => [ + servername => $vhost_name, + serveraliases => $vhost_aliases, + port => $vhost_port, + docroot => $vhost_docroot, + proxy_pass => [ { path => '/static', url => '!', }, { path => '/robots.txt', url => '!', }, { path => '/favicon.ico', url => '!', }, { path => '/', url => "http://${backend_listen_address}/", }, ], - directories => [ + directories => [ { path => '/api', provider => 'location', allow => 'from all', satisfy => 'Any', headers => ['add Access-Control-Allow-Origin "*"'], }, { path => $static_dir, options => ['-Indexes'], }, ] + $endpoint_directories, - aliases => [ + aliases => [ { alias => '/static', path => $static_dir, }, { alias => '/robots.txt', path => "${static_dir}/robots.txt", }, ], - require => [ + # work around fix for CVE-2019-0220 introduced in Apache2 2.4.25-3+deb9u7 + custom_fragment => 'MergeSlashes off', + require => [ File[$vhost_basic_auth_file], ], } $ssl_cert_names = ['star_softwareheritage_org', 'star_internal_softwareheritage_org'] include ::profile::hitch each($ssl_cert_names) |$ssl_cert_name| { realize(::Profile::Hitch::Ssl_cert[$ssl_cert_name]) } include ::profile::varnish ::profile::varnish::vhost {$vhost_name: aliases => $vhost_aliases, hsts_max_age => lookup('strict_transport_security::max_age'), } if $endpoint_directories { file {$vhost_basic_auth_file: ensure => present, owner => 'root', group => 'www-data', mode => '0640', content => $vhost_basic_auth_content, } } else { file {$vhost_basic_auth_file: ensure => absent, } } $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' @@::icinga2::object::service {"swh-webapp http redirect on ${::fqdn}": service_name => 'swh webapp http redirect', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $varnish_http_port, http_uri => '/', }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"swh-webapp https on ${::fqdn}": service_name => 'swh webapp', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, http_sni => true, http_uri => '/', http_onredirect => sticky }, target => $icinga_checks_file, tag => 'icinga2::exported', } @@::icinga2::object::service {"swh-webapp https certificate ${::fqdn}": service_name => 'swh webapp https certificate', import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, http_sni => true, http_certificate => 60, }, target => $icinga_checks_file, tag => 'icinga2::exported', } include ::profile::swh::deploy::webapp::icinga_checks } diff --git a/site-modules/profile/manifests/swh/deploy/webapp/icinga_checks.pp b/site-modules/profile/manifests/swh/deploy/webapp/icinga_checks.pp index 2b6b62c0..e665e7f9 100644 --- a/site-modules/profile/manifests/swh/deploy/webapp/icinga_checks.pp +++ b/site-modules/profile/manifests/swh/deploy/webapp/icinga_checks.pp @@ -1,89 +1,102 @@ class profile::swh::deploy::webapp::icinga_checks { $vhost_name = $::profile::swh::deploy::webapp::vhost_name $vhost_ssl_port = $::profile::swh::deploy::webapp::vhost_ssl_port $icinga_checks_file = '/etc/icinga2/conf.d/exported-checks.conf' $checks = { 'counters' => { http_uri => '/api/1/stat/counters/', http_string => '\"content\":', }, 'content known' => { http_uri => '/api/1/content/known/search/', http_post => 'q=8624bcdae55baeef00cd11d5dfcfa60f68710a02', http_string => '\"found\":true', }, 'content end to end' => { http_uri => '/browse/content/4dfc4478b1d5f7388b298fdfc06802485bdeae0c/', http_string => 'PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2', }, 'directory end to end' => { http_uri => '/browse/directory/977fc4b98c0e85816348cebd3b12026407c368b6/', http_linespan => true, http_expect_body_regex => 'Doc.*Grammar.*Include.*Lib.*Mac.*Misc.*Modules.*Objects.*PC.*PCbuild.*LICENSE.*README.rst', }, 'revision end to end' => { http_uri => '/browse/revision/f1b94134a4b879bc55c3dacdb496690c8ebdc03f/', http_linespan => true, http_expect_body_regex => join([ '-:"Allocate the output vlc pictures with dimensions padded,.*', 'as requested by the decoder \\\\(for alignments\\\\)."' ]), }, + 'revision log end to end' => { + http_uri => '/browse/revision/b9b0ecd1e2f9db10335383651f8317ed8cec8296/log/', + http_linespan => true, + http_expect_body_regex => join([ + '-:"', + join([ + '/browse/revision/b9b0ecd1e2f9db10335383651f8317ed8cec8296/', + 'Roberto Di Cosmo', + 'Moved to github', + ], '.*'), + '"', + ]), + }, 'release end to end' => { http_uri => '/browse/release/a9b7e3f1eada90250a6b2ab2ef3e0a846cb16831/', http_linespan => true, http_expect_body_regex => join([ '-:"Linux 4.9-rc8.*', '/revision/3e5de27e940d00d8d504dfb96625fb654f641509/"' ]), }, 'snapshot end to end' => { http_uri => '/browse/snapshot/baebc2109e4a2ec22a1129a3859647e191d04df4/branches/', http_linespan => true, http_expect_body_regex => join([ '-:"', join([ 'buster/main/4.13.13-1', 'buster/main/4.14.12-2', 'buster/main/4.14.13-1', 'buster/main/4.14.17-1', 'buster/main/4.15.4-1', 'buster/main/4.9.65-3', 'experimental/main/4.10~rc6-1~exp2', 'jessie-backports/main/3.16.39-1', 'jessie-backports/main/4.7.8-1~bpo8\\\\+1', 'jessie-backports/main/4.9.18-1~bpo8\\\\+1', 'jessie-backports/main/4.9.65-3\\\\+deb9u1~bpo8\\\\+1', 'jessie-backports/main/4.9.65-3\\\\+deb9u2~bpo8\\\\+1', 'jessie-kfreebsd/main/3.16.7-ckt9-2', 'jessie-proposed-updates/main/3.16.51-3', 'jessie-proposed-updates/main/3.16.51-3\\\\+deb8u1', 'jessie-updates/main/3.16.51-3', 'jessie/main/3.16.43-1', 'jessie/main/3.16.51-2', 'jessie/main/3.16.7-ckt2-1', 'jessie/main/3.16.7-ckt20-1\\\\+deb8u3', ], '.*'), '"', ]), } } each($checks) |$name, $args| { @@::icinga2::object::service {"swh-webapp ${name} ${::fqdn}": service_name => "swh webapp ${name}", import => ['generic-service'], host_name => $::fqdn, check_command => 'http', vars => { http_address => $vhost_name, http_vhost => $vhost_name, http_port => $vhost_ssl_port, http_ssl => true, } + $args, target => $icinga_checks_file, tag => 'icinga2::exported', } } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/base.pp b/site-modules/profile/manifests/swh/deploy/worker/base.pp index 087867ea..25b4fc7f 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/base.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/base.pp @@ -1,52 +1,37 @@ # Base worker profile class profile::swh::deploy::worker::base { include ::systemd::systemctl::daemon_reload $systemd_template_unit_name = 'swh-worker@.service' $systemd_unit_name = 'swh-worker.service' $systemd_slice_name = 'system-swh\x2dworker.slice' $systemd_generator = '/lib/systemd/system-generators/swh-worker-generator' - $config_directory = '/etc/softwareheritage/worker' package {'python3-swh.scheduler': ensure => installed, } ::systemd::unit_file {$systemd_template_unit_name: ensure => 'present', source => "puppet:///modules/profile/swh/deploy/worker/${systemd_template_unit_name}", } ::systemd::unit_file {$systemd_unit_name: ensure => 'present', source => "puppet:///modules/profile/swh/deploy/worker/${systemd_unit_name}", } ~> service {'swh-worker': ensure => running, enable => true, } ::systemd::unit_file {$systemd_slice_name: ensure => 'present', source => "puppet:///modules/profile/swh/deploy/worker/${systemd_slice_name}", } file {$systemd_generator: - ensure => 'present', - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/profile/swh/deploy/worker/swh-worker-generator', + ensure => 'absent', notify => Class['systemd::systemctl::daemon_reload'], } - - file {$config_directory: - ensure => 'directory', - owner => 'swhworker', - group => 'swhdev', - mode => '0644', - purge => true, - recurse => true, - } - } diff --git a/site-modules/profile/manifests/swh/deploy/worker/base_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/base_deposit.pp new file mode 100644 index 00000000..9b7f53b6 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/base_deposit.pp @@ -0,0 +1,7 @@ +class profile::swh::deploy::worker::base_deposit { + $packages = ['python3-swh.deposit.loader'] + + package {$packages: + ensure => 'present', + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp new file mode 100644 index 00000000..4fa1f6c8 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/checker_deposit.pp @@ -0,0 +1,10 @@ +# Deployment for deposit's archive checker +class profile::swh::deploy::worker::checker_deposit { + include ::profile::swh::deploy::worker::checker_deposit + + $private_tmp = lookup('swh::deploy::worker::checker_deposit::private_tmp') + ::profile::swh::deploy::worker::instance {'checker_deposit': + ensure => 'present', + private_tmp => $private_tmp, + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_ctags.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_ctags.pp new file mode 100644 index 00000000..553902be --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_ctags.pp @@ -0,0 +1,18 @@ +# Deployment for swh-indexer-ctags + +class profile::swh::deploy::worker::indexer_content_ctags { + include ::profile::swh::deploy::indexer + + $packages = ['universal-ctags'] + package {$packages: + ensure => 'present', + } + + Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_ctags': + ensure => 'stopped', + require => [ + Class['profile::swh::deploy::indexer'], + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp new file mode 100644 index 00000000..7614c706 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp @@ -0,0 +1,12 @@ +# Deployment for swh-indexer-mimetype + +class profile::swh::deploy::worker::indexer_content_mimetype { + include ::profile::swh::deploy::indexer + + Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': + ensure => present, + require => [ + Class['profile::swh::deploy::indexer'] + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp new file mode 100644 index 00000000..42377f82 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp @@ -0,0 +1,17 @@ +# Deployment for swh-indexer-fossology-license + +class profile::swh::deploy::worker::indexer_fossology_license { + include ::profile::swh::deploy::indexer + $packages = ['fossology-nomossa'] + package {$packages: + ensure => 'present', + } + + Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license': + ensure => present, + require => [ + Class['profile::swh::deploy::indexer'], + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp new file mode 100644 index 00000000..61bdfc28 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp @@ -0,0 +1,12 @@ +# Deployment for swh-indexer-origin-intrinsic-metadata + +class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata { + include ::profile::swh::deploy::indexer + + Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata': + ensure => present, + require => [ + Class['profile::swh::deploy::indexer'], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp new file mode 100644 index 00000000..f2cdc7ad --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp @@ -0,0 +1,12 @@ +# Deployment for swh-indexer-rehash + +class profile::swh::deploy::worker::indexer_rehash { + include ::profile::swh::deploy::indexer + + Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_rehash': + ensure => 'stopped', + require => [ + Class['profile::swh::deploy::indexer'] + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/instance.pp b/site-modules/profile/manifests/swh/deploy/worker/instance.pp index b3a60f2f..13397ccb 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/instance.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/instance.pp @@ -1,68 +1,61 @@ # Instance of a worker define profile::swh::deploy::worker::instance ( $ensure = present, - $task_broker = '', - $task_modules = [], - $task_queues = [], - $task_soft_time_limit = 0, - $concurrency = 10, - $loglevel = 'info', $max_tasks_per_child = 5, $instance_name = $title, $limit_no_file = undef, $private_tmp = undef) { include ::profile::swh::deploy::worker::base $service_basename = "swh-worker@${instance_name}" $service_name = "${service_basename}.service" - $config_directory = '/etc/softwareheritage/worker' - $instance_config = "${config_directory}/${instance_name}.ini" + $concurrency = lookup("swh::deploy::worker::${instance_name}::concurrency") + $loglevel = lookup("swh::deploy::worker::${instance_name}::loglevel") + $config_file = lookup("swh::deploy::worker::${instance_name}::config_file") + $config = lookup("swh::deploy::worker::${instance_name}::config", Hash, 'deep') case $ensure { 'present', 'running': { # Uses variables # - $concurrency # - $loglevel # - $max_tasks_per_child ::systemd::dropin_file {"${service_basename}/parameters.conf": ensure => present, unit => $service_name, filename => 'parameters.conf', content => template('profile/swh/deploy/worker/parameters.conf.erb'), } - # Uses variables - # - $task_broker - # - $task_modules - # - $task_queues - # - $task_soft_time_limit - file {$instance_config: - ensure => present, + file {$config_file: + ensure => 'present', owner => 'swhworker', - group => 'swhdev', - # contains a password for the broker - mode => '0640', - content => template('profile/swh/deploy/worker/instance_config.ini.erb'), + group => 'swhworker', + mode => '0644', + content => inline_template("<%= @config.to_yaml %>\n"), } + if $ensure == 'running' { - service {$service_basename: - ensure => $ensure, - require => [ - File[$instance_config], - ], - } + $service_ensure = 'running' + } else { + $service_ensure = undef + } + + service {$service_basename: + ensure => $service_ensure, + enable => true, + require => [ + File[$config_file], + ] } } default: { - file {$instance_config: - ensure => absent, - } ::systemd::dropin_file {"${service_basename}/parameters.conf": ensure => absent, unit => $service_name, filename => 'parameters.conf', } } } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister.pp b/site-modules/profile/manifests/swh/deploy/worker/lister.pp new file mode 100644 index 00000000..48baf28b --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/lister.pp @@ -0,0 +1,15 @@ +# Deployment for swh-lister-github +class profile::swh::deploy::worker::lister { + $packages = ['python3-swh.lister'] + + package {$packages: + ensure => present, + } + + ::profile::swh::deploy::worker::instance {'lister': + ensure => present, + require => [ + Package['python3-swh.lister'], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp new file mode 100644 index 00000000..2cc96f28 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp @@ -0,0 +1,15 @@ +# Deployment for swh-loader-debian +class profile::swh::deploy::worker::loader_debian { + $packages = ['python3-swh.loader.debian'] + + package {$packages: + ensure => 'present', + } + + ::profile::swh::deploy::worker::instance {'loader_debian': + ensure => present, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp new file mode 100644 index 00000000..7752de75 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp @@ -0,0 +1,8 @@ +# Deployment for deposit's loader +class profile::swh::deploy::worker::loader_deposit { + $private_tmp = lookup('swh::deploy::worker::loader_deposit::private_tmp') + ::profile::swh::deploy::worker::instance {'loader_deposit': + ensure => 'present', + private_tmp => $private_tmp, + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp new file mode 100644 index 00000000..31ab8eda --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp @@ -0,0 +1,12 @@ +# Deployment for swh-loader-git (remote) +class profile::swh::deploy::worker::loader_git { + include ::profile::swh::deploy::base_loader_git + + ::profile::swh::deploy::worker::instance {'loader_git': + ensure => present, + require => [ + Class['profile::swh::deploy::base_loader_git'], + ], + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp new file mode 100644 index 00000000..fd30730b --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp @@ -0,0 +1,17 @@ +# Deployment for swh-loader-mercurial (disk) +class profile::swh::deploy::worker::loader_mercurial { + $private_tmp = lookup('swh::deploy::worker::loader_mercurial::private_tmp') + $packages = ['python3-swh.loader.mercurial'] + + package {$packages: + ensure => 'present', + } + + ::profile::swh::deploy::worker::instance {'loader_mercurial': + ensure => 'present', + private_tmp => $private_tmp, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp new file mode 100644 index 00000000..19e912e2 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_npm.pp @@ -0,0 +1,18 @@ +# Deployment for swh-loader-npm +class profile::swh::deploy::worker::loader_npm { + $private_tmp = lookup('swh::deploy::worker::loader_npm::private_tmp') + + $packages = ['python3-swh.loader.npm'] + + package {$packages: + ensure => 'present', + } + + ::profile::swh::deploy::worker::instance {'loader_npm': + ensure => present, + private_tmp => $private_tmp, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp new file mode 100644 index 00000000..a614042a --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp @@ -0,0 +1,18 @@ +# Deployment for swh-loader-pypi +class profile::swh::deploy::worker::loader_pypi { + $private_tmp = lookup('swh::deploy::worker::loader_pypi::private_tmp') + + $packages = ['python3-swh.loader.pypi'] + + package {$packages: + ensure => 'present', + } + + ::profile::swh::deploy::worker::instance {'loader_pypi': + ensure => present, + private_tmp => $private_tmp, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp new file mode 100644 index 00000000..3f752e65 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp @@ -0,0 +1,19 @@ +# Deployment for swh-loader-svn +class profile::swh::deploy::worker::loader_svn { + $packages = ['python3-swh.loader.svn'] + $limit_no_file = lookup('swh::deploy::worker::loader_svn::limit_no_file') + $private_tmp = lookup('swh::deploy::worker::loader_svn::private_tmp') + + package {$packages: + ensure => 'present', + } + + ::profile::swh::deploy::worker::instance {'loader_svn': + ensure => present, + limit_no_file => $limit_no_file, + private_tmp => $private_tmp, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp deleted file mode 100644 index e5d83eb6..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Deployment for swh-indexer-ctags - -class profile::swh::deploy::worker::swh_indexer_ctags { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::ctags::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::ctags::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::ctags::task_broker') - - $config_file = '/etc/softwareheritage/indexer/ctags.yml' - $config = lookup('swh::deploy::worker::swh_indexer::ctags::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_ctags'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_ctags': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp deleted file mode 100644 index 66ea9494..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Deployment for swh-indexer-fossology-license - -class profile::swh::deploy::worker::swh_indexer_fossology_license { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::fossology_license::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::fossology_license::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::fossology_license::task_broker') - - $config_file = '/etc/softwareheritage/indexer/fossology_license.yml' - $config = lookup('swh::deploy::worker::swh_indexer::fossology_license::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_fossology_license'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_fossology_license': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp deleted file mode 100644 index de4bc02f..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Deployment for swh-indexer-language - -class profile::swh::deploy::worker::swh_indexer_language { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::language::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::language::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::language::task_broker') - - $config_file = '/etc/softwareheritage/indexer/language.yml' - $config = lookup('swh::deploy::worker::swh_indexer::language::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_language'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_language': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp deleted file mode 100644 index 79c001a1..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Deployment for swh-indexer-mimetype - -class profile::swh::deploy::worker::swh_indexer_mimetype { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::mimetype::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::mimetype::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::mimetype::task_broker') - - $config_file = '/etc/softwareheritage/indexer/mimetype.yml' - $config = lookup('swh::deploy::worker::swh_indexer::mimetype::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_mimetype'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_mimetype': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp deleted file mode 100644 index ade2f8ea..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_orchestrator.pp +++ /dev/null @@ -1,37 +0,0 @@ -# Deployment for swh-indexer - -class profile::swh::deploy::worker::swh_indexer_orchestrator { - - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::orchestrator::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::orchestrator::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::orchestrator::task_broker') - - $config_file = '/etc/softwareheritage/indexer/orchestrator.yml' - $config = lookup('swh::deploy::worker::swh_indexer::orchestrator::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_orchestrator_content_all'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_orchestrator': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp deleted file mode 100644 index a6a288d1..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_orchestrator_text.pp +++ /dev/null @@ -1,37 +0,0 @@ -# Deployment for swh-indexer - -class profile::swh::deploy::worker::swh_indexer_orchestrator_text { - - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::orchestrator_text::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::orchestrator_text::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::orchestrator_text::task_broker') - - $config_file = '/etc/softwareheritage/indexer/orchestrator_text.yml' - $config = lookup('swh::deploy::worker::swh_indexer::orchestrator_text::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_orchestrator_content_text'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_orchestrator_text': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp deleted file mode 100644 index ed5d0a1b..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Deployment for swh-indexer-rehash - -class profile::swh::deploy::worker::swh_indexer_rehash { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::rehash::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::rehash::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_indexer::rehash::task_broker') - - $config_file = '/etc/softwareheritage/indexer/rehash.yml' - $config = lookup('swh::deploy::worker::swh_indexer::rehash::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_rehash'] - - Package[$::profile::swh::deploy::indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_rehash': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp deleted file mode 100644 index 10992afc..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment for swh-lister-debian -class profile::swh::deploy::worker::swh_lister_debian { - $concurrency = lookup('swh::deploy::worker::swh_lister_debian::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_debian::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_debian::task_broker') - - $config_file = '/etc/softwareheritage/lister-debian.yml' - $config = lookup('swh::deploy::worker::swh_lister_debian::config', Hash, 'deep') - - $task_modules = ['swh.lister.debian.tasks'] - $task_queues = ['swh_lister_debian'] - - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'swh_lister_debian': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package['python3-swh.lister'], - File[$config_file], - ], - } - - # Contains passwords - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp deleted file mode 100644 index c33fda29..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment for swh-lister-github -class profile::swh::deploy::worker::swh_lister_github { - $concurrency = lookup('swh::deploy::worker::swh_lister_github::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_github::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_github::task_broker') - - $config_file = '/etc/softwareheritage/lister-github.com.yml' - $config = lookup('swh::deploy::worker::swh_lister_github::config', Hash, 'deep') - - $task_modules = ['swh.lister.github.tasks'] - $task_queues = ['swh_lister_github_discover', 'swh_lister_github_refresh'] - - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'swh_lister_github': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package['python3-swh.lister'], - File[$config_file], - ], - } - - # Contains passwords - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp deleted file mode 100644 index ddae6b32..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment for swh-lister-gitlab -class profile::swh::deploy::worker::swh_lister_gitlab { - $concurrency = lookup('swh::deploy::worker::swh_lister_gitlab::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_gitlab::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_gitlab::task_broker') - - $config_file = '/etc/softwareheritage/lister-gitlab.yml' - $config = lookup('swh::deploy::worker::swh_lister_gitlab::config', Hash, 'deep') - - $task_modules = ['swh.lister.gitlab.tasks'] - $task_queues = ['swh_lister_gitlab_discover', 'swh_lister_gitlab_refresh'] - - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'swh_lister_gitlab': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package['python3-swh.lister'], - File[$config_file], - ], - } - - # Contains passwords - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp deleted file mode 100644 index 8ac6f82e..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment for swh-lister-pypi -class profile::swh::deploy::worker::swh_lister_pypi { - $concurrency = lookup('swh::deploy::worker::swh_lister_pypi::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_pypi::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_lister_pypi::task_broker') - - $config_file = '/etc/softwareheritage/lister-pypi.yml' - $config = lookup('swh::deploy::worker::swh_lister_pypi::config', Hash, 'deep') - - $task_modules = ['swh.lister.pypi.tasks'] - $task_queues = ['swh_lister_pypi_refresh'] - - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'swh_lister_pypi': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package['python3-swh.lister'], - File[$config_file], - ], - } - - # Contains passwords - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp deleted file mode 100644 index d0e8c8bc..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp +++ /dev/null @@ -1,39 +0,0 @@ -# Deployment for swh-loader-debian -class profile::swh::deploy::worker::swh_loader_debian { - $concurrency = lookup('swh::deploy::worker::swh_loader_debian::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_debian::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_debian::task_broker') - - $config_file = '/etc/softwareheritage/loader/debian.yml' - $config = lookup('swh::deploy::worker::swh_loader_debian::config') - - $task_modules = ['swh.loader.debian.tasks'] - $task_queues = ['swh_loader_debian'] - - $packages = ['python3-swh.loader.debian'] - - package {$packages: - ensure => 'present', - } - - ::profile::swh::deploy::worker::instance {'swh_loader_debian': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package[$packages], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp deleted file mode 100644 index d224c486..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp +++ /dev/null @@ -1,65 +0,0 @@ -# Deployment for swh-loader-deposit -class profile::swh::deploy::worker::swh_loader_deposit { - $concurrency = lookup('swh::deploy::worker::swh_loader_deposit::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_deposit::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_deposit::task_broker') - - $deposit_config_directory = lookup('swh::deploy::deposit::conf_directory') - $config_file = lookup('swh::deploy::worker::swh_loader_deposit::swh_conf_file') - $config = lookup('swh::deploy::worker::swh_loader_deposit::config') - - $task_modules = ['swh.deposit.loader.tasks'] - $task_queues = ['swh_checker_deposit', 'swh_loader_deposit'] - - $packages = ['python3-swh.deposit.loader'] - $private_tmp = lookup('swh::deploy::worker::swh_loader_deposit::private_tmp') - - $service_name = 'swh_loader_deposit' - - package {$packages: - ensure => 'latest', - notify => Service["swh-worker@$service_name"], - } - - # This installs the swh-worker@$service_name service - ::profile::swh::deploy::worker::instance {$service_name: - ensure => running, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - private_tmp => $private_tmp, - require => [ - Package[$packages], - File[$config_file], - ], - } - - file {$deposit_config_directory: - ensure => directory, - owner => 'swhworker', - group => 'swhdev', - mode => '0750', - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - require => [ - File[$deposit_config_directory], - ], - } - - $swh_client_conf_file = lookup('swh::deploy::deposit::client::swh_conf_file') - $swh_client_config = lookup('swh::deploy::deposit::client::settings_private_data') - file {$swh_client_conf_file: - owner => 'swhworker', - group => 'swhdev', - mode => '0640', - content => inline_template("<%= @swh_client_config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp deleted file mode 100644 index 595397d9..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp +++ /dev/null @@ -1,35 +0,0 @@ -# Deployment for swh-loader-git (remote) -class profile::swh::deploy::worker::swh_loader_git { - include ::profile::swh::deploy::base_loader_git - - $concurrency = lookup('swh::deploy::worker::swh_loader_git::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_git::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_git::task_broker') - - $config_file = '/etc/softwareheritage/loader/git-updater.yml' - $config = lookup('swh::deploy::worker::swh_loader_git::config') - - $task_modules = ['swh.loader.git.tasks'] - $task_queues = ['swh_loader_git'] - - ::profile::swh::deploy::worker::instance {'swh_loader_git': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::base_loader_git'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git_disk.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git_disk.pp deleted file mode 100644 index c0d9b971..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git_disk.pp +++ /dev/null @@ -1,39 +0,0 @@ -# Deployment for swh-loader-git (disk) -class profile::swh::deploy::worker::swh_loader_git_disk { - include ::profile::swh::deploy::base_loader_git - - $concurrency = lookup('swh::deploy::worker::swh_loader_git_disk::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_git_disk::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_git_disk::task_broker') - - $config_file = '/etc/softwareheritage/loader/git-loader.yml' - $config = lookup('swh::deploy::worker::swh_loader_git_disk::config') - - $task_modules = ['swh.loader.git.tasks'] - $task_queues = ['swh_loader_git_express', 'swh_loader_git_archive'] - - $service_name = 'swh_loader_git_disk' - - Package[$::profile::swh::deploy::base_loader_git::packages] ~> Service["swh-worker@$service_name"] - - ::profile::swh::deploy::worker::instance {$service_name: - ensure => running, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Class['profile::swh::deploy::base_loader_git'], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp deleted file mode 100644 index 5cf92187..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp +++ /dev/null @@ -1,45 +0,0 @@ -# Deployment for swh-loader-mercurial (disk) -class profile::swh::deploy::worker::swh_loader_mercurial { - include ::profile::swh::deploy::base_loader_git - - $concurrency = lookup('swh::deploy::worker::swh_loader_mercurial::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_mercurial::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_mercurial::task_broker') - - $config_file = '/etc/softwareheritage/loader/hg.yml' - $config = lookup('swh::deploy::worker::swh_loader_mercurial::config') - - $task_modules = ['swh.loader.mercurial.tasks'] - $task_queues = ['swh_loader_mercurial', 'swh_loader_mercurial_archive'] - - $service_name = 'swh_loader_mercurial' - $private_tmp = lookup('swh::deploy::worker::swh_loader_mercurial::private_tmp') - - $packages = ['python3-swh.loader.mercurial'] - - package {$packages: - ensure => 'latest', - notify => Service["swh-worker@$service_name"] - } - - ::profile::swh::deploy::worker::instance {$service_name: - ensure => running, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - private_tmp => $private_tmp, - require => [ - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp deleted file mode 100644 index 7aa6ef23..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp +++ /dev/null @@ -1,41 +0,0 @@ -# Deployment for swh-loader-pypi -class profile::swh::deploy::worker::swh_loader_pypi { - $concurrency = lookup('swh::deploy::worker::swh_loader_pypi::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_pypi::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_pypi::task_broker') - - $config_file = '/etc/softwareheritage/loader/pypi.yml' - $config = lookup('swh::deploy::worker::swh_loader_pypi::config') - - $task_modules = ['swh.loader.pypi.tasks'] - $task_queues = ['swh_loader_pypi'] - $private_tmp = lookup('swh::deploy::worker::swh_loader_pypi::private_tmp') - - $packages = ['python3-swh.loader.pypi'] - - package {$packages: - ensure => 'latest', - } - - ::profile::swh::deploy::worker::instance {'swh_loader_pypi': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - private_tmp => $private_tmp, - require => [ - Package[$packages], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp deleted file mode 100644 index 55b94d0f..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp +++ /dev/null @@ -1,43 +0,0 @@ -# Deployment for swh-loader-svn -class profile::swh::deploy::worker::swh_loader_svn { - $concurrency = lookup('swh::deploy::worker::swh_loader_svn::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_svn::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_loader_svn::task_broker') - - $config_file = '/etc/softwareheritage/loader/svn.yml' - $config = lookup('swh::deploy::worker::swh_loader_svn::config') - - $task_modules = ['swh.loader.svn.tasks'] - $task_queues = ['swh_loader_svn', 'swh_loader_svn_mount_and_load'] - - $packages = ['python3-swh.loader.svn'] - $limit_no_file = lookup('swh::deploy::worker::swh_loader_svn::limit_no_file') - $private_tmp = lookup('swh::deploy::worker::swh_loader_svn::private_tmp') - - package {$packages: - ensure => 'latest', - } - - ::profile::swh::deploy::worker::instance {'swh_loader_svn': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - limit_no_file => $limit_no_file, - private_tmp => $private_tmp, - require => [ - Package[$packages], - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp deleted file mode 100644 index 6b11364d..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp +++ /dev/null @@ -1,37 +0,0 @@ -# Deployment for swh-storage-archiver -class profile::swh::deploy::worker::swh_storage_archiver { - include ::profile::swh::deploy::archiver - - $concurrency = lookup('swh::deploy::worker::swh_storage_archiver::concurrency') - $max_tasks_per_child = lookup('swh::deploy::worker::swh_storage_archiver::max_tasks_per_child') - $loglevel = lookup('swh::deploy::worker::swh_storage_archiver::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_storage_archiver::task_broker') - - $config_file = lookup('swh::deploy::worker::swh_storage_archiver::conf_file') - $config = lookup('swh::deploy::worker::swh_storage_archiver::config') - - $task_modules = ['swh.archiver.tasks'] - $task_queues = ['swh_storage_archive_worker'] - - ::profile::swh::deploy::worker::instance {'swh_storage_archiver': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - max_tasks_per_child => $max_tasks_per_child, - require => [ - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp deleted file mode 100644 index c7d475bf..00000000 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment of a vault cooker - -class profile::swh::deploy::worker::swh_vault_cooker { - include ::profile::swh::deploy::base_vault - - $concurrency = lookup('swh::deploy::worker::swh_vault_cooker::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_vault_cooker::loglevel') - $task_broker = lookup('swh::deploy::worker::swh_vault_cooker::task_broker') - - $conf_file = lookup('swh::deploy::worker::swh_vault_cooker::conf_file') - $config = lookup('swh::deploy::worker::swh_vault_cooker::config') - - $task_modules = ['swh.vault.cooking_tasks'] - $task_queues = ['swh_vault_cooking', 'swh_vault_batch_cooking'] - - ::profile::swh::deploy::worker::instance {'swh_vault_cooker': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - task_broker => $task_broker, - task_modules => $task_modules, - task_queues => $task_queues, - require => [ - Package[$packages], - File[$conf_file], - ], - } - - file {$conf_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp new file mode 100644 index 00000000..f4243376 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp @@ -0,0 +1,12 @@ +# Deployment of a vault cooker + +class profile::swh::deploy::worker::vault_cooker { + include ::profile::swh::deploy::base_vault + + ::profile::swh::deploy::worker::instance {'vault_cooker': + ensure => present, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/weekly_report_bot.pp b/site-modules/profile/manifests/weekly_report_bot.pp new file mode 100644 index 00000000..7be344b4 --- /dev/null +++ b/site-modules/profile/manifests/weekly_report_bot.pp @@ -0,0 +1,22 @@ +# A bot creating a thread for people to send their weekly report + +class profile::weekly_report_bot { + $command = '/usr/local/bin/weekly-report-bot' + + $weekly_report_user = lookup('weekly_report_bot::user') + $weekly_report_cron = lookup('weekly_report_bot::cron') + + file {$command: + ensure => present, + mode => '0755', + owner => 'root', + group => 'root', + source => 'puppet:///modules/profile/weekly_report_bot/weekly-report-bot', + } + + profile::cron::d {'weekly-report-bot': + command => $command, + user => $weekly_report_user, + * => $weekly_report_cron, + } +} diff --git a/site-modules/profile/manifests/zookeeper.pp b/site-modules/profile/manifests/zookeeper.pp index 7591f997..db6c5824 100644 --- a/site-modules/profile/manifests/zookeeper.pp +++ b/site-modules/profile/manifests/zookeeper.pp @@ -1,11 +1,18 @@ # Zookeeper cluster member profile class profile::zookeeper { + $zookeeper_servers = lookup('zookeeper::servers', Hash) + + $zookeeper_ids = $zookeeper_servers.filter |$id, $hostname| { + $hostname == $::swh_hostname['internal_fqdn'] + }.keys + class {'::zookeeper': - servers => lookup('zookeeper::servers', Hash), + servers => $zookeeper_servers, datastore => lookup('zookeeper::datastore'), client_port => lookup('zookeeper::client_port'), election_port => lookup('zookeeper::election_port'), leader_port => lookup('zookeeper::leader_port'), + id => $zookeeper_ids[0], } } diff --git a/site-modules/profile/templates/cron/snippet.erb b/site-modules/profile/templates/cron/snippet.erb new file mode 100644 index 00000000..ad084fd0 --- /dev/null +++ b/site-modules/profile/templates/cron/snippet.erb @@ -0,0 +1,6 @@ +# Cron snippet <%= @unique_tag %> +<% if @special -%> +<%= @special %> <%= @user %> <%= @command %> +<% else -%> +<%= @_params_hash['minute'] %> <%= @_params_hash['hour'] %> <%= @_params_hash['monthday'] %> <%= @_params_hash['month'] %> <%= @_params_hash['weekday'] %> <%= @user %> <%= @command %> +<% end -%> diff --git a/site-modules/profile/templates/debian_repository/uploaders.erb b/site-modules/profile/templates/debian_repository/uploaders.erb new file mode 100644 index 00000000..45e91496 --- /dev/null +++ b/site-modules/profile/templates/debian_repository/uploaders.erb @@ -0,0 +1,6 @@ +# File managed by puppet (class profile::debian_repository). +# Local modifications will be lost + +<% @gpg_keys.each do |fpr| -%> +allow * by key <%= fpr[-16..-1] %>+ +<% end %> diff --git a/site-modules/profile/templates/jenkins/agent/jenkins-agent.defaults.erb b/site-modules/profile/templates/jenkins/agent/jenkins-agent.defaults.erb new file mode 100644 index 00000000..abe4ce10 --- /dev/null +++ b/site-modules/profile/templates/jenkins/agent/jenkins-agent.defaults.erb @@ -0,0 +1,6 @@ +# File managed by puppet (module profile::jenkins::agent) +# Local modifications will be lost. + +AGENT_WORKDIR="<%= @workdir %>" +JNLP_URL="<%= @jenkins_jnlp_url %>" +JNLP_SECRET="<%= @jenkins_jnlp_token %>" diff --git a/site-modules/profile/templates/jenkins/agent/jenkins-agent.service.erb b/site-modules/profile/templates/jenkins/agent/jenkins-agent.service.erb new file mode 100644 index 00000000..46e81a5c --- /dev/null +++ b/site-modules/profile/templates/jenkins/agent/jenkins-agent.service.erb @@ -0,0 +1,20 @@ +[Unit] +Description=Jenkins agent +Documentation=https://wiki.jenkins.io/display/JENKINS/Distributed+builds +After=network.target + +[Service] +EnvironmentFile=/etc/default/jenkins-agent +ExecStart=/usr/bin/java -jar <%= @jenkins_agent_jar %> -jnlpUrl $JNLP_URL -secret $JNLP_SECRET -workDir $AGENT_WORKDIR +UnsetEnvironment=JNLP_URL JNLP_SECRET AGENT_WORKDIR + +User=jenkins +Group=jenkins + +KillMode=process + +Restart=always +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/profile/templates/jenkins/gitconfig.erb b/site-modules/profile/templates/jenkins/gitconfig.erb new file mode 100644 index 00000000..871a2f40 --- /dev/null +++ b/site-modules/profile/templates/jenkins/gitconfig.erb @@ -0,0 +1,9 @@ +# Git configuration for jenkins +# Handled by puppet (class profile::jenkins::base), manual changes will be lost + +[user] + name = Jenkins for Software Heritage + email = jenkins@<%= @fqdn %> + +[url "git@forge.softwareheritage.org:"] + pushInsteadOf = https://forge.softwareheritage.org diff --git a/site-modules/profile/templates/letsencrypt/fetch_certificate_file.erb b/site-modules/profile/templates/letsencrypt/fetch_certificate_file.erb new file mode 100644 index 00000000..39cc6b42 --- /dev/null +++ b/site-modules/profile/templates/letsencrypt/fetch_certificate_file.erb @@ -0,0 +1,13 @@ +<%= + dir = scope().call_function('lookup', ['letsencrypt::certificates::exported_directory']) + filename = "#{dir}/#{@basename}/#{@filename}" + begin + out = File.read(filename) + rescue Errno::ENOENT => e + out = "File not found: #{filename}\n" + rescue Errno::EACCES => e + out = "Permission denied : #{filename}\n" + end + + out +-%> diff --git a/site-modules/profile/templates/letsencrypt/letsencrypt_gandi_livedns.erb b/site-modules/profile/templates/letsencrypt/letsencrypt_gandi_livedns.erb new file mode 100755 index 00000000..25b38334 --- /dev/null +++ b/site-modules/profile/templates/letsencrypt/letsencrypt_gandi_livedns.erb @@ -0,0 +1,211 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import logging +import os +import sys +import urllib.parse + +import yaml +import requests + + +logger = logging.getLogger(__name__) + + +CONFIG_FILE = os.environ.get( + 'CERTBOT_GANDI_CONFIG', + '<%= @hook_configfile %>', +) + +CONFIG = None +DEFAULT_CONFIG = { + 'gandi_api': 'https://dns.api.gandi.net/api/v5/', + 'zones': {}, +} + + +def load_config(): + """Load the hook configuration from CONFIG_FILE""" + global CONFIG + + if CONFIG is not None: + return + + try: + with open(CONFIG_FILE, 'r') as f: + CONFIG = yaml.safe_load(f) + return True + except Exception as e: + logger.warning( + 'Could not open configuration file %s: %s', CONFIG_FILE, e + ) + CONFIG = DEFAULT_CONFIG + return False + + +def get_domain_config(domain): + """Retrieve the configuration for the zone containing `domain`.""" + labels = domain.split('.') + for i in range(len(labels)): + zone = '.'.join(labels[i:]) + if zone in CONFIG['zones']: + zone_config = CONFIG['zones'][zone] + if labels[0] == '*': + relative = '.'.join(labels[1:i]) + else: + relative = '.'.join(labels[:i]) + + acme_subdomain = '_acme-challenge%s' % ( + ('.%s' % relative) if relative else '' + ) + + return { + 'domain': domain, + 'zone': zone, + 'relative_subdomain': relative, + 'acme_subdomain': acme_subdomain, + 'api_key': zone_config['api_key'], + 'sharing_id': zone_config.get('sharing_id') + } + else: + logger.error( + 'Could not find zone for domain %s, available zones: %s', + domain, ', '.join(CONFIG['zones'].keys()), + ) + +def gandi_request(url, domain_config, method='GET', data=None): + """Perform a request to the Gandi website, with the given data""" + if url.startswith('https://'): + parsed_url = urllib.parse.urlparse(url) + else: + parsed_url = urllib.parse.urlparse(CONFIG['gandi_api']) + parsed_url = parsed_url._replace( + path='/'.join([parsed_url.path.rstrip('/'), url]) + ) + + # Add sharing_id to the query string if needed + if domain_config.get('sharing_id'): + qs = urllib.parse.parse_qs(parsed_url.query) + qs['sharing_id'] = domain_config['sharing_id'] + parsed_url = parsed_url._replace( + query=urllib.parse.urlencode(qs, doseq=True) + ) + + headers = { + 'X-Api-Key': domain_config['api_key'], + } + url = urllib.parse.urlunparse(parsed_url) + + method = method.lower() + response = getattr(requests, method)(url, headers=headers, json=data) + + if response.status_code < 400 or response.status_code == 404: + return response + + logger.warn('Got unexpected error %s from the Gandi API: %s', + response.status_code, response.text) + response.raise_for_status() + + +def get_zone_info(domain_config): + """Retrieve the zone information from Gandi's website""" + response = gandi_request('domains', domain_config) + for domain in response.json(): + if domain['fqdn'] == domain_config['zone']: + return domain + else: + return {} + + +def get_acme_url(domain_config): + """Get the URL for the acme records for the given domain config""" + zone_info = get_zone_info(domain_config) + acme_records_url = '%s/%s/TXT' % ( + zone_info['domain_records_href'], domain_config['acme_subdomain'] + ) + + return acme_records_url + + +def get_acme_records(domain_config): + """Retrieve existing ACME TXT records from the Gandi API""" + acme_records_url = get_acme_url(domain_config) + response = gandi_request(acme_records_url, domain_config) + if response.status_code == 404: + return set() + + rrset = response.json() + return {value.strip('"') for value in rrset['rrset_values']} + + +def set_acme_records(domain_config, acme_records): + """Set the ACME TXT records on the given domain to the given""" + acme_records_url = get_acme_url(domain_config) + if not acme_records: + response = gandi_request(acme_records_url, domain_config, + method='delete') + return True + + new_record = { + "rrset_ttl": 300, + "rrset_values": list(set(acme_records)), + } + + response = gandi_request(acme_records_url, domain_config, + method='put', data=new_record) + + if response.status_code == 404: + response.raise_for_status() + + return True + + +def usage(): + print(""" +Usage: %s {auth, cleanup, purge} + +Set the CERTBOT_DOMAIN environment variable to set the domain used by the hook. +Set the CERTBOT_VALIDATION environment variable to set the ACME challenge (only +for auth/cleanup). +""".strip() % sys.argv[0], file=sys.stderr) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + + certbot_domain = os.environ.get('CERTBOT_DOMAIN') + if len(sys.argv) != 2 or not certbot_domain: + usage() + sys.exit(1) + + certbot_validation = os.environ.get('CERTBOT_VALIDATION') + + load_config() + domain_config = get_domain_config(certbot_domain) + if not domain_config: + sys.exit(2) + + if sys.argv[1] == 'auth': + if not certbot_validation: + usage() + sys.exit(1) + acme_records = get_acme_records(domain_config) + acme_records.add(certbot_validation) + set_acme_records(domain_config, acme_records) + elif sys.argv[1] == 'cleanup': + if not certbot_validation: + usage() + sys.exit(1) + acme_records = get_acme_records(domain_config) + acme_records.add(certbot_validation) + set_acme_records(domain_config, acme_records) + elif sys.argv[1] == 'purge': + set_acme_records(domain_config, set()) + else: + usage() + sys.exit(1) diff --git a/site-modules/profile/templates/letsencrypt/letsencrypt_gandi_paas.erb b/site-modules/profile/templates/letsencrypt/letsencrypt_gandi_paas.erb new file mode 100644 index 00000000..6940561c --- /dev/null +++ b/site-modules/profile/templates/letsencrypt/letsencrypt_gandi_paas.erb @@ -0,0 +1,106 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import logging +import os +import xmlrpc.client + +import yaml + + +logger = logging.getLogger(__name__) + + +CONFIG_FILE = os.environ.get( + 'CERTBOT_GANDI_CONFIG', + '<%= @hook_configfile %>', +) + +CONFIG = None +DEFAULT_CONFIG = { + # Default: OT&E environment + 'gandi_xmlrpc': 'https://rpc.ote.gandi.net/xmlrpc/', + # Production environment + # 'gandi_xmlrpc': 'https://rpc.gandi.net/xmlrpc/', + 'zone_keys': {}, +} + + +def load_config(): + """Load the hook configuration from CONFIG_FILE""" + global CONFIG + + if CONFIG is not None: + return + + try: + with open(CONFIG_FILE, 'r') as f: + CONFIG = yaml.safe_load(f) + return True + except Exception as e: + logger.warning( + 'Could not open configuration file %s: %s', CONFIG_FILE, e + ) + CONFIG = DEFAULT_CONFIG + return False + + +def get_key_for_domain(domain): + """Retrieve the XML-RPC key for the zone containing `domain`.""" + labels = domain.split('.') + for i in range(len(labels)): + zone = '.'.join(labels[i:]) + if zone in CONFIG['zone_keys']: + return CONFIG['zone_keys'][zone] + else: + logger.error( + 'Could not find zone for domain %s, available zones: %s', + domain, ', '.join(CONFIG['zones'].keys()), + ) + + +def get_certificate_data(basepath): + """Get the certificate data from basepath into a Gandi-compatible format. + + https://doc.rpc.gandi.net/cert/reference.html#SSLCreateParams + """ + + fullchain = open(os.path.join(basepath, 'fullchain.pem')).read() + privkey = open(os.path.join(basepath, 'privkey.pem')).read() + + return { + 'crt': fullchain, + 'key': privkey, + } + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + + load_config() + + renewed_domains = os.environ['RENEWED_DOMAINS'].split() + base_path = os.environ['RENEWED_LINEAGE'] + + certificate_data = get_certificate_data(base_path) + + rpc_client = xmlrpc.client.ServerProxy(CONFIG['gandi_xmlrpc']) + + processed_api_keys = set() + for domain in renewed_domains: + api_key = get_key_for_domain(domain) + if not api_key or api_key in processed_api_keys: + continue + + try: + ret = rpc_client.cert.hosted.create(api_key, certificate_data) + except Exception: + logger.exception('Failed pushing certificate') + else: + processed_api_keys.add(api_key) + logger.info('Successfully pushed certificate with id %s', + ret['id']) diff --git a/site-modules/profile/templates/letsencrypt/letsencrypt_puppet_export.erb b/site-modules/profile/templates/letsencrypt/letsencrypt_puppet_export.erb new file mode 100644 index 00000000..3672b51f --- /dev/null +++ b/site-modules/profile/templates/letsencrypt/letsencrypt_puppet_export.erb @@ -0,0 +1,18 @@ +#!/bin/bash + +# Export renewed letsencrypt certificates to the puppet vardir, and make them +# accessible to the puppet user. +# Script managed by the ::profile::letsencrypt::puppet_export_hook puppet class + +set -e + +umask 077 + +puppet_cert_root=<%= scope().call_function('lookup', ['letsencrypt::certificates::exported_directory']) %> + +basename=$(basename "$RENEWED_LINEAGE") +destdir="$puppet_cert_root/$basename" + +rm -rf "$destdir" +cp -rL "$RENEWED_LINEAGE" "$destdir" +chown -R puppet: "$destdir" diff --git a/site-modules/profile/templates/phabricator/phabricator-aphlict.service.erb b/site-modules/profile/templates/phabricator/phabricator-aphlict.service.erb index 8f52e8d7..e3391f85 100644 --- a/site-modules/profile/templates/phabricator/phabricator-aphlict.service.erb +++ b/site-modules/profile/templates/phabricator/phabricator-aphlict.service.erb @@ -1,23 +1,22 @@ # Phabricator Notification Daemon unit file # Managed by puppet class profile::phabricator # Changes will be overwritten [Unit] Description=Phabricator Notification Daemon -Requires=network.target mysql.service apache2.service -After=network.target mysql.service apache2.service +After=network.target mariadb.service apache2.service Before=phabricator-phd.service [Service] User=<%= @phabricator_user %> Group=<%= @phabricator_user %> RuntimeDirectory=phabricator-aphlict Type=forking Environment="PATH=/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin" ExecStart=<%= @phabricator_basepath -%>/phabricator/bin/aphlict start ExecStop=<%= @phabricator_basepath -%>/phabricator/bin/aphlict stop Restart=always RestartSec=5s [Install] WantedBy=multi-user.target diff --git a/site-modules/profile/templates/phabricator/phabricator-phd.service.erb b/site-modules/profile/templates/phabricator/phabricator-phd.service.erb index 5b8f222c..ba8d625e 100644 --- a/site-modules/profile/templates/phabricator/phabricator-phd.service.erb +++ b/site-modules/profile/templates/phabricator/phabricator-phd.service.erb @@ -1,21 +1,20 @@ # Phabricator Daemon unit file # Managed by puppet class profile::phabricator # Changes will be overwritten [Unit] Description=Phabricator Daemons -Requires=network.target mysql.service apache2.service -After=network.target mysql.service apache2.service +After=network.target mariadb.service apache2.service [Service] User=<%= @phabricator_user %> Group=<%= @phabricator_user %> Type=forking Environment="PATH=/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin" ExecStart=<%= @phabricator_basepath -%>/phabricator/bin/phd start ExecStop=<%= @phabricator_basepath -%>/phabricator/bin/phd stop Restart=always RestartSec=5s [Install] WantedBy=multi-user.target diff --git a/site-modules/profile/templates/postgres/pg_service.conf.erb b/site-modules/profile/templates/postgres/pg_service.conf.erb index 9594fbcc..b9206679 100644 --- a/site-modules/profile/templates/postgres/pg_service.conf.erb +++ b/site-modules/profile/templates/postgres/pg_service.conf.erb @@ -1,6 +1,14 @@ # This file is managed by puppet. Local modifications will be overwritten. -[swh] -dbname=softwareheritage -host=db.internal.softwareheritage.org +<% @dbs.each do |db| -%> +[<%= db['alias'] %>] +dbname=<%= dname=db['name'] %> +host=<%= host=db['host'] %> +port=<%= port=db['port'] %> user=guest +[admin-<%= db['alias'] %>] +dbname=<%= dname=db['name'] %> +host=<%= host=db['host'] %> +port=<%= port=db['port'] %> +user=<%= port=db['user'] %> +<% end %> diff --git a/site-modules/profile/templates/postgres/pgpass.conf.erb b/site-modules/profile/templates/postgres/pgpass.conf.erb new file mode 100644 index 00000000..408e3b01 --- /dev/null +++ b/site-modules/profile/templates/postgres/pgpass.conf.erb @@ -0,0 +1,7 @@ +# This file is managed by puppet. Local modifications will be overwritten. + +#hostname:port:database:username:password +<% @dbs.each do |db| -%> +<%= host=db['host'] %>:<%= db['port'] %>:<%= db['name'] %>:guest:guest +<%= host=db['host'] %>:<%= db['port'] %>:<%= db['name'] %>:<%= db['user'] %>:<%= db['passwd'] %> +<% end %> diff --git a/site-modules/profile/templates/prometheus/node/scripts/apt.erb b/site-modules/profile/templates/prometheus/node/scripts/apt.erb new file mode 100755 index 00000000..9fc8c9ae --- /dev/null +++ b/site-modules/profile/templates/prometheus/node/scripts/apt.erb @@ -0,0 +1,195 @@ +#!/usr/bin/python3 + +from collections import defaultdict, namedtuple +import os +import shutil +import sys +import tempfile + +import apt + + +IS_ROOT = os.getuid() == 0 + +Labels = namedtuple('Labels', ['change_type', 'arch', 'origin']) + + +def update_cache(cache): + """Update the apt cache (equivalent to apt-get update)""" + cache.update() + cache.open(None) + + +def get_package_autoremoves(cache): + """Get the packages that can be autoremoved""" + for package in cache: + if package.is_auto_removable: + yield package + + +def get_package_upgrades(cache, *, dist_upgrade=False): + """Get the packages that need an upgrade. + + If dist_upgrade is true, pretend you're performing a dist-upgrade instead + of a plain upgrade. + + """ + cache.upgrade(dist_upgrade=dist_upgrade) + return cache.get_changes() + + +def origin_id(origin): + """Get an identifier for the given origin + + Filters out missing and/or bogus labels + """ + if origin.label in ('', None, 'stable'): + if not origin.origin: + return origin.site + return origin.origin + return origin.label + + +def pick_origin(origins): + """Pick an origin from the list of origins to set the label""" + origins = list(sorted(origins)) + if len(origins) == 1: + return origins[0] + for origin in origins: + if origin.startswith('Debian'): + return origin + else: + return origins[0] + + +def group_changes(changes): + """Group the changes by type, architecture, origin""" + groups = defaultdict(list) + + for package in changes: + if not package.marked_delete: + if not package.installed: + change_type = 'install' + elif package.candidate > package.installed: + change_type = 'upgrade' + elif package.candidate < package.installed: + change_type = 'downgrade' + else: + change_type = 'keep' + origin_ids = {origin_id(origin) + for origin in package.candidate.origins} + else: + change_type = 'removal' + origin_ids = {''} + architecture = package.architecture() + + if len(origin_ids) != 1: + print( + 'Multiple origins for package %s: %s' % + (package.candidate, ', '.join(origin_ids)), + file=sys.stderr, + ) + + labels = Labels(change_type, architecture, pick_origin(origin_ids)) + groups[labels].append(package) + + return groups + + +def print_prometheus_metric(metric, value, **labels): + """Print the given prometheus metric""" + labels = '' if not labels else '{%s}' % ','.join( + '%s="%s"' % (key, value.replace('"', r'\"')) + for key, value in sorted(labels.items()) + ) + print('%s%s %d' % (metric, labels, value)) + + +def format_upgrades_pending(upgrade_groups, dist_upgrade_groups): + """Format the upgrade groups for consumption by prometheus""" + def print_apt_upgrades(value, **labels): + return print_prometheus_metric('apt_upgrades_pending', value, **labels) + + print('# HELP apt_upgrades_pending Apt package pending upgrades by origin, arch and change type.') + print('# TYPE apt_upgrades_pending gauge') + + if not upgrade_groups and not dist_upgrade_groups: + print_apt_upgrades(0, origin="", arch="", change_type="", upgrade_type="") + return + + for labels, packages in upgrade_groups.items(): + print_apt_upgrades( + len(packages), + upgrade_type='upgrade', + **labels._asdict(), + ) + for labels, packages in dist_upgrade_groups.items(): + value = len(packages) - len(upgrade_groups[labels]) + if not value: + continue + print_apt_upgrades( + value, + upgrade_type='dist-upgrade', + **labels._asdict(), + ) + + +def get_reboot_required(): + """Check if the /run/reboot-required file exists""" + return os.path.exists('/run/reboot-required') + + +if __name__ == '__main__': + (fd, filename) = tempfile.mkstemp( + dir="<%= @textfile_directory %>", + prefix="<%= @script %>.prom.", + ) + try: + sys.stdout = os.fdopen(fd, 'w') + cache = apt.Cache() + if IS_ROOT: + print('# HELP apt_cache_update_failed Whether the apt cache update failed') + try: + update_cache(cache) + except Exception: + print_prometheus_metric('apt_cache_update_failed', 1) + else: + print_prometheus_metric('apt_cache_update_failed', 0) + + upgrades = get_package_upgrades(cache, dist_upgrade=False) + upgrade_groups = group_changes(upgrades) + cache.clear() + upgrades = get_package_upgrades(cache, dist_upgrade=True) + dist_upgrade_groups = group_changes(upgrades) + + format_upgrades_pending(upgrade_groups, dist_upgrade_groups) + + cache.clear() + print() + print( + '# HELP apt_autoremovals_pending Apt packages that can be ' + 'autoremoved.' + ) + print('# TYPE apt_autoremovals_pending gauge') + print_prometheus_metric( + 'apt_autoremovals_pending', + len(list(get_package_autoremoves(cache))) + ) + + print() + print( + '# HELP node_reboot_required Node reboot is required for software ' + 'updates.' + ) + print('# TYPE node_reboot_required gauge') + print_prometheus_metric( + 'node_reboot_required', + int(get_reboot_required()), + ) + sys.stdout.flush() + except Exception: + os.remove(filename) + raise + else: + os.chmod(fd, 0o0644) + shutil.move(filename, "<%= @textfile_directory %>/<%= @script %>.prom") diff --git a/site-modules/profile/templates/prometheus/node/scripts/puppet-classes.erb b/site-modules/profile/templates/prometheus/node/scripts/puppet-classes.erb new file mode 100644 index 00000000..b8359ed6 --- /dev/null +++ b/site-modules/profile/templates/prometheus/node/scripts/puppet-classes.erb @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +outfile="<%= @textfile_directory %>/<%= @script %>.prom" +tmpfile=`mktemp $outfile.XXXXXX` + +cleanup() { + rm -f $tmpfile + exit +} + +trap cleanup INT TERM EXIT + +{ +echo '# HELP puppet_class Puppet classes in use on this host' +echo '# TYPE puppet_class gauge' +sort /var/lib/puppet/state/classes.txt | while read cls; do + echo "puppet_class{class=\"$cls\"} 1" +done +} > $tmpfile + +chmod 644 $tmpfile +mv $tmpfile $outfile diff --git a/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.defaults.erb b/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.defaults.erb index c5cddbec..ce3c9a18 100644 --- a/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.defaults.erb +++ b/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.defaults.erb @@ -1,12 +1,10 @@ +# File managed by puppet (module profile::prometheus::sql) +# Changes will be overwritten. -CONFIG=/etc/prometheus-sql-exporter.yml +CONFIG=<%= @config_file %> PGHOST=/var/run/postgresql PGAPPNAME=prometheus-sql-exporter GOGC=40 -GOMAXPROCS=4 +GOMAXPROCS=1 LOGLEVEL=WARN -ARGS="\ -<%- scope.call_function('flatten_to_argument_list', [@defaults_config]).each do |argument| -%> - <%= argument %> \ -<%- end -%> -" +ARGS="-config.file <%= @config_file %> -web.listen-address <%= @listen_address %>" diff --git a/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.yml.in.erb b/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.yml.in.erb deleted file mode 100644 index 75c3f445..00000000 --- a/site-modules/profile/templates/prometheus/sql/prometheus-sql-exporter.yml.in.erb +++ /dev/null @@ -1,515 +0,0 @@ ---- -jobs: -# These checks are done for every cluster -- name: "cluster" - interval: '0' - connections: - # We have to omit the hostname, we use PGHOST as environment variable. - # See /etc/default/prometheus-sql-exporter - - 'postgres://postgres@/postgres?sslmode=disable' - queries: - - name: "running_queries" - help: "Number of running queries" - labels: - - "datname" - - "usename" - values: - - "count" - query: | - SELECT datname::text, usename::text, COUNT(*)::float AS count - FROM pg_stat_activity - WHERE NOT datname ~ '^template(0|1)$' - GROUP BY datname, usename; - - name: "pg_stat_activity" - help: "Running Backends by Database and state" - labels: - - "datname" - - "state" - values: - - "count" - - "max_tx_duration" - query: | - SELECT - pg_database.datname::text, - tmp.state::text, - COALESCE(count,0) as count, - COALESCE(max_tx_duration,0) as max_tx_duration - FROM - ( - VALUES ('active'), - ('idle'), - ('idle in transaction'), - ('idle in transaction (aborted)'), - ('fastpath function call'), - ('disabled') - ) AS tmp(state) CROSS JOIN pg_database - LEFT JOIN - ( - SELECT - datname, - state, - count(*) AS count, - MAX(EXTRACT(EPOCH FROM now() - xact_start))::float AS max_tx_duration - FROM pg_stat_activity - GROUP BY datname,state) AS tmp2 - ON tmp.state = tmp2.state AND pg_database.datname = tmp2.datname - WHERE NOT pg_database.datname ~ '^template(0|1)$' - - name: "settings" - help: "PostgreSQL settings" - labels: - - "settings" - values: - - "max_connections" - - "autovacuum_freeze_max_age" - - "superuser_reserved_connections" - - "max_wal_senders" - - "max_prepared_transactions" - query: | - SELECT - current_setting('max_connections')::float AS max_connections, - current_setting('autovacuum_freeze_max_age')::float AS autovacuum_freeze_max_age, - current_setting('superuser_reserved_connections')::float AS superuser_reserved_connections, - current_setting('max_wal_senders')::float AS max_wal_senders, - current_setting('max_prepared_transactions')::float AS max_prepared_transactions; - - name: "pg_locks_sum" - help: "number of current locks" - labels: - - "mode" - values: - - "count" - query: | - SELECT t.mode, count(*) - FROM - (VALUES - ('AccessShareLock'), - ('RowShareLock'), - ('RowExclusiveLock'), - ('ShareUpdateExclusiveLock'), - ('ShareLock'), - ('ShareRowExclusiveLock'), - ('ExclusiveLock'), - ('AccessExclusiveLock') - ) t(mode) - FULL JOIN pg_locks l ON t.mode = l.mode GROUP BY 1; - - name: "pg_stat_database" - help: "Database statistics" - type: "counter" - labels: - - "datname" - values: - - "numbackends:count" - - "xact_commit" - - "xact_rollback" - - "blks_read" - - "blks_hit" - - "tup_returned" - - "tup_fetched" - - "tup_inserted" - - "tup_updated" - - "tup_deleted" - - "conflicts" - - "temp_files" - - "temp_bytes" - - "deadlocks" - - "blk_read_time" - - "blk_write_time" - - "freeze_age" - - "dbsize" - query: | - SELECT - s.datname::text, - numbackends::float, - xact_commit::float, - xact_rollback::float, - blks_read::float, - blks_hit::float, - tup_returned::float, - tup_fetched::float, - tup_inserted::float, - tup_updated::float, - tup_deleted::float, - conflicts::float, - temp_files::float, - temp_bytes::float, - deadlocks::float, - blk_read_time, - blk_write_time, - age(d.datfrozenxid) AS freeze_age, - pg_database_size(s.datname)::float AS dbsize - FROM pg_stat_database s - LEFT JOIN pg_database d ON d.datname = s.datname - WHERE NOT s.datname ~ '^template(0|1)$' - - name: "pg_stat_archiver" - help: "pg_stat_archiver" - values: - - "archived_count" - - "last_archived_time" - - "failed_count" - - "last_failed_time" - - "stats_reset" - query: | - SELECT - archived_count, - EXTRACT(EPOCH FROM last_archived_time), - failed_count, - EXTRACT(EPOCH FROM last_failed_time), - EXTRACT(EPOCH FROM stats_reset) - FROM pg_stat_archiver - - name: "checkpoints" - help: "Requested and timed Checkpoints" - values: - - "timed" - - "requested" - query: | - SELECT - pg_stat_get_bgwriter_timed_checkpoints() timed, - pg_stat_get_bgwriter_requested_checkpoints() requested; - - name: "txid" - help: "current txid" - values: - - "txid_current" - query: | - SELECT - txid_current(); - - name: "pg_stat_statements" - help: "pg_stat_statements" - labels: - - "usename" - - "datname" - - "queryid" - - "query" - values: - - "calls" - - "total_time" - - "rows" - - "shared_blks_hit" - - "shared_blks_read" - - "shared_blks_dirtied" - - "shared_blks_writte" - - "local_blks_hit" - - "local_blks_read" - - "local_blks_dirtied" - - "local_blks_writte" - - "temp_blks_read" - - "temp_blks_written" - query: | - WITH w_pg_stat_statements AS ( - SELECT * FROM pg_stat_statements - ) - (SELECT - usename::text - , datname::text - , queryid::text - , substr(regexp_replace(query, E'[\\n\\r]+', ' ', 'g' ),1,1024) AS query - , calls - , total_time - , rows - , shared_blks_hit - , shared_blks_read - , shared_blks_dirtied - , shared_blks_written - , local_blks_hit - , local_blks_read - , local_blks_dirtied - , local_blks_written - , temp_blks_read - , temp_blks_written - FROM w_pg_stat_statements pss - JOIN pg_database pd - ON pss.dbid = pd.oid - JOIN pg_user pu - ON pss.userid = pu.usesysid - ORDER BY pss.total_time DESC - LIMIT 25) - UNION - SELECT - usename::text - , datname::text - , queryid::text - , substr(regexp_replace(query, E'[\\n\\r]+', ' ', 'g' ),1,1024) AS query - , calls - , total_time - , rows - , shared_blks_hit - , shared_blks_read - , shared_blks_dirtied - , shared_blks_written - , local_blks_hit - , local_blks_read - , local_blks_dirtied - , local_blks_written - , temp_blks_read - , temp_blks_written - FROM w_pg_stat_statements pss2 - JOIN pg_database pd2 - ON pss2.dbid = pd2.oid - JOIN pg_user pu2 - ON pss2.userid = pu2.usesysid - ORDER BY calls DESC - LIMIT 25; - - name: "prepared_transactions" - help: "Prepared Transactions" - labels: - - "datname" - values: - - "count" - query: | - SELECT - datname::text, - COUNT(transaction) AS count - FROM pg_database d - LEFT JOIN pg_prepared_xacts x ON d.datname = x.database - WHERE NOT d.datname ~ '^template(0|1)$' - GROUP BY datname - -- name: "cluster_9" - interval: '0' - min_version: "9.0" - max_version: "9.6" - connections: - # We have to omit the hostname, we use PGHOST as environment variable. - # See /etc/default/prometheus-sql-exporter - queries: - - name: "pg_stat_replication" - help: "pg_stat_replication" - labels: - - "application_name" - - "usename" - - "client_addr" - - "client_port" - values: - - "pid" - - "current_xlog_location_bytes" - - "sent_location_bytes" - - "flush_location_bytes" - - "replay_location_bytes" - - "send_lag_bytes" - - "flush_lag_bytes" - - "replay_lag_bytes" - query: | - SELECT - COALESCE(pid, 0) AS pid, - COALESCE(application_name, ' ')::text AS application_name, - COALESCE(usename, ' ')::text AS usename, - COALESCE(client_addr::text, 'local')::text AS client_addr, - COALESCE(client_port::text, ' ') AS client_port, - COALESCE(pg_xlog_location_diff(pg_current_xlog_location(), '0/0'), 0) AS current_xlog_location_bytes, - COALESCE(pg_xlog_location_diff(sent_location, '0/0'), 0) AS sent_location_bytes, - COALESCE(pg_xlog_location_diff(flush_location, '0/0'), 0) AS flush_location_bytes, - COALESCE(pg_xlog_location_diff(replay_location, '0/0'), 0) AS replay_location_bytes, - COALESCE(pg_xlog_location_diff(pg_current_xlog_location(), sent_location), 0) AS send_lag_bytes, - COALESCE(pg_xlog_location_diff(pg_current_xlog_location(), flush_location), 0) AS flush_lag_bytes, - COALESCE(pg_xlog_location_diff(pg_current_xlog_location(), replay_location), 0) AS replay_lag_bytes - FROM pg_stat_replication LEFT JOIN (VALUES(0)) filler(i) ON TRUE - - name: "waldistance" - help: "amount of wal written since creation" - values: - - "location" - query: | - SELECT - pg_xlog_location_diff(pg_current_xlog_location(),'0/0000') "location"; - - name: "LastCheckpointDistance" - help: "Distance to the last checkpoint" - values: - - "distance" - query: | - SELECT - pg_xlog_location_diff( pg_current_xlog_location(), checkpoint_location) "distance" FROM pg_control_checkpoint(); - - name: "archive_ready" - help: "archive_ready" - values: - - "archive_ready" - query: | - SELECT COUNT(1) AS archive_ready FROM pg_ls_dir('./pg_xlog/archive_status') WHERE pg_ls_dir ~ '^[0-9a-fA-F]{24}\.ready$'; - - -- name: "cluster_10" - interval: '0' - min_version: "10" - max_version: "999999" - connections: - # We have to omit the hostname, we use PGHOST as environment variable. - # See /etc/default/prometheus-sql-exporter - queries: - - name: "pg_stat_replication" - help: "pg_stat_replication" - labels: - - "application_name" - - "usename" - - "client_addr" - - "client_port" - values: - - "pid" - - "current_xlog_lsn_bytes" - - "sent_lsn_bytes" - - "flush_lsn_bytes" - - "replay_lsn_bytes" - - "send_lag_bytes" - - "flush_lag_bytes" - - "replay_lag_bytes" - query: | - SELECT - COALESCE(pid, 0) AS pid, - COALESCE(application_name, ' ')::text AS application_name, - COALESCE(usename, ' ')::text AS usename, - COALESCE(client_addr::text, 'local')::text AS client_addr, - COALESCE(client_port::text, ' ') AS client_port, - COALESCE(pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0'), 0) AS current_xlog_lsn_bytes, - COALESCE(pg_wal_lsn_diff(sent_lsn, '0/0'), 0) AS sent_location_bytes, - COALESCE(pg_wal_lsn_diff(flush_lsn, '0/0'), 0) AS flush_location_bytes, - COALESCE(pg_wal_lsn_diff(replay_lsn, '0/0'), 0) AS replay_location_bytes, - COALESCE(pg_wal_lsn_diff(pg_current_wal_lsn(), sent_lsn), 0) AS send_lag_bytes, - COALESCE(pg_wal_lsn_diff(pg_current_wal_lsn(), flush_lsn), 0) AS flush_lag_bytes, - COALESCE(pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), 0) AS replay_lag_bytes - FROM pg_stat_replication FULL JOIN (VALUES(0)) filler(i) ON TRUE - - name: "waldistance" - help: "amount of wal written since creation" - values: - - "location" - query: | - SELECT - pg_wal_lsn_diff(pg_current_wal_lsn(),'0/0000') "location"; - - name: "LastCheckpointDistance" - help: "Distance to the last checkpoint" - values: - - "distance" - query: | - SELECT - pg_wal_lsn_diff( pg_current_wal_lsn(), checkpoint_lsn) "distance" FROM pg_control_checkpoint(); - - name: "archive_ready" - help: "archive_ready" - values: - - "archive_ready" - query: | - SELECT COUNT(1) AS archive_ready FROM pg_ls_dir('./pg_wal/archive_status') WHERE pg_ls_dir ~ '^[0-9a-fA-F]{24}\.ready$'; - -# This checks are done for every database -- name: "database" - interval: '24h' - connections: - # We have to omit the hostname, we use PGHOST as environment variable. - # See /etc/default/prometheus-sql-exporter - - 'postgres://postgres@/postgres?sslmode=disable' - queries: - - name: "CREATE EXTENSION pg_stat_statements" - help: "Creates the needed extension pg_stat_statements and does not produce errors." - query: | - DO $$ - BEGIN - EXECUTE 'CREATE EXTENSION pg_stat_statements'; - EXCEPTION WHEN OTHERS THEN - RETURN; - END; - $$; -- name: "database" - interval: '0' - connections: - # We have to omit the hostname, we use PGHOST as environment variable. - # See /etc/default/prometheus-sql-exporter - - 'postgres://postgres@/postgres?sslmode=disable' - queries: - - name: "pg_stat_user_tables" - help: "Table stats" - labels: - - "datname" - - "schemaname" - - "relname" - values: - - "seq_scan" - - "seq_tup_read" - - "idx_scan" - - "idx_tup_fetch" - - "n_tup_ins" - - "n_tup_upd" - - "n_tup_del" - - "n_tup_hot_upd" - - "n_live_tup" - - "n_dead_tup" - - "vacuum_count" - - "autovacuum_count" - - "analyze_count" - - "autoanalyze_count" - query: | - SELECT - current_database()::text AS datname - , coalesce(schemaname::text, 'null') AS schemaname - , coalesce(relname::text, 'null') AS relname - , coalesce(seq_scan,0)::float AS seq_scan - , coalesce(seq_tup_read,0)::float AS seq_tup_read - , coalesce(idx_scan,0)::float AS idx_scan - , coalesce(idx_tup_fetch,0)::float AS idx_tup_fetch - , coalesce(n_tup_ins,0)::float AS n_tup_ins - , coalesce(n_tup_upd,0)::float AS n_tup_upd - , coalesce(n_tup_del,0)::float AS n_tup_del - , coalesce(n_tup_hot_upd,0)::float AS n_tup_hot_upd - , coalesce(n_live_tup,0)::float AS n_live_tup - , coalesce(n_dead_tup,0)::float AS n_dead_tup - , coalesce(vacuum_count,0)::float AS vacuum_count - , coalesce(autovacuum_count,0)::float AS autovacuum_count - , coalesce(analyze_count,0)::float AS analyze_count - , coalesce(autoanalyze_count,0)::float AS autoanalyze_count - FROM pg_stat_user_tables FULL JOIN (VALUES(0)) filler(i) ON TRUE; - - name: "pg_statio_user_tables" - help: "IO Stats" - labels: - - "datname" - - "schemaname" - - "relname" - values: - - "heap_blks_read" - - "heap_blks_hit" - - "idx_blks_read" - - "idx_blks_hit" - query: | - SELECT - current_database()::text AS datname - , COALESCE(schemaname::text, 'null') AS schemaname - , COALESCE(relname::text, 'null') AS relname - , COALESCE(heap_blks_read::float,0) AS heap_blks_read - , COALESCE(heap_blks_hit::float,0) AS heap_blks_hit - , COALESCE(idx_blks_read::float,0) AS idx_blks_read - , COALESCE(idx_blks_hit::float,0) AS idx_blks_hit - FROM pg_statio_user_tables FULL JOIN (VALUES(0)) filler(i) ON TRUE; - - name: "BufferAccess" - help: "Buffers Accessed" - labels: - - "datname" - - "schemaname" - - "relname" - values: - - "reads" - query: | - SELECT - current_database()::text AS datname, - COALESCE(schemaname::text, 'null') AS schemaname, - COALESCE(relname::text, 'null') AS relname, - sum( coalesce( heap_blks_read,0) - + coalesce(heap_blks_hit,0) - + coalesce(idx_blks_hit,0) - + coalesce(idx_blks_read,0) - + coalesce(toast_blks_hit,0) - + coalesce(toast_blks_read,0) - + coalesce(tidx_blks_hit,0) - + coalesce(tidx_blks_read,0))*8196::bigint as reads - FROM pg_statio_user_tables FULL JOIN (VALUES(0)) filler(i) ON TRUE - GROUP BY 1,2,3; - - name: "Maintenancecounters" - help: "Counters for maintenance jobs on user tables" - labels: - - "datname" - values: - - "vacuum_count" - - "autovacuum_count" - - "analyze_count" - - "autoanalyze_count" - query: | - SELECT - current_database()::text AS datname, - sum( vacuum_count) vacuum_count, - sum( autovacuum_count) autovacuum_count, - sum( analyze_count) analyze_count, - sum( autoanalyze_count) autoanalyze_count - FROM pg_stat_all_tables; -<% if @extra_config -%> -<%= @extra_config.to_yaml().gsub(/^---\n/, '') %> -<%- end -%> diff --git a/site-modules/profile/templates/prometheus/sql/systemd/update_config.conf.erb b/site-modules/profile/templates/prometheus/sql/systemd/update_config.conf.erb new file mode 100644 index 00000000..bfc84a8b --- /dev/null +++ b/site-modules/profile/templates/prometheus/sql/systemd/update_config.conf.erb @@ -0,0 +1,15 @@ +# File managed by puppet (module profile::prometheus::sql) +# Changes will be overwritten. + +[Unit] +# forget default config file location +ConditionPathExists= +# wait for PostgreSQL to start up +After=postgresql.service +# If no PostgreSQL cluster exist, we dont't start. +ConditionPathExistsGlob=/etc/postgresql/*/*/postgresql.conf + +[Service] +# before starting the sql exporter, generate config (as root) +PermissionsStartOnly=true +ExecStartPre=<%= @config_updater %> <%= @config_snippet_dir %> <%= @config_file %> diff --git a/site-modules/profile/templates/prometheus/statsd/prometheus-statsd-exporter.defaults.erb b/site-modules/profile/templates/prometheus/statsd/prometheus-statsd-exporter.defaults.erb new file mode 100644 index 00000000..2f184c18 --- /dev/null +++ b/site-modules/profile/templates/prometheus/statsd/prometheus-statsd-exporter.defaults.erb @@ -0,0 +1,36 @@ +# Set the command-line arguments to pass to the server. +# Due to shell scaping, to pass backslashes for regexes, you need to double +# them (\\d for \d). If running under systemd, you need to double them again +# (\\\\d to mean \d), and escape newlines too. +ARGS="\ +<%- scope.call_function('flatten_to_argument_list', [@defaults_config]).each do |argument| -%> + <%= argument %> \ +<%- end -%> +" + +# Prometheus-statsd-exporter supports the following options: +# --web.listen-address=":9102" +# The address on which to expose the web interface and generated Prometheus +# metrics. +# --web.telemetry-path="/metrics" +# Path under which to expose metrics. +# --statsd.listen-udp=":9125" +# The UDP address on which to receive statsd metric lines. "" disables it. +# --statsd.listen-tcp=":9125" +# The TCP address on which to receive statsd metric lines. "" disables it. +# --statsd.mapping-config=STATSD.MAPPING-CONFIG +# Metric mapping configuration file name. +# --statsd.read-buffer=STATSD.READ-BUFFER +# Size (in bytes) of the operating system's transmit read buffer +# associated with the UDP connection. Please make sure the kernel +# parameters net.core.rmem_max is set to a value greater than the +# value specified. +# --debug.dump-fsm="" +# The path to dump internal FSM generated for glob matching as Dot file. +# --log.level="info" +# Only log messages with the given severity or above. Valid levels: [debug, +# info, warn, error, fatal] +# --log.format="logger:stderr" +# Set the log target and format. Example: "logger:syslog?appname=bob&local=7" +# or "logger:stdout?json=true" + diff --git a/site-modules/profile/templates/puppet/swh-puppet-master-clean-certificate.sh.erb b/site-modules/profile/templates/puppet/swh-puppet-master-clean-certificate.sh.erb new file mode 100644 index 00000000..64c0bb73 --- /dev/null +++ b/site-modules/profile/templates/puppet/swh-puppet-master-clean-certificate.sh.erb @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Use: +# $0 CERTNAME + +# Example: +# $0 storage0.internal.staging.swh.network + +set -x + +CERTNAME=$1 +puppet node deactivate $CERTNAME +puppet cert clean $CERTNAME +systemctl restart apache2 diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb new file mode 100644 index 00000000..146035c0 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb @@ -0,0 +1,18 @@ +# Indexer Journal Client unit file +# Managed by puppet class profile::swh::deploy::indexer_journal_client +# Changes will be overwritten + +[Unit] +Description=Software Heritage Indexer Journal Client +After=network.target + +[Service] +User=<%= @user %> +Group=<%= @group %> +Type=simple +ExecStart=/usr/bin/swh indexer --config-file <%= @config_path %> journal-client +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-journal-publisher.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-journal-publisher.service.erb deleted file mode 100644 index f7d72b3c..00000000 --- a/site-modules/profile/templates/swh/deploy/journal/swh-journal-publisher.service.erb +++ /dev/null @@ -1,19 +0,0 @@ -# Journal Publisher unit file -# Managed by puppet class profile::swh::deploy::journal_publisher -# Changes will be overwritten - -[Unit] -Description=Software Heritage Journal Publisher -Requires=network.target kafka.service -After=network.target kafka.service - -[Service] -User=<%= @user %> -Group=<%= @group %> -Type=simple -ExecStart=/usr/bin/python3 -m swh.journal.publisher -Restart=always -RestartSec=5s - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/scheduler.ini.erb b/site-modules/profile/templates/swh/deploy/scheduler/scheduler.ini.erb deleted file mode 100644 index 90af16e7..00000000 --- a/site-modules/profile/templates/swh/deploy/scheduler/scheduler.ini.erb +++ /dev/null @@ -1,6 +0,0 @@ -# swh.scheduler configuration file -# File managed by puppet - modifications will be overwritten - -[main] -scheduling_db = <%= @database %> - diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb index 86b56d81..489dde2d 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb +++ b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-listener.service.erb @@ -1,19 +1,20 @@ # Scheduler Listener unit file # Managed by puppet class profile::swh::deploy::scheduler # Changes will be overwritten [Unit] Description=Software Heritage scheduler listener Requires=network.target rabbitmq-server.service After=network.target rabbitmq-server.service [Service] User=<%= @user %> Group=<%= @group %> Type=simple -ExecStart=/usr/bin/python3 -m swh.scheduler.celery_backend.listener +Environment=SWH_LOG_TARGET=journal +ExecStart=/usr/bin/swh --log-level <%= @listener_log_level %> scheduler --config-file <%= @config_file %> listener Restart=always RestartSec=10 [Install] WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb index cc6ec84a..9ae08542 100644 --- a/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb +++ b/site-modules/profile/templates/swh/deploy/scheduler/swh-scheduler-runner.service.erb @@ -1,19 +1,20 @@ # Scheduler Runner unit file # Managed by puppet class profile::swh::deploy::scheduler # Changes will be overwritten [Unit] Description=Software Heritage scheduler runner Requires=network.target rabbitmq-server.service After=network.target rabbitmq-server.service [Service] User=<%= @user %> Group=<%= @group %> Type=simple -ExecStart=/bin/sh -c 'while true; do echo running pending tasks at `/bin/date`...; /usr/bin/python3 -m swh.scheduler.celery_backend.runner; sleep 10; done' +Environment=SWH_LOG_TARGET=journal +ExecStart=/usr/bin/swh --log-level <%= @runner_log_level %> scheduler --config-file <%= @config_file %> runner --period 10 Restart=always RestartSec=10 [Install] WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/storage_listener/listener.ini.erb b/site-modules/profile/templates/swh/deploy/storage_listener/listener.ini.erb deleted file mode 100644 index 6ecc5158..00000000 --- a/site-modules/profile/templates/swh/deploy/storage_listener/listener.ini.erb +++ /dev/null @@ -1,9 +0,0 @@ -# swh.storage.listener configuration file -# File managed by puppet - modifications will be overwritten - -[main] -database = <%= @database %> -brokers = <%= @kafka_brokers.join(',') %> -topic_prefix = <%= @topic_prefix %> -poll_timeout = <%= @poll_timeout %> - diff --git a/site-modules/profile/templates/swh/deploy/storage_listener/swh-storage-listener.service.erb b/site-modules/profile/templates/swh/deploy/storage_listener/swh-storage-listener.service.erb deleted file mode 100644 index 5695fe62..00000000 --- a/site-modules/profile/templates/swh/deploy/storage_listener/swh-storage-listener.service.erb +++ /dev/null @@ -1,19 +0,0 @@ -# Storage Listener unit file -# Managed by puppet class profile::swh::deploy::storage_listener -# Changes will be overwritten - -[Unit] -Description=Software Heritage storage listener -Requires=network.target kafka.service -After=network.target kafka.service - -[Service] -User=<%= @user %> -Group=<%= @group %> -Type=simple -ExecStart=/usr/bin/python3 -m swh.storage.listener -Restart=always -RestartSec=5s - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/worker/instance_config.ini.erb b/site-modules/profile/templates/swh/deploy/worker/instance_config.ini.erb deleted file mode 100644 index 8539ab78..00000000 --- a/site-modules/profile/templates/swh/deploy/worker/instance_config.ini.erb +++ /dev/null @@ -1,8 +0,0 @@ -# Managed by puppet - modifications will be overwritten -# In defined class profile::swh::deploy::worker::instance - -[main] -task_broker = <%= @task_broker %> -task_modules = <%= @task_modules.join(', ') %> -task_queues = <%= @task_queues.join(', ') %> -task_soft_time_limit = <%= @task_soft_time_limit %> diff --git a/site-modules/profile/templates/swh/pg_cluster_backup.sh.erb b/site-modules/profile/templates/swh/pg_cluster_backup.sh.erb new file mode 100755 index 00000000..d3190ac5 --- /dev/null +++ b/site-modules/profile/templates/swh/pg_cluster_backup.sh.erb @@ -0,0 +1,26 @@ +#!/bin/sh + +TARGET_DIR=/srv/storage/space/postgres_secondary + +# banco has many *slow* CPU cores +# A full backup of the secondary cluster takes ~= 40 minutes with p=3 and c=1 +PARALLELISM=3 +COMPRESSION=1 + +DATE=`/bin/date +%Y-%m-%d` + +export PGHOST=belvedere.internal.softwareheritage.org +export PGPORT=5434 +export PGUSER=<%= @pg_backup_user %> +export PGPASSWORD=<%= @pg_backup_password %> + +DB_LIST=`/usr/bin/psql template1 -t -c \ + "SELECT datname from pg_database WHERE datname NOT IN ('postgres', 'template0', 'template1');"` + +cd ${TARGET_DIR} + +for database in ${DB_LIST} +do + echo "dumping database ${database}" + /usr/bin/pg_dump -Fd -j ${PARALLELISM} -Z ${COMPRESSION} -f pgdump-${DATE}.${database} -d ${database} -w +done diff --git a/site-modules/profile/templates/swh/rsnapshot.conf.erb b/site-modules/profile/templates/swh/rsnapshot.conf.erb new file mode 100644 index 00000000..e06c8e6e --- /dev/null +++ b/site-modules/profile/templates/swh/rsnapshot.conf.erb @@ -0,0 +1,259 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULE: # +# # +# This file requires tabs between elements # +# # +################################################# + +####################### +# CONFIG FILE VERSION # +####################### + +config_version 1.2 + +########################### +# SNAPSHOT ROOT DIRECTORY # +########################### + +# All snapshots will be stored under this root directory. +# +snapshot_root /srv/rsnapshot/ + +# If no_create_root is enabled, rsnapshot will not automatically create the +# snapshot_root directory. This is particularly useful if you are backing +# up to removable media, such as a FireWire or USB drive. +# +#no_create_root 1 + +################################# +# EXTERNAL PROGRAM DEPENDENCIES # +################################# + +# LINUX USERS: Be sure to uncomment "cmd_cp". This gives you extra features. +# EVERYONE ELSE: Leave "cmd_cp" commented out for compatibility. +# +# See the README file or the man page for more details. +# +cmd_cp /bin/cp + +# uncomment this to use the rm program instead of the built-in perl routine. +# +cmd_rm /bin/rm + +# rsync must be enabled for anything to work. This is the only command that +# must be enabled. +# +cmd_rsync /usr/bin/rsync + +# Uncomment this to enable remote ssh backups over rsync. +# +cmd_ssh /usr/bin/ssh + +# Comment this out to disable syslog support. +# +cmd_logger /usr/bin/logger + +# Uncomment this to specify the path to "du" for disk usage checks. +# If you have an older version of "du", you may also want to check the +# "du_args" parameter below. +# +#cmd_du /usr/bin/du + +# Uncomment this to specify the path to rsnapshot-diff. +# +#cmd_rsnapshot_diff /usr/bin/rsnapshot-diff + +# Specify the path to a script (and any optional arguments) to run right +# before rsnapshot syncs files +# +#cmd_preexec /path/to/preexec/script + +# Specify the path to a script (and any optional arguments) to run right +# after rsnapshot syncs files +# +#cmd_postexec /path/to/postexec/script + +# Paths to lvcreate, lvremove, mount and umount commands, for use with +# Linux LVMs. +# +#linux_lvm_cmd_lvcreate /sbin/lvcreate +#linux_lvm_cmd_lvremove /sbin/lvremove +#linux_lvm_cmd_mount /bin/mount +#linux_lvm_cmd_umount /bin/umount + +######################################### +# BACKUP LEVELS / INTERVALS # +# Must be unique and in ascending order # +# e.g. alpha, beta, gamma, etc. # +######################################### + +retain hourly 6 +retain daily 7 +retain weekly 4 +retain monthly 12 + +############################################ +# GLOBAL OPTIONS # +# All are optional, with sensible defaults # +############################################ + +# Verbose level, 1 through 5. +# 1 Quiet Print fatal errors only +# 2 Default Print errors and warnings only +# 3 Verbose Show equivalent shell commands being executed +# 4 Extra Verbose Show extra verbose information +# 5 Debug mode Everything +# +verbose 2 + +# Same as "verbose" above, but controls the amount of data sent to the +# logfile, if one is being used. The default is 3. +# If you want the rsync output, you have to set it to 4 +# +loglevel 3 + +# If you enable this, data will be written to the file you specify. The +# amount of data written is controlled by the "loglevel" parameter. +# +logfile /var/log/rsnapshot.log + +# If enabled, rsnapshot will write a lockfile to prevent two instances +# from running simultaneously (and messing up the snapshot_root). +# If you enable this, make sure the lockfile directory is not world +# writable. Otherwise anyone can prevent the program from running. +# +lockfile /var/run/rsnapshot.pid + +# By default, rsnapshot check lockfile, check if PID is running +# and if not, consider lockfile as stale, then start +# Enabling this stop rsnapshot if PID in lockfile is not running +# +#stop_on_stale_lockfile 0 + +# Default rsync args. All rsync commands have at least these options set. +# +rsync_short_args -aH +#rsync_long_args --delete --numeric-ids --relative --delete-excluded + +# ssh has no args passed by default, but you can specify some here. +# +#ssh_args -p 22 + +# Default arguments for the "du" program (for disk space reporting). +# The GNU version of "du" is preferred. See the man page for more details. +# If your version of "du" doesn't support the -h flag, try -k flag instead. +# +#du_args -csh + +# If this is enabled, rsync won't span filesystem partitions within a +# backup point. This essentially passes the -x option to rsync. +# The default is 0 (off). +# +#one_fs 0 + +# The include and exclude parameters, if enabled, simply get passed directly +# to rsync. If you have multiple include/exclude patterns, put each one on a +# separate line. Please look up the --include and --exclude options in the +# rsync man page for more details on how to specify file name patterns. +# +#include ??? +#include ??? +#exclude ??? +<% @backup_exclusion.each do |item| + # Directly using item here will modify @backup_exclusion + # and potentially impact other templates + str = String.new(item.to_s) + str.insert(0,'/') if (item.chars.first != '/') + str << '/' if (item.chars.last != '/') +%> +exclude <%= str -%> +<% end %> + +# The include_file and exclude_file parameters, if enabled, simply get +# passed directly to rsync. Please look up the --include-from and +# --exclude-from options in the rsync man page for more details. +# +#include_file /path/to/include/file +#exclude_file /path/to/exclude/file + +# If your version of rsync supports --link-dest, consider enabling this. +# This is the best way to support special files (FIFOs, etc) cross-platform. +# The default is 0 (off). +# +#link_dest 0 + +# When sync_first is enabled, it changes the default behaviour of rsnapshot. +# Normally, when rsnapshot is called with its lowest interval +# (i.e.: "rsnapshot alpha"), it will sync files AND rotate the lowest +# intervals. With sync_first enabled, "rsnapshot sync" handles the file sync, +# and all interval calls simply rotate files. See the man page for more +# details. The default is 0 (off). +# +#sync_first 0 + +# If enabled, rsnapshot will move the oldest directory for each interval +# to [interval_name].delete, then it will remove the lockfile and delete +# that directory just before it exits. The default is 0 (off). +# +#use_lazy_deletes 0 + +# Number of rsync re-tries. If you experience any network problems or +# network card issues that tend to cause ssh to fail with errors like +# "Corrupted MAC on input", for example, set this to a non-zero value +# to have the rsync operation re-tried. +# +#rsync_numtries 0 + +# LVM parameters. Used to backup with creating lvm snapshot before backup +# and removing it after. This should ensure consistency of data in some special +# cases +# +# LVM snapshot(s) size (lvcreate --size option). +# +#linux_lvm_snapshotsize 100M + +# Name to be used when creating the LVM logical volume snapshot(s). +# +#linux_lvm_snapshotname rsnapshot + +# Path to the LVM Volume Groups. +# +#linux_lvm_vgpath /dev + +# Mount point to use to temporarily mount the snapshot(s). +# +#linux_lvm_mountpath /path/to/mount/lvm/snapshot/during/backup + +############################### +### BACKUP POINTS / SCRIPTS ### +############################### + +# LOCALHOST +backup /etc/ banco.softwareheritage.org/ +backup /home/ banco.softwareheritage.org/ +backup /root/ banco.softwareheritage.org/ +#backup /var/log/rsnapshot localhost/ +#backup /etc/passwd localhost/ +#backup /home/foo/My Documents/ localhost/ +#backup /foo/bar/ localhost/ one_fs=1, rsync_short_args=-urltvpog +#backup_script /usr/local/bin/backup_pgsql.sh localhost/postgres/ +# You must set linux_lvm_* parameters below before using lvm snapshots +#backup lvm://vg0/xen-home/ lvm-vg0/xen-home/ + +# EXAMPLE.COM +#backup_exec /bin/date "+ backup of example.com started at %c" +#backup root@example.com:/home/ example.com/ +rsync_long_args=--bwlimit=16,exclude=core +#backup root@example.com:/etc/ example.com/ exclude=mtab,exclude=core +#backup_exec ssh root@example.com "mysqldump -A > /var/db/dump/mysql.sql" +#backup root@example.com:/var/db/dump/ example.com/ +#backup_exec /bin/date "+ backup of example.com ended at %c" + +# CVS.SOURCEFORGE.NET +#backup_script /usr/local/bin/backup_rsnapshot_cvsroot.sh rsnapshot.cvs.sourceforge.net/ + +# RSYNC.SAMBA.ORG +#backup rsync://rsync.samba.org/rsyncftp/ rsync.samba.org/rsyncftp/ + diff --git a/site-modules/profile/templates/unbound/forwarders.conf.erb b/site-modules/profile/templates/unbound/forwarders.conf.erb index 206c5761..b76096ae 100644 --- a/site-modules/profile/templates/unbound/forwarders.conf.erb +++ b/site-modules/profile/templates/unbound/forwarders.conf.erb @@ -1,15 +1,17 @@ # File managed by puppet (class profile::unbound), all changes will be lost +<% if not @forwarders.empty? -%> forward-zone: name: "." <% @forwarders.each do |forwarder| -%> forward-addr: <%= forwarder %> <% end -%> +<% end -%> <% @forward_zones.each do |zone, forwarders| -%> forward-zone: name: "<%= zone %>" <% forwarders.each do |forwarder| -%> forward-addr: <%= forwarder %> <% end -%> <% end -%> diff --git a/site-modules/role/manifests/postgresql_backup.pp b/site-modules/role/manifests/postgresql_backup.pp new file mode 100644 index 00000000..4a72ebbf --- /dev/null +++ b/site-modules/role/manifests/postgresql_backup.pp @@ -0,0 +1,3 @@ +class role::postgresql_backup { + include profile::postgresql::backup +} diff --git a/site-modules/role/manifests/postgresql_client.pp b/site-modules/role/manifests/postgresql_client.pp new file mode 100644 index 00000000..0f5ed846 --- /dev/null +++ b/site-modules/role/manifests/postgresql_client.pp @@ -0,0 +1,3 @@ +class role::postgresql_client { + include profile::postgresql::client +} diff --git a/site-modules/role/manifests/swh_apache_log_archiver.pp b/site-modules/role/manifests/swh_apache_log_archiver.pp deleted file mode 100644 index 79369ec0..00000000 --- a/site-modules/role/manifests/swh_apache_log_archiver.pp +++ /dev/null @@ -1,4 +0,0 @@ -class role::swh_apache_log_archiver inherits role::swh_base { - include profile::puppet::agent - include profile::filebeat -} diff --git a/site-modules/role/manifests/swh_api.pp b/site-modules/role/manifests/swh_api.pp index e8ea13c8..eb2a9341 100644 --- a/site-modules/role/manifests/swh_api.pp +++ b/site-modules/role/manifests/swh_api.pp @@ -1,10 +1,7 @@ -class role::swh_api inherits role::swh_server { +class role::swh_api inherits role::swh_base_api { include profile::network - include profile::puppet::agent - # Web UI - include profile::memcached - include profile::swh::deploy::storage - include profile::swh::deploy::webapp + # Extra deposit and storage services include profile::swh::deploy::deposit + include profile::swh::deploy::storage } diff --git a/site-modules/role/manifests/swh_api_azure.pp b/site-modules/role/manifests/swh_api_azure.pp deleted file mode 100644 index a0735a5a..00000000 --- a/site-modules/role/manifests/swh_api_azure.pp +++ /dev/null @@ -1,12 +0,0 @@ -# expansion of role::swh_api -# -network: incompatible with Azure infrastructure -# -deposit: not need for it - -class role::swh_api_azure inherits role::swh_server { - include profile::puppet::agent - - # Web UI - include profile::memcached - include profile::swh::deploy::storage - include profile::swh::deploy::webapp -} diff --git a/site-modules/role/manifests/swh_backup.pp b/site-modules/role/manifests/swh_backup.pp index d8950e78..2262e553 100644 --- a/site-modules/role/manifests/swh_backup.pp +++ b/site-modules/role/manifests/swh_backup.pp @@ -1,5 +1,7 @@ class role::swh_backup inherits role::swh_server { include profile::puppet::agent include profile::swh::deploy::objstorage include profile::swh::deploy::objstorage_ceph + include profile::megacli + include profile::rsnapshot::master } diff --git a/site-modules/role/manifests/swh_base.pp b/site-modules/role/manifests/swh_base.pp index 3ade0ce5..fd2d6354 100644 --- a/site-modules/role/manifests/swh_base.pp +++ b/site-modules/role/manifests/swh_base.pp @@ -1,14 +1,15 @@ class role::swh_base { include profile::base include profile::ssh::server include profile::unbound include profile::systemd_journal include profile::resolv_conf include profile::munin::node include profile::prometheus::node + include profile::prometheus::statsd include profile::icinga2 include profile::rsyslog include profile::swh } diff --git a/site-modules/role/manifests/swh_base_api.pp b/site-modules/role/manifests/swh_base_api.pp new file mode 100644 index 00000000..91bfb93b --- /dev/null +++ b/site-modules/role/manifests/swh_base_api.pp @@ -0,0 +1,10 @@ +class role::swh_base_api inherits role::swh_server { + include profile::puppet::agent + + # Web UI + include profile::memcached + include profile::swh::deploy::webapp + + # Apache logs + include profile::filebeat +} diff --git a/site-modules/role/manifests/swh_base_database.pp b/site-modules/role/manifests/swh_base_database.pp new file mode 100644 index 00000000..6f5f0c4c --- /dev/null +++ b/site-modules/role/manifests/swh_base_database.pp @@ -0,0 +1,4 @@ +class role::swh_base_database inherits role::swh_server { + include profile::puppet::agent + include profile::prometheus::sql +} diff --git a/site-modules/role/manifests/swh_base_storage.pp b/site-modules/role/manifests/swh_base_storage.pp new file mode 100644 index 00000000..40a53b46 --- /dev/null +++ b/site-modules/role/manifests/swh_base_storage.pp @@ -0,0 +1,6 @@ +class role::swh_base_storage inherits role::swh_server { + include profile::puppet::agent + include profile::swh::deploy::storage + include profile::swh::deploy::indexer_storage + include profile::swh::deploy::objstorage +} diff --git a/site-modules/role/manifests/swh_ci_agent.pp b/site-modules/role/manifests/swh_ci_agent.pp new file mode 100644 index 00000000..ac960c81 --- /dev/null +++ b/site-modules/role/manifests/swh_ci_agent.pp @@ -0,0 +1,3 @@ +class role::swh_ci_agent inherits role::swh_ci { + include profile::jenkins::agent +} diff --git a/site-modules/role/manifests/swh_ci_agent_debian.pp b/site-modules/role/manifests/swh_ci_agent_debian.pp new file mode 100644 index 00000000..3253808e --- /dev/null +++ b/site-modules/role/manifests/swh_ci_agent_debian.pp @@ -0,0 +1,3 @@ +class role::swh_ci_agent_debian inherits role::swh_ci_agent { + include profile::jenkins::agent::sbuild +} diff --git a/site-modules/role/manifests/swh_ci_server.pp b/site-modules/role/manifests/swh_ci_server.pp index 947002c9..6592ed49 100644 --- a/site-modules/role/manifests/swh_ci_server.pp +++ b/site-modules/role/manifests/swh_ci_server.pp @@ -1,9 +1,7 @@ class role::swh_ci_server inherits role::swh_ci { # Restore backups for ci server include profile::dar::client include profile::jenkins::server - - # single node setup for now - include profile::jenkins::worker + include profile::jenkins::agent::docker } diff --git a/site-modules/role/manifests/swh_database.pp b/site-modules/role/manifests/swh_database.pp index 4d9945ab..23738995 100644 --- a/site-modules/role/manifests/swh_database.pp +++ b/site-modules/role/manifests/swh_database.pp @@ -1,8 +1,5 @@ -class role::swh_database inherits role::swh_server { - include profile::puppet::agent - - include profile::prometheus::sql - +class role::swh_database inherits role::swh_base_database { include profile::munin::plugins::postgresql include profile::postgresql + include profile::megacli } diff --git a/site-modules/role/manifests/swh_deposit.pp b/site-modules/role/manifests/swh_deposit.pp new file mode 100644 index 00000000..52572cb0 --- /dev/null +++ b/site-modules/role/manifests/swh_deposit.pp @@ -0,0 +1,10 @@ +class role::swh_deposit inherits role::swh_server { + include profile::puppet::agent + include profile::network + + # Web UI + include profile::swh::deploy::deposit + + # Apache logs + include profile::filebeat +} diff --git a/site-modules/role/manifests/swh_elasticsearch.pp b/site-modules/role/manifests/swh_elasticsearch.pp index 5eea8645..b6d687fe 100644 --- a/site-modules/role/manifests/swh_elasticsearch.pp +++ b/site-modules/role/manifests/swh_elasticsearch.pp @@ -1,4 +1,6 @@ class role::swh_elasticsearch inherits role::swh_base { include profile::puppet::agent include profile::elasticsearch + + include profile::kafka::broker } diff --git a/site-modules/role/manifests/swh_eventlog.pp b/site-modules/role/manifests/swh_eventlog.pp index 86a94b9c..b118bd9d 100644 --- a/site-modules/role/manifests/swh_eventlog.pp +++ b/site-modules/role/manifests/swh_eventlog.pp @@ -1,6 +1,6 @@ -class role::swh_eventlog inherits role::swh_server { +class role::swh_eventlog inherits role::swh_base { include profile::puppet::agent include profile::kafka::broker - include profile::swh::deploy::storage_listener + include profile::swh::deploy::indexer_journal_client } diff --git a/site-modules/role/manifests/swh_gateway.pp b/site-modules/role/manifests/swh_gateway.pp new file mode 100644 index 00000000..0f8a9097 --- /dev/null +++ b/site-modules/role/manifests/swh_gateway.pp @@ -0,0 +1,4 @@ +class role::swh_gateway inherits role::swh_base { + include profile::network + include profile::puppet::agent +} diff --git a/site-modules/role/manifests/swh_hypervisor.pp b/site-modules/role/manifests/swh_hypervisor.pp index 372698b0..797366be 100644 --- a/site-modules/role/manifests/swh_hypervisor.pp +++ b/site-modules/role/manifests/swh_hypervisor.pp @@ -1,3 +1,4 @@ class role::swh_hypervisor inherits role::swh_server { include profile::puppet::agent + include profile::megacli } diff --git a/site-modules/role/manifests/swh_hypervisor_master.pp b/site-modules/role/manifests/swh_hypervisor_master.pp deleted file mode 100644 index 37278c08..00000000 --- a/site-modules/role/manifests/swh_hypervisor_master.pp +++ /dev/null @@ -1,3 +0,0 @@ -class role::swh_hypervisor_master inherits role::swh_hypervisor { - include profile::dar::server -} diff --git a/site-modules/role/manifests/swh_lsi_storage_adapter.pp b/site-modules/role/manifests/swh_lsi_storage_adapter.pp deleted file mode 100644 index 8a3083b5..00000000 --- a/site-modules/role/manifests/swh_lsi_storage_adapter.pp +++ /dev/null @@ -1,4 +0,0 @@ -class role::swh_lsi_storage_adapter inherits role::swh_base { - include profile::puppet::agent - include profile::megacli -} diff --git a/site-modules/role/manifests/swh_munin_master.pp b/site-modules/role/manifests/swh_munin_master.pp index a5169ad4..cf9f33f5 100644 --- a/site-modules/role/manifests/swh_munin_master.pp +++ b/site-modules/role/manifests/swh_munin_master.pp @@ -1,4 +1,4 @@ class role::swh_munin_master inherits role::swh_server { + include profile::puppet::agent include profile::munin::master - include profile::munin::stats_export } diff --git a/site-modules/role/manifests/swh_storage.pp b/site-modules/role/manifests/swh_storage.pp index bd83a6ce..e93addb6 100644 --- a/site-modules/role/manifests/swh_storage.pp +++ b/site-modules/role/manifests/swh_storage.pp @@ -1,9 +1,3 @@ -class role::swh_storage inherits role::swh_server { - include profile::puppet::agent - include profile::swh::deploy::storage - include profile::swh::deploy::indexer_storage - include profile::swh::deploy::objstorage - include profile::swh::deploy::worker - include profile::swh::deploy::objstorage_cloud +class role::swh_storage inherits role::swh_base_storage { include profile::swh::deploy::objstorage_ceph } diff --git a/site-modules/role/manifests/swh_storage_baremetal.pp b/site-modules/role/manifests/swh_storage_baremetal.pp new file mode 100644 index 00000000..59164b4b --- /dev/null +++ b/site-modules/role/manifests/swh_storage_baremetal.pp @@ -0,0 +1,4 @@ +class role::swh_storage_baremetal inherits role::swh_storage { + include profile::dar::server + include profile::megacli +} diff --git a/site-modules/role/manifests/swh_sysadmin.pp b/site-modules/role/manifests/swh_sysadmin.pp index 98c78a97..b3f2caa9 100644 --- a/site-modules/role/manifests/swh_sysadmin.pp +++ b/site-modules/role/manifests/swh_sysadmin.pp @@ -1,22 +1,26 @@ class role::swh_sysadmin inherits role::swh_server { include profile::network include profile::prometheus::server include profile::grafana include profile::prometheus::sql include profile::puppet::master + include profile::letsencrypt include profile::icinga2::icingaweb2 include profile::apache::simple_server include ::apache::mod::rewrite include profile::bind_server::primary include profile::munin::plugins::postgresql include profile::annex_web + include profile::stats_web include profile::docs_web include profile::debian_repository + + include profile::weekly_report_bot } diff --git a/site-modules/role/manifests/swh_vault.pp b/site-modules/role/manifests/swh_vault.pp index 811152d9..41c9c7e7 100644 --- a/site-modules/role/manifests/swh_vault.pp +++ b/site-modules/role/manifests/swh_vault.pp @@ -1,9 +1,4 @@ class role::swh_vault inherits role::swh_server { include profile::puppet::agent include profile::swh::deploy::vault - - include profile::munin::plugins::postgresql - include profile::postgresql - - include profile::swh::deploy::objstorage }