diff --git a/data/deployments/admin/common.yaml b/data/deployments/admin/common.yaml index 0f002480..5e2e9f02 100644 --- a/data/deployments/admin/common.yaml +++ b/data/deployments/admin/common.yaml @@ -1,18 +1,31 @@ swh::deploy::environment: admin swh::deploy::reverse_proxy::services: - hedgedoc +swh::postgresql::listen_addresses: + - 0.0.0.0 +swh::postgresql::network_accesses: + - 192.168.100.0/24 # Monitoring + swh::postgresql::shared_buffers: 4GB +swh::postgresql::port: 5432 +swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" +swh::postgresql::datadir_base: "/srv/postgresql" +swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" + +hedgedoc::db::database: hedgedoc +hedgedoc::db::username: hedgedoc +# swh::deploy::hedgedoc::db::password: in private-data swh::deploy::hedgedoc::reverse_proxy::backend_http_host: bardo.internal.admin.swh.network swh::deploy::hedgedoc::reverse_proxy::backend_http_port: "3000" swh::deploy::hedgedoc::reverse_proxy::websocket_support: true swh::deploy::hedgedoc::base_url: hedgedoc.softwareheritage.org swh::deploy::hedgedoc::vhost::letsencrypt_cert: hedgedoc hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 diff --git a/data/hostname/bardo.internal.admin.swh.network.yaml b/data/hostname/bardo.internal.admin.swh.network.yaml index 3630ecfa..d25c4ef3 100644 --- a/data/hostname/bardo.internal.admin.swh.network.yaml +++ b/data/hostname/bardo.internal.admin.swh.network.yaml @@ -1,46 +1,41 @@ hedgedoc::db::host: localhost -hedgedoc::db::database: hedgedoc -hedgedoc::db::username: hedgedoc -# hedgedoc::db::password: in private-data swh::postgresql::version: '12' swh::postgresql::port: 5433 -swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" + swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres" swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" -swh::postgresql::listen_addresses: - - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services -postgresql::server::config_entries: - shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" - cluster_name: "%{alias('swh::postgresql::cluster_name')}" - swh::dbs: hedgedoc: name: "%{alias('hedgedoc::db::database')}" user: "%{alias('hedgedoc::db::username')}" hedgedoc::release::version: 1.9.2 hedgedoc::release::digest: 052088a634731e0f9c28e40f9869281f24bf3fbb25173a341ba2c94496109f51 hedgedoc::release::digest_type: sha256 hedgedoc::allow_anonymous: true hedgedoc::allow_anonymous_edits: true # authentication hedgedoc::allow_email: true hedgedoc::allow_email_register: false hedgedoc::enable_keycloak: true hedgedoc::keycloak::provider_name: Software Heritage hedgedoc::keycloak::domain: auth.softwareheritage.org hedgedoc::keycloak::realm: SoftwareHeritage hedgedoc::keycloak::client::id: hedgedoc # hedgedoc::keycloak::client::secret in private-data hedgedoc::runtime_environment: production hedgedoc::log_level: info + +postgresql::server::config_entries: + shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" + cluster_name: "%{alias('swh::postgresql::cluster_name')}" diff --git a/data/hostname/dali.internal.admin.swh.network.yaml b/data/hostname/dali.internal.admin.swh.network.yaml new file mode 100644 index 00000000..1d03575f --- /dev/null +++ b/data/hostname/dali.internal.admin.swh.network.yaml @@ -0,0 +1,23 @@ +swh::postgresql::version: '14' +swh::postgresql::shared_buffers: 8GB + +swh::dbs: + netbox: + name: "%{alias('netbox::db::database')}" + user: "%{alias('netbox::db::username')}" + password: "%{alias('netbox::db::password')}" + hedgedoc: + name: "%{alias('hedgedoc::db::database')}" + user: "%{alias('hedgedoc::db::username')}" + grafana: + name: "%{alias('grafana::db::username')}" + user: "%{alias('grafana::db::username')}" + password: "%{alias('grafana::db::password')}" + sentry: + name: "%{alias('sentry::postgres::dbname')}" + user: "%{alias('sentry::postgres::user')}" + password: "%{alias('sentry::postgres::password')}" + keycloak: + name: "%{alias('keycloak::postgres::dbname')}" + user: "%{alias('keycloak::postgres::user')}" + password: "%{alias('keycloak::postgres::password')}" diff --git a/manifests/site.pp b/manifests/site.pp index 3435f022..4d01fdfa 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,212 +1,216 @@ node /^(pompidou|uffizi)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor } node /^(beaubourg|hypervisor\d+|branly)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor_with_ceph } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_rp_webapps } node 'webapp1.internal.softwareheritage.org' { include role::swh_rp_webapp } node /^search-esnode\d\.internal\.softwareheritage\.org$/ { include role::swh_elasticsearch } node /^search\d\.internal\.softwareheritage\.org$/ { include role::swh_search_with_journal_client } node /^counters\d\.internal\.softwareheritage\.org$/ { include role::swh_counters_with_journal_client } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler_with_journal_client } node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup include role::postgresql_backup } node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } node /^kafka\d+\./ { include role::swh_kafka_broker } node /^cassandra\d+\./ { include role::swh_cassandra_node } node 'granet.internal.softwareheritage.org' { include role::swh_graph_backend } node 'met.internal.softwareheritage.org' { include role::swh_provenance } node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } node /^saam\.(internal\.)?softwareheritage\.org$/ { include role::swh_storage_baremetal } node 'storage01.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cloud } node /^getty.(internal.)?softwareheritage.org$/ { include role::swh_journal_orchestrator_with_backfill_config } node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } node 'riverside.internal.softwareheritage.org' { include role::swh_sentry } node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'kelvingrove.internal.softwareheritage.org' { include role::swh_idp_primary } node 'giverny.softwareheritage.org' { include role::swh_desktop } node /^db\d\.internal\.staging\.swh\.network$/ { include role::swh_database include profile::postgresql::server include profile::pgbouncer include profile::postgresql::client } +node 'dali.internal.admin.swh.network' { + include role::swh_admin_database +} + node "bardo.internal.admin.swh.network" { include role::swh_hedgedoc } node 'scheduler0.internal.staging.swh.network' { include role::swh_scheduler_with_journal_client include profile::postgresql::client } node 'gateway.internal.staging.swh.network' { include role::swh_gateway } node /^storage\d\.internal\.staging\.swh\.network$/ { include role::swh_storage_with_journal } node /^worker\d\.internal\.staging\.swh\.network$/ { include role::swh_worker_inria } node /^search-esnode\d\.internal\.staging\.swh\.network$/ { include role::swh_elasticsearch } node /^search\d\.internal\.staging\.swh\.network$/ { include role::swh_search_with_journal_client } node /^counters\d\.internal\.staging\.swh\.network$/ { include role::swh_counters_with_journal_client } node 'webapp.internal.staging.swh.network' { include role::swh_webapp } node 'deposit.internal.staging.swh.network' { include role::swh_deposit } node 'vault.internal.staging.swh.network' { include role::swh_vault } node /^rp\d\.internal\.(staging|admin)\.swh\.network$/ { include role::swh_reverse_proxy } # Read-only storage for mirrors node 'objstorage0.internal.staging.swh.network' { include role::swh_remote_objstorage } node 'bojimans.internal.softwareheritage.org' { include role::swh_netbox } node /^mirror-test\.internal\.staging\.swh\.network$/ { include profile::postgresql::client } node default { include role::swh_base } diff --git a/site-modules/profile/manifests/postgresql/server.pp b/site-modules/profile/manifests/postgresql/server.pp index b3fa96f9..cddc76d1 100644 --- a/site-modules/profile/manifests/postgresql/server.pp +++ b/site-modules/profile/manifests/postgresql/server.pp @@ -1,123 +1,125 @@ # Install and configure a postgresql server class profile::postgresql::server { $swh_base_directory = lookup('swh::base_directory') $postgres_pass = lookup('swh::deploy::db::postgres::password') $listen_addresses = lookup('swh::postgresql::listen_addresses').join(',') # allow access through credentials $network_accesses = lookup('swh::postgresql::network_accesses').map | $nwk | { "host all all ${nwk} md5" } $postgres_version = lookup('swh::postgresql::version') $postgres_port = lookup('swh::postgresql::port') $postgres_datadir_base = lookup('swh::postgresql::datadir_base') $postgres_datadir = lookup('swh::postgresql::datadir') $postgres_max_connections = lookup('swh::postgresql::max_connections') $ip_mask_allow_all_users = '0.0.0.0/0' file { [ $postgres_datadir_base, "${postgres_datadir_base}/${postgres_version}" ] : ensure => directory, owner => 'root', group => 'root', mode => '0655', } -> class { 'postgresql::server': ip_mask_allow_all_users => $ip_mask_allow_all_users, ipv4acls => $network_accesses, postgres_password => $postgres_pass, port => $postgres_port, listen_addresses => [$listen_addresses], datadir => $postgres_datadir, needs_initdb => true, # Needed because managed_repo is false and data_dir is redefined by us ¯\_(ツ)_/¯ require => Class['profile::postgresql::apt_config'], pg_hba_conf_defaults => false, # see below for the actual default rules pg_hba_rules => { # Supersedes the default rules installed by puppetlab-postgres, thus # allowing pgbouncer/pgsql connection to the postgres user 'local access as postgres user' => { database => 'all', user => 'postgres', type => 'local', auth_method => 'ident', order => 1, }, 'local access to database with same name' => { database => 'all', user => 'all', type => 'local', auth_method => 'ident', order => 2, }, 'allow localhost TCP access to postgresql user' => { database => 'all', user => 'postgres', type => 'host', address => '127.0.0.1/32', auth_method => 'md5', order => 3, }, 'allow access to all users' => { database => 'all', user => 'all', type => 'host', address => $ip_mask_allow_all_users, auth_method => 'md5', order => 100, }, 'allow access to ipv6 localhost' => { database => 'all', user => 'all', type => 'host', address => '::1/128', auth_method => 'md5', order => 101, } }, } postgresql::server::config_entry{'max_connections': ensure => present, value => $postgres_max_connections, } postgresql::server::config_entry{'shared_preload_libraries': ensure => present, value => 'pg_stat_statements', } # read-only user $guest = 'guest' postgresql::server::role { $guest: password_hash => postgresql::postgresql_password($guest, 'guest'), require => Class['postgresql::server'] } $dbs = lookup('swh::dbs') each($dbs) | $db_type, $db_config | { # db_type in {storage, indexer, scheduler, etc...} $db_pass = pick( $db_config['password'], lookup("swh::deploy::${db_type}::db::password", {'default_value' => undef}) ) $db_name = $db_config['name'] $db_user = $db_config['user'] postgresql::server::db { $db_name: user => $db_user, password => $db_pass, owner => $db_user, + encoding => 'UTF8', + locale => 'C.UTF-8', require => Class['postgresql::server'] } # guest user has read access on tables postgresql::server::database_grant { $db_name: privilege => 'connect', db => $db_name, role => $guest, require => Postgresql::Server::Db[$db_name] } } } diff --git a/site-modules/role/manifests/swh_admin_database.pp b/site-modules/role/manifests/swh_admin_database.pp new file mode 100644 index 00000000..aa24765c --- /dev/null +++ b/site-modules/role/manifests/swh_admin_database.pp @@ -0,0 +1,5 @@ +class role::swh_admin_database inherits role::swh_base_database { + include profile::postgresql + include profile::postgresql::server + include profile::prometheus::sql +}