diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index a97b6b95..b7095a72 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,216 +1,217 @@ --- swh::deploy::environment: staging swh::deploy::worker::loader_nixguix::loglevel: debug swh::deploy::storage::db::host: db1.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db1.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: db1.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db1.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::worker::instances: - checker_deposit - loader_archive - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - indexer_origin_intrinsic_metadata #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage::config::storage0: cls: remote args: url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::deploy::deposit::url: http://deposit.internal.staging.swh.network # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt: '1': journal0.internal.staging.swh.network kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 swh::deploy::journal::brokers: - journal0.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging swh::postgresql::version: '12' swh::postgresql::port: 5433 swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" -swh::postgresql::datadir: "%{lookup('swh::base_directory')}/postgresql/%{lookup('swh::postgresql::cluster_name')}" +swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres" +swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" swh::postgresql::listen_addresses: - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services swh::postgresql::shared_buffers: 32GB postgresql::server::config_entries: shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" cluster_name: "%{alias('swh::postgresql::cluster_name')}" postgresql::globals::version: "%{alias('swh::postgresql::version')}" swh::dbs: storage: name: swh user: swh scheduler: name: swh-scheduler user: swh-scheduler vault: name: swh-vault user: swh-vault lister: name: swh-lister user: swh-lister deposit: name: swh-deposit user: swh-deposit indexer::storage: name: swh-indexer user: swh-indexer pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" pgbouncer::common::listen_addresses: - 0.0.0.0 pgbouncer::databases: - source_db: swh host: localhost auth_user: postgres port: 5433 alias: staging-swh - source_db: swh-scheduler host: localhost auth_user: postgres port: 5433 alias: staging-swh-scheduler - source_db: swh-vault host: localhost auth_user: postgres port: 5433 alias: staging-swh-vault - source_db: swh-lister host: localhost auth_user: postgres port: 5433 alias: staging-swh-lister - source_db: swh-deposit host: localhost auth_user: postgres port: 5433 alias: staging-swh-deposit - source_db: swh-indexer host: localhost auth_user: postgres port: 5433 alias: staging-swh-indexer # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 4 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" swh::deploy::objstorage::slicing: 0:1/1:5 swh::remote_service::objstorage::config: cls: pathslicing args: root: "%{hiera('swh::deploy::storage::directory')}" slicing: "%{hiera('swh::deploy::objstorage::slicing')}" # Deploy the storage server as a public resource swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 4 swh::deploy::storage::backend::max_requests: 100 swh::deploy::storage::backend::max_requests_jitter: 10 # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::backend::workers: 4 nginx::worker_processes: 4 swh::deploy::storage::config: storage: cls: local args: db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" journal_writer: cls: kafka args: brokers: "%{alias('swh::deploy::journal::brokers')}" prefix: "%{alias('swh::deploy::journal::prefix')}" client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" producer_config: message.max.bytes: 1000000000 diff --git a/site-modules/profile/manifests/postgresql/server.pp b/site-modules/profile/manifests/postgresql/server.pp index 88a92550..41cccde2 100644 --- a/site-modules/profile/manifests/postgresql/server.pp +++ b/site-modules/profile/manifests/postgresql/server.pp @@ -1,108 +1,109 @@ # Install and configure a postgresql server class profile::postgresql::server { $swh_base_directory = lookup('swh::base_directory') $postgres_pass = lookup('swh::deploy::db::postgres::password') $listen_addresses = lookup('swh::postgresql::listen_addresses').join(',') # allow access through credentials $network_accesses = lookup('swh::postgresql::network_accesses').map | $nwk | { "host all all ${nwk} md5" } $postgres_version = lookup('swh::postgresql::version') $postgres_port = lookup('swh::postgresql::port') + $postgres_datadir_base = lookup('swh::postgresql::datadir_base') $postgres_datadir = lookup('swh::postgresql::datadir') $ip_mask_allow_all_users = '0.0.0.0/0' - file { [ "${swh_base_directory}/postgresql", - "${swh_base_directory}/postgresql/${postgres_version}" ] : + file { [ "${postgres_datadir_base}", + "${postgres_datadir_base}/${postgres_version}" ] : ensure => directory, owner => 'root', group => 'root', mode => '0655', } -> class { 'postgresql::server': ip_mask_allow_all_users => $ip_mask_allow_all_users, ipv4acls => $network_accesses, postgres_password => $postgres_pass, port => $postgres_port, listen_addresses => [$listen_addresses], datadir => $postgres_datadir, needs_initdb => true, # Needed because managed_repo is false and data_dir is redefined by us ¯\_(ツ)_/¯ require => Class['profile::postgresql::apt_config'], pg_hba_conf_defaults => false, # see below for the actual default rules pg_hba_rules => { # Supersedes the default rules installed by puppetlab-postgres, thus # allowing pgbouncer/pgsql connection to the postgres user 'local access as postgres user' => { database => 'all', user => 'postgres', type => 'local', auth_method => 'ident', order => 1, }, 'local access to database with same name' => { database => 'all', user => 'all', type => 'local', auth_method => 'ident', order => 2, }, 'allow localhost TCP access to postgresql user' => { database => 'all', user => 'postgres', type => 'host', address => '127.0.0.1/32', auth_method => 'md5', order => 3, }, 'allow access to all users' => { database => 'all', user => 'all', type => 'host', address => $ip_mask_allow_all_users, auth_method => 'md5', order => 100, }, 'allow access to ipv6 localhost' => { database => 'all', user => 'all', type => 'host', address => '::1/128', auth_method => 'md5', order => 101, } } } # read-only user $guest = 'guest' postgresql::server::role { $guest: password_hash => postgresql_password($guest, 'guest'), require => Class['postgresql::server'] } $dbs = lookup('swh::dbs') each($dbs) | $db_type, $db_config | { # db_type in {storage, indexer, scheduler, etc...} $db_pass = lookup("swh::deploy::${db_type}::db::password") $db_name = $db_config['name'] $db_user = $db_config['user'] postgresql::server::db { $db_name: user => $db_user, password => $db_pass, owner => $db_user, require => Class['postgresql::server'] } # guest user has read access on tables postgresql::server::database_grant { $db_name: privilege => 'connect', db => $db_name, role => $guest, require => Postgresql::Server::Db[$db_name] } } }