diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index cc52d39a..a677ccc0 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,104 +1,177 @@ --- swh::deploy::environment: staging swh::deploy::worker::loader_nixguix::loglevel: debug swh::deploy::storage::db::host: db0.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db0.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db0.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: deposit.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db0.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db0.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::worker::instances: - checker_deposit - loader_archive - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - indexer_origin_intrinsic_metadata #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage::config::storage0: cls: remote args: url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::deploy::deposit::url: http://deposit.internal.staging.swh.network # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt: '1': journal0.internal.staging.swh.network kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 swh::deploy::journal::brokers: - journal0.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging + +swh::postgresql::version: '12' +swh::postgresql::port: 5433 +swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" +swh::postgresql::datadir: "%{lookup('swh::base_directory')}/postgresql/%{lookup('swh::postgresql::cluster_name')}" +swh::postgresql::listen_addresses: + - localhost + - 0.0.0.0 +swh::postgresql::network_accesses: + - 192.168.100.0/24 # Monitoring + - 192.168.130.0/24 # Staging services + +swh::postgresql::shared_buffers: 32GB + +postgresql::server::config_entries: + shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" + cluster_name: "%{alias('swh::postgresql::cluster_name')}" + +postgresql::globals::version: "%{alias('swh::postgresql::version')}" + +swh::dbs: + storage: + name: swh + user: swh + scheduler: + name: swh-scheduler + user: swh-scheduler + vault: + name: swh-vault + user: swh-vault + lister: + name: swh-lister + user: swh-lister + deposit: + name: swh-deposit + user: swh-deposit + indexer::storage: + name: swh-indexer + user: swh-indexer + +pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" +pgbouncer::listen_addr: 0.0.0.0 +pgbouncer::databases: + - source_db: swh + host: localhost + auth_user: postgres + port: 5433 + alias: staging-swh + - source_db: swh-scheduler + host: localhost + auth_user: postgres + port: 5433 + alias: staging-swh-scheduler + - source_db: swh-vault + host: localhost + auth_user: postgres + port: 5433 + alias: staging-swh-vault + - source_db: swh-lister + host: localhost + auth_user: postgres + port: 5433 + alias: staging-swh-lister + - source_db: swh-deposit + host: localhost + auth_user: postgres + port: 5433 + alias: staging-swh-deposit + - source_db: swh-indexer + host: localhost + auth_user: postgres + port: 5433 + alias: staging-swh-indexer diff --git a/data/deployments/staging/vagrant.yaml b/data/deployments/staging/vagrant.yaml new file mode 100644 index 00000000..e83dd3a6 --- /dev/null +++ b/data/deployments/staging/vagrant.yaml @@ -0,0 +1,2 @@ +--- +swh::postgresql::shared_buffers: 128MB diff --git a/data/hostname/db0.internal.staging.swh.network.yaml b/data/hostname/db0.internal.staging.swh.network.yaml index ea18127d..4a17d4f6 100644 --- a/data/hostname/db0.internal.staging.swh.network.yaml +++ b/data/hostname/db0.internal.staging.swh.network.yaml @@ -1,63 +1,13 @@ --- networks: eth0: address: 192.168.130.10 netmask: 255.255.255.0 gateway: 192.168.130.1 -swh::dbs: - storage: - name: swh - user: swh - indexer::storage: - name: swh-indexer - user: swh-indexer - scheduler: - name: swh-scheduler - user: swh-scheduler - vault: - name: swh-vault - user: swh-vault - lister: - name: swh-lister - user: swh-lister - -postgres::server::port: 5433 -postgres::server::listen_addresses: - - localhost - - 192.168.130.10 -postgres::server::network_access: - - 192.168.100.0/24 # Monitoring - - 192.168.130.0/24 # Staging services - -pgbouncer::auth_hba_file: /etc/postgresql/11/main/pg_hba.conf -pgbouncer::listen_addr: 192.168.130.10 -pgbouncer::databases: - - source_db: swh - host: localhost - auth_user: postgres - port: 5433 - alias: staging-swh - - source_db: swh-indexer - host: localhost - auth_user: postgres - port: 5433 - alias: staging-swh-indexer - - source_db: swh-scheduler - host: localhost - auth_user: postgres - port: 5433 - alias: staging-swh-scheduler - - source_db: swh-vault - host: localhost - auth_user: postgres - port: 5433 - alias: staging-swh-vault - - source_db: swh-lister - host: localhost - auth_user: postgres - port: 5433 - alias: staging-swh-lister - backups::exclude: - srv/softwareheritage/postgres + +swh::postgresql::shared_buffers: 8GB +swh::postgresql::version: '11' +swh::postgresql::datadir: '/var/lib/postgresql/11/main' diff --git a/site-modules/profile/manifests/postgresql/server.pp b/site-modules/profile/manifests/postgresql/server.pp index 3bfc7d0b..62becaf2 100644 --- a/site-modules/profile/manifests/postgresql/server.pp +++ b/site-modules/profile/manifests/postgresql/server.pp @@ -1,50 +1,63 @@ +# Install and configure a postgresql server class profile::postgresql::server { - class { 'postgresql::globals': - encoding => 'UTF-8', - locale => 'en_US.UTF-8', - manage_package_repo => true, - version => '11', - } + + $swh_base_directory = lookup('swh::base_directory') $postgres_pass = lookup('swh::deploy::db::postgres::password') - $server_port = lookup('postgres::server::port') - $server_addresses = lookup('postgres::server::listen_addresses').join(',') + $listen_addresses = lookup('swh::postgresql::listen_addresses').join(',') + # allow access through credentials - $network_access = lookup('postgres::server::network_access').map | $nwk | { + $network_accesses = lookup('swh::postgresql::network_accesses').map | $nwk | { "host all all ${nwk} md5" } + $postgres_version = lookup('swh::postgresql::version') + $postgres_port = lookup('swh::postgresql::port') + $postgres_datadir = lookup('swh::postgresql::datadir') - class { 'postgresql::server': - ip_mask_allow_all_users => '0.0.0.0/0', - ipv4acls => $network_access, - postgres_password => $postgres_pass, - port => $server_port, - listen_addresses => [$server_addresses], + file { [ "${swh_base_directory}/postgresql", + "${swh_base_directory}/postgresql/${postgres_version}" ] : + ensure => directory, + owner => 'root', + group => 'root', + mode => '0655', + } + -> class { 'postgresql::server': + ip_mask_allow_all_users => '0.0.0.0/0', + ipv4acls => $network_accesses, + postgres_password => $postgres_pass, + port => $postgres_port, + listen_addresses => [$listen_addresses], + datadir => $postgres_datadir, + needs_initdb => true, # Needed because managed_repo is false and data_dir is redefined by us ¯\_(ツ)_/¯ + require => Class['profile::postgresql::apt_config'] } $guest = 'guest' postgresql::server::role { $guest: password_hash => postgresql_password($guest, 'guest'), + require => Class['postgresql::server'] } $dbs = lookup('swh::dbs') each($dbs) | $db_type, $db_config | { # db_type in {storage, indexer, scheduler, etc...} $db_pass = lookup("swh::deploy::${db_type}::db::password") $db_name = $db_config['name'] $db_user = $db_config['user'] postgresql::server::db { $db_name: user => $db_user, password => $db_pass, - owner => $db_user + owner => $db_user, + require => Class['postgresql::server'] } # guest user has read access on tables postgresql::server::database_grant { $db_name: - privilege => 'connect', - db => $db_name, - role => $guest, + privilege => 'connect', + db => $db_name, + role => $guest, + require => Postgresql::Server::Db[$db_name] } } }