Page MenuHomeSoftware Heritage
Files F9345852

No OneTemporary
Actions

View Options

diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml
index cc52d39a..a677ccc0 100644
--- a/data/deployments/staging/common.yaml
+++ b/data/deployments/staging/common.yaml
@@ -1,104 +1,177 @@
---
swh::deploy::environment: staging
swh::deploy::worker::loader_nixguix::loglevel: debug
swh::deploy::storage::db::host: db0.internal.staging.swh.network
swh::deploy::storage::db::user: swh
swh::deploy::storage::db::dbname: swh
swh::deploy::indexer::storage::db::host: db0.internal.staging.swh.network
swh::deploy::indexer::storage::db::user: swh-indexer
swh::deploy::indexer::storage::db::dbname: swh-indexer
swh::deploy::scheduler::db::host: db0.internal.staging.swh.network
swh::deploy::scheduler::db::dbname: swh-scheduler
swh::deploy::scheduler::db::user: swh-scheduler
swh::deploy::deposit::db::host: deposit.internal.staging.swh.network
swh::deploy::deposit::db::dbuser: swh-deposit
swh::deploy::deposit::db::dbname: swh-deposit
swh::deploy::vault::db::host: db0.internal.staging.swh.network
swh::deploy::vault::db::user: swh-vault
swh::deploy::vault::db::dbname: swh-vault
swh::deploy::worker::lister::db::host: db0.internal.staging.swh.network
swh::deploy::worker::lister::db::user: swh-lister
swh::deploy::worker::lister::db::name: swh-lister
swh::deploy::worker::instances:
- checker_deposit
- loader_archive
- loader_cran
- loader_debian
- loader_deposit
- loader_nixguix
- loader_git
- loader_mercurial
- loader_npm
- loader_pypi
- loader_svn
- vault_cooker
- lister
- indexer_origin_intrinsic_metadata
#### Rabbitmq instance to use
# swh::deploy::worker::task_broker::password in private data
swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f"
#### Storage/Indexer/Vault/Scheduler services to use in staging area
swh::remote_service::storage::config::storage0:
cls: remote
args:
url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/"
swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}"
swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable
"%{alias('swh::remote_service::storage::config::storage0')}"
swh::remote_service::vault::config::vault0:
cls: remote
args:
url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/"
swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}"
swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}"
swh::remote_service::indexer::config::storage0:
cls: remote
url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/"
swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}"
swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}"
swh::remote_service::scheduler::config::scheduler0:
cls: remote
args:
url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/"
swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::deploy::deposit::url: http://deposit.internal.staging.swh.network
# do not save pack
swh::deploy::worker::loader_git::save_data_path: ""
swh::deploy::worker::loader_git::concurrency: 1
zookeeper::clusters:
rocquencourt:
'1': journal0.internal.staging.swh.network
kafka::clusters:
rocquencourt:
zookeeper::chroot: '/kafka/softwareheritage'
zookeeper::servers:
- journal0.internal.staging.swh.network
brokers:
journal0.internal.staging.swh.network:
id: 1
swh::deploy::journal::brokers:
- journal0.internal.staging.swh.network
swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging
swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging
+
+swh::postgresql::version: '12'
+swh::postgresql::port: 5433
+swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main"
+swh::postgresql::datadir: "%{lookup('swh::base_directory')}/postgresql/%{lookup('swh::postgresql::cluster_name')}"
+swh::postgresql::listen_addresses:
+ - localhost
+ - 0.0.0.0
+swh::postgresql::network_accesses:
+ - 192.168.100.0/24 # Monitoring
+ - 192.168.130.0/24 # Staging services
+
+swh::postgresql::shared_buffers: 32GB
+
+postgresql::server::config_entries:
+ shared_buffers: "%{alias('swh::postgresql::shared_buffers')}"
+ cluster_name: "%{alias('swh::postgresql::cluster_name')}"
+
+postgresql::globals::version: "%{alias('swh::postgresql::version')}"
+
+swh::dbs:
+ storage:
+ name: swh
+ user: swh
+ scheduler:
+ name: swh-scheduler
+ user: swh-scheduler
+ vault:
+ name: swh-vault
+ user: swh-vault
+ lister:
+ name: swh-lister
+ user: swh-lister
+ deposit:
+ name: swh-deposit
+ user: swh-deposit
+ indexer::storage:
+ name: swh-indexer
+ user: swh-indexer
+
+pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf"
+pgbouncer::listen_addr: 0.0.0.0
+pgbouncer::databases:
+ - source_db: swh
+ host: localhost
+ auth_user: postgres
+ port: 5433
+ alias: staging-swh
+ - source_db: swh-scheduler
+ host: localhost
+ auth_user: postgres
+ port: 5433
+ alias: staging-swh-scheduler
+ - source_db: swh-vault
+ host: localhost
+ auth_user: postgres
+ port: 5433
+ alias: staging-swh-vault
+ - source_db: swh-lister
+ host: localhost
+ auth_user: postgres
+ port: 5433
+ alias: staging-swh-lister
+ - source_db: swh-deposit
+ host: localhost
+ auth_user: postgres
+ port: 5433
+ alias: staging-swh-deposit
+ - source_db: swh-indexer
+ host: localhost
+ auth_user: postgres
+ port: 5433
+ alias: staging-swh-indexer
diff --git a/data/deployments/staging/vagrant.yaml b/data/deployments/staging/vagrant.yaml
new file mode 100644
index 00000000..e83dd3a6
--- /dev/null
+++ b/data/deployments/staging/vagrant.yaml
@@ -0,0 +1,2 @@
+---
+swh::postgresql::shared_buffers: 128MB
diff --git a/data/hostname/db0.internal.staging.swh.network.yaml b/data/hostname/db0.internal.staging.swh.network.yaml
index ea18127d..4a17d4f6 100644
--- a/data/hostname/db0.internal.staging.swh.network.yaml
+++ b/data/hostname/db0.internal.staging.swh.network.yaml
@@ -1,63 +1,13 @@
---
networks:
eth0:
address: 192.168.130.10
netmask: 255.255.255.0
gateway: 192.168.130.1
-swh::dbs:
- storage:
- name: swh
- user: swh
- indexer::storage:
- name: swh-indexer
- user: swh-indexer
- scheduler:
- name: swh-scheduler
- user: swh-scheduler
- vault:
- name: swh-vault
- user: swh-vault
- lister:
- name: swh-lister
- user: swh-lister
-
-postgres::server::port: 5433
-postgres::server::listen_addresses:
- - localhost
- - 192.168.130.10
-postgres::server::network_access:
- - 192.168.100.0/24 # Monitoring
- - 192.168.130.0/24 # Staging services
-
-pgbouncer::auth_hba_file: /etc/postgresql/11/main/pg_hba.conf
-pgbouncer::listen_addr: 192.168.130.10
-pgbouncer::databases:
- - source_db: swh
- host: localhost
- auth_user: postgres
- port: 5433
- alias: staging-swh
- - source_db: swh-indexer
- host: localhost
- auth_user: postgres
- port: 5433
- alias: staging-swh-indexer
- - source_db: swh-scheduler
- host: localhost
- auth_user: postgres
- port: 5433
- alias: staging-swh-scheduler
- - source_db: swh-vault
- host: localhost
- auth_user: postgres
- port: 5433
- alias: staging-swh-vault
- - source_db: swh-lister
- host: localhost
- auth_user: postgres
- port: 5433
- alias: staging-swh-lister
-
backups::exclude:
- srv/softwareheritage/postgres
+
+swh::postgresql::shared_buffers: 8GB
+swh::postgresql::version: '11'
+swh::postgresql::datadir: '/var/lib/postgresql/11/main'
diff --git a/site-modules/profile/manifests/postgresql/server.pp b/site-modules/profile/manifests/postgresql/server.pp
index 3bfc7d0b..62becaf2 100644
--- a/site-modules/profile/manifests/postgresql/server.pp
+++ b/site-modules/profile/manifests/postgresql/server.pp
@@ -1,50 +1,63 @@
+# Install and configure a postgresql server
class profile::postgresql::server {
- class { 'postgresql::globals':
- encoding => 'UTF-8',
- locale => 'en_US.UTF-8',
- manage_package_repo => true,
- version => '11',
- }
+
+ $swh_base_directory = lookup('swh::base_directory')
$postgres_pass = lookup('swh::deploy::db::postgres::password')
- $server_port = lookup('postgres::server::port')
- $server_addresses = lookup('postgres::server::listen_addresses').join(',')
+ $listen_addresses = lookup('swh::postgresql::listen_addresses').join(',')
+
# allow access through credentials
- $network_access = lookup('postgres::server::network_access').map | $nwk | {
+ $network_accesses = lookup('swh::postgresql::network_accesses').map | $nwk | {
"host all all ${nwk} md5"
}
+ $postgres_version = lookup('swh::postgresql::version')
+ $postgres_port = lookup('swh::postgresql::port')
+ $postgres_datadir = lookup('swh::postgresql::datadir')
- class { 'postgresql::server':
- ip_mask_allow_all_users => '0.0.0.0/0',
- ipv4acls => $network_access,
- postgres_password => $postgres_pass,
- port => $server_port,
- listen_addresses => [$server_addresses],
+ file { [ "${swh_base_directory}/postgresql",
+ "${swh_base_directory}/postgresql/${postgres_version}" ] :
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0655',
+ }
+ -> class { 'postgresql::server':
+ ip_mask_allow_all_users => '0.0.0.0/0',
+ ipv4acls => $network_accesses,
+ postgres_password => $postgres_pass,
+ port => $postgres_port,
+ listen_addresses => [$listen_addresses],
+ datadir => $postgres_datadir,
+ needs_initdb => true, # Needed because managed_repo is false and data_dir is redefined by us ¯\_(ツ)_/¯
+ require => Class['profile::postgresql::apt_config']
}
$guest = 'guest'
postgresql::server::role { $guest:
password_hash => postgresql_password($guest, 'guest'),
+ require => Class['postgresql::server']
}
$dbs = lookup('swh::dbs')
each($dbs) | $db_type, $db_config | {
# db_type in {storage, indexer, scheduler, etc...}
$db_pass = lookup("swh::deploy::${db_type}::db::password")
$db_name = $db_config['name']
$db_user = $db_config['user']
postgresql::server::db { $db_name:
user => $db_user,
password => $db_pass,
- owner => $db_user
+ owner => $db_user,
+ require => Class['postgresql::server']
}
# guest user has read access on tables
postgresql::server::database_grant { $db_name:
- privilege => 'connect',
- db => $db_name,
- role => $guest,
+ privilege => 'connect',
+ db => $db_name,
+ role => $guest,
+ require => Postgresql::Server::Db[$db_name]
}
}
}

File Metadata

Mime Type
text/x-diff
Expires
Fri, Jul 4, 3:33 PM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3272498

Event Timeline

About · Developer information · Legal