diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index de2ce99f..a97b6b95 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,177 +1,216 @@ --- swh::deploy::environment: staging swh::deploy::worker::loader_nixguix::loglevel: debug swh::deploy::storage::db::host: db1.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db1.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: db1.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db1.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::worker::instances: - checker_deposit - loader_archive - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - indexer_origin_intrinsic_metadata #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage::config::storage0: cls: remote args: - url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" + url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote - url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" + url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::deploy::deposit::url: http://deposit.internal.staging.swh.network # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt: '1': journal0.internal.staging.swh.network kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 swh::deploy::journal::brokers: - journal0.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging swh::postgresql::version: '12' swh::postgresql::port: 5433 swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" swh::postgresql::datadir: "%{lookup('swh::base_directory')}/postgresql/%{lookup('swh::postgresql::cluster_name')}" swh::postgresql::listen_addresses: - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services swh::postgresql::shared_buffers: 32GB postgresql::server::config_entries: shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" cluster_name: "%{alias('swh::postgresql::cluster_name')}" postgresql::globals::version: "%{alias('swh::postgresql::version')}" swh::dbs: storage: name: swh user: swh scheduler: name: swh-scheduler user: swh-scheduler vault: name: swh-vault user: swh-vault lister: name: swh-lister user: swh-lister deposit: name: swh-deposit user: swh-deposit indexer::storage: name: swh-indexer user: swh-indexer pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" pgbouncer::common::listen_addresses: - 0.0.0.0 pgbouncer::databases: - source_db: swh host: localhost auth_user: postgres port: 5433 alias: staging-swh - source_db: swh-scheduler host: localhost auth_user: postgres port: 5433 alias: staging-swh-scheduler - source_db: swh-vault host: localhost auth_user: postgres port: 5433 alias: staging-swh-vault - source_db: swh-lister host: localhost auth_user: postgres port: 5433 alias: staging-swh-lister - source_db: swh-deposit host: localhost auth_user: postgres port: 5433 alias: staging-swh-deposit - source_db: swh-indexer host: localhost auth_user: postgres port: 5433 alias: staging-swh-indexer + +# open objstorage api +swh::deploy::objstorage::backend::listen::host: 0.0.0.0 +swh::deploy::objstorage::backend::workers: 4 +swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" +swh::deploy::objstorage::slicing: 0:1/1:5 + +swh::remote_service::objstorage::config: + cls: pathslicing + args: + root: "%{hiera('swh::deploy::storage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + +# Deploy the storage server as a public resource +swh::deploy::storage::backend::listen::host: 0.0.0.0 +swh::deploy::storage::backend::workers: 4 +swh::deploy::storage::backend::max_requests: 100 +swh::deploy::storage::backend::max_requests_jitter: 10 + +# Deploy the indexer storage server as a public resource +swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 +swh::deploy::indexer::storage::backend::workers: 4 + +nginx::worker_processes: 4 + +swh::deploy::storage::config: + storage: + cls: local + args: + db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" + objstorage: "%{alias('swh::remote_service::objstorage::config')}" + journal_writer: + cls: kafka + args: + brokers: "%{alias('swh::deploy::journal::brokers')}" + prefix: "%{alias('swh::deploy::journal::prefix')}" + client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" + producer_config: + message.max.bytes: 1000000000 diff --git a/data/hostname/storage0.internal.staging.swh.network.yaml b/data/hostname/storage0.internal.staging.swh.network.yaml index 4eab62c1..2906b730 100644 --- a/data/hostname/storage0.internal.staging.swh.network.yaml +++ b/data/hostname/storage0.internal.staging.swh.network.yaml @@ -1,47 +1,8 @@ networks: eth0: address: 192.168.130.40 netmask: 255.255.255.0 gateway: 192.168.130.1 swh::postgres::service::users: - swhstorage - -# open objstorage api -swh::deploy::objstorage::backend::listen::host: 0.0.0.0 -swh::deploy::objstorage::backend::workers: 4 -swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" -swh::deploy::objstorage::slicing: 0:1/1:5 - -swh::remote_service::objstorage::config: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" - -# Deploy the storage server as a public resource -swh::deploy::storage::backend::listen::host: 0.0.0.0 -swh::deploy::storage::backend::workers: 4 -swh::deploy::storage::backend::max_requests: 100 -swh::deploy::storage::backend::max_requests_jitter: 10 - -# Deploy the indexer storage server as a public resource -swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 -swh::deploy::indexer::storage::backend::workers: 4 - -nginx::worker_processes: 4 - -swh::deploy::storage::config: - storage: - cls: local - args: - db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" - objstorage: "%{alias('swh::remote_service::objstorage::config')}" - journal_writer: - cls: kafka - args: - brokers: "%{alias('swh::deploy::journal::brokers')}" - prefix: "%{alias('swh::deploy::journal::prefix')}" - client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" - producer_config: - message.max.bytes: 1000000000 diff --git a/manifests/site.pp b/manifests/site.pp index a2fd8bf4..a21e0734 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,185 +1,185 @@ node 'louvre.internal.softwareheritage.org' { include role::swh_server } node /^(orsay|beaubourg|hypervisor\d+|branly)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin include profile::export_archive_counters } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_api } node 'webapp0.softwareheritage.org' { include role::swh_base_api } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler } node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup include role::postgresql_backup } node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } node /^zookeeper\d+.(internal.)?softwareheritage.org$/ { include role::swh_zookeeper } node /^kafka\d+\./ { include role::swh_kafka_broker } node /^cassandra\d+\./ { include role::swh_cassandra_node } node 'granet.internal.softwareheritage.org' { include role::swh_graph_backend } node /^(unibo-test).(internal.)?softwareheritage.org$/ { include role::swh_vault_test } node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } node /^uffizi\.(internal\.)?softwareheritage\.org$/ { include role::swh_storage_baremetal } node 'storage01.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cloud } node 'storage02.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cassandra } node /^getty.(internal.)?softwareheritage.org$/ { include role::swh_journal_orchestrator } node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } node 'riverside.internal.softwareheritage.org' { include role::swh_sentry } node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'kelvingrove.internal.softwareheritage.org' { include role::swh_idp_primary } node 'giverny.softwareheritage.org' { include role::swh_desktop } node /^db\d\.internal\.staging\.swh\.network$/ { include role::swh_database include profile::postgresql::server include profile::pgbouncer include profile::postgresql::client } node 'scheduler0.internal.staging.swh.network' { include role::swh_scheduler include profile::postgresql::client } node 'gateway.internal.staging.swh.network' { include role::swh_gateway } -node 'storage0.internal.staging.swh.network' { +node /^storage\d\.internal\.staging\.swh\.network$/ { include role::swh_base_storage include profile::postgresql::client } node /^worker\d\.internal\.staging\.swh\.network$/ { include role::swh_worker_inria } node 'webapp.internal.staging.swh.network' { include role::swh_base_api include profile::network } node 'deposit.internal.staging.swh.network' { include role::swh_deposit } node 'vault.internal.staging.swh.network' { include role::swh_vault } node 'journal0.internal.staging.swh.network' { include role::swh_journal_allinone } node 'bojimans.internal.softwareheritage.org' { include role::swh_netbox } node default { include role::swh_base }