diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index 61fd3058..5d4234d5 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,238 +1,246 @@ --- swh::deploy::environment: staging swh::deploy::worker::loader_nixguix::loglevel: debug swh::deploy::storage::db::host: db1.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db1.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: db1.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db1.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::worker::instances: - checker_deposit - loader_archive - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - indexer_origin_intrinsic_metadata #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage::config::storage0: cls: remote args: url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::deploy::deposit::url: https://deposit.staging.swh.network swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network" # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt_staging: '1': journal0.internal.staging.swh.network kafka::broker::heap_opts: "-Xmx3G -Xms3G" kafka::clusters: rocquencourt_staging: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 broker::heap_opts: "%{alias('kafka::broker::heap_opts')}" superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 cluster_config_overrides: offsets.topic.replication.factor: 1 # this is mandatory with only one node # public_listener_network: 128.93.166.0/26 swh::deploy::journal::brokers: - journal0.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging swh::deploy::deposit::reverse_proxy::backend_http_host: deposit.internal.staging.swh.network swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging swh::deploy::webapp::reverse_proxy::backend_http_host: webapp.internal.staging.swh.network swh::postgresql::version: '12' swh::postgresql::port: 5433 swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres" swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" swh::postgresql::listen_addresses: - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services swh::postgresql::shared_buffers: 32GB postgresql::server::config_entries: shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" cluster_name: "%{alias('swh::postgresql::cluster_name')}" postgresql::globals::version: "%{alias('swh::postgresql::version')}" swh::dbs: storage: name: swh user: swh scheduler: name: swh-scheduler user: swh-scheduler vault: name: swh-vault user: swh-vault lister: name: swh-lister user: swh-lister deposit: name: swh-deposit user: swh-deposit indexer::storage: name: swh-indexer user: swh-indexer pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" pgbouncer::common::listen_addresses: - 0.0.0.0 pgbouncer::databases: - source_db: swh host: localhost auth_user: postgres port: 5433 alias: staging-swh - source_db: swh-scheduler host: localhost auth_user: postgres port: 5433 alias: staging-swh-scheduler - source_db: swh-vault host: localhost auth_user: postgres port: 5433 alias: staging-swh-vault - source_db: swh-lister host: localhost auth_user: postgres port: 5433 alias: staging-swh-lister - source_db: swh-deposit host: localhost auth_user: postgres port: 5433 alias: staging-swh-deposit - source_db: swh-indexer host: localhost auth_user: postgres port: 5433 alias: staging-swh-indexer # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 4 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" swh::deploy::objstorage::slicing: 0:1/1:5 -swh::remote_service::objstorage::config: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" - # Deploy the storage server as a public resource swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 4 swh::deploy::storage::backend::max_requests: 100 swh::deploy::storage::backend::max_requests_jitter: 10 # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::backend::workers: 4 + +swh::remote_service::objstorage::config::rw: + cls: remote + url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::objstorage::port')}/" + +swh::remote_service::objstorage::config::ro: + cls: filtered + storage_conf: "%{alias('swh::remote_service::objstorage::config::rw')}" + filters_conf: + - type: readonly + +swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::ro')}" +swh::remote_service::objstorage::config::writable: "%{alias('swh::remote_service::objstorage::config::rw')}" + swh::deploy::indexer::storage::config: indexer_storage: cls: local db: "host=%{hiera('swh::deploy::indexer::storage::db::host')} port=%{hiera('swh::deploy::indexer::storage::db::port')} user=%{hiera('swh::deploy::indexer::storage::db::user')} dbname=%{hiera('swh::deploy::indexer::storage::db::dbname')} password=%{hiera('swh::deploy::indexer::storage::db::password')}" journal_writer: "%{alias('swh::deploy::indexer::journal::writer::config')}" nginx::worker_processes: 4 ## Reverse-proxy and frontend hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disable default vhost on port 80 apache::default_vhost: false diff --git a/data/hostname/storage1.internal.staging.swh.network.yaml b/data/hostname/storage1.internal.staging.swh.network.yaml index ed11950c..ef7059ef 100644 --- a/data/hostname/storage1.internal.staging.swh.network.yaml +++ b/data/hostname/storage1.internal.staging.swh.network.yaml @@ -1,59 +1,63 @@ --- networks: enp2s0f0: type: manual order: 10 extras: mtu: "9000" enp2s0f1: type: manual order: 10 extras: mtu: "9000" bond0: type: manual order: 20 extras: mtu: "9000" bond-miimon: 100 bond-mode: 802.3ad bond-xmit_hash_policy: layer3+4 bond-slaves: enp2s0f0 enp2s0f1 bond-lacp-rate: 1 vlan443: type: static order: 30 address: 192.168.130.41 netmask: 24 gateway: 192.168.130.1 mtu: "9000" extras: vlan-raw-device: bond0 backups::exclude: - srv/softwareheritage/objects swh::apt_config::enable_non_free: true packages: - intel-microcode - vlan - ifenslave swh::apt_config::backported_packages: buster: - linux-image-amd64 - linux-headers-amd64 - libnvpair1linux - libuutil1linux - libzfs2linux - libzpool2linux - zfs-dkms - zfsutils-linux - zfs-zed swh::deploy::storage::config::local: cls: local args: db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" - objstorage: "%{alias('swh::remote_service::objstorage::config')}" + objstorage: + cls: pathslicing + args: + root: "%{hiera('swh::deploy::storage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" journal_writer: "%{alias('swh::deploy::journal::writer::config')}"