diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index 3e06a4fe..027e7d43 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,392 +1,390 @@ --- swh::deploy::environment: staging dns::search_domains: - internal.staging.swh.network swh::deploy::storage::db::host: db1.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db1.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: db1.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db1.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::scrubber::db::host: db1.internal.staging.swh.network swh::deploy::webapp::db::host: db1.internal.staging.swh.network # swh::deploy::webapp::db::password in private data swh::deploy::webapp::inbound_email::domain: "%{lookup('swh::deploy::webapp::inbound_email::staging::domain')}" swh::deploy::worker::instances: - indexer_content_mimetype - indexer_fossology_license - indexer_origin_intrinsic_metadata - indexer_origin_extrinsic_metadata - checker_deposit - loader_archive - loader_bzr - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - loader_high_priority - loader_opam # only in staging - loader_cvs - loader_maven swh::deploy::lister::queues: - swh.lister.bitbucket.tasks.IncrementalBitBucketLister - swh.lister.bitbucket.tasks.FullBitBucketRelister - swh.lister.cgit.tasks.CGitListerTask - swh.lister.cran.tasks.CRANListerTask - swh.lister.debian.tasks.DebianListerTask - swh.lister.gitea.tasks.IncrementalGiteaLister - swh.lister.gitea.tasks.RangeGiteaLister - swh.lister.gitea.tasks.FullGiteaRelister - swh.lister.gitlab.tasks.IncrementalGitLabLister - swh.lister.gitlab.tasks.RangeGitLabLister - swh.lister.gitlab.tasks.FullGitLabRelister - swh.lister.gnu.tasks.GNUListerTask - swh.lister.launchpad.tasks.FullLaunchpadLister - swh.lister.launchpad.tasks.IncrementalLaunchpadLister - swh.lister.opam.tasks.OpamListerTask - swh.lister.npm.tasks.NpmListerTask - swh.lister.phabricator.tasks.FullPhabricatorLister - swh.lister.pypi.tasks.PyPIListerTask - swh.lister.sourceforge.tasks.FullSourceForgeLister - swh.lister.sourceforge.tasks.IncrementalSourceForgeLister # extra queues - swh.lister.maven.tasks.FullMavenLister - swh.lister.maven.tasks.IncrementalMavenLister #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage0::url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config::storage0: cls: remote url: "%{alias('swh::remote_service::storage0::url')}" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::counters::url: "http://counters0.internal.staging.swh.network:%{hiera('swh::remote_service::counters::port')}/" swh::deploy::deposit::url: https://deposit.staging.swh.network swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network" # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt_staging: '2': storage1.internal.staging.swh.network kafka::broker::heap_opts: "-Xmx3G -Xms3G" swh::deploy::journal::brokers: - journal1.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging swh::deploy::deposit::reverse_proxy::backend_http_host: deposit.internal.staging.swh.network swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging swh::deploy::webapp::reverse_proxy::backend_http_host: webapp.internal.staging.swh.network swh::deploy::graphql::vhost::letsencrypt_cert: graphql_staging swh::deploy::graphql::vhost::ssl_protocol: "%{hiera('apache::ssl_protocol')}" swh::deploy::graphql::vhost::ssl_honorcipherorder: "%{hiera('apache::ssl_honorcipherorder')}" swh::deploy::graphql::vhost::ssl_cipher: "%{hiera('apache::ssl_cipher')}" swh::deploy::graphql::vhost::hsts_header: "%{hiera('apache::hsts_header')}" swh::deploy::graphql::vhost::access_log_format: combined_with_duration - -swh::deploy::graphql::reverse_proxy::basic_auth: true swh::deploy::graphql::reverse_proxy::backend_http_host: graphql-worker0.internal.staging.swh.network swh::deploy::graphql::reverse_proxy::backend_http_port: "80" # swh::deploy::graphql::reverse_proxy::basic_auth::swh-stg in private data swh::deploy::graphql::reverse_proxy::basic_auth::users: - swh-stg swh::remote_service::objstorage::config::rw: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::objstorage::port')}/" swh::remote_service::objstorage::config::ro: cls: filtered storage_conf: "%{alias('swh::remote_service::objstorage::config::rw')}" filters_conf: - type: readonly swh::deploy::objstorage::vhost::letsencrypt_cert: objstorage_staging swh::deploy::objstorage::reverse_proxy::backend_http_host: objstorage0.internal.staging.swh.network swh::deploy::objstorage::reverse_proxy::basic_auth::users: - swh-stg - enea-stg - snyk-stg-01 swh::deploy::objstorage::backend::public_server_name: objstorage.staging.swh.network objstorage.internal.staging.swh.network swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::ro')}" swh::remote_service::objstorage::config::writable: "%{alias('swh::remote_service::objstorage::config::rw')}" swh::deploy::objstorage::backend::server_names: - "%{alias('swh::deploy::objstorage::backend::public_server_name')}" - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" swh::deploy::reverse_proxy::services: - deposit - webapp - objstorage - graphql swh::postgresql::version: '12' swh::postgresql::port: 5433 swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres" swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" swh::postgresql::listen_addresses: - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services swh::postgresql::shared_buffers: 32GB postgresql::server::config_entries: shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" cluster_name: "%{alias('swh::postgresql::cluster_name')}" swh::dbs: storage: name: swh user: swh scheduler: name: swh-scheduler user: swh-scheduler vault: name: swh-vault user: swh-vault lister: name: swh-lister user: swh-lister deposit: name: swh-deposit user: swh-deposit indexer::storage: name: swh-indexer user: swh-indexer webapp: name: swh-web user: swh-web scrubber: name: swh-scrubber user: swh-scrubber mirror: name: swh-mirror user: swh-mirror password: "%{lookup('swh::deploy::mirror::db::password')}" pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" pgbouncer::common::listen_addresses: - 0.0.0.0 pgbouncer::databases: - source_db: swh host: localhost auth_user: postgres port: 5433 alias: staging-swh - source_db: swh-scheduler host: localhost auth_user: postgres port: 5433 alias: staging-swh-scheduler - source_db: swh-vault host: localhost auth_user: postgres port: 5433 alias: staging-swh-vault - source_db: swh-lister host: localhost auth_user: postgres port: 5433 alias: staging-swh-lister - source_db: swh-deposit host: localhost auth_user: postgres port: 5433 alias: staging-swh-deposit - source_db: swh-indexer host: localhost auth_user: postgres port: 5433 alias: staging-swh-indexer - source_db: swh-web host: localhost auth_user: postgres port: 5433 alias: staging-swh-web - source_db: swh-mirror host: localhost auth_user: postgres port: 5433 alias: swh-mirror - source_db: swh-scrubber host: localhost auth_user: postgres port: 5433 alias: staging-swh-scrubber # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 16 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" swh::deploy::objstorage::slicing: 0:1/1:5 # Deploy the storage server as a public resource swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 4 swh::deploy::storage::backend::max_requests: 100 swh::deploy::storage::backend::max_requests_jitter: 10 # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::backend::workers: 4 nginx::worker_processes: 4 ## Reverse-proxy and frontend hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disable default vhost on port 80 apache::default_vhost: false # Elasticsearch elasticsearch::config::cluster::name: swh-search elasticsearch::config::discovery::seed_hosts: - search-esnode0.internal.staging.swh.network elasticsearch::config::cluster::initial_master_nodes: - search-esnode0 elasticsearch::jvm_options::heap_size: 16g elasticsearch::config::prometheus::indices: true swh::elasticsearch::search_nodes: - host: search-esnode0.internal.staging.swh.network port: 9200 swh::deploy::search::journal_client::service_types: - objects - indexed swh::deploy::search::journal_client::objects::consumer_group: swh.search.journal_client-v0.11 swh::deploy::search::journal_client::indexed::consumer_group: swh.search.journal_client.indexed-v0.11 swh::deploy::webapp::url: "https://webapp.staging.swh.network" swh::deploy::vault::e2e::storage: "%{alias('swh::remote_service::storage0::url')}" swh::config::keycloak::realm_name: SoftwareHeritageStaging # No historical file on staging swh::deploy::counters::cache_static_file: swh::deploy::counters::live_data_start: 1609462861 # 2021-01-01 swh::deploy::webapp::snapshot_e2e: uri: '/browse/snapshot/48dcf76ec1a3bd57ec117b1dace633691fdfd70d/branches/' regexp: - 'refs/tags/syslinux-3.20-pre2.*refs/tags/syslinux-3.20-pre3.*' swh::deploy::worker::loader_high_priority::queues: # bzr - save_code_now:swh.loader.bzr.tasks.LoadBazaar # cvs - save_code_now:swh.loader.cvs.tasks.LoadCvsRepository # git - save_code_now:swh.loader.git.tasks.UpdateGitRepository # mercurial - save_code_now:swh.loader.mercurial.tasks.LoadMercurial - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - save_code_now:swh.loader.svn.tasks.LoadSvnRepository - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # archives - save_code_now:swh.loader.package.archive.tasks.LoadArchive swh::deploy::scheduler::swh-scheduler-runner-priority::config::task_types: - load-bzr - load-cvs - load-git - load-svn - load-archive-files - load-hg syncoid::public_keys::storage1: type: ssh-ed25519 key: "AAAAC3NzaC1lZDI1NTE5AAAAIB0y7dvB0cBluC+Dy+w51P6JCbB18whd/IekP5148XsS" syncoid::public_keys::db1: type: ssh-ed25519 key: "AAAAC3NzaC1lZDI1NTE5AAAAILRVodfvLudSiOdWOPDSoN5MIwZPbyZAyClfr/SQUK4w" swh::deploy::maven_index_exporter::url: maven-exporter.internal.staging.swh.network