diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml index 7794da3f..584955f6 100644 --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -1,335 +1,331 @@ --- swh::deploy::environment: staging swh::deploy::worker::loader_nixguix::loglevel: debug swh::deploy::storage::db::host: db1.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db1.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: db1.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db1.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::webapp::db::host: db1.internal.staging.swh.network # swh::deploy::webapp::db::password in private data swh::deploy::worker::instances: - checker_deposit - loader_archive - loader_cran - loader_debian - loader_deposit - loader_nixguix - loader_git - loader_mercurial - loader_npm - loader_pypi - loader_svn - vault_cooker - lister - loader_high_priority #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage0::url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config::storage0: cls: remote args: url: "%{alias('swh::remote_service::storage0::url')}" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::counters::url: "http://counters0.internal.staging.swh.network:%{hiera('swh::remote_service::counters::port')}/" swh::deploy::deposit::url: https://deposit.staging.swh.network swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network" # do not save pack swh::deploy::worker::loader_git::save_data_path: "" swh::deploy::worker::loader_git::concurrency: 1 zookeeper::clusters: rocquencourt_staging: '1': journal0.internal.staging.swh.network kafka::broker::heap_opts: "-Xmx3G -Xms3G" kafka::clusters: rocquencourt_staging: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 public_hostname: broker0.journal.staging.swh.network broker::heap_opts: "%{alias('kafka::broker::heap_opts')}" superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 cluster_config_overrides: offsets.topic.replication.factor: 1 # this is mandatory with only one node public_listener_network: "%{alias('kafka::cluster::public_network')}" swh::deploy::journal::brokers: - journal0.internal.staging.swh.network swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging swh::deploy::deposit::reverse_proxy::backend_http_host: deposit.internal.staging.swh.network swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging swh::deploy::webapp::reverse_proxy::backend_http_host: webapp.internal.staging.swh.network swh::remote_service::objstorage::config::rw: cls: remote url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::objstorage::port')}/" swh::remote_service::objstorage::config::ro: cls: filtered storage_conf: "%{alias('swh::remote_service::objstorage::config::rw')}" filters_conf: - type: readonly swh::deploy::objstorage::vhost::letsencrypt_cert: objstorage_staging swh::deploy::objstorage::reverse_proxy::backend_http_host: objstorage0.internal.staging.swh.network swh::deploy::objstorage::reverse_proxy::backend_http_port: "%{lookup('swh::remote_service::objstorage::port')}" swh::deploy::objstorage::backend::public_server_name: objstorage.staging.swh.network objstorage.internal.staging.swh.network swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::ro')}" swh::remote_service::objstorage::config::writable: "%{alias('swh::remote_service::objstorage::config::rw')}" swh::deploy::objstorage::backend::server_names: - "%{alias('swh::deploy::objstorage::backend::public_server_name')}" - "%{::swh_hostname.internal_fqdn}" - "%{::hostname}" - 127.0.0.1 - localhost - "::1" swh::deploy::reverse_proxy::services: - deposit - webapp - objstorage swh::postgresql::version: '12' swh::postgresql::port: 5433 swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main" swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres" swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}" swh::postgresql::listen_addresses: - 0.0.0.0 swh::postgresql::network_accesses: - 192.168.100.0/24 # Monitoring - 192.168.130.0/24 # Staging services swh::postgresql::shared_buffers: 32GB postgresql::server::config_entries: shared_buffers: "%{alias('swh::postgresql::shared_buffers')}" cluster_name: "%{alias('swh::postgresql::cluster_name')}" postgresql::globals::version: "%{alias('swh::postgresql::version')}" swh::dbs: storage: name: swh user: swh scheduler: name: swh-scheduler user: swh-scheduler vault: name: swh-vault user: swh-vault lister: name: swh-lister user: swh-lister deposit: name: swh-deposit user: swh-deposit indexer::storage: name: swh-indexer user: swh-indexer webapp: name: swh-web user: swh-web mirror: name: swh-mirror user: swh-mirror password: "%{lookup('swh::deploy::mirror::db::password')}" pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf" pgbouncer::common::listen_addresses: - 0.0.0.0 pgbouncer::databases: - source_db: swh host: localhost auth_user: postgres port: 5433 alias: staging-swh - source_db: swh-scheduler host: localhost auth_user: postgres port: 5433 alias: staging-swh-scheduler - source_db: swh-vault host: localhost auth_user: postgres port: 5433 alias: staging-swh-vault - source_db: swh-lister host: localhost auth_user: postgres port: 5433 alias: staging-swh-lister - source_db: swh-deposit host: localhost auth_user: postgres port: 5433 alias: staging-swh-deposit - source_db: swh-indexer host: localhost auth_user: postgres port: 5433 alias: staging-swh-indexer - source_db: swh-web host: localhost auth_user: postgres port: 5433 alias: staging-swh-web - source_db: swh-mirror host: localhost auth_user: postgres port: 5433 alias: swh-mirror # open objstorage api swh::deploy::objstorage::backend::listen::host: 0.0.0.0 swh::deploy::objstorage::backend::workers: 4 swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}" swh::deploy::objstorage::slicing: 0:1/1:5 # Deploy the storage server as a public resource swh::deploy::storage::backend::listen::host: 0.0.0.0 swh::deploy::storage::backend::workers: 4 swh::deploy::storage::backend::max_requests: 100 swh::deploy::storage::backend::max_requests_jitter: 10 # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 swh::deploy::indexer::storage::backend::workers: 4 nginx::worker_processes: 4 ## Reverse-proxy and frontend hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disable default vhost on port 80 apache::default_vhost: false # Elasticsearch elastic::elk_version: '7.9.3' elasticsearch::config::cluster::name: swh-search elasticsearch::config::discovery::seed_hosts: - search-esnode0.internal.staging.swh.network elasticsearch::config::cluster::initial_master_nodes: - search-esnode0 elasticsearch::jvm_options::heap_size: 16g elasticsearch::config::prometheus::indices: true swh::elasticsearch::search_nodes: - host: search-esnode0.internal.staging.swh.network port: 9200 -swh::remote_service::search::config: - cls: remote - url: "http://search0.internal.staging.swh.network:%{hiera('swh::remote_service::search::port')}/" - # The webapp can query elasticsearch to get some info related to scheduler # tasks associated to save code now request. Modifying this so the staging # webapp does not access the production indexes swh::deploy::webapp::config::es_workers_index_url: http://search-esnode0.internal.staging.swh.network:9200/swh_workers-* swh::deploy::webapp::metadata_search_backend: swh-search swh::deploy::search::index: origin-v0.9.0 swh::deploy::search::journal_client::service_types: - objects - indexed swh::deploy::search::journal_client::objects::consumer_group: swh.search.journal_client-v0.9.0 swh::deploy::search::journal_client::indexed::consumer_group: swh.search.journal_client.indexed-v0.9.0 swh::deploy::webapp::url: "https://webapp.staging.swh.network" swh::deploy::vault::e2e::storage: "%{alias('swh::remote_service::storage0::url')}" swh::config::keycloak::realm_name: SoftwareHeritageStaging # No historical file on staging swh::deploy::counters::cache_static_file: swh::deploy::counters::live_data_start: 1609462861 # 2021-01-01 swh::deploy::webapp::snapshot_e2e: uri: '/browse/snapshot/48dcf76ec1a3bd57ec117b1dace633691fdfd70d/branches/' regexp: - 'refs/tags/syslinux-3.20-pre2.*refs/tags/syslinux-3.20-pre3.*' diff --git a/data/hostname/webapp.internal.staging.swh.network.yaml b/data/hostname/webapp.internal.staging.swh.network.yaml index a1825fa5..e33c758a 100644 --- a/data/hostname/webapp.internal.staging.swh.network.yaml +++ b/data/hostname/webapp.internal.staging.swh.network.yaml @@ -1,54 +1,60 @@ networks: eth0: address: 192.168.130.30 netmask: 255.255.255.0 gateway: 192.168.130.1 +swh::deploy::storage::db::user: guest +swh::deploy::storage::db::password: guest +swh::deploy::storage::backend::listen::host: "%{::fqdn}" + +swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::local_internal_network')}" + swh::deploy::webapp::backend::workers: 16 swh::deploy::webapp::backend::http_keepalive: 5 swh::deploy::webapp::backend::http_timeout: 3600 swh::deploy::webapp::backend::reload_mercy: 3600 swh::deploy::webapp::config::throttling: cache_uri: "%{hiera('memcached::server::bind')}:%{hiera('memcached::server::port')}" scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 129.168.130.0/24 swh_api_origin_search: limiter_rate: default: 10/m swh_api_origin_visit_latest: # This endpoint gets called a lot (by default, up to 70 times # per origin search), so it deserves a much higher rate-limit # than the rest of the API. limiter_rate: default: 700/m exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 192.168.130.0/24 swh_vault_cooking: limiter_rate: default: 120/h GET: 60/m exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 192.168.130.0/24 swh_save_origin: limiter_rate: default: 120/h POST: 10/h exempted_networks: - 127.0.0.0/8 - 192.168.100.0/23 - 129.168.130.0/24 swh::deploy::webapp::config::keycloak: server_url: "https://%{hiera('keycloak::vhost::name')}/auth/" realm_name: SoftwareHeritageStaging swh::deploy::webapp::instance_name: webapp.staging.swh.network diff --git a/data/hostname/webapp1.internal.softwareheritage.org.yaml b/data/hostname/webapp1.internal.softwareheritage.org.yaml index 43e93ee9..0db6fa32 100644 --- a/data/hostname/webapp1.internal.softwareheritage.org.yaml +++ b/data/hostname/webapp1.internal.softwareheritage.org.yaml @@ -1,28 +1,30 @@ hitch::frontend: "[*]:443" hitch::proxy_support: true varnish::http_port: 80 apache::http_port: 9080 # Disable default vhost on port 80 apache::default_vhost: false swh::deploy::webapp::vhost::letsencrypt_cert: archive_webapp1 swh::deploy::storage::db::host: somerset.internal.softwareheritage.org swh::deploy::storage::backend::workers: 16 swh::deploy::storage::backend::listen::host: "%{::fqdn}" +swh::deploy::storage::db::user: guest +swh::deploy::storage::db::password: guest swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::local_internal_network')}" swh::deploy::reverse_proxy::services: - webapp swh::deploy::webapp::metadata_search_backend: swh-search swh::deploy::search::index: origin-v0.9.0 swh::deploy::search::read_alias: origin-read-v0.9.0 swh::deploy::search::write_alias: origin-write-v0.9.0 # already running for the main instance so no need here swh::deploy::webapp::cron::refresh_statuses: false swh::deploy::savecodenow::e2e::activate: false diff --git a/manifests/site.pp b/manifests/site.pp index 9d11f80a..8446f95e 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,208 +1,208 @@ node /^(orsay|beaubourg|hypervisor\d+|branly|pompidou|uffizi)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin include profile::export_archive_counters } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_rp_webapps } node 'webapp1.internal.softwareheritage.org' { - include role::swh_rp_webapp_with_swh_search_and_storage + include role::swh_rp_webapp } node /^search-esnode\d\.internal\.softwareheritage\.org$/ { include role::swh_elasticsearch } node /^search\d\.internal\.softwareheritage\.org$/ { include role::swh_search_with_journal_client } node /^counters\d\.internal\.softwareheritage\.org$/ { include role::swh_counters_with_journal_client } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler_with_journal_client } node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup include role::postgresql_backup } node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } node /^kafka\d+\./ { include role::swh_kafka_broker } node /^cassandra\d+\./ { include role::swh_cassandra_node } node 'granet.internal.softwareheritage.org' { include role::swh_graph_backend } node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } node /^saam\.(internal\.)?softwareheritage\.org$/ { include role::swh_storage_baremetal } node 'storage01.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cloud } node /^getty.(internal.)?softwareheritage.org$/ { include role::swh_journal_orchestrator_with_backfill_config } node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } node 'riverside.internal.softwareheritage.org' { include role::swh_sentry } node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'kelvingrove.internal.softwareheritage.org' { include role::swh_idp_primary } node 'giverny.softwareheritage.org' { include role::swh_desktop } node /^db\d\.internal\.staging\.swh\.network$/ { include role::swh_database include profile::postgresql::server include profile::pgbouncer include profile::postgresql::client } node "bardo.internal.admin.swh.network" { include role::swh_hedgedoc } node 'scheduler0.internal.staging.swh.network' { include role::swh_scheduler_with_journal_client include profile::postgresql::client } node 'gateway.internal.staging.swh.network' { include role::swh_gateway } node /^storage\d\.internal\.staging\.swh\.network$/ { include role::swh_storage_with_backfill_config } node /^worker\d\.internal\.staging\.swh\.network$/ { include role::swh_worker_inria } node /^search-esnode\d\.internal\.staging\.swh\.network$/ { include role::swh_elasticsearch } node /^search\d\.internal\.staging\.swh\.network$/ { include role::swh_search_with_journal_client } node /^counters\d\.internal\.staging\.swh\.network$/ { include role::swh_counters_with_journal_client } node 'webapp.internal.staging.swh.network' { include role::swh_webapp } node 'deposit.internal.staging.swh.network' { include role::swh_deposit } node 'vault.internal.staging.swh.network' { include role::swh_vault } node /^rp\d\.internal\.(staging|admin)\.swh\.network$/ { include role::swh_reverse_proxy } node 'journal0.internal.staging.swh.network' { include role::swh_journal_allinone } # Read-only storage for mirrors node 'objstorage0.internal.staging.swh.network' { include role::swh_remote_objstorage } node 'bojimans.internal.softwareheritage.org' { include role::swh_netbox } node /^mirror-test\.internal\.staging\.swh\.network$/ { include profile::postgresql::client } node default { include role::swh_base } diff --git a/site-modules/role/manifests/swh_rp_webapp_with_swh_search_and_storage.pp b/site-modules/role/manifests/swh_rp_webapp_with_swh_search_and_storage.pp deleted file mode 100644 index 7d869780..00000000 --- a/site-modules/role/manifests/swh_rp_webapp_with_swh_search_and_storage.pp +++ /dev/null @@ -1,5 +0,0 @@ -# Web ui profile with reverse proxy, swh-search and swh-storage backend -class role::swh_rp_webapp_with_swh_search_and_storage inherits role::swh_rp_webapp { - include profile::swh::deploy::search - include profile::swh::deploy::storage -} diff --git a/site-modules/role/manifests/swh_rp_webapps.pp b/site-modules/role/manifests/swh_rp_webapps.pp index 232f7e8b..8f318313 100644 --- a/site-modules/role/manifests/swh_rp_webapps.pp +++ b/site-modules/role/manifests/swh_rp_webapps.pp @@ -1,6 +1,8 @@ +# Declare: +# through swh_rp_webapp +# - webapp + associated packages (r/o storage, search) +# - apache reverse proxy +# - deposit class role::swh_rp_webapps inherits role::swh_rp_webapp { - # Extra deposit and storage services include profile::swh::deploy::deposit - include profile::swh::deploy::storage - include profile::swh::deploy::search } diff --git a/site-modules/role/manifests/swh_webapp.pp b/site-modules/role/manifests/swh_webapp.pp index 96fafce6..8daf1127 100644 --- a/site-modules/role/manifests/swh_webapp.pp +++ b/site-modules/role/manifests/swh_webapp.pp @@ -1,6 +1,10 @@ -# Webapp api without reverse proxy +# - Webapp api without reverse proxy +# - storage +# - search class role::swh_webapp inherits role::swh_server { include profile::memcached + include profile::swh::deploy::storage + include profile::swh::deploy::search include profile::swh::deploy::webapp include profile::filebeat }