Page MenuHomeSoftware Heritage
Files F8394445

No OneTemporary
Actions

View Options

diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml
index 329bddea..11ac7a56 100644
--- a/data/deployments/staging/common.yaml
+++ b/data/deployments/staging/common.yaml
@@ -1,307 +1,306 @@
---
swh::deploy::environment: staging
swh::deploy::worker::loader_nixguix::loglevel: debug
swh::deploy::storage::db::host: db1.internal.staging.swh.network
swh::deploy::storage::db::user: swh
swh::deploy::storage::db::dbname: swh
swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network
swh::deploy::indexer::storage::db::user: swh-indexer
swh::deploy::indexer::storage::db::dbname: swh-indexer
swh::deploy::scheduler::db::host: db1.internal.staging.swh.network
swh::deploy::scheduler::db::dbname: swh-scheduler
swh::deploy::scheduler::db::user: swh-scheduler
swh::deploy::deposit::db::host: db1.internal.staging.swh.network
swh::deploy::deposit::db::dbuser: swh-deposit
swh::deploy::deposit::db::dbname: swh-deposit
swh::deploy::vault::db::host: db1.internal.staging.swh.network
swh::deploy::vault::db::user: swh-vault
swh::deploy::vault::db::dbname: swh-vault
swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network
swh::deploy::worker::lister::db::user: swh-lister
swh::deploy::worker::lister::db::name: swh-lister
swh::deploy::worker::instances:
- checker_deposit
- loader_archive
- loader_cran
- loader_debian
- loader_deposit
- loader_nixguix
- loader_git
- loader_mercurial
- loader_npm
- loader_pypi
- loader_svn
- vault_cooker
- lister
- - indexer_origin_intrinsic_metadata
#### Rabbitmq instance to use
# swh::deploy::worker::task_broker::password in private data
swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f"
#### Storage/Indexer/Vault/Scheduler services to use in staging area
swh::remote_service::storage::config::storage0:
cls: remote
args:
url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/"
swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}"
swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable
"%{alias('swh::remote_service::storage::config::storage0')}"
swh::remote_service::vault::config::vault0:
cls: remote
args:
url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/"
swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}"
swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}"
swh::remote_service::indexer::config::storage0:
cls: remote
url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/"
swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}"
swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}"
swh::remote_service::scheduler::config::scheduler0:
cls: remote
args:
url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/"
swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::deploy::deposit::url: https://deposit.staging.swh.network
swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network"
# do not save pack
swh::deploy::worker::loader_git::save_data_path: ""
swh::deploy::worker::loader_git::concurrency: 1
zookeeper::clusters:
rocquencourt_staging:
'1': journal0.internal.staging.swh.network
kafka::broker::heap_opts: "-Xmx3G -Xms3G"
kafka::clusters:
rocquencourt_staging:
zookeeper::chroot: '/kafka/softwareheritage'
zookeeper::servers:
- journal0.internal.staging.swh.network
brokers:
journal0.internal.staging.swh.network:
id: 1
public_hostname: broker0.journal.staging.swh.network
broker::heap_opts: "%{alias('kafka::broker::heap_opts')}"
superusers:
- User:swh-admin-olasd
# Users connecting in the plaintext endpoint are ANONYMOUS
# TODO: remove when explicit ACLs are given to producers
- User:ANONYMOUS
tls: true
plaintext_port: 9092
public_tls_port: 9093
internal_tls_port: 9094
cluster_config_overrides:
offsets.topic.replication.factor: 1 # this is mandatory with only one node
public_listener_network: "%{alias('kafka::cluster::public_network')}"
swh::deploy::journal::brokers:
- journal0.internal.staging.swh.network
swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging
swh::deploy::deposit::reverse_proxy::backend_http_host: deposit.internal.staging.swh.network
swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging
swh::deploy::webapp::reverse_proxy::backend_http_host: webapp.internal.staging.swh.network
swh::remote_service::objstorage::config::rw:
cls: remote
url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::objstorage::port')}/"
swh::remote_service::objstorage::config::ro:
cls: filtered
storage_conf: "%{alias('swh::remote_service::objstorage::config::rw')}"
filters_conf:
- type: readonly
swh::deploy::objstorage::vhost::letsencrypt_cert: objstorage_staging
swh::deploy::objstorage::reverse_proxy::backend_http_host: objstorage0.internal.staging.swh.network
swh::deploy::objstorage::reverse_proxy::backend_http_port: "%{lookup('swh::remote_service::objstorage::port')}"
swh::deploy::objstorage::backend::public_server_name:
objstorage.staging.swh.network
objstorage.internal.staging.swh.network
swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::ro')}"
swh::remote_service::objstorage::config::writable: "%{alias('swh::remote_service::objstorage::config::rw')}"
swh::deploy::objstorage::backend::server_names:
- "%{alias('swh::deploy::objstorage::backend::public_server_name')}"
- "%{::swh_hostname.internal_fqdn}"
- "%{::hostname}"
- 127.0.0.1
- localhost
- "::1"
swh::deploy::reverse_proxy::services:
- deposit
- webapp
- objstorage
swh::postgresql::version: '12'
swh::postgresql::port: 5433
swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main"
swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres"
swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}"
swh::postgresql::listen_addresses:
- 0.0.0.0
swh::postgresql::network_accesses:
- 192.168.100.0/24 # Monitoring
- 192.168.130.0/24 # Staging services
swh::postgresql::shared_buffers: 32GB
postgresql::server::config_entries:
shared_buffers: "%{alias('swh::postgresql::shared_buffers')}"
cluster_name: "%{alias('swh::postgresql::cluster_name')}"
postgresql::globals::version: "%{alias('swh::postgresql::version')}"
swh::dbs:
storage:
name: swh
user: swh
scheduler:
name: swh-scheduler
user: swh-scheduler
vault:
name: swh-vault
user: swh-vault
lister:
name: swh-lister
user: swh-lister
deposit:
name: swh-deposit
user: swh-deposit
indexer::storage:
name: swh-indexer
user: swh-indexer
clearly_defined:
name: clearly-defined
user: clearly-defined
pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf"
pgbouncer::common::listen_addresses:
- 0.0.0.0
pgbouncer::databases:
- source_db: swh
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh
- source_db: swh-scheduler
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-scheduler
- source_db: swh-vault
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-vault
- source_db: swh-lister
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-lister
- source_db: swh-deposit
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-deposit
- source_db: swh-indexer
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-indexer
- source_db: clearly-defined
host: localhost
auth_user: postgres
port: 5433
alias: staging-clearly-defined
# open objstorage api
swh::deploy::objstorage::backend::listen::host: 0.0.0.0
swh::deploy::objstorage::backend::workers: 4
swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}"
swh::deploy::objstorage::slicing: 0:1/1:5
# Deploy the storage server as a public resource
swh::deploy::storage::backend::listen::host: 0.0.0.0
swh::deploy::storage::backend::workers: 4
swh::deploy::storage::backend::max_requests: 100
swh::deploy::storage::backend::max_requests_jitter: 10
# Deploy the indexer storage server as a public resource
swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0
swh::deploy::indexer::storage::backend::workers: 4
swh::deploy::indexer::storage::config:
indexer_storage:
cls: local
db: "host=%{hiera('swh::deploy::indexer::storage::db::host')} port=%{hiera('swh::deploy::indexer::storage::db::port')} user=%{hiera('swh::deploy::indexer::storage::db::user')} dbname=%{hiera('swh::deploy::indexer::storage::db::dbname')} password=%{hiera('swh::deploy::indexer::storage::db::password')}"
journal_writer: "%{alias('swh::deploy::indexer::journal::writer::config')}"
nginx::worker_processes: 4
## Reverse-proxy and frontend
hitch::frontend: "[*]:443"
hitch::proxy_support: true
varnish::http_port: 80
apache::http_port: 9080
# Disable default vhost on port 80
apache::default_vhost: false
# Elasticsearch
elastic::elk_version: '7.9.3'
elasticsearch::config::cluster::name: swh-search
elasticsearch::config::discovery::seed_hosts:
- search-esnode0.internal.staging.swh.network
elasticsearch::config::cluster::initial_master_nodes:
- search-esnode0
elasticsearch::jvm_options::heap_size: 16g
elasticsearch::config::prometheus::indices: true
swh::elasticsearch::nodes:
- host: search-esnode0.internal.staging.swh.network
port: 9200
swh::deploy::search::backend::listen::host: 0.0.0.0
swh::remote_service::search::config:
cls: remote
url: "http://search0.internal.staging.swh.network:%{hiera('swh::remote_service::search::port')}/"
# The webapp can query elasticsearch to get some info related to scheduler
# tasks associated to save code now request. Modifying this so the staging
# webapp does not access the production indexes
swh::deploy::webapp::config::es_workers_index_url: http://search-esnode0.internal.staging.swh.network:9200/swh_workers-*
swh::deploy::search::journal_client::service_types:
- objects
- indexed
diff --git a/data/hostname/worker3.internal.staging.swh.network.yaml b/data/hostname/worker3.internal.staging.swh.network.yaml
new file mode 100644
index 00000000..e2a5bcb9
--- /dev/null
+++ b/data/hostname/worker3.internal.staging.swh.network.yaml
@@ -0,0 +1,5 @@
+swh::deploy::worker::instances:
+ - indexer_content_mimetype
+ - indexer_fossology_license
+ - indexer_origin_intrinsic_metadata
+

File Metadata

Mime Type
text/x-diff
Expires
Wed, Jun 4, 7:25 PM (1 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3254553

Event Timeline

About · Developer information · Legal