Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F8394445
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
11 KB
Subscribers
None
View Options
diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml
index 329bddea..11ac7a56 100644
--- a/data/deployments/staging/common.yaml
+++ b/data/deployments/staging/common.yaml
@@ -1,307 +1,306 @@
---
swh::deploy::environment: staging
swh::deploy::worker::loader_nixguix::loglevel: debug
swh::deploy::storage::db::host: db1.internal.staging.swh.network
swh::deploy::storage::db::user: swh
swh::deploy::storage::db::dbname: swh
swh::deploy::indexer::storage::db::host: db1.internal.staging.swh.network
swh::deploy::indexer::storage::db::user: swh-indexer
swh::deploy::indexer::storage::db::dbname: swh-indexer
swh::deploy::scheduler::db::host: db1.internal.staging.swh.network
swh::deploy::scheduler::db::dbname: swh-scheduler
swh::deploy::scheduler::db::user: swh-scheduler
swh::deploy::deposit::db::host: db1.internal.staging.swh.network
swh::deploy::deposit::db::dbuser: swh-deposit
swh::deploy::deposit::db::dbname: swh-deposit
swh::deploy::vault::db::host: db1.internal.staging.swh.network
swh::deploy::vault::db::user: swh-vault
swh::deploy::vault::db::dbname: swh-vault
swh::deploy::worker::lister::db::host: db1.internal.staging.swh.network
swh::deploy::worker::lister::db::user: swh-lister
swh::deploy::worker::lister::db::name: swh-lister
swh::deploy::worker::instances:
- checker_deposit
- loader_archive
- loader_cran
- loader_debian
- loader_deposit
- loader_nixguix
- loader_git
- loader_mercurial
- loader_npm
- loader_pypi
- loader_svn
- vault_cooker
- lister
- - indexer_origin_intrinsic_metadata
#### Rabbitmq instance to use
# swh::deploy::worker::task_broker::password in private data
swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672/%2f"
#### Storage/Indexer/Vault/Scheduler services to use in staging area
swh::remote_service::storage::config::storage0:
cls: remote
args:
url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/"
swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}"
swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable
"%{alias('swh::remote_service::storage::config::storage0')}"
swh::remote_service::vault::config::vault0:
cls: remote
args:
url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/"
swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}"
swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}"
swh::remote_service::indexer::config::storage0:
cls: remote
url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/"
swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}"
swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}"
swh::remote_service::scheduler::config::scheduler0:
cls: remote
args:
url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/"
swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::deploy::deposit::url: https://deposit.staging.swh.network
swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network"
# do not save pack
swh::deploy::worker::loader_git::save_data_path: ""
swh::deploy::worker::loader_git::concurrency: 1
zookeeper::clusters:
rocquencourt_staging:
'1': journal0.internal.staging.swh.network
kafka::broker::heap_opts: "-Xmx3G -Xms3G"
kafka::clusters:
rocquencourt_staging:
zookeeper::chroot: '/kafka/softwareheritage'
zookeeper::servers:
- journal0.internal.staging.swh.network
brokers:
journal0.internal.staging.swh.network:
id: 1
public_hostname: broker0.journal.staging.swh.network
broker::heap_opts: "%{alias('kafka::broker::heap_opts')}"
superusers:
- User:swh-admin-olasd
# Users connecting in the plaintext endpoint are ANONYMOUS
# TODO: remove when explicit ACLs are given to producers
- User:ANONYMOUS
tls: true
plaintext_port: 9092
public_tls_port: 9093
internal_tls_port: 9094
cluster_config_overrides:
offsets.topic.replication.factor: 1 # this is mandatory with only one node
public_listener_network: "%{alias('kafka::cluster::public_network')}"
swh::deploy::journal::brokers:
- journal0.internal.staging.swh.network
swh::deploy::deposit::vhost::letsencrypt_cert: deposit_staging
swh::deploy::deposit::reverse_proxy::backend_http_host: deposit.internal.staging.swh.network
swh::deploy::webapp::vhost::letsencrypt_cert: archive_staging
swh::deploy::webapp::reverse_proxy::backend_http_host: webapp.internal.staging.swh.network
swh::remote_service::objstorage::config::rw:
cls: remote
url: "http://storage1.internal.staging.swh.network:%{hiera('swh::remote_service::objstorage::port')}/"
swh::remote_service::objstorage::config::ro:
cls: filtered
storage_conf: "%{alias('swh::remote_service::objstorage::config::rw')}"
filters_conf:
- type: readonly
swh::deploy::objstorage::vhost::letsencrypt_cert: objstorage_staging
swh::deploy::objstorage::reverse_proxy::backend_http_host: objstorage0.internal.staging.swh.network
swh::deploy::objstorage::reverse_proxy::backend_http_port: "%{lookup('swh::remote_service::objstorage::port')}"
swh::deploy::objstorage::backend::public_server_name:
objstorage.staging.swh.network
objstorage.internal.staging.swh.network
swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::ro')}"
swh::remote_service::objstorage::config::writable: "%{alias('swh::remote_service::objstorage::config::rw')}"
swh::deploy::objstorage::backend::server_names:
- "%{alias('swh::deploy::objstorage::backend::public_server_name')}"
- "%{::swh_hostname.internal_fqdn}"
- "%{::hostname}"
- 127.0.0.1
- localhost
- "::1"
swh::deploy::reverse_proxy::services:
- deposit
- webapp
- objstorage
swh::postgresql::version: '12'
swh::postgresql::port: 5433
swh::postgresql::cluster_name: "%{lookup('swh::postgresql::version')}/main"
swh::postgresql::datadir_base: "%{lookup('swh::base_directory')}/postgres"
swh::postgresql::datadir: "%{lookup('swh::postgresql::datadir_base')}/%{lookup('swh::postgresql::cluster_name')}"
swh::postgresql::listen_addresses:
- 0.0.0.0
swh::postgresql::network_accesses:
- 192.168.100.0/24 # Monitoring
- 192.168.130.0/24 # Staging services
swh::postgresql::shared_buffers: 32GB
postgresql::server::config_entries:
shared_buffers: "%{alias('swh::postgresql::shared_buffers')}"
cluster_name: "%{alias('swh::postgresql::cluster_name')}"
postgresql::globals::version: "%{alias('swh::postgresql::version')}"
swh::dbs:
storage:
name: swh
user: swh
scheduler:
name: swh-scheduler
user: swh-scheduler
vault:
name: swh-vault
user: swh-vault
lister:
name: swh-lister
user: swh-lister
deposit:
name: swh-deposit
user: swh-deposit
indexer::storage:
name: swh-indexer
user: swh-indexer
clearly_defined:
name: clearly-defined
user: clearly-defined
pgbouncer::auth_hba_file: "/etc/postgresql/%{lookup('swh::postgresql::cluster_name')}/pg_hba.conf"
pgbouncer::common::listen_addresses:
- 0.0.0.0
pgbouncer::databases:
- source_db: swh
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh
- source_db: swh-scheduler
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-scheduler
- source_db: swh-vault
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-vault
- source_db: swh-lister
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-lister
- source_db: swh-deposit
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-deposit
- source_db: swh-indexer
host: localhost
auth_user: postgres
port: 5433
alias: staging-swh-indexer
- source_db: clearly-defined
host: localhost
auth_user: postgres
port: 5433
alias: staging-clearly-defined
# open objstorage api
swh::deploy::objstorage::backend::listen::host: 0.0.0.0
swh::deploy::objstorage::backend::workers: 4
swh::deploy::objstorage::directory: "%{hiera('swh::deploy::storage::directory')}"
swh::deploy::objstorage::slicing: 0:1/1:5
# Deploy the storage server as a public resource
swh::deploy::storage::backend::listen::host: 0.0.0.0
swh::deploy::storage::backend::workers: 4
swh::deploy::storage::backend::max_requests: 100
swh::deploy::storage::backend::max_requests_jitter: 10
# Deploy the indexer storage server as a public resource
swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0
swh::deploy::indexer::storage::backend::workers: 4
swh::deploy::indexer::storage::config:
indexer_storage:
cls: local
db: "host=%{hiera('swh::deploy::indexer::storage::db::host')} port=%{hiera('swh::deploy::indexer::storage::db::port')} user=%{hiera('swh::deploy::indexer::storage::db::user')} dbname=%{hiera('swh::deploy::indexer::storage::db::dbname')} password=%{hiera('swh::deploy::indexer::storage::db::password')}"
journal_writer: "%{alias('swh::deploy::indexer::journal::writer::config')}"
nginx::worker_processes: 4
## Reverse-proxy and frontend
hitch::frontend: "[*]:443"
hitch::proxy_support: true
varnish::http_port: 80
apache::http_port: 9080
# Disable default vhost on port 80
apache::default_vhost: false
# Elasticsearch
elastic::elk_version: '7.9.3'
elasticsearch::config::cluster::name: swh-search
elasticsearch::config::discovery::seed_hosts:
- search-esnode0.internal.staging.swh.network
elasticsearch::config::cluster::initial_master_nodes:
- search-esnode0
elasticsearch::jvm_options::heap_size: 16g
elasticsearch::config::prometheus::indices: true
swh::elasticsearch::nodes:
- host: search-esnode0.internal.staging.swh.network
port: 9200
swh::deploy::search::backend::listen::host: 0.0.0.0
swh::remote_service::search::config:
cls: remote
url: "http://search0.internal.staging.swh.network:%{hiera('swh::remote_service::search::port')}/"
# The webapp can query elasticsearch to get some info related to scheduler
# tasks associated to save code now request. Modifying this so the staging
# webapp does not access the production indexes
swh::deploy::webapp::config::es_workers_index_url: http://search-esnode0.internal.staging.swh.network:9200/swh_workers-*
swh::deploy::search::journal_client::service_types:
- objects
- indexed
diff --git a/data/hostname/worker3.internal.staging.swh.network.yaml b/data/hostname/worker3.internal.staging.swh.network.yaml
new file mode 100644
index 00000000..e2a5bcb9
--- /dev/null
+++ b/data/hostname/worker3.internal.staging.swh.network.yaml
@@ -0,0 +1,5 @@
+swh::deploy::worker::instances:
+ - indexer_content_mimetype
+ - indexer_fossology_license
+ - indexer_origin_intrinsic_metadata
+
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Wed, Jun 4, 7:25 PM (1 w, 14 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3254553
Attached To
rSPSITE puppet-swh-site
Event Timeline
Log In to Comment