diff --git a/data/location/sesi_rocquencourt_staging.yaml b/data/location/sesi_rocquencourt_staging.yaml index 3369b281..a6638b3d 100644 --- a/data/location/sesi_rocquencourt_staging.yaml +++ b/data/location/sesi_rocquencourt_staging.yaml @@ -1,131 +1,152 @@ --- dar::backup::enable: false dns::local_cache: false dns::nameservers: - 192.168.100.29 dns::search_domains: - internal.staging.swh.network dns::forward_zones: 'internal.softwareheritage.org.': - 192.168.100.29 '100.168.192.in-addr.arpa.': - 192.168.100.29 '101.168.192.in-addr.arpa.': - 192.168.100.29 'internal.staging.swh.network': - 192.168.100.29 '128.168.192.in-addr.arpa.': - 192.168.100.29 dns::forwarders: - 193.51.196.130 - 193.51.196.131 dns::forwarder_insecure: true ntp::servers: - sesi-ntp1.inria.fr - sesi-ntp2.inria.fr internal_network: 192.168.128.0/24 smtp::relayhost: '[smtp.inria.fr]' swh::deploy::storage::db::host: db0.internal.staging.swh.network swh::deploy::storage::db::user: swh swh::deploy::storage::db::dbname: swh swh::deploy::indexer::storage::db::host: db0.internal.staging.swh.network swh::deploy::indexer::storage::db::user: swh-indexer swh::deploy::indexer::storage::db::dbname: swh-indexer swh::deploy::scheduler::db::host: db0.internal.staging.swh.network swh::deploy::scheduler::db::dbname: swh-scheduler swh::deploy::scheduler::db::user: swh-scheduler swh::deploy::deposit::db::host: deposit.internal.staging.swh.network swh::deploy::deposit::db::dbuser: swh-deposit swh::deploy::deposit::db::dbname: swh-deposit swh::deploy::vault::db::host: db0.internal.staging.swh.network swh::deploy::vault::db::user: swh-vault swh::deploy::vault::db::dbname: swh-vault swh::deploy::worker::lister::db::host: db0.internal.staging.swh.network swh::deploy::worker::lister::db::user: swh-lister swh::deploy::worker::lister::db::name: swh-lister swh::deploy::worker::instances: - checker_deposit + - loader_debian - loader_deposit - loader_git - vault_cooker - lister - indexer_origin_intrinsic_metadata #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@scheduler0.internal.staging.swh.network:5672//" #### Storage/Indexer/Vault/Scheduler services to use in staging area swh::remote_service::storage::config::storage0: cls: remote args: url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote args: url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::indexer::config::storage0: cls: remote args: url: "http://storage0.internal.staging.swh.network:%{hiera('swh::remote_service::indexer::port')}/" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::indexer::config::writable: "%{alias('swh::remote_service::indexer::config::storage0')}" swh::remote_service::scheduler::config::scheduler0: cls: remote args: url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::deploy::deposit::url: http://deposit.internal.staging.swh.network swh::deploy::worker::loader_git::config: storage: "%{alias('swh::remote_service::storage::config::writable')}" save_data: false directory_packet_size: 100 celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_modules: - swh.loader.git.tasks task_queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository zookeeper::clusters: rocquencourt: 1: journal0.internal.staging.swh.network kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal0.internal.staging.swh.network brokers: journal0.internal.staging.swh.network: id: 1 swh::deploy::journal::brokers: - journal0.internal.staging.swh.network + +swh::deploy::worker::loader_debian::config: + storage: + cls: filter + args: + storage: + cls: buffer + args: + storage: "%{alias('swh::remote_service::storage::config::writable')}" + min_batch_size: + content: 100000 + content_bytes: 104857600 + directory: 1000 + revision: 1000 + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_modules: + - swh.loader.package.tasks + task_queues: + - swh.loader.package.tasks.LoadDebian diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp index cec46a2d..5c5152f8 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp @@ -1,11 +1,11 @@ # Deployment for loader-debian class profile::swh::deploy::worker::loader_debian { - include ::profile::swh::deploy::loader_package + include ::profile::swh::deploy::worker::loader_package ::profile::swh::deploy::worker::instance {'loader_debian': ensure => present, require => [ Package[$packages], ], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_package.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_package.pp index 6f76d076..3d05a0b1 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/loader_package.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_package.pp @@ -1,8 +1,8 @@ # Base loader package configuration -class profile::swh::deploy::loader_package { +class profile::swh::deploy::worker::loader_package { $packages = ['python3-swh.loader.core'] package {$packages: ensure => 'present', } }