diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -2233,10 +2233,7 @@ # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@rabbitmq:5672/%2f" -swh::deploy::worker::instances: - - loader_debian - - loader_git - - lister +swh::deploy::worker::instances: [] swh::deploy::loader_git::sentry_swh_package: swh.loader.git swh::deploy::loader_git::sentry_environment: "%{alias('swh::deploy::environment')}" @@ -2256,6 +2253,26 @@ - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository +# high priority loader (save-code-now, ...) +swh::deploy::worker::loader_high_priority::config_file: "%{hiera('swh::conf_directory')}/loader_high_priority.yml" +swh::deploy::worker::loader_high_priority::concurrency: 1 +swh::deploy::worker::loader_high_priority::max_tasks_per_child: 100 +swh::deploy::worker::loader_high_priority::loglevel: info +swh::deploy::worker::loader_high_priority::config: + storage: "%{alias('swh::deploy::worker::storage::pipeline')}" + max_content_size: 104857600 + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_queues: + # loader-git + - swh.loader.git.tasks.UpdateGitRepositoryHigh + # mercurial + - swh.loader.mercurial.tasks.LoadMercurialHigh + - swh.loader.mercurial.tasks.LoadArchiveMercurialHigh + # svn + - swh.loader.svn.tasks.LoadSvnRepositoryHigh + - swh.loader.svn.tasks.MountAndLoadSvnRepositoryHigh + # for all loader packages swh::deploy::loader_core::sentry_swh_package: swh.loader.core swh::deploy::loader_core::sentry_environment: "%{hiera('swh::deploy::environment')}" diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -44,6 +44,7 @@ - loader_svn - vault_cooker - lister + - loader_high_priority #### Rabbitmq instance to use # swh::deploy::worker::task_broker::password in private data diff --git a/data/subnets/sesi_rocquencourt.yaml b/data/subnets/sesi_rocquencourt.yaml --- a/data/subnets/sesi_rocquencourt.yaml +++ b/data/subnets/sesi_rocquencourt.yaml @@ -33,3 +33,4 @@ - loader_npm - loader_pypi - loader_svn + - loader_high_priority diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -38,6 +38,7 @@ - loader_npm - loader_pypi - loader_svn + - loader_high_priority dns::forward_zones: { } diff --git a/site-modules/profile/manifests/swh/deploy/base_loader_git.pp b/site-modules/profile/manifests/swh/deploy/base_loader_git.pp --- a/site-modules/profile/manifests/swh/deploy/base_loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/base_loader_git.pp @@ -1,5 +1,4 @@ # Git Loader base configuration - class profile::swh::deploy::base_loader_git { include ::profile::swh::deploy::loader diff --git a/site-modules/profile/manifests/swh/deploy/base_loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/base_loader_mercurial.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/base_loader_mercurial.pp @@ -0,0 +1,11 @@ +# Mercurial Loader base configuration +class profile::swh::deploy::base_loader_mercurial { + include ::profile::swh::deploy::loader + + $packages = ['python3-swh.loader.mercurial'] + + package {$packages: + ensure => 'present', + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/base_loader_svn.pp b/site-modules/profile/manifests/swh/deploy/base_loader_svn.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/base_loader_svn.pp @@ -0,0 +1,11 @@ +# Svn Loader base configuration +class profile::swh::deploy::base_loader_svn { + include ::profile::swh::deploy::loader + + $packages = ['python3-swh.loader.svn'] + + package {$packages: + ensure => 'present', + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp @@ -7,5 +7,8 @@ ensure => 'present', sentry_name => 'loader_core', private_tmp => $private_tmp, + require => [ + Package[$::profile::swh::deploy::base_loader_git::packages], + ], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp @@ -5,8 +5,7 @@ ::profile::swh::deploy::worker::instance {'loader_git': ensure => present, require => [ - Class['profile::swh::deploy::base_loader_git'], + Package[$::profile::swh::deploy::base_loader_git::packages], ], } - } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_high_priority.pp @@ -0,0 +1,14 @@ +# Deployment for high priority loader +class profile::swh::deploy::worker::loader_high_priority { + include ::profile::swh::deploy::base_loader_git + include ::profile::swh::deploy::base_loader_mercurial + include ::profile::swh::deploy::base_loader_svn + + $packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages + + ::profile::swh::deploy::worker::instance {'loader_high_priority': + ensure => present, + require => Package[$packages], + } + +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp @@ -1,17 +1,13 @@ # Deployment for swh-loader-mercurial (disk) class profile::swh::deploy::worker::loader_mercurial { + include ::profile::swh::deploy::base_loader_mercurial $private_tmp = lookup('swh::deploy::worker::loader_mercurial::private_tmp') - $packages = ['python3-swh.loader.mercurial'] - - package {$packages: - ensure => 'present', - } ::profile::swh::deploy::worker::instance {'loader_mercurial': ensure => 'present', private_tmp => $private_tmp, require => [ - Package[$packages], + Package[$::profile::swh::deploy::base_loader_mercurial::packages], ], } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp @@ -1,19 +1,13 @@ # Deployment for swh-loader-svn class profile::swh::deploy::worker::loader_svn { - $packages = ['python3-swh.loader.svn'] + include ::profile::swh::deploy::base_loader_svn $limit_no_file = lookup('swh::deploy::worker::loader_svn::limit_no_file') $private_tmp = lookup('swh::deploy::worker::loader_svn::private_tmp') - package {$packages: - ensure => 'present', - } - ::profile::swh::deploy::worker::instance {'loader_svn': ensure => present, limit_no_file => $limit_no_file, private_tmp => $private_tmp, - require => [ - Package[$packages], - ], + require => Package[$::profile::swh::deploy::base_loader_svn::packages], } }