diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -2372,6 +2372,20 @@ - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository +# One shot worker deactivated by default (e.g. first time ingestion on forge with +# restrictive loading ingestion parallelism policies) +swh::deploy::worker::loader_oneshot::config_file: "%{hiera('swh::conf_directory')}/loader_oneshot.yml" +swh::deploy::worker::loader_oneshot::concurrency: 1 +swh::deploy::worker::loader_oneshot::max_tasks_per_child: 100 +swh::deploy::worker::loader_oneshot::loglevel: info +swh::deploy::worker::loader_oneshot::task_queues: [] +swh::deploy::worker::loader_oneshot::config: + storage: "%{alias('swh::deploy::worker::storage::pipeline')}" + max_content_size: 104857600 + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_queues: "%{alias('swh::deploy::worker::loader_oneshot::task_queues')}" + # for all loader packages swh::deploy::loader_core::sentry_swh_package: swh.loader.core swh::deploy::loader_core::sentry_environment: "%{hiera('swh::deploy::environment')}" diff --git a/data/hostname/worker17.softwareheritage.org.yaml b/data/hostname/worker17.softwareheritage.org.yaml --- a/data/hostname/worker17.softwareheritage.org.yaml +++ b/data/hostname/worker17.softwareheritage.org.yaml @@ -1 +1,15 @@ -swh::deploy::worker::instances: [] +swh::deploy::worker::instances: + - loader_oneshot + +swh::deploy::worker::loader_oneshot::concurrency: 4 +swh::deploy::worker::loader_oneshot::task_queues: + # loader-git + - oneshot:swh.loader.git.tasks.UpdateGitRepository + # mercurial (not yet) + # - oneshot:swh.loader.mercurial.tasks.LoadMercurial + # - oneshot:swh.loader.mercurial.tasks.LoadArchiveMercurial + # svn + - oneshot:swh.loader.svn.tasks.LoadSvnRepository + - oneshot:swh.loader.svn.tasks.MountAndLoadSvnRepository + - oneshot:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository + diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp @@ -0,0 +1,14 @@ +# Deployment for oneshot loader +class profile::swh::deploy::worker::loader_oneshot { + include ::profile::swh::deploy::base_loader_git + include ::profile::swh::deploy::base_loader_mercurial + include ::profile::swh::deploy::base_loader_svn + + $packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages + + ::profile::swh::deploy::worker::instance {'loader_oneshot': + ensure => present, + require => Package[$packages], + } + +}