diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1965,6 +1965,8 @@ extraction_dir: /tmp/swh-deposit/archive/ swh_authority_url: "%{hiera('swh::deploy::deposit::url')}/" +swh::deploy::worker::loader::max_content_size: 104857600 + swh::deploy::worker::loader_deposit::config_file: "%{hiera('swh::conf_directory')}/loader_deposit.yml" swh::deploy::worker::loader_deposit::concurrency: 1 swh::deploy::worker::loader_deposit::private_tmp: true @@ -1972,7 +1974,7 @@ # deposit_basic_auth_swhworker_{username|password} in private_data swh::deploy::worker::loader_deposit::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2341,7 +2343,7 @@ swh::deploy::worker::loader_git::loglevel: info swh::deploy::worker::loader_git::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" temp_file_cutoff: "%{alias('swh::deploy::worker::loader_git::temp_file_cutoff')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" @@ -2358,7 +2360,7 @@ swh::deploy::worker::loader_high_priority::loglevel: info swh::deploy::worker::loader_high_priority::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2372,6 +2374,20 @@ - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository +# One shot worker deactivated by default (e.g. first time ingestion on forge with +# restrictive loading ingestion parallelism policies) +swh::deploy::worker::loader_oneshot::config_file: "%{hiera('swh::conf_directory')}/loader_oneshot.yml" +swh::deploy::worker::loader_oneshot::concurrency: 1 +swh::deploy::worker::loader_oneshot::max_tasks_per_child: 100 +swh::deploy::worker::loader_oneshot::loglevel: info +swh::deploy::worker::loader_oneshot::task_queues: [] +swh::deploy::worker::loader_oneshot::config: + storage: "%{alias('swh::deploy::worker::storage::pipeline')}" + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" + celery: + task_broker: "%{alias('swh::deploy::worker::task_broker')}" + task_queues: "%{alias('swh::deploy::worker::loader_oneshot::task_queues')}" + # for all loader packages swh::deploy::loader_core::sentry_swh_package: swh.loader.core swh::deploy::loader_core::sentry_environment: "%{hiera('swh::deploy::environment')}" @@ -2383,7 +2399,7 @@ swh::deploy::worker::loader_debian::loglevel: info swh::deploy::worker::loader_debian::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2395,7 +2411,7 @@ swh::deploy::worker::loader_archive::loglevel: info swh::deploy::worker::loader_archive::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2407,7 +2423,7 @@ swh::deploy::worker::loader_cran::loglevel: info swh::deploy::worker::loader_cran::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2419,7 +2435,7 @@ swh::deploy::worker::loader_nixguix::loglevel: info swh::deploy::worker::loader_nixguix::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2489,7 +2505,7 @@ swh::deploy::worker::loader_mercurial::loglevel: info swh::deploy::worker::loader_mercurial::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" reduce_effort: False clone_timeout_seconds: 7200 celery: @@ -2504,7 +2520,7 @@ swh::deploy::worker::loader_pypi::loglevel: info swh::deploy::worker::loader_pypi::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2516,7 +2532,7 @@ swh::deploy::worker::loader_npm::loglevel: info swh::deploy::worker::loader_npm::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: @@ -2533,7 +2549,7 @@ # Contains a password: in private data swh::deploy::worker::loader_svn::config: storage: "%{alias('swh::deploy::worker::storage::pipeline')}" - max_content_size: 104857600 + max_content_size: "%{alias('swh::deploy::worker::loader::max_content_size')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_queues: diff --git a/data/hostname/worker17.softwareheritage.org.yaml b/data/hostname/worker17.softwareheritage.org.yaml --- a/data/hostname/worker17.softwareheritage.org.yaml +++ b/data/hostname/worker17.softwareheritage.org.yaml @@ -1 +1,15 @@ -swh::deploy::worker::instances: [] +swh::deploy::worker::instances: + - loader_oneshot + +swh::deploy::worker::loader_oneshot::concurrency: 4 +swh::deploy::worker::loader_oneshot::task_queues: + # loader-git + - oneshot:swh.loader.git.tasks.UpdateGitRepository + # mercurial (not yet) + # - oneshot:swh.loader.mercurial.tasks.LoadMercurial + # - oneshot:swh.loader.mercurial.tasks.LoadArchiveMercurial + # svn + - oneshot:swh.loader.svn.tasks.LoadSvnRepository + - oneshot:swh.loader.svn.tasks.MountAndLoadSvnRepository + - oneshot:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository + diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_oneshot.pp @@ -0,0 +1,14 @@ +# Deployment for oneshot loader +class profile::swh::deploy::worker::loader_oneshot { + include ::profile::swh::deploy::base_loader_git + include ::profile::swh::deploy::base_loader_mercurial + include ::profile::swh::deploy::base_loader_svn + + $packages = $::profile::swh::deploy::base_loader_git::packages + $::profile::swh::deploy::base_loader_mercurial::packages + $::profile::swh::deploy::base_loader_svn::packages + + ::profile::swh::deploy::worker::instance {'loader_oneshot': + ensure => present, + require => Package[$packages], + } + +}