diff --git a/data/hostname/worker17.softwareheritage.org.yaml b/data/hostname/worker17.softwareheritage.org.yaml index 931efc76..9ed63b36 100644 --- a/data/hostname/worker17.softwareheritage.org.yaml +++ b/data/hostname/worker17.softwareheritage.org.yaml @@ -1,38 +1,21 @@ swh::deploy::worker::instances: - loader_oneshot - loader_git swh::deploy::worker::loader_oneshot::concurrency: 15 swh::deploy::worker::loader_oneshot::task_queues: # loader-git - oneshot:swh.loader.git.tasks.UpdateGitRepository # mercurial - oneshot:swh.loader.mercurial.tasks.LoadMercurial - oneshot:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - oneshot:swh.loader.svn.tasks.LoadSvnRepository - oneshot:swh.loader.svn.tasks.MountAndLoadSvnRepository - oneshot:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository -# Pipeline storage with retry, filter, buffer and finally writable storage -swh::deploy::worker::storage::pipeline: - cls: pipeline - steps: - - cls: buffer - min_batch_size: - content: 10000 - content_bytes: 1048576000 # 1Gib - directory: 10000 - revision: 10000 - release: 10000 - extid: 10000 - - cls: filter - - cls: retry - - "%{alias('swh::remote_service::storage::config::writable')}" - # Make the loader git service consume from a queue dedicated to subside lag swh::deploy::loader_git::queues: - oneshot3:swh.loader.git.tasks.UpdateGitRepository - swh::deploy::worker::loader_git::concurrency: 6 diff --git a/data/hostname/worker18.softwareheritage.org.yaml b/data/hostname/worker18.softwareheritage.org.yaml index 30d0c378..f3b85f03 100644 --- a/data/hostname/worker18.softwareheritage.org.yaml +++ b/data/hostname/worker18.softwareheritage.org.yaml @@ -1,38 +1,22 @@ swh::deploy::worker::instances: - loader_oneshot - loader_git swh::deploy::worker::loader_oneshot::concurrency: 7 swh::deploy::worker::loader_oneshot::task_queues: # loader-git - oneshot3:swh.loader.git.tasks.UpdateGitRepository - oneshot2:swh.loader.git.tasks.UpdateGitRepository # mercurial - oneshot2:swh.loader.mercurial.tasks.LoadMercurial - oneshot2:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - oneshot2:swh.loader.svn.tasks.LoadSvnRepository - oneshot2:swh.loader.svn.tasks.MountAndLoadSvnRepository - oneshot2:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository -# Pipeline storage with retry, filter, buffer and finally writable storage -swh::deploy::worker::storage::pipeline: - cls: pipeline - steps: - - cls: buffer - min_batch_size: - content: 10000 - content_bytes: 1048576000 # 1Gib - directory: 10000 - revision: 10000 - release: 10000 - extid: 10000 - - cls: filter - - cls: retry - - "%{alias('swh::remote_service::storage::config::writable')}" - # Make the loader git service consume from a queue dedicated to subside lag swh::deploy::loader_git::queues: - oneshot3:swh.loader.git.tasks.UpdateGitRepository swh::deploy::worker::loader_git::concurrency: 6