diff --git a/data/hostname/worker17.softwareheritage.org.yaml b/data/hostname/worker17.softwareheritage.org.yaml index 04b7039b..931efc76 100644 --- a/data/hostname/worker17.softwareheritage.org.yaml +++ b/data/hostname/worker17.softwareheritage.org.yaml @@ -1,35 +1,38 @@ swh::deploy::worker::instances: - loader_oneshot - loader_git swh::deploy::worker::loader_oneshot::concurrency: 15 swh::deploy::worker::loader_oneshot::task_queues: # loader-git - oneshot:swh.loader.git.tasks.UpdateGitRepository # mercurial - oneshot:swh.loader.mercurial.tasks.LoadMercurial - oneshot:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - oneshot:swh.loader.svn.tasks.LoadSvnRepository - oneshot:swh.loader.svn.tasks.MountAndLoadSvnRepository - oneshot:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # Pipeline storage with retry, filter, buffer and finally writable storage swh::deploy::worker::storage::pipeline: cls: pipeline steps: - cls: buffer min_batch_size: content: 10000 content_bytes: 1048576000 # 1Gib directory: 10000 revision: 10000 release: 10000 extid: 10000 - cls: filter - cls: retry - "%{alias('swh::remote_service::storage::config::writable')}" # Make the loader git service consume from a queue dedicated to subside lag swh::deploy::loader_git::queues: - oneshot3:swh.loader.git.tasks.UpdateGitRepository + + +swh::deploy::worker::loader_git::concurrency: 6 diff --git a/data/hostname/worker18.softwareheritage.org.yaml b/data/hostname/worker18.softwareheritage.org.yaml index 3240b01e..c75aa487 100644 --- a/data/hostname/worker18.softwareheritage.org.yaml +++ b/data/hostname/worker18.softwareheritage.org.yaml @@ -1,35 +1,37 @@ swh::deploy::worker::instances: - loader_oneshot - loader_git swh::deploy::worker::loader_oneshot::concurrency: 7 swh::deploy::worker::loader_oneshot::task_queues: # loader-git - oneshot2:swh.loader.git.tasks.UpdateGitRepository # mercurial - oneshot2:swh.loader.mercurial.tasks.LoadMercurial - oneshot2:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - oneshot2:swh.loader.svn.tasks.LoadSvnRepository - oneshot2:swh.loader.svn.tasks.MountAndLoadSvnRepository - oneshot2:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # Pipeline storage with retry, filter, buffer and finally writable storage swh::deploy::worker::storage::pipeline: cls: pipeline steps: - cls: buffer min_batch_size: content: 10000 content_bytes: 1048576000 # 1Gib directory: 10000 revision: 10000 release: 10000 extid: 10000 - cls: filter - cls: retry - "%{alias('swh::remote_service::storage::config::writable')}" # Make the loader git service consume from a queue dedicated to subside lag swh::deploy::loader_git::queues: - oneshot3:swh.loader.git.tasks.UpdateGitRepository + +swh::deploy::worker::loader_git::concurrency: 6