diff --git a/data/hostname/worker17.softwareheritage.org.yaml b/data/hostname/worker17.softwareheritage.org.yaml index 2e2367b7..345b0021 100644 --- a/data/hostname/worker17.softwareheritage.org.yaml +++ b/data/hostname/worker17.softwareheritage.org.yaml @@ -1,15 +1,30 @@ swh::deploy::worker::instances: - loader_oneshot swh::deploy::worker::loader_oneshot::concurrency: 15 swh::deploy::worker::loader_oneshot::task_queues: # loader-git - oneshot:swh.loader.git.tasks.UpdateGitRepository # mercurial - oneshot:swh.loader.mercurial.tasks.LoadMercurial - oneshot:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - oneshot:swh.loader.svn.tasks.LoadSvnRepository - oneshot:swh.loader.svn.tasks.MountAndLoadSvnRepository - oneshot:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository +# Pipeline storage with retry, filter, buffer and finally writable storage +swh::deploy::worker::storage::pipeline: + cls: pipeline + steps: + - cls: buffer + min_batch_size: + content: 10000 + content_bytes: 1048576000 # 1Gib + directory: 10000 + revision: 10000 + release: 10000 + extid: 10000 + - cls: filter + - cls: retry + - "%{alias('swh::remote_service::storage::config::writable')}" diff --git a/data/hostname/worker18.softwareheritage.org.yaml b/data/hostname/worker18.softwareheritage.org.yaml index 7cd9c0e6..73df94ef 100644 --- a/data/hostname/worker18.softwareheritage.org.yaml +++ b/data/hostname/worker18.softwareheritage.org.yaml @@ -1,15 +1,30 @@ swh::deploy::worker::instances: - loader_oneshot swh::deploy::worker::loader_oneshot::concurrency: 7 swh::deploy::worker::loader_oneshot::task_queues: # loader-git - oneshot2:swh.loader.git.tasks.UpdateGitRepository # mercurial - oneshot2:swh.loader.mercurial.tasks.LoadMercurial - oneshot2:swh.loader.mercurial.tasks.LoadArchiveMercurial # svn - oneshot2:swh.loader.svn.tasks.LoadSvnRepository - oneshot2:swh.loader.svn.tasks.MountAndLoadSvnRepository - oneshot2:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository +# Pipeline storage with retry, filter, buffer and finally writable storage +swh::deploy::worker::storage::pipeline: + cls: pipeline + steps: + - cls: buffer + min_batch_size: + content: 10000 + content_bytes: 1048576000 # 1Gib + directory: 10000 + revision: 10000 + release: 10000 + extid: 10000 + - cls: filter + - cls: retry + - "%{alias('swh::remote_service::storage::config::writable')}"