diff --git a/docker/conf/loader.yml b/docker/conf/loader.yml index 75464ca..e45acb1 100644 --- a/docker/conf/loader.yml +++ b/docker/conf/loader.yml @@ -1,51 +1,50 @@ storage: - cls: filter - args: - storage: - cls: buffer - args: - storage: - cls: remote - args: - url: http://swh-storage:5002/ - min_batch_size: - content: 10000 - content_bytes: 104857600 - directory: 1000 - revision: 1000 + cls: pipeline + steps: + - cls: validate + - cls: filter + - cls: buffer + min_batch_size: + content: 10000 + content_bytes: 104857600 + directory: 1000 + revision: 1000 + - cls: remote + url: http://swh-storage:5002/ + scheduler: cls: remote args: url: http://swh-scheduler:5008/ deposit: url: http://swh-deposit:5006/1/private auth: username: test password: test celery: task_broker: amqp://guest:guest@amqp// task_queues: - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - swh.loader.git.tasks.UpdateGitRepository - swh.loader.mercurial.tasks.LoadArchiveMercurial - swh.loader.mercurial.tasks.LoadMercurial - swh.loader.package.archive.tasks.LoadArchive - swh.loader.package.cran.tasks.LoadCRAN - swh.loader.package.debian.tasks.LoadDebian - swh.loader.package.deposit.tasks.LoadDeposit - swh.loader.package.npm.tasks.LoadNpm - swh.loader.package.pypi.tasks.LoadPyPI - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - swh.deposit.loader.tasks.ChecksDepositTsk lister_db_url: postgresql://postgres@swh-listers-db/swh-listers url: 'http://swh-deposit:5006'