diff --git a/docker/conf/loader-deposit.yml b/docker/conf/loader-deposit.yml new file mode 100644 --- /dev/null +++ b/docker/conf/loader-deposit.yml @@ -0,0 +1,26 @@ +storage: + cls: pipeline + steps: + - cls: buffer + min_batch_size: + content: 10000 + content_bytes: 104857600 + directory: 1000 + revision: 1000 + - cls: filter + - cls: remote + url: http://swh-storage:5002/ + +celery: + task_broker: amqp://guest:guest@amqp// + task_queues: + - swh.loader.package.deposit.tasks.LoadDeposit + - swh.deposit.loader.tasks.ChecksDepositTsk + +deposit: + url: http://swh-deposit:5006/1/private + auth: + username: test + password: test + +url: 'http://swh-deposit:5006' diff --git a/docker/conf/loader.yml b/docker/conf/loader.yml --- a/docker/conf/loader.yml +++ b/docker/conf/loader.yml @@ -11,20 +11,8 @@ - cls: remote url: http://swh-storage:5002/ - -scheduler: - cls: remote - url: http://swh-scheduler:5008/ - -deposit: - url: http://swh-deposit:5006/1/private - auth: - username: test - password: test - celery: task_broker: amqp://guest:guest@amqp// - task_queues: - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository @@ -35,14 +23,8 @@ - swh.loader.package.archive.tasks.LoadArchive - swh.loader.package.cran.tasks.LoadCRAN - swh.loader.package.debian.tasks.LoadDebian - - swh.loader.package.deposit.tasks.LoadDeposit - swh.loader.package.npm.tasks.LoadNpm - swh.loader.package.pypi.tasks.LoadPyPI - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - - swh.deposit.loader.tasks.ChecksDepositTsk - -lister_db_url: postgresql://postgres@swh-listers-db/swh-listers - -url: 'http://swh-deposit:5006' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -337,12 +337,31 @@ depends_on: - swh-storage - swh-scheduler - - swh-deposit - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" + swh-loader-deposit: + image: swh/stack + build: ./ + env_file: + - ./env/common_python.env + - ./env/workers.env + user: swh + environment: + SWH_WORKER_INSTANCE: loader + SWH_CONFIG_FILENAME: /loader.yml + entrypoint: /entrypoint.sh + depends_on: + - swh-storage + - swh-scheduler + - swh-deposit + - amqp + volumes: + - "./conf/loader-deposit.yml:/loader.yml:ro" + - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" + # Indexer Celery workers swh-indexer: