diff --git a/docker/conf/loader-deposit.yml b/docker/conf/loader-deposit.yml new file mode 100644 --- /dev/null +++ b/docker/conf/loader-deposit.yml @@ -0,0 +1,26 @@ +storage: + cls: pipeline + steps: + - cls: buffer + min_batch_size: + content: 10000 + content_bytes: 104857600 + directory: 1000 + revision: 1000 + - cls: filter + - cls: remote + url: http://swh-storage:5002/ + +celery: + task_broker: amqp://guest:guest@amqp// + task_queues: + - swh.loader.package.deposit.tasks.LoadDeposit + - swh.deposit.loader.tasks.ChecksDepositTsk + +deposit: + url: http://swh-deposit:5006/1/private + auth: + username: test + password: test + +url: 'http://swh-deposit:5006' diff --git a/docker/conf/loader.yml b/docker/conf/loader.yml --- a/docker/conf/loader.yml +++ b/docker/conf/loader.yml @@ -11,20 +11,8 @@ - cls: remote url: http://swh-storage:5002/ - -scheduler: - cls: remote - url: http://swh-scheduler:5008/ - -deposit: - url: http://swh-deposit:5006/1/private - auth: - username: test - password: test - celery: task_broker: amqp://guest:guest@amqp// - task_queues: - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository @@ -35,14 +23,8 @@ - swh.loader.package.archive.tasks.LoadArchive - swh.loader.package.cran.tasks.LoadCRAN - swh.loader.package.debian.tasks.LoadDebian - - swh.loader.package.deposit.tasks.LoadDeposit - swh.loader.package.npm.tasks.LoadNpm - swh.loader.package.pypi.tasks.LoadPyPI - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - - swh.deposit.loader.tasks.ChecksDepositTsk - -lister_db_url: postgresql://postgres@swh-listers-db/swh-listers - -url: 'http://swh-deposit:5006' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -337,12 +337,31 @@ depends_on: - swh-storage - swh-scheduler - - swh-deposit - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" + swh-loader-deposit: + image: swh/stack + build: ./ + env_file: + - ./env/common_python.env + - ./env/workers.env + user: swh + environment: + SWH_WORKER_INSTANCE: loader-deposit + SWH_CONFIG_FILENAME: /loader-deposit.yml + entrypoint: /entrypoint.sh + depends_on: + - swh-storage + - swh-scheduler + - swh-deposit + - amqp + volumes: + - "./conf/loader-deposit.yml:/loader-deposit.yml:ro" + - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" + # Indexer Celery workers swh-indexer: diff --git a/docker/env/workers.env b/docker/env/workers.env --- a/docker/env/workers.env +++ b/docker/env/workers.env @@ -1,3 +1,4 @@ CONCURRENCY=1 MAX_TASKS_PER_CHILD=10 LOGLEVEL=DEBUG +SWH_SCHEDULER_INSTANCE=http://swh-scheduler:5008 diff --git a/docker/services/swh-worker/entrypoint.sh b/docker/services/swh-worker/entrypoint.sh --- a/docker/services/swh-worker/entrypoint.sh +++ b/docker/services/swh-worker/entrypoint.sh @@ -15,7 +15,7 @@ echo Register task types in scheduler database wait-for-it swh-scheduler:5008 -s --timeout=0 - swh scheduler -C ${SWH_CONFIG_FILENAME} task-type register + swh scheduler --url ${SWH_SCHEDULER_INSTANCE} task-type register echo Starting the swh Celery worker for ${SWH_WORKER_INSTANCE} exec python -m celery \