diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -100,7 +100,6 @@ - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml @@ -120,7 +119,6 @@ - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml @@ -140,7 +138,6 @@ - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml @@ -175,7 +172,6 @@ env_file: - ./env/common_python.env - ./env/storage-db.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /storage.yml STORAGE_BACKEND: postgresql @@ -193,7 +189,6 @@ - 5003:5003 env_file: - ./env/common_python.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /objstorage.yml entrypoint: /entrypoint.sh @@ -241,7 +236,6 @@ - swh-idx-storage env_file: - ./env/common_python.env - - ./env/statsd.env environment: VERBOSITY: 3 DJANGO_SETTINGS_MODULE: swh.web.settings.development @@ -269,7 +263,6 @@ - swh-scheduler env_file: - ./env/common_python.env - - ./env/statsd.env - ./env/deposit-db.env environment: VERBOSITY: 3 @@ -294,7 +287,6 @@ env_file: - ./env/common_python.env - ./env/vault-db.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /vault.yml command: server @@ -316,7 +308,6 @@ command: worker env_file: - ./env/common_python.env - - ./env/statsd.env environment: SWH_CONFIG_FILENAME: /cooker.yml depends_on: @@ -344,8 +335,7 @@ env_file: - ./env/common_python.env - ./env/listers-db.env - - ./env/listers.env - - ./env/statsd.env + - ./env/workers.env user: swh environment: SWH_WORKER_INSTANCE: listers @@ -361,15 +351,14 @@ - "./conf/lister.yml:/lister.yml:ro" - "./services/swh-listers-worker/entrypoint.sh:/entrypoint.sh:ro" -# Loader Celery workers +# Loader + deposit checker Celery workers swh-loader: image: swh/stack build: ./ env_file: - ./env/common_python.env - - ./env/listers.env - - ./env/statsd.env + - ./env/workers.env user: swh environment: SWH_WORKER_INSTANCE: loader @@ -378,10 +367,11 @@ depends_on: - swh-storage - swh-scheduler + - swh-deposit - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" - - "./services/swh-loaders-worker/entrypoint.sh:/entrypoint.sh:ro" + - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" # Indexer Celery workers @@ -392,8 +382,11 @@ env_file: - ./env/common_python.env - ./env/indexers-db.env - - ./env/indexers.env - - ./env/statsd.env + - ./env/workers.env + environment: + SWH_WORKER_INSTANCE: indexer + SWH_CONFIG_FILENAME: /indexer.yml + CONCURRENCY: 4 entrypoint: /entrypoint.sh depends_on: - swh-scheduler-runner diff --git a/docker/env/common_python.env b/docker/env/common_python.env --- a/docker/env/common_python.env +++ b/docker/env/common_python.env @@ -1,5 +1,10 @@ # Forces Python's stdout to be shown in docker logs before they exit: PYTHONUNBUFFERED=1 +# Send statsd probes to prometheus-statsd-exporter so they are visible in +# http://localhost:5080/grafana +STATSD_HOST=prometheus-statsd-exporter +STATSD_PORT=9125 + # Uncomment and set this to a Sentry DSN to report errors to Sentry: # SWH_SENTRY_DSN= diff --git a/docker/env/indexers.env b/docker/env/indexers.env deleted file mode 100644 --- a/docker/env/indexers.env +++ /dev/null @@ -1,5 +0,0 @@ -CONCURRENCY=4 -MAX_TASKS_PER_CHILD=10 -LOGLEVEL=DEBUG -SWH_WORKER_INSTANCE=indexer -SWH_CONFIG_FILENAME=/indexer.yml diff --git a/docker/env/statsd.env b/docker/env/statsd.env deleted file mode 100644 --- a/docker/env/statsd.env +++ /dev/null @@ -1,2 +0,0 @@ -STATSD_HOST=prometheus-statsd-exporter -STATSD_PORT=9125 diff --git a/docker/env/listers.env b/docker/env/workers.env rename from docker/env/listers.env rename to docker/env/workers.env diff --git a/docker/services/swh-loaders-worker/entrypoint.sh b/docker/services/swh-worker/entrypoint.sh rename from docker/services/swh-loaders-worker/entrypoint.sh rename to docker/services/swh-worker/entrypoint.sh --- a/docker/services/swh-loaders-worker/entrypoint.sh +++ b/docker/services/swh-worker/entrypoint.sh @@ -16,7 +16,7 @@ echo Register task types in scheduler database swh scheduler -C ${SWH_CONFIG_FILENAME} task-type register - echo Starting the swh-loader Celery worker for ${SWH_WORKER_INSTANCE} + echo Starting the swh Celery worker for ${SWH_WORKER_INSTANCE} exec python -m celery worker \ --app=swh.scheduler.celery_backend.config.app \ --pool=prefork --events \