Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9348309
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
13 KB
Subscribers
None
View Options
diff --git a/docker/conf/lister.yml b/docker/conf/lister.yml
index ba5bbec..5a3c89b 100644
--- a/docker/conf/lister.yml
+++ b/docker/conf/lister.yml
@@ -1,41 +1,31 @@
-storage:
- cls: remote
- url: http://swh-storage:5002/
-
scheduler:
cls: remote
url: http://swh-scheduler:5008/
-lister:
- cls: local
- args:
- db: postgresql://postgres@swh-listers-db/swh-listers
-
celery:
task_broker: amqp://guest:guest@amqp//
task_queues:
- swh.lister.bitbucket.tasks.FullBitBucketRelister
- swh.lister.bitbucket.tasks.IncrementalBitBucketLister
- swh.lister.bitbucket.tasks.RangeBitBucketLister
- swh.lister.cgit.tasks.CGitListerTask
- swh.lister.cran.tasks.CRANListerTask
- swh.lister.debian.tasks.DebianListerTask
- swh.lister.gitea.tasks.FullGiteaRelister
- swh.lister.gitea.tasks.IncrementalGiteaLister
- swh.lister.gitea.tasks.RangeGiteaLister
- swh.lister.github.tasks.FullGitHubRelister
- swh.lister.github.tasks.IncrementalGitHubLister
- swh.lister.github.tasks.RangeGitHubLister
- swh.lister.gitlab.tasks.FullGitLabRelister
- swh.lister.gitlab.tasks.IncrementalGitLabLister
- swh.lister.gitlab.tasks.RangeGitLabLister
- swh.lister.gnu.tasks.GNUListerTask
- swh.lister.npm.tasks.NpmIncrementalListerTask
- swh.lister.npm.tasks.NpmListerTask
- swh.lister.launchpad.tasks.IncrementalLaunchpadLister
- swh.lister.launchpad.tasks.FullLaunchpadLister
- - swh.lister.launchpad.tasks.NewLaunchpadLister
- swh.lister.packagist.tasks.PackagistListerTask
- swh.lister.phabricator.tasks.FullPhabricatorLister
- swh.lister.phabricator.tasks.IncrementalPhabricatorLister
- swh.lister.pypi.tasks.PyPIListerTask
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 1a3cc06..36a2e5f 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -1,419 +1,411 @@
version: '2'
services:
amqp:
image: rabbitmq:3.6-management
ports:
- 5072:5672
# flower:
# image: mher/flower
# command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower
# ports:
# - 5055:5555
# depends_on:
# - amqp
zookeeper:
image: wurstmeister/zookeeper
restart: always
kafka:
image: wurstmeister/kafka
ports:
- "5092:9092"
env_file: ./env/kafka.env
depends_on:
- zookeeper
kafka-manager:
image: hlebalbau/kafka-manager:stable
ports:
- "5093:9000"
environment:
ZK_HOSTS: zookeeper:2181
APPLICATION_SECRET: random-secret
depends_on:
- zookeeper
command: -Dpidfile.path=/dev/null
volumes:
- "./conf/kafka-manager_logback.xml:/kafka-manager/conf/logback.xml:ro"
prometheus:
image: prom/prometheus
depends_on:
- prometheus-statsd-exporter
command:
# Needed for the reverse-proxy
- "--web.external-url=/prometheus"
- "--config.file=/etc/prometheus/prometheus.yml"
volumes:
- "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
restart: unless-stopped
prometheus-statsd-exporter:
image: prom/statsd-exporter
command:
- "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml"
volumes:
- "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro"
restart: unless-stopped
prometheus-rabbitmq-exporter:
image: kbudde/rabbitmq-exporter
restart: unless-stopped
environment:
SKIP_QUEUES: "RPC_.*"
MAX_QUEUES: 5000
RABBIT_URL: http://amqp:15672
LOG_LEVEL: warning
grafana:
image: grafana/grafana
restart: unless-stopped
depends_on:
- prometheus
environment:
GF_SERVER_ROOT_URL: http://localhost:5080/grafana
volumes:
- "./conf/grafana/provisioning:/etc/grafana/provisioning:ro"
- "./conf/grafana/dashboards:/var/lib/grafana/dashboards"
nginx:
image: nginx
volumes:
- "./conf/nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
- 5080:5080
# Scheduler
swh-scheduler-db:
image: postgres:12
env_file:
- ./env/common_python.env
- ./env/scheduler-db.env
swh-scheduler:
image: swh/stack
build: ./
env_file:
- ./env/common_python.env
- ./env/scheduler-db.env
- ./env/scheduler.env
environment:
SWH_CONFIG_FILENAME: /scheduler.yml
SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml
entrypoint: /entrypoint.sh
depends_on:
- swh-scheduler-db
ports:
- 5008:5008
volumes:
- "./conf/scheduler.yml:/scheduler.yml:ro"
- "./services/swh-scheduler/entrypoint.sh:/entrypoint.sh:ro"
swh-scheduler-listener:
image: swh/stack
build: ./
env_file:
- ./env/common_python.env
- ./env/scheduler-db.env
- ./env/scheduler.env
environment:
SWH_CONFIG_FILENAME: /scheduler.yml
SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml
entrypoint: /entrypoint.sh
command: start-listener
depends_on:
- swh-scheduler
- amqp
volumes:
- "./conf/scheduler.yml:/scheduler.yml:ro"
- "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro"
swh-scheduler-runner:
image: swh/stack
build: ./
env_file:
- ./env/common_python.env
- ./env/scheduler-db.env
- ./env/scheduler.env
environment:
SWH_CONFIG_FILENAME: /scheduler.yml
SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml
entrypoint: /entrypoint.sh
command: start-runner -p 10
depends_on:
- swh-scheduler
- amqp
volumes:
- "./conf/scheduler.yml:/scheduler.yml:ro"
- "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro"
# Graph storage
swh-storage-db:
image: postgres:12
env_file:
- ./env/storage-db.env
swh-storage:
image: swh/stack
build: ./
ports:
- 5002:5002
depends_on:
- swh-storage-db
- swh-objstorage
- kafka
env_file:
- ./env/common_python.env
- ./env/storage.env
environment:
SWH_CONFIG_FILENAME: /storage.yml
STORAGE_BACKEND: postgresql
entrypoint: /entrypoint.sh
volumes:
- "./conf/storage.yml:/storage.yml:ro"
- "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro"
# Object storage
swh-objstorage:
build: ./
image: swh/stack
ports:
- 5003:5003
env_file:
- ./env/common_python.env
environment:
SWH_CONFIG_FILENAME: /objstorage.yml
entrypoint: /entrypoint.sh
volumes:
- "./conf/objstorage.yml:/objstorage.yml:ro"
- "./services/swh-objstorage/entrypoint.sh:/entrypoint.sh:ro"
# Indexer storage
swh-idx-storage-db:
image: postgres:12
env_file:
- ./env/indexers-db.env
swh-idx-storage:
image: swh/stack
build: ./
ports:
- 5007:5007
depends_on:
- swh-idx-storage-db
env_file:
- ./env/common_python.env
- ./env/indexers-db.env
- ./env/indexers.env
environment:
SWH_CONFIG_FILENAME: /indexer_storage.yml
entrypoint: /entrypoint.sh
volumes:
- "./conf/indexer_storage.yml:/indexer_storage.yml:ro"
- "./services/swh-indexer-storage/entrypoint.sh:/entrypoint.sh:ro"
# Web interface
swh-web:
build: ./
image: swh/stack
ports:
- 5004:5004
depends_on:
- swh-storage
- swh-idx-storage
env_file:
- ./env/common_python.env
environment:
VERBOSITY: 3
DJANGO_SETTINGS_MODULE: swh.web.settings.production
SWH_CONFIG_FILENAME: /web.yml
entrypoint: /entrypoint.sh
volumes:
- "./conf/web.yml:/web.yml:ro"
- "./services/swh-web/entrypoint.sh:/entrypoint.sh:ro"
swh-deposit-db:
image: postgres:12
env_file:
- ./env/deposit-db.env
swh-deposit:
image: swh/stack
build: ./
ports:
- 5006:5006
depends_on:
- swh-deposit-db
- swh-scheduler
env_file:
- ./env/common_python.env
- ./env/deposit-db.env
- ./env/deposit.env
environment:
VERBOSITY: 3
SWH_CONFIG_FILENAME: /deposit.yml
DJANGO_SETTINGS_MODULE: swh.deposit.settings.production
entrypoint: /entrypoint.sh
volumes:
- "./conf/deposit.yml:/deposit.yml:ro"
- "./services/swh-deposit/entrypoint.sh:/entrypoint.sh:ro"
swh-vault-db:
image: postgres:12
env_file:
- ./env/vault-db.env
swh-vault:
image: swh/stack
build: ./
env_file:
- ./env/common_python.env
- ./env/vault-db.env
- ./env/vault.env
environment:
SWH_CONFIG_FILENAME: /vault.yml
command: server
ports:
- 5005:5005
depends_on:
- swh-vault-db
- swh-objstorage
- swh-storage
- swh-scheduler
entrypoint: /entrypoint.sh
volumes:
- "./conf/vault.yml:/vault.yml:ro"
- "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro"
swh-vault-worker:
image: swh/stack
build: ./
command: worker
env_file:
- ./env/common_python.env
- ./env/workers.env
environment:
SWH_CONFIG_FILENAME: /cooker.yml
depends_on:
- swh-vault
- swh-storage
entrypoint: /entrypoint.sh
volumes:
- "./conf/vault-worker.yml:/cooker.yml:ro"
- "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro"
# Lister Celery workers
- swh-listers-db:
- image: postgres:12
- env_file:
- - ./env/listers-db.env
-
swh-lister:
image: swh/stack
build: ./
env_file:
- ./env/common_python.env
- - ./env/listers-db.env
- ./env/listers.env
- ./env/workers.env
user: swh
environment:
SWH_WORKER_INSTANCE: listers
SWH_CONFIG_FILENAME: /lister.yml
depends_on:
- - swh-listers-db
- swh-scheduler
- swh-scheduler-runner
- - swh-storage
- amqp
entrypoint: /entrypoint.sh
volumes:
- "./conf/lister.yml:/lister.yml:ro"
- "./services/swh-listers-worker/entrypoint.sh:/entrypoint.sh:ro"
# Loader + deposit checker Celery workers
swh-loader:
image: swh/stack
build: ./
env_file:
- ./env/common_python.env
- ./env/workers.env
user: swh
environment:
SWH_WORKER_INSTANCE: loader
SWH_CONFIG_FILENAME: /loader.yml
entrypoint: /entrypoint.sh
depends_on:
- swh-storage
- swh-scheduler
- swh-deposit
- amqp
volumes:
- "./conf/loader.yml:/loader.yml:ro"
- "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro"
# Indexer Celery workers
swh-indexer:
image: swh/stack
build: ./
user: swh
env_file:
- ./env/common_python.env
- ./env/indexers-db.env
- ./env/indexers.env
- ./env/workers.env
environment:
SWH_WORKER_INSTANCE: indexer
SWH_CONFIG_FILENAME: /indexer.yml
CONCURRENCY: 4
entrypoint: /entrypoint.sh
depends_on:
- swh-scheduler-runner
- swh-idx-storage
- swh-storage
- swh-objstorage
- amqp
volumes:
- "./conf/indexer.yml:/indexer.yml:ro"
- "./services/swh-indexer-worker/entrypoint.sh:/entrypoint.sh:ro"
# Journal related
swh-indexer-journal-client:
image: swh/stack
build: ./
entrypoint: /entrypoint.sh
env_file:
- ./env/common_python.env
depends_on:
- kafka
- swh-storage
- swh-scheduler
volumes:
- "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro"
- "./services/swh-indexer-journal-client/entrypoint.sh:/entrypoint.sh:ro"
swh-scheduler-journal-client:
image: swh/stack
build: ./
entrypoint: /entrypoint.sh
env_file:
- ./env/common_python.env
depends_on:
- kafka
- swh-scheduler
volumes:
- "./conf/scheduler_journal_client.yml:/etc/softwareheritage/scheduler/journal_client.yml:ro"
- "./services/swh-scheduler-journal-client/entrypoint.sh:/entrypoint.sh:ro"
diff --git a/docker/env/listers-db.env b/docker/env/listers-db.env
deleted file mode 100644
index d5ffd4e..0000000
--- a/docker/env/listers-db.env
+++ /dev/null
@@ -1,2 +0,0 @@
-POSTGRES_PASSWORD=testpassword
-POSTGRES_DB=swh-listers
diff --git a/docker/env/listers.env b/docker/env/listers.env
index 6711ace..e69de29 100644
--- a/docker/env/listers.env
+++ b/docker/env/listers.env
@@ -1,2 +0,0 @@
-PGHOST=swh-listers-db
-PGUSER=postgres
diff --git a/docker/services/swh-listers-worker/entrypoint.sh b/docker/services/swh-listers-worker/entrypoint.sh
index 52eb4b5..e049766 100755
--- a/docker/services/swh-listers-worker/entrypoint.sh
+++ b/docker/services/swh-listers-worker/entrypoint.sh
@@ -1,40 +1,31 @@
#!/bin/bash
set -e
source /srv/softwareheritage/utils/pyutils.sh
setup_pip
-source /srv/softwareheritage/utils/pgsql.sh
-setup_pgsql
-
case "$1" in
"shell")
exec bash -i
;;
*)
- wait_pgsql ${POSTGRES_DB}
-
- echo Setup ${POSTGRES_DB} database for ${SWH_WORKER_INSTANCE}
-
- echo Initialize database
- swh lister -C ${SWH_CONFIG_FILENAME} db-init
echo Register task types in scheduler database
wait-for-it swh-scheduler:5008 -s --timeout=0
swh scheduler -C ${SWH_CONFIG_FILENAME} task-type register
echo Waiting for RabbitMQ to start
wait-for-it amqp:5672 -s --timeout=0
echo Starting the swh-lister Celery worker for ${SWH_WORKER_INSTANCE}
exec python -m celery \
--app=swh.scheduler.celery_backend.config.app \
worker \
--pool=prefork --events \
--concurrency=${CONCURRENCY} \
--max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
-Ofair --loglevel=${LOGLEVEL} \
--hostname "${SWH_WORKER_INSTANCE}@%h"
;;
esac
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Jul 4 2025, 6:23 PM (5 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3273378
Attached To
rDENV Development environment
Event Timeline
Log In to Comment