diff --git a/dockerfiles/swh-deposit/private.yml b/conf/deposit_private.yml rename from dockerfiles/swh-deposit/private.yml rename to conf/deposit_private.yml diff --git a/dockerfiles/swh-deposit/server.yml b/conf/deposit_server.yml rename from dockerfiles/swh-deposit/server.yml rename to conf/deposit_server.yml diff --git a/dockerfiles/swh-indexer-worker/indexer.yml b/conf/indexer.yml rename from dockerfiles/swh-indexer-worker/indexer.yml rename to conf/indexer.yml --- a/dockerfiles/swh-indexer-worker/indexer.yml +++ b/conf/indexer.yml @@ -16,3 +16,5 @@ url: http://swh-scheduler-api:5008/ celery: task_broker: amqp://guest:guest@amqp// + task_modules: + - swh.indexer.tasks diff --git a/dockerfiles/swh-indexer-storage/indexer_storage.yml b/conf/indexer_storage.yml rename from dockerfiles/swh-indexer-storage/indexer_storage.yml rename to conf/indexer_storage.yml diff --git a/conf/journal_client.yml b/conf/journal_client.yml new file mode 100644 --- /dev/null +++ b/conf/journal_client.yml @@ -0,0 +1,8 @@ +brokers: + - kafka +# small number of batch +max_messages: 1 +scheduler: + cls: remote + args: + url: http://swh-scheduler-api:5008/ diff --git a/dockerfiles/swh-journal-publisher/publisher.yml b/conf/journal_publisher.yml rename from dockerfiles/swh-journal-publisher/publisher.yml rename to conf/journal_publisher.yml diff --git a/conf/lister.yml b/conf/lister.yml new file mode 100644 --- /dev/null +++ b/conf/lister.yml @@ -0,0 +1,26 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ + +scheduler: + cls: remote + args: + url: http://swh-scheduler-api:5008/ + +lister: + cls: local + args: + db: postgresql:///?service=swh + +celery: + task_broker: amqp://guest:guest@amqp// + task_modules: + - swh.lister.bitbucket.tasks + - swh.lister.debian.tasks + - swh.lister.github.tasks + - swh.lister.gitlab.tasks + - swh.lister.npm.tasks + - swh.lister.pypi.tasks + +lister_db_url: postgresql:///?service=swh diff --git a/conf/loader.yml b/conf/loader.yml new file mode 100644 --- /dev/null +++ b/conf/loader.yml @@ -0,0 +1,19 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ +lister: + cls: local + args: + db: postgresql:///?service=swh +celery: + task_broker: amqp://guest:guest@amqp// + task_modules: + - swh.loader.debian.tasks + - swh.loader.dir.tasks + - swh.loader.git.tasks + - swh.loader.mercurial.tasks + - swh.loader.pypi.tasks + - swh.loader.svn.tasks + - swh.loader.tar.tasks +lister_db_url: postgresql:///?service=swh diff --git a/nginx.conf b/conf/nginx.conf rename from nginx.conf rename to conf/nginx.conf diff --git a/dockerfiles/swh-objstorage/objstorage.yml b/conf/objstorage.yml rename from dockerfiles/swh-objstorage/objstorage.yml rename to conf/objstorage.yml diff --git a/prometheus-statsd-mapping.yml b/conf/prometheus-statsd-mapping.yml rename from prometheus-statsd-mapping.yml rename to conf/prometheus-statsd-mapping.yml diff --git a/prometheus.yml b/conf/prometheus.yml rename from prometheus.yml rename to conf/prometheus.yml diff --git a/dockerfiles/swh-scheduler-worker/scheduler.yml b/conf/scheduler.yml rename from dockerfiles/swh-scheduler-worker/scheduler.yml rename to conf/scheduler.yml diff --git a/dockerfiles/swh-storage/storage.yml b/conf/storage.yml rename from dockerfiles/swh-storage/storage.yml rename to conf/storage.yml diff --git a/dockerfiles/swh-storage-listener/listener.yml b/conf/storage_listener.yml rename from dockerfiles/swh-storage-listener/listener.yml rename to conf/storage_listener.yml diff --git a/dockerfiles/swh-web/web.yml b/conf/web.yml rename from dockerfiles/swh-web/web.yml rename to conf/web.yml diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,13 +7,13 @@ ports: - 5072:5672 - flower: - image: mher/flower - command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower - ports: - - 5055:5555 - depends_on: - - amqp +# flower: +# image: mher/flower +# command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower +# ports: +# - 5055:5555 +# depends_on: +# - amqp zookeeper: image: wurstmeister/zookeeper @@ -22,7 +22,7 @@ image: wurstmeister/kafka ports: - 5092:9092 - env_file: ./kafka.env + env_file: ./env/kafka.env depends_on: - zookeeper @@ -33,7 +33,7 @@ - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - - "./prometheus.yml:/etc/prometheus/prometheus.yml:ro" + - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: @@ -41,13 +41,13 @@ command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - - "./prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" + - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped nginx: image: nginx volumes: - - "./nginx.conf:/etc/nginx/nginx.conf:ro" + - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 @@ -62,29 +62,35 @@ swh-scheduler-api: image: swh/scheduler-api build: ./dockerfiles/swh-scheduler-api - env_file: ./scheduler.env + env_file: ./env/scheduler.env depends_on: - swh-scheduler-db ports: - 5008:5008 + volumes: + - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-listener: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker - env_file: ./scheduler.env + env_file: ./env/scheduler.env command: listener depends_on: - swh-scheduler-api - amqp + volumes: + - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-runner: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker - env_file: ./scheduler.env + env_file: ./env/scheduler.env command: runner -p 10 depends_on: - swh-scheduler-api - amqp + volumes: + - "./conf/scheduler.yml:/scheduler.yml:ro" # Graph storage @@ -102,7 +108,9 @@ depends_on: - swh-storage-db - swh-objstorage - env_file: ./storage.env + env_file: ./env/storage.env + volumes: + - "./conf/storage.yml:/storage.yml:ro" # Object storage @@ -111,6 +119,8 @@ image: swh/objstorage ports: - 5003:5003 + volumes: + - "./conf/objstorage.yml:/objstorage.yml:ro" # Indexer storage @@ -132,6 +142,8 @@ POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres + volumes: + - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" # Web interface @@ -144,10 +156,14 @@ - swh-objstorage - swh-storage - swh-idx-storage + environment: + VERBOSITY: 3 + volumes: + - "./conf/web.yml:/etc/softwareheritage/web/web.yml:ro" swh-deposit-db: image: postgres:10 - env_file: ./deposit.env + env_file: ./env/deposit.env swh-deposit: build: ./dockerfiles/swh-deposit @@ -157,9 +173,12 @@ depends_on: - swh-deposit-db - swh-scheduler-api - env_file: ./deposit.env + env_file: ./env/deposit.env environment: PGHOST: swh-deposit-db + volumes: + - "./conf/deposit_server.yml:/etc/softwareheritage/deposit/server.yml:ro" + - "./conf/deposit_private.yml:/etc/softwareheritage/deposit/private.yml:ro" # Lister Celery workers @@ -168,102 +187,47 @@ environment: POSTGRES_PASSWORD: testpassword - swh-lister-debian: + swh-lister: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker - env_file: ./listers.env + env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: debian - depends_on: - - swh-listers-db - - swh-scheduler-api - - swh-scheduler-runner - - swh-storage - - amqp - - swh-lister-bitbucket: - image: swh/listers-worker - build: ./dockerfiles/swh-listers-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: bitbucket - depends_on: - - swh-listers-db - - swh-scheduler-api - - swh-scheduler-runner - - swh-storage - - amqp - - swh-lister-github: - image: swh/listers-worker - build: ./dockerfiles/swh-listers-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: github - depends_on: - - swh-listers-db - - swh-scheduler-api - - swh-scheduler-runner - - swh-storage - - amqp - - swh-lister-gitlab: - image: swh/listers-worker - build: ./dockerfiles/swh-listers-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: gitlab + SWH_WORKER_INSTANCE: listers + SWH_CONFIG_FILENAME: /lister.yml depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/lister.yml:ro" - swh-lister-npm: - image: swh/listers-worker - build: ./dockerfiles/swh-listers-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: npm - depends_on: - - swh-listers-db - - swh-scheduler-api - - swh-scheduler-runner - - swh-storage - - amqp +# Loader Celery workers - swh-lister-pypi: - image: swh/listers-worker - build: ./dockerfiles/swh-listers-worker - env_file: ./listers.env + swh-loader: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: pypi + SWH_WORKER_INSTANCE: loader + SWH_CONFIG_FILENAME: /loader.yml depends_on: - - swh-listers-db - - swh-scheduler-api - - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/loader.yml:/loader.yml:ro" # Indexer Celery workers swh-indexer: image: swh/indexer-worker build: ./dockerfiles/swh-indexer-worker - env_file: ./indexers.env + env_file: ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 @@ -273,13 +237,8 @@ - swh-storage - swh-objstorage - amqp - - depends_on: - - swh-scheduler-runner - - swh-idx-storage - - swh-storage - - swh-objstorage - - amqp + volumes: + - "./conf/indexer.yml:/indexer.yml:ro" swh-indexer-journal-client: image: swh/indexer-journal-client @@ -287,16 +246,20 @@ depends_on: - swh-journal-publisher - swh-scheduler-api + volumes: + - "./conf/journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" # Journal related swh-storage-listener: image: swh/storage-listener build: ./dockerfiles/swh-storage-listener - env_file: ./storage.env + env_file: ./env/storage.env depends_on: - swh-storage-db - kafka + volumes: + - "./conf/storage_listener.yml:/etc/softwareheritage/storage/listener.yml:ro" swh-journal-publisher: image: swh/journal-publisher @@ -304,95 +267,13 @@ depends_on: - kafka - swh-storage-listener + volumes: + - "./conf/journal_publisher.yml:/etc/softwareheritage/journal/publisher.yml:ro" swh-journal-client: image: swh/journal-client build: ./dockerfiles/swh-journal-client depends_on: - swh-journal-publisher - -# Loader Celery workers - - swh-loader-debian: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: debian - depends_on: - - swh-storage - - amqp - - swh-loader-dir: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: dir - depends_on: - - swh-storage - - amqp - - swh-loader-git: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: git - depends_on: - - swh-storage - - amqp - - swh-loader-mercurial: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: mercurial - depends_on: - - swh-storage - - amqp - - swh-loader-pypi: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: pypi - depends_on: - - swh-storage - - amqp - - swh-loader-svn: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: svn - depends_on: - - swh-storage - - amqp - - swh-loader-tar: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker - env_file: ./listers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: tar - depends_on: - - swh-storage - - amqp + volumes: + - "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro" diff --git a/dockerfiles/swh-deposit/Dockerfile b/dockerfiles/swh-deposit/Dockerfile --- a/dockerfiles/swh-deposit/Dockerfile +++ b/dockerfiles/swh-deposit/Dockerfile @@ -8,8 +8,6 @@ RUN pip install swh-deposit gunicorn RUN pip install psycopg2-binary # to make pip silent -COPY server.yml /etc/softwareheritage/deposit/ -COPY private.yml /etc/softwareheritage/deposit/ COPY entrypoint.sh / EXPOSE 5006 diff --git a/dockerfiles/swh-indexer-journal-client/Dockerfile b/dockerfiles/swh-indexer-journal-client/Dockerfile --- a/dockerfiles/swh-indexer-journal-client/Dockerfile +++ b/dockerfiles/swh-indexer-journal-client/Dockerfile @@ -5,8 +5,6 @@ apt-get install -y libsystemd-dev RUN pip install swh-indexer -COPY journal_client.yml /etc/softwareheritage/indexer/ COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] - diff --git a/dockerfiles/swh-indexer-storage/Dockerfile b/dockerfiles/swh-indexer-storage/Dockerfile --- a/dockerfiles/swh-indexer-storage/Dockerfile +++ b/dockerfiles/swh-indexer-storage/Dockerfile @@ -8,7 +8,6 @@ RUN pip install --upgrade pip setuptools wheel RUN pip install swh-indexer -COPY indexer_storage.yml / COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-storage/entrypoint.sh b/dockerfiles/swh-indexer-storage/entrypoint.sh --- a/dockerfiles/swh-indexer-storage/entrypoint.sh +++ b/dockerfiles/swh-indexer-storage/entrypoint.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +export PATH=${HOME}/.local/bin:${PATH} if [[ -d /src ]] ; then for srcrepo in /src/swh-* ; do diff --git a/dockerfiles/swh-indexer-worker/Dockerfile b/dockerfiles/swh-indexer-worker/Dockerfile --- a/dockerfiles/swh-indexer-worker/Dockerfile +++ b/dockerfiles/swh-indexer-worker/Dockerfile @@ -12,6 +12,5 @@ USER swh COPY entrypoint.sh / -COPY indexer.yml /home/swh/ ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-worker/entrypoint.sh b/dockerfiles/swh-indexer-worker/entrypoint.sh --- a/dockerfiles/swh-indexer-worker/entrypoint.sh +++ b/dockerfiles/swh-indexer-worker/entrypoint.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +export PATH=${HOME}/.local/bin:${PATH} if [[ -d /src ]] ; then for srcrepo in /src/swh-* ; do @@ -12,22 +13,11 @@ done fi -mkdir -p ~/.config/swh/worker - -cat > ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini < ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini < ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini <&2 >/dev/null; do sleep 0.1; done echo "Starting swh-storage's listener" - exec python3 -m swh.storage.listener --verbose + exec python3 -m swh.storage.listener ;; esac diff --git a/dockerfiles/swh-storage/Dockerfile b/dockerfiles/swh-storage/Dockerfile --- a/dockerfiles/swh-storage/Dockerfile +++ b/dockerfiles/swh-storage/Dockerfile @@ -8,7 +8,6 @@ RUN pip install --upgrade pip setuptools wheel RUN pip install swh-storage -COPY storage.yml / COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-web/Dockerfile b/dockerfiles/swh-web/Dockerfile --- a/dockerfiles/swh-web/Dockerfile +++ b/dockerfiles/swh-web/Dockerfile @@ -10,7 +10,6 @@ RUN pip install psycopg2-binary # to make pip silent RUN python3 -m swh.web.manage migrate -COPY web.yml /etc/softwareheritage/web/ COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/deposit.env b/env/deposit.env rename from deposit.env rename to env/deposit.env diff --git a/indexers.env b/env/indexers.env rename from indexers.env rename to env/indexers.env --- a/indexers.env +++ b/env/indexers.env @@ -2,4 +2,4 @@ MAX_TASKS_PER_CHILD=10 LOGLEVEL=DEBUG SWH_WORKER_INSTANCE=indexer -SWH_CONFIG_FILENAME=/home/swh/indexer.yml +SWH_CONFIG_FILENAME=/indexer.yml diff --git a/kafka.env b/env/kafka.env rename from kafka.env rename to env/kafka.env diff --git a/listers.env b/env/listers.env rename from listers.env rename to env/listers.env diff --git a/scheduler.env b/env/scheduler.env rename from scheduler.env rename to env/scheduler.env diff --git a/storage.env b/env/storage.env rename from storage.env rename to env/storage.env