diff --git a/Dockerfile b/Dockerfile index 304d15f..1a5ab6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,46 +1,42 @@ FROM python:3.6 RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ libapr1-dev \ libaprutil1-dev \ libpq-dev \ libsvn-dev \ libsystemd-dev \ postgresql-client \ wait-for-it \ ngrep && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip setuptools wheel RUN pip install swh-deposit swh-indexer swh-journal swh-lister swh-loader-debian \ swh-loader-dir swh-loader-git swh-loader-mercurial swh-loader-pypi \ swh-loader-svn swh-loader-tar swh-storage swh-objstorage \ swh-scheduler swh-vault swh-web RUN pip install gunicorn COPY services/swh-deposit/entrypoint.sh /swh-deposit/entrypoint.sh COPY services/swh-indexer-journal-client/entrypoint.sh /swh-indexer-journal-client/entrypoint.sh COPY services/swh-indexer-storage/entrypoint.sh /swh-indexer-storage/entrypoint.sh COPY services/swh-indexer-worker/entrypoint.sh /swh-indexer-worker/entrypoint.sh -COPY services/swh-journal-client/entrypoint.sh /swh-journal-client/entrypoint.sh COPY services/swh-listers-worker/entrypoint.sh /swh-listers-worker/entrypoint.sh COPY services/swh-loaders-worker/entrypoint.sh /swh-loaders-worker/entrypoint.sh COPY services/swh-objstorage/entrypoint.sh /swh-objstorage/entrypoint.sh COPY services/swh-scheduler-api/entrypoint.sh /swh-scheduler-api/entrypoint.sh COPY services/swh-scheduler-worker/entrypoint.sh /swh-scheduler-worker/entrypoint.sh COPY services/swh-storage/entrypoint.sh /swh-storage/entrypoint.sh COPY services/swh-vault/entrypoint.sh /swh-vault/entrypoint.sh COPY services/swh-web/entrypoint.sh /swh-web/entrypoint.sh -COPY services/swh-journal-client/client.py /swh-journal-client/ - COPY utils/pgsql.sh /swh-utils/pgsql.sh RUN useradd -ms /bin/bash swh RUN mkdir -p /srv/softwareheritage/objects - diff --git a/conf/journal_client.yml b/conf/journal_client.yml deleted file mode 100644 index ba096c4..0000000 --- a/conf/journal_client.yml +++ /dev/null @@ -1,7 +0,0 @@ -brokers: - - kafka -max_messages: 50 -scheduler: - cls: remote - args: - url: http://swh-scheduler-api:5008/ diff --git a/docker-compose.yml b/docker-compose.yml index effc7ca..46b0155 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,375 +1,366 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5072:5672 # flower: # image: mher/flower # command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower # ports: # - 5055:5555 # depends_on: # - amqp zookeeper: image: wurstmeister/zookeeper kafka: image: wurstmeister/kafka ports: - "5092:9092" env_file: ./env/kafka.env depends_on: - zookeeper kafka-manager: image: hlebalbau/kafka-manager:stable ports: - "5093:9000" environment: ZK_HOSTS: zookeeper:2181 APPLICATION_SECRET: random-secret command: -Dpidfile.path=/dev/null prometheus: image: prom/prometheus command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped grafana: image: grafana/grafana restart: unless-stopped environment: GF_SERVER_ROOT_URL: http://localhost:5080/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards" nginx: image: nginx volumes: - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 # Scheduler swh-scheduler-db: image: postgres:11 env_file: - ./env/scheduler-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-scheduler-api: image: swh/stack build: ./ env_file: - ./env/scheduler-db.env - ./env/scheduler.env entrypoint: /swh-scheduler-api/entrypoint.sh environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml depends_on: - swh-scheduler-db ports: - 5008:5008 volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-listener: image: swh/stack build: ./ entrypoint: /swh-scheduler-worker/entrypoint.sh env_file: - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml command: listener depends_on: - swh-scheduler-api - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-runner: image: swh/stack build: ./ entrypoint: /swh-scheduler-worker/entrypoint.sh env_file: - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml command: runner -p 10 depends_on: - swh-scheduler-api - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" # Graph storage swh-storage-db: image: postgres:11 env_file: - ./env/storage-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-storage: build: ./ image: swh/stack entrypoint: /swh-storage/entrypoint.sh ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage env_file: - ./env/storage-db.env environment: SWH_CONFIG_FILENAME: /storage.yml volumes: - "./conf/storage.yml:/storage.yml:ro" # Object storage swh-objstorage: build: ./ image: swh/stack entrypoint: /swh-objstorage/entrypoint.sh ports: - 5003:5003 environment: SWH_CONFIG_FILENAME: /objstorage.yml volumes: - "./conf/objstorage.yml:/objstorage.yml:ro" # Indexer storage swh-idx-storage-db: image: postgres:11 env_file: - ./env/indexers-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-idx-storage: build: ./ image: swh/stack entrypoint: /swh-indexer-storage/entrypoint.sh ports: - 5007:5007 depends_on: - swh-idx-storage-db env_file: - ./env/indexers-db.env environment: SWH_CONFIG_FILENAME: /indexer_storage.yml volumes: - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" # Web interface swh-web: build: ./ image: swh/stack entrypoint: /swh-web/entrypoint.sh ports: - 5004:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage environment: VERBOSITY: 3 DJANGO_SETTINGS_MODULE: swh.web.settings.development SWH_CONFIG_FILENAME: /web.yml PYTHONPATH: /tmp/swh volumes: - "./conf/web.yml:/web.yml:ro" swh-deposit-db: image: postgres:11 env_file: - ./env/deposit-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-deposit: build: ./ entrypoint: /swh-deposit/entrypoint.sh image: swh/stack ports: - 5006:5006 depends_on: - swh-deposit-db - swh-scheduler-api env_file: - ./env/deposit-db.env - ./env/deposit.env volumes: - "./conf/deposit.yml:/deposit.yml:ro" swh-vault-db: image: postgres:11 env_file: - ./env/vault-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-vault-api: build: ./ image: swh/stack entrypoint: /swh-vault/entrypoint.sh env_file: - ./env/vault-db.env environment: SWH_CONFIG_FILENAME: /vault-api.yml command: server ports: - 5005:5005 depends_on: - swh-vault-db - swh-objstorage - swh-storage - swh-scheduler-api volumes: - "./conf/vault-api.yml:/vault-api.yml:ro" swh-vault-worker: build: ./ image: swh/stack entrypoint: /swh-vault/entrypoint.sh command: worker environment: SWH_CONFIG_FILENAME: /cooker.yml depends_on: - swh-vault-api - swh-storage volumes: - "./conf/vault-worker.yml:/cooker.yml:ro" # Lister Celery workers swh-listers-db: image: postgres:11 env_file: - ./env/listers-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-lister: image: swh/stack build: ./ entrypoint: /swh-listers-worker/entrypoint.sh env_file: - ./env/listers-db.env - ./env/listers.env user: swh environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: listers SWH_CONFIG_FILENAME: /lister.yml depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp volumes: - "./conf/lister.yml:/lister.yml:ro" # Loader Celery workers swh-loader: image: swh/stack build: ./ entrypoint: /swh-loaders-worker/entrypoint.sh env_file: - ./env/listers.env user: swh environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: loader SWH_CONFIG_FILENAME: /loader.yml depends_on: - swh-storage - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" # Indexer Celery workers swh-indexer: image: swh/stack build: ./ user: swh entrypoint: /swh-indexer-worker/entrypoint.sh env_file: - ./env/indexers-db.env - ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp volumes: - "./conf/indexer.yml:/indexer.yml:ro" # Journal related - swh-journal-client: - image: swh/stack - build: ./ - entrypoint: /swh-journal-client/entrypoint.sh - depends_on: - - swh-journal-publisher - volumes: - - "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro" - swh-indexer-journal-client: image: swh/stack build: ./ entrypoint: /swh-indexer-journal-client/entrypoint.sh depends_on: - kafka - swh-storage-api - swh-scheduler-api volumes: - "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" diff --git a/services/swh-journal-client/client.py b/services/swh-journal-client/client.py deleted file mode 100644 index 5b005dc..0000000 --- a/services/swh-journal-client/client.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import logging - -from swh.journal.client import JournalClient - - -class JournalClientLogger(JournalClient): - """Client in charge of listing new received origins and origin_visits - in the swh journal. - - """ - CONFIG_BASE_FILENAME = 'journal/logger' - - def __init__(self): - # Only interested in content here so override the configuration - super().__init__(extra_configuration={'object_types': - ['origin', 'origin_visit']}) - - def process_objects(self, messages): - """Simply log messages received. - - """ - logging.info('client received the following messages: %s' % messages) - - -if __name__ == '__main__': - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s %(process)d %(levelname)s %(message)s' - ) - - import click - - @click.command() - def main(): - """Log the new received origin and origin_visits. - - """ - JournalClientLogger().process() - - main() diff --git a/services/swh-journal-client/entrypoint.sh b/services/swh-journal-client/entrypoint.sh deleted file mode 100755 index 4a6c621..0000000 --- a/services/swh-journal-client/entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list - -case "$1" in - "shell") - exec bash -i - ;; - *) - echo "Starting a swh-journal client" - exec wait-for-it kafka:9092 -s --timeout=0 -- python3 /swh-journal-client/client.py - ;; -esac