diff --git a/docker-compose.yml b/docker-compose.yml index d330b71..adf06b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,216 +1,223 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5018:15672 zookeeper: image: wurstmeister/zookeeper ports: - "2181:2181" kafka: image: wurstmeister/kafka ports: - "9092:9092" env_file: ./kafka.env volumes: - /var/run/docker.sock:/var/run/docker.sock depends_on: - zookeeper # Scheduler swh-scheduler-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-scheduler swh-scheduler-api: image: swh/scheduler-api build: ./dockerfiles/swh-scheduler-api env_file: ./scheduler.env depends_on: - swh-scheduler-db ports: - 5008:5008 swh-scheduler-listener: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./scheduler.env command: listener depends_on: - swh-scheduler-api - amqp swh-scheduler-runner: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./scheduler.env command: runner depends_on: - swh-scheduler-api - amqp # Graph storage swh-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage swh-storage: build: ./dockerfiles/swh-storage image: swh/storage ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage env_file: ./storage.env # Object storage swh-objstorage: build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - 5003:5003 # Indexer storage swh-idx-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage swh-idx-storage: build: ./dockerfiles/swh-indexer-storage image: swh/indexer-storage ports: - 5007:5007 depends_on: - swh-idx-storage-db environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres # Web interface swh-web: build: ./dockerfiles/swh-web image: swh/web ports: - 8080:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage # Lister Celery workers swh-listers-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword swh-lister-debian: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: debian depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-github: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: github depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-gitlab: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: gitlab depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-npm: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: npm depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-pypi: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: pypi depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp # Indexer Celery workers swh-indexer-mimetype: image: swh/indexer-worker build: ./dockerfiles/swh-indexer-worker env_file: ./indexers.env environment: SWH_WORKER_INSTANCE: mimetype depends_on: - swh-scheduler-api - swh-idx-storage - swh-storage - swh-objstorage - amqp # Journal related swh-storage-listener: image: swh/storage-listener build: ./dockerfiles/swh-storage-listener env_file: ./storage.env depends_on: - swh-storage-db - kafka swh-journal-publisher: image: swh/journal-publisher build: ./dockerfiles/swh-journal-publisher depends_on: - kafka - swh-storage-listener + + swh-journal-client: + image: swh/journal-client + build: ./dockerfiles/swh-journal-client + depends_on: + - swh-journal-publisher + diff --git a/dockerfiles/swh-journal-client/Dockerfile b/dockerfiles/swh-journal-client/Dockerfile new file mode 100644 index 0000000..b614527 --- /dev/null +++ b/dockerfiles/swh-journal-client/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y libsystemd-dev + +RUN pip install swh-journal +COPY logger.yml /etc/softwareheritage/journal/ +COPY client.py / +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-journal-client/client.py b/dockerfiles/swh-journal-client/client.py new file mode 100644 index 0000000..1dadd64 --- /dev/null +++ b/dockerfiles/swh-journal-client/client.py @@ -0,0 +1,46 @@ +# Copyright (C) 2018 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import logging + +from swh.journal.client import JournalClient + + +class JournalClientLogger(JournalClient): + """Client in charge of listing new received origins and origin_visits + in the swh journal. + + """ + CONFIG_BASE_FILENAME = 'journal/logger' + + def __init__(self): + # Only interested in content here so override the configuration + super().__init__(extra_configuration={'object_types': + ['origin', 'origin_visit']}) + + def process_objects(self, messages): + """Simply log messages received. + + """ + for msg in messages: + logging.info('msg: %s' % msg) + + +if __name__ == '__main__': + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(process)d %(levelname)s %(message)s' + ) + + import click + + @click.command() + def main(): + """Log the new received origin and origin_visits. + + """ + JournalClientLogger().process() + + main() diff --git a/dockerfiles/swh-journal-client/entrypoint.sh b/dockerfiles/swh-journal-client/entrypoint.sh new file mode 100755 index 0000000..8784ae7 --- /dev/null +++ b/dockerfiles/swh-journal-client/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +case "$1" in + "shell") + exec bash -i + ;; + *) + echo "Starting an swh-journal client" + exec python3 -m client + ;; +esac diff --git a/dockerfiles/swh-journal-client/logger.yml b/dockerfiles/swh-journal-client/logger.yml new file mode 100644 index 0000000..292a7ac --- /dev/null +++ b/dockerfiles/swh-journal-client/logger.yml @@ -0,0 +1,2 @@ +brokers: + - kafka