diff --git a/Dockerfile b/Dockerfile --- a/Dockerfile +++ b/Dockerfile @@ -26,23 +26,17 @@ COPY services/swh-indexer-journal-client/entrypoint.sh /swh-indexer-journal-client/entrypoint.sh COPY services/swh-indexer-storage/entrypoint.sh /swh-indexer-storage/entrypoint.sh COPY services/swh-indexer-worker/entrypoint.sh /swh-indexer-worker/entrypoint.sh -COPY services/swh-journal-client/entrypoint.sh /swh-journal-client/entrypoint.sh -COPY services/swh-journal-publisher/entrypoint.sh /swh-journal-publisher/entrypoint.sh COPY services/swh-listers-worker/entrypoint.sh /swh-listers-worker/entrypoint.sh COPY services/swh-loaders-worker/entrypoint.sh /swh-loaders-worker/entrypoint.sh COPY services/swh-objstorage/entrypoint.sh /swh-objstorage/entrypoint.sh COPY services/swh-scheduler-api/entrypoint.sh /swh-scheduler-api/entrypoint.sh COPY services/swh-scheduler-worker/entrypoint.sh /swh-scheduler-worker/entrypoint.sh COPY services/swh-storage/entrypoint.sh /swh-storage/entrypoint.sh -COPY services/swh-storage-listener/entrypoint.sh /swh-storage-listener/entrypoint.sh COPY services/swh-vault/entrypoint.sh /swh-vault/entrypoint.sh COPY services/swh-web/entrypoint.sh /swh-web/entrypoint.sh -COPY services/swh-journal-client/client.py /swh-journal-client/ - COPY utils/pgsql.sh /swh-utils/pgsql.sh RUN useradd -ms /bin/bash swh RUN mkdir -p /srv/softwareheritage/objects - diff --git a/conf/journal_client.yml b/conf/journal_client.yml deleted file mode 100644 --- a/conf/journal_client.yml +++ /dev/null @@ -1,7 +0,0 @@ -brokers: - - kafka -max_messages: 50 -scheduler: - cls: remote - args: - url: http://swh-scheduler-api:5008/ diff --git a/conf/journal_publisher.yml b/conf/journal_publisher.yml deleted file mode 100644 --- a/conf/journal_publisher.yml +++ /dev/null @@ -1,22 +0,0 @@ -brokers: - - kafka - -temporary_prefix: swh.tmp.journal.new -final_prefix: swh.journal.objects -consumer_id: swh.journal.publisher -publisher_id: swh.journal.publisher - -object_types: - - content - - revision - - release - - origin - - origin_visit - - snapshot - -storage: - cls: remote - args: - url: http://swh-storage:5002/ - -max_messages: 50 diff --git a/conf/storage_listener.yml b/conf/storage_listener.yml deleted file mode 100644 --- a/conf/storage_listener.yml +++ /dev/null @@ -1,4 +0,0 @@ -database: postgresql:///?service=swh-storage -brokers: - - kafka -topic_prefix: swh.tmp.journal.new diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -354,45 +354,13 @@ # Journal related - swh-storage-listener: - image: swh/stack - build: ./ - entrypoint: /swh-storage-listener/entrypoint.sh - env_file: - - ./env/storage-db.env - depends_on: - - swh-storage-db - - kafka - volumes: - - "./conf/storage_listener.yml:/etc/softwareheritage/storage/listener.yml:ro" - - swh-journal-publisher: - image: swh/stack - build: ./ - entrypoint: /swh-journal-publisher/entrypoint.sh - environment: - SWH_CONFIG_FILENAME: /journal_publisher.yml - depends_on: - - kafka - - swh-storage-listener - volumes: - - "./conf/journal_publisher.yml:/journal_publisher.yml:ro" - - swh-journal-client: - image: swh/stack - build: ./ - entrypoint: /swh-journal-client/entrypoint.sh - depends_on: - - swh-journal-publisher - volumes: - - "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro" - swh-indexer-journal-client: image: swh/stack build: ./ entrypoint: /swh-indexer-journal-client/entrypoint.sh depends_on: - - swh-journal-publisher + - kafka + - swh-storage-api - swh-scheduler-api volumes: - "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" diff --git a/services/swh-journal-client/client.py b/services/swh-journal-client/client.py deleted file mode 100644 --- a/services/swh-journal-client/client.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import logging - -from swh.journal.client import JournalClient - - -class JournalClientLogger(JournalClient): - """Client in charge of listing new received origins and origin_visits - in the swh journal. - - """ - CONFIG_BASE_FILENAME = 'journal/logger' - - def __init__(self): - # Only interested in content here so override the configuration - super().__init__(extra_configuration={'object_types': - ['origin', 'origin_visit']}) - - def process_objects(self, messages): - """Simply log messages received. - - """ - logging.info('client received the following messages: %s' % messages) - - -if __name__ == '__main__': - logging.basicConfig( - level=logging.INFO, - format='%(asctime)s %(process)d %(levelname)s %(message)s' - ) - - import click - - @click.command() - def main(): - """Log the new received origin and origin_visits. - - """ - JournalClientLogger().process() - - main() diff --git a/services/swh-journal-client/entrypoint.sh b/services/swh-journal-client/entrypoint.sh deleted file mode 100755 --- a/services/swh-journal-client/entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list - -case "$1" in - "shell") - exec bash -i - ;; - *) - echo "Starting a swh-journal client" - exec wait-for-it kafka:9092 -s --timeout=0 -- python3 /swh-journal-client/client.py - ;; -esac