diff --git a/Dockerfile b/Dockerfile --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,7 @@ COPY services/swh-storage-listener/entrypoint.sh /swh-storage-listener/entrypoint.sh COPY services/swh-vault/entrypoint.sh /swh-vault/entrypoint.sh COPY services/swh-web/entrypoint.sh /swh-web/entrypoint.sh +COPY services/swh-journal-backfiller/entrypoint.sh /swh-journal-backfiller/entrypoint.sh COPY services/swh-journal-client/client.py /swh-journal-client/ @@ -45,4 +46,3 @@ RUN useradd -ms /bin/bash swh RUN mkdir -p /srv/softwareheritage/objects - diff --git a/conf/journal_backfiller.yml b/conf/journal_backfiller.yml new file mode 100644 --- /dev/null +++ b/conf/journal_backfiller.yml @@ -0,0 +1,11 @@ +brokers: + - kafka + +final_prefix: swh.journal.objects +client_id: swh.journal.backfiller +object_types: + - content + +publisher_id: swh.journal.publisher + +storage_dbconn: postgresql:///?service=swh-storage diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -378,6 +378,20 @@ volumes: - "./conf/journal_publisher.yml:/journal_publisher.yml:ro" + swh-journal-backfiller: + image: swh/stack + build: ./ + entrypoint: /swh-journal-backfiller/entrypoint.sh + environment: + SWH_CONFIG_FILENAME: /journal_backfiller.yml + env_file: + - ./env/storage-db.env + depends_on: + - swh-storage-db + - kafka + volumes: + - "./conf/journal_backfiller.yml:/journal_backfiller.yml:ro" + swh-journal-client: image: swh/stack build: ./ diff --git a/services/swh-journal-backfiller/entrypoint.sh b/services/swh-journal-backfiller/entrypoint.sh new file mode 100755 --- /dev/null +++ b/services/swh-journal-backfiller/entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo Installed Python packages: +pip list + +source /swh-utils/pgsql.sh + +setup_pgsql + +case "$1" in + "shell") + exec bash -i + ;; + *) + wait_pgsql + echo "Starting swh-journal-backfiller" + exec wait-for-it kafka:9092 -s --timeout=0 -- swh-journal backfiller + ;; +esac