diff --git a/conf/journal_backfiller.yml b/conf/journal_backfiller.yml new file mode 100644 index 0000000..268b752 --- /dev/null +++ b/conf/journal_backfiller.yml @@ -0,0 +1,9 @@ +brokers: + - kafka + +final_prefix: swh.journal.objects +client_id: swh.journal.backfiller +object_types: + - content + +storage_dbconn: postgresql:///?service=swh-storage diff --git a/docker-compose.storage-replica.yml b/docker-compose.storage-replica.yml index 533cf3f..2824615 100644 --- a/docker-compose.storage-replica.yml +++ b/docker-compose.storage-replica.yml @@ -1,51 +1,66 @@ version: '2' services: # override web app to use the replica swh-web: environment: SWH_CONFIG_FILENAME: /web-replica.yml volumes: - "./conf/web-replica.yml:/web-replica.yml:ro" # create a dedicated db for the replica swh-storage-replica-db: image: postgres:11 env_file: - ./env/storage-db-replica.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # and an RPC server swh-storage-replica: image: swh/stack build: ./ depends_on: - swh-storage-replica-db - swh-objstorage env_file: - ./env/storage-db-replica.env environment: SWH_CONFIG_FILENAME: /storage-replica.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-replica.yml:/storage-replica.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # and the background process that keeps the replica in sync with the # main graph swh-storage-replica-replayer: image: swh/stack build: ./ depends_on: - swh-storage-replica-db - swh-objstorage env_file: - ./env/storage-db-replica.env environment: SWH_CONFIG_FILENAME: /storage-replica.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-replica.yml:/storage-replica.yml:ro" - "./services/swh-storage-replayer/entrypoint.sh:/entrypoint.sh:ro" + + swh-journal-backfiller: + image: swh/stack + build: ./ + entrypoint: /entrypoint.sh + environment: + SWH_CONFIG_FILENAME: /journal_backfiller.yml + env_file: + - ./env/storage-db.env + depends_on: + - swh-storage-db + - kafka + volumes: + - "./conf/journal_backfiller.yml:/journal_backfiller.yml:ro" + - "./services/swh-journal-backfiller/entrypoint.sh:/entrypoint.sh:ro" diff --git a/services/swh-journal-backfiller/entrypoint.sh b/services/swh-journal-backfiller/entrypoint.sh new file mode 100755 index 0000000..d24bf1b --- /dev/null +++ b/services/swh-journal-backfiller/entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +source /srv/softwareheritage/utils/pyutils.sh +setup_pip + +source /srv/softwareheritage/utils/pgsql.sh +setup_pgsql + +case "$1" in + "shell") + exec bash -i + ;; + *) + wait_pgsql + echo "Starting swh-journal-backfiller" + exec wait-for-it kafka:9092 -s --timeout=0 -- swh-journal backfiller $@ + ;; +esac