diff --git a/docker/conf/journal_backfiller.yml b/docker/conf/journal_backfiller.yml deleted file mode 100644 index 268b752..0000000 --- a/docker/conf/journal_backfiller.yml +++ /dev/null @@ -1,9 +0,0 @@ -brokers: - - kafka - -final_prefix: swh.journal.objects -client_id: swh.journal.backfiller -object_types: - - content - -storage_dbconn: postgresql:///?service=swh-storage diff --git a/docker/conf/storage-mirror-backfiller.yml b/docker/conf/storage-mirror-backfiller.yml new file mode 100644 index 0000000..323fbbf --- /dev/null +++ b/docker/conf/storage-mirror-backfiller.yml @@ -0,0 +1,8 @@ +storage: + cls: local + db: postgresql:///?service=swh-storage +journal_writer: + brokers: + - kafka + prefix: swh.journal.objects + client_id: swh.journal.backfiller diff --git a/docker/conf/storage-mirror.yml b/docker/conf/storage-mirror.yml index 6237bc7..9e806fa 100644 --- a/docker/conf/storage-mirror.yml +++ b/docker/conf/storage-mirror.yml @@ -1,6 +1,17 @@ storage: cls: local db: postgresql:///?service=swh-storage-mirror objstorage: cls: remote url: http://swh-objstorage:5003/ +journal_client: + cls: kafka + brokers: + - kafka + prefix: swh.journal.objects + group_id: swh.storage.mirror.replayer +journal_writer: + brokers: + - kafka + prefix: swh.journal.objects + client_id: swh.journal.backfiller diff --git a/docker/docker-compose.storage-mirror.yml b/docker/docker-compose.storage-mirror.yml index 885cd9e..2c8b6ad 100644 --- a/docker/docker-compose.storage-mirror.yml +++ b/docker/docker-compose.storage-mirror.yml @@ -1,69 +1,69 @@ version: "2.1" services: # override web app to use the mirror swh-web: environment: SWH_CONFIG_FILENAME: /web-mirror.yml volumes: - "./conf/web-mirror.yml:/web-mirror.yml:ro" # create a dedicated db for the mirror swh-storage-mirror-db: image: postgres:12 env_file: - ./env/storage-db-mirror.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: # and an RPC server swh-storage-mirror: image: swh/stack build: ./ depends_on: - swh-storage-mirror-db - swh-objstorage env_file: - ./env/storage-db-mirror.env environment: SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # and the background process that keeps the mirror in sync with the # main graph swh-storage-mirror-replayer: image: swh/stack build: ./ depends_on: - swh-storage-mirror-db - swh-objstorage env_file: - ./env/storage-db-mirror.env environment: SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage-replayer/entrypoint.sh:/entrypoint.sh:ro" swh-journal-backfiller: image: swh/stack build: ./ entrypoint: /entrypoint.sh environment: - SWH_CONFIG_FILENAME: /journal_backfiller.yml + SWH_CONFIG_FILENAME: /storage-mirror-backfiller.yml env_file: - ./env/storage-db.env depends_on: - swh-storage-db - kafka volumes: - - "./conf/journal_backfiller.yml:/journal_backfiller.yml:ro" + - "./conf/storage-mirror-backfiller.yml:/storage-mirror-backfiller.yml:ro" - "./services/swh-journal-backfiller/entrypoint.sh:/entrypoint.sh:ro" diff --git a/docker/services/swh-storage-replayer/entrypoint.sh b/docker/services/swh-storage-replayer/entrypoint.sh index 6623860..2e4e823 100755 --- a/docker/services/swh-storage-replayer/entrypoint.sh +++ b/docker/services/swh-storage-replayer/entrypoint.sh @@ -1,28 +1,29 @@ #!/bin/bash set -e source /srv/softwareheritage/utils/pyutils.sh setup_pip source /srv/softwareheritage/utils/pgsql.sh setup_pgsql case "$1" in "shell") exec bash -i ;; *) - wait_pgsql + wait_pgsql template1 - echo Setup the database - PGPASSWORD=${POSTGRES_PASSWORD} swh db-init \ - --db-name ${POSTGRES_DB} storage + echo Database setup + if ! check_pgsql_db_created; then + echo Creating database and extensions... + swh db create --db-name ${POSTGRES_DB} storage + fi + echo Initializing the database... + swh db init --db-name ${POSTGRES_DB} --flavor mirror storage echo Starting the swh-storage Kafka storage replayer - exec swh storage replay \ - --broker kafka \ - --prefix swh.journal.objects \ - --consumer-id swh.storage.mirror + exec swh storage replay ;; esac