diff --git a/docker/README.md b/docker/README.md --- a/docker/README.md +++ b/docker/README.md @@ -585,16 +585,16 @@ ``` -## Starting a kafka-powered replica of the storage +## Starting a kafka-powered mirror of the storage -This repo comes with an optional `docker-compose.storage-replica.yml` -docker compose file that can be used to test the kafka-powered replication +This repo comes with an optional `docker-compose.storage-mirror.yml` +docker compose file that can be used to test the kafka-powered mirror mecanism for the main storage. This can be used like: ``` -~/swh-environment/docker$ docker-compose -f docker-compose.yml -f docker-compose.storage-replica.yml up -d +~/swh-environment/docker$ docker-compose -f docker-compose.yml -f docker-compose.storage-mirror.yml up -d [...] ``` @@ -602,15 +602,15 @@ - overrides the swh-storage service to activate the kafka direct writer on swh.journal.objects prefixed topics using thw swh.storage.master ID, -- overrides the swh-web service to make it use the replica instead of the +- overrides the swh-web service to make it use the mirror instead of the master storage, -- starts a db for the replica, +- starts a db for the mirror, - starts a storage service based on this db, - starts a replayer service that runs the process that listen to kafka to - keeps the replica in sync. + keeps the mirror in sync. When using it, you will have a setup in which the master storage is used by -workers and most other services, whereas the storage replica will be used to +workers and most other services, whereas the storage mirror will be used to by the web application and should be kept in sync with the master storage by kafka. @@ -626,8 +626,8 @@ ``` (swh)$ docker-compose \ -f docker-compose.yml \ - -f docker-compose.storage-replica.yml \ - -f docker-compose.storage-replica.override.yml \ + -f docker-compose.storage-mirror.yml \ + -f docker-compose.storage-mirror.override.yml \ run \ swh-journal-backfiller \ snapshot \ diff --git a/docker/conf/storage-replica.yml b/docker/conf/storage-mirror.yml rename from docker/conf/storage-replica.yml rename to docker/conf/storage-mirror.yml --- a/docker/conf/storage-replica.yml +++ b/docker/conf/storage-mirror.yml @@ -1,7 +1,7 @@ storage: cls: local args: - db: postgresql:///?service=swh-storage-replica + db: postgresql:///?service=swh-storage-mirror objstorage: cls: remote args: diff --git a/docker/conf/web-replica.yml b/docker/conf/web-mirror.yml rename from docker/conf/web-replica.yml rename to docker/conf/web-mirror.yml --- a/docker/conf/web-replica.yml +++ b/docker/conf/web-mirror.yml @@ -1,7 +1,7 @@ storage: cls: remote args: - url: http://swh-storage-replica:5002/ + url: http://swh-storage-mirror:5002/ timeout: 1 objstorage: diff --git a/docker/docker-compose.storage-replica.yml b/docker/docker-compose.storage-mirror.yml rename from docker/docker-compose.storage-replica.yml rename to docker/docker-compose.storage-mirror.yml --- a/docker/docker-compose.storage-replica.yml +++ b/docker/docker-compose.storage-mirror.yml @@ -1,53 +1,53 @@ version: '2' services: - # override web app to use the replica + # override web app to use the mirror swh-web: environment: - SWH_CONFIG_FILENAME: /web-replica.yml + SWH_CONFIG_FILENAME: /web-mirror.yml volumes: - - "./conf/web-replica.yml:/web-replica.yml:ro" + - "./conf/web-mirror.yml:/web-mirror.yml:ro" - # create a dedicated db for the replica - swh-storage-replica-db: + # create a dedicated db for the mirror + swh-storage-mirror-db: image: postgres:11 env_file: - - ./env/storage-db-replica.env + - ./env/storage-db-mirror.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # and an RPC server - swh-storage-replica: + swh-storage-mirror: image: swh/stack build: ./ depends_on: - - swh-storage-replica-db + - swh-storage-mirror-db - swh-objstorage env_file: - - ./env/storage-db-replica.env + - ./env/storage-db-mirror.env environment: - SWH_CONFIG_FILENAME: /storage-replica.yml + SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - - "./conf/storage-replica.yml:/storage-replica.yml:ro" + - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" - # and the background process that keeps the replica in sync with the + # and the background process that keeps the mirror in sync with the # main graph - swh-storage-replica-replayer: + swh-storage-mirror-replayer: image: swh/stack build: ./ depends_on: - - swh-storage-replica-db + - swh-storage-mirror-db - swh-objstorage env_file: - - ./env/storage-db-replica.env + - ./env/storage-db-mirror.env environment: - SWH_CONFIG_FILENAME: /storage-replica.yml + SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - - "./conf/storage-replica.yml:/storage-replica.yml:ro" + - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage-replayer/entrypoint.sh:/entrypoint.sh:ro" swh-journal-backfiller: diff --git a/docker/env/storage-db-mirror.yml b/docker/env/storage-db-mirror.yml new file mode 100644 --- /dev/null +++ b/docker/env/storage-db-mirror.yml @@ -0,0 +1,4 @@ +PGHOST=swh-storage-mirror-db +PGUSER=postgres +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=swh-storage-mirror diff --git a/docker/env/storage-db-replica.env b/docker/env/storage-db-replica.env deleted file mode 100644 --- a/docker/env/storage-db-replica.env +++ /dev/null @@ -1,4 +0,0 @@ -PGHOST=swh-storage-replica-db -PGUSER=postgres -POSTGRES_PASSWORD=testpassword -POSTGRES_DB=swh-storage-replica diff --git a/docker/services/swh-storage-replayer/entrypoint.sh b/docker/services/swh-storage-replayer/entrypoint.sh --- a/docker/services/swh-storage-replayer/entrypoint.sh +++ b/docker/services/swh-storage-replayer/entrypoint.sh @@ -23,6 +23,6 @@ exec swh journal replay \ --broker kafka \ --prefix swh.journal.objects \ - --consumer-id swh.storage.replica + --consumer-id swh.storage.mirror ;; esac