diff --git a/Dockerfile b/Dockerfile --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,8 @@ COPY services/swh-journal-client/client.py /swh-journal-client/ +COPY utils/pgsql.sh /swh-utils/pgsql.sh + RUN useradd -ms /bin/bash swh RUN mkdir -p /srv/softwareheritage/objects diff --git a/conf/deposit.yml b/conf/deposit.yml --- a/conf/deposit.yml +++ b/conf/deposit.yml @@ -8,7 +8,7 @@ db: host: swh-deposit-db port: 5432 - name: softwareheritage-deposit + name: swh-deposit user: postgres password: testpassword media_root: /tmp/swh-deposit/uploads diff --git a/conf/indexer_storage.yml b/conf/indexer_storage.yml --- a/conf/indexer_storage.yml +++ b/conf/indexer_storage.yml @@ -1,4 +1,4 @@ indexer_storage: cls: local args: - db: postgresql:///?service=swh-indexer + db: postgresql:///?service=swh-indexers diff --git a/conf/lister.yml b/conf/lister.yml --- a/conf/lister.yml +++ b/conf/lister.yml @@ -11,7 +11,7 @@ lister: cls: local args: - db: postgresql:///?service=swh + db: postgresql:///?service=swh-listers celery: task_broker: amqp://guest:guest@amqp// diff --git a/conf/loader.yml b/conf/loader.yml --- a/conf/loader.yml +++ b/conf/loader.yml @@ -2,10 +2,6 @@ cls: remote args: url: http://swh-storage:5002/ -lister: - cls: local - args: - db: postgresql:///?service=swh celery: task_broker: amqp://guest:guest@amqp// task_modules: diff --git a/conf/storage.yml b/conf/storage.yml --- a/conf/storage.yml +++ b/conf/storage.yml @@ -1,7 +1,7 @@ storage: cls: local args: - db: postgresql:///?service=swh + db: postgresql:///?service=swh-storage objstorage: cls: remote args: diff --git a/conf/storage_listener.yml b/conf/storage_listener.yml --- a/conf/storage_listener.yml +++ b/conf/storage_listener.yml @@ -1,4 +1,4 @@ -database: postgresql:///?service=swh +database: postgresql:///?service=swh-storage brokers: - kafka topic_prefix: swh.tmp.journal.new diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,14 +64,18 @@ swh-scheduler-db: image: postgres:11 + env_file: + - ./env/scheduler-db.env environment: - POSTGRES_PASSWORD: testpassword - POSTGRES_DB: swh-scheduler + # unset PGHOST as db service crashes otherwise + PGHOST: swh-scheduler-api: image: swh/stack build: ./ - env_file: ./env/scheduler.env + env_file: + - ./env/scheduler-db.env + - ./env/scheduler.env entrypoint: /swh-scheduler-api/entrypoint.sh environment: SWH_CONFIG_FILENAME: /scheduler.yml @@ -87,7 +91,9 @@ image: swh/stack build: ./ entrypoint: /swh-scheduler-worker/entrypoint.sh - env_file: ./env/scheduler.env + env_file: + - ./env/scheduler-db.env + - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml @@ -102,7 +108,9 @@ image: swh/stack build: ./ entrypoint: /swh-scheduler-worker/entrypoint.sh - env_file: ./env/scheduler.env + env_file: + - ./env/scheduler-db.env + - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml @@ -117,9 +125,11 @@ swh-storage-db: image: postgres:11 + env_file: + - ./env/storage-db.env environment: - POSTGRES_PASSWORD: testpassword - POSTGRES_DB: swh-storage + # unset PGHOST as db service crashes otherwise + PGHOST: swh-storage: build: ./ @@ -130,7 +140,10 @@ depends_on: - swh-storage-db - swh-objstorage - env_file: ./env/storage.env + env_file: + - ./env/storage-db.env + environment: + SWH_CONFIG_FILENAME: /storage.yml volumes: - "./conf/storage.yml:/storage.yml:ro" @@ -151,9 +164,11 @@ swh-idx-storage-db: image: postgres:11 + env_file: + - ./env/indexers-db.env environment: - POSTGRES_PASSWORD: testpassword - POSTGRES_DB: swh-idx-storage + # unset PGHOST as db service crashes otherwise + PGHOST: swh-idx-storage: build: ./ @@ -163,11 +178,9 @@ - 5007:5007 depends_on: - swh-idx-storage-db + env_file: + - ./env/indexers-db.env environment: - POSTGRES_PASSWORD: testpassword - POSTGRES_DB: swh-idx-storage - PGHOST: swh-idx-storage-db - PGUSER: postgres SWH_CONFIG_FILENAME: /indexer_storage.yml volumes: - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" @@ -194,7 +207,11 @@ swh-deposit-db: image: postgres:11 - env_file: ./env/deposit.env + env_file: + - ./env/deposit-db.env + environment: + # unset PGHOST as db service crashes otherwise + PGHOST: swh-deposit: build: ./ @@ -205,23 +222,28 @@ depends_on: - swh-deposit-db - swh-scheduler-api - env_file: ./env/deposit.env - environment: - PGHOST: swh-deposit-db + env_file: + - ./env/deposit-db.env + - ./env/deposit.env volumes: - "./conf/deposit.yml:/deposit.yml:ro" swh-vault-db: image: postgres:11 + env_file: + - ./env/vault-db.env environment: - POSTGRES_PASSWORD: testpassword - POSTGRES_DB: swh-vault + # unset PGHOST as db service crashes otherwise + PGHOST: swh-vault-api: build: ./ image: swh/stack entrypoint: /swh-vault/entrypoint.sh - env_file: ./env/vault.env + env_file: + - ./env/vault-db.env + environment: + SWH_CONFIG_FILENAME: /vault-api.yml command: server ports: - 5005:5005 @@ -251,14 +273,19 @@ swh-listers-db: image: postgres:11 + env_file: + - ./env/listers-db.env environment: - POSTGRES_PASSWORD: testpassword + # unset PGHOST as db service crashes otherwise + PGHOST: swh-lister: image: swh/stack build: ./ entrypoint: /swh-listers-worker/entrypoint.sh - env_file: ./env/listers.env + env_file: + - ./env/listers-db.env + - ./env/listers.env user: swh environment: STATSD_HOST: prometheus-statsd-exporter @@ -280,7 +307,8 @@ image: swh/stack build: ./ entrypoint: /swh-loaders-worker/entrypoint.sh - env_file: ./env/listers.env + env_file: + - ./env/listers.env user: swh environment: STATSD_HOST: prometheus-statsd-exporter @@ -300,7 +328,9 @@ build: ./ user: swh entrypoint: /swh-indexer-worker/entrypoint.sh - env_file: ./env/indexers.env + env_file: + - ./env/indexers-db.env + - ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 @@ -319,7 +349,8 @@ image: swh/stack build: ./ entrypoint: /swh-storage-listener/entrypoint.sh - env_file: ./env/storage.env + env_file: + - ./env/storage-db.env depends_on: - swh-storage-db - kafka diff --git a/env/deposit-db.env b/env/deposit-db.env new file mode 100644 --- /dev/null +++ b/env/deposit-db.env @@ -0,0 +1,5 @@ +PGHOST=swh-deposit-db +PGUSER=postgres +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=swh-deposit + diff --git a/env/deposit.env b/env/deposit.env --- a/env/deposit.env +++ b/env/deposit.env @@ -1,5 +1,2 @@ -POSTGRES_DB=softwareheritage-deposit -POSTGRES_PASSWORD=testpassword -PGUSER=postgres SWH_CONFIG_FILENAME=/deposit.yml -DJANGO_SETTINGS_MODULE=swh.deposit.settings.production \ No newline at end of file +DJANGO_SETTINGS_MODULE=swh.deposit.settings.production diff --git a/env/indexers-db.env b/env/indexers-db.env new file mode 100644 --- /dev/null +++ b/env/indexers-db.env @@ -0,0 +1,4 @@ +PGHOST=swh-idx-storage-db +PGUSER=postgres +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=swh-idx-storage \ No newline at end of file diff --git a/env/listers.env b/env/listers-db.env copy from env/listers.env copy to env/listers-db.env --- a/env/listers.env +++ b/env/listers-db.env @@ -1,6 +1,4 @@ -CONCURRENCY=1 -MAX_TASKS_PER_CHILD=10 -LOGLEVEL=DEBUG -POSTGRES_PASSWORD=testpassword PGHOST=swh-listers-db PGUSER=postgres +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=swh-listers diff --git a/env/listers.env b/env/listers.env --- a/env/listers.env +++ b/env/listers.env @@ -1,6 +1,3 @@ CONCURRENCY=1 MAX_TASKS_PER_CHILD=10 LOGLEVEL=DEBUG -POSTGRES_PASSWORD=testpassword -PGHOST=swh-listers-db -PGUSER=postgres diff --git a/env/scheduler.env b/env/scheduler-db.env copy from env/scheduler.env copy to env/scheduler-db.env --- a/env/scheduler.env +++ b/env/scheduler-db.env @@ -1,7 +1,4 @@ -POSTGRES_PASSWORD=testpassword -POSTGRES_DB=swh-scheduler PGHOST=swh-scheduler-db PGUSER=postgres -SWH_WORKER_INSTANCE=scheduler -LOGLEVEL=INFO -CELERY_BROKER_URL=amqp://amqp// +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=swh-scheduler diff --git a/env/scheduler.env b/env/scheduler.env --- a/env/scheduler.env +++ b/env/scheduler.env @@ -1,7 +1,3 @@ -POSTGRES_PASSWORD=testpassword -POSTGRES_DB=swh-scheduler -PGHOST=swh-scheduler-db -PGUSER=postgres SWH_WORKER_INSTANCE=scheduler LOGLEVEL=INFO CELERY_BROKER_URL=amqp://amqp// diff --git a/env/storage-db.env b/env/storage-db.env new file mode 100644 --- /dev/null +++ b/env/storage-db.env @@ -0,0 +1,4 @@ +PGHOST=swh-storage-db +PGUSER=postgres +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=swh-storage diff --git a/env/vault-db.env b/env/vault-db.env new file mode 100644 --- /dev/null +++ b/env/vault-db.env @@ -0,0 +1,4 @@ +POSTGRES_DB=swh-vault +POSTGRES_PASSWORD=testpassword +PGUSER=postgres +PGHOST=swh-vault-db diff --git a/services/swh-deposit/entrypoint.sh b/services/swh-deposit/entrypoint.sh --- a/services/swh-deposit/entrypoint.sh +++ b/services/swh-deposit/entrypoint.sh @@ -12,22 +12,15 @@ done fi -echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass -echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" >> ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass -cat > ~/.pg_service.conf < ~/.pgpass - cat > ~/.pg_service.conf < ~/.pgpass + echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" >> ~/.pgpass + cat > ~/.pg_service.conf < /dev/null 2> /dev/null; do sleep 1; done +} \ No newline at end of file