diff --git a/conf/vault-api.yml b/conf/vault-api.yml new file mode 100644 index 0000000..b3ec6a3 --- /dev/null +++ b/conf/vault-api.yml @@ -0,0 +1,17 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ +scheduler: + cls: remote + args: + url: http://swh-scheduler-api:5008/ +vault: + cls: local + args: + db: postgresql:///?service=swh-vault +cache: + cls: pathslicing + args: + root: /srv/softwareheritage/vault + slicing: 0:5 diff --git a/conf/vault-worker.yml b/conf/vault-worker.yml new file mode 100644 index 0000000..f0500d4 --- /dev/null +++ b/conf/vault-worker.yml @@ -0,0 +1,13 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ +vault: + cls: remote + args: + url: http://swh-vault-api:5005/ +celery: + task_broker: amqp://guest:guest@amqp// + task_modules: + - swh.vault.cooking_tasks +max_bundle_size: 536870912 diff --git a/conf/web.yml b/conf/web.yml index 9592cd8..04f8875 100644 --- a/conf/web.yml +++ b/conf/web.yml @@ -1,24 +1,39 @@ storage: cls: remote args: url: http://swh-storage:5002/ timeout: 1 objstorage: cls: remote args: url: http://swh-objstorage:5003/ indexer_storage: cls: remote args: url: http://swh-idx-storage:5007/ +scheduler: + cls: remote + args: + url: http://swh-scheduler-api:5008/ + +vault: + cls: remote + args: + url: http://swh-vault-api:5005/ + +deposit: + private_api_url: https://swh-deposit:5006/1/private/ + private_api_user: swhworker + private_api_password: '' + allowed_hosts: - "*" debug: yes grecaptcha: activated: false site_key: '' diff --git a/docker-compose.yml b/docker-compose.yml index 17e665e..85d2a25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,296 +1,329 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5072:5672 # flower: # image: mher/flower # command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower # ports: # - 5055:5555 # depends_on: # - amqp zookeeper: image: wurstmeister/zookeeper kafka: image: wurstmeister/kafka ports: - 5092:9092 env_file: ./env/kafka.env depends_on: - zookeeper prometheus: image: prom/prometheus command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped grafana: image: grafana/grafana restart: unless-stopped environment: GF_SERVER_ROOT_URL: http://localhost:5080/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards" nginx: image: nginx volumes: - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 # Scheduler swh-scheduler-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-scheduler swh-scheduler-api: image: swh/scheduler-api build: ./dockerfiles/swh-scheduler-api env_file: ./env/scheduler.env depends_on: - swh-scheduler-db ports: - 5008:5008 volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-listener: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./env/scheduler.env command: listener depends_on: - swh-scheduler-api - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-runner: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./env/scheduler.env command: runner -p 10 depends_on: - swh-scheduler-api - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" # Graph storage swh-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage swh-storage: build: ./dockerfiles/swh-storage image: swh/storage ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage env_file: ./env/storage.env volumes: - "./conf/storage.yml:/storage.yml:ro" # Object storage swh-objstorage: build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - 5003:5003 volumes: - "./conf/objstorage.yml:/objstorage.yml:ro" # Indexer storage swh-idx-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage swh-idx-storage: build: ./dockerfiles/swh-indexer-storage image: swh/indexer-storage ports: - 5007:5007 depends_on: - swh-idx-storage-db environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres volumes: - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" # Web interface swh-web: build: ./dockerfiles/swh-web image: swh/web command: - runserver - --verbosity 3 - --insecure - 0.0.0.0:5004 ports: - 5004:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage environment: VERBOSITY: 3 + #DJANGO_SETTINGS_MODULE: djangosettings PYTHONPATH: /tmp/swh volumes: - "./conf/web.yml:/etc/softwareheritage/web/web.yml:ro" - "./conf/djangosettings.py:/tmp/swh/djangosettings.py" swh-deposit-db: image: postgres:10 env_file: ./env/deposit.env swh-deposit: build: ./dockerfiles/swh-deposit image: swh/deposit ports: - 5006:5006 depends_on: - swh-deposit-db - swh-scheduler-api env_file: ./env/deposit.env environment: PGHOST: swh-deposit-db volumes: - "./conf/deposit_server.yml:/etc/softwareheritage/deposit/server.yml:ro" - "./conf/deposit_private.yml:/etc/softwareheritage/deposit/private.yml:ro" + swh-vault-db: + image: postgres:10 + env_file: ./env/vault.env + + swh-vault-api: + build: ./dockerfiles/swh-vault + image: swh/vault + env_file: ./env/vault.env + command: server + ports: + - 5005:5005 + depends_on: + - swh-vault-db + - swh-objstorage + - swh-storage + - swh-scheduler-api + volumes: + - "./conf/vault-api.yml:/vault-api.yml:ro" + + swh-vault-worker: + build: ./dockerfiles/swh-vault + image: swh/vault + command: worker + environment: + SWH_CONFIG_FILENAME: /cooker.yml + depends_on: + - swh-vault-api + - swh-storage + volumes: + - "./conf/vault-worker.yml:/cooker.yml:ro" + + # Lister Celery workers swh-listers-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword swh-lister: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: listers SWH_CONFIG_FILENAME: /lister.yml depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp volumes: - "./conf/lister.yml:/lister.yml:ro" # Loader Celery workers swh-loader: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: loader SWH_CONFIG_FILENAME: /loader.yml depends_on: - swh-storage - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" # Indexer Celery workers swh-indexer: image: swh/indexer-worker build: ./dockerfiles/swh-indexer-worker env_file: ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp volumes: - "./conf/indexer.yml:/indexer.yml:ro" swh-indexer-journal-client: image: swh/indexer-journal-client build: ./dockerfiles/swh-indexer-journal-client depends_on: - swh-journal-publisher - swh-scheduler-api volumes: - "./conf/journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" # Journal related swh-storage-listener: image: swh/storage-listener build: ./dockerfiles/swh-storage-listener env_file: ./env/storage.env depends_on: - swh-storage-db - kafka volumes: - "./conf/storage_listener.yml:/etc/softwareheritage/storage/listener.yml:ro" swh-journal-publisher: image: swh/journal-publisher build: ./dockerfiles/swh-journal-publisher depends_on: - kafka - swh-storage-listener volumes: - "./conf/journal_publisher.yml:/etc/softwareheritage/journal/publisher.yml:ro" swh-journal-client: image: swh/journal-client build: ./dockerfiles/swh-journal-client depends_on: - swh-journal-publisher volumes: - "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro" diff --git a/dockerfiles/swh-vault/Dockerfile b/dockerfiles/swh-vault/Dockerfile new file mode 100644 index 0000000..826b3bf --- /dev/null +++ b/dockerfiles/swh-vault/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.6 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install --upgrade pip setuptools wheel +RUN pip install swh-vault + +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-vault/entrypoint.sh b/dockerfiles/swh-vault/entrypoint.sh new file mode 100755 index 0000000..f7268c8 --- /dev/null +++ b/dockerfiles/swh-vault/entrypoint.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo Installed Python packages: +pip list + +if [[ -n $PGHOST ]]; then + echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass + cat > ~/.pg_service.conf <&1 > /dev/null; do sleep 0.1; done + + echo Setup the swh-vault API database + PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init vault \ + --db-name ${POSTGRES_DB} + + echo Starting the swh-vault API server + exec swh-vault -C /vault-api.yml +esac diff --git a/env/vault.env b/env/vault.env new file mode 100644 index 0000000..684350e --- /dev/null +++ b/env/vault.env @@ -0,0 +1,4 @@ +POSTGRES_DB=softwareheritage-vault +POSTGRES_PASSWORD=testpassword +PGUSER=postgres +PGHOST=swh-vault-db