diff --git a/conf/djangosettings.py b/conf/djangosettings.py new file mode 100644 --- /dev/null +++ b/conf/djangosettings.py @@ -0,0 +1,3 @@ +from swh.web.settings.common import * # noqa + +AUTH_PASSWORD_VALIDATORS = [] # disable any pwd validation mechanism diff --git a/conf/vault-api.yml b/conf/vault-api.yml new file mode 100644 --- /dev/null +++ b/conf/vault-api.yml @@ -0,0 +1,17 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ +scheduler: + cls: remote + args: + url: http://swh-scheduler-api:5008/ +vault: + cls: local + args: + db: postgresql:///?service=swh-vault +cache: + cls: pathslicing + args: + root: /srv/softwareheritage/vault + slicing: 0:5 diff --git a/conf/vault-worker.yml b/conf/vault-worker.yml new file mode 100644 --- /dev/null +++ b/conf/vault-worker.yml @@ -0,0 +1,13 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ +vault: + cls: remote + args: + url: http://swh-vault-api:5005/ +celery: + task_broker: amqp://guest:guest@amqp// + task_modules: + - swh.vault.cooking_tasks +max_bundle_size: 536870912 diff --git a/conf/web.yml b/conf/web.yml --- a/conf/web.yml +++ b/conf/web.yml @@ -14,6 +14,21 @@ args: url: http://swh-idx-storage:5007/ +scheduler: + cls: remote + args: + url: http://swh-scheduler-api:5008/ + +vault: + cls: remote + args: + url: http://swh-vault-api:5005/ + +deposit: + private_api_url: https://swh-deposit:5006/1/private/ + private_api_user: swhworker + private_api_password: '' + allowed_hosts: - "*" diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,6 +159,12 @@ swh-web: build: ./dockerfiles/swh-web image: swh/web + command: + - runserver + - --verbosity 3 + - --insecure + - 0.0.0.0:5004 + ports: - 5004:5004 depends_on: @@ -167,8 +173,11 @@ - swh-idx-storage environment: VERBOSITY: 3 + #DJANGO_SETTINGS_MODULE: djangosettings + PYTHONPATH: /tmp/swh volumes: - "./conf/web.yml:/etc/softwareheritage/web/web.yml:ro" + - "./conf/djangosettings.py:/tmp/swh/djangosettings.py" swh-deposit-db: image: postgres:10 @@ -189,6 +198,38 @@ - "./conf/deposit_server.yml:/etc/softwareheritage/deposit/server.yml:ro" - "./conf/deposit_private.yml:/etc/softwareheritage/deposit/private.yml:ro" + swh-vault-db: + image: postgres:10 + env_file: ./env/vault.env + + swh-vault-api: + build: ./dockerfiles/swh-vault + image: swh/vault + env_file: ./env/vault.env + command: server + ports: + - 5005:5005 + depends_on: + - swh-vault-db + - swh-objstorage + - swh-storage + - swh-scheduler-api + volumes: + - "./conf/vault-api.yml:/vault-api.yml:ro" + + swh-vault-worker: + build: ./dockerfiles/swh-vault + image: swh/vault + command: worker + environment: + SWH_CONFIG_FILENAME: /cooker.yml + depends_on: + - swh-vault-api + - swh-storage + volumes: + - "./conf/vault-worker.yml:/cooker.yml:ro" + + # Lister Celery workers swh-listers-db: diff --git a/dockerfiles/swh-vault/Dockerfile b/dockerfiles/swh-vault/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-vault/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.6 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install --upgrade pip setuptools wheel +RUN pip install swh-vault + +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-vault/entrypoint.sh b/dockerfiles/swh-vault/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-vault/entrypoint.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo Installed Python packages: +pip list + +if [[ -n $PGHOST ]]; then + echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass + cat > ~/.pg_service.conf <&1 > /dev/null; do sleep 0.1; done + + echo Setup the swh-vault API database + PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init vault \ + --db-name ${POSTGRES_DB} + + echo Starting the swh-vault API server + exec swh-vault -C /vault-api.yml +esac diff --git a/dockerfiles/swh-web/Dockerfile b/dockerfiles/swh-web/Dockerfile --- a/dockerfiles/swh-web/Dockerfile +++ b/dockerfiles/swh-web/Dockerfile @@ -13,5 +13,6 @@ COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] +WORKDIR /tmp/swh EXPOSE 5004 diff --git a/dockerfiles/swh-web/entrypoint.sh b/dockerfiles/swh-web/entrypoint.sh --- a/dockerfiles/swh-web/entrypoint.sh +++ b/dockerfiles/swh-web/entrypoint.sh @@ -14,16 +14,12 @@ echo Installed Python packages: pip list -if [ "$1" = 'shell' ] ; then - exec bash -i -else - echo "starting the swh-web server" - # options: - # --verbosity to have sensible output - # --insecure to serve the static css/js - # 0.0.0.0 so that we can actually reach the service. - exec python3 -m swh.web.manage runserver \ - --verbosity 3 \ - --insecure \ - 0.0.0.0:5004 -fi +case "$1" in + "shell") + exec bash -i + ;; + *) + echo "starting the swh-web server" + exec python3 -m swh.web.manage $@ + ;; +esac diff --git a/env/vault.env b/env/vault.env new file mode 100644 --- /dev/null +++ b/env/vault.env @@ -0,0 +1,4 @@ +POSTGRES_DB=softwareheritage-vault +POSTGRES_PASSWORD=testpassword +PGUSER=postgres +PGHOST=swh-vault-db