diff --git a/docker-compose.yml b/docker-compose.yml index feba92a..713d128 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,66 +1,106 @@ version: '2' services: + amqp: + image: rabbitmq:3.6-management + ports: + - 5018:15672 + +# Scheduler + + swh-scheduler-db: + image: postgres:10 + environment: + POSTGRES_PASSWORD: testpassword + POSTGRES_DB: swh-scheduler + + swh-scheduler-api: + image: swh/scheduler-api + build: ./dockerfiles/swh-scheduler-api + env_file: ./scheduler.env + depends_on: + - swh-scheduler-db + ports: + - 5008:5008 + + swh-scheduler-listener: + image: swh/scheduler-worker + build: ./dockerfiles/swh-scheduler-worker + env_file: ./scheduler.env + command: listener + depends_on: + - swh-scheduler-api + - amqp + + swh-scheduler-runner: + image: swh/scheduler-worker + env_file: ./scheduler.env + command: runner + depends_on: + - swh-scheduler-listener # for the image only + - swh-scheduler-api + - amqp + # Graph storage swh-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage swh-storage: build: ./dockerfiles/swh-storage image: swh/storage ports: - - 5002:5002 + - 5002:5002 depends_on: - swh-storage-db - swh-objstorage environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage PGHOST: swh-storage-db PGUSER: postgres # Object storage swh-objstorage: build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - - 5003:5003 + - 5003:5003 # Indexer storage swh-idx-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage swh-idx-storage: build: ./dockerfiles/swh-indexer-storage image: swh/indexer-storage ports: - 5007:5007 depends_on: - swh-idx-storage-db environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres # Web interface swh-web: build: ./dockerfiles/swh-web image: swh/web ports: - - 5004:5004 + - 5004:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage diff --git a/dockerfiles/swh-scheduler-api/Dockerfile b/dockerfiles/swh-scheduler-api/Dockerfile new file mode 100644 index 0000000..0369f1b --- /dev/null +++ b/dockerfiles/swh-scheduler-api/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install swh-scheduler +COPY scheduler.yml / +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 5008 diff --git a/dockerfiles/swh-scheduler-api/entrypoint.sh b/dockerfiles/swh-scheduler-api/entrypoint.sh new file mode 100755 index 0000000..3978e2d --- /dev/null +++ b/dockerfiles/swh-scheduler-api/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass +cat > ~/.pg_service.conf <&1 > /dev/null; do sleep 0.1; done + + echo Setup the swh-scheduler API database + PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init scheduler \ + --db-name ${POSTGRES_DB} + + echo Starting the swh-scheduler API server + exec python -m swh.scheduler.api.server /scheduler.yml +esac diff --git a/dockerfiles/swh-scheduler-api/scheduler.yml b/dockerfiles/swh-scheduler-api/scheduler.yml new file mode 100644 index 0000000..b3e2ae5 --- /dev/null +++ b/dockerfiles/swh-scheduler-api/scheduler.yml @@ -0,0 +1,4 @@ +scheduler: + cls: local + args: + db: service=swh diff --git a/dockerfiles/swh-scheduler-worker/Dockerfile b/dockerfiles/swh-scheduler-worker/Dockerfile new file mode 100644 index 0000000..f9b3681 --- /dev/null +++ b/dockerfiles/swh-scheduler-worker/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install swh-scheduler +RUN mkdir /etc/softwareheritage/ +COPY scheduler.yml /etc/softwareheritage/ +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 5008 diff --git a/dockerfiles/swh-scheduler-worker/entrypoint.sh b/dockerfiles/swh-scheduler-worker/entrypoint.sh new file mode 100755 index 0000000..b7bcb49 --- /dev/null +++ b/dockerfiles/swh-scheduler-worker/entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass +cat > ~/.pg_service.conf <