diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,46 @@ services: + amqp: + image: rabbitmq:3.6-management + ports: + - 5018:15672 + +# Scheduler + + swh-scheduler-db: + image: postgres:10 + environment: + POSTGRES_PASSWORD: testpassword + POSTGRES_DB: swh-scheduler + + swh-scheduler-api: + image: swh/scheduler-api + build: ./dockerfiles/swh-scheduler-api + env_file: ./scheduler.env + depends_on: + - swh-scheduler-db + ports: + - 5008:5008 + + swh-scheduler-listener: + image: swh/scheduler-worker + build: ./dockerfiles/swh-scheduler-worker + env_file: ./scheduler.env + command: listener + depends_on: + - swh-scheduler-api + - amqp + + swh-scheduler-runner: + image: swh/scheduler-worker + env_file: ./scheduler.env + command: runner + depends_on: + - swh-scheduler-listener # for the image only + - swh-scheduler-api + - amqp + # Graph storage swh-storage-db: @@ -14,7 +54,7 @@ build: ./dockerfiles/swh-storage image: swh/storage ports: - - 5002:5002 + - 5002:5002 depends_on: - swh-storage-db - swh-objstorage @@ -30,7 +70,7 @@ build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - - 5003:5003 + - 5003:5003 # Indexer storage @@ -59,7 +99,7 @@ build: ./dockerfiles/swh-web image: swh/web ports: - - 5004:5004 + - 5004:5004 depends_on: - swh-objstorage - swh-storage diff --git a/dockerfiles/swh-scheduler-api/Dockerfile b/dockerfiles/swh-scheduler-api/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-scheduler-api/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install swh-scheduler +COPY scheduler.yml / +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 5008 diff --git a/dockerfiles/swh-scheduler-api/entrypoint.sh b/dockerfiles/swh-scheduler-api/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-scheduler-api/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass +cat > ~/.pg_service.conf <&1 > /dev/null; do sleep 0.1; done + + echo Setup the swh-scheduler API database + PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init scheduler \ + --db-name ${POSTGRES_DB} + + echo Starting the swh-scheduler API server + exec python -m swh.scheduler.api.server /scheduler.yml +esac diff --git a/dockerfiles/swh-scheduler-api/scheduler.yml b/dockerfiles/swh-scheduler-api/scheduler.yml new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-scheduler-api/scheduler.yml @@ -0,0 +1,4 @@ +scheduler: + cls: local + args: + db: service=swh diff --git a/dockerfiles/swh-scheduler-worker/Dockerfile b/dockerfiles/swh-scheduler-worker/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-scheduler-worker/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install swh-scheduler +RUN mkdir /etc/softwareheritage/ +COPY scheduler.yml /etc/softwareheritage/ +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 5008 diff --git a/dockerfiles/swh-scheduler-worker/entrypoint.sh b/dockerfiles/swh-scheduler-worker/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-scheduler-worker/entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass +cat > ~/.pg_service.conf <