Page MenuHomeSoftware Heritage
Files F9345686

D818.id2644.diff
No OneTemporary
Actions

D818.id2644.diff
View Options

diff --git a/docker-compose.yml b/docker-compose.yml
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,6 +2,46 @@
services:
+ amqp:
+ image: rabbitmq:3.6-management
+ ports:
+ - 5018:15672
+
+# Scheduler
+
+ swh-scheduler-db:
+ image: postgres:10
+ environment:
+ POSTGRES_PASSWORD: testpassword
+ POSTGRES_DB: swh-scheduler
+
+ swh-scheduler-api:
+ image: swh/scheduler-api
+ build: ./dockerfiles/swh-scheduler-api
+ env_file: ./scheduler.env
+ depends_on:
+ - swh-scheduler-db
+ ports:
+ - 5008:5008
+
+ swh-scheduler-listener:
+ image: swh/scheduler-worker
+ build: ./dockerfiles/swh-scheduler-worker
+ env_file: ./scheduler.env
+ command: listener
+ depends_on:
+ - swh-scheduler-api
+ - amqp
+
+ swh-scheduler-runner:
+ image: swh/scheduler-worker
+ env_file: ./scheduler.env
+ command: runner
+ depends_on:
+ - swh-scheduler-listener # for the image only
+ - swh-scheduler-api
+ - amqp
+
# Graph storage
swh-storage-db:
@@ -14,7 +54,7 @@
build: ./dockerfiles/swh-storage
image: swh/storage
ports:
- - 5002:5002
+ - 5002:5002
depends_on:
- swh-storage-db
- swh-objstorage
@@ -30,7 +70,7 @@
build: ./dockerfiles/swh-objstorage
image: swh/objstorage
ports:
- - 5003:5003
+ - 5003:5003
# Indexer storage
@@ -59,7 +99,7 @@
build: ./dockerfiles/swh-web
image: swh/web
ports:
- - 5004:5004
+ - 5004:5004
depends_on:
- swh-objstorage
- swh-storage
diff --git a/dockerfiles/swh-scheduler-api/Dockerfile b/dockerfiles/swh-scheduler-api/Dockerfile
new file mode 100644
--- /dev/null
+++ b/dockerfiles/swh-scheduler-api/Dockerfile
@@ -0,0 +1,14 @@
+FROM python:3
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update && \
+ apt-get install -y \
+ libsystemd-dev postgresql-client
+
+RUN pip install swh-scheduler
+COPY scheduler.yml /
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+EXPOSE 5008
diff --git a/dockerfiles/swh-scheduler-api/entrypoint.sh b/dockerfiles/swh-scheduler-api/entrypoint.sh
new file mode 100755
--- /dev/null
+++ b/dockerfiles/swh-scheduler-api/entrypoint.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+
+if [[ -d /src ]] ; then
+ for srcrepo in /src/swh-* ; do
+ pushd $srcrepo
+ pip install -e .
+ popd
+ done
+fi
+
+echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass
+cat > ~/.pg_service.conf <<EOF
+[swh]
+dbname=${POSTGRES_DB}
+host=${PGHOST}
+port=5432
+user=${PGUSER}
+EOF
+
+chmod 0400 ~/.pgpass
+
+case "$1" in
+ "shell")
+ exec bash -i
+ ;;
+ *)
+ echo Waiting for postgresql to start
+ until psql service=swh -c "select 1" 2>&1 > /dev/null; do sleep 0.1; done
+
+ echo Setup the swh-scheduler API database
+ PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init scheduler \
+ --db-name ${POSTGRES_DB}
+
+ echo Starting the swh-scheduler API server
+ exec python -m swh.scheduler.api.server /scheduler.yml
+esac
diff --git a/dockerfiles/swh-scheduler-api/scheduler.yml b/dockerfiles/swh-scheduler-api/scheduler.yml
new file mode 100644
--- /dev/null
+++ b/dockerfiles/swh-scheduler-api/scheduler.yml
@@ -0,0 +1,4 @@
+scheduler:
+ cls: local
+ args:
+ db: service=swh
diff --git a/dockerfiles/swh-scheduler-worker/Dockerfile b/dockerfiles/swh-scheduler-worker/Dockerfile
new file mode 100644
--- /dev/null
+++ b/dockerfiles/swh-scheduler-worker/Dockerfile
@@ -0,0 +1,15 @@
+FROM python:3
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update && \
+ apt-get install -y \
+ libsystemd-dev postgresql-client
+
+RUN pip install swh-scheduler
+RUN mkdir /etc/softwareheritage/
+COPY scheduler.yml /etc/softwareheritage/
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+EXPOSE 5008
diff --git a/dockerfiles/swh-scheduler-worker/entrypoint.sh b/dockerfiles/swh-scheduler-worker/entrypoint.sh
new file mode 100755
--- /dev/null
+++ b/dockerfiles/swh-scheduler-worker/entrypoint.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+if [[ -d /src ]] ; then
+ for srcrepo in /src/swh-* ; do
+ pushd $srcrepo
+ pip install -e .
+ popd
+ done
+fi
+
+echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass
+cat > ~/.pg_service.conf <<EOF
+[swh]
+dbname=${POSTGRES_DB}
+host=${PGHOST}
+port=5432
+user=${PGUSER}
+EOF
+
+chmod 0400 ~/.pgpass
+
+case "$1" in
+ "shell")
+ exec bash -i
+ ;;
+ "listener")
+ echo Starting the swh-scheduler listener
+ exec python -m swh.scheduler.celery_backend.listener
+ ;;
+ "runner")
+ echo Starting the swh-scheduler runner
+ exec sh -c 'while true; do
+ echo running pending tasks at `/bin/date`;
+ python -m swh.scheduler.celery_backend.runner;
+ sleep 10;
+ done' # beuark
+ ;;
+ *)
+ echo "Provide a command (shell|listener|runner)"
+ exit 1
+ ;;
+esac
diff --git a/dockerfiles/swh-scheduler-worker/scheduler.yml b/dockerfiles/swh-scheduler-worker/scheduler.yml
new file mode 100644
--- /dev/null
+++ b/dockerfiles/swh-scheduler-worker/scheduler.yml
@@ -0,0 +1,2 @@
+task_broker: amqp://guest@amqp//
+scheduling_db: service=swh
diff --git a/scheduler.env b/scheduler.env
new file mode 100644
--- /dev/null
+++ b/scheduler.env
@@ -0,0 +1,4 @@
+POSTGRES_PASSWORD=testpassword
+POSTGRES_DB=swh-scheduler
+PGHOST=swh-scheduler-db
+PGUSER=postgres

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 3:28 PM (1 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3218668

Event Timeline

About · Developer information · Legal