diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -171,3 +171,18 @@ - swh-scheduler-api - swh-storage - amqp + +# Indexer Celery workers + + swh-indexer-mimetype: + image: swh/indexer-worker + build: ./dockerfiles/swh-indexer-worker + env_file: ./indexers.env + environment: + SWH_WORKER_INSTANCE: mimetype + depends_on: + - swh-scheduler-api + - swh-idx-storage + - swh-storage + - swh-objstorage + - amqp diff --git a/dockerfiles/swh-indexer-worker/Dockerfile b/dockerfiles/swh-indexer-worker/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-indexer-worker/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev + +RUN pip install swh-indexer swh-scheduler +COPY indexer/ / +COPY entrypoint.sh / + +# Celery does not want to run as root with 'pickle' enabled. +RUN useradd -m swh-worker +USER swh-worker:swh-worker + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-worker/entrypoint.sh b/dockerfiles/swh-indexer-worker/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-indexer-worker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +mkdir -p ~/.config/swh/worker + +cat > ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini <