diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./scheduler.env - command: runner + command: runner -p 10 depends_on: - swh-scheduler-api - amqp @@ -280,3 +280,74 @@ depends_on: - swh-journal-publisher +# Loader Celery workers + + swh-loader-debian: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: debian + depends_on: + - swh-storage + - amqp + + swh-loader-dir: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: dir + depends_on: + - swh-storage + - amqp + + swh-loader-git: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: git + depends_on: + - swh-storage + - amqp + + swh-loader-mercurial: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: mercurial + depends_on: + - swh-storage + - amqp + + swh-loader-pypi: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: pypi + depends_on: + - swh-storage + - amqp + + swh-loader-svn: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: svn + depends_on: + - swh-storage + - amqp + + swh-loader-tar: + image: swh/loaders-worker + build: ./dockerfiles/swh-loaders-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: tar + depends_on: + - swh-storage + - amqp diff --git a/dockerfiles/swh-listers-worker/entrypoint.sh b/dockerfiles/swh-listers-worker/entrypoint.sh --- a/dockerfiles/swh-listers-worker/entrypoint.sh +++ b/dockerfiles/swh-listers-worker/entrypoint.sh @@ -35,7 +35,6 @@ [main] task_broker = amqp://guest@amqp// task_modules = swh.lister.${SWH_WORKER_INSTANCE}.tasks -task_queues = swh_lister_${SWH_WORKER_INSTANCE} task_soft_time_limit = 0 EOF diff --git a/dockerfiles/swh-listers-worker/lister.yml b/dockerfiles/swh-listers-worker/lister.yml --- a/dockerfiles/swh-listers-worker/lister.yml +++ b/dockerfiles/swh-listers-worker/lister.yml @@ -6,4 +6,4 @@ cls: remote args: url: http://swh-scheduler-api:5008/ -lister_db_url: service=swh +lister_db_url: postgresql:///?service=swh diff --git a/dockerfiles/swh-loaders-worker/Dockerfile b/dockerfiles/swh-loaders-worker/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-loaders-worker/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev libapr1-dev libaprutil1-dev libsvn-dev + +RUN pip install \ + swh-loader-debian \ + swh-loader-dir \ + swh-loader-git \ + swh-loader-mercurial \ + swh-loader-pypi \ + swh-loader-svn \ + swh-loader-tar + + +RUN useradd -ms /bin/bash swh +COPY entrypoint.sh / + +COPY loader.yml /home/swh/.config/swh/ +RUN chown -R swh: /home/swh/.config/ + +USER swh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-loaders-worker/entrypoint.sh b/dockerfiles/swh-loaders-worker/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-loaders-worker/entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + echo "WARNING: $srcrepo will NOT be pip installed in dev mode" + echo " due to permission limitations." + pip install --user . + popd + done +fi + +echo Installed Python packages: +pip list + +mkdir -p ~/.config/swh/worker + +cat > ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini <