diff --git a/docker-compose.yml b/docker-compose.yml index 713d128..3fc5a8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,106 +1,173 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5018:15672 # Scheduler swh-scheduler-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-scheduler swh-scheduler-api: image: swh/scheduler-api build: ./dockerfiles/swh-scheduler-api env_file: ./scheduler.env depends_on: - swh-scheduler-db ports: - 5008:5008 swh-scheduler-listener: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./scheduler.env command: listener depends_on: - swh-scheduler-api - amqp swh-scheduler-runner: image: swh/scheduler-worker env_file: ./scheduler.env command: runner depends_on: - swh-scheduler-listener # for the image only - swh-scheduler-api - amqp # Graph storage swh-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage swh-storage: build: ./dockerfiles/swh-storage image: swh/storage ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage PGHOST: swh-storage-db PGUSER: postgres # Object storage swh-objstorage: build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - 5003:5003 # Indexer storage swh-idx-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage swh-idx-storage: build: ./dockerfiles/swh-indexer-storage image: swh/indexer-storage ports: - 5007:5007 depends_on: - swh-idx-storage-db environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres # Web interface swh-web: build: ./dockerfiles/swh-web image: swh/web ports: - 5004:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage + +# Lister Celery workers + + swh-listers-db: + image: postgres:10 + environment: + POSTGRES_PASSWORD: testpassword + + swh-lister-debian: + image: swh/listers-worker + build: ./dockerfiles/swh-listers-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: debian + depends_on: + - swh-listers-db + - swh-scheduler-api + - swh-storage + - amqp + + swh-lister-github: + image: swh/listers-worker + build: ./dockerfiles/swh-listers-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: github + depends_on: + - swh-listers-db + - swh-scheduler-api + - swh-storage + - amqp + + swh-lister-gitlab: + image: swh/listers-worker + build: ./dockerfiles/swh-listers-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: gitlab + depends_on: + - swh-listers-db + - swh-scheduler-api + - swh-storage + - amqp + + swh-lister-npm: + image: swh/listers-worker + build: ./dockerfiles/swh-listers-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: npm + depends_on: + - swh-listers-db + - swh-scheduler-api + - swh-storage + - amqp + + swh-lister-pypi: + image: swh/listers-worker + build: ./dockerfiles/swh-listers-worker + env_file: ./listers.env + environment: + SWH_WORKER_INSTANCE: pypi + depends_on: + - swh-listers-db + - swh-scheduler-api + - swh-storage + - amqp diff --git a/dockerfiles/swh-listers-worker/Dockerfile b/dockerfiles/swh-listers-worker/Dockerfile new file mode 100644 index 0000000..2cfacbf --- /dev/null +++ b/dockerfiles/swh-listers-worker/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install swh-lister +RUN useradd -ms /bin/bash swh +COPY entrypoint.sh / + +COPY lister.yml /home/swh/.config/swh/ +RUN chown -R swh: /home/swh/.config/ + +USER swh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-listers-worker/entrypoint.sh b/dockerfiles/swh-listers-worker/entrypoint.sh new file mode 100755 index 0000000..a1671d6 --- /dev/null +++ b/dockerfiles/swh-listers-worker/entrypoint.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + echo "WARNING: $srcrepo wil NOT be pip installed in dev mode" + echo " due to permission limitations." + pip install --user . + popd + done +fi + +export POSTGRES_DB=swh-lister-${SWH_WORKER_INSTANCE} + +echo "${PGHOST}:5432:postgres:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass +echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" >> ~/.pgpass +cat > ~/.pg_service.conf < ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini <