diff --git a/dockerfiles/swh-deposit/private.yml b/conf/deposit_private.yml similarity index 100% rename from dockerfiles/swh-deposit/private.yml rename to conf/deposit_private.yml diff --git a/dockerfiles/swh-deposit/server.yml b/conf/deposit_server.yml similarity index 100% rename from dockerfiles/swh-deposit/server.yml rename to conf/deposit_server.yml diff --git a/dockerfiles/swh-indexer-worker/indexer.yml b/conf/indexer.yml similarity index 89% rename from dockerfiles/swh-indexer-worker/indexer.yml rename to conf/indexer.yml index cd1aca8..dbebd35 100644 --- a/dockerfiles/swh-indexer-worker/indexer.yml +++ b/conf/indexer.yml @@ -1,18 +1,20 @@ storage: cls: remote args: url: http://swh-storage:5002/ objstorage: cls: remote args: url: http://swh-objstorage:5003/ indexer_storage: cls: remote args: url: http://swh-idx-storage:5007/ scheduler: cls: remote args: url: http://swh-scheduler-api:5008/ celery: task_broker: amqp://guest:guest@amqp// + task_modules: + - swh.indexer.tasks diff --git a/dockerfiles/swh-indexer-storage/indexer_storage.yml b/conf/indexer_storage.yml similarity index 100% rename from dockerfiles/swh-indexer-storage/indexer_storage.yml rename to conf/indexer_storage.yml diff --git a/dockerfiles/swh-journal-client/logger.yml b/conf/journal_client.yml similarity index 100% rename from dockerfiles/swh-journal-client/logger.yml rename to conf/journal_client.yml diff --git a/dockerfiles/swh-journal-publisher/publisher.yml b/conf/journal_publisher.yml similarity index 100% rename from dockerfiles/swh-journal-publisher/publisher.yml rename to conf/journal_publisher.yml diff --git a/dockerfiles/swh-listers-worker/lister.yml b/conf/lister.yml similarity index 100% rename from dockerfiles/swh-listers-worker/lister.yml rename to conf/lister.yml diff --git a/dockerfiles/swh-loaders-worker/loader.yml b/conf/loader.yml similarity index 100% rename from dockerfiles/swh-loaders-worker/loader.yml rename to conf/loader.yml diff --git a/dockerfiles/swh-objstorage/objstorage.yml b/conf/objstorage.yml similarity index 100% rename from dockerfiles/swh-objstorage/objstorage.yml rename to conf/objstorage.yml diff --git a/dockerfiles/swh-scheduler-worker/scheduler.yml b/conf/scheduler.yml similarity index 100% rename from dockerfiles/swh-scheduler-worker/scheduler.yml rename to conf/scheduler.yml diff --git a/dockerfiles/swh-storage/storage.yml b/conf/storage.yml similarity index 100% rename from dockerfiles/swh-storage/storage.yml rename to conf/storage.yml diff --git a/dockerfiles/swh-storage-listener/listener.yml b/conf/storage_listener.yml similarity index 100% rename from dockerfiles/swh-storage-listener/listener.yml rename to conf/storage_listener.yml diff --git a/dockerfiles/swh-web/web.yml b/conf/web.yml similarity index 100% rename from dockerfiles/swh-web/web.yml rename to conf/web.yml diff --git a/docker-compose.yml b/docker-compose.yml index bb5efc5..a20a663 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,398 +1,451 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5072:5672 flower: image: mher/flower command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower ports: - 5055:5555 depends_on: - amqp zookeeper: image: wurstmeister/zookeeper kafka: image: wurstmeister/kafka ports: - 5092:9092 env_file: ./env/kafka.env depends_on: - zookeeper prometheus: image: prom/prometheus command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped nginx: image: nginx volumes: - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 # Scheduler swh-scheduler-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-scheduler swh-scheduler-api: image: swh/scheduler-api build: ./dockerfiles/swh-scheduler-api env_file: ./env/scheduler.env depends_on: - swh-scheduler-db ports: - 5008:5008 + volumes: + - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-listener: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./env/scheduler.env command: listener depends_on: - swh-scheduler-api - amqp + volumes: + - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-runner: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./env/scheduler.env command: runner -p 10 depends_on: - swh-scheduler-api - amqp + volumes: + - "./conf/scheduler.yml:/scheduler.yml:ro" # Graph storage swh-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage swh-storage: build: ./dockerfiles/swh-storage image: swh/storage ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage env_file: ./env/storage.env + volumes: + - "./conf/storage.yml:/storage.yml:ro" # Object storage swh-objstorage: build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - 5003:5003 + volumes: + - "./conf/objstorage.yml:/objstorage.yml:ro" # Indexer storage swh-idx-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage swh-idx-storage: build: ./dockerfiles/swh-indexer-storage image: swh/indexer-storage ports: - 5007:5007 depends_on: - swh-idx-storage-db environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres + volumes: + - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" # Web interface swh-web: build: ./dockerfiles/swh-web image: swh/web ports: - 5004:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage + volumes: + - "./conf/web.yml:/etc/softwareheritage/web/web.yml:ro" swh-deposit-db: image: postgres:10 env_file: ./env/deposit.env swh-deposit: build: ./dockerfiles/swh-deposit image: swh/deposit ports: - 5006:5006 depends_on: - swh-deposit-db - swh-scheduler-api env_file: ./env/deposit.env environment: PGHOST: swh-deposit-db + volumes: + - "./conf/deposit_server.yml:/etc/softwareheritage/deposit/server.yml:ro" + - "./conf/deposit_private.yml:/etc/softwareheritage/deposit/private.yml:ro" # Lister Celery workers swh-listers-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword swh-lister-debian: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: debian depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/home/swh/.config/swh/lister.yml:ro" swh-lister-bitbucket: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: bitbucket depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/home/swh/.config/swh/lister.yml:ro" swh-lister-github: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: github depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/home/swh/.config/swh/lister.yml:ro" swh-lister-gitlab: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: gitlab depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/home/swh/.config/swh/lister.yml:ro" swh-lister-npm: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: npm depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/home/swh/.config/swh/lister.yml:ro" swh-lister-pypi: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: pypi depends_on: - swh-listers-db - swh-scheduler-api - swh-scheduler-runner - swh-storage - amqp + volumes: + - "./conf/lister.yml:/home/swh/.config/swh/lister.yml:ro" # Indexer Celery workers swh-indexer: image: swh/indexer-worker build: ./dockerfiles/swh-indexer-worker env_file: ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp + volumes: + - "./conf/indexer.yml:/home/swh/.config/swh/indexer.yml:ro" swh-indexer-journal-client: image: swh/indexer-journal-client build: ./dockerfiles/swh-indexer-journal-client depends_on: - swh-journal-publisher - swh-scheduler-api + volumes: + - "./conf/journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" # Journal related swh-storage-listener: image: swh/storage-listener build: ./dockerfiles/swh-storage-listener env_file: ./env/storage.env depends_on: - swh-storage-db - kafka + volumes: + - "./conf/storage_listener.yml:/etc/softwareheritage/storage/listener.yml:ro" swh-journal-publisher: image: swh/journal-publisher build: ./dockerfiles/swh-journal-publisher depends_on: - kafka - swh-storage-listener + volumes: + - "./conf/journal_publisher.yml:/etc/softwareheritage/journal/publisher.yml:ro" swh-journal-client: image: swh/journal-client build: ./dockerfiles/swh-journal-client depends_on: - swh-journal-publisher + volumes: + - "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro" # Loader Celery workers swh-loader-debian: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: debian depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" swh-loader-dir: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: dir depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" swh-loader-git: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: git depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" swh-loader-mercurial: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: mercurial depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" swh-loader-pypi: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: pypi depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" swh-loader-svn: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: svn depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" swh-loader-tar: image: swh/loaders-worker build: ./dockerfiles/swh-loaders-worker env_file: ./env/listers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 SWH_WORKER_INSTANCE: tar depends_on: - swh-storage - amqp + volumes: + - "./conf/loader.yml:/home/swh/.config/swh/loader.yml:ro" diff --git a/dockerfiles/swh-deposit/Dockerfile b/dockerfiles/swh-deposit/Dockerfile index d280ca4..04e2b93 100644 --- a/dockerfiles/swh-deposit/Dockerfile +++ b/dockerfiles/swh-deposit/Dockerfile @@ -1,16 +1,14 @@ FROM python:3.6 RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y libsystemd-dev postgresql-client && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN pip install swh-deposit gunicorn RUN pip install psycopg2-binary # to make pip silent -COPY server.yml /etc/softwareheritage/deposit/ -COPY private.yml /etc/softwareheritage/deposit/ COPY entrypoint.sh / EXPOSE 5006 ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-journal-client/Dockerfile b/dockerfiles/swh-indexer-journal-client/Dockerfile index d57fbd0..f3f2ab7 100644 --- a/dockerfiles/swh-indexer-journal-client/Dockerfile +++ b/dockerfiles/swh-indexer-journal-client/Dockerfile @@ -1,12 +1,10 @@ FROM python:3.6 RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y libsystemd-dev RUN pip install swh-indexer -COPY journal_client.yml /etc/softwareheritage/indexer/ COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] - diff --git a/dockerfiles/swh-indexer-storage/Dockerfile b/dockerfiles/swh-indexer-storage/Dockerfile index 2c83afe..718fe14 100644 --- a/dockerfiles/swh-indexer-storage/Dockerfile +++ b/dockerfiles/swh-indexer-storage/Dockerfile @@ -1,16 +1,15 @@ FROM python:3.6 RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ libsystemd-dev postgresql-client RUN pip install --upgrade pip setuptools wheel RUN pip install swh-indexer -COPY indexer_storage.yml / COPY entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] EXPOSE 5002 diff --git a/dockerfiles/swh-indexer-worker/Dockerfile b/dockerfiles/swh-indexer-worker/Dockerfile index eb3144a..285d524 100644 --- a/dockerfiles/swh-indexer-worker/Dockerfile +++ b/dockerfiles/swh-indexer-worker/Dockerfile @@ -1,17 +1,16 @@ FROM python:3.6 RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ libsystemd-dev postgresql-client RUN pip install --upgrade pip setuptools wheel RUN pip install swh-indexer RUN useradd -ms /bin/bash swh USER swh COPY entrypoint.sh / -COPY indexer.yml /home/swh/ ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-worker/entrypoint.sh b/dockerfiles/swh-indexer-worker/entrypoint.sh index a5940a3..d87f436 100755 --- a/dockerfiles/swh-indexer-worker/entrypoint.sh +++ b/dockerfiles/swh-indexer-worker/entrypoint.sh @@ -1,41 +1,30 @@ #!/bin/bash set -e if [[ -d /src ]] ; then for srcrepo in /src/swh-* ; do pushd $srcrepo echo "WARNING: $srcrepo will NOT be pip installed in dev mode" echo " due to permission limitations." pip install --user . popd done fi -mkdir -p ~/.config/swh/worker - -cat > ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini <