diff --git a/docker/Dockerfile b/docker/Dockerfile index c7e373c..a80b798 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,48 +1,51 @@ FROM python:3.7 +RUN . /etc/os-release && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ libapr1-dev \ libaprutil1-dev \ libpq-dev \ libsvn-dev \ libsystemd-dev \ memcached \ - postgresql-client \ + postgresql-client-12 \ wait-for-it \ ngrep \ rsync && \ apt-get install -y --no-install-recommends \ r-base-core \ r-cran-jsonlite && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN useradd -md /srv/softwareheritage -s /bin/bash swh USER swh RUN python3 -m venv /srv/softwareheritage/venv ENV PATH="/srv/softwareheritage/venv/bin:${PATH}" RUN pip install --upgrade pip setuptools wheel RUN pip install gunicorn httpie RUN pip install \ swh-core[db,http] \ swh-deposit[server] \ swh-indexer \ swh-journal \ swh-lister \ swh-loader-core \ swh-loader-git \ swh-loader-mercurial \ swh-loader-svn \ swh-storage \ swh-objstorage \ swh-scheduler \ swh-vault \ swh-web COPY utils/*.sh /srv/softwareheritage/utils/ RUN mkdir -p /srv/softwareheritage/objects RUN rm -rd /srv/softwareheritage/.cache diff --git a/docker/docker-compose.storage-mirror.yml b/docker/docker-compose.storage-mirror.yml index f0494cd..096b462 100644 --- a/docker/docker-compose.storage-mirror.yml +++ b/docker/docker-compose.storage-mirror.yml @@ -1,69 +1,69 @@ version: '2' services: # override web app to use the mirror swh-web: environment: SWH_CONFIG_FILENAME: /web-mirror.yml volumes: - "./conf/web-mirror.yml:/web-mirror.yml:ro" # create a dedicated db for the mirror swh-storage-mirror-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/storage-db-mirror.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: # and an RPC server swh-storage-mirror: image: swh/stack build: ./ depends_on: - swh-storage-mirror-db - swh-objstorage env_file: - ./env/storage-db-mirror.env environment: SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # and the background process that keeps the mirror in sync with the # main graph swh-storage-mirror-replayer: image: swh/stack build: ./ depends_on: - swh-storage-mirror-db - swh-objstorage env_file: - ./env/storage-db-mirror.env environment: SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage-replayer/entrypoint.sh:/entrypoint.sh:ro" swh-journal-backfiller: image: swh/stack build: ./ entrypoint: /entrypoint.sh environment: SWH_CONFIG_FILENAME: /journal_backfiller.yml env_file: - ./env/storage-db.env depends_on: - swh-storage-db - kafka volumes: - "./conf/journal_backfiller.yml:/journal_backfiller.yml:ro" - "./services/swh-journal-backfiller/entrypoint.sh:/entrypoint.sh:ro" diff --git a/docker/docker-compose.storage-read-replica.yml b/docker/docker-compose.storage-read-replica.yml index 3c4985d..d75753e 100644 --- a/docker/docker-compose.storage-read-replica.yml +++ b/docker/docker-compose.storage-read-replica.yml @@ -1,66 +1,66 @@ version: '2' services: swh-storage-read-replica-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/storage-read-replica-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling it ourselves POSTGRES_DB: swh-storage-read-replica: image: swh/stack build: ./ ports: - 5042:5002 depends_on: - swh-storage - swh-storage-db - swh-storage-read-replica-db - swh-objstorage env_file: - ./env/common_python.env - ./env/storage-read-replica-db.env environment: SWH_CONFIG_FILENAME: /storage.yml STORAGE_BACKEND: postgresql entrypoint: /entrypoint.sh volumes: - "./conf/storage-read-replica.yml:/storage.yml:ro" - "./services/swh-storage-read-replica/entrypoint.sh:/entrypoint.sh:ro" # override storage db to enable wal_level=logical swh-storage-db: command: postgres -c wal_level=logical # override web app to use the replica swh-web: depends_on: - swh-storage-read-replica - swh-idx-storage environment: SWH_CONFIG_FILENAME: /web-read-replica.yml volumes: - "./conf/web-read-replica.yml:/web-read-replica.yml:ro" # override vault to use the replica swh-vault: depends_on: - swh-storage-read-replica environment: SWH_CONFIG_FILENAME: /vault-read-replica.yml volumes: - "./conf/vault-read-replica.yml:/vault-read-replica.yml:ro" swh-vault-worker: depends_on: - swh-storage-read-replica environment: SWH_CONFIG_FILENAME: /cooker-read-replica.yml volumes: - "./conf/vault-worker-read-replica.yml:/cooker-read-replica.yml:ro" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d73bc2f..6d08c2b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,431 +1,431 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5072:5672 # flower: # image: mher/flower # command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower # ports: # - 5055:5555 # depends_on: # - amqp zookeeper: image: wurstmeister/zookeeper restart: always kafka: image: wurstmeister/kafka ports: - "5092:9092" env_file: ./env/kafka.env depends_on: - zookeeper kafka-manager: image: hlebalbau/kafka-manager:stable ports: - "5093:9000" environment: ZK_HOSTS: zookeeper:2181 APPLICATION_SECRET: random-secret depends_on: - zookeeper command: -Dpidfile.path=/dev/null volumes: - "./conf/kafka-manager_logback.xml:/kafka-manager/conf/logback.xml:ro" prometheus: image: prom/prometheus depends_on: - prometheus-statsd-exporter command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped prometheus-rabbitmq-exporter: image: kbudde/rabbitmq-exporter restart: unless-stopped environment: SKIP_QUEUES: "RPC_.*" MAX_QUEUES: 5000 RABBIT_URL: http://amqp:15672 LOG_LEVEL: warning grafana: image: grafana/grafana restart: unless-stopped depends_on: - prometheus environment: GF_SERVER_ROOT_URL: http://localhost:5080/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards" nginx: image: nginx volumes: - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 # Scheduler swh-scheduler-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/common_python.env - ./env/scheduler-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: swh-scheduler: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh depends_on: - swh-scheduler-db ports: - 5008:5008 volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-listener: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh command: start-listener depends_on: - swh-scheduler - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-runner: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh command: start-runner -p 10 depends_on: - swh-scheduler - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro" # Graph storage swh-storage-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/storage-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: swh-storage: image: swh/stack build: ./ ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage - kafka env_file: - ./env/common_python.env - ./env/storage-db.env environment: SWH_CONFIG_FILENAME: /storage.yml STORAGE_BACKEND: postgresql entrypoint: /entrypoint.sh volumes: - "./conf/storage.yml:/storage.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # Object storage swh-objstorage: build: ./ image: swh/stack ports: - 5003:5003 env_file: - ./env/common_python.env environment: SWH_CONFIG_FILENAME: /objstorage.yml entrypoint: /entrypoint.sh volumes: - "./conf/objstorage.yml:/objstorage.yml:ro" - "./services/swh-objstorage/entrypoint.sh:/entrypoint.sh:ro" # Indexer storage swh-idx-storage-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/indexers-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: swh-idx-storage: image: swh/stack build: ./ ports: - 5007:5007 depends_on: - swh-idx-storage-db env_file: - ./env/common_python.env - ./env/indexers-db.env environment: SWH_CONFIG_FILENAME: /indexer_storage.yml entrypoint: /entrypoint.sh volumes: - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" - "./services/swh-indexer-storage/entrypoint.sh:/entrypoint.sh:ro" # Web interface swh-web: build: ./ image: swh/stack ports: - 5004:5004 depends_on: - swh-storage - swh-idx-storage env_file: - ./env/common_python.env environment: VERBOSITY: 3 DJANGO_SETTINGS_MODULE: swh.web.settings.production SWH_CONFIG_FILENAME: /web.yml entrypoint: /entrypoint.sh volumes: - "./conf/web.yml:/web.yml:ro" - "./services/swh-web/entrypoint.sh:/entrypoint.sh:ro" swh-deposit-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/deposit-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-deposit: image: swh/stack build: ./ ports: - 5006:5006 depends_on: - swh-deposit-db - swh-scheduler env_file: - ./env/common_python.env - ./env/deposit-db.env environment: VERBOSITY: 3 SWH_CONFIG_FILENAME: /deposit.yml DJANGO_SETTINGS_MODULE: swh.deposit.settings.production entrypoint: /entrypoint.sh volumes: - "./conf/deposit.yml:/deposit.yml:ro" - "./services/swh-deposit/entrypoint.sh:/entrypoint.sh:ro" swh-vault-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/vault-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: swh-vault: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/vault-db.env environment: SWH_CONFIG_FILENAME: /vault.yml command: server ports: - 5005:5005 depends_on: - swh-vault-db - swh-objstorage - swh-storage - swh-scheduler entrypoint: /entrypoint.sh volumes: - "./conf/vault.yml:/vault.yml:ro" - "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro" swh-vault-worker: image: swh/stack build: ./ command: worker env_file: - ./env/common_python.env environment: SWH_CONFIG_FILENAME: /cooker.yml depends_on: - swh-vault - swh-storage entrypoint: /entrypoint.sh volumes: - "./conf/vault-worker.yml:/cooker.yml:ro" - "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro" # Lister Celery workers swh-listers-db: - image: postgres:11 + image: postgres:12 env_file: - ./env/listers-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: swh-lister: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/listers-db.env - ./env/workers.env user: swh environment: SWH_WORKER_INSTANCE: listers SWH_CONFIG_FILENAME: /lister.yml depends_on: - swh-listers-db - swh-scheduler - swh-scheduler-runner - swh-storage - amqp entrypoint: /entrypoint.sh volumes: - "./conf/lister.yml:/lister.yml:ro" - "./services/swh-listers-worker/entrypoint.sh:/entrypoint.sh:ro" # Loader + deposit checker Celery workers swh-loader: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/workers.env user: swh environment: SWH_WORKER_INSTANCE: loader SWH_CONFIG_FILENAME: /loader.yml entrypoint: /entrypoint.sh depends_on: - swh-storage - swh-scheduler - swh-deposit - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" # Indexer Celery workers swh-indexer: image: swh/stack build: ./ user: swh env_file: - ./env/common_python.env - ./env/indexers-db.env - ./env/workers.env environment: SWH_WORKER_INSTANCE: indexer SWH_CONFIG_FILENAME: /indexer.yml CONCURRENCY: 4 entrypoint: /entrypoint.sh depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp volumes: - "./conf/indexer.yml:/indexer.yml:ro" - "./services/swh-indexer-worker/entrypoint.sh:/entrypoint.sh:ro" # Journal related swh-indexer-journal-client: image: swh/stack build: ./ entrypoint: /entrypoint.sh env_file: - ./env/common_python.env depends_on: - kafka - swh-storage - swh-scheduler volumes: - "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" - "./services/swh-indexer-journal-client/entrypoint.sh:/entrypoint.sh:ro"