diff --git a/docker/docker-compose.cassandra.yml b/docker/docker-compose.cassandra.yml index ae843a4..659cf8c 100644 --- a/docker/docker-compose.cassandra.yml +++ b/docker/docker-compose.cassandra.yml @@ -1,61 +1,61 @@ -version: '2' +version: "2.1" services: cassandra-seed: # This container starts a Cassandra instance that must be used as the # contact-point for clients. This container will then make the client # discover other cassandra containers. # This container must not be scaled up; scale up th 'cassandra' # container instead. image: cassandra env_file: - ./env/cassandra.env entrypoint: /swh_entrypoint.sh volumes: - "./services/cassandra/swh_entrypoint.sh:/swh_entrypoint.sh:ro" - "./conf/cassandra.yaml:/cassandra.yaml:ro" cassandra: # Additional Cassandra instance(s), which may be scaled up, but not # down. They will automatically connect to 'cassandra-seed', and # 'cassandra-seed' will tell clients to connect to these 'cassandra' # containers to load-balance. image: cassandra entrypoint: /swh_entrypoint.sh volumes: - "./services/cassandra/swh_entrypoint.sh:/swh_entrypoint.sh:ro" - "./conf/cassandra.yaml:/cassandra.yaml:ro" env_file: - ./env/cassandra.env prometheus: # just to add the dep on the cassandra-jmx-exporter-cassandra depends_on: - prometheus-statsd-exporter - prometheus-jmx-exporter-cassandra prometheus-jmx-exporter-cassandra: image: sscaling/jmx-prometheus-exporter environment: JVM_OPTS: "-Djava.util.logging.config.file=/logging.properties" volumes: - "./conf/prometheus-jmx-exporter-cassandra.yml:/opt/jmx_exporter/config.yml:ro" - "./conf/prometheus-jmx-exporter-logging.properties:/logging.properties:ro" ports: - "5556:5556" swh-storage: volumes: # note: you need to be on the cassandra-backend2 branch - "../swh-storage:/src/swh-storage" - "./conf/storage_cassandra.yml:/storage.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" depends_on: - swh-storage-db - cassandra-seed - swh-objstorage - kafka environment: CASSANDRA_SEED: cassandra-seed STORAGE_BACKEND: cassandra PYTHONUNBUFFERED: 1 diff --git a/docker/docker-compose.keycloak.yml b/docker/docker-compose.keycloak.yml index 785dedb..c43a62e 100644 --- a/docker/docker-compose.keycloak.yml +++ b/docker/docker-compose.keycloak.yml @@ -1,35 +1,35 @@ -version: '2' +version: "2.1" services: keycloak-db: image: postgres:11 env_file: - ./env/keycloak-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: keycloak: build: services/keycloak env_file: - ./env/keycloak.env entrypoint: /entrypoint.sh environment: JAVA_TOOL_OPTIONS: "-Dkeycloak.profile=preview -Dkeycloak.profile.feature.token_exchange=enabled" volumes: - "./services/keycloak/entrypoint.sh:/entrypoint.sh:ro" - "./services/keycloak/keycloak_swh_setup.py:/keycloak_swh_setup.py:ro" expose: - "8080" depends_on: - keycloak-db swh-web: depends_on: - keycloak environment: SWH_CONFIG_FILENAME: /web-keycloak.yml volumes: - - "./conf/web-keycloak.yml:/web-keycloak.yml:ro" \ No newline at end of file + - "./conf/web-keycloak.yml:/web-keycloak.yml:ro" diff --git a/docker/docker-compose.override.yml.example b/docker/docker-compose.override.yml.example index 258829a..524c7d5 100644 --- a/docker/docker-compose.override.yml.example +++ b/docker/docker-compose.override.yml.example @@ -1,6 +1,6 @@ -version: '2' +version: "2.1" services: swh-objstorage: volumes: - "../swh-objstorage:/src/swh-objstorage" diff --git a/docker/docker-compose.search.yml b/docker/docker-compose.search.yml index feaaac5..e596d09 100644 --- a/docker/docker-compose.search.yml +++ b/docker/docker-compose.search.yml @@ -1,62 +1,62 @@ -version: "2" +version: "2.1" services: elasticsearch: image: elasticsearch:7.9.3 env_file: - ./env/elasticsearch.env ports: - 9200:9200 volumes: - elasticsearch-data:/usr/share/elasticsearch/data swh-search: image: swh/stack build: ./ entrypoint: /entrypoint.sh ports: - 5010:5010 depends_on: - elasticsearch environment: SWH_CONFIG_FILENAME: /search.yml volumes: - "./conf/search.yml:/search.yml:ro" - "./services/swh-search/entrypoint.sh:/entrypoint.sh:ro" swh-search-journal-client-objects: image: swh/stack build: ./ entrypoint: /entrypoint.sh depends_on: kafka: condition: service_healthy swh-search: condition: service_started volumes: - "./conf/search_journal_client_objects.yml:/etc/softwareheritage/search/journal_client.yml:ro" - "./services/swh-search-journal-client/entrypoint.sh:/entrypoint.sh:ro" swh-search-journal-client-indexed: image: swh/stack build: ./ entrypoint: /entrypoint.sh depends_on: kafka: condition: service_healthy swh-search: condition: service_started volumes: - "./conf/search_journal_client_indexed.yml:/etc/softwareheritage/search/journal_client.yml:ro" - "./services/swh-search-journal-client/entrypoint.sh:/entrypoint.sh:ro" swh-web: depends_on: - swh-search environment: SWH_CONFIG_FILENAME: /web-search.yml volumes: - "./conf/web-search.yml:/web-search.yml:ro" volumes: elasticsearch-data: diff --git a/docker/docker-compose.storage-mirror.yml b/docker/docker-compose.storage-mirror.yml index 096b462..885cd9e 100644 --- a/docker/docker-compose.storage-mirror.yml +++ b/docker/docker-compose.storage-mirror.yml @@ -1,69 +1,69 @@ -version: '2' +version: "2.1" services: # override web app to use the mirror swh-web: environment: SWH_CONFIG_FILENAME: /web-mirror.yml volumes: - "./conf/web-mirror.yml:/web-mirror.yml:ro" # create a dedicated db for the mirror swh-storage-mirror-db: image: postgres:12 env_file: - ./env/storage-db-mirror.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling db creation ourselves in the backend # service entrypoint POSTGRES_DB: # and an RPC server swh-storage-mirror: image: swh/stack build: ./ depends_on: - swh-storage-mirror-db - swh-objstorage env_file: - ./env/storage-db-mirror.env environment: SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # and the background process that keeps the mirror in sync with the # main graph swh-storage-mirror-replayer: image: swh/stack build: ./ depends_on: - swh-storage-mirror-db - swh-objstorage env_file: - ./env/storage-db-mirror.env environment: SWH_CONFIG_FILENAME: /storage-mirror.yml entrypoint: /entrypoint.sh volumes: - "./conf/storage-mirror.yml:/storage-mirror.yml:ro" - "./services/swh-storage-replayer/entrypoint.sh:/entrypoint.sh:ro" swh-journal-backfiller: image: swh/stack build: ./ entrypoint: /entrypoint.sh environment: SWH_CONFIG_FILENAME: /journal_backfiller.yml env_file: - ./env/storage-db.env depends_on: - swh-storage-db - kafka volumes: - "./conf/journal_backfiller.yml:/journal_backfiller.yml:ro" - "./services/swh-journal-backfiller/entrypoint.sh:/entrypoint.sh:ro" diff --git a/docker/docker-compose.storage-read-replica.yml b/docker/docker-compose.storage-read-replica.yml index d75753e..04867dc 100644 --- a/docker/docker-compose.storage-read-replica.yml +++ b/docker/docker-compose.storage-read-replica.yml @@ -1,66 +1,66 @@ -version: '2' +version: "2.1" services: swh-storage-read-replica-db: image: postgres:12 env_file: - ./env/storage-read-replica-db.env environment: # unset PGHOST as db service crashes otherwise PGHOST: # unset POSTGRES_DB: we're handling it ourselves POSTGRES_DB: swh-storage-read-replica: image: swh/stack build: ./ ports: - 5042:5002 depends_on: - swh-storage - swh-storage-db - swh-storage-read-replica-db - swh-objstorage env_file: - ./env/common_python.env - ./env/storage-read-replica-db.env environment: SWH_CONFIG_FILENAME: /storage.yml STORAGE_BACKEND: postgresql entrypoint: /entrypoint.sh volumes: - "./conf/storage-read-replica.yml:/storage.yml:ro" - "./services/swh-storage-read-replica/entrypoint.sh:/entrypoint.sh:ro" # override storage db to enable wal_level=logical swh-storage-db: command: postgres -c wal_level=logical # override web app to use the replica swh-web: depends_on: - swh-storage-read-replica - swh-idx-storage environment: SWH_CONFIG_FILENAME: /web-read-replica.yml volumes: - "./conf/web-read-replica.yml:/web-read-replica.yml:ro" # override vault to use the replica swh-vault: depends_on: - swh-storage-read-replica environment: SWH_CONFIG_FILENAME: /vault-read-replica.yml volumes: - "./conf/vault-read-replica.yml:/vault-read-replica.yml:ro" swh-vault-worker: depends_on: - swh-storage-read-replica environment: SWH_CONFIG_FILENAME: /cooker-read-replica.yml volumes: - "./conf/vault-worker-read-replica.yml:/cooker-read-replica.yml:ro" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 58282f0..4a1bab7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,405 +1,405 @@ -version: "2" +version: "2.1" services: amqp: image: rabbitmq:3.6-management ports: - 5072:5672 zookeeper: image: wurstmeister/zookeeper restart: always kafka: image: wurstmeister/kafka ports: - "5092:9092" env_file: ./env/kafka.env environment: KAFKA_CREATE_TOPICS: swh.journal.objects.origin:1:1, swh.journal.objects.origin_visit:1:1, swh.journal.objects.origin_visit_status:1:1, swh.journal.indexed.origin_intrinsic_metadata:1:1 depends_on: - zookeeper healthcheck: test: "[ `JMX_PORT= kafka-topics.sh --list --zookeeper zookeeper:2181 | wc -l` -ge 4 ]" interval: 10s timeout: 5s retries: 10 prometheus: image: prom/prometheus depends_on: - prometheus-statsd-exporter command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" restart: unless-stopped prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" restart: unless-stopped prometheus-rabbitmq-exporter: image: kbudde/rabbitmq-exporter restart: unless-stopped environment: SKIP_QUEUES: "RPC_.*" MAX_QUEUES: 5000 RABBIT_URL: http://amqp:15672 LOG_LEVEL: warning grafana: image: grafana/grafana restart: unless-stopped depends_on: - prometheus environment: GF_SERVER_ROOT_URL: http://localhost:5080/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards" nginx: image: nginx volumes: - "./conf/nginx.conf:/etc/nginx/nginx.conf:ro" ports: - 5080:5080 # Scheduler swh-scheduler-db: image: postgres:12 env_file: - ./env/common_python.env - ./env/scheduler-db.env swh-scheduler: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh depends_on: - swh-scheduler-db ports: - 5008:5008 volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-listener: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh command: start-listener depends_on: - swh-scheduler - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-runner: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/scheduler-db.env - ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml entrypoint: /entrypoint.sh command: start-runner -p 10 depends_on: - swh-scheduler - amqp volumes: - "./conf/scheduler.yml:/scheduler.yml:ro" - "./services/swh-scheduler-worker/entrypoint.sh:/entrypoint.sh:ro" # Graph storage swh-storage-db: image: postgres:12 env_file: - ./env/storage-db.env swh-storage: image: swh/stack build: ./ ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage - kafka env_file: - ./env/common_python.env - ./env/storage.env environment: SWH_CONFIG_FILENAME: /storage.yml STORAGE_BACKEND: postgresql entrypoint: /entrypoint.sh volumes: - "./conf/storage.yml:/storage.yml:ro" - "./services/swh-storage/entrypoint.sh:/entrypoint.sh:ro" # Object storage swh-objstorage: build: ./ image: swh/stack ports: - 5003:5003 env_file: - ./env/common_python.env environment: SWH_CONFIG_FILENAME: /objstorage.yml entrypoint: /entrypoint.sh volumes: - "./conf/objstorage.yml:/objstorage.yml:ro" - "./services/swh-objstorage/entrypoint.sh:/entrypoint.sh:ro" # Indexer storage swh-idx-storage-db: image: postgres:12 env_file: - ./env/indexers-db.env swh-idx-storage: image: swh/stack build: ./ ports: - 5007:5007 depends_on: - swh-idx-storage-db env_file: - ./env/common_python.env - ./env/indexers-db.env - ./env/indexers.env environment: SWH_CONFIG_FILENAME: /indexer_storage.yml entrypoint: /entrypoint.sh volumes: - "./conf/indexer_storage.yml:/indexer_storage.yml:ro" - "./services/swh-indexer-storage/entrypoint.sh:/entrypoint.sh:ro" # Web interface swh-web: build: ./ image: swh/stack ports: - 5004:5004 depends_on: - swh-storage - swh-idx-storage env_file: - ./env/common_python.env environment: VERBOSITY: 3 DJANGO_SETTINGS_MODULE: swh.web.settings.production SWH_CONFIG_FILENAME: /web.yml entrypoint: /entrypoint.sh volumes: - "./conf/web.yml:/web.yml:ro" - "./services/swh-web/entrypoint.sh:/entrypoint.sh:ro" swh-deposit-db: image: postgres:12 env_file: - ./env/deposit-db.env swh-deposit: image: swh/stack build: ./ ports: - 5006:5006 depends_on: - swh-deposit-db - swh-scheduler env_file: - ./env/common_python.env - ./env/deposit-db.env - ./env/deposit.env environment: VERBOSITY: 3 SWH_CONFIG_FILENAME: /deposit.yml DJANGO_SETTINGS_MODULE: swh.deposit.settings.production entrypoint: /entrypoint.sh volumes: - "./conf/deposit.yml:/deposit.yml:ro" - "./services/swh-deposit/entrypoint.sh:/entrypoint.sh:ro" swh-vault-db: image: postgres:12 env_file: - ./env/vault-db.env swh-vault: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/vault-db.env - ./env/vault.env environment: SWH_CONFIG_FILENAME: /vault.yml command: server ports: - 5005:5005 depends_on: - swh-vault-db - swh-objstorage - swh-storage - swh-scheduler entrypoint: /entrypoint.sh volumes: - "./conf/vault.yml:/vault.yml:ro" - "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro" swh-vault-worker: image: swh/stack build: ./ command: worker env_file: - ./env/common_python.env - ./env/workers.env environment: SWH_CONFIG_FILENAME: /cooker.yml depends_on: - swh-vault - swh-storage entrypoint: /entrypoint.sh volumes: - "./conf/vault-worker.yml:/cooker.yml:ro" - "./services/swh-vault/entrypoint.sh:/entrypoint.sh:ro" # Lister Celery workers swh-lister: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/listers.env - ./env/workers.env user: swh environment: SWH_WORKER_INSTANCE: listers SWH_CONFIG_FILENAME: /lister.yml depends_on: - swh-scheduler - swh-scheduler-runner - amqp entrypoint: /entrypoint.sh volumes: - "./conf/lister.yml:/lister.yml:ro" - "./services/swh-listers-worker/entrypoint.sh:/entrypoint.sh:ro" # Loader + deposit checker Celery workers swh-loader: image: swh/stack build: ./ env_file: - ./env/common_python.env - ./env/workers.env user: swh environment: SWH_WORKER_INSTANCE: loader SWH_CONFIG_FILENAME: /loader.yml entrypoint: /entrypoint.sh depends_on: - swh-storage - swh-scheduler - swh-deposit - amqp volumes: - "./conf/loader.yml:/loader.yml:ro" - "./services/swh-worker/entrypoint.sh:/entrypoint.sh:ro" # Indexer Celery workers swh-indexer: image: swh/stack build: ./ user: swh env_file: - ./env/common_python.env - ./env/indexers-db.env - ./env/indexers.env - ./env/workers.env environment: SWH_WORKER_INSTANCE: indexer SWH_CONFIG_FILENAME: /indexer.yml CONCURRENCY: 4 entrypoint: /entrypoint.sh depends_on: - swh-scheduler-runner - swh-idx-storage - swh-storage - swh-objstorage - amqp volumes: - "./conf/indexer.yml:/indexer.yml:ro" - "./services/swh-indexer-worker/entrypoint.sh:/entrypoint.sh:ro" # Journal related swh-indexer-journal-client: image: swh/stack build: ./ entrypoint: /entrypoint.sh env_file: - ./env/common_python.env depends_on: kafka: condition: service_healthy swh-storage: condition: service_started swh-scheduler: condition: service_started volumes: - "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" - "./services/swh-indexer-journal-client/entrypoint.sh:/entrypoint.sh:ro" swh-scheduler-journal-client: image: swh/stack build: ./ entrypoint: /entrypoint.sh env_file: - ./env/common_python.env depends_on: kafka: condition: service_healthy swh-scheduler: condition: service_started volumes: - "./conf/scheduler_journal_client.yml:/etc/softwareheritage/scheduler/journal_client.yml:ro" - "./services/swh-scheduler-journal-client/entrypoint.sh:/entrypoint.sh:ro"