diff --git a/base-services.yml b/base-services.yml index 93ba944..36c6e2e 100644 --- a/base-services.yml +++ b/base-services.yml @@ -1,185 +1,199 @@ version: "3.7" services: memcache: image: memcached deploy: replicas: 1 db-storage: image: postgres:13 deploy: mode: global placement: constraints: - node.labels.org.softwareheritage.mirror.volumes.storage-db == true command: ['-c', 'shared_buffers=4GB', '-c', 'effective_cache_size=4GB', '-c', 'random_page_cost=1.5', '-c', 'max_wal_size=4GB'] environment: POSTGRES_PASSWORD_FILE: /run/secrets/postgres-password POSTGRES_USER: swh POSTGRES_DB: # unset POSTGRES_DB: we're handling db creation ourselves in the backend volumes: - "storage-db:/var/lib/postgresql/data:rw,Z" secrets: - source: swh-mirror-db-postgres-password target: postgres-password uid: '999' mode: 0400 db-web: image: postgres:13 deploy: mode: global placement: constraints: - node.labels.org.softwareheritage.mirror.volumes.web-db == true command: ['-c', 'shared_buffers=4GB', '-c', 'effective_cache_size=4GB', '-c', 'random_page_cost=1.5', '-c', 'max_wal_size=4GB'] environment: POSTGRES_PASSWORD_FILE: /run/secrets/postgres-password POSTGRES_USER: swh POSTGRES_DB: swh-web volumes: - "web-db:/var/lib/postgresql/data:rw,Z" secrets: - source: swh-mirror-web-postgres-password target: postgres-password uid: '999' mode: 0400 web: image: softwareheritage/web:${SWH_IMAGE_TAG:-latest} configs: - source: web target: /etc/softwareheritage/config.yml command: serve environment: PORT: "5004" PGHOST: db-web PGUSER: swh POSTGRES_DB: swh-web depends_on: - db-web - memcache secrets: - source: swh-mirror-web-postgres-password target: postgres-password mode: 0400 objstorage: image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} deploy: # needed to allow actual and dynamic load balancing endpoint_mode: dnsrr # a real life replicas value better be in the 16 to 64 range replicas: 2 placement: # note: if using a local volume, you need to pin the objstorage # instances on the node hosting the volume, eg. the manager, otherwise, # if using a remote/distributed objstorage backend (seaweedfs, cloud, # etc.) you want to remove this placement constraint constraints: - node.labels.org.softwareheritage.mirror.volumes.objstorage == true volumes: - "objstorage:/srv/softwareheritage/objects:rw,Z" configs: - source: objstorage target: /etc/softwareheritage/config.yml environment: PORT: "5003" STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 command: objstorage storage: image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} deploy: # needed to allow actual and dynammic load balancing endpoint_mode: dnsrr # a real life replicas value better be in the 16 to 64 range replicas: 2 configs: - source: storage target: /etc/softwareheritage/config.yml environment: PGHOST: db-storage PGUSER: swh POSTGRES_DB: swh-storage FLAVOR: mirror PORT: "5002" STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 command: storage depends_on: - db-storage secrets: - source: swh-mirror-db-postgres-password target: postgres-password mode: 0400 nginx: image: nginx configs: - source: nginx target: /etc/nginx/nginx.conf ports: - "5081:5081/tcp" deploy: placement: constraints: - node.role == manager prometheus: image: prom/prometheus depends_on: - prometheus-statsd-exporter command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z" deploy: mode: global prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro,Z" grafana: image: grafana/grafana depends_on: - prometheus environment: GF_SERVER_ROOT_URL: http://localhost:5081/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro,Z" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards:rw,Z" + redis: + image: redis:6.2.6 + deploy: + mode: global + placement: + constraints: + - node.labels.org.softwareheritage.mirror.volumes.redis == true + command: + - redis-server + - --save 60 1 + - --loglevel warning + volumes: + - redis:/data + volumes: objstorage: storage-db: web-db: secrets: swh-mirror-db-postgres-password: external: true swh-mirror-web-postgres-password: external: true configs: storage: file: conf/storage.yml name: storage objstorage: file: conf/objstorage.yml name: objstorage nginx: file: conf/nginx.conf name: nginx web: file: conf/web.yml name: web diff --git a/conf/content-replayer.yml.example b/conf/content-replayer.yml.example index 2958224..9c7cebf 100644 --- a/conf/content-replayer.yml.example +++ b/conf/content-replayer.yml.example @@ -1,28 +1,36 @@ objstorage: cls: remote url: http://storage0.euwest.azure.internal.softwareheritage.org:5003 max_retries: 5 pool_connections: 100 pool_maxsize: 200 objstorage_dst: cls: remote args: url: http://objstorage:5003 journal_client: cls: kafka brokers: - kafka1 - kafka2 - kafka3 group_id: test-user-content-replayer-x-change-me sasl.username: test-user sasl.password: change-me security.protocol: sasl_ssl sasl.mechanism: SCRAM-SHA-512 session.timeout.ms: 600000 max.poll.interval.ms: 3600000 message.max.bytes: 1000000000 privileged: true batch_size: 2000 + +replayer: + error_reporter: + # used to track objects that the replayer really failed at replication from + # the soure objstorage to the destination one + host: redis + port: 6379 + db: 0 diff --git a/conf/graph-replayer.yml.example b/conf/graph-replayer.yml.example index 32a8a22..2bd05c1 100644 --- a/conf/graph-replayer.yml.example +++ b/conf/graph-replayer.yml.example @@ -1,40 +1,48 @@ storage: cls: pipeline step: - cls: filter - cls: tenacious error_rate_limit: # fail after 10 errors for 1000 operations errors: 10 window_size: 1000 - cls: remote url: http://storage:5002/ max_retries: 5 pool_connections: 100 pool_maxsize: 200 journal_client: cls: kafka brokers: - kafka1 - kafka2 - kafka3 group_id: test-user-graph-replayer-x-change-me sasl.username: test-user sasl.password: change-me security.protocol: sasl_ssl sasl.mechanism: SCRAM-SHA-512 session.timeout.ms: 600000 max.poll.interval.ms: 3600000 message.max.bytes: 1000000000 object_types: - content - skipped_content - directory - origin - origin_visit - origin_visit_status - release - revision - snapshot privileged: true + +replayer: + error_reporter: + # used to track objects that the replayer really failed at storing in the + # storage + host: redis + port: 6379 + db: 0 diff --git a/content-replayer-override.yml b/content-replayer-override.yml index 7050a44..438ba70 100644 --- a/content-replayer-override.yml +++ b/content-replayer-override.yml @@ -1,21 +1,22 @@ # Mirror specific services version: '3.7' services: content-replayer: image: softwareheritage/replayer:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: content-replayer target: /etc/softwareheritage/config.yml command: - content-replayer depends_on: - objstorage + - redis configs: content-replayer: file: conf/content-replayer.yml name: content-replayer diff --git a/graph-replayer-override.yml b/graph-replayer-override.yml index 6e8d282..be87aa4 100644 --- a/graph-replayer-override.yml +++ b/graph-replayer-override.yml @@ -1,21 +1,22 @@ # Mirror specific services version: '3.7' services: graph-replayer: image: softwareheritage/replayer:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage + - redis configs: graph-replayer: file: conf/graph-replayer.yml name: graph-replayer