diff --git a/conf/graph-replayer/remote/all.yml.example b/conf/graph-replayer/remote/all.yml.example new file mode 100644 index 0000000..4246095 --- /dev/null +++ b/conf/graph-replayer/remote/all.yml.example @@ -0,0 +1,22 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + # object_types: + # - content diff --git a/conf/graph-replayer/remote/content.yml.example b/conf/graph-replayer/remote/content.yml.example new file mode 100644 index 0000000..c79d498 --- /dev/null +++ b/conf/graph-replayer/remote/content.yml.example @@ -0,0 +1,23 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - content diff --git a/conf/graph-replayer/remote/directory.yml.example b/conf/graph-replayer/remote/directory.yml.example new file mode 100644 index 0000000..4967436 --- /dev/null +++ b/conf/graph-replayer/remote/directory.yml.example @@ -0,0 +1,23 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - directory diff --git a/conf/graph-replayer/remote/origin.yml.example b/conf/graph-replayer/remote/origin.yml.example new file mode 100644 index 0000000..50cb945 --- /dev/null +++ b/conf/graph-replayer/remote/origin.yml.example @@ -0,0 +1,24 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - origin + - origin_visit diff --git a/conf/graph-replayer/remote/other.yml.example b/conf/graph-replayer/remote/other.yml.example new file mode 100644 index 0000000..96f1149 --- /dev/null +++ b/conf/graph-replayer/remote/other.yml.example @@ -0,0 +1,26 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - origin + - origin_visit + - snapshot + - release diff --git a/conf/graph-replayer/remote/release.yml.example b/conf/graph-replayer/remote/release.yml.example new file mode 100644 index 0000000..6400f74 --- /dev/null +++ b/conf/graph-replayer/remote/release.yml.example @@ -0,0 +1,23 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - release diff --git a/conf/graph-replayer/remote/revision.yml.example b/conf/graph-replayer/remote/revision.yml.example new file mode 100644 index 0000000..af56f6b --- /dev/null +++ b/conf/graph-replayer/remote/revision.yml.example @@ -0,0 +1,23 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - revision diff --git a/conf/graph-replayer/remote/snapshot.yml.example b/conf/graph-replayer/remote/snapshot.yml.example new file mode 100644 index 0000000..b6f58cc --- /dev/null +++ b/conf/graph-replayer/remote/snapshot.yml.example @@ -0,0 +1,23 @@ +storage: + cls: remote + args: + url: http://storage:5002/ + pool_connections: 100 + pool_maxsize: 200 + +journal_client: + cls: kafka + brokers: + - kafka01.softwareheritage.org:9093 + - kafka02.softwareheritage.org:9093 + - kafka03.softwareheritage.org:9093 + - kafka04.softwareheritage.org:9093 + - kafka05.softwareheritage.org:9093 + - kafka06.softwareheritage.org:9093 + group_id: test-graph-replayer-XXX + sasl.username: XXX + sasl.password: XXX + security.protocol: sasl_ssl + sasl.mechanism: SCRAM-SHA-512 + object_types: + - snapshot diff --git a/docker-compose.yml b/docker-compose.yml index ba33f8d..8bac168 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,132 +1,132 @@ version: "3.7" services: memcache: image: memcached deploy: replicas: 1 db-storage: image: postgres:11 deploy: replicas: 1 placement: constraints: - node.labels.role == db command: ['-c', 'shared_buffers=4GB', '-c', 'effective_cache_size=4GB', '-c', 'random_page_cost=1.5', '-c', 'max_wal_size=4GB'] environment: POSTGRES_PASSWORD_FILE: /run/secrets/postgres-password POSTGRES_USER: swh volumes: - "storage:/var/lib/postgresql/data" secrets: - source: postgres-password uid: '999' mode: 0400 web: - image: softwareheritage/web:20200617-135429 + image: softwareheritage/web:${SWH_IMAGE_TAG:-latest} configs: - source: web target: /etc/softwareheritage/config.yml command: - serve environment: PORT: "5004" depends_on: - memcache objstorage: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} deploy: replicas: 1 placement: constraints: - node.role == manager volumes: - "objstorage:/srv/softwareheritage/objects" configs: - source: objstorage target: /etc/softwareheritage/config.yml environment: PORT: "5003" STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 command: - objstorage storage: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} configs: - source: storage target: /etc/softwareheritage/config.yml environment: PGHOST: db-storage PORT: "5002" STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 command: - storage depends_on: - db-storage secrets: - source: postgres-password mode: 0400 nginx: image: nginx configs: - source: nginx target: /etc/nginx/nginx.conf ports: - "5081:5081/tcp" deploy: placement: constraints: - node.role == manager prometheus: image: prom/prometheus depends_on: - prometheus-statsd-exporter command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro" prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro" grafana: image: grafana/grafana depends_on: - prometheus environment: GF_SERVER_ROOT_URL: http://localhost:5081/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards" volumes: objstorage: storage: secrets: postgres-password: external: true configs: web: file: conf/web.yml storage: file: conf/storage.yml objstorage: file: conf/objstorage.yml nginx: file: conf/nginx.conf diff --git a/graph-replayer-remote-bytopic.yml b/graph-replayer-remote-bytopic.yml index 0a7bc95..4e77af0 100644 --- a/graph-replayer-remote-bytopic.yml +++ b/graph-replayer-remote-bytopic.yml @@ -1,95 +1,95 @@ # Mirror specific services version: '3.7' services: graph-replayer-content: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer-remote-content target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage graph-replayer-revision: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer-remote-revision target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage graph-replayer-directory: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer-remote-directory target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage graph-replayer-origin: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer-remote-origin target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage graph-replayer-snapshot: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer-remote-snapshot target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage graph-replayer-release: - image: softwareheritage/base:20200617-135429 + image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 configs: - source: graph-replayer-remote-release target: /etc/softwareheritage/config.yml command: - graph-replayer depends_on: - storage configs: graph-replayer-remote-content: file: conf/graph-replayer/remote/content.yml graph-replayer-remote-revision: file: conf/graph-replayer/remote/revision.yml graph-replayer-remote-directory: file: conf/graph-replayer/remote/directory.yml graph-replayer-remote-origin: file: conf/graph-replayer/remote/origin.yml graph-replayer-remote-release: file: conf/graph-replayer/remote/release.yml graph-replayer-remote-snapshot: file: conf/graph-replayer/remote/snapshot.yml