diff --git a/docker/Dockerfile b/docker/Dockerfile --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,6 +32,7 @@ RUN pip install \ swh-core[db,http] \ + swh-counters \ swh-deposit[server] \ swh-indexer \ swh-journal \ diff --git a/docker/conf/counters.yml b/docker/conf/counters.yml new file mode 100644 --- /dev/null +++ b/docker/conf/counters.yml @@ -0,0 +1,3 @@ +counters: + cls: redis + host: redis:6379 diff --git a/docker/conf/counters_journal_client.yml b/docker/conf/counters_journal_client.yml new file mode 100644 --- /dev/null +++ b/docker/conf/counters_journal_client.yml @@ -0,0 +1,12 @@ +counters: + cls: remote + url: http://swh-counters:5011/ +journal: + brokers: + - kafka + prefix: swh.journal.objects + group_id: swh.counters.journal_client + object_types: + - content + - origin + - revision diff --git a/docker/docker-compose-counters.yml b/docker/docker-compose-counters.yml new file mode 100644 --- /dev/null +++ b/docker/docker-compose-counters.yml @@ -0,0 +1,60 @@ +version: "2.1" + +services: + swh-counters: + image: swh/stack + build: ./ + entrypoint: /entrypoint.sh + environment: + SWH_CONFIG_FILENAME: /etc/softwareheritage/counters/server.yml + env_file: + - ./env/common_python.env + ports: + - 5011:5011 + depends_on: + redis: + condition: service_healthy + volumes: + - "./conf/counters.yml:/etc/softwareheritage/counters/server.yml:ro" + - "./services/swh-counters/entrypoint.sh:/entrypoint.sh:ro" + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:5011/" ] + interval: 10s + timeout: 5s + retries: 10 + + swh-counters-journal-client: + image: swh/stack + build: ./ + entrypoint: /entrypoint.sh + env_file: + - ./env/common_python.env + depends_on: + kafka: + condition: service_healthy + redis: + condition: service_healthy + swh-counters: + condition: service_healthy + volumes: + - "./conf/counters_journal_client.yml:/etc/softwareheritage/counters/journal_client.yml:ro" + - "./services/swh-counters-journal-client/entrypoint.sh:/entrypoint.sh:ro" + + redis: + image: redis + volumes: + - redis-data:/data + ports: + - 6379 + command: + - "--save" + - "60" + - "1" # flush every minutes + healthcheck: + test: [ "CMD", "redis-cli", "PING" ] + interval: 1s + timeout: 5s + retries: 10 + +volumes: + redis-data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,14 +16,16 @@ - "5092:9092" env_file: ./env/kafka.env environment: - KAFKA_CREATE_TOPICS: swh.journal.objects.origin:1:1, + KAFKA_CREATE_TOPICS: swh.journal.objects.content:1:1, + swh.journal.objects.origin:1:1, swh.journal.objects.origin_visit:1:1, swh.journal.objects.origin_visit_status:1:1, + swh.journal.objects.revision:1:1, swh.journal.indexed.origin_intrinsic_metadata:1:1 depends_on: - zookeeper healthcheck: - test: "[ `JMX_PORT= kafka-topics.sh --list --zookeeper zookeeper:2181 | wc -l` -ge 4 ]" + test: "[ `JMX_PORT= kafka-topics.sh --list --zookeeper zookeeper:2181 | wc -l` -ge 6 ]" interval: 10s timeout: 5s retries: 10 diff --git a/docker/services/swh-counters-journal-client/entrypoint.sh b/docker/services/swh-counters-journal-client/entrypoint.sh new file mode 100755 --- /dev/null +++ b/docker/services/swh-counters-journal-client/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +source /srv/softwareheritage/utils/pyutils.sh +setup_pip + +case "$1" in + "shell") + exec bash -i + ;; + *) + echo "Starting swh-counters-journal client" + exec wait-for-it kafka:9092 -s --timeout=0 -- \ + swh --log-level DEBUG counters --config-file /etc/softwareheritage/counters/journal_client.yml journal-client + ;; +esac diff --git a/docker/services/swh-counters/entrypoint.sh b/docker/services/swh-counters/entrypoint.sh new file mode 100755 --- /dev/null +++ b/docker/services/swh-counters/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +source /srv/softwareheritage/utils/pyutils.sh +setup_pip + +case "$1" in + "shell") + exec bash -i + ;; + *) + echo "Starting the swh-counters API server" + wait-for-it redis:6379 -s --timeout=0 + exec gunicorn --bind 0.0.0.0:5011 \ + --reload \ + --threads 4 \ + --workers 2 \ + --log-level DEBUG \ + --timeout 3600 \ + --config 'python:swh.core.api.gunicorn_config' \ + 'swh.counters.api.server:make_app_from_configfile()' + ;; +esac