diff --git a/docker/conf/indexer_journal_client.yml b/docker/conf/indexer_journal_client.yml --- a/docker/conf/indexer_journal_client.yml +++ b/docker/conf/indexer_journal_client.yml @@ -2,6 +2,7 @@ brokers: - kafka group_id: swh.indexer.journal_client + prefix: swh.journal.objects max_messages: 50 diff --git a/docker/docker-compose.search.yml b/docker/docker-compose.search.yml --- a/docker/docker-compose.search.yml +++ b/docker/docker-compose.search.yml @@ -1,4 +1,4 @@ -version: '2' +version: "2" services: elasticsearch: @@ -29,8 +29,10 @@ build: ./ entrypoint: /entrypoint.sh depends_on: - - swh-search - - kafka + kafka: + condition: service_healthy + swh-search: + condition: service_started volumes: - "./conf/search_journal_client_objects.yml:/etc/softwareheritage/search/journal_client.yml:ro" - "./services/swh-search-journal-client/entrypoint.sh:/entrypoint.sh:ro" @@ -40,8 +42,10 @@ build: ./ entrypoint: /entrypoint.sh depends_on: - - swh-search - - kafka + kafka: + condition: service_healthy + swh-search: + condition: service_started volumes: - "./conf/search_journal_client_indexed.yml:/etc/softwareheritage/search/journal_client.yml:ro" - "./services/swh-search-journal-client/entrypoint.sh:/entrypoint.sh:ro" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -15,8 +15,18 @@ ports: - "5092:9092" env_file: ./env/kafka.env + environment: + KAFKA_CREATE_TOPICS: swh.journal.objects.origin:1:1, + swh.journal.objects.origin_visit:1:1, + swh.journal.objects.origin_visit_status:1:1, + swh.journal.indexed.origin_intrinsic_metadata:1:1 depends_on: - zookeeper + healthcheck: + test: "[ `JMX_PORT= kafka-topics.sh --list --zookeeper zookeeper:2181 | wc -l` -ge 4 ]" + interval: 10s + timeout: 5s + retries: 10 prometheus: image: prom/prometheus @@ -368,9 +378,13 @@ env_file: - ./env/common_python.env depends_on: - - kafka - - swh-storage - - swh-scheduler + kafka: + condition: service_healthy + swh-storage: + condition: service_started + swh-scheduler: + condition: service_started + volumes: - "./conf/indexer_journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro" - "./services/swh-indexer-journal-client/entrypoint.sh:/entrypoint.sh:ro" @@ -382,8 +396,10 @@ env_file: - ./env/common_python.env depends_on: - - kafka - - swh-scheduler + kafka: + condition: service_healthy + swh-scheduler: + condition: service_started volumes: - "./conf/scheduler_journal_client.yml:/etc/softwareheritage/scheduler/journal_client.yml:ro" - "./services/swh-scheduler-journal-client/entrypoint.sh:/entrypoint.sh:ro" diff --git a/docker/services/swh-indexer-journal-client/entrypoint.sh b/docker/services/swh-indexer-journal-client/entrypoint.sh --- a/docker/services/swh-indexer-journal-client/entrypoint.sh +++ b/docker/services/swh-indexer-journal-client/entrypoint.sh @@ -11,7 +11,8 @@ ;; *) echo "Starting swh-indexer-journal client" - exec wait-for-it kafka:9092 -s --timeout=0 -- swh indexer --config-file /etc/softwareheritage/indexer/journal_client.yml journal-client + wait-for-it kafka:9092 -s --timeout=0 -- \ + swh --log-level DEBUG indexer --config-file /etc/softwareheritage/indexer/journal_client.yml journal-client ;; esac