diff --git a/docker-compose.yml b/docker-compose.yml index 64eda00..0a3f2ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,188 +1,211 @@ version: '2' services: amqp: image: rabbitmq:3.6-management ports: - 5018:15672 + zookeeper: + image: wurstmeister/zookeeper + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka + ports: + - "9092:9092" + # env_file: ./kafka.env + environment: + BROKER_ID_COMMAND: "hostname | awk -F'-' '{print $$2}'" + # KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n'" + KAFKA_CREATE_TOPICS: "swh.tmp_journal.new.content:1:1 +swh.tmp_journal.new.revision:1:1, +swh.tmp_journal.new.release:1:1, +swh.journal.objects.contents:1:1, +swh.journal.objects.revision:1:1, +swh.journal.objects.release:1:1" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + # Scheduler swh-scheduler-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-scheduler swh-scheduler-api: image: swh/scheduler-api build: ./dockerfiles/swh-scheduler-api env_file: ./scheduler.env depends_on: - swh-scheduler-db ports: - 5008:5008 swh-scheduler-listener: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./scheduler.env command: listener depends_on: - swh-scheduler-api - amqp swh-scheduler-runner: image: swh/scheduler-worker build: ./dockerfiles/swh-scheduler-worker env_file: ./scheduler.env command: runner depends_on: - swh-scheduler-api - amqp # Graph storage swh-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage swh-storage: build: ./dockerfiles/swh-storage image: swh/storage ports: - 5002:5002 depends_on: - swh-storage-db - swh-objstorage environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-storage PGHOST: swh-storage-db PGUSER: postgres # Object storage swh-objstorage: build: ./dockerfiles/swh-objstorage image: swh/objstorage ports: - 5003:5003 # Indexer storage swh-idx-storage-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage swh-idx-storage: build: ./dockerfiles/swh-indexer-storage image: swh/indexer-storage ports: - 5007:5007 depends_on: - swh-idx-storage-db environment: POSTGRES_PASSWORD: testpassword POSTGRES_DB: swh-idx-storage PGHOST: swh-idx-storage-db PGUSER: postgres # Web interface swh-web: build: ./dockerfiles/swh-web image: swh/web ports: - 8080:5004 depends_on: - swh-objstorage - swh-storage - swh-idx-storage # Lister Celery workers swh-listers-db: image: postgres:10 environment: POSTGRES_PASSWORD: testpassword swh-lister-debian: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: debian depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-github: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: github depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-gitlab: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: gitlab depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-npm: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: npm depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp swh-lister-pypi: image: swh/listers-worker build: ./dockerfiles/swh-listers-worker env_file: ./listers.env environment: SWH_WORKER_INSTANCE: pypi depends_on: - swh-listers-db - swh-scheduler-api - swh-storage - amqp # Indexer Celery workers swh-indexer-mimetype: image: swh/indexer-worker build: ./dockerfiles/swh-indexer-worker env_file: ./indexers.env environment: SWH_WORKER_INSTANCE: mimetype depends_on: - swh-scheduler-api - swh-idx-storage - swh-storage - swh-objstorage - amqp diff --git a/kafka.env b/kafka.env new file mode 100644 index 0000000..f204fae --- /dev/null +++ b/kafka.env @@ -0,0 +1,10 @@ +BROKER_ID_COMMAND="hostname | awk -F'-' '{print $$2}'" +KAFKA_ADVERTISED_HOST_NAME="192.168.99.100" +KAFKA_CREATE_TOPICS_SEPARATOR="$$'\n'" +KAFKA_CREATE_TOPICS="swh.tmp_journal.new.content:1:1 +swh.tmp_journal.new.revision:1:1, +swh.tmp_journal.new.release:1:1, +swh.journal.objects.contents:1:1, +swh.journal.objects.revision:1:1, +swh.journal.objects.release:1:1" +KAFKA_ZOOKEEPER_CONNECT="zookeeper:2181"