Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9344211
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Subscribers
None
View Options
diff --git a/docker-compose.yml b/docker-compose.yml
index 68c23a7..fe155ae 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,365 +1,371 @@
version: '2'
services:
amqp:
image: rabbitmq:3.6-management
ports:
- 5018:15672
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
kafka:
image: wurstmeister/kafka
ports:
- "9092:9092"
env_file: ./kafka.env
depends_on:
- zookeeper
flower:
image: mher/flower
command: --broker=amqp://guest:guest@amqp:5672//
ports:
- 5555:5555
depends_on:
- amqp
# Scheduler
swh-scheduler-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-scheduler
swh-scheduler-api:
image: swh/scheduler-api
build: ./dockerfiles/swh-scheduler-api
env_file: ./scheduler.env
depends_on:
- swh-scheduler-db
ports:
- 5008:5008
swh-scheduler-listener:
image: swh/scheduler-worker
build: ./dockerfiles/swh-scheduler-worker
env_file: ./scheduler.env
command: listener
depends_on:
- swh-scheduler-api
- amqp
swh-scheduler-runner:
image: swh/scheduler-worker
build: ./dockerfiles/swh-scheduler-worker
env_file: ./scheduler.env
command: runner -p 10
depends_on:
- swh-scheduler-api
- amqp
# Graph storage
swh-storage-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-storage
swh-storage:
build: ./dockerfiles/swh-storage
image: swh/storage
ports:
- 5002:5002
depends_on:
- swh-storage-db
- swh-objstorage
env_file: ./storage.env
# Object storage
swh-objstorage:
build: ./dockerfiles/swh-objstorage
image: swh/objstorage
ports:
- 5003:5003
# Indexer storage
swh-idx-storage-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-idx-storage
swh-idx-storage:
build: ./dockerfiles/swh-indexer-storage
image: swh/indexer-storage
ports:
- 5007:5007
depends_on:
- swh-idx-storage-db
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-idx-storage
PGHOST: swh-idx-storage-db
PGUSER: postgres
# Web interface
swh-web:
build: ./dockerfiles/swh-web
image: swh/web
ports:
- 8080:5004
depends_on:
- swh-objstorage
- swh-storage
- swh-idx-storage
swh-deposit-db:
image: postgres:10
env_file: ./deposit.env
swh-deposit:
build: ./dockerfiles/swh-deposit
image: swh/deposit
ports:
- 5006:5006
depends_on:
- swh-deposit-db
- swh-scheduler-api
env_file: ./deposit.env
environment:
PGHOST: swh-deposit-db
# Lister Celery workers
swh-listers-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
swh-lister-debian:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: debian
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-bitbucket:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: bitbucket
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-github:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: github
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-gitlab:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: gitlab
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-npm:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: npm
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-pypi:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: pypi
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
# Indexer Celery workers
swh-indexer-mimetype:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./indexers.env
environment:
SWH_WORKER_INSTANCE: content_mimetype
depends_on:
- swh-scheduler-api
- swh-idx-storage
- swh-storage
- swh-objstorage
- amqp
swh-indexer-origin-head:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./indexers.env
environment:
SWH_WORKER_INSTANCE: origin_head
depends_on:
- swh-scheduler-api
- swh-idx-storage
- swh-storage
- amqp
swh-indexer-revision-metadata:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./indexers.env
environment:
SWH_WORKER_INSTANCE: revision_metadata
depends_on:
- swh-scheduler-api
- swh-idx-storage
- swh-storage
- swh-objstorage
- amqp
swh-indexer-origin-intrinsic-metadata:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./indexers.env
environment:
SWH_WORKER_INSTANCE: origin_intrinsic_metadata
depends_on:
- swh-scheduler-api
- swh-idx-storage
- swh-storage
- amqp
+ swh-indexer-journal-client:
+ image: swh/indexer-journal-client
+ build: ./dockerfiles/swh-indexer-journal-client
+ depends_on:
+ - swh-journal-publisher
+
# Journal related
swh-storage-listener:
image: swh/storage-listener
build: ./dockerfiles/swh-storage-listener
env_file: ./storage.env
depends_on:
- swh-storage-db
- kafka
swh-journal-publisher:
image: swh/journal-publisher
build: ./dockerfiles/swh-journal-publisher
depends_on:
- kafka
- swh-storage-listener
swh-journal-client:
image: swh/journal-client
build: ./dockerfiles/swh-journal-client
depends_on:
- swh-journal-publisher
# Loader Celery workers
swh-loader-debian:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: debian
depends_on:
- swh-storage
- amqp
swh-loader-dir:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: dir
depends_on:
- swh-storage
- amqp
swh-loader-git:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: git
depends_on:
- swh-storage
- amqp
swh-loader-mercurial:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: mercurial
depends_on:
- swh-storage
- amqp
swh-loader-pypi:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: pypi
depends_on:
- swh-storage
- amqp
swh-loader-svn:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: svn
depends_on:
- swh-storage
- amqp
swh-loader-tar:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: tar
depends_on:
- swh-storage
- amqp
diff --git a/dockerfiles/swh-indexer-journal-client/Dockerfile b/dockerfiles/swh-indexer-journal-client/Dockerfile
new file mode 100644
index 0000000..8f8d9a4
--- /dev/null
+++ b/dockerfiles/swh-indexer-journal-client/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update && \
+ apt-get install -y libsystemd-dev
+
+RUN pip install swh-indexer
+COPY journal_client.yml /etc/softwareheritage/indexer/
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
diff --git a/dockerfiles/swh-indexer-journal-client/entrypoint.sh b/dockerfiles/swh-indexer-journal-client/entrypoint.sh
new file mode 100755
index 0000000..51f2908
--- /dev/null
+++ b/dockerfiles/swh-indexer-journal-client/entrypoint.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+if [[ -d /src ]] ; then
+ for srcrepo in /src/swh-* ; do
+ pushd $srcrepo
+ pip install -e .
+ popd
+ done
+fi
+
+echo Installed Python packages:
+pip list
+
+case "$1" in
+ "shell")
+ exec bash -i
+ ;;
+ *)
+ echo "Starting swh-journal client"
+ python3 -m swh.indexer.journal_client
+ ;;
+esac
+
diff --git a/dockerfiles/swh-indexer-journal-client/journal_client.yml b/dockerfiles/swh-indexer-journal-client/journal_client.yml
new file mode 100644
index 0000000..d11ae93
--- /dev/null
+++ b/dockerfiles/swh-indexer-journal-client/journal_client.yml
@@ -0,0 +1,10 @@
+brokers:
+ - kafka
+# small number of batch
+max_messages: 1
+consumer_id: swh.indexer.journal_client
+
+scheduler:
+ cls: remote
+ args:
+ url: http://swh-scheduler-api:5008/
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Fri, Jul 4, 2:13 PM (1 d, 2 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3236165
Attached To
rCDFD Dockerfiles for developers
Event Timeline
Log In to Comment