Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9340866
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
10 KB
Subscribers
None
View Options
diff --git a/docker-compose.yml b/docker-compose.yml
index 50c2633..28e1d81 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,331 +1,333 @@
version: '2'
services:
amqp:
image: rabbitmq:3.6-management
ports:
- 5072:5672
# flower:
# image: mher/flower
# command: --broker=amqp://guest:guest@amqp:5672// --url_prefix=flower
# ports:
# - 5055:5555
# depends_on:
# - amqp
zookeeper:
image: wurstmeister/zookeeper
kafka:
image: wurstmeister/kafka
ports:
- 5092:9092
env_file: ./env/kafka.env
depends_on:
- zookeeper
prometheus:
image: prom/prometheus
command:
# Needed for the reverse-proxy
- "--web.external-url=/prometheus"
- "--config.file=/etc/prometheus/prometheus.yml"
volumes:
- "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
restart: unless-stopped
prometheus-statsd-exporter:
image: prom/statsd-exporter
command:
- "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml"
volumes:
- "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro"
restart: unless-stopped
grafana:
image: grafana/grafana
restart: unless-stopped
environment:
GF_SERVER_ROOT_URL: http://localhost:5080/grafana
volumes:
- "./conf/grafana/provisioning:/etc/grafana/provisioning:ro"
- "./conf/grafana/dashboards:/var/lib/grafana/dashboards"
nginx:
image: nginx
volumes:
- "./conf/nginx.conf:/etc/nginx/nginx.conf:ro"
ports:
- 5080:5080
# Scheduler
swh-scheduler-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-scheduler
swh-scheduler-api:
image: swh/scheduler-api
build: ./dockerfiles/swh-scheduler-api
env_file: ./env/scheduler.env
+ environment:
+ SWH_CONFIG_FILENAME: /scheduler.yml
depends_on:
- swh-scheduler-db
ports:
- 5008:5008
volumes:
- "./conf/scheduler.yml:/scheduler.yml:ro"
swh-scheduler-listener:
image: swh/scheduler-worker
build: ./dockerfiles/swh-scheduler-worker
env_file: ./env/scheduler.env
command: listener
depends_on:
- swh-scheduler-api
- amqp
volumes:
- "./conf/scheduler.yml:/scheduler.yml:ro"
swh-scheduler-runner:
image: swh/scheduler-worker
build: ./dockerfiles/swh-scheduler-worker
env_file: ./env/scheduler.env
command: runner -p 10
depends_on:
- swh-scheduler-api
- amqp
volumes:
- "./conf/scheduler.yml:/scheduler.yml:ro"
# Graph storage
swh-storage-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-storage
swh-storage:
build: ./dockerfiles/swh-storage
image: swh/storage
ports:
- 5002:5002
depends_on:
- swh-storage-db
- swh-objstorage
env_file: ./env/storage.env
volumes:
- "./conf/storage.yml:/storage.yml:ro"
# Object storage
swh-objstorage:
build: ./dockerfiles/swh-objstorage
image: swh/objstorage
ports:
- 5003:5003
environment:
SWH_CONFIG_FILENAME: /objstorage.yml
volumes:
- "./conf/objstorage.yml:/objstorage.yml:ro"
# Indexer storage
swh-idx-storage-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-idx-storage
swh-idx-storage:
build: ./dockerfiles/swh-indexer-storage
image: swh/indexer-storage
ports:
- 5007:5007
depends_on:
- swh-idx-storage-db
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-idx-storage
PGHOST: swh-idx-storage-db
PGUSER: postgres
SWH_CONFIG_FILENAME: /indexer_storage.yml
volumes:
- "./conf/indexer_storage.yml:/indexer_storage.yml:ro"
# Web interface
swh-web:
build: ./dockerfiles/swh-web
image: swh/web
command:
- runserver
- --verbosity 3
- --insecure
- 0.0.0.0:5004
ports:
- 5004:5004
depends_on:
- swh-objstorage
- swh-storage
- swh-idx-storage
environment:
VERBOSITY: 3
#DJANGO_SETTINGS_MODULE: djangosettings
PYTHONPATH: /tmp/swh
volumes:
- "./conf/web.yml:/etc/softwareheritage/web/web.yml:ro"
- "./conf/djangosettings.py:/tmp/swh/djangosettings.py"
swh-deposit-db:
image: postgres:10
env_file: ./env/deposit.env
swh-deposit:
build: ./dockerfiles/swh-deposit
image: swh/deposit
ports:
- 5006:5006
depends_on:
- swh-deposit-db
- swh-scheduler-api
env_file: ./env/deposit.env
environment:
PGHOST: swh-deposit-db
volumes:
- "./conf/deposit.yml:/deposit.yml:ro"
swh-vault-db:
image: postgres:10
env_file: ./env/vault.env
swh-vault-api:
build: ./dockerfiles/swh-vault
image: swh/vault
env_file: ./env/vault.env
command: server
ports:
- 5005:5005
depends_on:
- swh-vault-db
- swh-objstorage
- swh-storage
- swh-scheduler-api
volumes:
- "./conf/vault-api.yml:/vault-api.yml:ro"
swh-vault-worker:
build: ./dockerfiles/swh-vault
image: swh/vault
command: worker
environment:
SWH_CONFIG_FILENAME: /cooker.yml
depends_on:
- swh-vault-api
- swh-storage
volumes:
- "./conf/vault-worker.yml:/cooker.yml:ro"
# Lister Celery workers
swh-listers-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
swh-lister:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./env/listers.env
environment:
STATSD_HOST: prometheus-statsd-exporter
STATSD_PORT: 9125
SWH_WORKER_INSTANCE: listers
SWH_CONFIG_FILENAME: /lister.yml
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-scheduler-runner
- swh-storage
- amqp
volumes:
- "./conf/lister.yml:/lister.yml:ro"
# Loader Celery workers
swh-loader:
image: swh/loaders-worker
build: ./dockerfiles/swh-loaders-worker
env_file: ./env/listers.env
environment:
STATSD_HOST: prometheus-statsd-exporter
STATSD_PORT: 9125
SWH_WORKER_INSTANCE: loader
SWH_CONFIG_FILENAME: /loader.yml
depends_on:
- swh-storage
- amqp
volumes:
- "./conf/loader.yml:/loader.yml:ro"
# Indexer Celery workers
swh-indexer:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./env/indexers.env
environment:
STATSD_HOST: prometheus-statsd-exporter
STATSD_PORT: 9125
depends_on:
- swh-scheduler-runner
- swh-idx-storage
- swh-storage
- swh-objstorage
- amqp
volumes:
- "./conf/indexer.yml:/indexer.yml:ro"
swh-indexer-journal-client:
image: swh/indexer-journal-client
build: ./dockerfiles/swh-indexer-journal-client
depends_on:
- swh-journal-publisher
- swh-scheduler-api
volumes:
- "./conf/journal_client.yml:/etc/softwareheritage/indexer/journal_client.yml:ro"
# Journal related
swh-storage-listener:
image: swh/storage-listener
build: ./dockerfiles/swh-storage-listener
env_file: ./env/storage.env
depends_on:
- swh-storage-db
- kafka
volumes:
- "./conf/storage_listener.yml:/etc/softwareheritage/storage/listener.yml:ro"
swh-journal-publisher:
image: swh/journal-publisher
build: ./dockerfiles/swh-journal-publisher
depends_on:
- kafka
- swh-storage-listener
volumes:
- "./conf/journal_publisher.yml:/etc/softwareheritage/journal/publisher.yml:ro"
swh-journal-client:
image: swh/journal-client
build: ./dockerfiles/swh-journal-client
depends_on:
- swh-journal-publisher
volumes:
- "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro"
diff --git a/dockerfiles/swh-scheduler-api/Dockerfile b/dockerfiles/swh-scheduler-api/Dockerfile
index 7670778..5bc3dad 100644
--- a/dockerfiles/swh-scheduler-api/Dockerfile
+++ b/dockerfiles/swh-scheduler-api/Dockerfile
@@ -1,15 +1,15 @@
FROM python:3.6
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get install -y \
libsystemd-dev postgresql-client
RUN pip install --upgrade pip setuptools wheel
-RUN pip install swh-scheduler
+RUN pip install swh-scheduler gunicorn
COPY entrypoint.sh /
ENTRYPOINT ["/entrypoint.sh"]
EXPOSE 5008
diff --git a/dockerfiles/swh-scheduler-api/entrypoint.sh b/dockerfiles/swh-scheduler-api/entrypoint.sh
index 34a5d3f..a597c8e 100755
--- a/dockerfiles/swh-scheduler-api/entrypoint.sh
+++ b/dockerfiles/swh-scheduler-api/entrypoint.sh
@@ -1,44 +1,48 @@
#!/bin/bash
set -e
if [[ -d /src ]] ; then
for srcrepo in /src/swh-* ; do
pushd $srcrepo
pip install -e .
popd
done
fi
echo Installed Python packages:
pip list
echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass
cat > ~/.pg_service.conf <<EOF
[swh-scheduler]
dbname=${POSTGRES_DB}
host=${PGHOST}
port=5432
user=${PGUSER}
EOF
chmod 0600 ~/.pgpass
case "$1" in
"shell")
exec bash -i
;;
"swh-scheduler")
exec $@
;;
*)
echo Waiting for postgresql to start
until psql postgresql:///?service=swh-scheduler -c "select 1" 2>&1 > /dev/null; do sleep 0.1; done
echo Setup the swh-scheduler API database
PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init scheduler \
--db-name ${POSTGRES_DB}
echo Starting the swh-scheduler API server
- exec swh-scheduler --log-level ${LOGLEVEL} -C /scheduler.yml api-server
+ exec gunicorn --bind 0.0.0.0:5008 \
+ --log-level DEBUG \
+ --reload \
+ swh.scheduler.api.wsgi
+
esac
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Fri, Jul 4, 11:15 AM (3 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3271348
Attached To
rDENV Development environment
Event Timeline
Log In to Comment