Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9338771
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
7 KB
Subscribers
None
View Options
diff --git a/docker-compose.yml b/docker-compose.yml
index 19ee0a4..46ec11a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,221 +1,221 @@
version: '2'
services:
amqp:
image: rabbitmq:3.6-management
ports:
- 5018:15672
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
kafka:
image: wurstmeister/kafka
ports:
- "9092:9092"
env_file: ./kafka.env
depends_on:
- zookeeper
# Scheduler
swh-scheduler-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-scheduler
swh-scheduler-api:
image: swh/scheduler-api
build: ./dockerfiles/swh-scheduler-api
env_file: ./scheduler.env
depends_on:
- swh-scheduler-db
ports:
- 5008:5008
swh-scheduler-listener:
image: swh/scheduler-worker
build: ./dockerfiles/swh-scheduler-worker
env_file: ./scheduler.env
command: listener
depends_on:
- swh-scheduler-api
- amqp
swh-scheduler-runner:
image: swh/scheduler-worker
build: ./dockerfiles/swh-scheduler-worker
env_file: ./scheduler.env
command: runner
depends_on:
- swh-scheduler-api
- amqp
# Graph storage
swh-storage-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-storage
swh-storage:
build: ./dockerfiles/swh-storage
image: swh/storage
ports:
- 5002:5002
depends_on:
- swh-storage-db
- swh-objstorage
env_file: ./storage.env
# Object storage
swh-objstorage:
build: ./dockerfiles/swh-objstorage
image: swh/objstorage
ports:
- 5003:5003
# Indexer storage
swh-idx-storage-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-idx-storage
swh-idx-storage:
build: ./dockerfiles/swh-indexer-storage
image: swh/indexer-storage
ports:
- 5007:5007
depends_on:
- swh-idx-storage-db
environment:
POSTGRES_PASSWORD: testpassword
POSTGRES_DB: swh-idx-storage
PGHOST: swh-idx-storage-db
PGUSER: postgres
# Web interface
swh-web:
build: ./dockerfiles/swh-web
image: swh/web
ports:
- 8080:5004
depends_on:
- swh-objstorage
- swh-storage
- swh-idx-storage
# Lister Celery workers
swh-listers-db:
image: postgres:10
environment:
POSTGRES_PASSWORD: testpassword
swh-lister-debian:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: debian
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-github:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: github
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-gitlab:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: gitlab
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-npm:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: npm
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
swh-lister-pypi:
image: swh/listers-worker
build: ./dockerfiles/swh-listers-worker
env_file: ./listers.env
environment:
SWH_WORKER_INSTANCE: pypi
depends_on:
- swh-listers-db
- swh-scheduler-api
- swh-storage
- amqp
# Indexer Celery workers
swh-indexer-mimetype:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./indexers.env
environment:
- SWH_WORKER_INSTANCE: mimetype
+ SWH_WORKER_INSTANCE: content_mimetype
depends_on:
- swh-scheduler-api
- swh-idx-storage
- swh-storage
- swh-objstorage
- amqp
# Journal related
swh-storage-listener:
image: swh/storage-listener
build: ./dockerfiles/swh-storage-listener
env_file: ./storage.env
depends_on:
- swh-storage-db
- kafka
swh-journal-publisher:
image: swh/journal-publisher
build: ./dockerfiles/swh-journal-publisher
depends_on:
- kafka
- swh-storage-listener
swh-journal-client:
image: swh/journal-client
build: ./dockerfiles/swh-journal-client
depends_on:
- swh-journal-publisher
diff --git a/dockerfiles/swh-indexer-storage/entrypoint.sh b/dockerfiles/swh-indexer-storage/entrypoint.sh
index 7ed6900..3296e6a 100755
--- a/dockerfiles/swh-indexer-storage/entrypoint.sh
+++ b/dockerfiles/swh-indexer-storage/entrypoint.sh
@@ -1,40 +1,42 @@
#!/bin/bash
set -e
if [[ -d /src ]] ; then
for srcrepo in /src/swh-* ; do
pushd $srcrepo
- pip install -e .
+ echo "WARNING: $srcrepo will NOT be pip installed in dev mode"
+ echo " due to permission limitations."
+ pip install --user .
popd
done
fi
echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass
cat > ~/.pg_service.conf <<EOF
[swh-indexer]
dbname=${POSTGRES_DB}
host=${PGHOST}
port=5432
user=${PGUSER}
EOF
chmod 0600 ~/.pgpass
case "$1" in
"shell")
exec bash -i
;;
*)
echo Waiting for postgresql to start
until psql service=swh-indexer -c "select 1" 2>&1 > /dev/null; do sleep 0.1; done
echo Setup the database
PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init indexer \
--db-name ${POSTGRES_DB}
echo Starting the swh-indexer-storage API server
- exec python -m swh.indexer.storage.api.server /indexer_storage.yml
+ exec python -m swh.indexer.storage.api.server /indexer_storage
;;
esac
diff --git a/dockerfiles/swh-indexer-storage/indexer_storage.yml b/dockerfiles/swh-indexer-storage/indexer_storage.yml
index d2f5276..657ee90 100644
--- a/dockerfiles/swh-indexer-storage/indexer_storage.yml
+++ b/dockerfiles/swh-indexer-storage/indexer_storage.yml
@@ -1,4 +1,4 @@
-storage:
+indexer_storage:
cls: local
args:
db: service=swh-indexer
diff --git a/dockerfiles/swh-indexer-worker/entrypoint.sh b/dockerfiles/swh-indexer-worker/entrypoint.sh
index 2ebe8b8..ec4592c 100755
--- a/dockerfiles/swh-indexer-worker/entrypoint.sh
+++ b/dockerfiles/swh-indexer-worker/entrypoint.sh
@@ -1,32 +1,34 @@
#!/bin/bash
set -e
if [[ -d /src ]] ; then
for srcrepo in /src/swh-* ; do
pushd $srcrepo
- pip install -e .
+ echo "WARNING: $srcrepo will NOT be pip installed in dev mode"
+ echo " due to permission limitations."
+ pip install --user .
popd
done
fi
mkdir -p ~/.config/swh/worker
cat > ~/.config/swh/worker/${SWH_WORKER_INSTANCE}.ini <<EOF
[main]
task_broker = amqp://guest@amqp//
task_modules = swh.indexer.tasks
task_queues = swh_indexer_${SWH_WORKER_INSTANCE}
task_soft_time_limit = 0
EOF
case "$1" in
"shell")
exec bash -i
;;
*)
echo Starting swh-indexer worker
exec python -m celery worker --app=swh.scheduler.celery_backend.config.app --pool=prefork --events --concurrency=${CONCURRENCY} --maxtasksperchild=${MAX_TASKS_PER_CHILD} -Ofair --loglevel=${LOGLEVEL} --without-gossip --without-mingle --without-heartbeat -n ${SWH_WORKER_INSTANCE}.${HOSTNAME}
;;
esac
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Jul 4 2025, 9:06 AM (6 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3252165
Attached To
rCDFD Dockerfiles for developers
Event Timeline
Log In to Comment