diff --git a/services/swh-deposit/entrypoint.sh b/services/swh-deposit/entrypoint.sh index 863cf78..d4f8c41 100755 --- a/services/swh-deposit/entrypoint.sh +++ b/services/swh-deposit/entrypoint.sh @@ -1,34 +1,26 @@ #!/bin/bash set -ex -if [[ -d /src ]] ; then - pwd - for src_repo in /src/swh-* ; do - pushd $src_repo - echo "Installing ${src_repo}" - pip install -e . - popd - done -fi +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql if [ "$1" = 'shell' ] ; then exec bash -i else wait_pgsql echo "Migrating db" django-admin migrate --settings=swh.deposit.settings.production echo "starting swh-deposit server" exec gunicorn --bind 0.0.0.0:5006 \ --reload \ --log-level DEBUG \ --timeout 3600 \ swh.deposit.wsgi fi diff --git a/services/swh-indexer-storage/entrypoint.sh b/services/swh-indexer-storage/entrypoint.sh index 841957f..c775d65 100755 --- a/services/swh-indexer-storage/entrypoint.sh +++ b/services/swh-indexer-storage/entrypoint.sh @@ -1,42 +1,30 @@ #!/bin/bash set -e -export PATH=${HOME}/.local/bin:${PATH} -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - echo "WARNING: $srcrepo will NOT be pip installed in dev mode" - echo " due to permission limitations." - pip install --user . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) wait_pgsql echo Setup the database PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init indexer \ --db-name ${POSTGRES_DB} echo Starting the swh-indexer-storage API server exec gunicorn --bind 0.0.0.0:5007 \ --reload \ --log-level DEBUG \ --timeout 3600 \ swh.indexer.storage.api.wsgi ;; esac diff --git a/services/swh-indexer-worker/entrypoint.sh b/services/swh-indexer-worker/entrypoint.sh index c2c53b6..dea17ef 100755 --- a/services/swh-indexer-worker/entrypoint.sh +++ b/services/swh-indexer-worker/entrypoint.sh @@ -1,42 +1,32 @@ #!/bin/bash set -e -export PATH=${HOME}/.local/bin:${PATH} - -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - echo "WARNING: $srcrepo will NOT be pip installed in dev mode" - echo " due to permission limitations." - pip install --user . - popd - done -fi +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) echo Waiting for RabbitMQ to start wait-for-it amqp:5672 -s --timeout=0 wait_pgsql echo Starting swh-indexer worker exec python -m celery worker \ --app=swh.scheduler.celery_backend.config.app \ --pool=prefork --events \ --concurrency=${CONCURRENCY} \ --maxtasksperchild=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} --without-gossip \ --without-mingle \ --heartbeat-interval 10 \ --hostname "${SWH_WORKER_INSTANCE}@%h" ;; esac diff --git a/services/swh-listers-worker/entrypoint.sh b/services/swh-listers-worker/entrypoint.sh index 964f3f6..33ec36a 100755 --- a/services/swh-listers-worker/entrypoint.sh +++ b/services/swh-listers-worker/entrypoint.sh @@ -1,57 +1,45 @@ #!/bin/bash set -e -export PATH=${HOME}/.local/bin:${PATH} -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - echo "WARNING: $srcrepo wil NOT be pip installed in dev mode" - echo " due to permission limitations." - pip install --user . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) wait_pgsql echo Setup ${POSTGRES_DB} database for ${SWH_WORKER_INSTANCE} if psql -lqt | cut -d \| -f 1 | grep -qw ${POSTGRES_DB}; then echo Database already exists, nothing to do else echo Creating database createdb ${POSTGRES_DB} echo Initialize database python -m swh.lister.cli \ --db-url postgres://${PGUSER}@${PGHOST}/${POSTGRES_DB} \ all fi echo Waiting for RabbitMQ to start wait-for-it amqp:5672 -s --timeout=0 echo Starting the swh-lister Celery worker for ${SWH_WORKER_INSTANCE} exec python -m celery worker \ --app=swh.scheduler.celery_backend.config.app \ --pool=prefork --events \ --concurrency=${CONCURRENCY} \ --maxtasksperchild=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} --without-gossip \ --without-mingle \ --heartbeat-interval 10 \ --hostname "${SWH_WORKER_INSTANCE}@%h" ;; esac diff --git a/services/swh-loaders-worker/entrypoint.sh b/services/swh-loaders-worker/entrypoint.sh index d135d21..2a15927 100755 --- a/services/swh-loaders-worker/entrypoint.sh +++ b/services/swh-loaders-worker/entrypoint.sh @@ -1,39 +1,27 @@ #!/bin/bash set -e -export PATH=${HOME}/.local/bin:${PATH} - -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - echo "WARNING: $srcrepo will NOT be pip installed in dev mode" - echo " due to permission limitations." - pip install --user . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip case "$1" in "shell") exec bash -i ;; *) echo Waiting for RabbitMQ to start wait-for-it amqp:5672 -s --timeout=0 echo Starting the swh-loader Celery worker for ${SWH_WORKER_INSTANCE} exec python -m celery worker \ --app=swh.scheduler.celery_backend.config.app \ --pool=prefork --events \ --concurrency=${CONCURRENCY} \ --maxtasksperchild=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} --without-gossip \ --without-mingle \ --heartbeat-interval 10 \ --hostname "${SWH_WORKER_INSTANCE}@%h" ;; esac diff --git a/services/swh-objstorage/entrypoint.sh b/services/swh-objstorage/entrypoint.sh index 182afe6..a0ef87b 100755 --- a/services/swh-objstorage/entrypoint.sh +++ b/services/swh-objstorage/entrypoint.sh @@ -1,30 +1,22 @@ #!/bin/bash set -e -echo Step 1 -if [[ -d /src ]] ; then - echo Yes - for srcrepo in /src/swh-* ; do - echo installing $srcrepo - pushd $srcrepo - pip install -e . - popd - done -fi +source /srv/softwareheritage/utils/pyutils.sh +setup_pip echo Installed Python packages: pip list if [ "$1" = 'shell' ] ; then exec bash -i else echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ --reload \ --timeout 3600 \ swh.objstorage.api.wsgi fi diff --git a/services/swh-scheduler-api/entrypoint.sh b/services/swh-scheduler-api/entrypoint.sh index f9bab4b..e023451 100755 --- a/services/swh-scheduler-api/entrypoint.sh +++ b/services/swh-scheduler-api/entrypoint.sh @@ -1,41 +1,32 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; "swh-scheduler") exec $@ ;; *) wait_pgsql echo Setup the swh-scheduler API database PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init scheduler \ --db-name ${POSTGRES_DB} echo Starting the swh-scheduler API server exec gunicorn --bind 0.0.0.0:5008 \ --log-level DEBUG \ --reload \ --timeout 3600 \ swh.scheduler.api.wsgi esac diff --git a/services/swh-scheduler-worker/entrypoint.sh b/services/swh-scheduler-worker/entrypoint.sh index 50a962d..697cd0f 100755 --- a/services/swh-scheduler-worker/entrypoint.sh +++ b/services/swh-scheduler-worker/entrypoint.sh @@ -1,30 +1,21 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) wait_pgsql echo "Starting the swh-scheduler $1" exec wait-for-it amqp:5672 -s --timeout=0 -- swh-scheduler --log-level ${LOGLEVEL} -C /scheduler.yml $@ ;; esac diff --git a/services/swh-storage-listener/entrypoint.sh b/services/swh-storage-listener/entrypoint.sh index 44e01d9..135fbef 100755 --- a/services/swh-storage-listener/entrypoint.sh +++ b/services/swh-storage-listener/entrypoint.sh @@ -1,30 +1,21 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) wait_pgsql echo "Starting swh-storage-listener" exec wait-for-it kafka:9092 -s --timeout=0 -- python3 -m swh.storage.listener ;; esac diff --git a/services/swh-storage-replayer/entrypoint.sh b/services/swh-storage-replayer/entrypoint.sh index 5c0dc75..47a7e88 100755 --- a/services/swh-storage-replayer/entrypoint.sh +++ b/services/swh-storage-replayer/entrypoint.sh @@ -1,35 +1,28 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) wait_pgsql echo Setup the database PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init storage \ --db-name ${POSTGRES_DB} echo Starting the swh-storage Kafka storage replayer - exec swh-journal replay --broker kafka --prefix swh.journal.objects \ + exec swh-journal replay \ + --broker kafka \ + --prefix swh.journal.objects \ --consumer-id swh.storage.replica ;; esac diff --git a/services/swh-storage/entrypoint.sh b/services/swh-storage/entrypoint.sh index 5bdb942..bd69924 100755 --- a/services/swh-storage/entrypoint.sh +++ b/services/swh-storage/entrypoint.sh @@ -1,38 +1,29 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; *) wait_pgsql echo Setup the database PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init storage \ --db-name ${POSTGRES_DB} echo Starting the swh-storage API server exec gunicorn --bind 0.0.0.0:5002 \ --reload \ --log-level DEBUG \ --timeout 3600 \ swh.storage.api.wsgi ;; esac diff --git a/services/swh-vault/entrypoint.sh b/services/swh-vault/entrypoint.sh index 7024c8a..26ac367 100755 --- a/services/swh-vault/entrypoint.sh +++ b/services/swh-vault/entrypoint.sh @@ -1,47 +1,38 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip source /srv/softwareheritage/utils/pgsql.sh - setup_pgsql case "$1" in "shell") exec bash -i ;; "worker") echo Starting the swh-vault Celery worker for exec python -m celery worker \ --app=swh.scheduler.celery_backend.config.app \ --pool=prefork --events \ --concurrency=${CONCURRENCY:-1} \ --maxtasksperchild=${MAX_TASKS_PER_CHILD:-10} \ -Ofair --loglevel=${LOGLEVEL:-INFO} --without-gossip \ --without-mingle --without-heartbeat \ --hostname "vault@%h" ;; "server") # ensure the pathslicing root dir for the cache exists mkdir -p /srv/softwareheritage/vault wait_pgsql echo Setup the swh-vault API database PGPASSWORD=${POSTGRES_PASSWORD} swh-db-init vault \ --db-name ${POSTGRES_DB} echo Starting the swh-vault API server exec swh-vault -C ${SWH_CONFIG_FILENAME} esac diff --git a/services/swh-web/entrypoint.sh b/services/swh-web/entrypoint.sh index cf4251e..ddd6ad7 100755 --- a/services/swh-web/entrypoint.sh +++ b/services/swh-web/entrypoint.sh @@ -1,32 +1,23 @@ #!/bin/bash set -e -if [[ -d /src ]] ; then - for srcrepo in /src/swh-* ; do - pushd $srcrepo - echo installing $srcrepo - pip install -e . - popd - done -fi - -echo Installed Python packages: -pip list +source /srv/softwareheritage/utils/pyutils.sh +setup_pip case "$1" in "shell") exec bash -i ;; *) - echo "Migrating db" + echo "Migrating db using ${DJANGO_SETTINGS_MODULE}" django-admin migrate --settings=${DJANGO_SETTINGS_MODULE} echo "Creating admin user" - echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('admin', 'admin@swh-web.org', 'admin')" | python3 -m swh.web.manage shell + echo "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('admin', 'admin@swh-web.org', 'admin')" | python3 -m swh.web.manage shell || true echo "starting the swh-web server" exec gunicorn --bind 0.0.0.0:5004 \ --timeout 3600 \ swh.web.wsgi esac diff --git a/utils/pyutils.sh b/utils/pyutils.sh new file mode 100755 index 0000000..01fb18b --- /dev/null +++ b/utils/pyutils.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +setup_pip () { + export PATH=${HOME}/.local/bin:${PATH} + + if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pip install -e $srcrepo + done + fi + + echo Installed Python packages: + pip list + +}