diff --git a/images/base/Dockerfile b/images/base/Dockerfile index 2127cae..81cf3e7 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -1,50 +1,51 @@ ARG debianversion=buster FROM debian:${debianversion:-buster} LABEL maintainer="Software Heritage " ENV PROJECT_NAME swh-base RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ apt-transport-https curl lsb-release RUN echo deb http://deb.debian.org/debian/ $(lsb_release -sc)-backports main \ > /etc/apt/sources.list.d/backports.list RUN echo deb [trusted=yes] https://debian.softwareheritage.org/ $(lsb_release -sc)-swh main \ > /etc/apt/sources.list.d/softwareheritage.list RUN mkdir /etc/softwareheritage RUN mkdir -p /var/run/gunicorn/swh RUN mkdir -p /srv/softwareheritage/objects ENV SWH_CONFIG_FILENAME=/etc/softwareheritage/config.yml ENV LC_ALL=C.UTF-8 ARG SWH_VER=unknown RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ -t $(lsb_release -sc)-backports \ gunicorn \ postgresql-client \ python3-dulwich \ python3-swh.objstorage \ python3-swh.objstorage.replayer \ python3-swh.model \ python3-swh.scheduler \ python3-swh.storage \ python3-swh.journal \ wait-for-it \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* COPY conf/logconfig.ini /etc/gunicorn/logconfig.ini COPY conf/gunicorn.cfg /etc/gunicorn/swh.cfg COPY tools/*.sh /srv/softwareheritage/utils/ +COPY tools/*.sql /srv/softwareheritage/utils/ RUN chmod +x /srv/softwareheritage/utils/*.sh COPY base/entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/images/base/entrypoint.sh b/images/base/entrypoint.sh index a1a00eb..3ef53df 100755 --- a/images/base/entrypoint.sh +++ b/images/base/entrypoint.sh @@ -1,69 +1,78 @@ #!/bin/bash set -e source /srv/softwareheritage/utils/pgsql.sh # generate the config file from the 'template' if [ -f /etc/softwareheritage/config.yml.tmpl ]; then # I know... I know! eval "echo \"`cat /etc/softwareheritage/config.yml.tmpl`\"" > \ /etc/softwareheritage/config.yml fi # generate the pgservice file if any if [ -f /run/secrets/postgres-password ]; then echo 'Available secrets:' ls -l /run/secrets/ setup_pgsql fi # For debugging purpose echo "### CONFIG FILE ###" cat /etc/softwareheritage/config.yml echo "###################" -case "$1" in - "shell") - exec bash -i - ;; - "db-init") - wait_pgsql - echo "Initialize the SWH database" - exec swh db init - ;; - "graph-replayer") - echo "Starting the SWH mirror graph replayer" - exec sh -c "trap : TERM INT; while :; do swh --log-level ${LOG_LEVEL:-WARNING} storage replay; done" - ;; - "content-replayer") - echo "Starting the SWH mirror content replayer" - exec sh -c "trap : TERM INT; while :; do swh --log-level ${LOG_LEVEL:-WARNING} objstorage replay; done" - ;; - "objstorage") - echo "Starting the SWH $1 RPC server" - exec gunicorn3 \ - --bind 0.0.0.0:${PORT:-5000} \ - --bind unix:/var/run/gunicorn/swh/$1.sock \ - --worker-class aiohttp.worker.GunicornWebWorker \ - --threads 4 \ - --workers 2 \ - --log-level "${LOG_LEVEL:-WARNING}" \ - --timeout 3600 \ - "swh.$1.api.server:make_app_from_configfile()" - ;; - *) - wait_pgsql - echo "Starting the SWH $1 RPC server" - swh db init || echo "swh db init failed, database is probably already initialized; ignored" - exec gunicorn3 \ - --bind 0.0.0.0:${PORT:-5000} \ - --bind unix:/var/run/gunicorn/swh/$1.sock \ - --threads 4 \ - --workers 2 \ - --log-level "${LOG_LEVEL:-WARNING}" \ - --timeout 3600 \ - "swh.$1.api.server:make_app_from_configfile()" - ;; -esac +echo "Arguments: $@" + +while (( "$#" )); do + case "$1" in + "shell") + exec bash -i + ;; + "db-init") + wait_pgsql + echo "Initialize the SWH database" + exec swh db init + ;; + "fix-mirror") + db_init + echo "Fixing the database for the mirror use case (drop constraints, if any)" + fix_storage_for_mirror + ;; + "graph-replayer") + echo "Starting the SWH mirror graph replayer" + exec sh -c "trap : TERM INT; while :; do swh --log-level ${LOG_LEVEL:-WARNING} storage replay; done" + ;; + "content-replayer") + echo "Starting the SWH mirror content replayer" + exec sh -c "trap : TERM INT; while :; do swh --log-level ${LOG_LEVEL:-WARNING} objstorage replay; done" + ;; + "objstorage") + echo "Starting the SWH $1 RPC server" + exec gunicorn3 \ + --bind 0.0.0.0:${PORT:-5000} \ + --bind unix:/var/run/gunicorn/swh/$1.sock \ + --worker-class aiohttp.worker.GunicornWebWorker \ + --threads 4 \ + --workers 2 \ + --log-level "${LOG_LEVEL:-WARNING}" \ + --timeout 3600 \ + "swh.$1.api.server:make_app_from_configfile()" + ;; + *) + db_init + echo "Starting the SWH $1 RPC server" + exec gunicorn3 \ + --bind 0.0.0.0:${PORT:-5000} \ + --bind unix:/var/run/gunicorn/swh/$1.sock \ + --threads 4 \ + --workers 2 \ + --log-level "${LOG_LEVEL:-WARNING}" \ + --timeout 3600 \ + "swh.$1.api.server:make_app_from_configfile()" + ;; + esac + shift +done diff --git a/images/tools/fix-storage.sql b/images/tools/fix-storage.sql new file mode 100644 index 0000000..32dd145 --- /dev/null +++ b/images/tools/fix-storage.sql @@ -0,0 +1,11 @@ +alter table skipped_content drop constraint if exists skipped_content_origin_fkey; +alter table revision drop constraint if exists revision_author_fkey; +alter table revision drop constraint if exists revision_committer_fkey; +alter table revision_history drop constraint if exists revision_history_id_fkey; +alter table snapshot_branches drop constraint if exists snapshot_branches_snapshot_id_fkey; +alter table snapshot_branches drop constraint if exists snapshot_branches_branch_id_fkey; +alter table origin_visit drop constraint if exists origin_visit_origin_fkey; +alter table origin_visit_status drop constraint if exists origin_visit_status_origin_visit_fkey; +alter table release drop constraint if exists release_author_fkey; +alter table raw_extrinsic_metadata drop constraint if exists raw_extrinsic_metadata_authority_fkey; +alter table raw_extrinsic_metadata drop constraint if exists raw_extrinsic_metadata_fetcher_fkey; diff --git a/images/tools/pgsql.sh b/images/tools/pgsql.sh index a453bf5..89b8683 100755 --- a/images/tools/pgsql.sh +++ b/images/tools/pgsql.sh @@ -1,20 +1,35 @@ #!/bin/bash setup_pgsql () { echo "${PGHOST}:5432:${PGDATABASE:-swh}:${PGUSER:-swh}:$(cat /run/secrets/postgres-password)" \ > ~/.pgpass cat > ~/.pg_service.conf < /dev/null 2> /dev/null; do sleep 1; done } + +db_init () { + wait_pgsql + if version=$(psql postgresql:///?service=swh -qtA \ + -c "select version from dbversion order by version desc limit 1" 2>/dev/null) ; then + echo "Database seems already initialized at version $version" + else + echo "Database seems not initialized yet; initialize it" + swh db init + fi +} + +fix_storage_for_mirror () { + psql postgresql:///?service=swh -f /srv/softwareheritage/utils/fix-storage.sql +}