diff --git a/images/Dockerfile b/images/Dockerfile index bdf22ab..bd92b6d 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -1,130 +1,131 @@ ARG debianversion=buster FROM debian:${debianversion:-buster} as swh-common LABEL maintainer="Software Heritage " ENV PROJECT_NAME swh-base RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ apt-transport-https \ curl \ lsb-release \ wait-for-it RUN echo deb http://deb.debian.org/debian/ $(lsb_release -sc)-backports main \ > /etc/apt/sources.list.d/backports.list RUN echo deb [trusted=yes] https://debian.softwareheritage.org/ $(lsb_release -sc)-swh main \ > /etc/apt/sources.list.d/softwareheritage.list RUN mkdir /etc/softwareheritage RUN mkdir -p /var/run/gunicorn/swh RUN mkdir -p /var/lib/swh -RUN mkdir -p /srv/softwareheritage/objects +RUN mkdir -p /srv/softwareheritage ENV SWH_CONFIG_FILENAME=/etc/softwareheritage/config.yml ENV LC_ALL=C.UTF-8 ################################## # BASE services ################################## FROM swh-common as swh-base ARG SWH_VER ENV SWH_VER=${SWH_VER} RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ -t $(lsb_release -sc)-backports \ --no-install-recommends \ gunicorn \ postgresql-client \ python3-dulwich \ python3-swh.journal \ python3-swh.objstorage \ python3-swh.objstorage.cloud \ # python3-swh.objstorage.rados \ python3-swh.scheduler \ python3-swh.storage \ zstd \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* ## still missing: vault, deposit COPY conf/logconfig.ini /etc/gunicorn/logconfig.ini COPY conf/gunicorn.cfg /etc/gunicorn/swh.cfg COPY tools/*.sh /srv/softwareheritage/utils/ COPY tools/*.sql /srv/softwareheritage/utils/ +COPY tools/*.py /srv/softwareheritage/utils/ RUN chmod +x /srv/softwareheritage/utils/*.sh COPY base/entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] ################################## # WEB ################################## FROM swh-common as swh-web ARG SWH_VER ENV SWH_VER=${SWH_VER} ENV DJANGO_SETTINGS_MODULE=swh.web.settings.production RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y --no-install-recommends \ -t $(lsb_release -sc)-backports \ gunicorn \ postgresql-client \ python3-magic \ python3-swh.web \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* COPY conf/logconfig.ini /etc/gunicorn/logconfig.ini COPY conf/gunicorn.cfg /etc/gunicorn/swh.cfg COPY tools/*.sh /srv/softwareheritage/utils/ COPY tools/*.sql /srv/softwareheritage/utils/ RUN chmod +x /srv/softwareheritage/utils/*.sh COPY web/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ################################## # Mirror (replayer) ################################## FROM swh-common as swh-replayer ARG SWH_VER ENV SWH_VER=${SWH_VER} RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ -t $(lsb_release -sc)-backports \ --no-install-recommends \ python3-swh.journal \ python3-swh.objstorage \ python3-swh.objstorage.replayer \ python3-swh.storage \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* COPY tools/*.sh /srv/softwareheritage/utils/ COPY tools/*.sql /srv/softwareheritage/utils/ RUN chmod +x /srv/softwareheritage/utils/*.sh COPY replayer/entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] ### # Test image ### FROM swh-base as swh-test COPY restore_kafka.py / #ENTRYPOINT ["python3", "-u", "/restore_kafka.py"] diff --git a/images/base/entrypoint.sh b/images/base/entrypoint.sh old mode 100755 new mode 100644 index d6e79af..72d2f2a --- a/images/base/entrypoint.sh +++ b/images/base/entrypoint.sh @@ -1,61 +1,66 @@ #!/bin/bash set -e source /srv/softwareheritage/utils/pgsql.sh # generate the config file from the 'template' if [ -f /etc/softwareheritage/config.yml.tmpl ]; then # I know... I know! eval "echo \"`cat /etc/softwareheritage/config.yml.tmpl`\"" > \ /etc/softwareheritage/config.yml fi # generate the pgservice file if any if [ -f /run/secrets/postgres-password ]; then POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password setup_pgsql fi +# ensure root dirs used for pathslicer objstorage exist (if any) +if [ -v SWH_CONFIG_FILENAME ]; then + python3 /srv/softwareheritage/utils/init_pathslicer_root.py +fi + # For debugging purpose echo "### CONFIG FILE ###" cat /etc/softwareheritage/config.yml | grep -v password || true echo "###################" echo "Arguments: $@" case "$1" in "shell") exec bash -i ;; *) if [ -v POSTGRES_DB ]; then wait_pgsql template1 echo Database setup if ! check_pgsql_db_created; then echo Creating database and extensions... swh db create --db-name ${POSTGRES_DB} $1 fi echo Initializing the database... echo " step 1: init-admin" swh db init-admin --db-name ${POSTGRES_DB} $1 echo " step 2: init" swh db init --flavor ${FLAVOR:-default} $1 echo " step 3: upgrade" swh db upgrade --non-interactive $1 fi echo "Starting the SWH $1 RPC server" exec gunicorn3 \ --bind 0.0.0.0:${PORT:-5000} \ --bind unix:/var/run/gunicorn/swh/$1.sock \ --threads ${GUNICORN_THREADS:-4} \ --workers ${GUNICORN_WORKERS:-16} \ --log-level "${LOG_LEVEL:-WARNING}" \ --timeout ${GUNICORN_TIMEOUT:-3600} \ --statsd-host=prometheus-statsd-exporter:9125 \ --statsd-prefix=service.app.$1 \ "swh.$1.api.server:make_app_from_configfile()" ;; esac diff --git a/images/tools/init_pathslicer_root.py b/images/tools/init_pathslicer_root.py new file mode 100755 index 0000000..2bec039 --- /dev/null +++ b/images/tools/init_pathslicer_root.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +# very simple tool to ensure any pathslicer based objstorage found in the SWH +# config file do have its root directory created + +import os +from swh.core.config import read as config_read + + +def ensure_pathslicer_root(cfg): + if cfg.get("cls") == "pathslicing": + root = cfg.get("root") + if root: + ensure_root(root) + else: + for k, v in cfg.items(): + if isinstance(v, dict): + ensure_pathslicer_root(v) + + +def ensure_root(root): + print(f"Create directorty {root}") + try: + os.makedirs(root, exist_ok=True) + except PermissionError as exc: + print(f"Failed to create directory: {exc}") + + +config_file = os.environ.get("SWH_CONFIG_FILENAME") +cfg = config_read(config_file) +ensure_pathslicer_root(cfg)