diff --git a/base-services.yml b/base-services.yml index 60e86fb..6839162 100644 --- a/base-services.yml +++ b/base-services.yml @@ -1,138 +1,168 @@ version: "3.7" services: memcache: image: memcached deploy: replicas: 1 db-storage: - image: postgres:12 + image: postgres:13 deploy: mode: global placement: constraints: - node.role == manager command: ['-c', 'shared_buffers=4GB', '-c', 'effective_cache_size=4GB', '-c', 'random_page_cost=1.5', '-c', 'max_wal_size=4GB'] environment: - POSTGRES_PASSWORD_FILE: /run/secrets/postgres-password + POSTGRES_PASSWORD_FILE: /run/secrets/postgres-storage-password POSTGRES_USER: swh POSTGRES_DB: # unset POSTGRES_DB: we're handling db creation ourselves in the backend volumes: - - "storage:/var/lib/postgresql/data:rw,Z" + - "storage-db:/var/lib/postgresql/data:rw,Z" secrets: - - source: postgres-password + - source: postgres-storage-password + uid: '999' + mode: 0400 + + db-web: + image: postgres:13 + deploy: + mode: global + placement: + constraints: + - node.role == manager + command: ['-c', 'shared_buffers=4GB', '-c', 'effective_cache_size=4GB', '-c', 'random_page_cost=1.5', '-c', 'max_wal_size=4GB'] + environment: + POSTGRES_PASSWORD_FILE: /run/secrets/postgres-web-password + POSTGRES_USER: swh + POSTGRES_DB: + # unset POSTGRES_DB: we're handling db creation ourselves in the backend + volumes: + - "web-db:/var/lib/postgresql/data:rw,Z" + secrets: + - source: postgres-web-password uid: '999' mode: 0400 web: image: softwareheritage/web:${SWH_IMAGE_TAG:-latest} configs: - source: web target: /etc/softwareheritage/config.yml command: serve environment: PORT: "5004" + PGHOST: db-web + PGUSER: swh + POSTGRES_DB: swh-web depends_on: + - db-web - memcache + secrets: + - source: postgres-web-password + mode: 0400 objstorage: image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} deploy: placement: constraints: - node.role == manager volumes: - "objstorage:/srv/softwareheritage/objects:rw,Z" configs: - source: objstorage target: /etc/softwareheritage/config.yml environment: PORT: "5003" STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 command: objstorage storage: image: softwareheritage/base:${SWH_IMAGE_TAG:-latest} configs: - source: storage target: /etc/softwareheritage/config.yml environment: PGHOST: db-storage PGUSER: swh POSTGRES_DB: swh-storage PORT: "5002" STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 command: storage depends_on: - db-storage secrets: - - source: postgres-password + - source: postgres-storage-password mode: 0400 nginx: image: nginx configs: - source: nginx target: /etc/nginx/nginx.conf ports: - "5081:5081/tcp" deploy: placement: constraints: - node.role == manager prometheus: image: prom/prometheus depends_on: - prometheus-statsd-exporter command: # Needed for the reverse-proxy - "--web.external-url=/prometheus" - "--config.file=/etc/prometheus/prometheus.yml" volumes: - "./conf/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z" deploy: mode: global prometheus-statsd-exporter: image: prom/statsd-exporter command: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" volumes: - "./conf/prometheus-statsd-mapping.yml:/etc/prometheus/statsd-mapping.yml:ro,Z" grafana: image: grafana/grafana depends_on: - prometheus environment: GF_SERVER_ROOT_URL: http://localhost:5081/grafana volumes: - "./conf/grafana/provisioning:/etc/grafana/provisioning:ro,Z" - "./conf/grafana/dashboards:/var/lib/grafana/dashboards:rw,Z" volumes: objstorage: - storage: + storage-db: + web-db: secrets: - postgres-password: + postgres-storage-password: + external: true + postgres-web-password: external: true configs: storage: file: conf/storage.yml name: storage objstorage: file: conf/objstorage.yml name: objstorage nginx: file: conf/nginx.conf name: nginx web: file: conf/web.yml name: web diff --git a/conf/web.yml b/conf/web.yml index e0a0b1a..d29a2c9 100644 --- a/conf/web.yml +++ b/conf/web.yml @@ -1,69 +1,71 @@ storage: cls: remote args: url: http://storage:5002/ timeout: 1 objstorage: cls: remote args: url: http://objstorage:5003/ indexer_storage: cls: remote args: url: http://indexer-storage:5007/ scheduler: cls: remote args: url: http://scheduler:5008/ vault: cls: remote args: url: http://vault:5005/ deposit: private_api_url: https://deposit:5006/1/private/ private_api_user: swhworker private_api_password: '' allowed_hosts: - appserver debug: yes serve_assets: yes +production_db: + name: postgresql:///?service=swh + throttling: cache_uri: 127.0.0.1:11211 scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_api_origin_search: limiter_rate: default: 70/m exempted_networks: - 0.0.0.0/0 swh_api_origin_visit_latest: limiter_rate: default: 700/m exempted_networks: - 0.0.0.0/0 swh_vault_cooking: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_save_origin: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 search: {} - diff --git a/images/Dockerfile b/images/Dockerfile index a7ff69d..8adfe60 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -1,116 +1,119 @@ ARG debianversion=buster FROM debian:${debianversion:-buster} as swh-common LABEL maintainer="Software Heritage " ENV PROJECT_NAME swh-base RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ apt-transport-https \ curl \ lsb-release \ wait-for-it RUN echo deb http://deb.debian.org/debian/ $(lsb_release -sc)-backports main \ > /etc/apt/sources.list.d/backports.list RUN echo deb [trusted=yes] https://debian.softwareheritage.org/ $(lsb_release -sc)-swh main \ > /etc/apt/sources.list.d/softwareheritage.list RUN mkdir /etc/softwareheritage RUN mkdir -p /var/run/gunicorn/swh RUN mkdir -p /var/lib/swh RUN mkdir -p /srv/softwareheritage/objects ENV SWH_CONFIG_FILENAME=/etc/softwareheritage/config.yml ENV LC_ALL=C.UTF-8 ################################## # BASE services ################################## FROM swh-common as swh-base ARG SWH_VER ENV SWH_VER=${SWH_VER} RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ -t $(lsb_release -sc)-backports \ --no-install-recommends \ gunicorn \ postgresql-client \ python3-dulwich \ python3-swh.journal \ python3-swh.objstorage \ # python3-swh.objstorage.cloud \ # python3-swh.objstorage.rados \ python3-swh.scheduler \ python3-swh.storage \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* ## still missing: vault, deposit COPY conf/logconfig.ini /etc/gunicorn/logconfig.ini COPY conf/gunicorn.cfg /etc/gunicorn/swh.cfg COPY tools/*.sh /srv/softwareheritage/utils/ COPY tools/*.sql /srv/softwareheritage/utils/ RUN chmod +x /srv/softwareheritage/utils/*.sh COPY base/entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] ################################## # WEB ################################## FROM swh-common as swh-web ARG SWH_VER ENV SWH_VER=${SWH_VER} ENV DJANGO_SETTINGS_MODULE=swh.web.settings.production RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y --no-install-recommends \ -t $(lsb_release -sc)-backports \ gunicorn \ python3-magic \ python3-swh.web \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* COPY conf/logconfig.ini /etc/gunicorn/logconfig.ini COPY conf/gunicorn.cfg /etc/gunicorn/swh.cfg +COPY tools/*.sh /srv/softwareheritage/utils/ +COPY tools/*.sql /srv/softwareheritage/utils/ +RUN chmod +x /srv/softwareheritage/utils/*.sh COPY web/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ################################## # Mirror (replayer) ################################## FROM swh-common as swh-replayer ARG SWH_VER ENV SWH_VER=${SWH_VER} RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y \ -t $(lsb_release -sc)-backports \ --no-install-recommends \ python3-swh.journal \ python3-swh.objstorage \ python3-swh.objstorage.replayer \ python3-swh.storage \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* COPY tools/*.sh /srv/softwareheritage/utils/ COPY tools/*.sql /srv/softwareheritage/utils/ RUN chmod +x /srv/softwareheritage/utils/*.sh COPY replayer/entrypoint.sh / ENTRYPOINT ["/entrypoint.sh"] diff --git a/images/web/entrypoint.sh b/images/web/entrypoint.sh index 00b42be..e282405 100755 --- a/images/web/entrypoint.sh +++ b/images/web/entrypoint.sh @@ -1,50 +1,51 @@ #!/bin/bash set -e -create_admin_script=" -from django.contrib.auth import get_user_model; - -username = 'admin'; -password = 'admin'; -email = 'admin@swh-web.org'; - -User = get_user_model(); - -if not User.objects.filter(username = username).exists(): - User.objects.create_superuser(username, email, password); -" +source /srv/softwareheritage/utils/pgsql.sh -# generate the config file from the 'template' -if [ -f /etc/softwareheritage/config.yml.tmpl ]; then - # I know... I know! - eval "echo \"`cat /etc/softwareheritage/config.yml.tmpl`\"" > \ - /etc/softwareheritage/config.yml +# generate the pgservice file if any +if [ -f /run/secrets/postgres-password ]; then + POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password + setup_pgsql fi if [ "$1" = 'shell' ] ; then shift if (( $# == 0)); then exec bash -i else "$@" fi else + create_admin_script=" +from django.contrib.auth import get_user_model; + +username = 'admin'; +password = 'admin'; +email = 'admin@swh-web.org'; + +User = get_user_model(); + +if not User.objects.filter(username = username).exists(): + User.objects.create_superuser(username, email, password); +" echo "Migrating db using ${DJANGO_SETTINGS_MODULE}" django-admin migrate --settings=${DJANGO_SETTINGS_MODULE} echo "Creating admin user" echo "$create_admin_script" | python3 -m swh.web.manage shell echo "starting the swh-web server" mkdir -p /var/run/gunicorn/swh/web - exec gunicorn3 \ + gunicorn3 \ --bind 0.0.0.0:5004 \ --bind unix:/var/run/gunicorn/swh/web/sock \ --threads 2 \ --workers 2 \ --timeout 3600 \ 'django.core.wsgi:get_wsgi_application()' - + # give some time to log in and check a few things before dying + sleep 180 fi