diff --git a/Dockerfile b/Dockerfile new file mode 100644 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM python:3.6 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && apt-get upgrade -y && \ + apt-get install -y \ + libapr1-dev \ + libaprutil1-dev \ + libpq-dev \ + libsvn-dev \ + libsystemd-dev \ + postgresql-client \ + wait-for-it && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --upgrade pip setuptools wheel +RUN pip install swh-deposit swh-indexer swh-journal swh-lister swh-loader-debian \ + swh-loader-dir swh-loader-git swh-loader-mercurial swh-loader-pypi \ + swh-loader-svn swh-loader-tar swh-storage swh-objstorage \ + swh-scheduler swh-vault swh-web + +RUN pip install gunicorn psycopg2-binary # to make pip silent + +COPY services/swh-deposit/entrypoint.sh /swh-deposit/entrypoint.sh +COPY services/swh-indexer-journal-client/entrypoint.sh /swh-indexer-journal-client/entrypoint.sh +COPY services/swh-indexer-storage/entrypoint.sh /swh-indexer-storage/entrypoint.sh +COPY services/swh-indexer-worker/entrypoint.sh /swh-indexer-worker/entrypoint.sh +COPY services/swh-journal-client/entrypoint.sh /swh-journal-client/entrypoint.sh +COPY services/swh-journal-publisher/entrypoint.sh /swh-journal-publisher/entrypoint.sh +COPY services/swh-listers-worker/entrypoint.sh /swh-listers-worker/entrypoint.sh +COPY services/swh-loaders-worker/entrypoint.sh /swh-loaders-worker/entrypoint.sh +COPY services/swh-objstorage/entrypoint.sh /swh-objstorage/entrypoint.sh +COPY services/swh-scheduler-api/entrypoint.sh /swh-scheduler-api/entrypoint.sh +COPY services/swh-scheduler-worker/entrypoint.sh /swh-scheduler-worker/entrypoint.sh +COPY services/swh-storage/entrypoint.sh /swh-storage/entrypoint.sh +COPY services/swh-storage-listener/entrypoint.sh /swh-storage-listener/entrypoint.sh +COPY services/swh-vault/entrypoint.sh /swh-vault/entrypoint.sh +COPY services/swh-web/entrypoint.sh /swh-web/entrypoint.sh + +COPY services/swh-journal-client/client.py /swh-journal-client/ + +RUN useradd -ms /bin/bash swh + +RUN mkdir -p /srv/softwareheritage/objects + diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -69,9 +69,10 @@ POSTGRES_DB: swh-scheduler swh-scheduler-api: - image: swh/scheduler-api - build: ./dockerfiles/swh-scheduler-api + image: swh/stack + build: ./ env_file: ./env/scheduler.env + entrypoint: /swh-scheduler-api/entrypoint.sh environment: SWH_CONFIG_FILENAME: /scheduler.yml SWH_SCHEDULER_CONFIG_FILE: /scheduler.yml @@ -83,8 +84,9 @@ - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-listener: - image: swh/scheduler-worker - build: ./dockerfiles/swh-scheduler-worker + image: swh/stack + build: ./ + entrypoint: /swh-scheduler-worker/entrypoint.sh env_file: ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml @@ -97,8 +99,9 @@ - "./conf/scheduler.yml:/scheduler.yml:ro" swh-scheduler-runner: - image: swh/scheduler-worker - build: ./dockerfiles/swh-scheduler-worker + image: swh/stack + build: ./ + entrypoint: /swh-scheduler-worker/entrypoint.sh env_file: ./env/scheduler.env environment: SWH_CONFIG_FILENAME: /scheduler.yml @@ -119,8 +122,9 @@ POSTGRES_DB: swh-storage swh-storage: - build: ./dockerfiles/swh-storage - image: swh/storage + build: ./ + image: swh/stack + entrypoint: /swh-storage/entrypoint.sh ports: - 5002:5002 depends_on: @@ -133,8 +137,9 @@ # Object storage swh-objstorage: - build: ./dockerfiles/swh-objstorage - image: swh/objstorage + build: ./ + image: swh/stack + entrypoint: /swh-objstorage/entrypoint.sh ports: - 5003:5003 environment: @@ -151,8 +156,9 @@ POSTGRES_DB: swh-idx-storage swh-idx-storage: - build: ./dockerfiles/swh-indexer-storage - image: swh/indexer-storage + build: ./ + image: swh/stack + entrypoint: /swh-indexer-storage/entrypoint.sh ports: - 5007:5007 depends_on: @@ -169,9 +175,9 @@ # Web interface swh-web: - build: ./dockerfiles/swh-web - image: swh/web - + build: ./ + image: swh/stack + entrypoint: /swh-web/entrypoint.sh ports: - 5004:5004 depends_on: @@ -191,8 +197,9 @@ env_file: ./env/deposit.env swh-deposit: - build: ./dockerfiles/swh-deposit - image: swh/deposit + build: ./ + entrypoint: /swh-deposit/entrypoint.sh + image: swh/stack ports: - 5006:5006 depends_on: @@ -211,8 +218,9 @@ POSTGRES_DB: swh-vault swh-vault-api: - build: ./dockerfiles/swh-vault - image: swh/vault + build: ./ + image: swh/stack + entrypoint: /swh-vault/entrypoint.sh env_file: ./env/vault.env command: server ports: @@ -226,8 +234,9 @@ - "./conf/vault-api.yml:/vault-api.yml:ro" swh-vault-worker: - build: ./dockerfiles/swh-vault - image: swh/vault + build: ./ + image: swh/stack + entrypoint: /swh-vault/entrypoint.sh command: worker environment: SWH_CONFIG_FILENAME: /cooker.yml @@ -246,9 +255,11 @@ POSTGRES_PASSWORD: testpassword swh-lister: - image: swh/listers-worker - build: ./dockerfiles/swh-listers-worker + image: swh/stack + build: ./ + entrypoint: /swh-listers-worker/entrypoint.sh env_file: ./env/listers.env + user: swh environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 @@ -266,9 +277,11 @@ # Loader Celery workers swh-loader: - image: swh/loaders-worker - build: ./dockerfiles/swh-loaders-worker + image: swh/stack + build: ./ + entrypoint: /swh-loaders-worker/entrypoint.sh env_file: ./env/listers.env + user: swh environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 @@ -283,8 +296,10 @@ # Indexer Celery workers swh-indexer: - image: swh/indexer-worker - build: ./dockerfiles/swh-indexer-worker + image: swh/stack + build: ./ + user: swh + entrypoint: /swh-indexer-worker/entrypoint.sh env_file: ./env/indexers.env environment: STATSD_HOST: prometheus-statsd-exporter @@ -301,8 +316,9 @@ # Journal related swh-storage-listener: - image: swh/storage-listener - build: ./dockerfiles/swh-storage-listener + image: swh/stack + build: ./ + entrypoint: /swh-storage-listener/entrypoint.sh env_file: ./env/storage.env depends_on: - swh-storage-db @@ -311,8 +327,9 @@ - "./conf/storage_listener.yml:/etc/softwareheritage/storage/listener.yml:ro" swh-journal-publisher: - image: swh/journal-publisher - build: ./dockerfiles/swh-journal-publisher + image: swh/stack + build: ./ + entrypoint: /swh-journal-publisher/entrypoint.sh depends_on: - kafka - swh-storage-listener @@ -320,16 +337,18 @@ - "./conf/journal_publisher.yml:/etc/softwareheritage/journal/publisher.yml:ro" swh-journal-client: - image: swh/journal-client - build: ./dockerfiles/swh-journal-client + image: swh/stack + build: ./ + entrypoint: /swh-journal-client/entrypoint.sh depends_on: - swh-journal-publisher volumes: - "./conf/journal_client.yml:/etc/softwareheritage/journal/logger.yml:ro" swh-indexer-journal-client: - image: swh/indexer-journal-client - build: ./dockerfiles/swh-indexer-journal-client + image: swh/stack + build: ./ + entrypoint: /swh-indexer-journal-client/entrypoint.sh depends_on: - swh-journal-publisher - swh-scheduler-api diff --git a/dockerfiles/swh-deposit/Dockerfile b/dockerfiles/swh-deposit/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-deposit/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && apt-get upgrade -y && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install swh-deposit gunicorn -RUN pip install psycopg2-binary # to make pip silent - -COPY entrypoint.sh / -EXPOSE 5006 -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-journal-client/Dockerfile b/dockerfiles/swh-indexer-journal-client/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-indexer-journal-client/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y wait-for-it - -RUN pip install swh-indexer -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-storage/Dockerfile b/dockerfiles/swh-indexer-storage/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-indexer-storage/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-indexer gunicorn - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-worker/Dockerfile b/dockerfiles/swh-indexer-worker/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-indexer-worker/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-indexer - -RUN useradd -ms /bin/bash swh -USER swh - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-journal-client/Dockerfile b/dockerfiles/swh-journal-client/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-journal-client/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y wait-for-it - -RUN pip install swh-journal -COPY client.py / -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-journal-publisher/Dockerfile b/dockerfiles/swh-journal-publisher/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-journal-publisher/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y wait-for-it - -RUN pip install swh-journal -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-listers-worker/Dockerfile b/dockerfiles/swh-listers-worker/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-listers-worker/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-lister - -RUN useradd -ms /bin/bash swh - -COPY entrypoint.sh / - -USER swh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-loaders-worker/Dockerfile b/dockerfiles/swh-loaders-worker/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-loaders-worker/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y libapr1-dev libaprutil1-dev && \ - apt-get install -y libsvn-dev && \ - apt-get install -y wait-for-it - -RUN pip install \ - swh-loader-debian \ - swh-loader-dir \ - swh-loader-git \ - swh-loader-mercurial \ - swh-loader-pypi \ - swh-loader-svn \ - swh-loader-tar - - -RUN useradd -ms /bin/bash swh -COPY entrypoint.sh / - -USER swh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-objstorage/Dockerfile b/dockerfiles/swh-objstorage/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-objstorage/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y \ - libsystemd-dev - -RUN pip install -U pip setuptools wheel -RUN pip install swh-objstorage gunicorn - -COPY entrypoint.sh / - -RUN mkdir -p /srv/softwareheritage/objects - -ENTRYPOINT ["/entrypoint.sh"] - -EXPOSE 5003 diff --git a/dockerfiles/swh-scheduler-api/Dockerfile b/dockerfiles/swh-scheduler-api/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-scheduler-api/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-scheduler gunicorn - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] - -EXPOSE 5008 diff --git a/dockerfiles/swh-scheduler-worker/Dockerfile b/dockerfiles/swh-scheduler-worker/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-scheduler-worker/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client libpq-dev && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-scheduler --no-binary psycopg2 - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-storage-listener/Dockerfile b/dockerfiles/swh-storage-listener/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-storage-listener/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install swh-storage kafka_python -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-storage/Dockerfile b/dockerfiles/swh-storage/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-storage/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-storage gunicorn - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] - -EXPOSE 5002 diff --git a/dockerfiles/swh-vault/Dockerfile b/dockerfiles/swh-vault/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-vault/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y postgresql-client && \ - apt-get install -y wait-for-it - -RUN pip install --upgrade pip setuptools wheel -RUN pip install swh-vault - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/swh-web/Dockerfile b/dockerfiles/swh-web/Dockerfile deleted file mode 100644 --- a/dockerfiles/swh-web/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.6 - -RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && apt-get upgrade -y && \ - apt-get install -y libsystemd-dev && \ - apt-get install -y pandoc && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN pip install -U pip -RUN pip install swh-web -RUN pip install psycopg2-binary # to make pip silent -RUN pip install gunicorn -RUN python3 -m swh.web.manage migrate - -COPY entrypoint.sh / - -ENTRYPOINT ["/entrypoint.sh"] -WORKDIR /tmp/swh - -EXPOSE 5004 diff --git a/dockerfiles/swh-deposit/entrypoint.sh b/services/swh-deposit/entrypoint.sh rename from dockerfiles/swh-deposit/entrypoint.sh rename to services/swh-deposit/entrypoint.sh diff --git a/dockerfiles/swh-indexer-journal-client/entrypoint.sh b/services/swh-indexer-journal-client/entrypoint.sh rename from dockerfiles/swh-indexer-journal-client/entrypoint.sh rename to services/swh-indexer-journal-client/entrypoint.sh diff --git a/dockerfiles/swh-indexer-journal-client/journal_client.yml b/services/swh-indexer-journal-client/journal_client.yml rename from dockerfiles/swh-indexer-journal-client/journal_client.yml rename to services/swh-indexer-journal-client/journal_client.yml diff --git a/dockerfiles/swh-indexer-storage/entrypoint.sh b/services/swh-indexer-storage/entrypoint.sh rename from dockerfiles/swh-indexer-storage/entrypoint.sh rename to services/swh-indexer-storage/entrypoint.sh diff --git a/dockerfiles/swh-indexer-worker/entrypoint.sh b/services/swh-indexer-worker/entrypoint.sh rename from dockerfiles/swh-indexer-worker/entrypoint.sh rename to services/swh-indexer-worker/entrypoint.sh diff --git a/dockerfiles/swh-journal-client/client.py b/services/swh-journal-client/client.py rename from dockerfiles/swh-journal-client/client.py rename to services/swh-journal-client/client.py diff --git a/dockerfiles/swh-journal-client/entrypoint.sh b/services/swh-journal-client/entrypoint.sh rename from dockerfiles/swh-journal-client/entrypoint.sh rename to services/swh-journal-client/entrypoint.sh --- a/dockerfiles/swh-journal-client/entrypoint.sh +++ b/services/swh-journal-client/entrypoint.sh @@ -18,7 +18,7 @@ exec bash -i ;; *) - echo "Starting an swh-journal client" - exec wait-for-it kafka:9092 -s --timeout=0 -- python3 -m client + echo "Starting a swh-journal client" + exec wait-for-it kafka:9092 -s --timeout=0 -- python3 /swh-journal-client/client.py ;; esac diff --git a/dockerfiles/swh-journal-publisher/entrypoint.sh b/services/swh-journal-publisher/entrypoint.sh rename from dockerfiles/swh-journal-publisher/entrypoint.sh rename to services/swh-journal-publisher/entrypoint.sh diff --git a/dockerfiles/swh-listers-worker/entrypoint.sh b/services/swh-listers-worker/entrypoint.sh rename from dockerfiles/swh-listers-worker/entrypoint.sh rename to services/swh-listers-worker/entrypoint.sh diff --git a/dockerfiles/swh-loaders-worker/entrypoint.sh b/services/swh-loaders-worker/entrypoint.sh rename from dockerfiles/swh-loaders-worker/entrypoint.sh rename to services/swh-loaders-worker/entrypoint.sh diff --git a/dockerfiles/swh-objstorage/entrypoint.sh b/services/swh-objstorage/entrypoint.sh rename from dockerfiles/swh-objstorage/entrypoint.sh rename to services/swh-objstorage/entrypoint.sh diff --git a/dockerfiles/swh-scheduler-api/entrypoint.sh b/services/swh-scheduler-api/entrypoint.sh rename from dockerfiles/swh-scheduler-api/entrypoint.sh rename to services/swh-scheduler-api/entrypoint.sh diff --git a/dockerfiles/swh-scheduler-worker/entrypoint.sh b/services/swh-scheduler-worker/entrypoint.sh rename from dockerfiles/swh-scheduler-worker/entrypoint.sh rename to services/swh-scheduler-worker/entrypoint.sh diff --git a/dockerfiles/swh-storage-listener/entrypoint.sh b/services/swh-storage-listener/entrypoint.sh rename from dockerfiles/swh-storage-listener/entrypoint.sh rename to services/swh-storage-listener/entrypoint.sh diff --git a/dockerfiles/swh-storage/entrypoint.sh b/services/swh-storage/entrypoint.sh rename from dockerfiles/swh-storage/entrypoint.sh rename to services/swh-storage/entrypoint.sh diff --git a/dockerfiles/swh-vault/entrypoint.sh b/services/swh-vault/entrypoint.sh rename from dockerfiles/swh-vault/entrypoint.sh rename to services/swh-vault/entrypoint.sh diff --git a/dockerfiles/swh-web/entrypoint.sh b/services/swh-web/entrypoint.sh rename from dockerfiles/swh-web/entrypoint.sh rename to services/swh-web/entrypoint.sh --- a/dockerfiles/swh-web/entrypoint.sh +++ b/services/swh-web/entrypoint.sh @@ -19,6 +19,9 @@ exec bash -i ;; *) + echo "Migrating db" + django-admin migrate --settings=${DJANGO_SETTINGS_MODULE} + echo "starting the swh-web server" exec gunicorn --bind 0.0.0.0:5004 \ --timeout 3600 \