diff --git a/Makefile b/Makefile deleted file mode 100644 --- a/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -ALL_DOCKERFILES := $(wildcard dockerfiles/Dockerfile-*) -ALL_BUILD_TARGETS := $(subst dockerfiles/Dockerfile-,build-,$(ALL_DOCKERFILES)) - -all: build - -build: $(ALL_BUILD_TARGETS) - -clean: - # Discard existing volumes - docker-compose down --volumes - -run: build - # Runs containers in the foreground - docker-compose up - -build-%: dockerfiles/Dockerfile-% - @echo "" - @echo "+----------------------------------------------------+" - @printf '| %-50s |\n' "Building $(subst build-,,$@)" - @echo "+----------------------------------------------------+" - @echo "" - docker build -f $< -t $(subst build-,,$@) $(BUILD_CONTEXT) .. diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -8,16 +8,68 @@ The end goal is to smooth the contributors/developers workflow. Focus on coding, not configuring! +## Dependencies + +This uses docker with docker-compose, so ensure you have a working +docker environment and docker-compose is installed. + ## How to use +Initialise the database with: + +``` +docker-compose run swh-storage init +``` + +then start the services with: + ``` -make run +docker-compose up ``` -This will build docker images and run them using docker-compose. +This will build docker images and run them. Press Ctrl-C when you want to stop it. +Note: the db initialization process is a manual step for now because it +is not yet "idempotent", but (hopefully) this will be fixed any time soon. + +To run them in a detached (background) mode: + +``` +docker-compose up -d +``` + +To run only the objstorage API: + +``` +docker-compose up swh-objstorage +``` + +### Install a package from sources + +It is possible to run a docker with some swh packages installed from sources +instead of from pypi. To do this you must write a docker-compose override +file. An example is given in docker-compose.override.yml.example: + +``` +version: '2' + +services: + swh-objstorage: + volumes: + - "/home/ddouard/src/swh-environment/swh-objstorage:/src/swh-objstorage" +``` + +A file named docker-compose.override.yml will automatically be loaded by +docker-compose. + +This example shows the simple case of the swh-objstorage package: you just have to +mount it in the container in /src and the entrypoint will ensure every +swh-* package found in /src/ is installed (using `pip install -e` so you can +easily hack your code. If the application you play with have autoreload support, +there is even no need for restarting the impacted container.) + ## Details This runs the following services on their respectively standard ports: diff --git a/docker-compose.override.yml.example b/docker-compose.override.yml.example new file mode 100644 --- /dev/null +++ b/docker-compose.override.yml.example @@ -0,0 +1,6 @@ +version: '2' + +services: + swh-objstorage: + volumes: + - "/home/ddouard/src/swh-environment/swh-objstorage:/src/swh-objstorage" diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,26 @@ -version: '2.0' +version: '2' services: - - pgsql-storage: - image: swh-storage-pg + swh-storage-db: + image: postgres:10 environment: POSTGRES_PASSWORD: testpassword - + POSTGRES_DB: swh-storage swh-storage: - image: swh-storage + build: ./dockerfiles/swh-storage + image: swh/storage ports: - 5002:5002 depends_on: - - pgsql-storage + - swh-storage-db - swh-objstorage - + environment: + POSTGRES_PASSWORD: testpassword + POSTGRES_DB: swh-storage + PGHOST: swh-storage-db + PGUSER: postgres swh-objstorage: - image: swh-objstorage + build: ./dockerfiles/swh-objstorage + image: swh/objstorage ports: - 5003:5003 diff --git a/dockerfiles/Dockerfile-swh-objstorage b/dockerfiles/Dockerfile-swh-objstorage deleted file mode 100644 --- a/dockerfiles/Dockerfile-swh-objstorage +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3 - -RUN \ - export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev - -RUN mkdir -p /srv/softwareheritage/objects - -WORKDIR /usr/local/src/ -COPY ./swh-objstorage/requirements*.txt ./swh-objstorage/ - -RUN pip install -r ./swh-objstorage/requirements.txt -r ./swh-objstorage/requirements-swh.txt - -COPY ./swh-docker-dev/config/ /etc/softwareheritage/ - -COPY . . -RUN pip install -e ./swh-objstorage/ - -ENTRYPOINT ["python", "-m", "swh.objstorage.api.server", "/etc/softwareheritage/objstorage/server.yml"] - -EXPOSE 5003 diff --git a/dockerfiles/Dockerfile-swh-storage b/dockerfiles/Dockerfile-swh-storage deleted file mode 100644 --- a/dockerfiles/Dockerfile-swh-storage +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3 - -RUN \ - export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install -y libsystemd-dev - -WORKDIR /usr/local/src/ -COPY ./swh-storage/requirements*.txt ./swh-storage/ - -RUN pip install -r ./swh-storage/requirements.txt -r ./swh-storage/requirements-swh.txt - -COPY ./swh-docker-dev/config/ /etc/softwareheritage/ - -COPY . . -RUN pip install -e ./swh-storage/ - -RUN cat /etc/softwareheritage/storage/storage.yml - -ENTRYPOINT ["python", "-m", "swh.storage.api.server", "/etc/softwareheritage/storage/storage.yml"] - -EXPOSE 5002 - diff --git a/dockerfiles/Dockerfile-swh-storage-pg b/dockerfiles/Dockerfile-swh-storage-pg deleted file mode 100644 --- a/dockerfiles/Dockerfile-swh-storage-pg +++ /dev/null @@ -1,4 +0,0 @@ -FROM postgres - -# Create the swh-storage's database when the container starts. -COPY ./swh-storage/swh/storage/sql/ /docker-entrypoint-initdb.d diff --git a/dockerfiles/swh-objstorage/Dockerfile b/dockerfiles/swh-objstorage/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-objstorage/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3 + +export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev + +RUN pip install swh-objstorage +COPY objstorage.yml / +COPY entrypoint.sh / + +RUN mkdir -p /srv/softwareheritage/objects + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 5003 diff --git a/dockerfiles/swh-objstorage/entrypoint.sh b/dockerfiles/swh-objstorage/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-objstorage/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +echo Step 1 +if [[ -d /src ]] ; then + echo Yes + for srcrepo in /src/swh-* ; do + echo installing $srcrepo + pushd $srcrepo + pip install -e . + popd + done +fi + +if [ "$1" = 'shell' ] ; then + exec bash -i +else + echo Starting the swh-objstorage API server + exec python -m swh.objstorage.api.server /objstorage.yml +fi diff --git a/config/objstorage/server.yml b/dockerfiles/swh-objstorage/objstorage.yml rename from config/objstorage/server.yml rename to dockerfiles/swh-objstorage/objstorage.yml diff --git a/dockerfiles/swh-storage/Dockerfile b/dockerfiles/swh-storage/Dockerfile new file mode 100644 --- /dev/null +++ b/dockerfiles/swh-storage/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3 + +export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + libsystemd-dev postgresql-client + +RUN pip install swh-storage +COPY storage.yml / +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +EXPOSE 5002 diff --git a/dockerfiles/swh-storage/entrypoint.sh b/dockerfiles/swh-storage/entrypoint.sh new file mode 100755 --- /dev/null +++ b/dockerfiles/swh-storage/entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +if [[ -d /src ]] ; then + for srcrepo in /src/swh-* ; do + pushd $srcrepo + pip install -e . + popd + done +fi + +echo "${PGHOST}:5432:${POSTGRES_DB}:${PGUSER}:${POSTGRES_PASSWORD}" > ~/.pgpass +chmod 0400 ~/.pgpass + +case "$1" in + "shell") + exec bash -i + ;; + "init") + echo Setup the database + swh-db-init storage -d ${POSTGRES_DB} + ;; + *) + echo Starting the swh-storage API server + exec python -m swh.storage.api.server /storage.yml +esac diff --git a/config/storage/storage.yml b/dockerfiles/swh-storage/storage.yml rename from config/storage/storage.yml rename to dockerfiles/swh-storage/storage.yml --- a/config/storage/storage.yml +++ b/dockerfiles/swh-storage/storage.yml @@ -1,8 +1,8 @@ storage: cls: local args: - db: "host=pgsql-storage port=5432 user=postgres password=testpassword" # FIXME: this should not be hardcoded - objstorage: + db: "host=swh-storage-db port=5432 user=postgres password=testpassword" # FIXME: this should not be hardcoded + objstorage: cls: remote args: url: http://swh-objstorage:5003/ # FIXME: this should not be hardcoded