diff --git a/Dockerfile b/Dockerfile index 2a8226a..83636be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,64 +1,55 @@ -# If true, install from locally cloned modules -# If false, use pip -ARG LOCAL_MODULES=true -ARG SWH_MODULES="swh-core swh-model swh-objstorage" - ### # Base image mother of all FROM python:3.7 as swh-base RUN . /etc/os-release && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ libapr1-dev \ libaprutil1-dev \ libpq-dev \ libsvn-dev \ libsystemd-dev \ memcached \ postgresql-client-12 \ wait-for-it \ ngrep \ rsync && \ apt-get install -y --no-install-recommends \ r-base-core \ r-cran-jsonlite && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN useradd -md /srv/softwareheritage -s /bin/bash swh RUN mkdir /app WORKDIR /app USER swh RUN python3 -m venv /srv/softwareheritage/venv ENV PATH="/srv/softwareheritage/venv/bin:${PATH}" RUN pip install --upgrade pip setuptools wheel RUN pip install flask gunicorn httpie ### # swh-packages FROM swh-base as swh-packages # Install requirements.txt only if there is any changes COPY swh-model/requirements.txt /app/requirements-model.txt COPY swh-core/requirements.txt /app/requirements-core.txt COPY swh-journal/requirements.txt /app/requirements-journal.txt -COPY swh-objstorage/requirements.txt /app/requirements-objstorage.txt -COPY swh-storage/requirements.txt /app/requirements-storage.txt RUN . /srv/softwareheritage/venv/bin/activate && cat /app/requirements-*.txt > /tmp/requirements.txt && pip install -r /tmp/requirements.txt \ && pip install decorator aiohttp_utils blinker # && rm /srv/requirements* # Copy source code COPY swh-model/swh /app/swh COPY swh-core/swh /app/swh -COPY swh-model/swh /app/swh -COPY swh-objstorage/swh /app/swh -COPY swh-storage/swh /app/swh +COPY swh-journal/swh /app/swh diff --git a/Dockerfile.objstorage b/Dockerfile.objstorage new file mode 100644 index 0000000..3caa05e --- /dev/null +++ b/Dockerfile.objstorage @@ -0,0 +1,8 @@ +ARG BASE + +FROM $BASE +COPY swh-objstorage/requirements.txt /app/requirements-objstorage.txt + +RUN . /srv/softwareheritage/venv/bin/activate && pip install -r /app/requirements-objstorage.txt + +COPY swh-objstorage/swh /app/swh diff --git a/Dockerfile.storage b/Dockerfile.storage new file mode 100644 index 0000000..2e5e792 --- /dev/null +++ b/Dockerfile.storage @@ -0,0 +1,10 @@ +ARG BASE + +FROM $BASE +COPY swh-storage/requirements.txt /app/requirements-storage.txt +COPY swh-objstorage/requirements.txt /app/requirements-obstorage.txt + +RUN . /srv/softwareheritage/venv/bin/activate && pip install -r /app/requirements-storage.txt + +COPY swh-objstorage/swh /app/swh +COPY swh-storage/swh /app/swh diff --git a/kubernetes/10-objstorage.yml b/kubernetes/10-objstorage.yml index a6b520d..6490d56 100644 --- a/kubernetes/10-objstorage.yml +++ b/kubernetes/10-objstorage.yml @@ -1,151 +1,155 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: objstorage data: config.yml: | objstorage: cls: pathslicing args: root: "/srv/softwareheritage/objects" slicing: 0:5 client_max_size: 1073741824 entrypoint.sh: | #!/bin/bash set -e # source /srv/softwareheritage/utils/pyutils.sh # setup_pip # echo Installed Python packages: # pip list echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ --threads 4 \ --workers 2 \ --reload \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.objstorage.api.server:make_app_from_configfile()' --- apiVersion: v1 kind: PersistentVolume metadata: name: objstorage-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: objstorage-pv local: path: /srv/softwareheritage-kube/objects nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: objstorage-pvc spec: accessModes: - ReadWriteOnce storageClassName: objstorage-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: objstorage labels: app: objstorage spec: replicas: 1 selector: matchLabels: app: objstorage + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 template: metadata: labels: app: objstorage spec: containers: - name: objstorage - image: swh/stack:latest + image: swh/objstorage:latest command: - /entrypoint.sh ports: - containerPort: 5003 env: - name: PORT value: "5003" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - mountPath: "/srv/softwareheritage/objects" name: objstorage-pvc - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true volumes: - name: config configMap: name: objstorage defaultMode: 0777 - name: objstorage-pvc persistentVolumeClaim: claimName: objstorage-pvc --- apiVersion: v1 kind: Service metadata: name: objstorage spec: type: ClusterIP selector: app: objstorage ports: - port: 5003 targetPort: 5003 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: objstorage spec: rules: - host: objstorage.default http: paths: - path: / pathType: Prefix backend: service: name: objstorage port: number: 5003 diff --git a/kubernetes/11-storage.yml b/kubernetes/11-storage.yml index 0c475c3..db01c10 100644 --- a/kubernetes/11-storage.yml +++ b/kubernetes/11-storage.yml @@ -1,132 +1,141 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: local - args: - db: postgresql:///?service=swh - objstorage: - cls: remote - args: - url: http://objstorage:5000/ + db: postgresql:///?service=swh-storage + objstorage: + cls: remote + url: http://objstorage:5003/ + journal_writer: + cls: kafka + args: + brokers: + - kafka + prefix: swh.journal.objects + client_id: swh.storage.master entrypoint.sh: | #!/bin/bash set -e - # source /srv/softwareheritage/utils/pyutils.sh - # setup_pip - - # echo Installed Python packages: - # pip list - - echo Starting the swh-objstorage API server - exec gunicorn --bind 0.0.0.0:5003 \ - --worker-class aiohttp.worker.GunicornWebWorker \ - --log-level DEBUG \ - --threads 4 \ - --workers 2 \ - --reload \ - --timeout 3600 \ - --config 'python:swh.core.api.gunicorn_config' \ - 'swh.objstorage.api.server:make_app_from_configfile()' + echo Starting the swh-storage API server + exec gunicorn --bind 0.0.0.0:5002 \ + --reload \ + --threads 4 \ + --workers 2 \ + --log-level DEBUG \ + --timeout 3600 \ + --config 'python:swh.core.api.gunicorn_config' \ + 'swh.storage.api.server:make_app_from_configfile()' --- apiVersion: apps/v1 kind: Deployment metadata: name: storage labels: app: storage spec: replicas: 1 selector: matchLabels: app: storage + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 template: metadata: labels: app: storage spec: containers: - name: storage - image: swh/stack:latest + image: swh/storage:latest imagePullPolicy: Always - args: - - storage + command: + - /entrypoint.sh ports: - containerPort: 5002 env: - name: PORT value: "5002" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: POSTGRES_PASSWORD_FILE value: "/etc/softwareheritage/db-password" - name: PGHOST value: "storage-db" - name: PGUSER valueFrom: configMapKeyRef: name: storage-db key: POSTGRES_USER - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: storage-db key: POSTGRES_PASSWORD - name: POSTGRES_DB valueFrom: configMapKeyRef: name: storage-db key: POSTGRES_DB + - name: SWH_CONFIG_FILENAME + value: /etc/softwareheritage/config.yml volumeMounts: + - name: db-password + mountPath: /run/secrets/postgres-password + subPath: POSTGRES_PASSWORD + readOnly: true - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - - name: db-password - mountPath: /run/secrets/postgres-password - subPath: POSTGRES_PASSWORD + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint.sh readOnly: true volumes: - name: config configMap: name: storage + defaultMode: 0777 - name: db-password secret: secretName: storage-db --- apiVersion: v1 kind: Service metadata: name: storage spec: type: ClusterIP selector: app: storage ports: - port: 5002 targetPort: 5002 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: storage spec: rules: - host: storage.default http: paths: - path: / pathType: Prefix backend: service: name: storage port: - number: 5003 + number: 5002 diff --git a/skaffold.yaml b/skaffold.yaml index c027eb9..4e6c663 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1,15 +1,27 @@ apiVersion: skaffold/v2beta13 kind: Config metadata: name: swh-environment build: artifacts: - - image: swh/stack + - image: swh/stack-base docker: dockerfile: Dockerfile + - image: swh/objstorage + docker: + dockerfile: Dockerfile.objstorage + requires: + - image: swh/stack-base + alias: BASE + - image: swh/storage + docker: + dockerfile: Dockerfile.storage + requires: + - image: swh/stack-base + alias: BASE deploy: kubectl: manifests: - kubernetes/05-storage-db.yml - kubernetes/10-objstorage.yml - kubernetes/11-storage.yml