diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2a8226a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,64 @@ +# If true, install from locally cloned modules +# If false, use pip +ARG LOCAL_MODULES=true +ARG SWH_MODULES="swh-core swh-model swh-objstorage" + +### +# Base image mother of all +FROM python:3.7 as swh-base + +RUN . /etc/os-release && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && apt-get upgrade -y && \ + apt-get install -y \ + libapr1-dev \ + libaprutil1-dev \ + libpq-dev \ + libsvn-dev \ + libsystemd-dev \ + memcached \ + postgresql-client-12 \ + wait-for-it \ + ngrep \ + rsync && \ + apt-get install -y --no-install-recommends \ + r-base-core \ + r-cran-jsonlite && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -md /srv/softwareheritage -s /bin/bash swh + +RUN mkdir /app +WORKDIR /app + +USER swh + +RUN python3 -m venv /srv/softwareheritage/venv +ENV PATH="/srv/softwareheritage/venv/bin:${PATH}" + +RUN pip install --upgrade pip setuptools wheel +RUN pip install flask gunicorn httpie + +### +# swh-packages +FROM swh-base as swh-packages + +# Install requirements.txt only if there is any changes +COPY swh-model/requirements.txt /app/requirements-model.txt +COPY swh-core/requirements.txt /app/requirements-core.txt +COPY swh-journal/requirements.txt /app/requirements-journal.txt +COPY swh-objstorage/requirements.txt /app/requirements-objstorage.txt +COPY swh-storage/requirements.txt /app/requirements-storage.txt +RUN . /srv/softwareheritage/venv/bin/activate && cat /app/requirements-*.txt > /tmp/requirements.txt && pip install -r /tmp/requirements.txt \ + && pip install decorator aiohttp_utils blinker + # && rm /srv/requirements* + +# Copy source code +COPY swh-model/swh /app/swh +COPY swh-core/swh /app/swh +COPY swh-model/swh /app/swh +COPY swh-objstorage/swh /app/swh +COPY swh-storage/swh /app/swh diff --git a/kubernetes/05-storage-db.yml b/kubernetes/05-storage-db.yml new file mode 100644 index 0000000..68a4e36 --- /dev/null +++ b/kubernetes/05-storage-db.yml @@ -0,0 +1,118 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage-db-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: storage-db + local: + path: /srv/softwareheritage/dev/storage-db + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + # TODO adapt for your needs + - key: kubernetes.io/os + operator: In + values: + - linux +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: storage-db-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: storage-db + resources: + requests: + storage: 10Gi + +--- +## TODO Change this to your real postgresql password +apiVersion: v1 +kind: Secret +metadata: + name: storage-db +type: Opaque +# data: +# POSTGRES_PASSWORD: | +# "echo 'strong password' | base64" +stringData: + POSTGRES_PASSWORD: swh +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: storage-db +data: + # property-like keys; each key maps to a simple value + POSTGRES_USER: swh + POSTGRES_DB: swh +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: storage-db + labels: + app: storage-db +spec: + replicas: 1 + selector: + matchLabels: + app: storage-db + template: + metadata: + labels: + app: storage-db + spec: + containers: + - name: storage-db + image: postgres:13.0 + imagePullPolicy: Always + ports: + - containerPort: 5432 + env: + - name: PGOPTIONS + value: -c shared_buffers=4GB -c effective_cache_size=4GB -c random_page_cost=1.5 -c max_wal_size=4GB + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: storage-db + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_DB + volumeMounts: + - mountPath: "/var/lib/postgresql/data" + name: storage-db-pvc + volumes: + - name: storage-db-pvc + persistentVolumeClaim: + claimName: storage-db-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: storage-db +spec: + type: ClusterIP + selector: + app: storage-db + ports: + - port: 5432 + targetPort: 5432 diff --git a/kubernetes/10-objstorage.yml b/kubernetes/10-objstorage.yml new file mode 100644 index 0000000..a6b520d --- /dev/null +++ b/kubernetes/10-objstorage.yml @@ -0,0 +1,151 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: objstorage +data: + config.yml: | + objstorage: + cls: pathslicing + args: + root: "/srv/softwareheritage/objects" + slicing: 0:5 + client_max_size: 1073741824 + entrypoint.sh: | + #!/bin/bash + + set -e + + # source /srv/softwareheritage/utils/pyutils.sh + # setup_pip + + # echo Installed Python packages: + # pip list + + echo Starting the swh-objstorage API server + exec gunicorn --bind 0.0.0.0:5003 \ + --worker-class aiohttp.worker.GunicornWebWorker \ + --log-level DEBUG \ + --threads 4 \ + --workers 2 \ + --reload \ + --timeout 3600 \ + --config 'python:swh.core.api.gunicorn_config' \ + 'swh.objstorage.api.server:make_app_from_configfile()' +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: objstorage-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: objstorage-pv + local: + path: /srv/softwareheritage-kube/objects + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + # TODO adapt for your needs + - key: kubernetes.io/os + operator: In + values: + - linux +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: objstorage-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: objstorage-pv + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: objstorage + labels: + app: objstorage +spec: + replicas: 1 + selector: + matchLabels: + app: objstorage + template: + metadata: + labels: + app: objstorage + spec: + containers: + - name: objstorage + image: swh/stack:latest + command: + - /entrypoint.sh + ports: + - containerPort: 5003 + env: + - name: PORT + value: "5003" + - name: STATSD_HOST + value: "prometheus-statsd-exporter" + - name: STATSD_PORT + value: "9125" + - name: SWH_CONFIG_FILENAME + value: /etc/softwareheritage/config.yml + volumeMounts: + - mountPath: "/srv/softwareheritage/objects" + name: objstorage-pvc + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + readOnly: true + volumes: + - name: config + configMap: + name: objstorage + defaultMode: 0777 + - name: objstorage-pvc + persistentVolumeClaim: + claimName: objstorage-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: objstorage +spec: + type: ClusterIP + selector: + app: objstorage + ports: + - port: 5003 + targetPort: 5003 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: objstorage +spec: + rules: + - host: objstorage.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: objstorage + port: + number: 5003 diff --git a/kubernetes/11-storage.yml b/kubernetes/11-storage.yml new file mode 100644 index 0000000..0c475c3 --- /dev/null +++ b/kubernetes/11-storage.yml @@ -0,0 +1,132 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: storage +data: + config.yml: | + storage: + cls: local + args: + db: postgresql:///?service=swh + objstorage: + cls: remote + args: + url: http://objstorage:5000/ + entrypoint.sh: | + #!/bin/bash + + set -e + + # source /srv/softwareheritage/utils/pyutils.sh + # setup_pip + + # echo Installed Python packages: + # pip list + + echo Starting the swh-objstorage API server + exec gunicorn --bind 0.0.0.0:5003 \ + --worker-class aiohttp.worker.GunicornWebWorker \ + --log-level DEBUG \ + --threads 4 \ + --workers 2 \ + --reload \ + --timeout 3600 \ + --config 'python:swh.core.api.gunicorn_config' \ + 'swh.objstorage.api.server:make_app_from_configfile()' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: storage + labels: + app: storage +spec: + replicas: 1 + selector: + matchLabels: + app: storage + template: + metadata: + labels: + app: storage + spec: + containers: + - name: storage + image: swh/stack:latest + imagePullPolicy: Always + args: + - storage + ports: + - containerPort: 5002 + env: + - name: PORT + value: "5002" + - name: STATSD_HOST + value: "prometheus-statsd-exporter" + - name: STATSD_PORT + value: "9125" + - name: POSTGRES_PASSWORD_FILE + value: "/etc/softwareheritage/db-password" + - name: PGHOST + value: "storage-db" + - name: PGUSER + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: storage-db + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_DB + volumeMounts: + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: db-password + mountPath: /run/secrets/postgres-password + subPath: POSTGRES_PASSWORD + readOnly: true + volumes: + - name: config + configMap: + name: storage + - name: db-password + secret: + secretName: storage-db +--- +apiVersion: v1 +kind: Service +metadata: + name: storage +spec: + type: ClusterIP + selector: + app: storage + ports: + - port: 5002 + targetPort: 5002 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: storage +spec: + rules: + - host: storage.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: storage + port: + number: 5003 diff --git a/kubernetes/Readme.md b/kubernetes/Readme.md new file mode 100644 index 0000000..21cef98 --- /dev/null +++ b/kubernetes/Readme.md @@ -0,0 +1,123 @@ + +## Prerquisite + +### Directories +``` +# sudo mkdir -p /srv/softwareheritage-kube/storage-db +# sudo mkdir -p /srv/softwareheritage-kube/objects +``` +Must match the content of `02-storage-db.yaml` + +### Registry + +- Add the following line on your `/etc/hosts` file. It's needed to be able to + push the image to it from docker +``` +127.0.0.1 registry.default +``` +- Start the registry in kubernetes +``` +# cd kubernetes +# kubectl apply -f registry/00-registry.yml +``` + +## Build the base image + +``` +# cd docker +# docker build --no-cache -t swh/stack . + +# docker tag swh/stack:latest registry.default/swh/stack:latest +# docker push registry.default/swh/stack:latest + +``` + +## start the objstorage + + +- start the service +``` +# cd kubernetes + +# kubectl apply -f 01-objstorage.yml +configmap/objstorage created +persistentvolume/objstorage-pv created +persistentvolumeclaim/objstorage-pvc created +deployment.apps/objstorage created +service/objstorage created +``` +- test it +``` +# kubectl get pods +NAME READY STATUS RESTARTS AGE +registry-deployment-7595868dc8-657ps 1/1 Running 0 46m +objstorage-8587d58b68-76jbn 1/1 Running 0 12m + +# kubectl get services objstorage +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +objstorage ClusterIP 10.43.185.191 5003/TCP 17m + +# curl http://$(kubectl get services objstorage -o jsonpath='{.spec.clusterIP}'):5003 +SWH Objstorage API server% +``` + + +## Start the storage + +- Start the db +``` +# cd kubernetes + +# kubectl apply -f 02-storage-db.yml +persistentvolume/storage-db-pv created +persistentvolumeclaim/storage-db-pvc created +secret/storage-db created +configmap/storage-db created +deployment.apps/storage-db created +service/storage-db created + +# kubectl get pods +NAME READY STATUS RESTARTS AGE +registry-deployment-7595868dc8-657ps 1/1 Running 0 46m +objstorage-8587d58b68-76jbn 1/1 Running 0 15m +storage-db-64b7f8b684-48n7w 1/1 Running 0 4m52s + +# kubectl get services storage-db +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +storage-db ClusterIP 10.43.213.178 5432/TCP 8m19s +``` +- Start the storage +``` +# cd kubernetes + +# kubectl apply -f 03-storage.yml +configmap/storage created +deployment.apps/storage created +service/storage created +``` + +- Test the service +``` +# kubectl get pods +NAME READY STATUS RESTARTS AGE +registry-deployment-7595868dc8-657ps 1/1 Running 0 49m +storage-db-64b7f8b684-48n7w 1/1 Running 0 7m40s +storage-6b759fb974-w9rzj 1/1 Running 0 66s + +# kubectl get services storage +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +storage ClusterIP 10.43.212.116 5002/TCP 2m24s + +# curl http://$(kubectl get services storage -o jsonpath='{.spec.clusterIP}'):5002 + +Software Heritage storage server + +

You have reached the +Software Heritage +storage server.
+See its +documentation +and API for more information

+ + +``` diff --git a/kubernetes/registry/00-registry.yml b/kubernetes/registry/00-registry.yml new file mode 100644 index 0000000..9bd7f00 --- /dev/null +++ b/kubernetes/registry/00-registry.yml @@ -0,0 +1,51 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: registry +spec: + selector: + app: registry + ports: + - protocol: TCP + port: 80 + targetPort: 5000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: registry +spec: + rules: + - host: registry.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: registry + port: + number: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry-deployment + labels: + app: registry +spec: + replicas: 1 + selector: + matchLabels: + app: registry + template: + metadata: + labels: + app: registry + spec: + containers: + - name: registry + image: registry:latest + ports: + - containerPort: 5000 diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..c027eb9 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,15 @@ +apiVersion: skaffold/v2beta13 +kind: Config +metadata: + name: swh-environment +build: + artifacts: + - image: swh/stack + docker: + dockerfile: Dockerfile +deploy: + kubectl: + manifests: + - kubernetes/05-storage-db.yml + - kubernetes/10-objstorage.yml + - kubernetes/11-storage.yml