diff --git a/images/base/entrypoint.sh b/images/base/entrypoint.sh index ec6582f..944eaf0 100755 --- a/images/base/entrypoint.sh +++ b/images/base/entrypoint.sh @@ -1,64 +1,65 @@ #!/bin/bash set -e source /srv/softwareheritage/utils/pgsql.sh # generate the config file from the 'template' if [ -f /etc/softwareheritage/config.yml.tmpl ]; then # I know... I know! eval "echo \"`cat /etc/softwareheritage/config.yml.tmpl`\"" > \ /etc/softwareheritage/config.yml fi # generate the pgservice file if any if [ -f /run/secrets/postgres-password ]; then POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password setup_pgsql fi # For debugging purpose echo "### CONFIG FILE ###" cat /etc/softwareheritage/config.yml echo "###################" echo "Arguments: $@" case "$1" in "shell") exec bash -i ;; "objstorage") echo "Starting the SWH $1 RPC server" exec gunicorn3 \ --bind 0.0.0.0:${PORT:-5000} \ --bind unix:/var/run/gunicorn/swh/$1.sock \ --worker-class aiohttp.worker.GunicornWebWorker \ --threads 4 \ --workers 2 \ --log-level "${LOG_LEVEL:-WARNING}" \ --timeout 3600 \ "swh.$1.api.server:make_app_from_configfile()" ;; *) wait_pgsql template1 echo Database setup if ! check_pgsql_db_created; then echo Creating database and extensions... swh db create --db-name ${POSTGRES_DB} storage fi echo Initializing the database... + swh db init-admin --db-name ${POSTGRES_DB} storage swh db init --db-name ${POSTGRES_DB} --flavor ${FLAVOR:-default} storage echo "Starting the SWH $1 RPC server" exec gunicorn3 \ --bind 0.0.0.0:${PORT:-5000} \ --bind unix:/var/run/gunicorn/swh/$1.sock \ --threads 4 \ --workers 2 \ --log-level "${LOG_LEVEL:-WARNING}" \ --timeout 3600 \ "swh.$1.api.server:make_app_from_configfile()" ;; esac diff --git a/kubernetes/01-objstorage.yml b/kubernetes/01-objstorage.yml new file mode 100644 index 0000000..db1d239 --- /dev/null +++ b/kubernetes/01-objstorage.yml @@ -0,0 +1,107 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: objstorage +data: + config.yml: | + objstorage: + cls: pathslicing + args: + root: "/srv/softwareheritage/objects" + slicing: 0:1/1:5 + client_max_size: 1073741824 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: objstorage-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: objstorage-pv + local: + path: /srv/softwareheritage-kube/objects + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + # TODO adapt for your needs + - key: kubernetes.io/os + operator: In + values: + - linux +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: objstorage-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: objstorage-pv + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: objstorage + labels: + app: objstorage +spec: + replicas: 1 + selector: + matchLabels: + app: objstorage + template: + metadata: + labels: + app: objstorage + spec: + containers: + - name: objstorage + image: registry.default/softwareheritage/base:latest + command: + - /entrypoint.sh + - objstorage + ports: + - containerPort: 5003 + env: + - name: PORT + value: "5003" + - name: STATSD_HOST + value: "prometheus-statsd-exporter" + - name: STATSD_PORT + value: "9125" + volumeMounts: + - mountPath: "/srv/softwareheritage/objects" + name: objstorage-pvc + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + volumes: + - name: config + configMap: + name: objstorage + - name: objstorage-pvc + persistentVolumeClaim: + claimName: objstorage-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: objstorage +spec: + type: ClusterIP + selector: + app: objstorage + ports: + - port: 5003 + targetPort: 5003 diff --git a/kubernetes/02_storage-db.yml b/kubernetes/02_storage-db.yml new file mode 100644 index 0000000..cb9b5fc --- /dev/null +++ b/kubernetes/02_storage-db.yml @@ -0,0 +1,117 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage-db-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: storage-db + local: + path: /srv/softwareheritage-kube/storage-db + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + # TODO adapt for your needs + - key: kubernetes.io/os + operator: In + values: + - linux +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: storage-db-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: storage-db + resources: + requests: + storage: 10Gi + +--- +## TODO Change this to your real postgresql password +apiVersion: v1 +kind: Secret +metadata: + name: storage-db +type: Opaque +# data: +# POSTGRES_PASSWORD: | +# "echo 'strong password' | base64" +stringData: + POSTGRES_PASSWORD: swh +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: storage-db +data: + # property-like keys; each key maps to a simple value + POSTGRES_USER: swh + POSTGRES_DB: swh +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: storage-db + labels: + app: storage-db +spec: + replicas: 1 + selector: + matchLabels: + app: storage-db + template: + metadata: + labels: + app: storage-db + spec: + containers: + - name: storage-db + image: postgres:13.0 + ports: + - containerPort: 5432 + env: + - name: PGOPTIONS + value: -c shared_buffers=4GB -c effective_cache_size=4GB -c random_page_cost=1.5 -c max_wal_size=4GB + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: storage-db + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_DB + volumeMounts: + - mountPath: "/var/lib/postgresql/data" + name: storage-db-pvc + volumes: + - name: storage-db-pvc + persistentVolumeClaim: + claimName: storage-db-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: storage-db +spec: + type: ClusterIP + selector: + app: storage-db + ports: + - port: 5432 + targetPort: 5432 diff --git a/kubernetes/03-storage.yml b/kubernetes/03-storage.yml new file mode 100644 index 0000000..24461cf --- /dev/null +++ b/kubernetes/03-storage.yml @@ -0,0 +1,95 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: storage +data: + config.yml: | + storage: + cls: local + args: + db: postgresql:///?service=swh + objstorage: + cls: remote + args: + url: http://objstorage:5000/ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: storage + labels: + app: storage +spec: + replicas: 1 + selector: + matchLabels: + app: storage + template: + metadata: + labels: + app: storage + spec: + containers: + - name: storage + image: registry.default/softwareheritage/base:latest + imagePullPolicy: Always + command: + - /entrypoint.sh + - storage + ports: + - containerPort: 5002 + env: + - name: PORT + value: "5002" + - name: STATSD_HOST + value: "prometheus-statsd-exporter" + - name: STATSD_PORT + value: "9125" + - name: POSTGRES_PASSWORD_FILE + value: "/etc/softwareheritage/db-password" + - name: PGHOST + value: "storage-db" + - name: PGUSER + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: storage-db + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: storage-db + key: POSTGRES_DB + volumeMounts: + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: db-password + mountPath: /run/secrets/postgres-password + subPath: POSTGRES_PASSWORD + readOnly: true + volumes: + - name: config + configMap: + name: storage + - name: db-password + secret: + secretName: storage-db +--- +apiVersion: v1 +kind: Service +metadata: + name: storage +spec: + type: ClusterIP + selector: + app: storage + ports: + - port: 5002 + targetPort: 5002 diff --git a/kubernetes/Readme.md b/kubernetes/Readme.md new file mode 100644 index 0000000..958ac14 --- /dev/null +++ b/kubernetes/Readme.md @@ -0,0 +1,122 @@ + +## Prerquisite + +### Directories +``` +# sudo mkdir -p /srv/softwareheritage-kube/storage-db +# sudo mkdir -p /srv/softwareheritage-kube/objects +``` +Must match the content of `02-storage-db.yaml` + +### Registry + +- Add the following line on your `/etc/hosts` file. It's needed to be able to + push the image to it from docker +``` +127.0.0.1 registry.default +``` +- Start the registry in kubernetes +``` +# cd kubernetes +# kubectl apply -f registry/00-registry.yml +``` + +## Build the base image + +``` +# cd images +# ./build_images.sh + +# docker tag softwareheritage/base:latest registry.default/softwareheritage/base:latest +# docker push registry.default/softwareheritage/base:latest + +``` + +## start the objstorage + +- start the service +``` +# cd kubernetes + +# kubectl apply -f 01-objstorage.yml +configmap/objstorage created +persistentvolume/objstorage-pv created +persistentvolumeclaim/objstorage-pvc created +deployment.apps/objstorage created +service/objstorage created +``` +- test it +From a node of the cluster : +``` +# kubectl get pods +NAME READY STATUS RESTARTS AGE +registry-deployment-7595868dc8-657ps 1/1 Running 0 46m +objstorage-8587d58b68-76jbn 1/1 Running 0 12m + +# kubectl get services objstorage +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +objstorage ClusterIP 10.43.185.191 5003/TCP 17m + +# curl http://$(kubectl get services objstorage -o jsonpath='{.spec.clusterIP}'):5003 +SWH Objstorage API server% +``` +## Start the storage + +- Start the db +``` +# cd kubernetes + +# kubectl apply -f 02-storage-db.yml +persistentvolume/storage-db-pv created +persistentvolumeclaim/storage-db-pvc created +secret/storage-db created +configmap/storage-db created +deployment.apps/storage-db created +service/storage-db created + +# kubectl get pods +NAME READY STATUS RESTARTS AGE +registry-deployment-7595868dc8-657ps 1/1 Running 0 46m +objstorage-8587d58b68-76jbn 1/1 Running 0 15m +storage-db-64b7f8b684-48n7w 1/1 Running 0 4m52s + +# kubectl get services storage-db +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +storage-db ClusterIP 10.43.213.178 5432/TCP 8m19s +``` +- Start the storage +``` +# cd kubernetes + +# kubectl apply -f 03-storage.yml +configmap/storage created +deployment.apps/storage created +service/storage created +``` + +- Test the service +From a node of the cluster : +``` +# kubectl get pods +NAME READY STATUS RESTARTS AGE +registry-deployment-7595868dc8-657ps 1/1 Running 0 49m +storage-db-64b7f8b684-48n7w 1/1 Running 0 7m40s +storage-6b759fb974-w9rzj 1/1 Running 0 66s + +# kubectl get services storage +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +storage ClusterIP 10.43.212.116 5002/TCP 2m24s + +# curl http://$(kubectl get services storage -o jsonpath='{.spec.clusterIP}'):5002 + +Software Heritage storage server + +

You have reached the +Software Heritage +storage server.
+See its +documentation +and API for more information

+ + +``` diff --git a/kubernetes/init.sh b/kubernetes/init.sh new file mode 100755 index 0000000..2715e6b --- /dev/null +++ b/kubernetes/init.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +mkdir -v -p /srv/softwareheritage-kube/storage-db +mkdir -v -p /srv/softwareheritage-kube/objects diff --git a/kubernetes/registry/00-registry.yml b/kubernetes/registry/00-registry.yml new file mode 100644 index 0000000..9bd7f00 --- /dev/null +++ b/kubernetes/registry/00-registry.yml @@ -0,0 +1,51 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: registry +spec: + selector: + app: registry + ports: + - protocol: TCP + port: 80 + targetPort: 5000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: registry +spec: + rules: + - host: registry.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: registry + port: + number: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry-deployment + labels: + app: registry +spec: + replicas: 1 + selector: + matchLabels: + app: registry + template: + metadata: + labels: + app: registry + spec: + containers: + - name: registry + image: registry:latest + ports: + - containerPort: 5000