diff --git a/kubernetes/10-objstorage.yml b/kubernetes/10-objstorage.yml index 5c6b1ca..367a15e 100644 --- a/kubernetes/10-objstorage.yml +++ b/kubernetes/10-objstorage.yml @@ -1,162 +1,162 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: objstorage data: config.yml: | objstorage: cls: pathslicing args: root: "/srv/softwareheritage/objects" slicing: 0:5 client_max_size: 1073741824 entrypoint.sh: | #!/bin/bash set -e # source /srv/softwareheritage/utils/pyutils.sh # setup_pip # echo Installed Python packages: # pip list echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ --threads 4 \ --workers 2 \ --reload \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.objstorage.api.server:make_app_from_configfile()' --- apiVersion: v1 kind: PersistentVolume metadata: name: objstorage-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: objstorage-pv local: - path: /srv/softwareheritage-kube/objects + path: /srv/softwareheritage-kube/dev/objects nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: objstorage-pvc spec: accessModes: - ReadWriteOnce storageClassName: objstorage-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: objstorage labels: app: objstorage spec: replicas: 1 selector: matchLabels: app: objstorage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: objstorage spec: containers: - name: objstorage image: swh/objstorage:latest command: - /entrypoint.sh ports: - containerPort: 5003 env: - name: PORT value: "5003" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - mountPath: "/srv/softwareheritage/objects" name: objstorage-pvc - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true resources: requests: memory: "512Mi" cpu: "250m" limits: memory: "1024Mi" cpu: "500m" volumes: - name: config configMap: name: objstorage defaultMode: 0777 - name: objstorage-pvc persistentVolumeClaim: claimName: objstorage-pvc --- apiVersion: v1 kind: Service metadata: name: objstorage spec: type: ClusterIP selector: app: objstorage ports: - port: 5003 targetPort: 5003 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: objstorage spec: rules: - host: objstorage.default http: paths: - path: / pathType: Prefix backend: service: name: objstorage port: number: 5003 diff --git a/kubernetes/15-scheduler-db.yml b/kubernetes/15-scheduler-db.yml index 81ef08b..0847158 100644 --- a/kubernetes/15-scheduler-db.yml +++ b/kubernetes/15-scheduler-db.yml @@ -1,129 +1,129 @@ --- apiVersion: v1 kind: PersistentVolume metadata: name: scheduler-db-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: scheduler-db local: - path: /srv/softwareheritage/dev/scheduler-db + path: /srv/softwareheritage-kube/dev/scheduler-db nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: scheduler-db-pvc spec: accessModes: - ReadWriteOnce storageClassName: scheduler-db resources: requests: storage: 10Gi --- ## TODO Change this to your real postgresql password apiVersion: v1 kind: Secret metadata: name: scheduler-db type: Opaque stringData: POSTGRES_PASSWORD: swh --- apiVersion: v1 kind: ConfigMap metadata: name: scheduler-db data: # property-like keys; each key maps to a simple value POSTGRES_USER: swh POSTGRES_DB: swh-scheduler --- apiVersion: apps/v1 kind: Deployment metadata: name: scheduler-db labels: app: scheduler-db spec: replicas: 1 selector: matchLabels: app: scheduler-db template: metadata: labels: app: scheduler-db spec: containers: - name: scheduler-db image: postgres:13.0 imagePullPolicy: Always ports: - containerPort: 5432 args: - "-c" - "shared_buffers=512MB" - "-c" - "effective_cache_size=512MB" - "-c" - "random_page_cost=1.5" - "-c" - "max_wal_size=512MB" env: - name: POSTGRES_USER valueFrom: configMapKeyRef: name: scheduler-db key: POSTGRES_USER - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: scheduler-db key: POSTGRES_PASSWORD - name: POSTGRES_DB valueFrom: configMapKeyRef: name: scheduler-db key: POSTGRES_DB volumeMounts: - mountPath: "/var/lib/postgresql" name: scheduler-db-pvc resources: requests: memory: "512Mi" cpu: "250m" limits: memory: "1024Mi" cpu: "500m" volumes: - name: scheduler-db-pvc persistentVolumeClaim: claimName: scheduler-db-pvc --- apiVersion: v1 kind: Service metadata: name: scheduler-db spec: type: ClusterIP selector: app: scheduler-db ports: - port: 5432 targetPort: 5432 diff --git a/kubernetes/Readme.md b/kubernetes/Readme.md index bcdd18b..7d2a180 100644 --- a/kubernetes/Readme.md +++ b/kubernetes/Readme.md @@ -1,197 +1,196 @@ ## Prerequisite ### Directories ``` -# sudo mkdir -p /srv/softwareheritage-kube/objects -# sudo mkdir -p /srv/softwareheritage-kube/storage-db -# sudo mkdir -p /srv/softwareheritage-kube/scheduler-db +# sudo mkdir -p /srv/softwareheritage-kube/dev/{objects,storage-db,scheduler-db} ``` + Must match the content of `05-storage-db.yaml` ### Registry - Add the following line on your `/etc/hosts` file. It's needed to be able to push the image to it from docker ``` 127.0.0.1 registry.default ``` - Start the registry in kubernetes ``` # cd kubernetes # kubectl apply -f registry/00-registry.yml ``` ## Build the base image ``` # cd docker # docker build --no-cache -t swh/stack . # docker tag swh/stack:latest registry.default/swh/stack:latest # docker push registry.default/swh/stack:latest ``` ## start the objstorage - build image ``` # docker build -f Dockerfile.objstorage -t swh/objstorage --build-arg BASE=swh/stack . # docker tag swh/objstorage:latest registry.default/swh/objstorage:latest # docker push registry.default/swh/objstorage:latest ``` - start the service ``` # cd kubernetes # kubectl apply -f 10-objstorage.yml configmap/objstorage created persistentvolume/objstorage-pv created persistentvolumeclaim/objstorage-pvc created deployment.apps/objstorage created service/objstorage created ``` - test it ``` # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 46m objstorage-8587d58b68-76jbn 1/1 Running 0 12m # kubectl get services objstorage NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE objstorage ClusterIP 10.43.185.191 5003/TCP 17m # curl http://$(kubectl get services objstorage -o jsonpath='{.spec.clusterIP}'):5003 SWH Objstorage API server% ``` ## Start the storage - Start the db ``` # cd kubernetes # kubectl apply -f 05-storage-db.yml persistentvolume/storage-db-pv created persistentvolumeclaim/storage-db-pvc created secret/storage-db created configmap/storage-db created deployment.apps/storage-db created service/storage-db created # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 46m objstorage-8587d58b68-76jbn 1/1 Running 0 15m storage-db-64b7f8b684-48n7w 1/1 Running 0 4m52s # kubectl get services storage-db NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE storage-db ClusterIP 10.43.213.178 5432/TCP 8m19s ``` - Start the storage ``` # cd kubernetes # kubectl apply -f 11-storage.yml configmap/storage created deployment.apps/storage created service/storage created ``` - Test the service ``` # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 49m storage-db-64b7f8b684-48n7w 1/1 Running 0 7m40s storage-6b759fb974-w9rzj 1/1 Running 0 66s # kubectl get services storage NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE storage ClusterIP 10.43.212.116 5002/TCP 2m24s # curl http://$(kubectl get services storage -o jsonpath='{.spec.clusterIP}'):5002 Software Heritage storage server

You have reached the Software Heritage storage server.
See its documentation and API for more information

``` ## Start the scheduler - Start the db ``` # cd kubernetes # kubectl apply -f 15-scheduler-db.yml persistentvolume/scheduler-db-pv unchanged persistentvolumeclaim/scheduler-db-pvc created secret/scheduler-db configured configmap/scheduler-db unchanged deployment.apps/scheduler-db unchanged service/scheduler-db unchanged # kubectl get services scheduler-db NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE scheduler-db ClusterIP 10.43.115.249 5433/TCP 110s ``` - Test the service ``` # kubectl apply -f 20-scheduler.yml configmap/scheduler created deployment.apps/scheduler created service/scheduler created ingress.networking.k8s.io/scheduler created # kubectl get services scheduler NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE scheduler ClusterIP 10.43.218.74 5008/TCP 23s # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-5f6894c5b-9wkmr 1/1 Running 0 28m objstorage-5b87c549b6-f6jvc 1/1 Running 0 12m storage-db-79bfbff68-mg7fr 1/1 Running 0 107s storage-6bfcb87b6-7s7t8 1/1 Running 0 87s scheduler-db-666c8dc8b4-qxm9d 1/1 Running 0 73s scheduler-595f944854-hbsj4 1/1 Running 0 62s # curl http://$(kubectl get services scheduler -o jsonpath='{.spec.clusterIP}'):5008 Software Heritage scheduler RPC server

You have reached the Software Heritage scheduler RPC server.
See its documentation and API for more information

% ``` ## Development ### Skaffold To start the development environment using skaffold, use the following command: ``` skaffold --default-repo registry.default dev ``` It will build the images, deploy them on the local registry and start the services. It will monitor the projects to detect the changes and restart the containers when needed