diff --git a/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml b/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml index 48c0bda..d527e13 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml @@ -1,187 +1,194 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: objstorage data: config.yml: | objstorage: cls: pathslicing args: root: "/srv/softwareheritage/objects" slicing: 0:2/2:4/4:6 client_max_size: 1073741824 entrypoint.sh: | #!/bin/bash set -e # source /srv/softwareheritage/utils/pyutils.sh # setup_pip # echo Installed Python packages: # pip list echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ --workers 200 \ --reload \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.objstorage.api.server:make_app_from_configfile()' --- apiVersion: v1 kind: PersistentVolume metadata: name: objstorage-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: objstorage-pv local: path: /srv/data/softwareheritage-kube/objects nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # - key: kubernetes.io/hostname # operator: In # values: # - parasilo-19.rennes.grid5000.fr - - key: type + - key: worker operator: In values: - - loader + - 'true' --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: objstorage-pvc spec: accessModes: - ReadWriteOnce storageClassName: objstorage-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: objstorage labels: app: objstorage spec: replicas: 4 selector: matchLabels: app: objstorage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: objstorage spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - # - key: kubernetes.io/hostname - # operator: In - # values: - # - parasilo-19.rennes.grid5000.fr - - key: type + - key: worker operator: In values: - - loader + - 'true' + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - objstorage + topologyKey: kubernetes.io/hostname containers: - name: objstorage image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5003 readinessProbe: httpGet: path: / port: 5003 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5003 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5003" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - mountPath: "/srv/softwareheritage/objects" name: objstorage-pvc - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true volumes: - name: config configMap: name: objstorage defaultMode: 0777 - name: objstorage-pvc persistentVolumeClaim: claimName: objstorage-pvc --- apiVersion: v1 kind: Service metadata: name: objstorage spec: type: ClusterIP selector: app: objstorage ports: - port: 5003 targetPort: 5003 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: objstorage spec: rules: - host: objstorage.default http: paths: - path: / pathType: Prefix backend: service: name: objstorage port: number: 5003 diff --git a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml index 47a5f4d..3eb7bc3 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml @@ -1,154 +1,170 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: cassandra hosts: - parasilo-2.rennes.grid5000.fr - parasilo-3.rennes.grid5000.fr - parasilo-4.rennes.grid5000.fr - parasilo-5.rennes.grid5000.fr - parasilo-6.rennes.grid5000.fr - parasilo-7.rennes.grid5000.fr - parasilo-8.rennes.grid5000.fr keyspace: swh consistency_level: LOCAL_QUORUM + #select_missing_algo: concurrent + #select_missing_algo: grouped-naive + #select_missing_algo: grouped-pk-serial + select_missing_algo: grouped-pk-concurrent + directory_entries_insert_algo: batch objstorage: cls: remote url: http://objstorage:5003 entrypoint.sh: | #!/bin/bash set -e echo Starting the swh-storage API server exec gunicorn --bind 0.0.0.0:5002 \ --reload \ --access-logfile /dev/stdout \ --access-logformat "%(t)s %(s)s %(r)s %(M)s" \ --workers 128 \ --log-level INFO \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.storage.api.server:make_app_from_configfile()' --- apiVersion: apps/v1 kind: Deployment metadata: name: storage labels: app: storage spec: replicas: 1 selector: matchLabels: app: storage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: storage spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: type + - key: worker operator: In values: - - loader + - 'true' + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - objstorage + topologyKey: kubernetes.io/hostname containers: - name: storage - image: vsellier/swh-base + image: vsellier/swh-base:D6495 imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 readinessProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5002" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true # resources: # requests: # memory: "10000Mi" # cpu: "5000m" # limits: # memory: "15000Mi" # cpu: "10000m" volumes: - name: config configMap: name: storage defaultMode: 0777 - name: db-password secret: secretName: storage-db --- apiVersion: v1 kind: Service metadata: name: storage spec: type: ClusterIP selector: app: storage ports: - port: 5002 targetPort: 5002 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: storage spec: rules: - host: storage.default http: paths: - path: / pathType: Prefix backend: service: name: storage port: number: 5002 diff --git a/sysadmin/grid5000/cassandra/kubernetes/15-scheduler-db.yml b/sysadmin/grid5000/cassandra/kubernetes/15-scheduler-db.yml index 5c4f570..caf8bd9 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/15-scheduler-db.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/15-scheduler-db.yml @@ -1,138 +1,138 @@ --- apiVersion: v1 kind: PersistentVolume metadata: name: scheduler-db-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: scheduler-db local: path: /srv/data/softwareheritage-kube/scheduler-db nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - - key: kubernetes.io/hostname + - key: storage operator: In values: - - parasilo-19.rennes.grid5000.fr + - 'true' --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: scheduler-db-pvc spec: accessModes: - ReadWriteOnce storageClassName: scheduler-db resources: requests: storage: 10Gi --- ## TODO Change this to your real postgresql password apiVersion: v1 kind: Secret metadata: name: scheduler-db type: Opaque stringData: POSTGRES_PASSWORD: swh --- apiVersion: v1 kind: ConfigMap metadata: name: scheduler-db data: # property-like keys; each key maps to a simple value POSTGRES_USER: swh POSTGRES_DB: swh-scheduler --- apiVersion: apps/v1 kind: Deployment metadata: name: scheduler-db labels: app: scheduler-db spec: replicas: 1 selector: matchLabels: app: scheduler-db template: metadata: labels: app: scheduler-db spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: kubernetes.io/hostname + - key: storage operator: In values: - - parasilo-19.rennes.grid5000.fr + - 'true' containers: - name: scheduler-db image: postgres:13.0 imagePullPolicy: Always ports: - containerPort: 5432 args: - "-c" - "shared_buffers=5GB" - "-c" - "effective_cache_size=512MB" - "-c" - "random_page_cost=1.5" - "-c" - "max_wal_size=1GB" env: - name: POSTGRES_USER valueFrom: configMapKeyRef: name: scheduler-db key: POSTGRES_USER - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: scheduler-db key: POSTGRES_PASSWORD - name: POSTGRES_DB valueFrom: configMapKeyRef: name: scheduler-db key: POSTGRES_DB volumeMounts: - mountPath: "/var/lib/postgresql/data" name: scheduler-db-pvc resources: requests: memory: "5Gi" cpu: "2000m" limits: memory: "10Gi" cpu: "5000m" volumes: - name: scheduler-db-pvc persistentVolumeClaim: claimName: scheduler-db-pvc --- apiVersion: v1 kind: Service metadata: name: scheduler-db spec: type: ClusterIP selector: app: scheduler-db ports: - port: 5432 targetPort: 5432 diff --git a/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml b/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml index ec3ff1c..d2b04ba 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml @@ -1,65 +1,65 @@ --- apiVersion: apps/v1 kind: Deployment metadata: name: amqp labels: app: amqp spec: replicas: 1 selector: matchLabels: app: amqp template: metadata: labels: app: amqp spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: kubernetes.io/hostname + - key: admin operator: In values: - - parasilo-19.rennes.grid5000.fr + - 'true' containers: - name: amqp image: rabbitmq:3.7-management imagePullPolicy: Always ports: - containerPort: 5672 --- apiVersion: v1 kind: Service metadata: name: amqp spec: type: ClusterIP selector: app: amqp ports: - port: 5672 targetPort: 5672 name: rabbitmq - port: 15672 targetPort: 15672 name: rabbitmq-admin --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: amqp spec: rules: - host: rabbitmq.default http: paths: - path: / pathType: Prefix backend: service: name: amqp port: number: 15672 diff --git a/sysadmin/grid5000/cassandra/kubernetes/20-scheduler.yml b/sysadmin/grid5000/cassandra/kubernetes/20-scheduler.yml index 82f4159..148dbf3 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/20-scheduler.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/20-scheduler.yml @@ -1,212 +1,211 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: scheduler data: config.yml: | scheduler: cls: local db: postgresql:///?service=swh-scheduler celery: task_broker: amqp://guest:guest@amqp/%2f broker_transport_options: max_retries: 1 entrypoint-init-db.sh: | #!/bin/bash set -e echo -n "waiting for database availability" set +e ko=1 while [[ $ko -gt 0 ]]; do PGCONNECT_TIMEOUT=2 psql --list &>/dev/null;ko=$?; echo -n .; sleep 0.5; done echo set -e echo Init swh-scheduler database echo Creating extensions... swh db init-admin --db-name ${PGDATABASE} scheduler echo Initializing the database... swh db init --db-name ${PGDATABASE} scheduler entrypoint.sh: | #!/bin/bash set -e echo "${PGHOST}:5432:${PGDATABASE}:${PGUSER}:${PGPASSWORD}" >> ~/.pgpass cat >> ~/.pg_service.conf </dev/null;ko=$?; echo -n .; sleep 0.5; done - echo - set -e - - echo Init swh-scheduler database + source /srv/softwareheritage/utils/pgsql.sh + wait_pgsql ${PGDATABASE} + echo Init swh-vault database echo Creating extensions... - swh db init-admin --db-name ${PGDATABASE} scheduler + swh db init-admin --db-name ${PGDATABASE} vault echo Initializing the database... - swh db init --db-name ${PGDATABASE} scheduler + swh db init --db-name ${PGDATABASE} vault + entrypoint.sh: | #!/bin/bash set -e echo "${PGHOST}:5432:${PGDATABASE}:${PGUSER}:${PGPASSWORD}" >> ~/.pgpass cat >> ~/.pg_service.conf <