diff --git a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml b/sysadmin/grid5000/cassandra/kubernetes/11-storage-vault.yml similarity index 90% copy from sysadmin/grid5000/cassandra/kubernetes/11-storage.yml copy to sysadmin/grid5000/cassandra/kubernetes/11-storage-vault.yml index 3eb7bc3..74048b1 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/11-storage-vault.yml @@ -1,170 +1,170 @@ --- apiVersion: v1 kind: ConfigMap metadata: - name: storage + name: storage-vault data: config.yml: | storage: cls: cassandra hosts: - parasilo-2.rennes.grid5000.fr - parasilo-3.rennes.grid5000.fr - parasilo-4.rennes.grid5000.fr - parasilo-5.rennes.grid5000.fr - parasilo-6.rennes.grid5000.fr - parasilo-7.rennes.grid5000.fr - parasilo-8.rennes.grid5000.fr keyspace: swh consistency_level: LOCAL_QUORUM #select_missing_algo: concurrent #select_missing_algo: grouped-naive #select_missing_algo: grouped-pk-serial select_missing_algo: grouped-pk-concurrent directory_entries_insert_algo: batch objstorage: cls: remote - url: http://objstorage:5003 + url: http://objstorage-vault:5003 entrypoint.sh: | #!/bin/bash set -e echo Starting the swh-storage API server exec gunicorn --bind 0.0.0.0:5002 \ --reload \ --access-logfile /dev/stdout \ --access-logformat "%(t)s %(s)s %(r)s %(M)s" \ --workers 128 \ --log-level INFO \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.storage.api.server:make_app_from_configfile()' --- apiVersion: apps/v1 kind: Deployment metadata: - name: storage + name: storage-vault labels: - app: storage + app: storage-vault spec: replicas: 1 selector: matchLabels: - app: storage + app: storage-vault strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: - app: storage + app: storage-vault spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: worker operator: In values: - 'true' podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - - objstorage + - storage topologyKey: kubernetes.io/hostname containers: - - name: storage + - name: storage-vault image: vsellier/swh-base:D6495 imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 readinessProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5002" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true # resources: # requests: # memory: "10000Mi" # cpu: "5000m" # limits: # memory: "15000Mi" # cpu: "10000m" volumes: - name: config configMap: - name: storage + name: storage-vault defaultMode: 0777 - name: db-password secret: secretName: storage-db --- apiVersion: v1 kind: Service metadata: - name: storage + name: storage-vault spec: type: ClusterIP selector: - app: storage + app: storage-vault ports: - port: 5002 targetPort: 5002 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: storage + name: storage-vault spec: rules: - - host: storage.default + - host: storage-vault.default http: paths: - path: / pathType: Prefix backend: service: - name: storage + name: storage-vault port: number: 5002 diff --git a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml index 3eb7bc3..5c75071 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml @@ -1,170 +1,170 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: cassandra hosts: - parasilo-2.rennes.grid5000.fr - parasilo-3.rennes.grid5000.fr - parasilo-4.rennes.grid5000.fr - parasilo-5.rennes.grid5000.fr - parasilo-6.rennes.grid5000.fr - parasilo-7.rennes.grid5000.fr - parasilo-8.rennes.grid5000.fr keyspace: swh consistency_level: LOCAL_QUORUM #select_missing_algo: concurrent #select_missing_algo: grouped-naive #select_missing_algo: grouped-pk-serial select_missing_algo: grouped-pk-concurrent directory_entries_insert_algo: batch objstorage: cls: remote url: http://objstorage:5003 entrypoint.sh: | #!/bin/bash set -e echo Starting the swh-storage API server exec gunicorn --bind 0.0.0.0:5002 \ --reload \ --access-logfile /dev/stdout \ --access-logformat "%(t)s %(s)s %(r)s %(M)s" \ --workers 128 \ --log-level INFO \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.storage.api.server:make_app_from_configfile()' --- apiVersion: apps/v1 kind: Deployment metadata: name: storage labels: app: storage spec: replicas: 1 selector: matchLabels: app: storage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: storage spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: worker operator: In values: - 'true' podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 1 podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - - objstorage + - storage topologyKey: kubernetes.io/hostname containers: - name: storage image: vsellier/swh-base:D6495 imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 readinessProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5002" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true # resources: # requests: # memory: "10000Mi" # cpu: "5000m" # limits: # memory: "15000Mi" # cpu: "10000m" volumes: - name: config configMap: name: storage defaultMode: 0777 - name: db-password secret: secretName: storage-db --- apiVersion: v1 kind: Service metadata: name: storage spec: type: ClusterIP selector: app: storage ports: - port: 5002 targetPort: 5002 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: storage spec: rules: - host: storage.default http: paths: - path: / pathType: Prefix backend: service: name: storage port: number: 5002 diff --git a/sysadmin/grid5000/cassandra/kubernetes/21-scheduler-runner.yml b/sysadmin/grid5000/cassandra/kubernetes/21-scheduler-runner.yml index e45fca1..61da25a 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/21-scheduler-runner.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/21-scheduler-runner.yml @@ -1,142 +1,196 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: scheduler-runner data: config.yml: | scheduler: cls: remote url: http://scheduler:5008 celery: task_broker: amqp://guest:guest@amqp/%2f broker_transport_options: max_retries: 1 entrypoint-init.sh: | #!/bin/bash set -e echo Waiting on the scheduler service wait-for-it scheduler:5008 -s --timeout=0 echo Waiting on the rabbit exec wait-for-it amqp:5672 -s --timeout=0 entrypoint.sh: | #!/bin/bash set -e echo "Starting the swh-scheduler" - swh --log-level ${LOGLEVEL} scheduler -C /etc/softwareheritage/config.yml start-runner --period 10 + swh --log-level ${LOGLEVEL} scheduler -C /etc/softwareheritage/config.yml start-runner --period 10 ${PRIOTITY_ARG} --- apiVersion: apps/v1 kind: Deployment metadata: name: scheduler-runner labels: app: scheduler-runner spec: replicas: 1 selector: matchLabels: app: scheduler-runner strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: scheduler-runner spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: admin operator: In values: - 'true' initContainers: - name: scheduler-runner-init image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint-init.sh volumeMounts: - name: config mountPath: /entrypoint-init.sh subPath: entrypoint-init.sh readOnly: true containers: - name: scheduler-runner image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5008 resources: requests: cpu: "50m" memory: "100Mi" limits: cpu: "200m" memory: "200Mi" env: - name: PORT value: "5008" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml - name: LOGLEVEL value: INFO volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true volumes: - name: config configMap: name: scheduler-runner defaultMode: 0777 --- -apiVersion: v1 -kind: Service +apiVersion: apps/v1 +kind: Deployment metadata: - name: scheduler-runner + name: scheduler-runner-priority + labels: + app: scheduler-runner-priority spec: - type: ClusterIP + replicas: 1 selector: - app: scheduler-runner - ports: - - port: 5008 - targetPort: 5008 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: scheduler-runner -spec: - rules: - - host: scheduler-runner.default - http: - paths: - - path: / - pathType: Prefix - backend: - service: + matchLabels: + app: scheduler-runner-priority + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + template: + metadata: + labels: + app: scheduler-runner-priority + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: admin + operator: In + values: + - 'true' + initContainers: + - name: scheduler-runner-priority-init + image: vsellier/swh-base + imagePullPolicy: Always + command: + - /entrypoint-init.sh + volumeMounts: + - name: config + mountPath: /entrypoint-init.sh + subPath: entrypoint-init.sh + readOnly: true + containers: + - name: scheduler-runner-priority + image: vsellier/swh-base + imagePullPolicy: Always + command: + - /entrypoint.sh + ports: + - containerPort: 5008 + resources: + requests: + cpu: "50m" + memory: "100Mi" + limits: + cpu: "200m" + memory: "200Mi" + env: + - name: PORT + value: "5008" + - name: STATSD_HOST + value: "prometheus-statsd-exporter" + - name: STATSD_PORT + value: "9125" + - name: SWH_CONFIG_FILENAME + value: /etc/softwareheritage/config.yml + - name: LOGLEVEL + value: INFO + - name: PRIORITY_ARG + value: --with-priority + volumeMounts: + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + readOnly: true + volumes: + - name: config + configMap: name: scheduler-runner - port: - number: 5008 + defaultMode: 0777 diff --git a/sysadmin/grid5000/cassandra/kubernetes/81-vault.yml b/sysadmin/grid5000/cassandra/kubernetes/81-vault.yml index ed93716..4a66fd5 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/81-vault.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/81-vault.yml @@ -1,273 +1,276 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: vault data: config.yml: | vault: cls: local db: postgresql:///?service=swh cache: cls: pathslicing root: /srv/softwareheritage/vault slicing: 0:5 storage: - cls: remote - url: http://storage:5002/ + cls: pipeline + steps: + - cls: retry + - cls: remote + url: http://storage-vault:5002/ scheduler: cls: remote url: http://scheduler:5008/ entrypoint-init-db.sh: | #!/bin/bash set -e source /srv/softwareheritage/utils/pgsql.sh wait_pgsql ${PGDATABASE} echo Init swh-vault database echo Creating extensions... swh db init-admin --db-name ${PGDATABASE} vault echo Initializing the database... swh db init --db-name ${PGDATABASE} vault entrypoint.sh: | #!/bin/bash set -e echo "${PGHOST}:5432:${PGDATABASE}:${PGUSER}:${PGPASSWORD}" >> ~/.pgpass cat >> ~/.pg_service.conf <