diff --git a/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml b/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml index 0037406..440db29 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/10-objstorage.yml @@ -1,180 +1,180 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: objstorage data: config.yml: | objstorage: cls: pathslicing args: root: "/srv/softwareheritage/objects" slicing: 0:2/2:4/4:6 client_max_size: 1073741824 entrypoint.sh: | #!/bin/bash set -e # source /srv/softwareheritage/utils/pyutils.sh # setup_pip # echo Installed Python packages: # pip list echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ - --workers 300 \ + --workers 200 \ --reload \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.objstorage.api.server:make_app_from_configfile()' --- apiVersion: v1 kind: PersistentVolume metadata: name: objstorage-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: objstorage-pv local: path: /srv/data/softwareheritage-kube/objects nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/hostname operator: In values: - parasilo-19.rennes.grid5000.fr --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: objstorage-pvc spec: accessModes: - ReadWriteOnce storageClassName: objstorage-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: objstorage labels: app: objstorage spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: objstorage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: objstorage spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - parasilo-19.rennes.grid5000.fr containers: - name: objstorage image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5003 readinessProbe: httpGet: path: / port: 5003 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5003 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5003" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - mountPath: "/srv/softwareheritage/objects" name: objstorage-pvc - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true volumes: - name: config configMap: name: objstorage defaultMode: 0777 - name: objstorage-pvc persistentVolumeClaim: claimName: objstorage-pvc --- apiVersion: v1 kind: Service metadata: name: objstorage spec: type: ClusterIP selector: app: objstorage ports: - port: 5003 targetPort: 5003 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: objstorage spec: rules: - host: objstorage.default http: paths: - path: / pathType: Prefix backend: service: name: objstorage port: number: 5003 diff --git a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml index 630796d..3f0793c 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml @@ -1,146 +1,146 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: cassandra hosts: - parasilo-2.rennes.grid5000.fr keyspace: swh consistency_level: LOCAL_QUORUM objstorage: cls: remote url: http://objstorage:5003 entrypoint.sh: | #!/bin/bash set -e echo Starting the swh-storage API server exec gunicorn --bind 0.0.0.0:5002 \ --reload \ --workers 128 \ --log-level DEBUG \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.storage.api.server:make_app_from_configfile()' --- apiVersion: apps/v1 kind: Deployment metadata: name: storage labels: app: storage spec: replicas: 1 selector: matchLabels: app: storage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: storage spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: In values: - loader containers: - name: storage image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 readinessProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5002" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - resources: - requests: - memory: "10000Mi" - cpu: "5000m" - limits: - memory: "15000Mi" - cpu: "10000m" + # resources: + # requests: + # memory: "10000Mi" + # cpu: "5000m" + # limits: + # memory: "15000Mi" + # cpu: "10000m" volumes: - name: config configMap: name: storage defaultMode: 0777 - name: db-password secret: secretName: storage-db --- apiVersion: v1 kind: Service metadata: name: storage spec: type: ClusterIP selector: app: storage ports: - port: 5002 targetPort: 5002 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: storage spec: rules: - host: storage.default http: paths: - path: / pathType: Prefix backend: service: name: storage port: number: 5002 diff --git a/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml b/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml index 83f7f3b..ec3ff1c 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/16-rabbitmq.yml @@ -1,65 +1,65 @@ --- apiVersion: apps/v1 kind: Deployment metadata: name: amqp labels: app: amqp spec: replicas: 1 selector: matchLabels: app: amqp template: metadata: labels: app: amqp spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - parasilo-19.rennes.grid5000.fr containers: - name: amqp - image: rabbitmq:3.6-management + image: rabbitmq:3.7-management imagePullPolicy: Always ports: - containerPort: 5672 --- apiVersion: v1 kind: Service metadata: name: amqp spec: type: ClusterIP selector: app: amqp ports: - port: 5672 targetPort: 5672 name: rabbitmq - port: 15672 targetPort: 15672 name: rabbitmq-admin --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: amqp spec: rules: - host: rabbitmq.default http: paths: - path: / pathType: Prefix backend: service: name: amqp port: number: 15672 diff --git a/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml b/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml index 96f81e4..4279323 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml @@ -1,160 +1,160 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: loaders data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 10000 content_bytes: 104857600 directory: 1000 revision: 1000 - cls: filter - cls: retry - cls: remote url: http://storage:5002/ max_content_size: 104857600 temp_file_cutoff: 536870912 - celery: + task_ignore_result: True task_broker: amqp://guest:guest@amqp// task_queues: - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - swh.loader.git.tasks.UpdateGitRepository - swh.loader.package.archive.tasks.LoadArchive - swh.loader.package.cran.tasks.LoadCRAN - swh.loader.package.debian.tasks.LoadDebian - swh.loader.package.npm.tasks.LoadNpm - swh.loader.package.pypi.tasks.LoadPyPI - save_code_now:swh.loader.git.tasks.UpdateGitRepository # - swh.loader.mercurial.tasks.LoadArchiveMercurial # - swh.loader.mercurial.tasks.LoadMercurial # - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # - swh.loader.svn.tasks.LoadSvnRepository # - swh.loader.svn.tasks.MountAndLoadSvnRepository entrypoint-init.sh: | #!/bin/bash set -e # echo Waiting for RabbitMQ to start wait-for-it amqp:5672 -s --timeout=0 # echo Register task types in scheduler database wait-for-it scheduler:5008 -s --timeout=0 swh scheduler --url http://scheduler:5008 task-type register entrypoint.sh: | #!/bin/bash set -e echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ - --pool=solo --events \ + --pool=prefork --events \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" --- apiVersion: apps/v1 kind: Deployment metadata: name: loaders labels: app: loaders spec: replicas: 1 selector: matchLabels: app: loaders strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: loaders spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: In values: - loader initContainers: - name: loaders-init image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint-init.sh readOnly: true containers: - name: loaders image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 resources: requests: memory: "2000Mi" cpu: "500m" limits: memory: "4000Mi" cpu: "1000m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - name: tmp mountPath: /tmp volumes: - name: config configMap: name: loaders defaultMode: 0777 - name: tmp emptyDir: {} diff --git a/sysadmin/grid5000/cassandra/kubernetes/kubernetes-dashboard-user.yaml b/sysadmin/grid5000/cassandra/kubernetes/kubernetes-dashboard-user.yaml new file mode 100644 index 0000000..f38e9e6 --- /dev/null +++ b/sysadmin/grid5000/cassandra/kubernetes/kubernetes-dashboard-user.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin-user + namespace: kubernetes-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-user +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: admin-user + namespace: kubernetes-dashboard + diff --git a/sysadmin/grid5000/cassandra/kubernetes/kubernetes-dashboard.yaml b/sysadmin/grid5000/cassandra/kubernetes/kubernetes-dashboard.yaml new file mode 100644 index 0000000..1f0dd44 --- /dev/null +++ b/sysadmin/grid5000/cassandra/kubernetes/kubernetes-dashboard.yaml @@ -0,0 +1,303 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: kubernetes-dashboard + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: kubernetes-dashboard +type: Opaque + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: kubernetes-dashboard +type: Opaque +data: + csrf: "" + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: kubernetes-dashboard +type: Opaque + +--- + +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: kubernetes-dashboard + +--- + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard +rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - name: kubernetes-dashboard + image: kubernetesui/dashboard:v2.3.1 + imagePullPolicy: Always + ports: + - containerPort: 8443 + protocol: TCP + args: + - --auto-generate-certificates + - --namespace=kubernetes-dashboard + - --token-ttl=0 + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + nodeSelector: + "kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + +--- + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: dashboard-metrics-scraper + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: dashboard-metrics-scraper + template: + metadata: + labels: + k8s-app: dashboard-metrics-scraper + annotations: + seccomp.security.alpha.kubernetes.io/pod: 'runtime/default' + spec: + containers: + - name: dashboard-metrics-scraper + image: kubernetesui/metrics-scraper:v1.0.6 + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + serviceAccountName: kubernetes-dashboard + nodeSelector: + "kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + - name: tmp-volume + emptyDir: {}