diff --git a/kubernetes/01-journal.yml b/kubernetes/01-journal.yml index 613323c..5894a0e 100644 --- a/kubernetes/01-journal.yml +++ b/kubernetes/01-journal.yml @@ -1,238 +1,238 @@ apiVersion: v1 kind: PersistentVolume metadata: name: zookeeper-data-pv spec: capacity: storage: 1Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: zookeeper-data-pv local: path: /srv/softwareheritage-kube/dev/zookeeper/data nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- apiVersion: v1 kind: PersistentVolume metadata: name: zookeeper-datalog-pv spec: capacity: storage: 1Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: zookeeper-datalog-pv local: path: /srv/softwareheritage-kube/dev/zookeeper/datalog nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: zookeeper-data-pvc spec: accessModes: - ReadWriteOnce storageClassName: zookeeper-data-pv resources: requests: storage: 1Gi --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: zookeeper-datalog-pvc spec: accessModes: - ReadWriteOnce storageClassName: zookeeper-datalog-pv resources: requests: storage: 1Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: zookeeper labels: app: zookeeper spec: replicas: 1 selector: matchLabels: app: zookeeper template: metadata: labels: app: zookeeper spec: containers: - name: zookeeper image: zookeeper:3.6 imagePullPolicy: Always ports: - containerPort: 2181 resources: requests: - memory: "128Mi" + memory: "100Mi" cpu: "50m" limits: - memory: "256Mi" + memory: "200Mi" cpu: "100m" volumeMounts: - mountPath: "/data" name: zookeeper-data-pvc - mountPath: "/datalog" name: zookeeper-datalog-pvc volumes: - name: zookeeper-data-pvc persistentVolumeClaim: claimName: zookeeper-data-pvc - name: zookeeper-datalog-pvc persistentVolumeClaim: claimName: zookeeper-datalog-pvc --- apiVersion: v1 kind: Service metadata: name: zookeeper spec: type: ClusterIP selector: app: zookeeper ports: - port: 2181 targetPort: 2181 --- apiVersion: v1 kind: ConfigMap metadata: name: kafka data: KAFKA_ADVERTISED_HOST_NAME: kafka KAFKA_ADVERTISED_PORT: "9092" KAFKA_PORT: "9092" KAFKA_LISTENERS: PLAINTEXT://:9092 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_MESSAGE_MAX_BYTES: "104857600" KAFKA_LOG_DIRS: /kafka/logs KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099 JMX_PORT: "1099" LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER: DEBUG, authorizerAppender --- apiVersion: v1 kind: PersistentVolume metadata: name: kafka-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: kafka-pv local: path: /srv/softwareheritage-kube/dev/kafka nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: kafka-pvc spec: accessModes: - ReadWriteOnce storageClassName: kafka-pv resources: requests: storage: 10Gi --- apiVersion: v1 kind: Service metadata: name: kafka spec: selector: app: kafka ports: - port: 9092 targetPort: 9092 --- apiVersion: apps/v1 kind: Deployment metadata: name: kafka labels: app: kafka spec: replicas: 1 selector: matchLabels: app: kafka template: metadata: labels: app: kafka spec: containers: - name: kafka image: wurstmeister/kafka imagePullPolicy: Always ports: - containerPort: 9092 env: - name: KAFKA_CREATE_TOPICS value: swh.journal.objects.content:10:1:compact, swh.journal.objects.origin:10:1:compact, swh.journal.objects.origin_visit:10:1:compact, swh.journal.objects.origin_visit_status:10:1:compact, swh.journal.objects.skipped_content:10:1:compact, swh.journal.objects.revision:10:1:compact, swh.journal.indexed.origin_intrinsic_metadata:10:1:compact envFrom: - configMapRef: name: kafka resources: requests: + memory: "512Mi" + cpu: "100m" + limits: memory: "1024Mi" cpu: "200m" - limits: - memory: "1536Mi" - cpu: "300m" volumeMounts: - mountPath: "/kafka" name: kafka-pvc volumes: - name: kafka-pvc persistentVolumeClaim: claimName: kafka-pvc diff --git a/kubernetes/02-monitoring.yml b/kubernetes/02-monitoring.yml index 042f835..201e586 100644 --- a/kubernetes/02-monitoring.yml +++ b/kubernetes/02-monitoring.yml @@ -1,334 +1,337 @@ # ------- # Prometheus statsd exporter configuration # ------- --- apiVersion: v1 kind: ConfigMap metadata: name: prometheus-statsd-exporter data: config.yml: | defaults: timer_type: histogram buckets: - .005 - .01 - .025 - .05 - .1 - .25 - .5 - .75 - 1 - 2 - 5 - 10 - 15 - 30 - 45 - 60 - 120 - 300 - 600 - 900 - 1800 - 2700 - 3600 - 7200 --- apiVersion: apps/v1 kind: Deployment metadata: name: prometheus-statsd-exporter labels: app: prometheus-statsd-exporter spec: replicas: 1 selector: matchLabels: app: prometheus-statsd-exporter template: metadata: labels: app: prometheus-statsd-exporter spec: containers: - name: prometheus-statsd-exporter image: prom/statsd-exporter imagePullPolicy: Always args: - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml" ports: - containerPort: 9125 volumeMounts: - name: config mountPath: /etc/prometheus/statsd-mapping.yml subPath: config.yml readOnly: true volumes: - name: config configMap: name: prometheus-statsd-exporter --- apiVersion: v1 kind: Service metadata: name: prometheus-statsd-exporter spec: type: ClusterIP selector: app: prometheus-statsd-exporter ports: - name: statsd port: 9125 targetPort: 9125 - name: http port: 9102 targetPort: 9102 --- # ------- # Prometheus configuration # ------- apiVersion: v1 kind: ConfigMap metadata: name: prometheus data: config.yml: | # my global config global: scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. # scrape_timeout is set to the global default (10s). scrape_configs: - job_name: prometheus static_configs: - targets: - prometheus:9090 metrics_path: /prometheus/metrics - job_name: statsd-exporter static_configs: - targets: - prometheus-statsd-exporter:9102 - job_name: jmx-exporter-cassandra static_configs: - targets: - prometheus-jmx-exporter-cassandra:5556 --- apiVersion: v1 kind: PersistentVolume metadata: name: prometheus-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: prometheus-pv local: path: /srv/softwareheritage-kube/dev/prometheus nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- apiVersion: v1 kind: Service metadata: name: prometheus spec: type: ClusterIP selector: app: prometheus ports: - port: 9090 targetPort: 9090 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: prometheus spec: rules: - host: prometheus.default http: paths: - path: / pathType: Prefix backend: service: name: prometheus port: number: 9090 --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: prometheus-pvc spec: accessModes: - ReadWriteOnce storageClassName: prometheus-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: prometheus labels: app: prometheus spec: replicas: 1 selector: matchLabels: app: prometheus template: metadata: labels: app: prometheus spec: containers: - name: prometheus image: prom/prometheus imagePullPolicy: Always args: - "--config.file=/etc/prometheus/prometheus.yml" ports: - containerPort: 9090 volumeMounts: - name: config mountPath: /etc/prometheus/prometheus.yml subPath: config.yml readOnly: true - mountPath: "/prometheus/data" name: prometheus-pvc resources: + requests: + cpu: "20m" + memory: "100Mi" limits: cpu: "100m" memory: "256Mi" volumes: - name: config configMap: name: prometheus - name: prometheus-pvc persistentVolumeClaim: claimName: prometheus-pvc --- apiVersion: v1 kind: PersistentVolume metadata: name: grafana-pv spec: capacity: storage: 1Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: grafana-pv local: path: /srv/softwareheritage-kube/dev/grafana nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: grafana-pvc spec: accessModes: - ReadWriteOnce storageClassName: grafana-pv resources: requests: storage: 1Gi --- apiVersion: v1 kind: Service metadata: name: grafana spec: type: ClusterIP selector: app: grafana ports: - port: 3000 targetPort: 3000 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: grafana spec: rules: - host: grafana.default http: paths: - path: / pathType: Prefix backend: service: name: grafana port: number: 3000 --- apiVersion: apps/v1 kind: Deployment metadata: name: grafana labels: app: grafana spec: replicas: 1 selector: matchLabels: app: grafana template: metadata: labels: app: grafana spec: containers: - name: grafana image: swh/grafana imagePullPolicy: Always env: - name: GF_SERVER_ROOT_URL value: http://grafana.default ports: - containerPort: 3000 resources: requests: cpu: "50m" memory: "128Mi" limits: cpu: "100m" memory: "256Mi" volumeMounts: - mountPath: "/var/lib/grafana" name: grafana-pvc volumes: - name: grafana-pvc persistentVolumeClaim: claimName: grafana-pvc diff --git a/kubernetes/05-storage-db.yml b/kubernetes/05-storage-db.yml index 631e51e..3e4c7ae 100644 --- a/kubernetes/05-storage-db.yml +++ b/kubernetes/05-storage-db.yml @@ -1,131 +1,131 @@ --- apiVersion: v1 kind: PersistentVolume metadata: name: storage-db-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: storage-db local: path: /srv/softwareheritage-kube/dev/storage-db nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: storage-db-pvc spec: accessModes: - ReadWriteOnce storageClassName: storage-db resources: requests: storage: 10Gi --- ## TODO Change this to your real postgresql password apiVersion: v1 kind: Secret metadata: name: storage-db type: Opaque # data: # POSTGRES_PASSWORD: | # "echo 'strong password' | base64" stringData: POSTGRES_PASSWORD: swh --- apiVersion: v1 kind: ConfigMap metadata: name: storage-db data: # property-like keys; each key maps to a simple value POSTGRES_USER: swh POSTGRES_DB: swh --- apiVersion: apps/v1 kind: Deployment metadata: name: storage-db labels: app: storage-db spec: replicas: 1 selector: matchLabels: app: storage-db template: metadata: labels: app: storage-db spec: containers: - name: storage-db image: postgres:13.0 imagePullPolicy: Always ports: - containerPort: 5432 args: - "-c" - - "shared_buffers=512MB" + - "shared_buffers=256MB" - "-c" - - "effective_cache_size=512MB" + - "effective_cache_size=256MB" - "-c" - "random_page_cost=1.5" - "-c" - - "max_wal_size=512MB" + - "max_wal_size=256MB" env: - name: POSTGRES_USER valueFrom: configMapKeyRef: name: storage-db key: POSTGRES_USER - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: storage-db key: POSTGRES_PASSWORD - name: POSTGRES_DB valueFrom: configMapKeyRef: name: storage-db key: POSTGRES_DB volumeMounts: - mountPath: "/var/lib/postgresql/data" name: storage-db-pvc resources: requests: + memory: "256Mi" + cpu: "100m" + limits: memory: "512Mi" cpu: "250m" - limits: - memory: "1024Mi" - cpu: "500m" volumes: - name: storage-db-pvc persistentVolumeClaim: claimName: storage-db-pvc --- apiVersion: v1 kind: Service metadata: name: storage-db spec: type: ClusterIP selector: app: storage-db ports: - port: 5432 targetPort: 5432 diff --git a/kubernetes/10-objstorage.yml b/kubernetes/10-objstorage.yml index 367a15e..05e6635 100644 --- a/kubernetes/10-objstorage.yml +++ b/kubernetes/10-objstorage.yml @@ -1,162 +1,178 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: objstorage data: config.yml: | objstorage: cls: pathslicing args: root: "/srv/softwareheritage/objects" slicing: 0:5 client_max_size: 1073741824 entrypoint.sh: | #!/bin/bash set -e # source /srv/softwareheritage/utils/pyutils.sh # setup_pip # echo Installed Python packages: # pip list echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ --threads 4 \ --workers 2 \ --reload \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.objstorage.api.server:make_app_from_configfile()' --- apiVersion: v1 kind: PersistentVolume metadata: name: objstorage-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: objstorage-pv local: path: /srv/softwareheritage-kube/dev/objects nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: objstorage-pvc spec: accessModes: - ReadWriteOnce storageClassName: objstorage-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: objstorage labels: app: objstorage spec: replicas: 1 selector: matchLabels: app: objstorage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: objstorage spec: containers: - name: objstorage image: swh/objstorage:latest command: - /entrypoint.sh ports: - containerPort: 5003 + readinessProbe: + httpGet: + path: / + port: 5003 + scheme: "HTTP" + initialDelaySeconds: 0 + failureThreshold: 2 + periodSeconds: 10 + startupProbe: + httpGet: + path: / + port: 5003 + scheme: "HTTP" + initialDelaySeconds: 5 + failureThreshold: 30 + periodSeconds: 1 env: - name: PORT value: "5003" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - mountPath: "/srv/softwareheritage/objects" name: objstorage-pvc - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true resources: requests: memory: "512Mi" cpu: "250m" limits: - memory: "1024Mi" + memory: "768Mi" cpu: "500m" volumes: - name: config configMap: name: objstorage defaultMode: 0777 - name: objstorage-pvc persistentVolumeClaim: claimName: objstorage-pvc --- apiVersion: v1 kind: Service metadata: name: objstorage spec: type: ClusterIP selector: app: objstorage ports: - port: 5003 targetPort: 5003 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: objstorage spec: rules: - host: objstorage.default http: paths: - path: / pathType: Prefix backend: service: name: objstorage port: number: 5003 diff --git a/kubernetes/11-storage.yml b/kubernetes/11-storage.yml index 374931b..0aaa0b8 100644 --- a/kubernetes/11-storage.yml +++ b/kubernetes/11-storage.yml @@ -1,216 +1,232 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: local db: postgresql:///?service=swh objstorage: cls: remote url: http://objstorage:5003/ journal_writer: cls: kafka args: brokers: - kafka prefix: swh.journal.objects client_id: swh.storage.master entrypoint-init-db.sh: | #!/bin/bash set -e echo -n "waiting for database availability" set +e ko=1 while [[ $ko -gt 0 ]]; do PGCONNECT_TIMEOUT=2 psql --list &>/dev/null;ko=$?; echo -n .; sleep 0.5; done echo set -e echo Init swh-storage database echo Creating extensions... swh db init-admin --db-name ${PGDATABASE} storage echo Initializing the database... swh db init --db-name ${PGDATABASE} storage entrypoint.sh: | #!/bin/bash set -e echo "${PGHOST}:5432:${PGDATABASE}:${PGUSER}:${PGPASSWORD}" >> ~/.pgpass cat >> ~/.pg_service.conf <