diff --git a/sysadmin/T3592-elastic-workers/worker/README.md b/sysadmin/T3592-elastic-workers/worker/README.md new file mode 100644 index 0000000..00a6c5c --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/README.md @@ -0,0 +1,22 @@ + +# Goal + +- autoscaling workers depending on repositories to load and allocated resources. + +# keda + +Install KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): +``` +$ helm repo add kedacore https://kedacore.github.io/charts +$ helm repo update +swhworker@poc-rancher:~$ kubectl create namespace keda +namespace/keda created +swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda +NAME: keda +LAST DEPLOYED: Fri Oct 8 09:48:40 2021 +NAMESPACE: keda +STATUS: deployed +REVISION: 1 +TEST SUITE: None +``` +source: https://keda.sh/docs/2.4/deploy/ diff --git a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml index 276bc94..99931b4 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml @@ -1,19 +1,58 @@ --- -apiVersion: autoscaling/v2beta2 -kind: HorizontalPodAutoscaler +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject metadata: - name: loaders + name: loaders-operators spec: scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: loaders - minReplicas: {{ .Values.swh.loader.replicas.min }} - maxReplicas: {{ .Values.swh.loader.replicas.max }} - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 + apiVersion: apps/v1 # Optional. Default: apps/v1 + kind: Deployment # Optional. Default: Deployment + name: loaders # Mandatory. Must be in same namespace as ScaledObject + # envSourceContainerName: {container-name} # Optional. Default: + # .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Must be less than + # minReplicaCount + minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0 + maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100 + fallback: # Optional. Section to specify fallback + # options + failureThreshold: 3 # Mandatory if fallback section is + # included + replicas: 6 # Mandatory if fallback section is + # included + advanced: # Optional. Section to specify advanced + # options + restoreToOriginalReplicaCount: false # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA + # related options + behavior: # Optional. Use to modify HPA's scaling + # behavior + scaleDown: + stabilizationWindowSeconds: 60 # default 300 + policies: + - type: Percent + value: 2 + periodSeconds: 15 + triggers: + {{- range .Values.amqp.queues }} + - type: rabbitmq + metadata: + host: amqp://{{ $.Values.amqp.username }}:{{ $.Values.amqp.password }}@{{ $.Values.amqp.host }}// + # Optional. If not specified, it must be done + # by using TriggerAuthentication. + protocol: auto # Optional. Specifies protocol to use, + # either amqp or http, or auto to + # autodetect based on the `host` value. + # Default value is auto. + mode: QueueLength # QueueLength or MessageRate + value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec. + # target per instance + queueName: {{ . }} + vhostName: / # Optional. If not specified, use the vhost in the + # `host` connection string. Alternatively, you can + # use existing environment variables to read + # configuration from: See details in "Parameter + # list" section hostFromEnv: RABBITMQ_HOST% + {{- end }} diff --git a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml index 358638a..5345d7f 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml @@ -1,40 +1,41 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: loaders data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 10000 content_bytes: 104857600 directory: 10000 revision: 10000 - cls: filter - cls: retry - cls: remote url: http://{{ .Values.storage.host }}:5002/ celery: - task_broker: amqp://{{ .Values.amqp.user }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}// + task_broker: amqp://{{ .Values.amqp.username }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}// task_queues: - - swh.loader.git.tasks.UpdateGitRepository + {{- range .Values.amqp.queues }} + - {{ . }} + {{- end }} entrypoint.sh: | #!/bin/bash set -e echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ --pool=prefork \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" - diff --git a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml index 31f9e40..c713b3e 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml @@ -1,65 +1,65 @@ --- apiVersion: apps/v1 kind: Deployment metadata: name: loaders labels: app: loaders spec: - replicas: {{ .Values.swh.loader.replicas.default }} + replicas: {{ .Values.swh.loader.replicas.min }} selector: matchLabels: app: loaders strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: loaders spec: containers: - name: loaders image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} imagePullPolicy: Always command: - /entrypoint.sh resources: requests: memory: "256Mi" cpu: "200m" limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - mountPath: /tmp name: tmp-volume volumes: - name: config configMap: name: loaders defaultMode: 0777 - name: tmp-volume emptyDir: {} diff --git a/sysadmin/T3592-elastic-workers/worker/values.yaml b/sysadmin/T3592-elastic-workers/worker/values.yaml index 9f77fab..cb3b737 100644 --- a/sysadmin/T3592-elastic-workers/worker/values.yaml +++ b/sysadmin/T3592-elastic-workers/worker/values.yaml @@ -1,20 +1,20 @@ # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: user: guest password: guest host: amqp storage: host: swh-storage swh: loader: image: swh/loaders version: latest replicas: min: 1 - max: 10 - default: 5 + max: 5 +