diff --git a/sysadmin/T3592-elastic-workers/worker/README.md b/sysadmin/T3592-elastic-workers/worker/README.md index 8eb2141..3e3514f 100644 --- a/sysadmin/T3592-elastic-workers/worker/README.md +++ b/sysadmin/T3592-elastic-workers/worker/README.md @@ -1,81 +1,96 @@ # Goal - autoscaling workers depending on repositories to load and allocated resources. # keda This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): ``` $ helm repo add kedacore https://kedacore.github.io/charts $ helm repo update swhworker@poc-rancher:~$ kubectl create namespace keda namespace/keda created swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda NAME: keda LAST DEPLOYED: Fri Oct 8 09:48:40 2021 NAMESPACE: keda STATUS: deployed REVISION: 1 TEST SUITE: None ``` source: https://keda.sh/docs/2.4/deploy/ # helm Install the worker declaration from this directory in the cluster ``` $ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml -$ helm install -f ../loader-git.staging.values.yaml workers-git ../worker +$ TYPE=git; REL=workers-$TYPE; \ + helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker +$ TYPE=pypi; REL=workers-$TYPE; \ + helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` Where: ``` $ cat ../loader-git.staging.values.yaml # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: username: password: host: scheduler0.internal.staging.swh.network queue_threshold: 10 # spawn worker per increment of `value` messages queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository storage: host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: git +``` + +# Upgrade + +When changing something, apply the upgrade to the deployed chart: +``` +$ TYPE=git; REL=workers-$TYPE; \ + helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` # secrets This now uses metadata fetcher credentials `metadata-fetcher-credentials` installed as secret within the cluster. More details: ``` $ kubectl describe secrets/metadata-fetcher-credentials ``` Installed through: ``` -$ kubectl -f loader-git-metadata-fetcher-credentials.yaml apply +$ kubectl -f ./loader-git-metadata-fetcher-credentials.yaml apply # secret file $ cat loader-git-metadata-fetcher-credentials.yaml apiVersion: v1 kind: Secret metadata: name: metadata-fetcher-credentials type: Opaque stringData: data: | metadata_fetcher_credentials: github: github: - username: password: - ... ``` diff --git a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml index 99931b4..6aba442 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml @@ -1,58 +1,59 @@ --- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: - name: loaders-operators + name: loaders-{{ .Values.loader.name }}-{{ .Values.loader.type }}-operators spec: scaleTargetRef: apiVersion: apps/v1 # Optional. Default: apps/v1 kind: Deployment # Optional. Default: Deployment - name: loaders # Mandatory. Must be in same namespace as ScaledObject + # Mandatory. Must be in same namespace as ScaledObject + name: {{ .Values.loader.name }}-{{ .Values.loader.type }} # envSourceContainerName: {container-name} # Optional. Default: # .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds idleReplicaCount: 0 # Optional. Must be less than # minReplicaCount minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0 maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100 fallback: # Optional. Section to specify fallback # options failureThreshold: 3 # Mandatory if fallback section is # included replicas: 6 # Mandatory if fallback section is # included advanced: # Optional. Section to specify advanced # options restoreToOriginalReplicaCount: false # Optional. Default: false horizontalPodAutoscalerConfig: # Optional. Section to specify HPA # related options behavior: # Optional. Use to modify HPA's scaling # behavior scaleDown: stabilizationWindowSeconds: 60 # default 300 policies: - type: Percent value: 2 periodSeconds: 15 triggers: {{- range .Values.amqp.queues }} - type: rabbitmq metadata: host: amqp://{{ $.Values.amqp.username }}:{{ $.Values.amqp.password }}@{{ $.Values.amqp.host }}// # Optional. If not specified, it must be done # by using TriggerAuthentication. protocol: auto # Optional. Specifies protocol to use, # either amqp or http, or auto to # autodetect based on the `host` value. # Default value is auto. mode: QueueLength # QueueLength or MessageRate value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec. # target per instance queueName: {{ . }} vhostName: / # Optional. If not specified, use the vhost in the # `host` connection string. Alternatively, you can # use existing environment variables to read # configuration from: See details in "Parameter # list" section hostFromEnv: RABBITMQ_HOST% {{- end }} diff --git a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml index 9dc2b52..27fda2d 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml @@ -1,53 +1,53 @@ --- apiVersion: v1 kind: ConfigMap metadata: - name: loaders + name: {{ .Values.loader.name }}-{{ .Values.loader.type }} data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 1000 content_bytes: 52428800 # 50 MB directory: 1000 directory_entries: 12000 revision: 1000 revision_parents: 2000 revision_bytes: 52428800 release: 1000 release_bytes: 52428800 extid: 1000 - cls: filter - cls: retry - cls: remote url: http://{{ .Values.storage.host }}:5002/ celery: task_broker: amqp://{{ .Values.amqp.username }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}// task_queues: {{- range .Values.amqp.queues }} - {{ . }} {{- end }} entrypoint.sh: | #!/bin/bash set -e # Create the full config filename cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME # contains required credentials for git loader (with metadata loader inside) # ignored by the other loaders cat /tmp/secret-data/data >> $SWH_CONFIG_FILENAME echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ --pool=prefork \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" diff --git a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml index ab19917..8931a9c 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml @@ -1,72 +1,72 @@ --- apiVersion: apps/v1 kind: Deployment metadata: - name: loaders + name: {{ .Values.loader.name }}-{{ .Values.loader.type }} labels: - app: loaders + app: {{ .Values.loader.name }}-{{ .Values.loader.type }} spec: replicas: {{ .Values.swh.loader.replicas.min }} selector: matchLabels: - app: loaders + app: {{ .Values.loader.name }}-{{ .Values.loader.type }} strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: - app: loaders + app: {{ .Values.loader.name }}-{{ .Values.loader.type }} spec: containers: - name: loaders image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} imagePullPolicy: Always command: - /entrypoint.sh resources: requests: memory: "256Mi" cpu: "200m" limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME # FIXME: built by entrypoint.sh, determine how to properly declare this value: /tmp/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - name: metadata-fetcher-credentials mountPath: /tmp/secret-data readOnly: true - mountPath: /tmp name: tmp-volume volumes: - name: config configMap: - name: loaders + name: {{ .Values.loader.name }}-{{ .Values.loader.type }} defaultMode: 0777 - name: tmp-volume emptyDir: {} - name: metadata-fetcher-credentials secret: secretName: metadata-fetcher-credentials diff --git a/sysadmin/T3592-elastic-workers/worker/templates/services.yaml b/sysadmin/T3592-elastic-workers/worker/templates/services.yaml index 36e4bcd..569e18a 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/services.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/services.yaml @@ -1,18 +1,18 @@ --- apiVersion: v1 kind: Service metadata: - name: storage + name: storage-{{ .Values.loader.name }}-{{ .Values.loader.type }} spec: type: ExternalName externalName: {{ .Values.storage.host }} --- apiVersion: v1 kind: Service metadata: - name: amqp + name: amqp-{{ .Values.loader.name }}-{{ .Values.loader.type }} spec: type: ExternalName externalName: {{ .Values.amqp.host }} diff --git a/sysadmin/T3592-elastic-workers/worker/values.yaml b/sysadmin/T3592-elastic-workers/worker/values.yaml index 5034b42..36a918e 100644 --- a/sysadmin/T3592-elastic-workers/worker/values.yaml +++ b/sysadmin/T3592-elastic-workers/worker/values.yaml @@ -1,20 +1,24 @@ # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: user: guest password: guest host: amqp storage: host: swh-storage +loader: + name: loaders + type: + swh: loader: image: softwareheritage/loaders version: 2022-04-28 replicas: min: 1 max: 10