diff --git a/sysadmin/T3592-elastic-workers/worker/README.md b/sysadmin/T3592-elastic-workers/worker/README.md index b66d765..98aed47 100644 --- a/sysadmin/T3592-elastic-workers/worker/README.md +++ b/sysadmin/T3592-elastic-workers/worker/README.md @@ -1,135 +1,138 @@ # Goal - autoscaling workers depending on repositories to load and allocated resources. # keda This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): ``` $ helm repo add kedacore https://kedacore.github.io/charts $ helm repo update swhworker@poc-rancher:~$ kubectl create namespace keda namespace/keda created swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda NAME: keda LAST DEPLOYED: Fri Oct 8 09:48:40 2021 NAMESPACE: keda STATUS: deployed REVISION: 1 TEST SUITE: None ``` source: https://keda.sh/docs/2.4/deploy/ # helm We use helm to ease the cluster application management. # Install Install the worker declaration from this directory in the cluster ``` $ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml $ TYPE=git; REL=workers-$TYPE; \ helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker $ TYPE=pypi; REL=workers-$TYPE; \ helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` Where: ``` $ cat ../loader-git.staging.values.yaml # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: host: scheduler0.internal.staging.swh.network queue_threshold: 10 # spawn worker per increment of `value` messages queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository storage: host: storage1.internal.staging.swh.network loader: name: loaders type: git ``` # List List currently deployed applications: ``` $ helm list helm list NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION workers-bzr default 1 2022-04-29 12:59:32.111950055 +0200 CEST deployed worker-0.1.0 1.16.0 workers-git default 4 2022-04-29 12:50:12.322826487 +0200 CEST deployed worker-0.1.0 1.16.0 workers-pypi default 1 2022-04-29 12:51:22.506259018 +0200 CEST deployed worker-0.1.0 1.16.0 ``` # Upgrade When adapting the worker definition, you can apply the changes by upgrading the deployed application: ``` $ TYPE=git; REL=workers-$TYPE; \ helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` # Secrets The current work requires credentials (installed as secret within the cluster): - metadata fetcher credentials `metadata-fetcher-credentials` - amqp credentials `` More details describing the secrets: ``` $ kubectl describe secret metadata-fetcher-credentials ``` Installed through: ``` -$ TYPE=git +$ TYPE=git # Replace mentions below in the yaml files $ kubectl -f $SECRET_FILE apply # for secret file in { # loader-$TYPE-metadata-fetcher-credentials.yaml # loader-$TYPE-sentry-secrets.yaml # amqp-access-credentials.yaml # } $ cat loader-$TYPE-metadata-fetcher-credentials.yaml apiVersion: v1 kind: Secret metadata: name: metadata-fetcher-credentials + namespace: ns-loaders-git type: Opaque stringData: data: | metadata_fetcher_credentials: github: github: - username: password: - ... $ cat amqp-access-credentials.yaml apiVersion: v1 kind: Secret metadata: name: amqp-access-credentials + namespace: ns-loaders-$TYPE type: Opaque stringData: host: amqp://:@scheduler0.internal.staging.swh.network:5672/%2f $ cat loaders-$TYPE-sentry-secrets.yaml apiVersion: v1 kind: Secret metadata: name: loaders-$TYPE-sentry-secrets + namespace: ns-loaders-$TYPE type: Opaque stringData: sentry-dsn: https://@sentry.softwareheritage.org/8 ``` diff --git a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml index c53530a..a2fe028 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml @@ -1,73 +1,74 @@ --- apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: name: amqp-authentication - namespace: default # must be same namespace as the ScaledObject + namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} spec: secretTargetRef: # Optional. - parameter: host name: amqp-access-credentials key: host --- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: name: loaders-{{ .Values.loader.name }}-{{ .Values.loader.type }}-operators + namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} spec: scaleTargetRef: apiVersion: apps/v1 # Optional. Default: apps/v1 kind: Deployment # Optional. Default: Deployment # Mandatory. Must be in same namespace as ScaledObject name: {{ .Values.loader.name }}-{{ .Values.loader.type }} # envSourceContainerName: {container-name} # Optional. Default: # .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds idleReplicaCount: 0 # Optional. Must be less than # minReplicaCount minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0 maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100 fallback: # Optional. Section to specify fallback # options failureThreshold: 3 # Mandatory if fallback section is # included replicas: 6 # Mandatory if fallback section is # included advanced: # Optional. Section to specify advanced # options restoreToOriginalReplicaCount: false # Optional. Default: false horizontalPodAutoscalerConfig: # Optional. Section to specify HPA # related options behavior: # Optional. Use to modify HPA's scaling # behavior scaleDown: stabilizationWindowSeconds: 60 # default 300 policies: - type: Percent value: 2 periodSeconds: 15 triggers: {{- range .Values.amqp.queues }} - type: rabbitmq authenticationRef: name: amqp-authentication metadata: host: host # Optional. If not specified, it must be done # by using TriggerAuthentication. protocol: auto # Optional. Specifies protocol to use, # either amqp or http, or auto to # autodetect based on the `host` value. # Default value is auto. mode: QueueLength # QueueLength or MessageRate value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec. # target per instance queueName: {{ . }} vhostName: / # Optional. If not specified, use the vhost in the # `host` connection string. Alternatively, you can # use existing environment variables to read # configuration from: See details in "Parameter # list" section hostFromEnv: RABBITMQ_HOST% {{- end }} diff --git a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml index 6030ca8..b0100f7 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml @@ -1,56 +1,57 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: {{ .Values.loader.name }}-{{ .Values.loader.type }} + namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 1000 content_bytes: 52428800 directory: 1000 directory_entries: 12000 revision: 1000 revision_parents: 2000 revision_bytes: 52428800 release: 1000 release_bytes: 52428800 extid: 1000 - cls: filter - cls: retry - cls: remote url: http://{{ .Values.storage.host }}:5002/ celery: task_broker: ##amqp_host## task_queues: {{- range .Values.amqp.queues }} - {{ . }} {{- end }} entrypoint.sh: | #!/bin/bash set -e # Create the full config filename cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME # contains required credentials for git loader (with metadata loader inside) # ignored by the other loaders cat /etc/credentials/metadata-fetcher/data >> $SWH_CONFIG_FILENAME # Install the rabbitmq host information sed -i 's,##amqp_host##,'$RABBITMQ_HOST',g' $SWH_CONFIG_FILENAME echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ --pool=prefork \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" diff --git a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml index 0ad690f..3bf31e3 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml @@ -1,90 +1,91 @@ --- apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Values.loader.name }}-{{ .Values.loader.type }} + namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} labels: app: {{ .Values.loader.name }}-{{ .Values.loader.type }} spec: replicas: {{ .Values.swh.loader.replicas.min }} selector: matchLabels: app: {{ .Values.loader.name }}-{{ .Values.loader.type }} strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: {{ .Values.loader.name }}-{{ .Values.loader.type }} spec: containers: - name: loaders image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} imagePullPolicy: Always command: - /entrypoint.sh resources: requests: memory: "256Mi" cpu: "200m" limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME # FIXME: built by entrypoint.sh, determine how to properly declare this value: /tmp/config.yml - name: SWH_SENTRY_ENVIRONMENT value: {{ .Values.sentry.environment }} - name: SWH_MAIN_PACKAGE value: {{ .Values.sentry.swhpackage }} - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: name: {{ .Values.loader.name }}-{{ .Values.loader.type }}-sentry-secrets key: sentry-dsn # 'name' secret must exist & include key "host" optional: false - name: RABBITMQ_HOST valueFrom: secretKeyRef: name: amqp-access-credentials key: host # 'name' secret must exist & include key "host" optional: false volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - name: metadata-fetcher-credentials mountPath: /etc/credentials/metadata-fetcher readOnly: true - mountPath: /tmp name: tmp-volume volumes: - name: config configMap: name: {{ .Values.loader.name }}-{{ .Values.loader.type }} defaultMode: 0777 - name: tmp-volume emptyDir: {} - name: metadata-fetcher-credentials secret: secretName: metadata-fetcher-credentials diff --git a/sysadmin/T3592-elastic-workers/worker/templates/services.yaml b/sysadmin/T3592-elastic-workers/worker/templates/services.yaml index 569e18a..ecf4d19 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/services.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/services.yaml @@ -1,18 +1,20 @@ --- apiVersion: v1 kind: Service metadata: name: storage-{{ .Values.loader.name }}-{{ .Values.loader.type }} + namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} spec: type: ExternalName externalName: {{ .Values.storage.host }} --- apiVersion: v1 kind: Service metadata: name: amqp-{{ .Values.loader.name }}-{{ .Values.loader.type }} + namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} spec: type: ExternalName externalName: {{ .Values.amqp.host }}