diff --git a/sysadmin/T3592-elastic-workers/worker/.gitignore b/sysadmin/T3592-elastic-workers/worker/.gitignore index a20ef5b..a8cc11c 100644 --- a/sysadmin/T3592-elastic-workers/worker/.gitignore +++ b/sysadmin/T3592-elastic-workers/worker/.gitignore @@ -1,8 +1,10 @@ loader-bzr.staging.values.yaml loader-cvs.staging.values.yaml loader-git.staging.values.yaml loader-maven.staging.values.yaml loader-pypi.staging.values.yaml loader-svn.staging.values.yaml -loader-git-metadata-fetcher-credentials.yaml loader-npm.staging.values.yaml + +loader-git-metadata-fetcher-credentials.yaml +amqp-access-credentials.yaml diff --git a/sysadmin/T3592-elastic-workers/worker/README.md b/sysadmin/T3592-elastic-workers/worker/README.md index 57a692b..c573cff 100644 --- a/sysadmin/T3592-elastic-workers/worker/README.md +++ b/sysadmin/T3592-elastic-workers/worker/README.md @@ -1,114 +1,123 @@ # Goal - autoscaling workers depending on repositories to load and allocated resources. # keda This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): ``` $ helm repo add kedacore https://kedacore.github.io/charts $ helm repo update swhworker@poc-rancher:~$ kubectl create namespace keda namespace/keda created swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda NAME: keda LAST DEPLOYED: Fri Oct 8 09:48:40 2021 NAMESPACE: keda STATUS: deployed REVISION: 1 TEST SUITE: None ``` source: https://keda.sh/docs/2.4/deploy/ # helm We use helm to ease the cluster application management. # Install Install the worker declaration from this directory in the cluster ``` $ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml $ TYPE=git; REL=workers-$TYPE; \ helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker $ TYPE=pypi; REL=workers-$TYPE; \ helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` Where: ``` $ cat ../loader-git.staging.values.yaml # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: - username: - password: host: scheduler0.internal.staging.swh.network queue_threshold: 10 # spawn worker per increment of `value` messages queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository storage: host: storage1.internal.staging.swh.network loader: name: loaders type: git ``` # List List currently deployed applications: ``` $ helm list helm list NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION workers-bzr default 1 2022-04-29 12:59:32.111950055 +0200 CEST deployed worker-0.1.0 1.16.0 workers-git default 4 2022-04-29 12:50:12.322826487 +0200 CEST deployed worker-0.1.0 1.16.0 workers-pypi default 1 2022-04-29 12:51:22.506259018 +0200 CEST deployed worker-0.1.0 1.16.0 ``` # Upgrade When adapting the worker definition, you can apply the changes by upgrading the deployed application: ``` $ TYPE=git; REL=workers-$TYPE; \ helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` # Secrets -The current work requires metadata fetcher credentials `metadata-fetcher-credentials` -installed as secret within the cluster. +The current work requires credentials (installed as secret within the cluster): +- metadata fetcher credentials `metadata-fetcher-credentials` +- amqp credentials `` -More details: +More details describing the secrets: ``` -$ kubectl describe secrets/metadata-fetcher-credentials +$ kubectl describe secret metadata-fetcher-credentials ``` Installed through: ``` -$ kubectl -f ./loader-git-metadata-fetcher-credentials.yaml apply -# secret file +$ kubectl -f $SECRET_FILE apply +# for secret file in {loader-git-metadata-fetcher-credentials,amqp-access-credentials}.yaml $ cat loader-git-metadata-fetcher-credentials.yaml apiVersion: v1 kind: Secret metadata: name: metadata-fetcher-credentials type: Opaque stringData: data: | metadata_fetcher_credentials: github: github: - username: password: - ... +$ cat amqp-access-credentials.yaml +apiVersion: v1 +kind: Secret +metadata: + name: amqp-access-credentials +type: Opaque +data: + username: # output of: echo -n 'redacted-pass' | base64 + password: + ``` diff --git a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml index 27fda2d..bb18004 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml @@ -1,53 +1,60 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: {{ .Values.loader.name }}-{{ .Values.loader.type }} data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 1000 content_bytes: 52428800 # 50 MB directory: 1000 directory_entries: 12000 revision: 1000 revision_parents: 2000 revision_bytes: 52428800 release: 1000 release_bytes: 52428800 extid: 1000 - cls: filter - cls: retry - cls: remote url: http://{{ .Values.storage.host }}:5002/ celery: - task_broker: amqp://{{ .Values.amqp.username }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}// + task_broker: amqp://##username##:##password##@{{ .Values.amqp.host }}// task_queues: {{- range .Values.amqp.queues }} - {{ . }} {{- end }} entrypoint.sh: | #!/bin/bash set -e # Create the full config filename cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME # contains required credentials for git loader (with metadata loader inside) # ignored by the other loaders - cat /tmp/secret-data/data >> $SWH_CONFIG_FILENAME + cat /etc/credentials/metadata-fetcher/data >> $SWH_CONFIG_FILENAME + + # Work around configuration setup + amqp_username=$(cat /etc/credentials/amqp/username) + amqp_password=$(cat /etc/credentials/amqp/password) + + sed -i 's/##username##/'$amqp_username'/g' $SWH_CONFIG_FILENAME + sed -i 's/##password##/'$amqp_password'/g' $SWH_CONFIG_FILENAME echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ --pool=prefork \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" diff --git a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml index 8931a9c..6924f45 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml @@ -1,72 +1,78 @@ --- apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Values.loader.name }}-{{ .Values.loader.type }} labels: app: {{ .Values.loader.name }}-{{ .Values.loader.type }} spec: replicas: {{ .Values.swh.loader.replicas.min }} selector: matchLabels: app: {{ .Values.loader.name }}-{{ .Values.loader.type }} strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: {{ .Values.loader.name }}-{{ .Values.loader.type }} spec: containers: - name: loaders image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} imagePullPolicy: Always command: - /entrypoint.sh resources: requests: memory: "256Mi" cpu: "200m" limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME # FIXME: built by entrypoint.sh, determine how to properly declare this value: /tmp/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - name: metadata-fetcher-credentials - mountPath: /tmp/secret-data + mountPath: /etc/credentials/metadata-fetcher + readOnly: true + - name: amqp-access-credentials + mountPath: /etc/credentials/amqp readOnly: true - mountPath: /tmp name: tmp-volume volumes: - name: config configMap: name: {{ .Values.loader.name }}-{{ .Values.loader.type }} defaultMode: 0777 - name: tmp-volume emptyDir: {} - name: metadata-fetcher-credentials secret: secretName: metadata-fetcher-credentials + - name: amqp-access-credentials + secret: + secretName: amqp-access-credentials diff --git a/sysadmin/T3592-elastic-workers/worker/values.yaml b/sysadmin/T3592-elastic-workers/worker/values.yaml index 9afe622..b42aa0e 100644 --- a/sysadmin/T3592-elastic-workers/worker/values.yaml +++ b/sysadmin/T3592-elastic-workers/worker/values.yaml @@ -1,23 +1,21 @@ # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: - user: guest - password: guest host: amqp storage: host: swh-storage loader: name: loaders type: swh: loader: image: softwareheritage/loaders version: 2022-05-11 replicas: min: 1 max: 5