diff --git a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml index 48d6e5e..9dc2b52 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml @@ -1,47 +1,53 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: loaders data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 1000 content_bytes: 52428800 # 50 MB directory: 1000 directory_entries: 12000 revision: 1000 revision_parents: 2000 revision_bytes: 52428800 release: 1000 release_bytes: 52428800 extid: 1000 - cls: filter - cls: retry - cls: remote url: http://{{ .Values.storage.host }}:5002/ celery: task_broker: amqp://{{ .Values.amqp.username }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}// task_queues: {{- range .Values.amqp.queues }} - {{ . }} {{- end }} entrypoint.sh: | #!/bin/bash set -e + # Create the full config filename + cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME + # contains required credentials for git loader (with metadata loader inside) + # ignored by the other loaders + cat /tmp/secret-data/data >> $SWH_CONFIG_FILENAME + echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ --pool=prefork \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" diff --git a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml index c713b3e..ab19917 100644 --- a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml +++ b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml @@ -1,65 +1,72 @@ --- apiVersion: apps/v1 kind: Deployment metadata: name: loaders labels: app: loaders spec: replicas: {{ .Values.swh.loader.replicas.min }} selector: matchLabels: app: loaders strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: loaders spec: containers: - name: loaders image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} imagePullPolicy: Always command: - /entrypoint.sh resources: requests: memory: "256Mi" cpu: "200m" limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME - value: /etc/softwareheritage/config.yml + # FIXME: built by entrypoint.sh, determine how to properly declare this + value: /tmp/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true + - name: metadata-fetcher-credentials + mountPath: /tmp/secret-data + readOnly: true - mountPath: /tmp name: tmp-volume volumes: - name: config configMap: name: loaders defaultMode: 0777 - name: tmp-volume emptyDir: {} + - name: metadata-fetcher-credentials + secret: + secretName: metadata-fetcher-credentials