diff --git a/swh/templates/loaders/configmap.yaml b/swh/templates/loaders/configmap.yaml index 687c2ac..8160d14 100644 --- a/swh/templates/loaders/configmap.yaml +++ b/swh/templates/loaders/configmap.yaml @@ -1,64 +1,54 @@ {{ if .Values.loaders.enabled -}} {{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} {{- $loader_name := ( print "loader-" $loader_type ) -}} --- apiVersion: v1 kind: ConfigMap metadata: - name: {{ $loader_name }} + name: {{ $loader_name }}-template namespace: {{ $.Values.namespace }} data: - config.yml: | + config.yml.template: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 1000 content_bytes: 52428800 directory: 1000 directory_entries: 12000 revision: 1000 revision_parents: 2000 revision_bytes: 52428800 release: 1000 release_bytes: 52428800 extid: 1000 - cls: filter - cls: retry - cls: remote url: http://{{ $.Values.loaders.storage.host }}:{{ $.Values.loaders.storage.port }}/ celery: - task_broker: ##amqp_host## + task_broker: ${host} task_queues: {{- range $queue := get $deployment_config "queues" }} - {{ $queue }} {{- end }} - entrypoint.sh: | + metadata_fetcher_credentials: + init-container-entrypoint.sh: | #!/bin/bash set -e - # Create the full config filename - cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME - # contains required credentials for git loader (with metadata loader inside) - # ignored by the other loaders - cat /etc/credentials/metadata-fetcher/data >> $SWH_CONFIG_FILENAME + CONFIG_FILE=/etc/swh/config.yml - # Install the rabbitmq host information - sed -i 's,##amqp_host##,'$RABBITMQ_HOST',g' $SWH_CONFIG_FILENAME + # substitute environment variables when creating the default config.yml + eval echo \""$( $CONFIG_FILE - echo Starting the swh Celery worker - exec python -m celery \ - --app=swh.scheduler.celery_backend.config.app \ - worker \ - --pool=prefork \ - --concurrency=${CONCURRENCY} \ - --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ - -Ofair --loglevel=${LOGLEVEL} \ - --without-gossip \ - --without-mingle \ - --hostname "${HOSTNAME}" + CREDS_LISTER_PATH=/etc/credentials/metadata-fetcher/credentials + [ -f $CREDS_LISTER_PATH ] && \ + sed 's/^/ /g' $CREDS_LISTER_PATH >> $CONFIG_FILE {{ end }} {{- end -}} diff --git a/swh/templates/loaders/deployment.yaml b/swh/templates/loaders/deployment.yaml index 82200c2..ceea568 100644 --- a/swh/templates/loaders/deployment.yaml +++ b/swh/templates/loaders/deployment.yaml @@ -1,116 +1,125 @@ {{ if .Values.loaders.enabled -}} {{- $configurationChecksum := include (print $.Template.BasePath "/loaders/configmap.yaml") . -}} {{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} {{- $loader_name := ( print "loader-" $loader_type ) -}} {{- $image_name_key := ( print "swh_loader_" $loader_type "_image" ) -}} {{- $image_version_key := ( print $image_name_key "_version" ) -}} {{- $container_image := get $.Values $image_name_key }} {{- $container_image_version := get $.Values $image_version_key }} --- apiVersion: apps/v1 kind: Deployment metadata: name: {{ $loader_name }} namespace: {{ $.Values.namespace }} labels: app: {{ $loader_name }} spec: revisionHistoryLimit: 3 selector: matchLabels: app: {{ $loader_name }} strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: {{ $loader_name }} annotations: # Force a rollout upgrade if the configuration changes checksum/config: {{ $configurationChecksum | sha256sum }} spec: {{- if $.Values.loaders.affinity }} affinity: {{ toYaml $.Values.loaders.affinity | nindent 8 }} {{- end }} + initContainers: + - name: prepare-configuration + image: debian:bullseye + imagePullPolicy: Always + envFrom: + - secretRef: + # define the ${host) values for configuration "celery > task_broker: " key + name: amqp-access-credentials + command: + - /entrypoint.sh + volumeMounts: + - name: configuration-template + mountPath: /entrypoint.sh + subPath: "init-container-entrypoint.sh" + readOnly: true + - name: configuration + mountPath: /etc/swh + - name: configuration-template + mountPath: /etc/swh/configuration-template + - name: metadata-fetcher-credentials + mountPath: /etc/credentials/metadata-fetcher + readOnly: true containers: - name: loaders {{ if $container_image -}} image: {{ $container_image }}:{{ $container_image_version }} {{ else -}} image: {{ $.Values.swh_loaders_image }}:{{ $.Values.swh_loaders_image_version }} {{ end }} imagePullPolicy: Always command: - - /entrypoint.sh + - /opt/swh/entrypoint.sh resources: requests: memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }} cpu: {{ get $deployment_config "requestedCpu" | default "500m" }} limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: STATSD_HOST value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} - name: STATSD_PORT value: {{ $.Values.statsdPort | default "9125" | quote }} - name: CONCURRENCY - value: "1" + value: {{ get $deployment_config "concurrency" | default 1 | quote }} - name: MAX_TASKS_PER_CHILD - value: "5" + value: {{ get $deployment_config "maxTasksPerChild" | default 5 | quote }} - name: LOGLEVEL - value: "INFO" + value: {{ get $deployment_config "logLevel" | default "INFO" | quote }} - name: SWH_CONFIG_FILENAME - # FIXME: built by entrypoint.sh, determine how to properly declare this - value: /tmp/config.yml + value: /etc/swh/config.yml - name: SWH_SENTRY_ENVIRONMENT value: {{ $.Values.sentry.environment }} - name: SWH_MAIN_PACKAGE value: {{ get $deployment_config "sentrySwhPackage" }} - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: name: {{ $loader_name }}-sentry-secrets key: sentry-dsn # 'name' secret must exist & include key "host" optional: false - - name: RABBITMQ_HOST - valueFrom: - secretKeyRef: - name: amqp-access-credentials - key: host - # 'name' secret must exist & include key "host" - optional: false volumeMounts: - - name: config - mountPath: /etc/softwareheritage/config.yml - subPath: config.yml - readOnly: true - - name: config - mountPath: /entrypoint.sh - subPath: entrypoint.sh - readOnly: true - - name: metadata-fetcher-credentials - mountPath: /etc/credentials/metadata-fetcher - readOnly: true - - mountPath: /tmp - name: tmp-volume + - name: configuration + mountPath: /etc/swh volumes: - - name: config - configMap: - name: {{ $loader_name }} - defaultMode: 0777 - - name: tmp-volume - emptyDir: {} - - name: metadata-fetcher-credentials - secret: - secretName: metadata-fetcher-credentials + - name: configuration + emptyDir: {} + - name: configuration-template + configMap: + name: {{ $loader_name }}-template + defaultMode: 0777 + items: + - key: "config.yml.template" + path: "config.yml.template" + - key: "init-container-entrypoint.sh" + path: "init-container-entrypoint.sh" + - name: metadata-fetcher-credentials + secret: + secretName: metadata-fetcher-credentials + optional: true {{ end }} {{- end -}} diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml index 622a8b3..93ce8a1 100644 --- a/values-swh-application-versions.yaml +++ b/values-swh-application-versions.yaml @@ -1,13 +1,13 @@ # This file references the last version of all the softwareheritage images # It's used to manage the automatic update of the environments swh_storage_replayer_image: softwareheritage/storage-replayer swh_storage_replayer_image_version: "20220819.1" swh_loaders_image: softwareheritage/loaders swh_loaders_image_version: "20220826.1" swh_loader_git_image: softwareheritage/loader_git -swh_loader_git_image_version: "20220906.1" +swh_loader_git_image_version: "20220908.1" swh_graphql_image: softwareheritage/graphql swh_graphql_image_version: "20220826.1" swh_listers_image: softwareheritage/lister swh_listers_image_version: "20220907.4"