diff --git a/swh/templates/listers/configmap.yaml b/swh/templates/listers/configmap.yaml new file mode 100644 --- /dev/null +++ b/swh/templates/listers/configmap.yaml @@ -0,0 +1,41 @@ +{{ if .Values.listers.enabled -}} +{{- range $lister_type, $deployment_config := .Values.listers.deployments -}} +{{- $lister_name := ( print "lister-" $lister_type ) -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $lister_name }}-template + namespace: {{ $.Values.namespace }} +data: + config.yml.template: | + storage: + cls: remote + url: http://{{ $.Values.listers.storage.host }}:{{ $.Values.listers.storage.port }}/ + scheduler: + cls: remote + url: http://{{ $.Values.listers.scheduler.host }}:{{ $.Values.listers.scheduler.port }}/ + celery: + task_broker: ${host} + task_queues: + {{- range $queue := get $deployment_config "queues" }} + - {{ $queue }} + {{- end }} + credentials: + init-container-entrypoint.sh: | + #!/bin/bash + + set -e + + CONFIG_FILE=/etc/swh/config.yml + + # substitute environment variables when creating the default config.yml + eval echo \""$( $CONFIG_FILE + + CREDS_LISTER_PATH=/etc/credentials/listers/credentials + [ -f $CREDS_LISTER_PATH ] && \ + sed 's/^/ /g' $CREDS_LISTER_PATH >> $CONFIG_FILE + +{{ end }} +{{- end -}} diff --git a/swh/templates/listers/deployment.yaml b/swh/templates/listers/deployment.yaml new file mode 100644 --- /dev/null +++ b/swh/templates/listers/deployment.yaml @@ -0,0 +1,119 @@ +{{ if .Values.listers.enabled -}} +{{- $configurationChecksum := include (print $.Template.BasePath "/listers/configmap.yaml") . -}} +{{- range $lister_type, $deployment_config := .Values.listers.deployments -}} +{{- $lister_name := ( print "lister-" $lister_type ) -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $lister_name }} + namespace: {{ $.Values.namespace }} + labels: + app: {{ $lister_name }} +spec: + selector: + matchLabels: + app: {{ $lister_name }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + template: + metadata: + labels: + app: {{ $lister_name }} + annotations: + # Force a rollout upgrade if the configuration changes + checksum/config: {{ $configurationChecksum | sha256sum }} + spec: + {{- if $.Values.listers.affinity }} + affinity: + {{ toYaml $.Values.listers.affinity | nindent 8 }} + {{- end }} + initContainers: + - name: prepare-configuration + image: debian:bullseye + imagePullPolicy: Always + envFrom: + - secretRef: + # define the ${host) values for configuration "celery > task_broker: " key + name: amqp-access-credentials + command: + - /entrypoint.sh + volumeMounts: + - name: configuration-template + mountPath: /entrypoint.sh + subPath: "init-container-entrypoint.sh" + readOnly: true + - name: configuration + mountPath: /etc/swh + - name: configuration-template + mountPath: /etc/swh/configuration-template + - name: lister-credentials-secrets + mountPath: /etc/credentials/listers + readOnly: true + containers: + - name: listers + resources: + requests: + memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }} + cpu: {{ get $deployment_config "requestedCpu" | default "500m" }} + limits: + memory: "4000Mi" + cpu: "1200m" + image: {{ $.Values.swh_listers_image }}:{{ $.Values.swh_listers_image_version }} + imagePullPolicy: Always + command: + - /bin/bash + args: + - -c + - /opt/swh/entrypoint.sh + lifecycle: + preStop: + exec: + command: ["kill", "1"] + env: + - name: STATSD_HOST + value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} + - name: STATSD_PORT + value: {{ $.Values.statsdPort | default "9125" | quote }} + - name: CONCURRENCY + value: {{ get $deployment_config "concurrency" | default 1 | quote }} + - name: MAX_TASKS_PER_CHILD + value: {{ get $deployment_config "maxTasksPerChild" | default 1 | quote }} + - name: LOGLEVEL + value: {{ get $deployment_config "logLevel" | default "INFO" | quote }} + - name: SWH_CONFIG_FILENAME + value: /etc/swh/config.yml + - name: SWH_SENTRY_ENVIRONMENT + value: {{ $.Values.sentry.environment }} + - name: SWH_MAIN_PACKAGE + value: {{ $.Values.listers.sentrySwhPackage }} + - name: SWH_SENTRY_DSN + valueFrom: + secretKeyRef: + name: lister-sentry-secrets + key: sentry-dsn + # 'name' secret must exist & include key "host" + optional: false + volumeMounts: + - name: configuration + mountPath: /etc/swh + volumes: + - name: configuration + emptyDir: {} + - name: configuration-template + configMap: + name: {{ $lister_name }}-template + defaultMode: 0777 + items: + - key: "config.yml.template" + path: "config.yml.template" + - key: "init-container-entrypoint.sh" + path: "init-container-entrypoint.sh" + - name: lister-credentials-secrets + secret: + secretName: lister-credentials-secrets + optional: true +{{ end }} +{{- end -}} diff --git a/swh/templates/listers/keda-autoscaling.yaml b/swh/templates/listers/keda-autoscaling.yaml new file mode 100644 --- /dev/null +++ b/swh/templates/listers/keda-autoscaling.yaml @@ -0,0 +1,83 @@ +{{ if .Values.listers.enabled -}} +{{- range $lister_type, $deployment_config := .Values.listers.deployments -}} +{{ if get $deployment_config "autoScaling" }} +{{- $autoscalingConfig := get $deployment_config "autoScaling" -}} +{{- $lister_name := ( print "lister-" $lister_type ) -}} +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: amqp-authentication + namespace: {{ $.Values.namespace }} +spec: + secretTargetRef: # Optional. + - parameter: host + name: amqp-access-credentials + key: host + +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ $lister_name }}-operators + namespace: {{ $.Values.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 # Optional. Default: apps/v1 + kind: Deployment # Optional. Default: Deployment + # Mandatory. Must be in same namespace as ScaledObject + name: {{ $lister_name }} + # envSourceContainerName: {container-name} # Optional. Default: + # .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Must be less than + # minReplicaCount + minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }} + maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }} + fallback: # Optional. Section to specify fallback + # options + failureThreshold: 3 # Mandatory if fallback section is + # included + replicas: 6 # Mandatory if fallback section is + # included + advanced: # Optional. Section to specify advanced + # options + restoreToOriginalReplicaCount: false # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA + # related options + behavior: # Optional. Use to modify HPA's scaling + # behavior + scaleDown: + stabilizationWindowSeconds: 60 # default 300 + policies: + - type: Percent + value: 2 + periodSeconds: 15 + triggers: + {{- range $queue := get $deployment_config "queues" }} + - type: rabbitmq + authenticationRef: + name: amqp-authentication + metadata: + host: host + # Optional. If not specified, it must be done + # by using TriggerAuthentication. + protocol: auto # Optional. Specifies protocol to use, + # either amqp or http, or auto to + # autodetect based on the `host` value. + # Default value is auto. + mode: QueueLength # QueueLength or MessageRate + # message backlog or publish/sec. + # target per instance + value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }} + queueName: {{ $queue }} + vhostName: / # Optional. If not specified, use the vhost in the + # `host` connection string. Alternatively, you can + # use existing environment variables to read + # configuration from: See details in "Parameter + # list" section hostFromEnv: RABBITMQ_HOST% + {{- end }} +{{ end }} +{{ end }} +{{- end -}} diff --git a/swh/values/default.yaml b/swh/values/default.yaml --- a/swh/values/default.yaml +++ b/swh/values/default.yaml @@ -30,6 +30,17 @@ values: - "true" +listers: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "swh/lister" + operator: In + values: + - "true" + graphql: affinity: nodeAffinity: diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -16,7 +16,6 @@ # - swh.loader.bzr.tasks.LoadBazaar # autoScaling: # queueThreshold: 10 - # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.bzr # cvs: @@ -26,7 +25,6 @@ # - swh.loader.cvs.tasks.LoadCvsRepository # autoScaling: # queueThreshold: 10 - # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.cvs # hg: @@ -37,7 +35,6 @@ # - swh.loader.mercurial.tasks.LoadMercurial # autoScaling: # queueThreshold: 10 - # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.mercurial highpriority: @@ -56,7 +53,6 @@ - save_code_now:swh.loader.package.archive.tasks.LoadArchive autoScaling: queueThreshold: 10 - minReplicacount: 0 maxReplicaCount: 3 sentrySwhPackage: swh.loader.highpriority git: @@ -68,8 +64,7 @@ - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository autoScaling: queueThreshold: 5 # spawn worker per increment of `value` messages - minReplicacount: 1 - maxReplicaCount: 5 + maxReplicaCount: 3 sentrySwhPackage: swh.loader.git # maven: # requestedMemory: 256Mi @@ -78,7 +73,6 @@ # - swh.loader.package.maven.tasks.LoadMaven # autoScaling: # queueThreshold: 10 - # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.core # npm: @@ -88,7 +82,6 @@ # - swh.loader.package.npm.tasks.LoadNpm # autoScaling: # queueThreshold: 10 - # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.core pypi: @@ -98,8 +91,7 @@ - swh.loader.package.pypi.tasks.LoadPyPI autoScaling: queueThreshold: 10 - minReplicacount: 1 - maxReplicaCount: 3 + maxReplicaCount: 1 sentrySwhPackage: swh.loader.core # svn: # requestedMemory: 256Mi @@ -110,10 +102,29 @@ # - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # autoScaling: # queueThreshold: 10 - # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.svn +listers: + enabled: true + sentrySwhPackage: swh.lister + storage: + host: storage1.internal.staging.swh.network + port: 5002 + scheduler: + host: scheduler0.internal.staging.swh.network + port: 5008 + amqp: + host: scheduler0.internal.staging.swh.network + deployments: + gnu-full: + queues: + - swh.lister.gnu.tasks.GNUListerTask + autoScaling: + queueThreshold: 10 + minReplicaCount: 0 + maxReplicaCount: 1 + graphql: enabled: true sentry_enabled: true diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml --- a/values-swh-application-versions.yaml +++ b/values-swh-application-versions.yaml @@ -9,4 +9,5 @@ swh_loader_git_image_version: "20220906.1" swh_graphql_image: softwareheritage/graphql swh_graphql_image_version: "20220826.1" - +swh_listers_image: softwareheritage/lister +swh_listers_image_version: "20220907.4"