diff --git a/swh/templates/listers/configmap.yaml b/swh/templates/listers/configmap.yaml new file mode 100644 index 0000000..d90e47d --- /dev/null +++ b/swh/templates/listers/configmap.yaml @@ -0,0 +1,51 @@ +{{ if .Values.listers.enabled -}} +{{- range $lister_type, $deployment_config := .Values.listers.deployments -}} +{{- $lister_name := ( print "lister-" $lister_type ) -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $lister_name }} + namespace: {{ $.Values.namespace }} +data: + config.yml: | + storage: + cls: remote + url: http://{{ $.Values.listers.storage.host }}:{{ $.Values.listers.storage.port }}/ + scheduler: + cls: remote + url: http://{{ $.Values.listers.scheduler.host }}:{{ $.Values.listers.scheduler.port }}/ + + celery: + task_broker: ##amqp_host## + task_queues: + {{- range $queue := get $deployment_config "queues" }} + - {{ $queue }} + {{- end }} + entrypoint.sh: | + #!/bin/bash + + set -e + + # Create the full config filename + cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME + # contains required credentials for lister + cat /etc/credentials/listers/data >> $SWH_CONFIG_FILENAME + + # Install the rabbitmq host information + sed -i 's,##amqp_host##,'$RABBITMQ_HOST',g' $SWH_CONFIG_FILENAME + + echo Starting the swh Celery worker + exec python -m celery \ + --app=swh.scheduler.celery_backend.config.app \ + worker \ + --pool=prefork \ + --concurrency=${CONCURRENCY} \ + --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ + -Ofair --loglevel=${LOGLEVEL} \ + --without-gossip \ + --without-mingle \ + --without-heartbeat \ + --hostname "${HOSTNAME}" +{{ end }} +{{- end -}} diff --git a/swh/templates/listers/deployment.yaml b/swh/templates/listers/deployment.yaml new file mode 100644 index 0000000..943803b --- /dev/null +++ b/swh/templates/listers/deployment.yaml @@ -0,0 +1,107 @@ +{{ if .Values.listers.enabled -}} +{{- $configurationChecksum := include (print $.Template.BasePath "/listers/configmap.yaml") . -}} +{{- range $lister_type, $deployment_config := .Values.listers.deployments -}} +{{- $lister_name := ( print "lister-" $lister_type ) -}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $lister_name }} + namespace: {{ $.Values.namespace }} + labels: + app: {{ $lister_name }} +spec: + selector: + matchLabels: + app: {{ $lister_name }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + template: + metadata: + labels: + app: {{ $lister_name }} + annotations: + # Force a rollout upgrade if the configuration changes + checksum/config: {{ $configurationChecksum | sha256sum }} + spec: + {{- if $.Values.listers.affinity }} + affinity: + {{ toYaml $.Values.listers.affinity | nindent 8 }} + {{- end }} + containers: + - name: listers + image: {{ $.Values.swh_listers_image }}:{{ $.Values.swh_listers_image_version }} + imagePullPolicy: Always + command: + - /entrypoint.sh + resources: + requests: + memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }} + cpu: {{ get $deployment_config "requestedCpu" | default "500m" }} + limits: + memory: "4000Mi" + cpu: "1200m" + lifecycle: + preStop: + exec: + command: ["kill", "1"] + env: + - name: STATSD_HOST + value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} + - name: STATSD_PORT + value: {{ $.Values.statsdPort | default "9125" | quote }} + - name: CONCURRENCY + value: "1" + - name: MAX_TASKS_PER_CHILD + value: "1" + - name: LOGLEVEL + value: "INFO" + - name: SWH_CONFIG_FILENAME + # FIXME: built by entrypoint.sh, determine how to properly declare this + value: /tmp/config.yml + - name: SWH_SENTRY_ENVIRONMENT + value: {{ $.Values.sentry.environment }} + - name: SWH_MAIN_PACKAGE + value: {{ $.Values.listers.sentrySwhPackage }} + - name: SWH_SENTRY_DSN + valueFrom: + secretKeyRef: + name: lister-sentry-secrets + key: sentry-dsn + # 'name' secret must exist & include key "host" + optional: false + - name: RABBITMQ_HOST + valueFrom: + secretKeyRef: + name: amqp-access-credentials + key: host + # 'name' secret must exist & include key "host" + optional: false + volumeMounts: + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + readOnly: true + - name: lister-credentials-secrets + mountPath: /etc/credentials/listers + readOnly: true + - mountPath: /tmp + name: tmp-volume + volumes: + - name: config + configMap: + name: {{ $lister_name }} + defaultMode: 0777 + - name: tmp-volume + emptyDir: {} + - name: lister-credentials-secrets + secret: + secretName: lister-credentials-secrets +{{ end }} +{{- end -}} diff --git a/swh/templates/listers/keda-autoscaling.yaml b/swh/templates/listers/keda-autoscaling.yaml new file mode 100644 index 0000000..b0e49e3 --- /dev/null +++ b/swh/templates/listers/keda-autoscaling.yaml @@ -0,0 +1,83 @@ +{{ if .Values.listers.enabled -}} +{{- range $lister_type, $deployment_config := .Values.listers.deployments -}} +{{ if get $deployment_config "autoScaling" }} +{{- $autoscalingConfig := get $deployment_config "autoScaling" -}} +{{- $lister_name := ( print "lister-" $lister_type ) -}} +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: amqp-authentication + namespace: {{ $.Values.namespace }} +spec: + secretTargetRef: # Optional. + - parameter: host + name: amqp-access-credentials + key: host + +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ $lister_name }}-operators + namespace: {{ $.Values.namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 # Optional. Default: apps/v1 + kind: Deployment # Optional. Default: Deployment + # Mandatory. Must be in same namespace as ScaledObject + name: {{ $lister_name }} + # envSourceContainerName: {container-name} # Optional. Default: + # .spec.template.spec.containers[0] + pollingInterval: 30 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + idleReplicaCount: 0 # Optional. Must be less than + # minReplicaCount + minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }} + maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }} + fallback: # Optional. Section to specify fallback + # options + failureThreshold: 3 # Mandatory if fallback section is + # included + replicas: 6 # Mandatory if fallback section is + # included + advanced: # Optional. Section to specify advanced + # options + restoreToOriginalReplicaCount: false # Optional. Default: false + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA + # related options + behavior: # Optional. Use to modify HPA's scaling + # behavior + scaleDown: + stabilizationWindowSeconds: 60 # default 300 + policies: + - type: Percent + value: 2 + periodSeconds: 15 + triggers: + {{- range $queue := get $deployment_config "queues" }} + - type: rabbitmq + authenticationRef: + name: amqp-authentication + metadata: + host: host + # Optional. If not specified, it must be done + # by using TriggerAuthentication. + protocol: auto # Optional. Specifies protocol to use, + # either amqp or http, or auto to + # autodetect based on the `host` value. + # Default value is auto. + mode: QueueLength # QueueLength or MessageRate + # message backlog or publish/sec. + # target per instance + value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }} + queueName: {{ $queue }} + vhostName: / # Optional. If not specified, use the vhost in the + # `host` connection string. Alternatively, you can + # use existing environment variables to read + # configuration from: See details in "Parameter + # list" section hostFromEnv: RABBITMQ_HOST% + {{- end }} +{{ end }} +{{ end }} +{{- end -}} diff --git a/swh/values/default.yaml b/swh/values/default.yaml index d20de4b..dbeb773 100644 --- a/swh/values/default.yaml +++ b/swh/values/default.yaml @@ -1,45 +1,56 @@ namespace: swh storage_replayer: storageClass: cassandra # only cassandra is currently supported maxMessagesBytes: "524288000" journalBrokers: # The name of the secret containing the BROKER_USER_PASSWORD value secretName: storage-replayer-broker-secret cassandra: keySpace: swh consistencyLevel: LOCAL_QUORUM affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/etcd operator: NotIn values: - "true" loaders: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "swh/loader" operator: In values: - "true" +listers: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "swh/lister" + operator: In + values: + - "true" + graphql: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: swh/rpc operator: In values: - "true" statsd_exporter: enabled: true diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml index 8849748..d391113 100644 --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -1,137 +1,159 @@ sentry: environment: staging loaders: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 amqp: host: scheduler0.internal.staging.swh.network deployments: # bzr: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.bzr.tasks.LoadBazaar # autoScaling: # queueThreshold: 10 # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.bzr # cvs: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.cvs.tasks.LoadCvsRepository # autoScaling: # queueThreshold: 10 # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.cvs # hg: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.mercurial.tasks.LoadArchiveMercurial # - swh.loader.mercurial.tasks.LoadMercurial # autoScaling: # queueThreshold: 10 # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.mercurial highpriority: requestedMemory: 256Mi requestedCpu: 200m queues: - save_code_now:swh.loader.bzr.tasks.LoadBazaar - save_code_now:swh.loader.git.tasks.UpdateGitRepository - save_code_now:swh.loader.git.tasks.LoadDiskGitRepository - save_code_now:swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial - save_code_now:swh.loader.mercurial.tasks.LoadMercurial - save_code_now:swh.loader.svn.tasks.LoadSvnRepository - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - save_code_now:swh.loader.package.archive.tasks.LoadArchive autoScaling: queueThreshold: 10 minReplicacount: 0 maxReplicaCount: 3 sentrySwhPackage: swh.loader.highpriority git: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository autoScaling: queueThreshold: 5 # spawn worker per increment of `value` messages minReplicacount: 1 maxReplicaCount: 5 sentrySwhPackage: swh.loader.git # maven: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.package.maven.tasks.LoadMaven # autoScaling: # queueThreshold: 10 # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.core # npm: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.package.npm.tasks.LoadNpm # autoScaling: # queueThreshold: 10 # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.core pypi: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.package.pypi.tasks.LoadPyPI autoScaling: queueThreshold: 10 minReplicacount: 1 maxReplicaCount: 3 sentrySwhPackage: swh.loader.core # svn: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.svn.tasks.LoadSvnRepository # - swh.loader.svn.tasks.MountAndLoadSvnRepository # - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # autoScaling: # queueThreshold: 10 # minReplicacount: 1 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.svn +listers: + enabled: true + sentrySwhPackage: swh.lister + storage: + host: storage1.internal.staging.swh.network + port: 5002 + scheduler: + host: scheduler1.internal.staging.swh.network + port: 5008 + amqp: + host: scheduler0.internal.staging.swh.network + deployments: + gnu-full: + requestedMemory: 256Mi + requestedCpu: 200m + queues: + - swh.lister.gnu.tasks.GNUListerTask + autoScaling: + queueThreshold: 10 + minReplicacount: 0 + maxReplicaCount: 1 + graphql: enabled: true sentry_enabled: true backends: storage: host: webapp.internal.staging.swh.network port: 5002 search: host: search0.internal.staging.swh.network port: 5010 replicas: 1 gunicorn: threads: 4 workers: 2 timeout: 3600 ingress: enabled: true httpPath: / host: graphql.staging.swh.network logLevel: DEBUG diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml index 520e249..2a4ebff 100644 --- a/values-swh-application-versions.yaml +++ b/values-swh-application-versions.yaml @@ -1,12 +1,14 @@ # This file references the last version of all the softwareheritage images # It's used to manage the automatic update of the environments swh_storage_replayer_image: softwareheritage/storage-replayer swh_storage_replayer_image_version: "20220819.1" swh_loaders_image: softwareheritage/loaders swh_loaders_image_version: "20220826.1" swh_loader_git_image: softwareheritage/loader_git swh_loader_git_image_version: "20220906.1" swh_graphql_image: softwareheritage/graphql swh_graphql_image_version: "20220826.1" +swh_listers_image: softwareheritage/lister +swh_listers_image_version: "20220906.1"