diff --git a/swh/templates/listers/keda-autoscaling.yaml b/swh/templates/listers/keda-autoscaling.yaml index b0e49e3..71a56ad 100644 --- a/swh/templates/listers/keda-autoscaling.yaml +++ b/swh/templates/listers/keda-autoscaling.yaml @@ -1,83 +1,83 @@ {{ if .Values.listers.enabled -}} {{- range $lister_type, $deployment_config := .Values.listers.deployments -}} {{ if get $deployment_config "autoScaling" }} {{- $autoscalingConfig := get $deployment_config "autoScaling" -}} {{- $lister_name := ( print "lister-" $lister_type ) -}} --- apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: - name: amqp-authentication + name: amqp-authentication-{{ $lister_name }} namespace: {{ $.Values.namespace }} spec: secretTargetRef: # Optional. - parameter: host name: amqp-access-credentials key: host --- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: name: {{ $lister_name }}-operators namespace: {{ $.Values.namespace }} spec: scaleTargetRef: apiVersion: apps/v1 # Optional. Default: apps/v1 kind: Deployment # Optional. Default: Deployment # Mandatory. Must be in same namespace as ScaledObject name: {{ $lister_name }} # envSourceContainerName: {container-name} # Optional. Default: # .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds idleReplicaCount: 0 # Optional. Must be less than # minReplicaCount minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }} maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }} fallback: # Optional. Section to specify fallback # options failureThreshold: 3 # Mandatory if fallback section is # included replicas: 6 # Mandatory if fallback section is # included advanced: # Optional. Section to specify advanced # options restoreToOriginalReplicaCount: false # Optional. Default: false horizontalPodAutoscalerConfig: # Optional. Section to specify HPA # related options behavior: # Optional. Use to modify HPA's scaling # behavior scaleDown: stabilizationWindowSeconds: 60 # default 300 policies: - type: Percent value: 2 periodSeconds: 15 triggers: {{- range $queue := get $deployment_config "queues" }} - type: rabbitmq authenticationRef: - name: amqp-authentication + name: amqp-authentication-{{ $lister_name }} metadata: host: host # Optional. If not specified, it must be done # by using TriggerAuthentication. protocol: auto # Optional. Specifies protocol to use, # either amqp or http, or auto to # autodetect based on the `host` value. # Default value is auto. mode: QueueLength # QueueLength or MessageRate # message backlog or publish/sec. # target per instance value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }} queueName: {{ $queue }} vhostName: / # Optional. If not specified, use the vhost in the # `host` connection string. Alternatively, you can # use existing environment variables to read # configuration from: See details in "Parameter # list" section hostFromEnv: RABBITMQ_HOST% {{- end }} {{ end }} {{ end }} {{- end -}} diff --git a/swh/templates/loaders/keda-autoscaling.yaml b/swh/templates/loaders/keda-autoscaling.yaml index d20f073..bd34ac2 100644 --- a/swh/templates/loaders/keda-autoscaling.yaml +++ b/swh/templates/loaders/keda-autoscaling.yaml @@ -1,83 +1,83 @@ {{ if .Values.loaders.enabled -}} {{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} {{ if get $deployment_config "autoScaling" }} {{- $autoscalingConfig := get $deployment_config "autoScaling" -}} {{- $loader_name := ( print "loader-" $loader_type ) -}} --- apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: - name: amqp-authentication + name: amqp-authentication-{{ $loader_name }} namespace: {{ $.Values.namespace }} spec: secretTargetRef: # Optional. - parameter: host name: amqp-access-credentials key: host --- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: name: {{ $loader_name }}-operators namespace: {{ $.Values.namespace }} spec: scaleTargetRef: apiVersion: apps/v1 # Optional. Default: apps/v1 kind: Deployment # Optional. Default: Deployment # Mandatory. Must be in same namespace as ScaledObject name: {{ $loader_name }} # envSourceContainerName: {container-name} # Optional. Default: # .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds idleReplicaCount: 0 # Optional. Must be less than # minReplicaCount minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }} maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }} fallback: # Optional. Section to specify fallback # options failureThreshold: 3 # Mandatory if fallback section is # included replicas: 6 # Mandatory if fallback section is # included advanced: # Optional. Section to specify advanced # options restoreToOriginalReplicaCount: false # Optional. Default: false horizontalPodAutoscalerConfig: # Optional. Section to specify HPA # related options behavior: # Optional. Use to modify HPA's scaling # behavior scaleDown: stabilizationWindowSeconds: 60 # default 300 policies: - type: Percent value: 2 periodSeconds: 15 triggers: {{- range $queue := get $deployment_config "queues" }} - type: rabbitmq authenticationRef: - name: amqp-authentication + name: amqp-authentication-{{ $loader_name }} metadata: host: host # Optional. If not specified, it must be done # by using TriggerAuthentication. protocol: auto # Optional. Specifies protocol to use, # either amqp or http, or auto to # autodetect based on the `host` value. # Default value is auto. mode: QueueLength # QueueLength or MessageRate # message backlog or publish/sec. # target per instance value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }} queueName: {{ $queue }} vhostName: / # Optional. If not specified, use the vhost in the # `host` connection string. Alternatively, you can # use existing environment variables to read # configuration from: See details in "Parameter # list" section hostFromEnv: RABBITMQ_HOST% {{- end }} {{ end }} {{ end }} {{- end -}} diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml index 32b1633..9b3aadd 100644 --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -1,151 +1,166 @@ sentry: environment: staging loaders: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 amqp: host: scheduler0.internal.staging.swh.network deployments: - # bzr: - # requestedMemory: 256Mi - # requestedCpu: 200m - # queues: - # - swh.loader.bzr.tasks.LoadBazaar - # autoScaling: - # queueThreshold: 10 - # maxReplicaCount: 3 - # sentrySwhPackage: swh.loader.bzr + bzr: + requestedMemory: 256Mi + requestedCpu: 200m + queues: + - swh.loader.bzr.tasks.LoadBazaar + autoScaling: + queueThreshold: 10 + maxReplicaCount: 1 + sentrySwhPackage: swh.loader.bzr # cvs: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.cvs.tasks.LoadCvsRepository # autoScaling: # queueThreshold: 10 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.cvs # hg: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.mercurial.tasks.LoadArchiveMercurial # - swh.loader.mercurial.tasks.LoadMercurial # autoScaling: # queueThreshold: 10 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.mercurial highpriority: requestedMemory: 256Mi requestedCpu: 200m queues: - save_code_now:swh.loader.bzr.tasks.LoadBazaar - save_code_now:swh.loader.git.tasks.UpdateGitRepository - save_code_now:swh.loader.git.tasks.LoadDiskGitRepository - save_code_now:swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial - save_code_now:swh.loader.mercurial.tasks.LoadMercurial - save_code_now:swh.loader.svn.tasks.LoadSvnRepository - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - save_code_now:swh.loader.package.archive.tasks.LoadArchive autoScaling: queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.highpriority git: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository autoScaling: queueThreshold: 5 # spawn worker per increment of `value` messages maxReplicaCount: 3 sentrySwhPackage: swh.loader.git # maven: # requestedMemory: 256Mi # requestedCpu: 200m # image: swh_loader_package_image # queues: # - swh.loader.package.maven.tasks.LoadMaven # autoScaling: # queueThreshold: 10 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.core npm: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.npm.tasks.LoadNpm autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core pypi: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pypi.tasks.LoadPyPI autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core + pubdev: + requestedMemory: 256Mi + requestedCpu: 200m + image: swh_loader_package_image + queues: + - swh.loader.package.pubdev.tasks.LoadPubDev + autoScaling: + queueThreshold: 10 + maxReplicaCount: 1 + sentrySwhPackage: swh.loader.core # svn: # requestedMemory: 256Mi # requestedCpu: 200m # queues: # - swh.loader.svn.tasks.LoadSvnRepository # - swh.loader.svn.tasks.MountAndLoadSvnRepository # - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # autoScaling: # queueThreshold: 10 # maxReplicaCount: 3 # sentrySwhPackage: swh.loader.svn listers: enabled: true sentrySwhPackage: swh.lister storage: host: storage1.internal.staging.swh.network port: 5002 scheduler: host: scheduler0.internal.staging.swh.network port: 5008 amqp: host: scheduler0.internal.staging.swh.network deployments: gnu-full: queues: - swh.lister.gnu.tasks.GNUListerTask autoScaling: queueThreshold: 10 - minReplicaCount: 0 + maxReplicaCount: 1 + pubdev: + queues: + - swh.lister.pubdev.tasks.PubDevListerTask + autoScaling: + queueThreshold: 10 maxReplicaCount: 1 graphql: enabled: true sentry_enabled: true backends: storage: host: webapp.internal.staging.swh.network port: 5002 search: host: search0.internal.staging.swh.network port: 5010 replicas: 1 gunicorn: threads: 4 workers: 2 timeout: 3600 ingress: enabled: true httpPath: / host: graphql.staging.swh.network logLevel: DEBUG diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml index 4482185..0ed2f57 100644 --- a/values-swh-application-versions.yaml +++ b/values-swh-application-versions.yaml @@ -1,17 +1,19 @@ # This file references the last version of all the softwareheritage images # It's used to manage the automatic update of the environments swh_storage_replayer_image: softwareheritage/storage-replayer swh_storage_replayer_image_version: "20220819.1" swh_loaders_image: softwareheritage/loaders swh_loaders_image_version: "20220826.1" +swh_loader_bzr_image: softwareheritage/loader_bzr +swh_loader_bzr_image_version: "20220909.1" swh_loader_git_image: softwareheritage/loader_git swh_loader_git_image_version: "20220908.1" swh_loader_highpriority_image: softwareheritage/loader_highpriority swh_loader_highpriority_image_version: "20220908.1" swh_loader_package_image: softwareheritage/loader_package swh_loader_package_image_version: "20220909.1" swh_graphql_image: softwareheritage/graphql swh_graphql_image_version: "20220826.1" swh_listers_image: softwareheritage/lister swh_listers_image_version: "20220908.1"