diff --git a/swh/templates/loaders/deployment.yaml b/swh/templates/loaders/deployment.yaml index 482be7f..ffac064 100644 --- a/swh/templates/loaders/deployment.yaml +++ b/swh/templates/loaders/deployment.yaml @@ -1,163 +1,163 @@ {{ if .Values.loaders.enabled -}} {{- $configurationChecksum := include (print $.Template.BasePath "/loaders/configmap.yaml") . -}} {{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} {{- $loader_name := ( print "loader-" $loader_type ) -}} # if defined at the "typed" loader level {{- $local_container_image_key := get $deployment_config "image" }} {{- $local_container_image := get $.Values $local_container_image_key }} {{- $local_container_image_version_key := ( print $local_container_image_key "_version" ) }} {{- $local_container_image_version := get $.Values $local_container_image_version_key }} # otherwise if the global image is defined {{- $image_name_key := ( print "swh_loader_" $loader_type "_image" ) -}} {{- $image_version_key := ( print $image_name_key "_version" ) -}} {{- $container_image := get $.Values $image_name_key }} {{- $container_image_version := get $.Values $image_version_key }} --- apiVersion: apps/v1 kind: Deployment metadata: name: {{ $loader_name }} namespace: {{ $.Values.namespace }} labels: app: {{ $loader_name }} spec: revisionHistoryLimit: 2 selector: matchLabels: app: {{ $loader_name }} strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: {{ $loader_name }} annotations: # Force a rollout upgrade if the configuration changes checksum/config: {{ $configurationChecksum | sha256sum }} spec: {{- if $.Values.loaders.affinity }} affinity: {{ toYaml $.Values.loaders.affinity | nindent 8 }} {{- end }} terminationGracePeriodSeconds: 3600 initContainers: - name: prepare-configuration image: debian:bullseye imagePullPolicy: Always env: - name: AMQP_USERNAME valueFrom: secretKeyRef: name: common-secrets key: rabbitmq-amqp-username # 'name' secret must exist & include that ^ key optional: false - name: AMQP_PASSWORD valueFrom: secretKeyRef: name: common-secrets key: rabbitmq-amqp-password # 'name' secret must exist & include that ^ key optional: false command: - /entrypoint.sh volumeMounts: - name: configuration-template mountPath: /entrypoint.sh subPath: "init-container-entrypoint.sh" readOnly: true - name: configuration mountPath: /etc/swh - name: configuration-template mountPath: /etc/swh/configuration-template - name: metadata-fetcher-credentials mountPath: /etc/credentials/metadata-fetcher readOnly: true containers: - name: loaders {{ if $local_container_image -}} image: {{ $local_container_image }}:{{ $local_container_image_version }} {{ else if $container_image -}} image: {{ $container_image }}:{{ $container_image_version }} {{ else -}} image: {{ $.Values.swh_loaders_image }}:{{ $.Values.swh_loaders_image_version }} {{ end }} imagePullPolicy: Always command: - /opt/swh/entrypoint.sh resources: requests: memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }} cpu: {{ get $deployment_config "requestedCpu" | default "500m" }} limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: STATSD_HOST value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} - name: STATSD_PORT value: {{ $.Values.statsdPort | default "9125" | quote }} - name: MAX_TASKS_PER_CHILD - value: {{ get $deployment_config "maxTasksPerChild" | default 5 | quote }} + value: {{ get $deployment_config "maxTasksPerChild" | default 10 | quote }} - name: LOGLEVEL value: {{ get $deployment_config "logLevel" | default "INFO" | quote }} - name: SWH_CONFIG_FILENAME value: /etc/swh/config.yml - name: SWH_SENTRY_ENVIRONMENT value: {{ $.Values.sentry.environment }} - name: SWH_MAIN_PACKAGE value: {{ get $deployment_config "sentrySwhPackage" }} - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: name: common-secrets key: {{ $loader_name}}-sentry-dsn # 'name' secret must exist & include key "host" optional: false {{ if ( contains "deposit" $loader_name ) -}} - name: DEPOSIT_USERNAME valueFrom: secretKeyRef: name: common-secrets key: deposit-username # 'name' secret must exist & include key "host" optional: false - name: DEPOSIT_PASSWORD valueFrom: secretKeyRef: name: common-secrets key: deposit-username # 'name' secret must exist & include key "host" optional: false {{ end }} volumeMounts: - name: configuration mountPath: /etc/swh - name: localstorage mountPath: /tmp volumes: - name: localstorage emptyDir: {} - name: configuration emptyDir: {} - name: configuration-template configMap: name: {{ $loader_name }}-template defaultMode: 0777 items: - key: "config.yml.template" path: "config.yml.template" - key: "init-container-entrypoint.sh" path: "init-container-entrypoint.sh" - name: metadata-fetcher-credentials secret: secretName: metadata-fetcher-credentials optional: true {{ end }} {{- end -}} diff --git a/swh/templates/loaders/keda-autoscaling.yaml b/swh/templates/loaders/keda-autoscaling.yaml index 451ade1..cb3ab93 100644 --- a/swh/templates/loaders/keda-autoscaling.yaml +++ b/swh/templates/loaders/keda-autoscaling.yaml @@ -1,64 +1,64 @@ {{ if .Values.loaders.enabled -}} {{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} {{ if get $deployment_config "autoScaling" }} {{- $autoscalingConfig := get $deployment_config "autoScaling" -}} {{- $loader_name := ( print "loader-" $loader_type ) -}} --- apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: name: amqp-authentication-{{ $loader_name }} namespace: {{ $.Values.namespace }} spec: secretTargetRef: # Optional. - parameter: HTTP_HOST name: common-secrets key: rabbitmq-http-host --- apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: name: {{ $loader_name }}-operators namespace: {{ $.Values.namespace }} spec: scaleTargetRef: apiVersion: apps/v1 # Optional. Default: apps/v1 kind: Deployment # Optional. Default: Deployment # Mandatory. Must be in same namespace as ScaledObject name: {{ $loader_name }} # envSourceContainerName: {container-name} # Optional. Default: # .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds idleReplicaCount: 0 # Optional. Must be less than # minReplicaCount minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }} maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }} triggers: {{- range $queue := get $deployment_config "queues" }} - type: rabbitmq authenticationRef: name: amqp-authentication-{{ $loader_name }} metadata: host: HTTP_HOST # Optional. If not specified, it must be done # by using TriggerAuthentication. protocol: http # Optional. Specifies protocol to use, # either amqp or http, or auto to # autodetect based on the `host` value. # Default value is auto. mode: QueueLength # QueueLength to trigger on number of msgs in queue excludeUnacknowledged: "false" # QueueLength should include unacked messages # Implies "http" protocol is used - value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }} + value: {{ get $autoscalingConfig "queueThreshold" | default 10 | quote }} queueName: {{ $queue }} vhostName: / # Optional. If not specified, use the vhost in the # `host` connection string. Alternatively, you can # use existing environment variables to read # configuration from: See details in "Parameter # list" section hostFromEnv: RABBITMQ_HOST% {{- end }} {{ end }} {{ end }} {{- end -}} diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml index 1cebda0..93d3f27 100644 --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -1,269 +1,256 @@ sentry: environment: staging loaders: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 amqp: host: scheduler0.internal.staging.swh.network port: 5672 deployments: archive: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.archive.tasks.LoadArchive autoScaling: - queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core addforgenow: image: swh_loader_git_image requestedMemory: 256Mi requestedCpu: 200m queues: - add_forge_now:swh.loader.git.tasks.UpdateGitRepository autoScaling: queueThreshold: 1 maxReplicaCount: 5 sentrySwhPackage: swh.loader.git bzr: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.bzr.tasks.LoadBazaar autoScaling: - queueThreshold: 3000000 maxReplicaCount: 3 sentrySwhPackage: swh.loader.bzr cran: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.cran.tasks.LoadCRAN autoScaling: - queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core cvs: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.cvs.tasks.LoadCvsRepository autoScaling: - queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.cvs debian: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.debian.tasks.LoadDebian autoScaling: - queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core deposit: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.deposit.tasks.LoadDeposit autoScaling: queueThreshold: 1 maxReplicaCount: 3 sentrySwhPackage: swh.loader.core extraConfig: deposit: url: "https://deposit-rp.internal.staging.swh.network/1/private" auth: username: "${DEPOSIT_USERNAME}" password: "${DEPOSIT_PASSWORD}" default_filename: archive.tar mercurial: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.mercurial.tasks.LoadArchiveMercurial - swh.loader.mercurial.tasks.LoadMercurial autoScaling: - queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.mercurial nixguix: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.nixguix.tasks.LoadNixguix autoScaling: queueThreshold: 1 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core extraConfig: unsupported_file_extensions: - patch - iso - whl - gem - pom - msi - pod - png - rock - ttf - jar - c - el - rpm - diff highpriority: requestedMemory: 256Mi requestedCpu: 200m queues: - save_code_now:swh.loader.bzr.tasks.LoadBazaar - save_code_now:swh.loader.git.tasks.UpdateGitRepository - save_code_now:swh.loader.git.tasks.LoadDiskGitRepository - save_code_now:swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial - save_code_now:swh.loader.mercurial.tasks.LoadMercurial - save_code_now:swh.loader.svn.tasks.LoadSvnRepository - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - save_code_now:swh.loader.package.archive.tasks.LoadArchive autoScaling: queueThreshold: 1 maxReplicaCount: 3 sentrySwhPackage: swh.loader.highpriority git: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository autoScaling: - queueThreshold: 10 # spawn worker per increment of `value` messages maxReplicaCount: 3 sentrySwhPackage: swh.loader.git maven: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.maven.tasks.LoadMaven autoScaling: - queueThreshold: 500 maxReplicaCount: 3 sentrySwhPackage: swh.loader.core npm: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.npm.tasks.LoadNpm autoScaling: - queueThreshold: 2000 maxReplicaCount: 2 sentrySwhPackage: swh.loader.core opam: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.opam.tasks.LoadOpam autoScaling: - queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core extraConfig: # FIXME: Find a way to avoid this initialization step in loader task initialize_opam_root: true pypi: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pypi.tasks.LoadPyPI autoScaling: - queueThreshold: 1000 maxReplicaCount: 3 sentrySwhPackage: swh.loader.core pubdev: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pubdev.tasks.LoadPubDev autoScaling: - queueThreshold: 3000 maxReplicaCount: 3 sentrySwhPackage: swh.loader.core svn: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository autoScaling: - queueThreshold: 500 maxReplicaCount: 3 sentrySwhPackage: swh.loader.svn listers: enabled: true sentrySwhPackage: swh.lister storage: host: storage1.internal.staging.swh.network port: 5002 scheduler: host: scheduler0.internal.staging.swh.network port: 5008 amqp: host: scheduler0.internal.staging.swh.network deployments: gnu-full: queues: - swh.lister.gnu.tasks.GNUListerTask autoScaling: maxReplicaCount: 1 pubdev: logLevel: DEBUG queues: - swh.lister.pubdev.tasks.PubDevListerTask autoScaling: maxReplicaCount: 1 gogs-full: queues: - swh.lister.gogs.tasks.FullGogsRelister autoScaling: maxReplicaCount: 1 graphql: enabled: true sentry_enabled: true backends: storage: host: webapp.internal.staging.swh.network port: 5002 search: host: search0.internal.staging.swh.network port: 5010 replicas: 1 gunicorn: threads: 4 workers: 2 timeout: 3600 ingress: enabled: true httpPath: / host: graphql.staging.swh.network logLevel: DEBUG