Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7124069
D8400.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
13 KB
Subscribers
None
D8400.diff
View Options
diff --git a/swh/templates/listers/configmap.yaml b/swh/templates/listers/configmap.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/listers/configmap.yaml
@@ -0,0 +1,41 @@
+{{ if .Values.listers.enabled -}}
+{{- range $lister_type, $deployment_config := .Values.listers.deployments -}}
+{{- $lister_name := ( print "lister-" $lister_type ) -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ $lister_name }}-template
+ namespace: {{ $.Values.namespace }}
+data:
+ config.yml.template: |
+ storage:
+ cls: remote
+ url: http://{{ $.Values.listers.storage.host }}:{{ $.Values.listers.storage.port }}/
+ scheduler:
+ cls: remote
+ url: http://{{ $.Values.listers.scheduler.host }}:{{ $.Values.listers.scheduler.port }}/
+ celery:
+ task_broker: ${host}
+ task_queues:
+ {{- range $queue := get $deployment_config "queues" }}
+ - {{ $queue }}
+ {{- end }}
+ credentials:
+ init-container-entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ CONFIG_FILE=/etc/swh/config.yml
+
+ # substitute environment variables when creating the default config.yml
+ eval echo \""$(</etc/swh/configuration-template/config.yml.template)"\" \
+ > $CONFIG_FILE
+
+ CREDS_LISTER_PATH=/etc/credentials/listers/credentials
+ [ -f $CREDS_LISTER_PATH ] && \
+ sed 's/^/ /g' $CREDS_LISTER_PATH >> $CONFIG_FILE
+
+{{ end }}
+{{- end -}}
diff --git a/swh/templates/listers/deployment.yaml b/swh/templates/listers/deployment.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/listers/deployment.yaml
@@ -0,0 +1,119 @@
+{{ if .Values.listers.enabled -}}
+{{- $configurationChecksum := include (print $.Template.BasePath "/listers/configmap.yaml") . -}}
+{{- range $lister_type, $deployment_config := .Values.listers.deployments -}}
+{{- $lister_name := ( print "lister-" $lister_type ) -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ $lister_name }}
+ namespace: {{ $.Values.namespace }}
+ labels:
+ app: {{ $lister_name }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ $lister_name }}
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: {{ $lister_name }}
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: {{ $configurationChecksum | sha256sum }}
+ spec:
+ {{- if $.Values.listers.affinity }}
+ affinity:
+ {{ toYaml $.Values.listers.affinity | nindent 8 }}
+ {{- end }}
+ initContainers:
+ - name: prepare-configuration
+ image: debian:bullseye
+ imagePullPolicy: Always
+ envFrom:
+ - secretRef:
+ # define the ${host) values for configuration "celery > task_broker: " key
+ name: amqp-access-credentials
+ command:
+ - /entrypoint.sh
+ volumeMounts:
+ - name: configuration-template
+ mountPath: /entrypoint.sh
+ subPath: "init-container-entrypoint.sh"
+ readOnly: true
+ - name: configuration
+ mountPath: /etc/swh
+ - name: configuration-template
+ mountPath: /etc/swh/configuration-template
+ - name: lister-credentials-secrets
+ mountPath: /etc/credentials/listers
+ readOnly: true
+ containers:
+ - name: listers
+ resources:
+ requests:
+ memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }}
+ cpu: {{ get $deployment_config "requestedCpu" | default "500m" }}
+ limits:
+ memory: "4000Mi"
+ cpu: "1200m"
+ image: {{ $.Values.swh_listers_image }}:{{ $.Values.swh_listers_image_version }}
+ imagePullPolicy: Always
+ command:
+ - /bin/bash
+ args:
+ - -c
+ - /opt/swh/entrypoint.sh
+ lifecycle:
+ preStop:
+ exec:
+ command: ["kill", "1"]
+ env:
+ - name: STATSD_HOST
+ value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ $.Values.statsdPort | default "9125" | quote }}
+ - name: CONCURRENCY
+ value: {{ get $deployment_config "concurrency" | default 1 | quote }}
+ - name: MAX_TASKS_PER_CHILD
+ value: {{ get $deployment_config "maxTasksPerChild" | default 1 | quote }}
+ - name: LOGLEVEL
+ value: {{ get $deployment_config "logLevel" | default "INFO" | quote }}
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/swh/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ $.Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: {{ $.Values.listers.sentrySwhPackage }}
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: lister-sentry-secrets
+ key: sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+ volumeMounts:
+ - name: configuration
+ mountPath: /etc/swh
+ volumes:
+ - name: configuration
+ emptyDir: {}
+ - name: configuration-template
+ configMap:
+ name: {{ $lister_name }}-template
+ defaultMode: 0777
+ items:
+ - key: "config.yml.template"
+ path: "config.yml.template"
+ - key: "init-container-entrypoint.sh"
+ path: "init-container-entrypoint.sh"
+ - name: lister-credentials-secrets
+ secret:
+ secretName: lister-credentials-secrets
+ optional: true
+{{ end }}
+{{- end -}}
diff --git a/swh/templates/listers/keda-autoscaling.yaml b/swh/templates/listers/keda-autoscaling.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/listers/keda-autoscaling.yaml
@@ -0,0 +1,83 @@
+{{ if .Values.listers.enabled -}}
+{{- range $lister_type, $deployment_config := .Values.listers.deployments -}}
+{{ if get $deployment_config "autoScaling" }}
+{{- $autoscalingConfig := get $deployment_config "autoScaling" -}}
+{{- $lister_name := ( print "lister-" $lister_type ) -}}
+---
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+ name: amqp-authentication
+ namespace: {{ $.Values.namespace }}
+spec:
+ secretTargetRef: # Optional.
+ - parameter: host
+ name: amqp-access-credentials
+ key: host
+
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: {{ $lister_name }}-operators
+ namespace: {{ $.Values.namespace }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ # Mandatory. Must be in same namespace as ScaledObject
+ name: {{ $lister_name }}
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 300 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Optional. Must be less than
+ # minReplicaCount
+ minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }}
+ maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }}
+ fallback: # Optional. Section to specify fallback
+ # options
+ failureThreshold: 3 # Mandatory if fallback section is
+ # included
+ replicas: 6 # Mandatory if fallback section is
+ # included
+ advanced: # Optional. Section to specify advanced
+ # options
+ restoreToOriginalReplicaCount: false # Optional. Default: false
+ horizontalPodAutoscalerConfig: # Optional. Section to specify HPA
+ # related options
+ behavior: # Optional. Use to modify HPA's scaling
+ # behavior
+ scaleDown:
+ stabilizationWindowSeconds: 60 # default 300
+ policies:
+ - type: Percent
+ value: 2
+ periodSeconds: 15
+ triggers:
+ {{- range $queue := get $deployment_config "queues" }}
+ - type: rabbitmq
+ authenticationRef:
+ name: amqp-authentication
+ metadata:
+ host: host
+ # Optional. If not specified, it must be done
+ # by using TriggerAuthentication.
+ protocol: auto # Optional. Specifies protocol to use,
+ # either amqp or http, or auto to
+ # autodetect based on the `host` value.
+ # Default value is auto.
+ mode: QueueLength # QueueLength or MessageRate
+ # message backlog or publish/sec.
+ # target per instance
+ value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }}
+ queueName: {{ $queue }}
+ vhostName: / # Optional. If not specified, use the vhost in the
+ # `host` connection string. Alternatively, you can
+ # use existing environment variables to read
+ # configuration from: See details in "Parameter
+ # list" section hostFromEnv: RABBITMQ_HOST%
+ {{- end }}
+{{ end }}
+{{ end }}
+{{- end -}}
diff --git a/swh/values/default.yaml b/swh/values/default.yaml
--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -30,6 +30,17 @@
values:
- "true"
+listers:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: "swh/lister"
+ operator: In
+ values:
+ - "true"
+
graphql:
affinity:
nodeAffinity:
diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml
--- a/swh/values/staging.yaml
+++ b/swh/values/staging.yaml
@@ -16,7 +16,6 @@
# - swh.loader.bzr.tasks.LoadBazaar
# autoScaling:
# queueThreshold: 10
- # minReplicacount: 1
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.bzr
# cvs:
@@ -26,7 +25,6 @@
# - swh.loader.cvs.tasks.LoadCvsRepository
# autoScaling:
# queueThreshold: 10
- # minReplicacount: 1
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.cvs
# hg:
@@ -37,7 +35,6 @@
# - swh.loader.mercurial.tasks.LoadMercurial
# autoScaling:
# queueThreshold: 10
- # minReplicacount: 1
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.mercurial
highpriority:
@@ -56,7 +53,6 @@
- save_code_now:swh.loader.package.archive.tasks.LoadArchive
autoScaling:
queueThreshold: 10
- minReplicacount: 0
maxReplicaCount: 3
sentrySwhPackage: swh.loader.highpriority
git:
@@ -68,8 +64,7 @@
- swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
autoScaling:
queueThreshold: 5 # spawn worker per increment of `value` messages
- minReplicacount: 1
- maxReplicaCount: 5
+ maxReplicaCount: 3
sentrySwhPackage: swh.loader.git
# maven:
# requestedMemory: 256Mi
@@ -78,7 +73,6 @@
# - swh.loader.package.maven.tasks.LoadMaven
# autoScaling:
# queueThreshold: 10
- # minReplicacount: 1
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.core
# npm:
@@ -88,7 +82,6 @@
# - swh.loader.package.npm.tasks.LoadNpm
# autoScaling:
# queueThreshold: 10
- # minReplicacount: 1
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.core
pypi:
@@ -98,8 +91,7 @@
- swh.loader.package.pypi.tasks.LoadPyPI
autoScaling:
queueThreshold: 10
- minReplicacount: 1
- maxReplicaCount: 3
+ maxReplicaCount: 1
sentrySwhPackage: swh.loader.core
# svn:
# requestedMemory: 256Mi
@@ -110,10 +102,29 @@
# - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository
# autoScaling:
# queueThreshold: 10
- # minReplicacount: 1
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.svn
+listers:
+ enabled: true
+ sentrySwhPackage: swh.lister
+ storage:
+ host: storage1.internal.staging.swh.network
+ port: 5002
+ scheduler:
+ host: scheduler0.internal.staging.swh.network
+ port: 5008
+ amqp:
+ host: scheduler0.internal.staging.swh.network
+ deployments:
+ gnu-full:
+ queues:
+ - swh.lister.gnu.tasks.GNUListerTask
+ autoScaling:
+ queueThreshold: 10
+ minReplicaCount: 0
+ maxReplicaCount: 1
+
graphql:
enabled: true
sentry_enabled: true
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -9,4 +9,5 @@
swh_loader_git_image_version: "20220906.1"
swh_graphql_image: softwareheritage/graphql
swh_graphql_image_version: "20220826.1"
-
+swh_listers_image: softwareheritage/lister
+swh_listers_image_version: "20220907.4"
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Dec 20 2024, 12:01 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3224172
Attached To
D8400: archive-staging: Deploy listers in cluster
Event Timeline
Log In to Comment