Page MenuHomeSoftware Heritage
Files F7123575

D8400.id30332.diff
No OneTemporary
Actions

D8400.id30332.diff
View Options

diff --git a/swh/templates/listers/configmap.yaml b/swh/templates/listers/configmap.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/listers/configmap.yaml
@@ -0,0 +1,51 @@
+{{ if .Values.listers.enabled -}}
+{{- range $lister_type, $deployment_config := .Values.listers.deployments -}}
+{{- $lister_name := ( print "lister-" $lister_type ) -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ $lister_name }}
+ namespace: {{ $.Values.namespace }}
+data:
+ config.yml: |
+ storage:
+ cls: remote
+ url: http://{{ $.Values.listers.storage.host }}:{{ $.Values.listers.storage.port }}/
+ scheduler:
+ cls: remote
+ url: http://{{ $.Values.listers.scheduler.host }}:{{ $.Values.listers.scheduler.port }}/
+
+ celery:
+ task_broker: ##amqp_host##
+ task_queues:
+ {{- range $queue := get $deployment_config "queues" }}
+ - {{ $queue }}
+ {{- end }}
+ entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ # Create the full config filename
+ cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME
+ # contains required credentials for lister
+ cat /etc/credentials/listers/data >> $SWH_CONFIG_FILENAME
+
+ # Install the rabbitmq host information
+ sed -i 's,##amqp_host##,'$RABBITMQ_HOST',g' $SWH_CONFIG_FILENAME
+
+ echo Starting the swh Celery worker
+ exec python -m celery \
+ --app=swh.scheduler.celery_backend.config.app \
+ worker \
+ --pool=prefork \
+ --concurrency=${CONCURRENCY} \
+ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
+ -Ofair --loglevel=${LOGLEVEL} \
+ --without-gossip \
+ --without-mingle \
+ --without-heartbeat \
+ --hostname "${HOSTNAME}"
+{{ end }}
+{{- end -}}
diff --git a/swh/templates/listers/deployment.yaml b/swh/templates/listers/deployment.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/listers/deployment.yaml
@@ -0,0 +1,107 @@
+{{ if .Values.listers.enabled -}}
+{{- $configurationChecksum := include (print $.Template.BasePath "/listers/configmap.yaml") . -}}
+{{- range $lister_type, $deployment_config := .Values.listers.deployments -}}
+{{- $lister_name := ( print "lister-" $lister_type ) -}}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ $lister_name }}
+ namespace: {{ $.Values.namespace }}
+ labels:
+ app: {{ $lister_name }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ $lister_name }}
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: {{ $lister_name }}
+ annotations:
+ # Force a rollout upgrade if the configuration changes
+ checksum/config: {{ $configurationChecksum | sha256sum }}
+ spec:
+ {{- if $.Values.listers.affinity }}
+ affinity:
+ {{ toYaml $.Values.listers.affinity | nindent 8 }}
+ {{- end }}
+ containers:
+ - name: listers
+ image: {{ $.Values.swh_listers_image }}:{{ $.Values.swh_listers_image_version }}
+ imagePullPolicy: Always
+ command:
+ - /entrypoint.sh
+ resources:
+ requests:
+ memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }}
+ cpu: {{ get $deployment_config "requestedCpu" | default "500m" }}
+ limits:
+ memory: "4000Mi"
+ cpu: "1200m"
+ lifecycle:
+ preStop:
+ exec:
+ command: ["kill", "1"]
+ env:
+ - name: STATSD_HOST
+ value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
+ - name: STATSD_PORT
+ value: {{ $.Values.statsdPort | default "9125" | quote }}
+ - name: CONCURRENCY
+ value: "1"
+ - name: MAX_TASKS_PER_CHILD
+ value: "1"
+ - name: LOGLEVEL
+ value: "INFO"
+ - name: SWH_CONFIG_FILENAME
+ # FIXME: built by entrypoint.sh, determine how to properly declare this
+ value: /tmp/config.yml
+ - name: SWH_SENTRY_ENVIRONMENT
+ value: {{ $.Values.sentry.environment }}
+ - name: SWH_MAIN_PACKAGE
+ value: {{ $.Values.listers.sentrySwhPackage }}
+ - name: SWH_SENTRY_DSN
+ valueFrom:
+ secretKeyRef:
+ name: lister-sentry-secrets
+ key: sentry-dsn
+ # 'name' secret must exist & include key "host"
+ optional: false
+ - name: RABBITMQ_HOST
+ valueFrom:
+ secretKeyRef:
+ name: amqp-access-credentials
+ key: host
+ # 'name' secret must exist & include key "host"
+ optional: false
+ volumeMounts:
+ - name: config
+ mountPath: /etc/softwareheritage/config.yml
+ subPath: config.yml
+ readOnly: true
+ - name: config
+ mountPath: /entrypoint.sh
+ subPath: entrypoint.sh
+ readOnly: true
+ - name: lister-credentials-secrets
+ mountPath: /etc/credentials/listers
+ readOnly: true
+ - mountPath: /tmp
+ name: tmp-volume
+ volumes:
+ - name: config
+ configMap:
+ name: {{ $lister_name }}
+ defaultMode: 0777
+ - name: tmp-volume
+ emptyDir: {}
+ - name: lister-credentials-secrets
+ secret:
+ secretName: lister-credentials-secrets
+{{ end }}
+{{- end -}}
diff --git a/swh/templates/listers/keda-autoscaling.yaml b/swh/templates/listers/keda-autoscaling.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/listers/keda-autoscaling.yaml
@@ -0,0 +1,83 @@
+{{ if .Values.listers.enabled -}}
+{{- range $lister_type, $deployment_config := .Values.listers.deployments -}}
+{{ if get $deployment_config "autoScaling" }}
+{{- $autoscalingConfig := get $deployment_config "autoScaling" -}}
+{{- $lister_name := ( print "lister-" $lister_type ) -}}
+---
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+ name: amqp-authentication
+ namespace: {{ $.Values.namespace }}
+spec:
+ secretTargetRef: # Optional.
+ - parameter: host
+ name: amqp-access-credentials
+ key: host
+
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+ name: {{ $lister_name }}-operators
+ namespace: {{ $.Values.namespace }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ # Mandatory. Must be in same namespace as ScaledObject
+ name: {{ $lister_name }}
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 300 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Optional. Must be less than
+ # minReplicaCount
+ minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }}
+ maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }}
+ fallback: # Optional. Section to specify fallback
+ # options
+ failureThreshold: 3 # Mandatory if fallback section is
+ # included
+ replicas: 6 # Mandatory if fallback section is
+ # included
+ advanced: # Optional. Section to specify advanced
+ # options
+ restoreToOriginalReplicaCount: false # Optional. Default: false
+ horizontalPodAutoscalerConfig: # Optional. Section to specify HPA
+ # related options
+ behavior: # Optional. Use to modify HPA's scaling
+ # behavior
+ scaleDown:
+ stabilizationWindowSeconds: 60 # default 300
+ policies:
+ - type: Percent
+ value: 2
+ periodSeconds: 15
+ triggers:
+ {{- range $queue := get $deployment_config "queues" }}
+ - type: rabbitmq
+ authenticationRef:
+ name: amqp-authentication
+ metadata:
+ host: host
+ # Optional. If not specified, it must be done
+ # by using TriggerAuthentication.
+ protocol: auto # Optional. Specifies protocol to use,
+ # either amqp or http, or auto to
+ # autodetect based on the `host` value.
+ # Default value is auto.
+ mode: QueueLength # QueueLength or MessageRate
+ # message backlog or publish/sec.
+ # target per instance
+ value: {{ get $autoscalingConfig "queueThreshold" | default 100 | quote }}
+ queueName: {{ $queue }}
+ vhostName: / # Optional. If not specified, use the vhost in the
+ # `host` connection string. Alternatively, you can
+ # use existing environment variables to read
+ # configuration from: See details in "Parameter
+ # list" section hostFromEnv: RABBITMQ_HOST%
+ {{- end }}
+{{ end }}
+{{ end }}
+{{- end -}}
diff --git a/swh/values/default.yaml b/swh/values/default.yaml
--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -30,6 +30,17 @@
values:
- "true"
+listers:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: "swh/lister"
+ operator: In
+ values:
+ - "true"
+
graphql:
affinity:
nodeAffinity:
diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml
--- a/swh/values/staging.yaml
+++ b/swh/values/staging.yaml
@@ -114,6 +114,28 @@
# maxReplicaCount: 3
# sentrySwhPackage: swh.loader.svn
+listers:
+ enabled: true
+ sentrySwhPackage: swh.lister
+ storage:
+ host: storage1.internal.staging.swh.network
+ port: 5002
+ scheduler:
+ host: scheduler0.internal.staging.swh.network
+ port: 5008
+ amqp:
+ host: scheduler0.internal.staging.swh.network
+ deployments:
+ gnu-full:
+ requestedMemory: 256Mi
+ requestedCpu: 200m
+ queues:
+ - swh.lister.gnu.tasks.GNUListerTask
+ autoScaling:
+ queueThreshold: 10
+ minReplicacount: 0
+ maxReplicaCount: 1
+
graphql:
enabled: true
sentry_enabled: true
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -9,4 +9,6 @@
swh_loader_git_image_version: "20220906.1"
swh_graphql_image: softwareheritage/graphql
swh_graphql_image_version: "20220826.1"
+swh_listers_image: softwareheritage/lister
+swh_listers_image_version: "20220906.1"

File Metadata

Mime Type
text/plain
Expires
Thu, Dec 19, 11:24 AM (1 d, 21 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3234372

Event Timeline

About · Developer information · Legal