Page MenuHomeSoftware Heritage
Files F9337182

No OneTemporary
Actions

View Options

diff --git a/sysadmin/T3592-elastic-workers/worker/README.md b/sysadmin/T3592-elastic-workers/worker/README.md
new file mode 100644
index 0000000..00a6c5c
--- /dev/null
+++ b/sysadmin/T3592-elastic-workers/worker/README.md
@@ -0,0 +1,22 @@
+
+# Goal
+
+- autoscaling workers depending on repositories to load and allocated resources.
+
+# keda
+
+Install KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling):
+```
+$ helm repo add kedacore https://kedacore.github.io/charts
+$ helm repo update
+swhworker@poc-rancher:~$ kubectl create namespace keda
+namespace/keda created
+swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda
+NAME: keda
+LAST DEPLOYED: Fri Oct 8 09:48:40 2021
+NAMESPACE: keda
+STATUS: deployed
+REVISION: 1
+TEST SUITE: None
+```
+source: https://keda.sh/docs/2.4/deploy/
diff --git a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml
index 276bc94..99931b4 100644
--- a/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml
+++ b/sysadmin/T3592-elastic-workers/worker/templates/autoscale.yaml
@@ -1,19 +1,58 @@
---
-apiVersion: autoscaling/v2beta2
-kind: HorizontalPodAutoscaler
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
metadata:
- name: loaders
+ name: loaders-operators
spec:
scaleTargetRef:
- apiVersion: apps/v1
- kind: Deployment
- name: loaders
- minReplicas: {{ .Values.swh.loader.replicas.min }}
- maxReplicas: {{ .Values.swh.loader.replicas.max }}
- metrics:
- - type: Resource
- resource:
- name: cpu
- target:
- type: Utilization
- averageUtilization: 50
+ apiVersion: apps/v1 # Optional. Default: apps/v1
+ kind: Deployment # Optional. Default: Deployment
+ name: loaders # Mandatory. Must be in same namespace as ScaledObject
+ # envSourceContainerName: {container-name} # Optional. Default:
+ # .spec.template.spec.containers[0]
+ pollingInterval: 30 # Optional. Default: 30 seconds
+ cooldownPeriod: 300 # Optional. Default: 300 seconds
+ idleReplicaCount: 0 # Optional. Must be less than
+ # minReplicaCount
+ minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0
+ maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100
+ fallback: # Optional. Section to specify fallback
+ # options
+ failureThreshold: 3 # Mandatory if fallback section is
+ # included
+ replicas: 6 # Mandatory if fallback section is
+ # included
+ advanced: # Optional. Section to specify advanced
+ # options
+ restoreToOriginalReplicaCount: false # Optional. Default: false
+ horizontalPodAutoscalerConfig: # Optional. Section to specify HPA
+ # related options
+ behavior: # Optional. Use to modify HPA's scaling
+ # behavior
+ scaleDown:
+ stabilizationWindowSeconds: 60 # default 300
+ policies:
+ - type: Percent
+ value: 2
+ periodSeconds: 15
+ triggers:
+ {{- range .Values.amqp.queues }}
+ - type: rabbitmq
+ metadata:
+ host: amqp://{{ $.Values.amqp.username }}:{{ $.Values.amqp.password }}@{{ $.Values.amqp.host }}//
+ # Optional. If not specified, it must be done
+ # by using TriggerAuthentication.
+ protocol: auto # Optional. Specifies protocol to use,
+ # either amqp or http, or auto to
+ # autodetect based on the `host` value.
+ # Default value is auto.
+ mode: QueueLength # QueueLength or MessageRate
+ value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec.
+ # target per instance
+ queueName: {{ . }}
+ vhostName: / # Optional. If not specified, use the vhost in the
+ # `host` connection string. Alternatively, you can
+ # use existing environment variables to read
+ # configuration from: See details in "Parameter
+ # list" section hostFromEnv: RABBITMQ_HOST%
+ {{- end }}
diff --git a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml
index 358638a..5345d7f 100644
--- a/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml
+++ b/sysadmin/T3592-elastic-workers/worker/templates/config-map.yaml
@@ -1,40 +1,41 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: loaders
data:
config.yml: |
storage:
cls: pipeline
steps:
- cls: buffer
min_batch_size:
content: 10000
content_bytes: 104857600
directory: 10000
revision: 10000
- cls: filter
- cls: retry
- cls: remote
url: http://{{ .Values.storage.host }}:5002/
celery:
- task_broker: amqp://{{ .Values.amqp.user }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}//
+ task_broker: amqp://{{ .Values.amqp.username }}:{{ .Values.amqp.password }}@{{ .Values.amqp.host }}//
task_queues:
- - swh.loader.git.tasks.UpdateGitRepository
+ {{- range .Values.amqp.queues }}
+ - {{ . }}
+ {{- end }}
entrypoint.sh: |
#!/bin/bash
set -e
echo Starting the swh Celery worker
exec python -m celery \
--app=swh.scheduler.celery_backend.config.app \
worker \
--pool=prefork \
--concurrency=${CONCURRENCY} \
--max-tasks-per-child=${MAX_TASKS_PER_CHILD} \
-Ofair --loglevel=${LOGLEVEL} \
--hostname "${HOSTNAME}"
-
diff --git a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml
index 31f9e40..c713b3e 100644
--- a/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml
+++ b/sysadmin/T3592-elastic-workers/worker/templates/deployment.yaml
@@ -1,65 +1,65 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: loaders
labels:
app: loaders
spec:
- replicas: {{ .Values.swh.loader.replicas.default }}
+ replicas: {{ .Values.swh.loader.replicas.min }}
selector:
matchLabels:
app: loaders
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
template:
metadata:
labels:
app: loaders
spec:
containers:
- name: loaders
image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }}
imagePullPolicy: Always
command:
- /entrypoint.sh
resources:
requests:
memory: "256Mi"
cpu: "200m"
limits:
memory: "4000Mi"
cpu: "1200m"
lifecycle:
preStop:
exec:
command: ["kill", "1"]
env:
- name: CONCURRENCY
value: "1"
- name: MAX_TASKS_PER_CHILD
value: "5"
- name: LOGLEVEL
value: "INFO"
- name: SWH_CONFIG_FILENAME
value: /etc/softwareheritage/config.yml
volumeMounts:
- name: config
mountPath: /etc/softwareheritage/config.yml
subPath: config.yml
readOnly: true
- name: config
mountPath: /entrypoint.sh
subPath: entrypoint.sh
readOnly: true
- mountPath: /tmp
name: tmp-volume
volumes:
- name: config
configMap:
name: loaders
defaultMode: 0777
- name: tmp-volume
emptyDir: {}
diff --git a/sysadmin/T3592-elastic-workers/worker/values.yaml b/sysadmin/T3592-elastic-workers/worker/values.yaml
index 9f77fab..cb3b737 100644
--- a/sysadmin/T3592-elastic-workers/worker/values.yaml
+++ b/sysadmin/T3592-elastic-workers/worker/values.yaml
@@ -1,20 +1,20 @@
# Default values for worker.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
amqp:
user: guest
password: guest
host: amqp
storage:
host: swh-storage
swh:
loader:
image: swh/loaders
version: latest
replicas:
min: 1
- max: 10
- default: 5
+ max: 5
+

File Metadata

Mime Type
text/x-diff
Expires
Jul 4 2025, 7:56 AM (10 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3247375

Event Timeline

About · Developer information · Legal