diff --git a/worker/templates/config-map.yaml b/swh/templates/loaders/configmap.yaml rename from worker/templates/config-map.yaml rename to swh/templates/loaders/configmap.yaml --- a/worker/templates/config-map.yaml +++ b/swh/templates/loaders/configmap.yaml @@ -1,9 +1,12 @@ +{{ if .Values.loaders.enabled -}} +{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} +{{- $loader_name := ( print "loader-" $loader_type ) -}} --- apiVersion: v1 kind: ConfigMap metadata: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} + name: {{ $loader_name }} + namespace: {{ $.Values.namespace }} data: config.yml: | storage: @@ -24,13 +27,13 @@ - cls: filter - cls: retry - cls: remote - url: http://{{ .Values.storage.host }}:5002/ + url: http://{{ $.Values.loaders.storage.host }}:{{ $.Values.loaders.storage.port }}/ celery: task_broker: ##amqp_host## task_queues: - {{- range .Values.amqp.queues }} - - {{ . }} + {{- range $queue := get $deployment_config "task_queues" }} + - {{ $queue }} {{- end }} entrypoint.sh: | #!/bin/bash @@ -57,3 +60,5 @@ --without-gossip \ --without-mingle \ --hostname "${HOSTNAME}" +{{ end }} +{{- end -}} diff --git a/worker/templates/deployment.yaml b/swh/templates/loaders/deployment.yaml rename from worker/templates/deployment.yaml rename to swh/templates/loaders/deployment.yaml --- a/worker/templates/deployment.yaml +++ b/swh/templates/loaders/deployment.yaml @@ -1,15 +1,19 @@ +{{ if .Values.loaders.enabled -}} +{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} +{{- $loader_name := ( print "loader-" $loader_type ) -}} + --- apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} + name: {{ $loader_name }} + namespace: {{ $.Values.namespace }} labels: - app: {{ .Values.loader.name }}-{{ .Values.loader.type }} + app: {{ $loader_name }} spec: selector: matchLabels: - app: {{ .Values.loader.name }}-{{ .Values.loader.type }} + app: {{ $loader_name }} strategy: type: RollingUpdate rollingUpdate: @@ -17,18 +21,18 @@ template: metadata: labels: - app: {{ .Values.loader.name }}-{{ .Values.loader.type }} + app: {{ $loader_name }} spec: containers: - name: loaders - image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} + image: {{ $.Values.swh_loaders_image }}:{{ $.Values.swh_loaders_image_version }} imagePullPolicy: Always command: - /entrypoint.sh resources: requests: - memory: "256Mi" - cpu: "200m" + memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }} + cpu: {{ get $deployment_config "requestedCpu" | default "500m" }} limits: memory: "4000Mi" cpu: "1200m" @@ -38,9 +42,9 @@ command: ["kill", "1"] env: - name: STATSD_HOST - value: "prometheus-statsd-exporter.default" + value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} - name: STATSD_PORT - value: "9125" + value: {{ $.Values.statsdPort | default "9125" | quote }} - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD @@ -51,13 +55,13 @@ # FIXME: built by entrypoint.sh, determine how to properly declare this value: /tmp/config.yml - name: SWH_SENTRY_ENVIRONMENT - value: {{ .Values.sentry.environment }} + value: {{ $.Values.sentry.environment }} - name: SWH_MAIN_PACKAGE - value: {{ .Values.sentry.swhpackage }} + value: {{ get $deployment_config "swhpackage" }} - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }}-sentry-secrets + name: {{ $loader_name }}-sentry-secrets key: sentry-dsn # 'name' secret must exist & include key "host" optional: false @@ -85,10 +89,12 @@ volumes: - name: config configMap: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} + name: {{ $loader_name }} defaultMode: 0777 - name: tmp-volume emptyDir: {} - name: metadata-fetcher-credentials secret: secretName: metadata-fetcher-credentials +{{ end }} +{{- end -}} diff --git a/worker/templates/autoscale.yaml b/swh/templates/loaders/keda-autoscaling.yaml rename from worker/templates/autoscale.yaml rename to swh/templates/loaders/keda-autoscaling.yaml --- a/worker/templates/autoscale.yaml +++ b/swh/templates/loaders/keda-autoscaling.yaml @@ -1,9 +1,14 @@ +{{ if .Values.loaders.enabled -}} +{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} +{{ if get $deployment_config "autoScaling" }} +{{- $autoscalingConfig := get $deployment_config "autoScaling" -}} +{{- $loader_name := ( print "loader-" $loader_type ) -}} --- apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: name: amqp-authentication - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} + namespace: {{ $.Values.namespace }} spec: secretTargetRef: # Optional. - parameter: host @@ -14,22 +19,22 @@ apiVersion: keda.sh/v1alpha1 kind: ScaledObject metadata: - name: loaders-{{ .Values.loader.name }}-{{ .Values.loader.type }}-operators - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} + name: {{ $loader_name }}-operators + namespace: {{ $.Values.namespace }} spec: scaleTargetRef: apiVersion: apps/v1 # Optional. Default: apps/v1 kind: Deployment # Optional. Default: Deployment # Mandatory. Must be in same namespace as ScaledObject - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} + name: {{ $loader_name }} # envSourceContainerName: {container-name} # Optional. Default: # .spec.template.spec.containers[0] pollingInterval: 30 # Optional. Default: 30 seconds cooldownPeriod: 300 # Optional. Default: 300 seconds idleReplicaCount: 0 # Optional. Must be less than # minReplicaCount - minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0 - maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100 + minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }} + maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }} fallback: # Optional. Section to specify fallback # options failureThreshold: 3 # Mandatory if fallback section is @@ -50,7 +55,7 @@ value: 2 periodSeconds: 15 triggers: - {{- range .Values.amqp.queues }} + {{- range $queue := get $deployment_config "task_queues" }} - type: rabbitmq authenticationRef: name: amqp-authentication @@ -63,12 +68,16 @@ # autodetect based on the `host` value. # Default value is auto. mode: QueueLength # QueueLength or MessageRate - value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec. - # target per instance - queueName: {{ . }} + # message backlog or publish/sec. + # target per instance + value: {{ get $autoscalingConfig "queue_threshold" | default 100 }} + queueName: {{ $queue }} vhostName: / # Optional. If not specified, use the vhost in the # `host` connection string. Alternatively, you can # use existing environment variables to read # configuration from: See details in "Parameter # list" section hostFromEnv: RABBITMQ_HOST% {{- end }} +{{ end }} +{{ end }} +{{- end -}} diff --git a/swh/templates/loaders/services.yaml b/swh/templates/loaders/services.yaml new file mode 100644 --- /dev/null +++ b/swh/templates/loaders/services.yaml @@ -0,0 +1,24 @@ +{{ if .Values.loaders.enabled -}} +{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} +{{- $loader_name := ( print "loader-" $loader_type ) -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: storage-{{ $loader_name }} + namespace: {{ $.Values.namespace }} +spec: + type: ExternalName + externalName: {{ $.Values.loaders.storage.host }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: amqp-{{ $loader_name }} + namespace: {{ $.Values.namespace }} +spec: + type: ExternalName + externalName: {{ $.Values.loaders.amqp.host }} +{{ end }} +{{- end -}} diff --git a/swh/values.yaml b/swh/values.yaml --- a/swh/values.yaml +++ b/swh/values.yaml @@ -45,6 +45,12 @@ # lagThreashold: 1000 # minReplicaCount: 1 # maxReplicaCount: 10 +loaders: + enabled: false + deployments: + # Example of deployments + # git: + statsd_exporter: enabled: false diff --git a/swh/values/default.yaml b/swh/values/default.yaml --- a/swh/values/default.yaml +++ b/swh/values/default.yaml @@ -1,4 +1,6 @@ namespace: swh +sentry: + environment: staging storage_replayer: storageClass: cassandra # only cassandra is currently supported diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml new file mode 100644 --- /dev/null +++ b/swh/values/staging.yaml @@ -0,0 +1,21 @@ +loaders: + enabled: true + storage: + host: storage.internal.staging.swh.network + port: 5002 + amqp: + host: scheduler0.internal.staging.swh.network + deployments: + git: + requestedMemory: 256Mi + requestedCpu: 200m + task_queues: + - swh.loader.git.tasks.UpdateGitRepository + - swh.loader.git.tasks.LoadDiskGitRepository + - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository + autoScaling: + queue_threshold: 5 # spawn worker per increment of `value` messages + minReplicacount: 1 + maxReplicaCount: 3 + swhpackage: swh.loader.git + diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml --- a/values-swh-application-versions.yaml +++ b/values-swh-application-versions.yaml @@ -3,3 +3,5 @@ swh_storage_replayer_image: softwareheritage/storage-replayer swh_storage_replayer_image_version: "20220819.1" +swh_loaders_image: softwareheritage/loaders +swh_loaders_image_version: 2022-05-17 diff --git a/worker/.helmignore b/worker/.helmignore deleted file mode 100644 --- a/worker/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/worker/.gitignore b/worker/.gitignore deleted file mode 100644 --- a/worker/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.secret.yaml - diff --git a/worker/Chart.yaml b/worker/Chart.yaml deleted file mode 100644 --- a/worker/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: worker -description: A Helm chart to deploy dynamic workers on Kubernetes cluster - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/worker/README.md b/worker/README.md deleted file mode 100644 --- a/worker/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Goal - -- autoscaling workers depending on repositories to load and allocated resources. - -# keda - -This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): -``` -$ helm repo add kedacore https://kedacore.github.io/charts -$ helm repo update -swhworker@poc-rancher:~$ kubectl create namespace keda -namespace/keda created -swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda -NAME: keda -LAST DEPLOYED: Fri Oct 8 09:48:40 2021 -NAMESPACE: keda -STATUS: deployed -REVISION: 1 -TEST SUITE: None -``` -source: https://keda.sh/docs/2.4/deploy/ - -# helm - -We use helm to ease the cluster application management. - -# Install - -Install the worker declaration from this directory in the cluster -``` -$ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml -$ TYPE=git; REL=workers-$TYPE; \ - helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker -$ TYPE=pypi; REL=workers-$TYPE; \ - helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker -``` - -Where: -``` -$ cat ../loader-git.staging.values.yaml -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.git.tasks.UpdateGitRepository - - swh.loader.git.tasks.LoadDiskGitRepository - - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: git -``` - -# List - -List currently deployed applications: - -``` -$ helm list -helm list -NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION -workers-bzr default 1 2022-04-29 12:59:32.111950055 +0200 CEST deployed worker-0.1.0 1.16.0 -workers-git default 4 2022-04-29 12:50:12.322826487 +0200 CEST deployed worker-0.1.0 1.16.0 -workers-pypi default 1 2022-04-29 12:51:22.506259018 +0200 CEST deployed worker-0.1.0 1.16.0 -``` - -# Upgrade - -When adapting the worker definition, you can apply the changes by upgrading the -deployed application: - -``` -$ TYPE=git; REL=workers-$TYPE; \ - helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker -``` - -# Secrets - -The current work requires credentials (installed as secret within the cluster): -- metadata fetcher credentials `metadata-fetcher-credentials` -- amqp credentials `` - -More details describing the secrets: -``` -$ kubectl describe secret metadata-fetcher-credentials -``` - -Installed through: - -``` -$ TYPE=git # Replace mentions below in the yaml files -$ kubectl -f $SECRET_FILE apply --namespaces ns-loader-$TYPE -# for secret file in { -# instances/loaders-$TYPE-metadata-fetcher-credentials.secret.yaml -# ./loader-$TYPE-sentry.secret.yaml -# ./amqp-access-credentials.secret.yaml -# ... -# } -$ cat instances/loaders-metadata-fetcher.secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: metadata-fetcher-credentials -type: Opaque -stringData: - data: | - metadata_fetcher_credentials: - github: - github: - - username: - password: - - ... -$ cat ./amqp-access-credentials.secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: amqp-access-credentials - namespace: ns-loaders-$TYPE -type: Opaque -stringData: - host: amqp://:@scheduler0.internal.staging.swh.network:5672/%2f -$ cat ./loaders-$TYPE-sentry.secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: loaders-$TYPE-sentry-secrets - namespace: ns-loaders-$TYPE -type: Opaque -stringData: - sentry-dsn: https://@sentry.softwareheritage.org/8 -``` diff --git a/worker/instances/loaders-bzr.staging.values.yaml b/worker/instances/loaders-bzr.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-bzr.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.bzr.tasks.LoadBazaar - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: bzr diff --git a/worker/instances/loaders-cvs.staging.values.yaml b/worker/instances/loaders-cvs.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-cvs.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.cvs.tasks.LoadCvsRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: cvs diff --git a/worker/instances/loaders-git.staging.values.yaml b/worker/instances/loaders-git.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-git.staging.values.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 5 # spawn worker per increment of `value` messages - queues: - - swh.loader.git.tasks.UpdateGitRepository - - swh.loader.git.tasks.LoadDiskGitRepository - - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: git - -swh: - loader: - replicas: - min: 1 - max: 3 - -sentry: - swhpackage: swh.loader.git diff --git a/worker/instances/loaders-maven.staging.values.yaml b/worker/instances/loaders-maven.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-maven.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.package.maven.tasks.LoadMaven - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: maven diff --git a/worker/instances/loaders-npm.staging.values.yaml b/worker/instances/loaders-npm.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-npm.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.package.npm.tasks.LoadNpm - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: npm diff --git a/worker/instances/loaders-pypi.staging.values.yaml b/worker/instances/loaders-pypi.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-pypi.staging.values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.package.pypi.tasks.LoadPyPI - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: pypi - diff --git a/worker/instances/loaders-svn.staging.values.yaml b/worker/instances/loaders-svn.staging.values.yaml deleted file mode 100644 --- a/worker/instances/loaders-svn.staging.values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.svn.tasks.LoadSvnRepository - - swh.loader.svn.tasks.MountAndLoadSvnRepository - - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: svn diff --git a/worker/templates/services.yaml b/worker/templates/services.yaml deleted file mode 100644 --- a/worker/templates/services.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: storage-{{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - type: ExternalName - externalName: {{ .Values.storage.host }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: amqp-{{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - type: ExternalName - externalName: {{ .Values.amqp.host }} - diff --git a/worker/values.yaml b/worker/values.yaml deleted file mode 100644 --- a/worker/values.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: amqp - -storage: - host: swh-storage - -loader: - name: loaders - type: - -swh: - loader: - image: softwareheritage/loaders - version: 2022-05-17 - replicas: - min: 1 - max: 5 - -sentry: - environment: staging