Page MenuHomeSoftware Heritage
Files F7124121

D8315.diff
No OneTemporary
Actions

D8315.diff
View Options

diff --git a/monitoring/README.md b/monitoring/README.md
deleted file mode 100644
--- a/monitoring/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-monitoring
-----------
-
-Install prometheus exporter:
-```
-kubectl apply -f prometheus-exporter.yml
-```
diff --git a/monitoring/prometheus-statsd-exporter.yml b/monitoring/prometheus-statsd-exporter.yml
deleted file mode 100644
--- a/monitoring/prometheus-statsd-exporter.yml
+++ /dev/null
@@ -1,119 +0,0 @@
-# -------
-# Prometheus statsd exporter configuration
-# -------
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: prometheus-statsd-exporter
-data:
- config.yml: |
- defaults:
- timer_type: histogram
- buckets:
- - .005
- - .01
- - .025
- - .05
- - .1
- - .25
- - .5
- - .75
- - 1
- - 2
- - 5
- - 10
- - 15
- - 30
- - 45
- - 60
- - 120
- - 300
- - 600
- - 900
- - 1800
- - 2700
- - 3600
- - 7200
- mappings:
- - match: "(.*_percent)"
- name: "${1}"
- match_type: regex
- observer_type: histogram
- histogram_options:
- buckets:
- - 0.0
- - 0.05
- - 0.1
- - 0.15
- - 0.2
- - 0.25
- - 0.3
- - 0.35
- - 0.4
- - 0.45
- - 0.5
- - 0.55
- - 0.6
- - 0.65
- - 0.7
- - 0.75
- - 0.8
- - 0.85
- - 0.9
- - 0.95
- - 1.
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: prometheus-statsd-exporter
- labels:
- app: prometheus-statsd-exporter
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: prometheus-statsd-exporter
- template:
- metadata:
- labels:
- app: prometheus-statsd-exporter
- spec:
- containers:
- - name: prometheus-statsd-exporter
- image: prom/statsd-exporter
- imagePullPolicy: Always
- args:
- - "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml"
- ports:
- - containerPort: 9125
- volumeMounts:
- - name: config
- mountPath: /etc/prometheus/statsd-mapping.yml
- subPath: config.yml
- readOnly: true
- volumes:
- - name: config
- configMap:
- name: prometheus-statsd-exporter
----
-apiVersion: v1
-kind: Service
-metadata:
- name: prometheus-statsd-exporter-svc
- labels:
- app: prometheus-statsd-exporter
-spec:
- type: ClusterIP
- selector:
- app: prometheus-statsd-exporter
- ports:
- - name: statsd
- port: 9125
- targetPort: 9125
- protocol: UDP
- - name: http
- port: 9102
- targetPort: 9102
diff --git a/monitoring/swh-statsd-metrics-sm.yml b/monitoring/swh-statsd-metrics-sm.yml
deleted file mode 100644
--- a/monitoring/swh-statsd-metrics-sm.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-# The following matches the monitor currently installed in the cluster
-# kubectl -n cattle-monitoring-system get servicemonitors.monitoring.coreos.com -o yaml
----
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: swh-statsd-metrics-sm
- # namespace: cattle-monitoring-system
-spec:
- endpoints:
- - path: /metrics
- port: http
- interval: 10s
- selector:
- matchLabels:
- app: prometheus-statsd-exporter
- namespaceSelector:
- matchNames:
- - default
diff --git a/worker/templates/config-map.yaml b/swh/templates/loaders/configmap.yaml
rename from worker/templates/config-map.yaml
rename to swh/templates/loaders/configmap.yaml
--- a/worker/templates/config-map.yaml
+++ b/swh/templates/loaders/configmap.yaml
@@ -1,9 +1,12 @@
+{{ if .Values.loaders.enabled -}}
+{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}}
+{{- $loader_name := ( print "loader-" $loader_type ) -}}
---
apiVersion: v1
kind: ConfigMap
metadata:
- name: {{ .Values.loader.name }}-{{ .Values.loader.type }}
- namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }}
+ name: {{ $loader_name }}
+ namespace: {{ $.Values.namespace }}
data:
config.yml: |
storage:
@@ -24,13 +27,13 @@
- cls: filter
- cls: retry
- cls: remote
- url: http://{{ .Values.storage.host }}:5002/
+ url: http://storage-loaders:{{ $.Values.loaders.storage.port }}/
celery:
task_broker: ##amqp_host##
task_queues:
- {{- range .Values.amqp.queues }}
- - {{ . }}
+ {{- range $queue := get $deployment_config "queues" }}
+ - {{ $queue }}
{{- end }}
entrypoint.sh: |
#!/bin/bash
@@ -57,3 +60,5 @@
--without-gossip \
--without-mingle \
--hostname "${HOSTNAME}"
+{{ end }}
+{{- end -}}
diff --git a/worker/templates/deployment.yaml b/swh/templates/loaders/deployment.yaml
rename from worker/templates/deployment.yaml
rename to swh/templates/loaders/deployment.yaml
--- a/worker/templates/deployment.yaml
+++ b/swh/templates/loaders/deployment.yaml
@@ -1,15 +1,19 @@
+{{ if .Values.loaders.enabled -}}
+{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}}
+{{- $loader_name := ( print "loader-" $loader_type ) -}}
+
---
apiVersion: apps/v1
kind: Deployment
metadata:
- name: {{ .Values.loader.name }}-{{ .Values.loader.type }}
- namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }}
+ name: {{ $loader_name }}
+ namespace: {{ $.Values.namespace }}
labels:
- app: {{ .Values.loader.name }}-{{ .Values.loader.type }}
+ app: {{ $loader_name }}
spec:
selector:
matchLabels:
- app: {{ .Values.loader.name }}-{{ .Values.loader.type }}
+ app: {{ $loader_name }}
strategy:
type: RollingUpdate
rollingUpdate:
@@ -17,18 +21,22 @@
template:
metadata:
labels:
- app: {{ .Values.loader.name }}-{{ .Values.loader.type }}
+ app: {{ $loader_name }}
spec:
+ {{- if $.Values.loaders.affinity }}
+ affinity:
+ {{ toYaml $.Values.loaders.affinity | nindent 8 }}
+ {{- end }}
containers:
- name: loaders
- image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }}
+ image: {{ $.Values.swh_loaders_image }}:{{ $.Values.swh_loaders_image_version }}
imagePullPolicy: Always
command:
- /entrypoint.sh
resources:
requests:
- memory: "256Mi"
- cpu: "200m"
+ memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }}
+ cpu: {{ get $deployment_config "requestedCpu" | default "500m" }}
limits:
memory: "4000Mi"
cpu: "1200m"
@@ -38,9 +46,9 @@
command: ["kill", "1"]
env:
- name: STATSD_HOST
- value: "prometheus-statsd-exporter.default"
+ value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }}
- name: STATSD_PORT
- value: "9125"
+ value: {{ $.Values.statsdPort | default "9125" | quote }}
- name: CONCURRENCY
value: "1"
- name: MAX_TASKS_PER_CHILD
@@ -51,13 +59,13 @@
# FIXME: built by entrypoint.sh, determine how to properly declare this
value: /tmp/config.yml
- name: SWH_SENTRY_ENVIRONMENT
- value: {{ .Values.sentry.environment }}
+ value: {{ $.Values.sentry.environment }}
- name: SWH_MAIN_PACKAGE
- value: {{ .Values.sentry.swhpackage }}
+ value: {{ get $deployment_config "sentrySwhPackage" }}
- name: SWH_SENTRY_DSN
valueFrom:
secretKeyRef:
- name: {{ .Values.loader.name }}-{{ .Values.loader.type }}-sentry-secrets
+ name: {{ $loader_name }}-sentry-secrets
key: sentry-dsn
# 'name' secret must exist & include key "host"
optional: false
@@ -85,10 +93,12 @@
volumes:
- name: config
configMap:
- name: {{ .Values.loader.name }}-{{ .Values.loader.type }}
+ name: {{ $loader_name }}
defaultMode: 0777
- name: tmp-volume
emptyDir: {}
- name: metadata-fetcher-credentials
secret:
secretName: metadata-fetcher-credentials
+{{ end }}
+{{- end -}}
diff --git a/worker/templates/autoscale.yaml b/swh/templates/loaders/keda-autoscaling.yaml
rename from worker/templates/autoscale.yaml
rename to swh/templates/loaders/keda-autoscaling.yaml
--- a/worker/templates/autoscale.yaml
+++ b/swh/templates/loaders/keda-autoscaling.yaml
@@ -1,9 +1,14 @@
+{{ if .Values.loaders.enabled -}}
+{{- range $loader_type, $deployment_config := .Values.loaders.deployments -}}
+{{ if get $deployment_config "autoScaling" }}
+{{- $autoscalingConfig := get $deployment_config "autoScaling" -}}
+{{- $loader_name := ( print "loader-" $loader_type ) -}}
---
apiVersion: keda.sh/v1alpha1
kind: TriggerAuthentication
metadata:
name: amqp-authentication
- namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }}
+ namespace: {{ $.Values.namespace }}
spec:
secretTargetRef: # Optional.
- parameter: host
@@ -14,22 +19,22 @@
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
- name: loaders-{{ .Values.loader.name }}-{{ .Values.loader.type }}-operators
- namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }}
+ name: {{ $loader_name }}-operators
+ namespace: {{ $.Values.namespace }}
spec:
scaleTargetRef:
apiVersion: apps/v1 # Optional. Default: apps/v1
kind: Deployment # Optional. Default: Deployment
# Mandatory. Must be in same namespace as ScaledObject
- name: {{ .Values.loader.name }}-{{ .Values.loader.type }}
+ name: {{ $loader_name }}
# envSourceContainerName: {container-name} # Optional. Default:
# .spec.template.spec.containers[0]
pollingInterval: 30 # Optional. Default: 30 seconds
cooldownPeriod: 300 # Optional. Default: 300 seconds
idleReplicaCount: 0 # Optional. Must be less than
# minReplicaCount
- minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0
- maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100
+ minReplicaCount: {{ get $autoscalingConfig "minReplicaCount" | default 0 }}
+ maxReplicaCount: {{ get $autoscalingConfig "maxReplicaCount" | default 5 }}
fallback: # Optional. Section to specify fallback
# options
failureThreshold: 3 # Mandatory if fallback section is
@@ -50,7 +55,7 @@
value: 2
periodSeconds: 15
triggers:
- {{- range .Values.amqp.queues }}
+ {{- range $queue := get $deployment_config "queues" }}
- type: rabbitmq
authenticationRef:
name: amqp-authentication
@@ -63,12 +68,16 @@
# autodetect based on the `host` value.
# Default value is auto.
mode: QueueLength # QueueLength or MessageRate
- value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec.
- # target per instance
- queueName: {{ . }}
+ # message backlog or publish/sec.
+ # target per instance
+ value: {{ get $autoscalingConfig "queueThreshold" | default 100 }}
+ queueName: {{ $queue }}
vhostName: / # Optional. If not specified, use the vhost in the
# `host` connection string. Alternatively, you can
# use existing environment variables to read
# configuration from: See details in "Parameter
# list" section hostFromEnv: RABBITMQ_HOST%
{{- end }}
+{{ end }}
+{{ end }}
+{{- end -}}
diff --git a/swh/templates/loaders/services.yaml b/swh/templates/loaders/services.yaml
new file mode 100644
--- /dev/null
+++ b/swh/templates/loaders/services.yaml
@@ -0,0 +1,11 @@
+{{ if .Values.loaders.enabled -}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: storage-loaders
+ namespace: {{ $.Values.namespace }}
+spec:
+ type: ExternalName
+ externalName: {{ .Values.loaders.storage.host }}
+{{- end -}}
diff --git a/swh/values.yaml b/swh/values.yaml
--- a/swh/values.yaml
+++ b/swh/values.yaml
@@ -45,6 +45,12 @@
# lagThreashold: 1000
# minReplicaCount: 1
# maxReplicaCount: 10
+loaders:
+ enabled: false
+ deployments:
+ # Example of deployments
+ # git:
+
statsd_exporter:
enabled: false
diff --git a/swh/values/default.yaml b/swh/values/default.yaml
--- a/swh/values/default.yaml
+++ b/swh/values/default.yaml
@@ -19,5 +19,16 @@
values:
- "true"
+loaders:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: swh/worker
+ operator: In
+ values:
+ - "true"
+
statsd_exporter:
enabled: true
diff --git a/swh/values/production.yaml b/swh/values/production.yaml
--- a/swh/values/production.yaml
+++ b/swh/values/production.yaml
@@ -1,3 +1,6 @@
+sentry:
+ environment: production
+
storage_replayer:
enabled: true
journalBrokers:
diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml
new file mode 100644
--- /dev/null
+++ b/swh/values/staging.yaml
@@ -0,0 +1,24 @@
+sentry:
+ environment: staging
+
+loaders:
+ enabled: true
+ storage:
+ host: storage.internal.staging.swh.network
+ port: 5002
+ amqp:
+ host: scheduler0.internal.staging.swh.network
+ deployments:
+ git:
+ requestedMemory: 256Mi
+ requestedCpu: 200m
+ queues:
+ - swh.loader.git.tasks.UpdateGitRepository
+ - swh.loader.git.tasks.LoadDiskGitRepository
+ - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
+ autoScaling:
+ queueThreshold: 5 # spawn worker per increment of `value` messages
+ minReplicacount: 1
+ maxReplicaCount: 3
+ sentrySwhPackage: swh.loader.git
+
diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml
--- a/values-swh-application-versions.yaml
+++ b/values-swh-application-versions.yaml
@@ -3,3 +3,5 @@
swh_storage_replayer_image: softwareheritage/storage-replayer
swh_storage_replayer_image_version: "20220819.1"
+swh_loaders_image: softwareheritage/loaders
+swh_loaders_image_version: 2022-05-17
diff --git a/worker/.helmignore b/worker/.helmignore
deleted file mode 100644
--- a/worker/.helmignore
+++ /dev/null
@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
diff --git a/worker/.gitignore b/worker/.gitignore
deleted file mode 100644
--- a/worker/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*.secret.yaml
-
diff --git a/worker/Chart.yaml b/worker/Chart.yaml
deleted file mode 100644
--- a/worker/Chart.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-apiVersion: v2
-name: worker
-description: A Helm chart to deploy dynamic workers on Kubernetes cluster
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"
diff --git a/worker/README.md b/worker/README.md
deleted file mode 100644
--- a/worker/README.md
+++ /dev/null
@@ -1,138 +0,0 @@
-# Goal
-
-- autoscaling workers depending on repositories to load and allocated resources.
-
-# keda
-
-This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling):
-```
-$ helm repo add kedacore https://kedacore.github.io/charts
-$ helm repo update
-swhworker@poc-rancher:~$ kubectl create namespace keda
-namespace/keda created
-swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda
-NAME: keda
-LAST DEPLOYED: Fri Oct 8 09:48:40 2021
-NAMESPACE: keda
-STATUS: deployed
-REVISION: 1
-TEST SUITE: None
-```
-source: https://keda.sh/docs/2.4/deploy/
-
-# helm
-
-We use helm to ease the cluster application management.
-
-# Install
-
-Install the worker declaration from this directory in the cluster
-```
-$ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml
-$ TYPE=git; REL=workers-$TYPE; \
- helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker
-$ TYPE=pypi; REL=workers-$TYPE; \
- helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker
-```
-
-Where:
-```
-$ cat ../loader-git.staging.values.yaml
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.git.tasks.UpdateGitRepository
- - swh.loader.git.tasks.LoadDiskGitRepository
- - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: git
-```
-
-# List
-
-List currently deployed applications:
-
-```
-$ helm list
-helm list
-NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
-workers-bzr default 1 2022-04-29 12:59:32.111950055 +0200 CEST deployed worker-0.1.0 1.16.0
-workers-git default 4 2022-04-29 12:50:12.322826487 +0200 CEST deployed worker-0.1.0 1.16.0
-workers-pypi default 1 2022-04-29 12:51:22.506259018 +0200 CEST deployed worker-0.1.0 1.16.0
-```
-
-# Upgrade
-
-When adapting the worker definition, you can apply the changes by upgrading the
-deployed application:
-
-```
-$ TYPE=git; REL=workers-$TYPE; \
- helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker
-```
-
-# Secrets
-
-The current work requires credentials (installed as secret within the cluster):
-- metadata fetcher credentials `metadata-fetcher-credentials`
-- amqp credentials ``
-
-More details describing the secrets:
-```
-$ kubectl describe secret metadata-fetcher-credentials
-```
-
-Installed through:
-
-```
-$ TYPE=git # Replace mentions below in the yaml files
-$ kubectl -f $SECRET_FILE apply --namespaces ns-loader-$TYPE
-# for secret file in {
-# instances/loaders-$TYPE-metadata-fetcher-credentials.secret.yaml
-# ./loader-$TYPE-sentry.secret.yaml
-# ./amqp-access-credentials.secret.yaml
-# ...
-# }
-$ cat instances/loaders-metadata-fetcher.secret.yaml
-apiVersion: v1
-kind: Secret
-metadata:
- name: metadata-fetcher-credentials
-type: Opaque
-stringData:
- data: |
- metadata_fetcher_credentials:
- github:
- github:
- - username: <redacted>
- password: <redacted>
- - ...
-$ cat ./amqp-access-credentials.secret.yaml
-apiVersion: v1
-kind: Secret
-metadata:
- name: amqp-access-credentials
- namespace: ns-loaders-$TYPE
-type: Opaque
-stringData:
- host: amqp://<redacted>:<redacted>@scheduler0.internal.staging.swh.network:5672/%2f
-$ cat ./loaders-$TYPE-sentry.secrets.yaml
-apiVersion: v1
-kind: Secret
-metadata:
- name: loaders-$TYPE-sentry-secrets
- namespace: ns-loaders-$TYPE
-type: Opaque
-stringData:
- sentry-dsn: https://<redacted>@sentry.softwareheritage.org/8
-```
diff --git a/worker/instances/loaders-bzr.staging.values.yaml b/worker/instances/loaders-bzr.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-bzr.staging.values.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.bzr.tasks.LoadBazaar
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: bzr
diff --git a/worker/instances/loaders-cvs.staging.values.yaml b/worker/instances/loaders-cvs.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-cvs.staging.values.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.cvs.tasks.LoadCvsRepository
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: cvs
diff --git a/worker/instances/loaders-git.staging.values.yaml b/worker/instances/loaders-git.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-git.staging.values.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 5 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.git.tasks.UpdateGitRepository
- - swh.loader.git.tasks.LoadDiskGitRepository
- - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: git
-
-swh:
- loader:
- replicas:
- min: 1
- max: 3
-
-sentry:
- swhpackage: swh.loader.git
diff --git a/worker/instances/loaders-maven.staging.values.yaml b/worker/instances/loaders-maven.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-maven.staging.values.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.package.maven.tasks.LoadMaven
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: maven
diff --git a/worker/instances/loaders-npm.staging.values.yaml b/worker/instances/loaders-npm.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-npm.staging.values.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.package.npm.tasks.LoadNpm
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: npm
diff --git a/worker/instances/loaders-pypi.staging.values.yaml b/worker/instances/loaders-pypi.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-pypi.staging.values.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.package.pypi.tasks.LoadPyPI
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: pypi
-
diff --git a/worker/instances/loaders-svn.staging.values.yaml b/worker/instances/loaders-svn.staging.values.yaml
deleted file mode 100644
--- a/worker/instances/loaders-svn.staging.values.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: scheduler0.internal.staging.swh.network
- queue_threshold: 10 # spawn worker per increment of `value` messages
- queues:
- - swh.loader.svn.tasks.LoadSvnRepository
- - swh.loader.svn.tasks.MountAndLoadSvnRepository
- - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository
-
-storage:
- host: storage1.internal.staging.swh.network
-
-loader:
- name: loaders
- type: svn
diff --git a/worker/templates/services.yaml b/worker/templates/services.yaml
deleted file mode 100644
--- a/worker/templates/services.yaml
+++ /dev/null
@@ -1,20 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
- name: storage-{{ .Values.loader.name }}-{{ .Values.loader.type }}
- namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }}
-spec:
- type: ExternalName
- externalName: {{ .Values.storage.host }}
-
----
-apiVersion: v1
-kind: Service
-metadata:
- name: amqp-{{ .Values.loader.name }}-{{ .Values.loader.type }}
- namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }}
-spec:
- type: ExternalName
- externalName: {{ .Values.amqp.host }}
-
diff --git a/worker/values.yaml b/worker/values.yaml
deleted file mode 100644
--- a/worker/values.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Default values for worker.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-amqp:
- host: amqp
-
-storage:
- host: swh-storage
-
-loader:
- name: loaders
- type: <override>
-
-swh:
- loader:
- image: softwareheritage/loaders
- version: 2022-05-17
- replicas:
- min: 1
- max: 5
-
-sentry:
- environment: staging

File Metadata

Mime Type
text/plain
Expires
Dec 20 2024, 4:57 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220991

Event Timeline

About · Developer information · Legal