diff --git a/sysadmin/T3592-elastic-workers/worker/.gitignore b/sysadmin/T3592-elastic-workers/worker/.gitignore index b687e2e..136a928 100644 --- a/sysadmin/T3592-elastic-workers/worker/.gitignore +++ b/sysadmin/T3592-elastic-workers/worker/.gitignore @@ -1,11 +1,2 @@ -loader-bzr.staging.values.yaml -loader-cvs.staging.values.yaml -loader-git.staging.values.yaml -loader-maven.staging.values.yaml -loader-pypi.staging.values.yaml -loader-svn.staging.values.yaml -loader-npm.staging.values.yaml +*.secret.yaml -loader-git-metadata-fetcher-credentials.yaml -amqp-access-credentials.yaml -loaders-git-sentry-secrets.yaml diff --git a/sysadmin/T3592-elastic-workers/worker/README.md b/sysadmin/T3592-elastic-workers/worker/README.md index 98aed47..dab1fca 100644 --- a/sysadmin/T3592-elastic-workers/worker/README.md +++ b/sysadmin/T3592-elastic-workers/worker/README.md @@ -1,138 +1,139 @@ # Goal - autoscaling workers depending on repositories to load and allocated resources. # keda This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): ``` $ helm repo add kedacore https://kedacore.github.io/charts $ helm repo update swhworker@poc-rancher:~$ kubectl create namespace keda namespace/keda created swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda NAME: keda LAST DEPLOYED: Fri Oct 8 09:48:40 2021 NAMESPACE: keda STATUS: deployed REVISION: 1 TEST SUITE: None ``` source: https://keda.sh/docs/2.4/deploy/ # helm We use helm to ease the cluster application management. # Install Install the worker declaration from this directory in the cluster ``` $ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml $ TYPE=git; REL=workers-$TYPE; \ helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker $ TYPE=pypi; REL=workers-$TYPE; \ helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` Where: ``` $ cat ../loader-git.staging.values.yaml # Default values for worker. # This is a YAML-formatted file. # Declare variables to be passed into your templates. amqp: host: scheduler0.internal.staging.swh.network queue_threshold: 10 # spawn worker per increment of `value` messages queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository storage: host: storage1.internal.staging.swh.network loader: name: loaders type: git ``` # List List currently deployed applications: ``` $ helm list helm list NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION workers-bzr default 1 2022-04-29 12:59:32.111950055 +0200 CEST deployed worker-0.1.0 1.16.0 workers-git default 4 2022-04-29 12:50:12.322826487 +0200 CEST deployed worker-0.1.0 1.16.0 workers-pypi default 1 2022-04-29 12:51:22.506259018 +0200 CEST deployed worker-0.1.0 1.16.0 ``` # Upgrade When adapting the worker definition, you can apply the changes by upgrading the deployed application: ``` $ TYPE=git; REL=workers-$TYPE; \ helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker ``` # Secrets The current work requires credentials (installed as secret within the cluster): - metadata fetcher credentials `metadata-fetcher-credentials` - amqp credentials `` More details describing the secrets: ``` $ kubectl describe secret metadata-fetcher-credentials ``` Installed through: ``` $ TYPE=git # Replace mentions below in the yaml files $ kubectl -f $SECRET_FILE apply # for secret file in { -# loader-$TYPE-metadata-fetcher-credentials.yaml -# loader-$TYPE-sentry-secrets.yaml -# amqp-access-credentials.yaml +# instances/loader-$TYPE-metadata-fetcher-credentials.secret.yaml +# ./loader-$TYPE-sentry.secret.yaml +# ./amqp-access-credentials.secret.yaml +# ... # } -$ cat loader-$TYPE-metadata-fetcher-credentials.yaml +$ cat instances/loader-git-metadata-fetcher-credentials.secret.yaml apiVersion: v1 kind: Secret metadata: name: metadata-fetcher-credentials namespace: ns-loaders-git type: Opaque stringData: data: | metadata_fetcher_credentials: github: github: - username: password: - ... -$ cat amqp-access-credentials.yaml +$ cat ./amqp-access-credentials.secret.yaml apiVersion: v1 kind: Secret metadata: name: amqp-access-credentials namespace: ns-loaders-$TYPE type: Opaque stringData: host: amqp://:@scheduler0.internal.staging.swh.network:5672/%2f -$ cat loaders-$TYPE-sentry-secrets.yaml +$ cat ./loaders-$TYPE-sentry.secrets.yaml apiVersion: v1 kind: Secret metadata: name: loaders-$TYPE-sentry-secrets namespace: ns-loaders-$TYPE type: Opaque stringData: sentry-dsn: https://@sentry.softwareheritage.org/8 ``` diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-bzr.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-bzr.staging.values.yaml new file mode 100644 index 0000000..896c426 --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-bzr.staging.values.yaml @@ -0,0 +1,16 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 10 # spawn worker per increment of `value` messages + queues: + - swh.loader.bzr.tasks.LoadBazaar + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: bzr diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-cvs.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-cvs.staging.values.yaml new file mode 100644 index 0000000..fe3594b --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-cvs.staging.values.yaml @@ -0,0 +1,16 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 10 # spawn worker per increment of `value` messages + queues: + - swh.loader.cvs.tasks.LoadCvsRepository + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: cvs diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-git.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-git.staging.values.yaml new file mode 100644 index 0000000..a0f8187 --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-git.staging.values.yaml @@ -0,0 +1,27 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 5 # spawn worker per increment of `value` messages + queues: + - swh.loader.git.tasks.UpdateGitRepository + - swh.loader.git.tasks.LoadDiskGitRepository + - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: git + +swh: + loader: + replicas: + min: 1 + max: 10 + +sentry: + swhpackage: swh.loader.git diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-maven.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-maven.staging.values.yaml new file mode 100644 index 0000000..c7a3c67 --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-maven.staging.values.yaml @@ -0,0 +1,16 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 10 # spawn worker per increment of `value` messages + queues: + - swh.loader.package.maven.tasks.LoadMaven + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: maven diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-npm.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-npm.staging.values.yaml new file mode 100644 index 0000000..8143f51 --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-npm.staging.values.yaml @@ -0,0 +1,16 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 10 # spawn worker per increment of `value` messages + queues: + - swh.loader.package.npm.tasks.LoadNpm + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: npm diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-pypi.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-pypi.staging.values.yaml new file mode 100644 index 0000000..9b040b3 --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-pypi.staging.values.yaml @@ -0,0 +1,17 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 10 # spawn worker per increment of `value` messages + queues: + - swh.loader.package.pypi.tasks.LoadPyPI + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: pypi + diff --git a/sysadmin/T3592-elastic-workers/worker/instances/loaders-svn.staging.values.yaml b/sysadmin/T3592-elastic-workers/worker/instances/loaders-svn.staging.values.yaml new file mode 100644 index 0000000..6bd88a4 --- /dev/null +++ b/sysadmin/T3592-elastic-workers/worker/instances/loaders-svn.staging.values.yaml @@ -0,0 +1,18 @@ +# Default values for worker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +amqp: + host: scheduler0.internal.staging.swh.network + queue_threshold: 10 # spawn worker per increment of `value` messages + queues: + - swh.loader.svn.tasks.LoadSvnRepository + - swh.loader.svn.tasks.MountAndLoadSvnRepository + - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository + +storage: + host: storage1.internal.staging.swh.network + +loader: + name: loaders + type: svn