diff --git a/swh/templates/loaders/configmap.yaml b/swh/templates/loaders/configmap.yaml index 8160d14..1c758f2 100644 --- a/swh/templates/loaders/configmap.yaml +++ b/swh/templates/loaders/configmap.yaml @@ -1,54 +1,60 @@ {{ if .Values.loaders.enabled -}} {{- range $loader_type, $deployment_config := .Values.loaders.deployments -}} {{- $loader_name := ( print "loader-" $loader_type ) -}} --- apiVersion: v1 kind: ConfigMap metadata: name: {{ $loader_name }}-template namespace: {{ $.Values.namespace }} data: config.yml.template: | storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 1000 content_bytes: 52428800 directory: 1000 directory_entries: 12000 revision: 1000 revision_parents: 2000 revision_bytes: 52428800 release: 1000 release_bytes: 52428800 extid: 1000 - cls: filter - cls: retry - cls: remote url: http://{{ $.Values.loaders.storage.host }}:{{ $.Values.loaders.storage.port }}/ + {{- if $deployment_config.extraConfig -}} + {{- range $option, $value := $deployment_config.extraConfig }} + {{ $option }}: {{ $value }} + {{- end }} + {{- end }} + celery: task_broker: ${host} task_queues: {{- range $queue := get $deployment_config "queues" }} - {{ $queue }} {{- end }} metadata_fetcher_credentials: init-container-entrypoint.sh: | #!/bin/bash set -e CONFIG_FILE=/etc/swh/config.yml # substitute environment variables when creating the default config.yml eval echo \""$( $CONFIG_FILE CREDS_LISTER_PATH=/etc/credentials/metadata-fetcher/credentials [ -f $CREDS_LISTER_PATH ] && \ sed 's/^/ /g' $CREDS_LISTER_PATH >> $CONFIG_FILE {{ end }} {{- end -}} diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml index 6432839..28332d0 100644 --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -1,258 +1,257 @@ sentry: environment: staging loaders: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 amqp: host: scheduler0.internal.staging.swh.network deployments: archive: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.archive.tasks.LoadArchive autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core addforgenow: image: swh_loader_git_image requestedMemory: 256Mi requestedCpu: 200m queues: - add_forge_now:swh.loader.git.tasks.UpdateGitRepository autoScaling: queueThreshold: 1 maxReplicaCount: 5 sentrySwhPackage: swh.loader.git bzr: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.bzr.tasks.LoadBazaar autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.bzr cran: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.cran.tasks.LoadCRAN autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core cvs: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.cvs.tasks.LoadCvsRepository autoScaling: queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.cvs debian: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.debian.tasks.LoadDebian autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core # deposit: # requestedMemory: 256Mi # requestedCpu: 200m # image: swh_loader_package_image # queues: # - swh.loader.package.cran.tasks.LoadCRAN # autoScaling: # queueThreshold: 10 # maxReplicaCount: 1 # sentrySwhPackage: swh.loader.core mercurial: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.mercurial.tasks.LoadArchiveMercurial - swh.loader.mercurial.tasks.LoadMercurial autoScaling: queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.mercurial - # nixguix: - # requestedMemory: 256Mi - # requestedCpu: 200m - # image: swh_loader_package_image - # queues: - # - swh.loader.package.nixguix.tasks.LoadNixguix - # autoScaling: - # queueThreshold: 10 - # maxReplicaCount: 1 - # sentrySwhPackage: swh.loader.core - # extra_config: - # unsupported_file_extensions: - # - patch - # - iso - # - whl - # - gem - # - pom - # - msi - # - pod - # - png - # - rock - # - ttf - # - jar - # - c - # - el - # - rpm - # - diff - + nixguix: + requestedMemory: 256Mi + requestedCpu: 200m + image: swh_loader_package_image + queues: + - swh.loader.package.nixguix.tasks.LoadNixguix + autoScaling: + queueThreshold: 10 + maxReplicaCount: 1 + sentrySwhPackage: swh.loader.core + extraConfig: + unsupported_file_extensions: + - patch + - iso + - whl + - gem + - pom + - msi + - pod + - png + - rock + - ttf + - jar + - c + - el + - rpm + - diff highpriority: requestedMemory: 256Mi requestedCpu: 200m queues: - save_code_now:swh.loader.bzr.tasks.LoadBazaar - save_code_now:swh.loader.git.tasks.UpdateGitRepository - save_code_now:swh.loader.git.tasks.LoadDiskGitRepository - save_code_now:swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial - save_code_now:swh.loader.mercurial.tasks.LoadMercurial - save_code_now:swh.loader.svn.tasks.LoadSvnRepository - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - save_code_now:swh.loader.package.archive.tasks.LoadArchive autoScaling: queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.highpriority git: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository autoScaling: queueThreshold: 5 # spawn worker per increment of `value` messages maxReplicaCount: 3 sentrySwhPackage: swh.loader.git maven: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.maven.tasks.LoadMaven autoScaling: queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.core npm: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.npm.tasks.LoadNpm autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core # opam: # requestedMemory: 256Mi # requestedCpu: 200m # image: swh_loader_package_image # queues: # - swh.loader.package.opam.tasks.LoadOpam # autoScaling: # queueThreshold: 10 # maxReplicaCount: 1 # sentrySwhPackage: swh.loader.core pypi: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pypi.tasks.LoadPyPI autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core pubdev: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pubdev.tasks.LoadPubDev autoScaling: queueThreshold: 10 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core svn: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository autoScaling: queueThreshold: 10 maxReplicaCount: 3 sentrySwhPackage: swh.loader.svn listers: enabled: true sentrySwhPackage: swh.lister storage: host: storage1.internal.staging.swh.network port: 5002 scheduler: host: scheduler0.internal.staging.swh.network port: 5008 amqp: host: scheduler0.internal.staging.swh.network deployments: gnu-full: queues: - swh.lister.gnu.tasks.GNUListerTask autoScaling: maxReplicaCount: 1 pubdev: queues: - swh.lister.pubdev.tasks.PubDevListerTask autoScaling: maxReplicaCount: 1 gogs-full: logLevel: DEBUG queues: - swh.lister.gogs.tasks.FullGogsRelister autoScaling: maxReplicaCount: 1 graphql: enabled: true sentry_enabled: true backends: storage: host: webapp.internal.staging.swh.network port: 5002 search: host: search0.internal.staging.swh.network port: 5010 replicas: 1 gunicorn: threads: 4 workers: 2 timeout: 3600 ingress: enabled: true httpPath: / host: graphql.staging.swh.network logLevel: DEBUG