diff --git a/swh/templates/indexers/deployment.yaml b/swh/templates/indexers/deployment.yaml index 1d2df8a..eae86df 100644 --- a/swh/templates/indexers/deployment.yaml +++ b/swh/templates/indexers/deployment.yaml @@ -1,126 +1,128 @@ {{ if .Values.indexers.enabled -}} {{- $configurationChecksum := include (print $.Template.BasePath "/indexers/configmap.yaml") . -}} {{- range $indexer_type, $deployment_config := .Values.indexers.deployments -}} {{- $indexer_name := ( print "indexer-" $indexer_type ) -}} --- apiVersion: apps/v1 kind: Deployment metadata: name: {{ $indexer_name }} namespace: {{ $.Values.namespace }} labels: app: {{ $indexer_name }} spec: revisionHistoryLimit: 2 selector: matchLabels: app: {{ $indexer_name }} strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: {{ $indexer_name }} annotations: # Force a rollout upgrade if the configuration changes checksum/config: {{ $configurationChecksum | sha256sum }} spec: {{- if $.Values.indexers.affinity }} affinity: {{ toYaml $.Values.indexers.affinity | nindent 8 }} {{- end }} terminationGracePeriodSeconds: 3600 initContainers: - name: prepare-configuration image: debian:bullseye imagePullPolicy: Always env: - name: JOURNAL_PASSWORD valueFrom: secretKeyRef: name: common-secrets key: journal-password optional: true command: - /entrypoint.sh volumeMounts: - name: configuration-template mountPath: /entrypoint.sh subPath: "init-container-entrypoint.sh" readOnly: true - name: configuration mountPath: /etc/swh - name: configuration-template mountPath: /etc/swh/configuration-template containers: - name: indexers image: {{ $.Values.swh_indexer_image }}:{{ $.Values.swh_indexer_image_version }} imagePullPolicy: Always command: - /opt/swh/entrypoint.sh resources: requests: memory: {{ get $deployment_config "requestedMemory" | default "512Mi" }} cpu: {{ get $deployment_config "requestedCpu" | default "500m" }} limits: memory: "4000Mi" cpu: "1200m" lifecycle: preStop: exec: command: ["/pre-stop.sh"] env: - name: STATSD_HOST value: {{ $.Values.statsdExternalHost | default "prometheus-statsd-exporter" }} - name: STATSD_PORT value: {{ $.Values.statsdPort | default "9125" | quote }} - name: MAX_TASKS_PER_CHILD value: {{ get $deployment_config "maxTasksPerChild" | default 10 | quote }} - name: LOGLEVEL value: {{ get $deployment_config "logLevel" | default "INFO" | quote }} - name: SWH_CONFIG_FILENAME value: /etc/swh/config.yml + - name: SWH_INDEXER_TYPE + value: {{ get $deployment_config "indexer_type" }} - name: SWH_SENTRY_ENVIRONMENT value: {{ $.Values.sentry.environment }} - name: SWH_MAIN_PACKAGE value: {{ get $deployment_config "sentrySwhPackage" }} - name: SWH_SENTRY_DSN valueFrom: secretKeyRef: name: common-secrets key: indexer-sentry-dsn # 'name' secret must exist & include key "host" optional: false volumeMounts: - name: indexer-utils mountPath: /pre-stop.sh subPath: "pre-stop.sh" - name: configuration mountPath: /etc/swh - name: localstorage mountPath: /tmp volumes: - name: localstorage emptyDir: {} - name: configuration emptyDir: {} - name: configuration-template configMap: name: {{ $indexer_name }}-template defaultMode: 0777 items: - key: "config.yml.template" path: "config.yml.template" - key: "init-container-entrypoint.sh" path: "init-container-entrypoint.sh" - name: indexer-utils configMap: name: indexer-utils defaultMode: 0777 items: - key: "pre-stop-idempotent.sh" path: "pre-stop.sh" {{ end }} {{- end -}} diff --git a/swh/values/staging.yaml b/swh/values/staging.yaml index 10ef0ca..aa27d93 100644 --- a/swh/values/staging.yaml +++ b/swh/values/staging.yaml @@ -1,425 +1,429 @@ sentry: environment: staging loaders: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 amqp: host: scheduler0.internal.staging.swh.network port: 5672 deployments: addforgenow: image: swh_loader_git_image requestedMemory: 256Mi requestedCpu: 200m queues: - add_forge_now:swh.loader.git.tasks.UpdateGitRepository ackLate: true autoScaling: queueThreshold: 1 maxReplicaCount: 2 sentrySwhPackage: swh.loader.git archive: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.archive.tasks.LoadArchive autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core bzr: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.bzr.tasks.LoadBazaar autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.bzr cran: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.cran.tasks.LoadCRAN autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core cvs: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.cvs.tasks.LoadCvsRepository autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.cvs debian: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.debian.tasks.LoadDebian autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core deposit: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.deposit.tasks.LoadDeposit autoScaling: queueThreshold: 1 maxReplicaCount: 2 sentrySwhPackage: swh.loader.core extraConfig: deposit: url: "https://deposit-rp.internal.staging.swh.network/1/private" auth: username: "${DEPOSIT_USERNAME}" password: "${DEPOSIT_PASSWORD}" default_filename: archive.tar git: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.git.tasks.UpdateGitRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository autoScaling: maxReplicaCount: 2 sentrySwhPackage: swh.loader.git golang: image: swh_loader_package_image requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.package.golang.tasks.LoadGolang autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core highpriority: requestedMemory: 256Mi requestedCpu: 200m queues: - save_code_now:swh.loader.bzr.tasks.LoadBazaar - save_code_now:swh.loader.git.tasks.UpdateGitRepository - save_code_now:swh.loader.git.tasks.LoadDiskGitRepository - save_code_now:swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - save_code_now:swh.loader.mercurial.tasks.LoadArchiveMercurial - save_code_now:swh.loader.mercurial.tasks.LoadMercurial - save_code_now:swh.loader.svn.tasks.LoadSvnRepository - save_code_now:swh.loader.svn.tasks.MountAndLoadSvnRepository - save_code_now:swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - save_code_now:swh.loader.package.archive.tasks.LoadArchive ackLate: true autoScaling: queueThreshold: 1 maxReplicaCount: 2 sentrySwhPackage: swh.loader.highpriority maven: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.maven.tasks.LoadMaven autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core mercurial: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.mercurial.tasks.LoadArchiveMercurial - swh.loader.mercurial.tasks.LoadMercurial autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.mercurial nixguix: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.nixguix.tasks.LoadNixguix autoScaling: queueThreshold: 1 maxReplicaCount: 1 sentrySwhPackage: swh.loader.core extraConfig: unsupported_file_extensions: - patch - iso - whl - gem - pom - msi - pod - png - rock - ttf - jar - c - el - rpm - diff npm: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.npm.tasks.LoadNpm autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core opam: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.opam.tasks.LoadOpam autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core extraConfig: # FIXME: Find a way to avoid this initialization step in loader task initialize_opam_root: true pypi: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pypi.tasks.LoadPyPI autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core pubdev: requestedMemory: 256Mi requestedCpu: 200m image: swh_loader_package_image queues: - swh.loader.package.pubdev.tasks.LoadPubDev autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.core svn: requestedMemory: 256Mi requestedCpu: 200m queues: - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository autoScaling: maxReplicaCount: 1 sentrySwhPackage: swh.loader.svn listers: enabled: true sentrySwhPackage: swh.lister storage: host: storage1.internal.staging.swh.network port: 5002 scheduler: host: scheduler0.internal.staging.swh.network port: 5008 amqp: host: scheduler0.internal.staging.swh.network deployments: all: maxTasksperchild: 3 queues: - swh.lister.cgit.tasks.CGitListerTask - swh.lister.cran.tasks.CRANListerTask - swh.lister.debian.tasks.DebianListerTask - swh.lister.gitea.tasks.IncrementalGiteaLister - swh.lister.gitea.tasks.RangeGiteaLister - swh.lister.gitea.tasks.FullGiteaRelister - swh.lister.gitlab.tasks.IncrementalGitLabLister - swh.lister.gitlab.tasks.RangeGitLabLister - swh.lister.gitlab.tasks.FullGitLabRelister - swh.lister.npm.tasks.NpmListerTask - swh.lister.phabricator.tasks.FullPhabricatorLister - swh.lister.pypi.tasks.PyPIListerTask autoScaling: maxReplicaCount: 2 bower: queues: - swh.lister.bower.tasks.BowerListerTask autoScaling: maxReplicaCount: 1 bitbucket: queues: - swh.lister.bitbucket.tasks.IncrementalBitBucketLister - swh.lister.bitbucket.tasks.FullBitBucketRelister autoScaling: maxReplicaCount: 1 gnu-full: queues: - swh.lister.gnu.tasks.GNUListerTask autoScaling: maxReplicaCount: 1 gogs-full: queues: - swh.lister.gogs.tasks.FullGogsRelister autoScaling: maxReplicaCount: 1 golang: queues: - swh.lister.golang.tasks.FullGolangLister - swh.lister.golang.tasks.IncrementalGolangLister autoScaling: maxReplicaCount: 1 launchpad: queues: - swh.lister.launchpad.tasks.FullLaunchpadLister - swh.lister.launchpad.tasks.IncrementalLaunchpadLister autoScaling: maxReplicaCount: 1 maven: queues: - swh.lister.maven.tasks.FullMavenLister - swh.lister.maven.tasks.IncrementalMavenLister autoScaling: maxReplicaCount: 1 opam: queues: - swh.lister.opam.tasks.OpamListerTask autoScaling: maxReplicaCount: 1 pubdev: queues: - swh.lister.pubdev.tasks.PubDevListerTask autoScaling: maxReplicaCount: 1 sourceforge: queues: - swh.lister.sourceforge.tasks.FullSourceForgeLister - swh.lister.sourceforge.tasks.IncrementalSourceForgeLister autoScaling: maxReplicaCount: 1 cookers: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 vault: host: vault.internal.staging.swh.network port: 5005 amqp: host: scheduler0.internal.staging.swh.network deployments: simple: queues: - swh.vault.cooking_tasks.SWHCookingTask autoScaling: maxReplicaCount: 2 batch: queues: - swh.vault.cooking_tasks.SWHBatchCookingTask autoScaling: maxReplicaCount: 2 checker_deposit: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 deposit: host: deposit-rp.internal.staging.swh.network amqp: host: scheduler0.internal.staging.swh.network autoScaling: maxReplicaCount: 2 indexers: enabled: true storage: host: storage1.internal.staging.swh.network port: 5002 scheduler: host: scheduler0.internal.staging.swh.network port: 5008 indexer_storage: host: storage1.internal.staging.swh.network port: 5007 objstorage: host: storage1.internal.staging.swh.network port: 5003 journalBrokers: hosts: - journal1.internal.staging.swh.network deployments: content-mimetype: + indexer_type: content_mimetype consumerGroup: swh.indexer.journal_client.content_mimetype prefix: swh.journal.objects extraConfig: tools: name: file # FIXME: Push this version to be autodetected by indexer version: 2:0.4.15-2 configuration: type: library debian-package: python3-magic write_batch_size: 1000 autoScaling: maxReplicaCount: 2 # FIXME: Dependency on nomossa need to be figured out prior to deploying it # content-fossology-license: + # indexer_type: content_fossology_license # consumerGroup: swh.indexer.journal_client.content_fossology_license # prefix: swh.journal.objects # extraConfig: # workdir: /tmp/swh/indexer.fossology.license/ # tools: # name: 'nomos' # version: '3.1-1~bpo9~swh+1' # configuration: # command_line: "nomossa " # write_batch_size: 1000 # autoScaling: # maxReplicaCount: 2 - # origin-extrinsic: - # consumerGroup: swh.indexer.journal_client.extrinsic_metadata - # prefix: swh.journal.objects - # extraConfig: - # tools: - # name: swh-metadata-detector - # version: 0.0.2 - # configuration: {} - # autoScaling: - # maxReplicaCount: 2 - # origin-intrinsic: - # consumerGroup: swh.indexer.journal_client.origin_intrinsic_metadata - # prefix: swh.journal.objects - # batch_size: 200 - # extraConfig: - # tools: - # name: swh-metadata-detector - # version: 0.0.2 - # configuration: {} - # autoScaling: - # maxReplicaCount: 2 + origin-extrinsic: + indexer_type: extrinsic_metadata + consumerGroup: swh.indexer.journal_client.extrinsic_metadata + prefix: swh.journal.objects + extraConfig: + tools: + name: swh-metadata-detector + version: 0.0.2 + configuration: {} + autoScaling: + maxReplicaCount: 2 + origin-intrinsic: + indexer_type: origin_intrinsic_metadata + consumerGroup: swh.indexer.journal_client.origin_intrinsic_metadata + prefix: swh.journal.objects + batch_size: 200 + extraConfig: + tools: + name: swh-metadata-detector + version: 0.0.2 + configuration: {} + autoScaling: + maxReplicaCount: 2 graphql: enabled: true sentry_enabled: true backends: storage: host: webapp.internal.staging.swh.network port: 5002 search: host: search0.internal.staging.swh.network port: 5010 replicas: 1 gunicorn: threads: 4 workers: 2 timeout: 3600 ingress: enabled: true httpPath: / host: graphql.staging.swh.network logLevel: DEBUG diff --git a/values-swh-application-versions.yaml b/values-swh-application-versions.yaml index a580b8b..b6d69b4 100644 --- a/values-swh-application-versions.yaml +++ b/values-swh-application-versions.yaml @@ -1,31 +1,31 @@ # This file references the last version of all the softwareheritage images # It's used to manage the automatic update of the environments swh_checker_deposit_image: softwareheritage/deposit_checkers swh_checker_deposit_image_version: "20220927.2" swh_cookers_image: softwareheritage/vault_cookers swh_cookers_image_version: "20220926.2" swh_graphql_image: softwareheritage/graphql swh_graphql_image_version: "20220913.2" swh_indexer_image: softwareheritage/indexer -swh_indexer_image_version: "20220927.1" +swh_indexer_image_version: "20220927.2" swh_listers_image: softwareheritage/lister swh_listers_image_version: "20220921.1" swh_loaders_image: softwareheritage/loaders swh_loaders_image_version: "20220826.1" swh_loader_bzr_image: softwareheritage/loader_bzr swh_loader_bzr_image_version: "20220926.1" swh_loader_cvs_image: softwareheritage/loader_cvs swh_loader_cvs_image_version: "20220926.1" swh_loader_git_image: softwareheritage/loader_git swh_loader_git_image_version: "20220926.1" swh_loader_highpriority_image: softwareheritage/loader_highpriority swh_loader_highpriority_image_version: "20220926.1" swh_loader_mercurial_image: softwareheritage/loader_mercurial swh_loader_mercurial_image_version: "20220926.1" swh_loader_package_image: softwareheritage/loader_package swh_loader_package_image_version: "20220926.1" swh_loader_svn_image: softwareheritage/loader_svn swh_loader_svn_image_version: "20220926.1" swh_storage_replayer_image: softwareheritage/storage-replayer swh_storage_replayer_image_version: "20220927.1"