diff --git a/kubernetes/45-listers.yml b/kubernetes/45-listers.yml new file mode 100644 index 0000000..3380257 --- /dev/null +++ b/kubernetes/45-listers.yml @@ -0,0 +1,137 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: listers +data: + config.yml: | + scheduler: + cls: remote + url: http://scheduler:5008/ + + celery: + task_broker: amqp://guest:guest@amqp// + task_queues: + - swh.lister.bitbucket.tasks.FullBitBucketRelister + - swh.lister.bitbucket.tasks.IncrementalBitBucketLister + - swh.lister.bitbucket.tasks.RangeBitBucketLister + - swh.lister.cgit.tasks.CGitListerTask + - swh.lister.cran.tasks.CRANListerTask + - swh.lister.debian.tasks.DebianListerTask + - swh.lister.gitea.tasks.FullGiteaRelister + - swh.lister.gitea.tasks.IncrementalGiteaLister + - swh.lister.gitea.tasks.RangeGiteaLister + - swh.lister.github.tasks.FullGitHubRelister + - swh.lister.github.tasks.IncrementalGitHubLister + - swh.lister.github.tasks.RangeGitHubLister + - swh.lister.gitlab.tasks.FullGitLabRelister + - swh.lister.gitlab.tasks.IncrementalGitLabLister + - swh.lister.gitlab.tasks.RangeGitLabLister + - swh.lister.gnu.tasks.GNUListerTask + - swh.lister.npm.tasks.NpmIncrementalListerTask + - swh.lister.npm.tasks.NpmListerTask + - swh.lister.launchpad.tasks.IncrementalLaunchpadLister + - swh.lister.launchpad.tasks.FullLaunchpadLister + - swh.lister.packagist.tasks.PackagistListerTask + - swh.lister.phabricator.tasks.FullPhabricatorLister + - swh.lister.phabricator.tasks.IncrementalPhabricatorLister + - swh.lister.pypi.tasks.PyPIListerTask + entrypoint-init.sh: | + #!/bin/bash + + set -e + + # echo Waiting for RabbitMQ to start + wait-for-it amqp:5672 -s --timeout=0 + + # echo Register task types in scheduler database + wait-for-it scheduler:5008 -s --timeout=0 + + swh scheduler --url http://scheduler:5008 task-type register + entrypoint.sh: | + #!/bin/bash + + set -e + + echo Starting the swh listers + exec python -m celery \ + --app=swh.scheduler.celery_backend.config.app \ + worker \ + --pool=prefork --events \ + --concurrency=${CONCURRENCY} \ + --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ + -Ofair --loglevel=${LOGLEVEL} \ + --hostname "${HOSTNAME}" + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: listers + labels: + app: listers +spec: + replicas: 1 + selector: + matchLabels: + app: listers + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + template: + metadata: + labels: + app: listers + spec: + initContainers: + - name: listers-init + image: swh/listers:latest + imagePullPolicy: Always + command: + - /entrypoint.sh + volumeMounts: + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint-init.sh + readOnly: true + containers: + - name: listers + image: swh/listers:latest + imagePullPolicy: Always + command: + - /entrypoint.sh + resources: + requests: + memory: "128Mi" + cpu: "200m" + limits: + memory: "256Mi" + cpu: "256m" + env: + - name: CONCURRENCY + value: "1" + - name: MAX_TASKS_PER_CHILD + value: "5" + - name: LOGLEVEL + value: "INFO" + - name: SWH_CONFIG_FILENAME + value: /etc/softwareheritage/config.yml + volumeMounts: + - name: config + mountPath: /etc/softwareheritage/config.yml + subPath: config.yml + readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + readOnly: true + volumes: + - name: config + configMap: + name: listers + defaultMode: 0777 diff --git a/skaffold.yaml b/skaffold.yaml index 5ac1f03..6d148e7 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1,66 +1,73 @@ apiVersion: skaffold/v2beta13 kind: Config metadata: name: swh-environment build: artifacts: - image: swh/stack-base docker: dockerfile: Dockerfile - image: swh/objstorage docker: dockerfile: Dockerfile.objstorage requires: - image: swh/stack-base alias: BASE - image: swh/storage docker: dockerfile: Dockerfile.storage requires: - image: swh/stack-base alias: BASE - image: swh/scheduler docker: dockerfile: Dockerfile.scheduler requires: - image: swh/stack-base alias: BASE - image: swh/webapp docker: dockerfile: Dockerfile.webapp requires: - image: swh/stack-base alias: BASE - image: swh/loaders docker: dockerfile: Dockerfile.loaders requires: - image: swh/stack-base alias: BASE + - image: swh/listers + docker: + dockerfile: Dockerfile.listers + requires: + - image: swh/stack-base + alias: BASE - image: swh/grafana docker: dockerfile: Dockerfile.grafana - image: swh/search docker: dockerfile: Dockerfile.search requires: - image: swh/stack-base alias: BASE deploy: kubectl: manifests: - kubernetes/01-journal.yml - kubernetes/02-monitoring.yml - kubernetes/05-storage-db.yml - kubernetes/10-objstorage.yml - kubernetes/11-storage.yml - kubernetes/15-scheduler-db.yml - kubernetes/16-rabbitmq.yml - kubernetes/20-scheduler.yml - kubernetes/21-scheduler-runner.yml - kubernetes/29-web-db.yml - kubernetes/30-webapp.yml - kubernetes/40-loaders.yml + - kubernetes/45-listers.yml - kubernetes/50-elasticsearch.yml - kubernetes/55-search.yml - kubernetes/22-scheduler-listener.yml