diff --git a/charts_code/Jenkinsfile b/charts_code/Jenkinsfile deleted file mode 100644 index a88cefa..0000000 --- a/charts_code/Jenkinsfile +++ /dev/null @@ -1,29 +0,0 @@ -node { - def app - - stage('Clone repository') { - checkout scm - } - - stage('Update GIT') { - script { - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - withCredentials([usernamePassword(credentialsId: 'github', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) { - def encodedPassword = URLEncoder.encode("$GIT_PASSWORD",'UTF-8') - sh "git config user.email amadouth6@gmail.com" - sh "git config user.name amadouth6" - sh "git switch minikube-cluster" - sh "cat swh-graphql/myvalues.yaml" - sh "sed -i 's+version:.*+version: \"${DOCKERTAG}\"+g' swh-graphql/myvalues.yaml" - //sh "sed -i 's+amadouth/graphql.*+amadouth/graphql:${DOCKERTAG}+g' swh-graphql/myvalues.yaml" - sh "cat swh-graphql/myvalues.yaml" - sh "git add ./swh-graphql/myvalues.yaml" - sh "git commit -m 'Done by Jenkins Job changemanifest: ${env.BUILD_NUMBER}'" - sh "echo '============DERNIERE ETAPE================'" - //sh " git pull https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${GIT_USERNAME}/argocd.git" - sh "git push -f https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${GIT_USERNAME}/argocd.git minikube-cluster" - } - } - } -} -} diff --git a/charts_code/swh-graphql/Chart.yaml b/charts_code/swh-graphql/Chart.yaml deleted file mode 100644 index 8441941..0000000 --- a/charts_code/swh-graphql/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: swh-graphql -description: A Helm chart to deploy the graphql service - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/charts_code/swh-graphql/README.md b/charts_code/swh-graphql/README.md deleted file mode 100644 index 8d99978..0000000 --- a/charts_code/swh-graphql/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Goal - -- Deploy a standalone graphql instance - -This chart will be merged with the global swh chart when available - -# helm - -We use helm to ease the cluster application management. - -# Install - -## Prerequisites -- Helm >= 3.0 -- Access to a kubernetes cluster -- The url of a reachable swh storage - -## Chart installation - -Install the worker declaration from this directory in the cluster -``` -$ helm install -f my-values.yaml graphql -``` - -With `my-values.yaml` containing some overrides of the default -values matching your environment. - -Detail of the possible values is visible with: -``` -swh-charts/swh-graphql $ helm show values . -``` diff --git a/charts_code/swh-graphql/minikube-value.yml b/charts_code/swh-graphql/minikube-value.yml deleted file mode 100644 index 4dab334..0000000 --- a/charts_code/swh-graphql/minikube-value.yml +++ /dev/null @@ -1,5 +0,0 @@ -# The ingress host to listen for -ingressHost: graphql.minikube-cluster - -#Minikube namespace deployment -namespace: graphql-minikube \ No newline at end of file diff --git a/charts_code/swh-graphql/myvalues.yaml b/charts_code/swh-graphql/myvalues.yaml deleted file mode 100644 index d200e5b..0000000 --- a/charts_code/swh-graphql/myvalues.yaml +++ /dev/null @@ -1,11 +0,0 @@ -image: - # The name of the image to use - name: amadouth/graphqll - # The version of the image to use - version: "48" - -# The storage url to query -storageUrl: http://webapp.internal.staging.swh.network:5002/ - -#Replicas value number -replicas: 3 diff --git a/charts_code/swh-graphql/ranchervalues.yaml b/charts_code/swh-graphql/ranchervalues.yaml deleted file mode 100644 index c9c3ada..0000000 --- a/charts_code/swh-graphql/ranchervalues.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# The ingress host to listen for -ingressHost: graphql.ranchercluster - -#namespace -namespace: graphql-rancher \ No newline at end of file diff --git a/charts_code/swh-graphql/templates/configmap.yaml b/charts_code/swh-graphql/templates/configmap.yaml deleted file mode 100644 index 81e6b99..0000000 --- a/charts_code/swh-graphql/templates/configmap.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: graphql - namespace: {{ .Values.namespace }} -data: - config.yml: | - storage: - cls: remote - url: {{ .Values.storageUrl }} - - debug: yes - - server-type: wsgi diff --git a/charts_code/swh-graphql/templates/deployment.yaml b/charts_code/swh-graphql/templates/deployment.yaml deleted file mode 100644 index 5ccae45..0000000 --- a/charts_code/swh-graphql/templates/deployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: graphql - namespace: {{ .Values.namespace }} - labels: - app: graphql -spec: - replicas: {{ .Values.replicas }} - selector: - matchLabels: - app: graphql - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - template: - metadata: - labels: - app: graphql - annotations: - # Force a rollout upgrade if the configuration changes - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - spec: - containers: - - name: graphql - image: {{ .Values.image.name }}:{{ .Values.image.version }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 5013 - env: - - name: PORT - value: "5013" - - name: THREADS - value: {{ .Values.gunicornThreads | quote }} - - name: WORKERS - value: {{ .Values.gunicornWorkers | quote }} - - name: LOG_LEVEL - value: {{ .Values.logLevel | quote }} - - name: TIMEOUT - value: {{ .Values.gunicornTimeout | quote }} - volumeMounts: - - name: config - mountPath: /etc/swh/config.yml - subPath: config.yml - readOnly: true - volumes: - - name: config - configMap: - name: graphql - defaultMode: 0444 - diff --git a/charts_code/swh-graphql/templates/ingress.yaml b/charts_code/swh-graphql/templates/ingress.yaml deleted file mode 100644 index 83e8a10..0000000 --- a/charts_code/swh-graphql/templates/ingress.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{ if .Values.enableIngress }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: graphql - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / -spec: - {{- if .Values.ingressClassName }} - ingressClassName: {{ .Values.ingressClassName }} - {{- end }} - rules: - - host: {{ .Values.ingressHost }} - http: - paths: - - path: {{ .Values.ingressHttpPath }} - pathType: Prefix - backend: - service: - name: graphql - port: - number: 5013 -{{ end }} diff --git a/charts_code/swh-graphql/templates/service.yaml b/charts_code/swh-graphql/templates/service.yaml deleted file mode 100644 index b983d61..0000000 --- a/charts_code/swh-graphql/templates/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: graphql - namespace: {{ .Values.namespace }} -spec: - type: ClusterIP - selector: - app: graphql - ports: - - port: 5013 - targetPort: 5013 diff --git a/charts_code/swh-graphql/values.yaml b/charts_code/swh-graphql/values.yaml deleted file mode 100644 index e5c5520..0000000 --- a/charts_code/swh-graphql/values.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Default values for graphql. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -namespace: default - -# Number of graphql pods to deploy -replicas: 2 - -image: - # The name of the image to use - name: softwareheritage/graphql - # The version of the image to use - version: latest - pullPolicy: Always - -# The storage url to query -storageUrl: http://storage.svc.cluster.local:5002 - -# Deploy or not an ingress controller -enableIngress: true -# Specify the ingress class to use if several are available in the cluster -ingressClassName: {} - -# The ingress host to listen for -ingressHost: graphql.localdomain - -# Configure the path of the ingress -ingressHttpPath: / - -# The graphql log level. Possible values TRACE, DEBUG, INFO, ERROR -logLevel: INFO - -gunicornThreads: 4 -gunicornWorkers: 2 -gunicornTimeout: 3600 diff --git a/charts_code/worker/.gitignore b/charts_code/worker/.gitignore deleted file mode 100644 index 136a928..0000000 --- a/charts_code/worker/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.secret.yaml - diff --git a/charts_code/worker/.helmignore b/charts_code/worker/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/charts_code/worker/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts_code/worker/Chart.yaml b/charts_code/worker/Chart.yaml deleted file mode 100644 index dc9e2eb..0000000 --- a/charts_code/worker/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: worker -description: A Helm chart to deploy dynamic workers on Kubernetes cluster - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/charts_code/worker/README.md b/charts_code/worker/README.md deleted file mode 100644 index d602318..0000000 --- a/charts_code/worker/README.md +++ /dev/null @@ -1,138 +0,0 @@ -# Goal - -- autoscaling workers depending on repositories to load and allocated resources. - -# keda - -This uses KEDA - K(ubernetes) E(vents)-D(riven) A(utoscaling): -``` -$ helm repo add kedacore https://kedacore.github.io/charts -$ helm repo update -swhworker@poc-rancher:~$ kubectl create namespace keda -namespace/keda created -swhworker@poc-rancher:~$ helm install keda kedacore/keda --namespace keda -NAME: keda -LAST DEPLOYED: Fri Oct 8 09:48:40 2021 -NAMESPACE: keda -STATUS: deployed -REVISION: 1 -TEST SUITE: None -``` -source: https://keda.sh/docs/2.4/deploy/ - -# helm - -We use helm to ease the cluster application management. - -# Install - -Install the worker declaration from this directory in the cluster -``` -$ export KUBECONFIG=export KUBECONFIG=staging-workers.yaml -$ TYPE=git; REL=workers-$TYPE; \ - helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker -$ TYPE=pypi; REL=workers-$TYPE; \ - helm install -f ./loader-$TYPE.staging.values.yaml $REL ../worker -``` - -Where: -``` -$ cat ../loader-git.staging.values.yaml -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.git.tasks.UpdateGitRepository - - swh.loader.git.tasks.LoadDiskGitRepository - - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: git -``` - -# List - -List currently deployed applications: - -``` -$ helm list -helm list -NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION -workers-bzr default 1 2022-04-29 12:59:32.111950055 +0200 CEST deployed worker-0.1.0 1.16.0 -workers-git default 4 2022-04-29 12:50:12.322826487 +0200 CEST deployed worker-0.1.0 1.16.0 -workers-pypi default 1 2022-04-29 12:51:22.506259018 +0200 CEST deployed worker-0.1.0 1.16.0 -``` - -# Upgrade - -When adapting the worker definition, you can apply the changes by upgrading the -deployed application: - -``` -$ TYPE=git; REL=workers-$TYPE; \ - helm upgrade -f ./loader-$TYPE.staging.values.yaml $REL ../worker -``` - -# Secrets - -The current work requires credentials (installed as secret within the cluster): -- metadata fetcher credentials `metadata-fetcher-credentials` -- amqp credentials `` - -More details describing the secrets: -``` -$ kubectl describe secret metadata-fetcher-credentials -``` - -Installed through: - -``` -$ TYPE=git # Replace mentions below in the yaml files -$ kubectl -f $SECRET_FILE apply --namespaces ns-loader-$TYPE -# for secret file in { -# instances/loaders-$TYPE-metadata-fetcher-credentials.secret.yaml -# ./loader-$TYPE-sentry.secret.yaml -# ./amqp-access-credentials.secret.yaml -# ... -# } -$ cat instances/loaders-metadata-fetcher.secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: metadata-fetcher-credentials -type: Opaque -stringData: - data: | - metadata_fetcher_credentials: - github: - github: - - username: - password: - - ... -$ cat ./amqp-access-credentials.secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: amqp-access-credentials - namespace: ns-loaders-$TYPE -type: Opaque -stringData: - host: amqp://:@scheduler0.internal.staging.swh.network:5672/%2f -$ cat ./loaders-$TYPE-sentry.secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: loaders-$TYPE-sentry-secrets - namespace: ns-loaders-$TYPE -type: Opaque -stringData: - sentry-dsn: https://@sentry.softwareheritage.org/8 -``` diff --git a/charts_code/worker/instances/loaders-bzr.staging.values.yaml b/charts_code/worker/instances/loaders-bzr.staging.values.yaml deleted file mode 100644 index 896c426..0000000 --- a/charts_code/worker/instances/loaders-bzr.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.bzr.tasks.LoadBazaar - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: bzr diff --git a/charts_code/worker/instances/loaders-cvs.staging.values.yaml b/charts_code/worker/instances/loaders-cvs.staging.values.yaml deleted file mode 100644 index fe3594b..0000000 --- a/charts_code/worker/instances/loaders-cvs.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.cvs.tasks.LoadCvsRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: cvs diff --git a/charts_code/worker/instances/loaders-git.staging.values.yaml b/charts_code/worker/instances/loaders-git.staging.values.yaml deleted file mode 100644 index a0f8187..0000000 --- a/charts_code/worker/instances/loaders-git.staging.values.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 5 # spawn worker per increment of `value` messages - queues: - - swh.loader.git.tasks.UpdateGitRepository - - swh.loader.git.tasks.LoadDiskGitRepository - - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: git - -swh: - loader: - replicas: - min: 1 - max: 10 - -sentry: - swhpackage: swh.loader.git diff --git a/charts_code/worker/instances/loaders-maven.staging.values.yaml b/charts_code/worker/instances/loaders-maven.staging.values.yaml deleted file mode 100644 index c7a3c67..0000000 --- a/charts_code/worker/instances/loaders-maven.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.package.maven.tasks.LoadMaven - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: maven diff --git a/charts_code/worker/instances/loaders-npm.staging.values.yaml b/charts_code/worker/instances/loaders-npm.staging.values.yaml deleted file mode 100644 index 8143f51..0000000 --- a/charts_code/worker/instances/loaders-npm.staging.values.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.package.npm.tasks.LoadNpm - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: npm diff --git a/charts_code/worker/instances/loaders-pypi.staging.values.yaml b/charts_code/worker/instances/loaders-pypi.staging.values.yaml deleted file mode 100644 index 9b040b3..0000000 --- a/charts_code/worker/instances/loaders-pypi.staging.values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.package.pypi.tasks.LoadPyPI - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: pypi - diff --git a/charts_code/worker/instances/loaders-svn.staging.values.yaml b/charts_code/worker/instances/loaders-svn.staging.values.yaml deleted file mode 100644 index 6bd88a4..0000000 --- a/charts_code/worker/instances/loaders-svn.staging.values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: scheduler0.internal.staging.swh.network - queue_threshold: 10 # spawn worker per increment of `value` messages - queues: - - swh.loader.svn.tasks.LoadSvnRepository - - swh.loader.svn.tasks.MountAndLoadSvnRepository - - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - -storage: - host: storage1.internal.staging.swh.network - -loader: - name: loaders - type: svn diff --git a/charts_code/worker/templates/autoscale.yaml b/charts_code/worker/templates/autoscale.yaml deleted file mode 100644 index a2fe028..0000000 --- a/charts_code/worker/templates/autoscale.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - name: amqp-authentication - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - secretTargetRef: # Optional. - - parameter: host - name: amqp-access-credentials - key: host - ---- -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: loaders-{{ .Values.loader.name }}-{{ .Values.loader.type }}-operators - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - scaleTargetRef: - apiVersion: apps/v1 # Optional. Default: apps/v1 - kind: Deployment # Optional. Default: Deployment - # Mandatory. Must be in same namespace as ScaledObject - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} - # envSourceContainerName: {container-name} # Optional. Default: - # .spec.template.spec.containers[0] - pollingInterval: 30 # Optional. Default: 30 seconds - cooldownPeriod: 300 # Optional. Default: 300 seconds - idleReplicaCount: 0 # Optional. Must be less than - # minReplicaCount - minReplicaCount: {{ .Values.swh.loader.replicas.min }} # Optional. Default: 0 - maxReplicaCount: {{ .Values.swh.loader.replicas.max }} # Optional. Default: 100 - fallback: # Optional. Section to specify fallback - # options - failureThreshold: 3 # Mandatory if fallback section is - # included - replicas: 6 # Mandatory if fallback section is - # included - advanced: # Optional. Section to specify advanced - # options - restoreToOriginalReplicaCount: false # Optional. Default: false - horizontalPodAutoscalerConfig: # Optional. Section to specify HPA - # related options - behavior: # Optional. Use to modify HPA's scaling - # behavior - scaleDown: - stabilizationWindowSeconds: 60 # default 300 - policies: - - type: Percent - value: 2 - periodSeconds: 15 - triggers: - {{- range .Values.amqp.queues }} - - type: rabbitmq - authenticationRef: - name: amqp-authentication - metadata: - host: host - # Optional. If not specified, it must be done - # by using TriggerAuthentication. - protocol: auto # Optional. Specifies protocol to use, - # either amqp or http, or auto to - # autodetect based on the `host` value. - # Default value is auto. - mode: QueueLength # QueueLength or MessageRate - value: {{ $.Values.amqp.queue_threshold | quote }} # message backlog or publish/sec. - # target per instance - queueName: {{ . }} - vhostName: / # Optional. If not specified, use the vhost in the - # `host` connection string. Alternatively, you can - # use existing environment variables to read - # configuration from: See details in "Parameter - # list" section hostFromEnv: RABBITMQ_HOST% - {{- end }} diff --git a/charts_code/worker/templates/config-map.yaml b/charts_code/worker/templates/config-map.yaml deleted file mode 100644 index b0100f7..0000000 --- a/charts_code/worker/templates/config-map.yaml +++ /dev/null @@ -1,57 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -data: - config.yml: | - storage: - cls: pipeline - steps: - - cls: buffer - min_batch_size: - content: 1000 - content_bytes: 52428800 - directory: 1000 - directory_entries: 12000 - revision: 1000 - revision_parents: 2000 - revision_bytes: 52428800 - release: 1000 - release_bytes: 52428800 - extid: 1000 - - cls: filter - - cls: retry - - cls: remote - url: http://{{ .Values.storage.host }}:5002/ - - celery: - task_broker: ##amqp_host## - task_queues: - {{- range .Values.amqp.queues }} - - {{ . }} - {{- end }} - entrypoint.sh: | - #!/bin/bash - - set -e - - # Create the full config filename - cat /etc/softwareheritage/config.yml > $SWH_CONFIG_FILENAME - # contains required credentials for git loader (with metadata loader inside) - # ignored by the other loaders - cat /etc/credentials/metadata-fetcher/data >> $SWH_CONFIG_FILENAME - - # Install the rabbitmq host information - sed -i 's,##amqp_host##,'$RABBITMQ_HOST',g' $SWH_CONFIG_FILENAME - - echo Starting the swh Celery worker - exec python -m celery \ - --app=swh.scheduler.celery_backend.config.app \ - worker \ - --pool=prefork \ - --concurrency=${CONCURRENCY} \ - --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ - -Ofair --loglevel=${LOGLEVEL} \ - --hostname "${HOSTNAME}" diff --git a/charts_code/worker/templates/deployment.yaml b/charts_code/worker/templates/deployment.yaml deleted file mode 100644 index 3bf31e3..0000000 --- a/charts_code/worker/templates/deployment.yaml +++ /dev/null @@ -1,91 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} - labels: - app: {{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - replicas: {{ .Values.swh.loader.replicas.min }} - selector: - matchLabels: - app: {{ .Values.loader.name }}-{{ .Values.loader.type }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - template: - metadata: - labels: - app: {{ .Values.loader.name }}-{{ .Values.loader.type }} - spec: - containers: - - name: loaders - image: {{ .Values.swh.loader.image }}:{{ .Values.swh.loader.version }} - imagePullPolicy: Always - command: - - /entrypoint.sh - resources: - requests: - memory: "256Mi" - cpu: "200m" - limits: - memory: "4000Mi" - cpu: "1200m" - lifecycle: - preStop: - exec: - command: ["kill", "1"] - env: - - name: CONCURRENCY - value: "1" - - name: MAX_TASKS_PER_CHILD - value: "5" - - name: LOGLEVEL - value: "INFO" - - name: SWH_CONFIG_FILENAME - # FIXME: built by entrypoint.sh, determine how to properly declare this - value: /tmp/config.yml - - name: SWH_SENTRY_ENVIRONMENT - value: {{ .Values.sentry.environment }} - - name: SWH_MAIN_PACKAGE - value: {{ .Values.sentry.swhpackage }} - - name: SWH_SENTRY_DSN - valueFrom: - secretKeyRef: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }}-sentry-secrets - key: sentry-dsn - # 'name' secret must exist & include key "host" - optional: false - - name: RABBITMQ_HOST - valueFrom: - secretKeyRef: - name: amqp-access-credentials - key: host - # 'name' secret must exist & include key "host" - optional: false - volumeMounts: - - name: config - mountPath: /etc/softwareheritage/config.yml - subPath: config.yml - readOnly: true - - name: config - mountPath: /entrypoint.sh - subPath: entrypoint.sh - readOnly: true - - name: metadata-fetcher-credentials - mountPath: /etc/credentials/metadata-fetcher - readOnly: true - - mountPath: /tmp - name: tmp-volume - volumes: - - name: config - configMap: - name: {{ .Values.loader.name }}-{{ .Values.loader.type }} - defaultMode: 0777 - - name: tmp-volume - emptyDir: {} - - name: metadata-fetcher-credentials - secret: - secretName: metadata-fetcher-credentials diff --git a/charts_code/worker/templates/services.yaml b/charts_code/worker/templates/services.yaml deleted file mode 100644 index ecf4d19..0000000 --- a/charts_code/worker/templates/services.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: storage-{{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - type: ExternalName - externalName: {{ .Values.storage.host }} - ---- -apiVersion: v1 -kind: Service -metadata: - name: amqp-{{ .Values.loader.name }}-{{ .Values.loader.type }} - namespace: ns-{{ .Values.loader.name }}-{{ .Values.loader.type }} -spec: - type: ExternalName - externalName: {{ .Values.amqp.host }} - diff --git a/charts_code/worker/values.yaml b/charts_code/worker/values.yaml deleted file mode 100644 index e835d82..0000000 --- a/charts_code/worker/values.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Default values for worker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -amqp: - host: amqp - -storage: - host: swh-storage - -loader: - name: loaders - type: - -swh: - loader: - image: softwareheritage/loaders - version: 2022-05-17 - replicas: - min: 1 - max: 5 - -sentry: - environment: staging diff --git a/image_code/Dockerfile b/image_code/Dockerfile deleted file mode 100644 index 348ae3f..0000000 --- a/image_code/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Deeply inspired from the Dockerfile of the swh-graph project -FROM python:3.10-bullseye - -RUN apt-get -y update && \ - apt-get -y upgrade && \ - apt-get install -y libcmph-dev librdkafka-dev && \ - rm -rf /var/lib/dpkg && \ - addgroup --gid 1000 swh && \ - useradd --gid 1000 --uid 1000 -m -d /opt/graphql swh && \ - mkdir /etc/swh - -USER swh -WORKDIR /opt/graphql - -COPY --chown=swh:swh requirements-frozen.txt /opt/graphql -COPY --chown=swh:swh entrypoint.sh /opt/graphql - -ENV PYTHONPATH=/opt/graphql -ENV PATH=/opt/graphql/.local/bin:$PATH - -RUN chmod u+x /opt/graphql/entrypoint.sh && \ - /usr/local/bin/python -m pip install --upgrade pip && \ - pip install --no-cache-dir -r requirements-frozen.txt && \ - pip install gunicorn - -ENV SWH_CONFIG_FILENAME=/etc/swh/config.yml -ENV PORT 5013 -EXPOSE $PORT -ENV THREADS 4 -ENV WORKERS 2 -ENV LOG_LEVEL INFO -ENV TIMEOUT 3600 - -ENTRYPOINT "/opt/graphql/entrypoint.sh" diff --git a/image_code/Jenkinsfile b/image_code/Jenkinsfile deleted file mode 100644 index e7fcb51..0000000 --- a/image_code/Jenkinsfile +++ /dev/null @@ -1,25 +0,0 @@ -node { - def app - - stage('Clone repository') { - checkout scm - } - - stage('Build image') { - - app = docker.build("amadouth/graphqll") - } - - stage('Push image') { - - docker.withRegistry('https://registry.hub.docker.com', 'dockerhub') { - app.push("${env.BUILD_NUMBER}") - } - } - - - stage('Trigger Mise à jour du cluster Minikube') { - echo "triggering updatemanifestjob" - build job: 'Minikube cluster update', parameters: [string(name: 'DOCKERTAG', value: env.BUILD_NUMBER)] - } -} diff --git a/image_code/entrypoint.sh b/image_code/entrypoint.sh deleted file mode 100644 index 2ac079b..0000000 --- a/image_code/entrypoint.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -e - -case "$1" in -"shell") - exec bash -i - ;; -*) - echo Starting the swh-graphql API server - - exec gunicorn --bind "0.0.0.0:${PORT}" \ - --threads "${THREADS}" \ - --workers "${WORKERS}" \ - --log-level "${LOG_LEVEL}" \ - --timeout "${TIMEOUT}" \ - --config 'python:swh.core.api.gunicorn_config' \ - 'swh.graphql.server:make_app_from_configfile()' - ;; -esac diff --git a/image_code/requirements-frozen.txt b/image_code/requirements-frozen.txt deleted file mode 100644 index ef08514..0000000 --- a/image_code/requirements-frozen.txt +++ /dev/null @@ -1,71 +0,0 @@ -aiohttp==3.8.1 -aiohttp-utils==3.1.1 -aiosignal==1.2.0 -ariadne==0.14.1 -asgiref==3.4.1 -async-timeout==4.0.2 -attrs==21.4.0 -attrs-strict==1.0.0 -blinker==1.4 -cassandra-driver==3.25.0 -certifi==2021.10.8 -cffi==1.15.0 -charset-normalizer==2.0.12 -click==8.0.1 -confluent-kafka==1.8.2 -Deprecated==1.2.13 -Flask==2.1.2 -frozenlist==1.3.0 -geomet==0.2.1.post1 -graphql-core==3.1.6 -gunicorn==20.1.0 -h11==0.12.0 -hypothesis==6.46.6 -idna==3.3 -importlib-metadata==4.11.3 -iniconfig==1.1.1 -iso8601==1.0.2 -itsdangerous==2.1.2 -Jinja2==3.1.2 -MarkupSafe==2.1.1 -mirakuru==2.4.2 -msgpack==1.0.3 -multidict==6.0.2 -mypy-extensions==0.4.3 -packaging==21.3 -pluggy==1.0.0 -port-for==0.6.2 -psutil==5.9.0 -psycopg2==2.9.3 -py==1.11.0 -pycparser==2.21 -pyparsing==3.0.9 -pytest==7.1.2 -pytest-postgresql==3.1.3 -python-dateutil==2.8.2 -python-magic==0.4.25 -python-mimeparse==1.6.0 -PyYAML==6.0 -redis==4.3.1 -requests==2.27.1 -sentry-sdk==1.5.12 -six==1.16.0 -sortedcontainers==2.4.0 -starlette==0.14.2 -swh.core==2.5.0 -swh.counters==0.9.1 -swh.graphql==0.0.1.post1 -swh.journal==1.0.0 -swh.model==6.2.0 -swh.objstorage==1.0.5 -swh.perfecthash==0.1.2 -swh.storage==1.4.0 -tenacity==8.0.1 -tomli==2.0.1 -typing-extensions==3.10.0.2 -urllib3==1.26.9 -uvicorn==0.17.4 -Werkzeug==2.1.2 -wrapt==1.14.1 -yarl==1.7.2 -zipp==3.8.0 diff --git a/image_code/requirements.txt b/image_code/requirements.txt deleted file mode 100644 index 61104e2..0000000 --- a/image_code/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -ariadne -asgiref -click -graphql-core -h11 -starlette -typing-extensions -swh-graphql diff --git a/secret.yaml b/secret.yaml deleted file mode 100644 index bca70f3..0000000 --- a/secret.yaml +++ /dev/null @@ -1 +0,0 @@ -q