diff --git a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml index 3f0793c..47a5f4d 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/11-storage.yml @@ -1,146 +1,154 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: cassandra hosts: - parasilo-2.rennes.grid5000.fr + - parasilo-3.rennes.grid5000.fr + - parasilo-4.rennes.grid5000.fr + - parasilo-5.rennes.grid5000.fr + - parasilo-6.rennes.grid5000.fr + - parasilo-7.rennes.grid5000.fr + - parasilo-8.rennes.grid5000.fr keyspace: swh consistency_level: LOCAL_QUORUM objstorage: cls: remote url: http://objstorage:5003 entrypoint.sh: | #!/bin/bash set -e echo Starting the swh-storage API server exec gunicorn --bind 0.0.0.0:5002 \ --reload \ + --access-logfile /dev/stdout \ + --access-logformat "%(t)s %(s)s %(r)s %(M)s" \ --workers 128 \ - --log-level DEBUG \ + --log-level INFO \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.storage.api.server:make_app_from_configfile()' --- apiVersion: apps/v1 kind: Deployment metadata: name: storage labels: app: storage spec: replicas: 1 selector: matchLabels: app: storage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: storage spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: In values: - loader containers: - name: storage image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 readinessProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 0 failureThreshold: 2 periodSeconds: 10 startupProbe: httpGet: path: / port: 5002 scheme: "HTTP" initialDelaySeconds: 5 failureThreshold: 30 periodSeconds: 1 env: - name: PORT value: "5002" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true # resources: # requests: # memory: "10000Mi" # cpu: "5000m" # limits: # memory: "15000Mi" # cpu: "10000m" volumes: - name: config configMap: name: storage defaultMode: 0777 - name: db-password secret: secretName: storage-db --- apiVersion: v1 kind: Service metadata: name: storage spec: type: ClusterIP selector: app: storage ports: - port: 5002 targetPort: 5002 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: storage spec: rules: - host: storage.default http: paths: - path: / pathType: Prefix backend: service: name: storage port: number: 5002 diff --git a/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml b/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml index 4279323..6955582 100644 --- a/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml +++ b/sysadmin/grid5000/cassandra/kubernetes/40-loaders.yml @@ -1,160 +1,162 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: loaders data: config.yml: | storage: cls: pipeline steps: - cls: buffer min_batch_size: - content: 10000 + content: 500 content_bytes: 104857600 directory: 1000 - revision: 1000 + revision: 50 - cls: filter - cls: retry - cls: remote url: http://storage:5002/ max_content_size: 104857600 temp_file_cutoff: 536870912 celery: task_ignore_result: True task_broker: amqp://guest:guest@amqp// task_queues: - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - swh.loader.git.tasks.UpdateGitRepository - swh.loader.package.archive.tasks.LoadArchive - swh.loader.package.cran.tasks.LoadCRAN - swh.loader.package.debian.tasks.LoadDebian - swh.loader.package.npm.tasks.LoadNpm - swh.loader.package.pypi.tasks.LoadPyPI - save_code_now:swh.loader.git.tasks.UpdateGitRepository # - swh.loader.mercurial.tasks.LoadArchiveMercurial # - swh.loader.mercurial.tasks.LoadMercurial # - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository # - swh.loader.svn.tasks.LoadSvnRepository # - swh.loader.svn.tasks.MountAndLoadSvnRepository entrypoint-init.sh: | #!/bin/bash set -e # echo Waiting for RabbitMQ to start wait-for-it amqp:5672 -s --timeout=0 # echo Register task types in scheduler database wait-for-it scheduler:5008 -s --timeout=0 swh scheduler --url http://scheduler:5008 task-type register entrypoint.sh: | #!/bin/bash set -e echo Starting the swh Celery worker exec python -m celery \ --app=swh.scheduler.celery_backend.config.app \ worker \ - --pool=prefork --events \ + --pool=prefork \ --concurrency=${CONCURRENCY} \ --max-tasks-per-child=${MAX_TASKS_PER_CHILD} \ -Ofair --loglevel=${LOGLEVEL} \ --hostname "${HOSTNAME}" --- apiVersion: apps/v1 kind: Deployment metadata: name: loaders labels: app: loaders spec: - replicas: 1 + replicas: 100 selector: matchLabels: app: loaders strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: loaders spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: type operator: In values: - loader initContainers: - name: loaders-init image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint-init.sh readOnly: true containers: - name: loaders image: vsellier/swh-base imagePullPolicy: Always command: - /entrypoint.sh ports: - containerPort: 5002 resources: requests: memory: "2000Mi" cpu: "500m" limits: memory: "4000Mi" cpu: "1000m" lifecycle: preStop: exec: command: ["kill", "1"] env: - name: CONCURRENCY value: "1" - name: MAX_TASKS_PER_CHILD value: "5" - name: LOGLEVEL value: "INFO" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml + - name: TMPDIR + value: /srv/tmp volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true - name: tmp - mountPath: /tmp + mountPath: /srv/tmp volumes: - name: config configMap: name: loaders defaultMode: 0777 - name: tmp emptyDir: {} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/swh-storage-stats.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/swh-storage-stats.json index 4d84b96..aac95ad 100644 --- a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/swh-storage-stats.json +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/swh-storage-stats.json @@ -1,311 +1,311 @@ { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "limit": 100, "matchAny": true, "name": "Annotations & Alerts", "tags": [ "configuration", "admin", "run" ], "type": "tags" } ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "id": 20, - "iteration": 1631017061345, + "iteration": 1633099886309, "links": [], "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 2, "panels": [], "repeat": "object", "title": "${object}", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 5, "interval": "1m", "legend": { "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.6", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum by (instance, endpoint) (rate(swh_storage_operations_total{object_type=~\"${object}\"}[$inter]))", "hide": false, "interval": "", "legendFormat": "{{instance}} - {{endpoint}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Creation rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:57", "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:58", - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], - "refresh": "10s", + "refresh": false, "schemaVersion": 30, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": null, "definition": "label_values(swh_storage_operations_total, instance)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": null, "multi": true, "name": "cassandra", "options": [], "query": { "query": "label_values(swh_storage_operations_total, instance)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": null, "definition": "label_values(swh_storage_operations_total, object_type)", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, "name": "object", "options": [], "query": { "query": "label_values(swh_storage_operations_total, object_type)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "2m", "value": "2m" }, "description": null, "error": null, "hide": 0, "label": null, "name": "inter", "options": [ { "selected": true, "text": "2m", "value": "2m" }, { "selected": false, "text": "5m", "value": "5m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": false, "text": "30d", "value": "30d" } ], "query": "2m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" } ] }, "time": { - "from": "now-6h", - "to": "now" + "from": "2021-09-30T22:41:47.094Z", + "to": "2021-10-01T00:03:55.968Z" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "SWH storage statsd", "uid": "ydJXRa47z", - "version": 8 + "version": 16 } \ No newline at end of file