Page MenuHomeSoftware Heritage
Files F9696770

No OneTemporary
Actions

View Options

diff --git a/kubernetes/02-monitoring.yml b/kubernetes/02-monitoring.yml
index 201e586..c54d3cc 100644
--- a/kubernetes/02-monitoring.yml
+++ b/kubernetes/02-monitoring.yml
@@ -1,337 +1,343 @@
# -------
# Prometheus statsd exporter configuration
# -------
---
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-statsd-exporter
data:
config.yml: |
defaults:
timer_type: histogram
buckets:
- .005
- .01
- .025
- .05
- .1
- .25
- .5
- .75
- 1
- 2
- 5
- 10
- 15
- 30
- 45
- 60
- 120
- 300
- 600
- 900
- 1800
- 2700
- 3600
- 7200
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-statsd-exporter
labels:
app: prometheus-statsd-exporter
spec:
replicas: 1
selector:
matchLabels:
app: prometheus-statsd-exporter
template:
metadata:
labels:
app: prometheus-statsd-exporter
spec:
containers:
- name: prometheus-statsd-exporter
image: prom/statsd-exporter
imagePullPolicy: Always
args:
- "--statsd.mapping-config=/etc/prometheus/statsd-mapping.yml"
ports:
- containerPort: 9125
volumeMounts:
- name: config
mountPath: /etc/prometheus/statsd-mapping.yml
subPath: config.yml
readOnly: true
volumes:
- name: config
configMap:
name: prometheus-statsd-exporter
---
apiVersion: v1
kind: Service
metadata:
name: prometheus-statsd-exporter
spec:
type: ClusterIP
selector:
app: prometheus-statsd-exporter
ports:
- name: statsd
port: 9125
targetPort: 9125
- name: http
port: 9102
targetPort: 9102
---
# -------
# Prometheus configuration
# -------
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus
data:
config.yml: |
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- prometheus:9090
metrics_path: /prometheus/metrics
- job_name: statsd-exporter
static_configs:
- targets:
- prometheus-statsd-exporter:9102
- job_name: jmx-exporter-cassandra
static_configs:
- targets:
- prometheus-jmx-exporter-cassandra:5556
+
+ - job_name: swh-counters-exporter
+ static_configs:
+ - targets:
+ - counters:5011
+ metrics_path: /metrics
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: prometheus-pv
spec:
capacity:
storage: 10Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
storageClassName: prometheus-pv
local:
path: /srv/softwareheritage-kube/dev/prometheus
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
# TODO adapt for your needs
- key: kubernetes.io/os
operator: In
values:
- linux
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
spec:
type: ClusterIP
selector:
app: prometheus
ports:
- port: 9090
targetPort: 9090
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: prometheus
spec:
rules:
- host: prometheus.default
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: prometheus
port:
number: 9090
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: prometheus-pvc
spec:
accessModes:
- ReadWriteOnce
storageClassName: prometheus-pv
resources:
requests:
storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus
labels:
app: prometheus
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
spec:
containers:
- name: prometheus
image: prom/prometheus
imagePullPolicy: Always
args:
- "--config.file=/etc/prometheus/prometheus.yml"
ports:
- containerPort: 9090
volumeMounts:
- name: config
mountPath: /etc/prometheus/prometheus.yml
subPath: config.yml
readOnly: true
- mountPath: "/prometheus/data"
name: prometheus-pvc
resources:
requests:
cpu: "20m"
memory: "100Mi"
limits:
cpu: "100m"
memory: "256Mi"
volumes:
- name: config
configMap:
name: prometheus
- name: prometheus-pvc
persistentVolumeClaim:
claimName: prometheus-pvc
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: grafana-pv
spec:
capacity:
storage: 1Gi
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Delete
storageClassName: grafana-pv
local:
path: /srv/softwareheritage-kube/dev/grafana
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
# TODO adapt for your needs
- key: kubernetes.io/os
operator: In
values:
- linux
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: grafana-pvc
spec:
accessModes:
- ReadWriteOnce
storageClassName: grafana-pv
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: Service
metadata:
name: grafana
spec:
type: ClusterIP
selector:
app: grafana
ports:
- port: 3000
targetPort: 3000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: grafana
spec:
rules:
- host: grafana.default
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grafana
port:
number: 3000
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana
labels:
app: grafana
spec:
replicas: 1
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
spec:
containers:
- name: grafana
image: swh/grafana
imagePullPolicy: Always
env:
- name: GF_SERVER_ROOT_URL
value: http://grafana.default
ports:
- containerPort: 3000
resources:
requests:
cpu: "50m"
memory: "128Mi"
limits:
cpu: "100m"
memory: "256Mi"
volumeMounts:
- mountPath: "/var/lib/grafana"
name: grafana-pvc
volumes:
- name: grafana-pvc
persistentVolumeClaim:
claimName: grafana-pvc
diff --git a/kubernetes/60-redis.yml b/kubernetes/60-redis.yml
new file mode 100644
index 0000000..3976864
--- /dev/null
+++ b/kubernetes/60-redis.yml
@@ -0,0 +1,93 @@
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: redis-data-pv
+spec:
+ capacity:
+ storage: 1Gi
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteOnce
+ persistentVolumeReclaimPolicy: Delete
+ storageClassName: redis-data-pv
+ local:
+ path: /srv/softwareheritage-kube/dev/redis
+ nodeAffinity:
+ required:
+ nodeSelectorTerms:
+ - matchExpressions:
+ # TODO adapt for your needs
+ - key: kubernetes.io/os
+ operator: In
+ values:
+ - linux
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: redis-data-pvc
+spec:
+ accessModes:
+ - ReadWriteOnce
+ storageClassName: redis-data-pv
+ resources:
+ requests:
+ storage: 1Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: redis
+spec:
+ type: ClusterIP
+ selector:
+ app: redis
+ ports:
+ - port: 6379
+ targetPort: 6379
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: redis
+ labels:
+ app: redis
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: redis
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: redis
+ spec:
+ containers:
+ - name: redis
+ image: redis:latest
+ imagePullPolicy: Always
+ args:
+ - "--save"
+ - "60"
+ - "1" # flush every minutes
+ ports:
+ - containerPort: 6379
+ resources:
+ requests:
+ memory: "50Mi"
+ cpu: "50m"
+ limits:
+ memory: "100Mi"
+ cpu: "100m"
+ volumeMounts:
+ - mountPath: "/data"
+ name: redis-data-pvc
+ volumes:
+ - name: redis-data-pvc
+ persistentVolumeClaim:
+ claimName: redis-data-pvc
diff --git a/kubernetes/61-counters.yml b/kubernetes/61-counters.yml
new file mode 100644
index 0000000..4730183
--- /dev/null
+++ b/kubernetes/61-counters.yml
@@ -0,0 +1,167 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: counters
+data:
+ config.yml: |
+ counters:
+ cls: redis
+ host: redis:6379
+ history:
+ cls: prometheus
+ prometheus_host: prometheus
+ prometheus_port: 9090
+ live_data_start: 1609462861 # 2021-01-01
+ cache_base_directory: /tmp
+ interval: 1h
+ entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ wait-for-it redis:6379 -s --timeout=0
+
+ exec gunicorn --bind 0.0.0.0:5011 \
+ --reload \
+ --threads 4 \
+ --workers 2 \
+ --log-level DEBUG \
+ --timeout 3600 \
+ --config 'python:swh.core.api.gunicorn_config' \
+ 'swh.counters.api.server:make_app_from_configfile()'
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: counters
+ labels:
+ app: counters
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: counters
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: counters
+ spec:
+ containers:
+ - name: counters
+ image: swh/counters:latest
+ imagePullPolicy: Always
+ command:
+ - /entrypoint.sh
+ ports:
+ - containerPort: 5011
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 5011
+ scheme: "HTTP"
+ initialDelaySeconds: 0
+ failureThreshold: 1
+ periodSeconds: 10
+ startupProbe:
+ httpGet:
+ path: /
+ port: 5011
+ scheme: "HTTP"
+ initialDelaySeconds: 5
+ failureThreshold: 30
+ periodSeconds: 1
+ env:
+ - name: PORT
+ value: "5010"
+ - name: STATSD_HOST
+ value: "prometheus-statsd-exporter"
+ - name: STATSD_PORT
+ value: "9125"
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/softwareheritage/config.yml
+ volumeMounts:
+ - name: config
+ mountPath: /etc/softwareheritage/config.yml
+ subPath: config.yml
+ readOnly: true
+ - name: config
+ mountPath: /entrypoint.sh
+ subPath: entrypoint.sh
+ readOnly: true
+ resources:
+ requests:
+ memory: "75Mi"
+ cpu: "20m"
+ limits:
+ memory: "150Mi"
+ cpu: "200m"
+ volumes:
+ - name: config
+ configMap:
+ name: counters
+ defaultMode: 0777
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: counters
+spec:
+ type: ClusterIP
+ selector:
+ app: counters
+ ports:
+ - port: 5011
+ targetPort: 5011
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: counters
+spec:
+ rules:
+ - host: counters.default
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: counters
+ port:
+ number: 5011
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+ name: counters-history
+ labels:
+ app: counters-history
+spec:
+ schedule: "*/1 * * * *"
+ successfulJobsHistoryLimit: 1
+ failedJobsHistoryLimit: 1
+ concurrencyPolicy: Forbid
+ jobTemplate:
+ spec:
+ template:
+ spec:
+ containers:
+ - name: counters-history
+ image: curlimages/curl
+ imagePullPolicy: Always
+ args:
+ - -v
+ - -f
+ - -X
+ - POST
+ - -H
+ - "Content-Type: application/json"
+ - "http://counters:5011/refresh_history"
+ - "-d"
+ - '{"cache_file": "history.json", "objects": ["content", "directory", "origin", "origin_visit", "origin_visit_status", "release", "revision", "snapshot"]}'
+ restartPolicy: Never
diff --git a/kubernetes/62-counters-journal-client.yml b/kubernetes/62-counters-journal-client.yml
new file mode 100644
index 0000000..2936b9c
--- /dev/null
+++ b/kubernetes/62-counters-journal-client.yml
@@ -0,0 +1,86 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: counters-journal-client
+data:
+ config.yml: |
+ counters:
+ cls: remote
+ url: http://counters:5011
+ journal:
+ brokers:
+ - kafka
+ group_id: swh.counters.journal_client
+ prefix: swh.journal.objects
+ object_types:
+ - content
+ - directory
+ - origin
+ - origin_visit
+ - origin_visit_status
+ - release
+ - revision
+ - skipped_content
+ - snapshot
+ message.max.bytes: 524288000
+ entrypoint.sh: |
+ #!/bin/bash
+
+ set -e
+
+ wait-for-it kafka:9092 -s --timeout=0
+ wait-for-it counters:5011 -s --timeout=0
+
+ swh --log-level INFO counters \
+ --config-file /etc/softwareheritage/config.yml \
+ journal-client
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: counters-journal-client
+ labels:
+ app: counters-journal-client
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: counters-journal-client
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ app: counters-journal-client
+ spec:
+ containers:
+ - name: counters-journal-client
+ image: swh/counters:latest
+ imagePullPolicy: Always
+ command:
+ - /entrypoint.sh
+ env:
+ - name: SWH_CONFIG_FILENAME
+ value: /etc/softwareheritage/config.yml
+ volumeMounts:
+ - name: config
+ mountPath: /etc/softwareheritage/config.yml
+ subPath: config.yml
+ readOnly: true
+ - name: config
+ mountPath: /entrypoint.sh
+ subPath: entrypoint.sh
+ readOnly: true
+ resources:
+ requests:
+ memory: "100Mi"
+ cpu: "20m"
+ limits:
+ memory: "150Mi"
+ cpu: "100m"
+ volumes:
+ - name: config
+ configMap:
+ name: counters-journal-client
+ defaultMode: 0777
diff --git a/skaffold.yaml b/skaffold.yaml
index 1513a4d..e208d66 100644
--- a/skaffold.yaml
+++ b/skaffold.yaml
@@ -1,75 +1,84 @@
apiVersion: skaffold/v2beta13
kind: Config
metadata:
name: swh-environment
build:
artifacts:
- image: swh/stack-base
docker:
dockerfile: Dockerfile
- image: swh/objstorage
docker:
dockerfile: Dockerfile.objstorage
requires:
- image: swh/stack-base
alias: BASE
- image: swh/storage
docker:
dockerfile: Dockerfile.storage
requires:
- image: swh/stack-base
alias: BASE
- image: swh/scheduler
docker:
dockerfile: Dockerfile.scheduler
requires:
- image: swh/stack-base
alias: BASE
- image: swh/webapp
docker:
dockerfile: Dockerfile.webapp
requires:
- image: swh/stack-base
alias: BASE
- image: swh/loaders
docker:
dockerfile: Dockerfile.loaders
requires:
- image: swh/stack-base
alias: BASE
- image: swh/listers
docker:
dockerfile: Dockerfile.listers
requires:
- image: swh/stack-base
alias: BASE
- image: swh/grafana
docker:
dockerfile: Dockerfile.grafana
- image: swh/search
docker:
dockerfile: Dockerfile.search
requires:
- image: swh/stack-base
alias: BASE
+ - image: swh/counters
+ docker:
+ dockerfile: Dockerfile.counters
+ requires:
+ - image: swh/stack-base
+ alias: BASE
deploy:
kubectl:
manifests:
- kubernetes/01-journal.yml
- kubernetes/02-monitoring.yml
- kubernetes/05-storage-db.yml
- kubernetes/10-objstorage.yml
- kubernetes/11-storage.yml
- kubernetes/15-scheduler-db.yml
- kubernetes/16-rabbitmq.yml
- kubernetes/20-scheduler.yml
- kubernetes/21-scheduler-runner.yml
- kubernetes/22-scheduler-listener.yml
- kubernetes/23-scheduler-journal-client.yml
- kubernetes/29-web-db.yml
- kubernetes/30-webapp.yml
- kubernetes/40-loaders.yml
- kubernetes/45-listers.yml
- kubernetes/50-elasticsearch.yml
- kubernetes/55-search.yml
- kubernetes/56-search-journal-client.yml
+ - kubernetes/60-redis.yml
+ - kubernetes/61-counters.yml
+ - kubernetes/62-counters-journal-client.yml

File Metadata

Mime Type
text/x-diff
Expires
Mon, Aug 18, 9:24 PM (1 d, 4 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3331164

Event Timeline

About · Developer information · Legal