No OneTemporary
Actions

Size

12 KB

Subscribers

None

View Options

	diff --git a/kubernetes/10-objstorage.yml b/kubernetes/10-objstorage.yml
	index 5c6b1ca..367a15e 100644
	--- a/kubernetes/10-objstorage.yml
	+++ b/kubernetes/10-objstorage.yml
	@@ -1,162 +1,162 @@
	---
	apiVersion: v1
	kind: ConfigMap
	metadata:
	name: objstorage
	data:
	config.yml: \|
	objstorage:
	cls: pathslicing
	args:
	root: "/srv/softwareheritage/objects"
	slicing: 0:5
	client_max_size: 1073741824
	entrypoint.sh: \|
	#!/bin/bash

	set -e

	# source /srv/softwareheritage/utils/pyutils.sh
	# setup_pip

	# echo Installed Python packages:
	# pip list

	echo Starting the swh-objstorage API server
	exec gunicorn --bind 0.0.0.0:5003 \
	--worker-class aiohttp.worker.GunicornWebWorker \
	--log-level DEBUG \
	--threads 4 \
	--workers 2 \
	--reload \
	--timeout 3600 \
	--config 'python:swh.core.api.gunicorn_config' \
	'swh.objstorage.api.server:make_app_from_configfile()'
	---
	apiVersion: v1
	kind: PersistentVolume
	metadata:
	name: objstorage-pv
	spec:
	capacity:
	storage: 10Gi
	volumeMode: Filesystem
	accessModes:
	- ReadWriteOnce
	persistentVolumeReclaimPolicy: Delete
	storageClassName: objstorage-pv
	local:
	- path: /srv/softwareheritage-kube/objects
	+ path: /srv/softwareheritage-kube/dev/objects
	nodeAffinity:
	required:
	nodeSelectorTerms:
	- matchExpressions:
	# TODO adapt for your needs
	- key: kubernetes.io/os
	operator: In
	values:
	- linux
	---
	kind: PersistentVolumeClaim
	apiVersion: v1
	metadata:
	name: objstorage-pvc
	spec:
	accessModes:
	- ReadWriteOnce
	storageClassName: objstorage-pv
	resources:
	requests:
	storage: 10Gi
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: objstorage
	labels:
	app: objstorage
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: objstorage
	strategy:
	type: RollingUpdate
	rollingUpdate:
	maxSurge: 1
	template:
	metadata:
	labels:
	app: objstorage
	spec:
	containers:
	- name: objstorage
	image: swh/objstorage:latest
	command:
	- /entrypoint.sh
	ports:
	- containerPort: 5003
	env:
	- name: PORT
	value: "5003"
	- name: STATSD_HOST
	value: "prometheus-statsd-exporter"
	- name: STATSD_PORT
	value: "9125"
	- name: SWH_CONFIG_FILENAME
	value: /etc/softwareheritage/config.yml
	volumeMounts:
	- mountPath: "/srv/softwareheritage/objects"
	name: objstorage-pvc
	- name: config
	mountPath: /etc/softwareheritage/config.yml
	subPath: config.yml
	readOnly: true
	- name: config
	mountPath: /entrypoint.sh
	subPath: entrypoint.sh
	readOnly: true
	resources:
	requests:
	memory: "512Mi"
	cpu: "250m"
	limits:
	memory: "1024Mi"
	cpu: "500m"
	volumes:
	- name: config
	configMap:
	name: objstorage
	defaultMode: 0777
	- name: objstorage-pvc
	persistentVolumeClaim:
	claimName: objstorage-pvc
	---
	apiVersion: v1
	kind: Service
	metadata:
	name: objstorage
	spec:
	type: ClusterIP
	selector:
	app: objstorage
	ports:
	- port: 5003
	targetPort: 5003
	---
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: objstorage
	spec:
	rules:
	- host: objstorage.default
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: objstorage
	port:
	number: 5003
	diff --git a/kubernetes/15-scheduler-db.yml b/kubernetes/15-scheduler-db.yml
	index 81ef08b..0847158 100644
	--- a/kubernetes/15-scheduler-db.yml
	+++ b/kubernetes/15-scheduler-db.yml
	@@ -1,129 +1,129 @@
	---
	apiVersion: v1
	kind: PersistentVolume
	metadata:
	name: scheduler-db-pv
	spec:
	capacity:
	storage: 10Gi
	volumeMode: Filesystem
	accessModes:
	- ReadWriteOnce
	persistentVolumeReclaimPolicy: Delete
	storageClassName: scheduler-db
	local:
	- path: /srv/softwareheritage/dev/scheduler-db
	+ path: /srv/softwareheritage-kube/dev/scheduler-db
	nodeAffinity:
	required:
	nodeSelectorTerms:
	- matchExpressions:
	# TODO adapt for your needs
	- key: kubernetes.io/os
	operator: In
	values:
	- linux
	---
	kind: PersistentVolumeClaim
	apiVersion: v1
	metadata:
	name: scheduler-db-pvc
	spec:
	accessModes:
	- ReadWriteOnce
	storageClassName: scheduler-db
	resources:
	requests:
	storage: 10Gi

	---
	## TODO Change this to your real postgresql password
	apiVersion: v1
	kind: Secret
	metadata:
	name: scheduler-db
	type: Opaque
	stringData:
	POSTGRES_PASSWORD: swh
	---
	apiVersion: v1
	kind: ConfigMap
	metadata:
	name: scheduler-db
	data:
	# property-like keys; each key maps to a simple value
	POSTGRES_USER: swh
	POSTGRES_DB: swh-scheduler
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: scheduler-db
	labels:
	app: scheduler-db
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: scheduler-db
	template:
	metadata:
	labels:
	app: scheduler-db
	spec:
	containers:
	- name: scheduler-db
	image: postgres:13.0
	imagePullPolicy: Always
	ports:
	- containerPort: 5432
	args:
	- "-c"
	- "shared_buffers=512MB"
	- "-c"
	- "effective_cache_size=512MB"
	- "-c"
	- "random_page_cost=1.5"
	- "-c"
	- "max_wal_size=512MB"
	env:
	- name: POSTGRES_USER
	valueFrom:
	configMapKeyRef:
	name: scheduler-db
	key: POSTGRES_USER
	- name: POSTGRES_PASSWORD
	valueFrom:
	secretKeyRef:
	name: scheduler-db
	key: POSTGRES_PASSWORD
	- name: POSTGRES_DB
	valueFrom:
	configMapKeyRef:
	name: scheduler-db
	key: POSTGRES_DB
	volumeMounts:
	- mountPath: "/var/lib/postgresql"
	name: scheduler-db-pvc
	resources:
	requests:
	memory: "512Mi"
	cpu: "250m"
	limits:
	memory: "1024Mi"
	cpu: "500m"
	volumes:
	- name: scheduler-db-pvc
	persistentVolumeClaim:
	claimName: scheduler-db-pvc
	---
	apiVersion: v1
	kind: Service
	metadata:
	name: scheduler-db
	spec:
	type: ClusterIP
	selector:
	app: scheduler-db
	ports:
	- port: 5432
	targetPort: 5432
	diff --git a/kubernetes/Readme.md b/kubernetes/Readme.md
	index bcdd18b..7d2a180 100644
	--- a/kubernetes/Readme.md
	+++ b/kubernetes/Readme.md
	@@ -1,197 +1,196 @@
	## Prerequisite

	### Directories

	```
	-# sudo mkdir -p /srv/softwareheritage-kube/objects
	-# sudo mkdir -p /srv/softwareheritage-kube/storage-db
	-# sudo mkdir -p /srv/softwareheritage-kube/scheduler-db
	+# sudo mkdir -p /srv/softwareheritage-kube/dev/{objects,storage-db,scheduler-db}
	```
	+
	Must match the content of `05-storage-db.yaml`

	### Registry

	- Add the following line on your `/etc/hosts` file. It's needed to be able to
	push the image to it from docker
	```
	127.0.0.1 registry.default
	```
	- Start the registry in kubernetes
	```
	# cd kubernetes
	# kubectl apply -f registry/00-registry.yml
	```

	## Build the base image

	```
	# cd docker
	# docker build --no-cache -t swh/stack .

	# docker tag swh/stack:latest registry.default/swh/stack:latest
	# docker push registry.default/swh/stack:latest

	```

	## start the objstorage

	- build image
	```
	# docker build -f Dockerfile.objstorage -t swh/objstorage --build-arg BASE=swh/stack .
	# docker tag swh/objstorage:latest registry.default/swh/objstorage:latest
	# docker push registry.default/swh/objstorage:latest
	```

	- start the service
	```
	# cd kubernetes

	# kubectl apply -f 10-objstorage.yml
	configmap/objstorage created
	persistentvolume/objstorage-pv created
	persistentvolumeclaim/objstorage-pvc created
	deployment.apps/objstorage created
	service/objstorage created
	```
	- test it
	```
	# kubectl get pods
	NAME READY STATUS RESTARTS AGE
	registry-deployment-7595868dc8-657ps 1/1 Running 0 46m
	objstorage-8587d58b68-76jbn 1/1 Running 0 12m

	# kubectl get services objstorage
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	objstorage ClusterIP 10.43.185.191 <none> 5003/TCP 17m

	# curl http://$(kubectl get services objstorage -o jsonpath='{.spec.clusterIP}'):5003
	SWH Objstorage API server%
	```

	## Start the storage

	- Start the db
	```
	# cd kubernetes

	# kubectl apply -f 05-storage-db.yml
	persistentvolume/storage-db-pv created
	persistentvolumeclaim/storage-db-pvc created
	secret/storage-db created
	configmap/storage-db created
	deployment.apps/storage-db created
	service/storage-db created

	# kubectl get pods
	NAME READY STATUS RESTARTS AGE
	registry-deployment-7595868dc8-657ps 1/1 Running 0 46m
	objstorage-8587d58b68-76jbn 1/1 Running 0 15m
	storage-db-64b7f8b684-48n7w 1/1 Running 0 4m52s

	# kubectl get services storage-db
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	storage-db ClusterIP 10.43.213.178 <none> 5432/TCP 8m19s
	```
	- Start the storage
	```
	# cd kubernetes

	# kubectl apply -f 11-storage.yml
	configmap/storage created
	deployment.apps/storage created
	service/storage created
	```

	- Test the service
	```
	# kubectl get pods
	NAME READY STATUS RESTARTS AGE
	registry-deployment-7595868dc8-657ps 1/1 Running 0 49m
	storage-db-64b7f8b684-48n7w 1/1 Running 0 7m40s
	storage-6b759fb974-w9rzj 1/1 Running 0 66s

	# kubectl get services storage
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	storage ClusterIP 10.43.212.116 <none> 5002/TCP 2m24s

	# curl http://$(kubectl get services storage -o jsonpath='{.spec.clusterIP}'):5002
	<html>
	<head><title>Software Heritage storage server</title></head>
	<body>
	<p>You have reached the
	<a href="https://www.softwareheritage.org/">Software Heritage</a>
	storage server.<br />
	See its
	<a href="https://docs.softwareheritage.org/devel/swh-storage/">documentation
	and API</a> for more information</p>
	</body>
	</html>
	```

	## Start the scheduler

	- Start the db

	```
	# cd kubernetes

	# kubectl apply -f 15-scheduler-db.yml
	persistentvolume/scheduler-db-pv unchanged
	persistentvolumeclaim/scheduler-db-pvc created
	secret/scheduler-db configured
	configmap/scheduler-db unchanged
	deployment.apps/scheduler-db unchanged
	service/scheduler-db unchanged

	# kubectl get services scheduler-db
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	scheduler-db ClusterIP 10.43.115.249 <none> 5433/TCP 110s
	```

	- Test the service

	```
	# kubectl apply -f 20-scheduler.yml
	configmap/scheduler created
	deployment.apps/scheduler created
	service/scheduler created
	ingress.networking.k8s.io/scheduler created

	# kubectl get services scheduler
	NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
	scheduler ClusterIP 10.43.218.74 <none> 5008/TCP 23s

	# kubectl get pods NAME READY STATUS RESTARTS AGE
	registry-deployment-5f6894c5b-9wkmr 1/1 Running 0 28m
	objstorage-5b87c549b6-f6jvc 1/1 Running 0 12m
	storage-db-79bfbff68-mg7fr 1/1 Running 0 107s
	storage-6bfcb87b6-7s7t8 1/1 Running 0 87s
	scheduler-db-666c8dc8b4-qxm9d 1/1 Running 0 73s
	scheduler-595f944854-hbsj4 1/1 Running 0 62s

	# curl http://$(kubectl get services scheduler -o jsonpath='{.spec.clusterIP}'):5008
	<html>
	<head><title>Software Heritage scheduler RPC server</title></head>
	<body>
	<p>You have reached the
	<a href="https://www.softwareheritage.org/">Software Heritage</a>
	scheduler RPC server.<br />
	See its
	<a href="https://docs.softwareheritage.org/devel/swh-scheduler/">documentation
	and API</a> for more information</p>
	</body>
	</html>%
	```

	## Development

	### Skaffold

	To start the development environment using skaffold, use the following command:

	```
	skaffold --default-repo registry.default dev
	```

	It will build the images, deploy them on the local registry and start the services.
	It will monitor the projects to detect the changes and restart the containers when needed

File Metadata

Mime Type: text/x-diff
Expires: Fri, Jul 4, 3:37 PM (1 w, 2 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3329222

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions