diff --git a/Dockerfile.webapp b/Dockerfile.webapp index 327d507..26693c5 100644 --- a/Dockerfile.webapp +++ b/Dockerfile.webapp @@ -1,23 +1,18 @@ ARG BASE FROM $BASE COPY swh-auth /app/swh-vault RUN pip install swh-auth COPY swh-vault /app/swh-vault RUN pip install swh-vault COPY swh-indexer /app/swh-indexer RUN pip install swh-indexer COPY swh-scheduler /app/swh-scheduler RUN pip install swh-scheduler COPY swh-search /app/swh-search RUN pip install swh-search COPY swh-storage /app/swh-storage RUN pip install swh-storage COPY swh-web /app/swh-web RUN pip install swh-web - -COPY --chown=swh:swh docker/services/swh-web/entrypoint.sh /entrypoint.sh -RUN chmod u+x /entrypoint.sh - -# RUN for module in $(ls -d swh-*); do echo Installing $module; pip install $module; done diff --git a/kubernetes/29-web-db.yml b/kubernetes/29-web-db.yml new file mode 100644 index 0000000..f3975d6 --- /dev/null +++ b/kubernetes/29-web-db.yml @@ -0,0 +1,142 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: web-db-pv +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: web-db + local: + path: /srv/softwareheritage-kube/dev/web-db + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + # TODO adapt for your needs + - key: kubernetes.io/os + operator: In + values: + - linux +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: web-db-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: web-db + resources: + requests: + storage: 10Gi + +--- +## TODO Change this to your real postgresql password +apiVersion: v1 +kind: Secret +metadata: + name: web-db +type: Opaque +stringData: + POSTGRES_PASSWORD: swh +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: web-db +data: + # property-like keys; each key maps to a simple value + POSTGRES_USER: swh + POSTGRES_DB: swh-web +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web-db + labels: + app: web-db +spec: + replicas: 1 + selector: + matchLabels: + app: web-db + template: + metadata: + labels: + app: web-db + spec: + containers: + - name: web-db + image: postgres:13.0 + imagePullPolicy: Always + ports: + - containerPort: 5432 + args: + - "-c" + - "shared_buffers=512MB" + - "-c" + - "effective_cache_size=512MB" + - "-c" + - "random_page_cost=1.5" + - "-c" + - "max_wal_size=512MB" + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: web-db + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: web-db + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: web-db + key: POSTGRES_DB + volumeMounts: + - mountPath: "/var/lib/postgresql" + name: web-db-pvc + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1024Mi" + cpu: "500m" + volumes: + - name: web-db-pvc + persistentVolumeClaim: + claimName: web-db-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: web-db +spec: + type: ClusterIP + selector: + app: web-db + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: v1 +kind: Service +metadata: + name: web-db-dev +spec: + type: NodePort + selector: + app: web-db + ports: + - port: 5437 + targetPort: 5432 # internal port of the service + diff --git a/kubernetes/30-webapp.yml b/kubernetes/30-webapp.yml index c90d5ee..b33d8c6 100644 --- a/kubernetes/30-webapp.yml +++ b/kubernetes/30-webapp.yml @@ -1,190 +1,272 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: webapp data: config.yml: | storage: cls: remote args: url: http://storage:5002/ timeout: 1 objstorage: cls: remote args: url: http://objstorage:5003/ # indexer_storage: # cls: remote # args: # url: http://indexer-storage:5007/ scheduler: cls: remote args: url: http://scheduler:5008/ # vault: # cls: remote # args: # url: http://vault:5005/ # deposit: # private_api_url: https://deposit:5006/1/private/ # private_api_user: swhworker # private_api_password: '' # TODO: remove the * allowed_hosts: - "*" - appserver debug: yes serve_assets: yes throttling: cache_uri: memcached:11211 scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_api_origin_search: limiter_rate: default: 70/m exempted_networks: - 0.0.0.0/0 swh_api_origin_visit_latest: limiter_rate: default: 700/m exempted_networks: - 0.0.0.0/0 swh_vault_cooking: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_save_origin: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 - development_db: /tmp/db.sqlite3 - production_db: /tmp/db.sqlite3 - search: {} + entrypoint-init.sh: | + #!/bin/bash + + set -e + + source /srv/softwareheritage/utils/pgsql.sh + + create_admin_script=" + from django.contrib.auth import get_user_model + + username = 'admin'; + password = 'admin'; + email = 'admin@swh-web.org'; + + User = get_user_model(); + + if not User.objects.filter(username = username).exists(): + User.objects.create_superuser(username, email, password); + " + + echo "waiting for database availability" + wait_pgsql ${PGDATABASE} + + echo "Migrating db using ${DJANGO_SETTINGS_MODULE}" + django-admin migrate --settings=${DJANGO_SETTINGS_MODULE} + + echo "Creating Django admin user" + echo "$create_admin_script" | python3 -m swh.web.manage shell + + entrypoint.sh: | + #!/bin/bash + + set -e + + echo "starting the swh-web server" + exec gunicorn --bind 0.0.0.0:5004 \ + --threads 2 \ + --workers 2 \ + --timeout 3600 \ + --access-logfile '-' \ + --config 'python:swh.web.gunicorn_config' \ + 'django.core.wsgi:get_wsgi_application()' --- apiVersion: v1 kind: Service metadata: name: memcached spec: type: ClusterIP selector: app: memcached ports: - port: 11211 targetPort: 11211 --- apiVersion: apps/v1 kind: Deployment metadata: name: memcached labels: app: memcached spec: replicas: 1 selector: matchLabels: app: memcached template: metadata: labels: app: memcached spec: containers: - name: memcached image: memcached imagePullPolicy: Always resources: limits: memory: "128Mi" cpu: "100m" --- apiVersion: v1 kind: Service metadata: name: webapp spec: selector: app: webapp ports: - port: 5004 targetPort: 5004 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: webapp labels: name: webapp spec: rules: - host: webapp.default http: paths: - pathType: Prefix path: "/" backend: service: name: webapp port: number: 5004 --- apiVersion: apps/v1 kind: Deployment metadata: name: webapp spec: selector: matchLabels: app: webapp template: metadata: labels: app: webapp spec: + initContainers: + - name: web-init + image: swh/webapp:latest + imagePullPolicy: Always + command: + - /entrypoint.sh + env: + - name: PGHOST + value: "web-db" + - name: PGUSER + valueFrom: + configMapKeyRef: + name: web-db + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: web-db + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + configMapKeyRef: + name: web-db + key: POSTGRES_DB + - name: DJANGO_SETTINGS_MODULE + value: swh.web.settings.production + volumeMounts: + - name: db-password + mountPath: /run/secrets/postgres-password + subPath: POSTGRES_PASSWORD + readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint-init.sh + readOnly: true containers: - name: webapp image: swh/webapp:latest command: - /entrypoint.sh resources: limits: memory: "256Mi" cpu: "500m" ports: - containerPort: 5004 env: - name: PORT value: "5004" - name: VERBOSITY value: "3" - name: DJANGO_SETTINGS_MODULE value: swh.web.settings.production - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true + - name: config + mountPath: /entrypoint.sh + subPath: entrypoint.sh + readOnly: true volumes: - name: config configMap: name: webapp + defaultMode: 0777 + - name: db-password + secret: + secretName: web-db diff --git a/kubernetes/Readme.md b/kubernetes/Readme.md index 5c5891e..b135c85 100644 --- a/kubernetes/Readme.md +++ b/kubernetes/Readme.md @@ -1,201 +1,201 @@ ## Prerequisite ### Directories ``` -# sudo mkdir -p /srv/softwareheritage-kube/dev/{objects,storage-db,scheduler-db,kafka} +# sudo mkdir -p /srv/softwareheritage-kube/dev/{objects,storage-db,scheduler-db,kafka,web-db} ``` Must match the content of `05-storage-db.yaml` ### Registry - Add the following line on your `/etc/hosts` file. It's needed to be able to push the image to it from docker ``` 127.0.0.1 registry.default ``` - Start the registry in kubernetes ``` # cd kubernetes # kubectl apply -f registry/00-registry.yml ``` ## Build the base image ``` # cd docker # docker build --no-cache -t swh/stack . # docker tag swh/stack:latest registry.default/swh/stack:latest # docker push registry.default/swh/stack:latest ``` ## start the objstorage - build image ``` # docker build -f Dockerfile.objstorage -t swh/objstorage --build-arg BASE=swh/stack . # docker tag swh/objstorage:latest registry.default/swh/objstorage:latest # docker push registry.default/swh/objstorage:latest ``` - start the service ``` # cd kubernetes # kubectl apply -f 10-objstorage.yml configmap/objstorage created persistentvolume/objstorage-pv created persistentvolumeclaim/objstorage-pvc created deployment.apps/objstorage created service/objstorage created ``` - test it ``` # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 46m objstorage-8587d58b68-76jbn 1/1 Running 0 12m # kubectl get services objstorage NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE objstorage ClusterIP 10.43.185.191 5003/TCP 17m # curl http://$(kubectl get services objstorage -o jsonpath='{.spec.clusterIP}'):5003 SWH Objstorage API server% ``` ## Start the storage - Start the db ``` # cd kubernetes # kubectl apply -f 05-storage-db.yml persistentvolume/storage-db-pv created persistentvolumeclaim/storage-db-pvc created secret/storage-db created configmap/storage-db created deployment.apps/storage-db created service/storage-db created # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 46m objstorage-8587d58b68-76jbn 1/1 Running 0 15m storage-db-64b7f8b684-48n7w 1/1 Running 0 4m52s # kubectl get services storage-db NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE storage-db ClusterIP 10.43.213.178 5432/TCP 8m19s ``` - Start the storage ``` # cd kubernetes # kubectl apply -f 11-storage.yml configmap/storage created deployment.apps/storage created service/storage created ``` - Test the service ``` # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 49m storage-db-64b7f8b684-48n7w 1/1 Running 0 7m40s storage-6b759fb974-w9rzj 1/1 Running 0 66s # kubectl get services storage NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE storage ClusterIP 10.43.212.116 5002/TCP 2m24s # curl http://$(kubectl get services storage -o jsonpath='{.spec.clusterIP}'):5002 Software Heritage storage server

You have reached the Software Heritage storage server.
See its documentation and API for more information

``` ## Start the scheduler - Start the db ``` # cd kubernetes # kubectl apply -f 15-scheduler-db.yml persistentvolume/scheduler-db-pv unchanged persistentvolumeclaim/scheduler-db-pvc created secret/scheduler-db configured configmap/scheduler-db unchanged deployment.apps/scheduler-db unchanged service/scheduler-db unchanged # kubectl get services scheduler-db NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE scheduler-db ClusterIP 10.43.115.249 5433/TCP 110s ``` - Test the service ``` # kubectl apply -f 20-scheduler.yml configmap/scheduler created deployment.apps/scheduler created service/scheduler created ingress.networking.k8s.io/scheduler created # kubectl get services scheduler NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE scheduler ClusterIP 10.43.218.74 5008/TCP 23s # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-5f6894c5b-9wkmr 1/1 Running 0 28m objstorage-5b87c549b6-f6jvc 1/1 Running 0 12m storage-db-79bfbff68-mg7fr 1/1 Running 0 107s storage-6bfcb87b6-7s7t8 1/1 Running 0 87s scheduler-db-666c8dc8b4-qxm9d 1/1 Running 0 73s scheduler-595f944854-hbsj4 1/1 Running 0 62s # curl http://$(kubectl get services scheduler -o jsonpath='{.spec.clusterIP}'):5008 Software Heritage scheduler RPC server

You have reached the Software Heritage scheduler RPC server.
See its documentation and API for more information

% ``` ## Development To access the services, they must be declared on the `/etc/hosts` file: ``` 127.0.0.1 objstorage.default storage.default webapp.default scheduler.default ``` ### Skaffold To start the development environment using skaffold, use the following command: ``` skaffold --default-repo registry.default dev ``` It will build the images, deploy them on the local registry and start the services. It will monitor the projects to detect the changes and restart the containers when needed diff --git a/skaffold.yaml b/skaffold.yaml index 69d05c8..4fd403f 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -1,51 +1,52 @@ apiVersion: skaffold/v2beta13 kind: Config metadata: name: swh-environment build: artifacts: - image: swh/stack-base docker: dockerfile: Dockerfile - image: swh/objstorage docker: dockerfile: Dockerfile.objstorage requires: - image: swh/stack-base alias: BASE - image: swh/storage docker: dockerfile: Dockerfile.storage requires: - image: swh/stack-base alias: BASE - image: swh/scheduler docker: dockerfile: Dockerfile.scheduler requires: - image: swh/stack-base alias: BASE - image: swh/webapp docker: dockerfile: Dockerfile.webapp requires: - image: swh/stack-base alias: BASE - image: swh/loaders docker: dockerfile: Dockerfile.loaders requires: - image: swh/stack-base alias: BASE deploy: kubectl: manifests: - kubernetes/01-journal.yml - kubernetes/05-storage-db.yml - kubernetes/10-objstorage.yml - kubernetes/11-storage.yml - kubernetes/15-scheduler-db.yml - kubernetes/20-scheduler.yml - kubernetes/25-rabbitmq.yml + - kubernetes/29-web-db.yml - kubernetes/30-webapp.yml - kubernetes/40-loaders.yml