diff --git a/Dockerfile b/Dockerfile index c93c2f8..b98d7a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,60 +1,60 @@ ### # Base image mother of all FROM python:3.7 as swh-base RUN . /etc/os-release && echo "deb http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ libapr1-dev \ libaprutil1-dev \ libpq-dev \ libsvn-dev \ libsystemd-dev \ memcached \ postgresql-client-12 \ wait-for-it \ ngrep \ rsync && \ apt-get install -y --no-install-recommends \ r-base-core \ r-cran-jsonlite && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN useradd -md /srv/softwareheritage -s /bin/bash swh RUN mkdir /app WORKDIR /app USER swh RUN python3 -m venv /srv/softwareheritage/venv ENV PATH="/srv/softwareheritage/venv/bin:${PATH}" RUN pip install --upgrade pip setuptools wheel RUN pip install flask gunicorn httpie decorator aiohttp_utils blinker ### # swh-packages FROM swh-base as swh-packages # Install command line # RUN pip install swh-core # Install requirements.txt only if there is any changes + # COPY swh-model/requirements.txt /app/requirements-model.txt # COPY swh-core/requirements.txt /app/requirements-core.txt # COPY swh-journal/requirements.txt /app/requirements-journal.txt - -# RUN cat /app/requirements-*.txt > /tmp/requirements.txt && pip install -r /tmp/requirements.txt \ +# RUN cat /app/requirements-*.txt > /tmp/requirements.txt && pip install -r /tmp/requirements.txt # && rm /srv/requirements* # Copy source code COPY swh-model /app/swh-model COPY swh-core /app/swh-core COPY swh-journal /app/swh-journal RUN pip install swh-model && pip install swh-core && pip install swh-journal diff --git a/kubernetes/05-storage-db.yml b/kubernetes/05-storage-db.yml index f842e17..48637bd 100644 --- a/kubernetes/05-storage-db.yml +++ b/kubernetes/05-storage-db.yml @@ -1,131 +1,131 @@ --- apiVersion: v1 kind: PersistentVolume metadata: name: storage-db-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: storage-db local: path: /srv/softwareheritage-kube/dev/storage-db nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: storage-db-pvc spec: accessModes: - ReadWriteOnce storageClassName: storage-db resources: requests: storage: 10Gi --- ## TODO Change this to your real postgresql password apiVersion: v1 kind: Secret metadata: name: storage-db type: Opaque # data: # POSTGRES_PASSWORD: | # "echo 'strong password' | base64" stringData: POSTGRES_PASSWORD: swh --- apiVersion: v1 kind: ConfigMap metadata: name: storage-db data: # property-like keys; each key maps to a simple value POSTGRES_USER: swh POSTGRES_DB: swh --- apiVersion: apps/v1 kind: Deployment metadata: name: storage-db labels: app: storage-db spec: replicas: 1 selector: matchLabels: app: storage-db template: metadata: labels: app: storage-db spec: containers: - name: storage-db image: postgres:13.0 imagePullPolicy: Always ports: - containerPort: 5432 args: - "-c" - "shared_buffers=512MB" - - "-c" + - "-c" - "effective_cache_size=512MB" - "-c" - "random_page_cost=1.5" - "-c" - "max_wal_size=512MB" env: - name: POSTGRES_USER valueFrom: configMapKeyRef: name: storage-db key: POSTGRES_USER - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: storage-db key: POSTGRES_PASSWORD - name: POSTGRES_DB valueFrom: configMapKeyRef: name: storage-db key: POSTGRES_DB volumeMounts: - mountPath: "/var/lib/postgresql" name: storage-db-pvc resources: requests: memory: "512Mi" cpu: "250m" limits: memory: "1024Mi" cpu: "500m" volumes: - name: storage-db-pvc persistentVolumeClaim: claimName: storage-db-pvc --- apiVersion: v1 kind: Service metadata: name: storage-db spec: type: ClusterIP selector: app: storage-db ports: - port: 5432 targetPort: 5432 diff --git a/kubernetes/10-objstorage.yml b/kubernetes/10-objstorage.yml index 062db97..5c6b1ca 100644 --- a/kubernetes/10-objstorage.yml +++ b/kubernetes/10-objstorage.yml @@ -1,162 +1,162 @@ --- apiVersion: v1 kind: ConfigMap metadata: name: objstorage data: config.yml: | objstorage: cls: pathslicing args: root: "/srv/softwareheritage/objects" slicing: 0:5 client_max_size: 1073741824 entrypoint.sh: | #!/bin/bash set -e # source /srv/softwareheritage/utils/pyutils.sh # setup_pip # echo Installed Python packages: # pip list echo Starting the swh-objstorage API server exec gunicorn --bind 0.0.0.0:5003 \ --worker-class aiohttp.worker.GunicornWebWorker \ --log-level DEBUG \ --threads 4 \ --workers 2 \ --reload \ --timeout 3600 \ --config 'python:swh.core.api.gunicorn_config' \ 'swh.objstorage.api.server:make_app_from_configfile()' --- apiVersion: v1 kind: PersistentVolume metadata: name: objstorage-pv spec: capacity: storage: 10Gi volumeMode: Filesystem accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete storageClassName: objstorage-pv local: path: /srv/softwareheritage-kube/objects nodeAffinity: required: nodeSelectorTerms: - matchExpressions: # TODO adapt for your needs - key: kubernetes.io/os operator: In values: - linux --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: objstorage-pvc spec: accessModes: - ReadWriteOnce storageClassName: objstorage-pv resources: requests: storage: 10Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: objstorage labels: app: objstorage spec: replicas: 1 selector: matchLabels: app: objstorage strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 template: metadata: labels: app: objstorage spec: containers: - name: objstorage image: swh/objstorage:latest - command: + command: - /entrypoint.sh ports: - containerPort: 5003 env: - name: PORT value: "5003" - name: STATSD_HOST value: "prometheus-statsd-exporter" - name: STATSD_PORT value: "9125" - name: SWH_CONFIG_FILENAME value: /etc/softwareheritage/config.yml volumeMounts: - mountPath: "/srv/softwareheritage/objects" name: objstorage-pvc - name: config mountPath: /etc/softwareheritage/config.yml subPath: config.yml readOnly: true - name: config mountPath: /entrypoint.sh subPath: entrypoint.sh readOnly: true resources: requests: memory: "512Mi" cpu: "250m" limits: memory: "1024Mi" cpu: "500m" volumes: - name: config configMap: name: objstorage defaultMode: 0777 - name: objstorage-pvc persistentVolumeClaim: claimName: objstorage-pvc --- apiVersion: v1 kind: Service metadata: name: objstorage spec: type: ClusterIP selector: app: objstorage ports: - port: 5003 targetPort: 5003 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: objstorage spec: rules: - host: objstorage.default http: paths: - path: / pathType: Prefix backend: service: name: objstorage - port: + port: number: 5003 diff --git a/kubernetes/11-storage.yml b/kubernetes/11-storage.yml index 47b32ca..0562916 100644 --- a/kubernetes/11-storage.yml +++ b/kubernetes/11-storage.yml @@ -1,218 +1,216 @@ ---- +-- apiVersion: v1 kind: ConfigMap metadata: name: storage data: config.yml: | storage: cls: local db: postgresql:///?service=swh objstorage: cls: remote url: http://objstorage:5003/ journal_writer: cls: kafka args: brokers: - kafka prefix: swh.journal.objects client_id: swh.storage.master entrypoint-init-db.sh: | #!/bin/bash set -e echo -n "waiting for database availability" set +e ko=1 while [[ $ko -gt 0 ]]; do PGCONNECT_TIMEOUT=2 psql --list &>/dev/null;ko=$?; echo -n .; sleep 0.5; done echo set -e echo Init swh-storage database echo Creating extensions... swh db init-admin --db-name ${PGDATABASE} storage echo Initializing the database... swh db init --db-name ${PGDATABASE} storage entrypoint.sh: | #!/bin/bash set -e echo "${PGHOST}:5432:${PGDATABASE}:${PGUSER}:${PGPASSWORD}" >> ~/.pgpass cat >> ~/.pg_service.conf < 5003/TCP 17m # curl http://$(kubectl get services objstorage -o jsonpath='{.spec.clusterIP}'):5003 -SWH Objstorage API server% +SWH Objstorage API server% ``` ## Start the storage - Start the db ``` # cd kubernetes # kubectl apply -f 02-storage-db.yml persistentvolume/storage-db-pv created persistentvolumeclaim/storage-db-pvc created secret/storage-db created configmap/storage-db created deployment.apps/storage-db created service/storage-db created # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 46m objstorage-8587d58b68-76jbn 1/1 Running 0 15m storage-db-64b7f8b684-48n7w 1/1 Running 0 4m52s # kubectl get services storage-db NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE storage-db ClusterIP 10.43.213.178 5432/TCP 8m19s ``` - Start the storage ``` # cd kubernetes # kubectl apply -f 03-storage.yml configmap/storage created deployment.apps/storage created service/storage created ``` - Test the service ``` # kubectl get pods NAME READY STATUS RESTARTS AGE registry-deployment-7595868dc8-657ps 1/1 Running 0 49m storage-db-64b7f8b684-48n7w 1/1 Running 0 7m40s storage-6b759fb974-w9rzj 1/1 Running 0 66s # kubectl get services storage NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE storage ClusterIP 10.43.212.116 5002/TCP 2m24s # curl http://$(kubectl get services storage -o jsonpath='{.spec.clusterIP}'):5002 Software Heritage storage server

You have reached the Software Heritage storage server.
See its documentation and API for more information

``` ## Development ### Skaffold To start the development environment using skaffold, use the following command: ``` skaffold --default-repo registry.default dev ``` It will build the images, deploy them on the local registry and start the services. It will monitor the projects to detect the changes and restart the containers when needed diff --git a/kubernetes/registry/00-registry.yml b/kubernetes/registry/00-registry.yml index 9bd7f00..31b3fdc 100644 --- a/kubernetes/registry/00-registry.yml +++ b/kubernetes/registry/00-registry.yml @@ -1,51 +1,51 @@ --- apiVersion: v1 kind: Service metadata: name: registry spec: selector: app: registry ports: - protocol: TCP port: 80 targetPort: 5000 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: registry spec: rules: - host: registry.default http: paths: - path: / pathType: Prefix backend: service: name: registry - port: + port: number: 80 --- apiVersion: apps/v1 kind: Deployment metadata: name: registry-deployment labels: app: registry spec: replicas: 1 selector: matchLabels: app: registry template: metadata: labels: app: registry spec: containers: - name: registry image: registry:latest ports: - containerPort: 5000