diff --git a/docker/conf/nginx.conf b/docker/conf/nginx.conf index b851484..b0f7d38 100644 --- a/docker/conf/nginx.conf +++ b/docker/conf/nginx.conf @@ -1,109 +1,120 @@ worker_processes 1; # Show startup logs on stderr; switch to debug to print, well, debug logs when # running nginx-debug error_log /dev/stderr info; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; client_max_body_size 100M; # Built-in Docker resolver. Needed to allow on-demand resolution of proxy # upstreams. resolver 127.0.0.11 valid=30s; server { listen 5080 default_server; # Add a trailing slash to top level requests (e.g. http://localhost:5080/flower) rewrite ^/([^/]+)$ /$1/ permanent; # In this pile of proxies, all upstreams are set using a variable. This # makes nginx DNS-resolve the name of the upstream when clients request # them, rather than on start. This avoids an unstarted container preventing # nginx from starting. # # Variables need to be set as early as possible, as they're statements from # the rewrite module and `rewrite [...] break;` will prevent these # statements from being executed. location /flower/ { set $upstream "http://flower:5555"; rewrite ^/flower/(.*)$ /$1 break; proxy_pass $upstream; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; proxy_redirect off; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; } location /rabbitmq/ { set $upstream "http://amqp:15672"; rewrite ^ $request_uri; rewrite ^/rabbitmq(/.*)$ $1 break; proxy_pass $upstream$uri; } location /scheduler { set $upstream "http://swh-scheduler:5008"; rewrite ^/scheduler/(.*)$ /$1 break; proxy_pass $upstream; } location /storage { set $upstream "http://swh-storage:5002"; rewrite ^/storage/(.*)$ /$1 break; proxy_pass $upstream; } location /indexer-storage { set $upstream "http://swh-idx-storage:5007"; rewrite ^/indexer-storage/(.*)$ /$1 break; proxy_pass $upstream; } location /deposit { set $upstream "http://swh-deposit:5006"; rewrite ^/deposit/(.*)$ /deposit/$1 break; proxy_pass $upstream; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Host $host:5080; proxy_set_header SCRIPT_NAME /deposit; proxy_redirect off; } location /objstorage { set $upstream "http://swh-objstorage:5003"; rewrite ^/objstorage/(.*)$ /$1 break; proxy_pass $upstream; } location /prometheus { set $upstream "http://prometheus:9090"; proxy_pass $upstream; } location /grafana { set $upstream "http://grafana:3000"; rewrite ^/grafana/(.*)$ /$1 break; proxy_pass $upstream; } location / { set $upstream "http://swh-web:5004"; proxy_pass $upstream; } + location /keycloak { + set $upstream "http://keycloak:8080"; + proxy_pass $upstream; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Proto $scheme; + } } } diff --git a/docker/conf/web-keycloak.yml b/docker/conf/web-keycloak.yml new file mode 100644 index 0000000..40a5f54 --- /dev/null +++ b/docker/conf/web-keycloak.yml @@ -0,0 +1,65 @@ +storage: + cls: remote + args: + url: http://swh-storage:5002/ + timeout: 1 + +objstorage: + cls: remote + args: + url: http://swh-objstorage:5003/ + +indexer_storage: + cls: remote + args: + url: http://swh-idx-storage:5007/ + +scheduler: + cls: remote + args: + url: http://swh-scheduler:5008/ + +vault: + cls: remote + args: + url: http://swh-vault:5005/ + +deposit: + private_api_url: https://swh-deposit:5006/1/private/ + private_api_user: swhworker + private_api_password: '' + +allowed_hosts: + - "*" + +debug: yes + +serve_assets: yes + +development_db: /tmp/db.sqlite3 +production_db: /tmp/db.sqlite3 + +throttling: + cache_uri: 127.0.0.1:11211 + scopes: + swh_api: + limiter_rate: + default: 120/h + swh_api_origin_search: + limiter_rate: + default: 70/m + swh_api_origin_visit_latest: + limiter_rate: + default: 700/m + swh_vault_cooking: + limiter_rate: + default: 120/h + swh_save_origin: + limiter_rate: + default: 120/h + +search: {} + +keycloak: + server_url: http://keycloak:8080/keycloak/auth/ + realm_name: SoftwareHeritage diff --git a/docker/docker-compose.keycloak.yml b/docker/docker-compose.keycloak.yml new file mode 100644 index 0000000..785dedb --- /dev/null +++ b/docker/docker-compose.keycloak.yml @@ -0,0 +1,35 @@ +version: '2' + +services: + + keycloak-db: + image: postgres:11 + env_file: + - ./env/keycloak-db.env + environment: + # unset PGHOST as db service crashes otherwise + PGHOST: + + keycloak: + build: services/keycloak + env_file: + - ./env/keycloak.env + entrypoint: /entrypoint.sh + environment: + JAVA_TOOL_OPTIONS: "-Dkeycloak.profile=preview + -Dkeycloak.profile.feature.token_exchange=enabled" + volumes: + - "./services/keycloak/entrypoint.sh:/entrypoint.sh:ro" + - "./services/keycloak/keycloak_swh_setup.py:/keycloak_swh_setup.py:ro" + expose: + - "8080" + depends_on: + - keycloak-db + + swh-web: + depends_on: + - keycloak + environment: + SWH_CONFIG_FILENAME: /web-keycloak.yml + volumes: + - "./conf/web-keycloak.yml:/web-keycloak.yml:ro" \ No newline at end of file diff --git a/docker/env/keycloak-db.env b/docker/env/keycloak-db.env new file mode 100644 index 0000000..a543508 --- /dev/null +++ b/docker/env/keycloak-db.env @@ -0,0 +1,4 @@ +PGHOST=keycloak-db +POSTGRES_USER=keycloak +POSTGRES_PASSWORD=testpassword +POSTGRES_DB=keycloak \ No newline at end of file diff --git a/docker/env/keycloak.env b/docker/env/keycloak.env new file mode 100644 index 0000000..cf0cca0 --- /dev/null +++ b/docker/env/keycloak.env @@ -0,0 +1,9 @@ +DB_VENDOR=POSTGRES +DB_ADDR=keycloak-db +DB_DATABASE=keycloak +DB_USER=keycloak +DB_SCHEMA=public +DB_PASSWORD=testpassword +KEYCLOAK_USER=admin +KEYCLOAK_PASSWORD=admin +PROXY_ADDRESS_FORWARDING=true diff --git a/docker/services/keycloak/Dockerfile b/docker/services/keycloak/Dockerfile new file mode 100644 index 0000000..717085d --- /dev/null +++ b/docker/services/keycloak/Dockerfile @@ -0,0 +1,19 @@ +FROM jboss/keycloak + +USER root +# install python3 and python-keycloak, this is needed to execute a +# custom config script after keycloak server is up +RUN microdnf install -y python3 && microdnf clean all +RUN pip3 install python-keycloak +# install wait-for-it script +RUN curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh > /wait-for-it.sh +RUN chmod +x /wait-for-it.sh + +USER jboss +# Modify some config files for keycloak to work properly behind a reverse proxy +# https://stackoverflow.com/questions/44624844/configure-reverse-proxy-for-keycloak-docker-with-custom-base-url +RUN sed -i -e 's/auth<\/web-context>/keycloak\/auth<\/web-context>/' $JBOSS_HOME/standalone/configuration/standalone.xml +RUN sed -i -e 's/auth<\/web-context>/keycloak\/auth<\/web-context>/' $JBOSS_HOME/standalone/configuration/standalone-ha.xml +RUN sed -i -e 's/name="\/"/name="\/keycloak\/"/' $JBOSS_HOME/standalone/configuration/standalone.xml +RUN sed -i -e 's/name="\/"/name="\/keycloak\/"/' $JBOSS_HOME/standalone/configuration/standalone-ha.xml +RUN sed -i -e 's/\/auth/\/keycloak\/auth"/' $JBOSS_HOME/welcome-content/index.html diff --git a/docker/services/keycloak/entrypoint.sh b/docker/services/keycloak/entrypoint.sh new file mode 100755 index 0000000..5ccea4c --- /dev/null +++ b/docker/services/keycloak/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# turn on bash's job control +set -m + +echo "Starting Keycloak" +/opt/jboss/tools/docker-entrypoint.sh -b 0.0.0.0& +echo "Waiting for Keycloak server to be up" +/wait-for-it.sh localhost:8080 -s --timeout=0 +echo "Configuring Keycloak to be used in docker environment" +echo "and creating some test users in the SoftwareHeritage realm" +/keycloak_swh_setup.py +fg %1 + diff --git a/docker/services/keycloak/keycloak_swh_setup.py b/docker/services/keycloak/keycloak_swh_setup.py new file mode 100755 index 0000000..dac1acf --- /dev/null +++ b/docker/services/keycloak/keycloak_swh_setup.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from keycloak import KeycloakAdmin + + +server_url = 'http://localhost:8080/keycloak/auth/' +realm_name = 'SoftwareHeritage' + +admin = { + 'username': 'admin', + 'password': 'admin' +} + + +def assign_client_base_url(keycloak_admin, client_name, base_url): + client_data = { + 'baseUrl': base_url, + 'clientId': client_name + } + client_id = keycloak_admin.get_client_id(client_name) + keycloak_admin.update_client(client_id, client_data) + + +def assign_client_role_to_user(keycloak_admin, client_name, client_role, + username): + client_id = keycloak_admin.get_client_id(client_name) + staff_user_role = keycloak_admin.get_client_role(client_id, client_role) + user_id = keycloak_admin.get_user_id(username) + keycloak_admin.assign_client_role(user_id, client_id, staff_user_role) + + +def assign_client_roles_to_user(keycloak_admin, client_name, client_roles, + username): + for client_role in client_roles: + assign_client_role_to_user(keycloak_admin, client_name, client_role, + username) + + +def create_user(keycloak_admin, user_data): + try: + keycloak_admin.create_user(user_data) + except Exception: + # user already created + pass + + +# login as admin in master realm +keycloak_admin = KeycloakAdmin(server_url, + admin['username'], + admin['password']) + +# update master realm clients base urls as we use a reverse proxy +assign_client_base_url( + keycloak_admin, + 'account', + '/keycloak/auth/realms/master/account' +) + +assign_client_base_url( + keycloak_admin, + 'security-admin-console', + '/keycloak/auth/admin/master/console/index.html' +) + +# create swh realm +keycloak_admin.create_realm(payload={ + 'realm': realm_name, + 'rememberMe': True, + 'attributes': { + 'frontendUrl': 'http://localhost:5080/keycloak/auth/' + }, + 'enabled': True, +}, skip_exists=True) + +# set swh realm name in order to create users in it +keycloak_admin.realm_name = realm_name + +# update swh realm clients base urls as we use a reverse proxy +assign_client_base_url( + keycloak_admin, + 'account', + f'/keycloak/auth/realms/{realm_name}/account' +) + +assign_client_base_url( + keycloak_admin, + 'security-admin-console', + f'/keycloak/auth/admin/{realm_name}/console/index.html' +) + +# create an admin user in the swh realm +user_data = { + 'email': 'admin@example.org', + 'username': admin['username'], + 'firstName': admin['username'], + 'lastName': admin['username'], + 'credentials': [{ + 'value': admin['username'], + 'type': admin['password'], + 'temporary': False + }], + 'enabled': True, + 'emailVerified': False, +} + +create_user(keycloak_admin, user_data) + +# assign realm admin roles to created user +realm_management_roles = [ + 'view-users', + 'view-events', + 'view-identity-providers', + 'manage-identity-providers', + 'create-client', + 'query-clients', + 'query-realms', + 'manage-events', + 'view-clients', + 'manage-realm', + 'impersonation', + 'manage-clients', + 'view-authorization', + 'query-users', + 'view-realm', + 'manage-authorization', + 'manage-users', + 'query-groups' +] +assign_client_roles_to_user(keycloak_admin, 'realm-management', + realm_management_roles, admin['username']) + + +# login as admin in swh realm +keycloak_admin = KeycloakAdmin(server_url, + admin['username'], + admin['password'], + realm_name) + +# create swh-web public client +keycloak_admin.create_client(payload={ + 'id': 'swh-web', + 'clientId': 'swh-web', + 'surrogateAuthRequired': False, + 'enabled': True, + 'redirectUris': [ + 'http://localhost:5004/*', + ], + 'bearerOnly': False, + 'consentRequired': False, + 'standardFlowEnabled': True, + 'implicitFlowEnabled': False, + 'directAccessGrantsEnabled': True, + 'serviceAccountsEnabled': False, + 'publicClient': True, + 'frontchannelLogout': False, + 'protocol': 'openid-connect', + 'fullScopeAllowed': True, + 'protocolMappers': [ + { + 'name': 'user groups', + 'protocol': 'openid-connect', + 'protocolMapper': 'oidc-group-membership-mapper', + 'consentRequired': False, + 'config': { + 'full.path': True, + 'userinfo.token.claim': True, + 'id.token.claim': True, + 'access.token.claim': True, + 'claim.name': 'groups', + 'jsonType.label': 'String' + } + }, + { + 'name': 'audience', + 'protocol': 'openid-connect', + 'protocolMapper': 'oidc-audience-mapper', + 'consentRequired': False, + 'config': { + 'included.client.audience': 'swh-web', + 'id.token.claim': True, + 'access.token.claim': True + } + } + ], +}, skip_exists=True) + +# create staff group +keycloak_admin.create_group(payload={ + 'name': 'staff', +}, skip_exists=True) + +groups = keycloak_admin.get_groups() + +admin_user_id = keycloak_admin.get_user_id(username=admin['username']) + +for group in groups: + if group['name'] == 'staff': + keycloak_admin.group_user_add(admin_user_id, group['id']) + break + +# create some test users +user_data = { + 'email': 'john.doe@example.org', + 'username': 'johndoe', + 'firstName': 'John', + 'lastName': 'Doe', + 'credentials': [{ + 'value': 'johndoe-swh', + 'type': 'password', + 'temporary': False + }], + 'enabled': True, + 'emailVerified': False, +} +create_user(keycloak_admin, user_data) + +user_data = { + 'email': 'jane.doe@example.org', + 'username': 'janedoe', + 'firstName': 'Jane', + 'lastName': 'Doe', + 'credentials': [{ + 'value': 'janedoe-swh', + 'type': 'password', + 'temporary': False + }], + 'enabled': True, + 'emailVerified': False, +} +create_user(keycloak_admin, user_data) diff --git a/docker/services/swh-web/entrypoint.sh b/docker/services/swh-web/entrypoint.sh index 0e9102a..b0c82b0 100755 --- a/docker/services/swh-web/entrypoint.sh +++ b/docker/services/swh-web/entrypoint.sh @@ -1,42 +1,42 @@ #!/bin/bash set -e create_admin_script=" from django.contrib.auth import get_user_model; username = 'admin'; password = 'admin'; email = 'admin@swh-web.org'; User = get_user_model(); if not User.objects.filter(username = username).exists(): User.objects.create_superuser(username, email, password); " source /srv/softwareheritage/utils/pyutils.sh setup_pip case "$1" in "shell") exec bash -i ;; *) echo "Starting memcached" memcached& echo "Migrating db using ${DJANGO_SETTINGS_MODULE}" django-admin migrate --settings=${DJANGO_SETTINGS_MODULE} - echo "Creating admin user" + echo "Creating Django admin user" echo "$create_admin_script" | python3 -m swh.web.manage shell echo "starting the swh-web server" exec gunicorn --bind 0.0.0.0:5004 \ --threads 2 \ --workers 2 \ --timeout 3600 \ --config 'python:swh.web.gunicorn_config' \ 'django.core.wsgi:get_wsgi_application()' esac