Page MenuHomeSoftware Heritage
Files F7066383

D2774.diff
No OneTemporary
Actions

D2774.diff
View Options

diff --git a/docker/conf/nginx.conf b/docker/conf/nginx.conf
--- a/docker/conf/nginx.conf
+++ b/docker/conf/nginx.conf
@@ -105,5 +105,16 @@
set $upstream "http://swh-web:5004";
proxy_pass $upstream;
}
+ location /keycloak {
+ set $upstream "http://keycloak:8080";
+ proxy_pass $upstream;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Host $host;
+ proxy_set_header X-Forwarded-Server $host;
+ proxy_set_header X-Forwarded-Port $server_port;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ }
}
}
diff --git a/docker/conf/web-keycloak.yml b/docker/conf/web-keycloak.yml
new file mode 100644
--- /dev/null
+++ b/docker/conf/web-keycloak.yml
@@ -0,0 +1,65 @@
+storage:
+ cls: remote
+ args:
+ url: http://swh-storage:5002/
+ timeout: 1
+
+objstorage:
+ cls: remote
+ args:
+ url: http://swh-objstorage:5003/
+
+indexer_storage:
+ cls: remote
+ args:
+ url: http://swh-idx-storage:5007/
+
+scheduler:
+ cls: remote
+ args:
+ url: http://swh-scheduler:5008/
+
+vault:
+ cls: remote
+ args:
+ url: http://swh-vault:5005/
+
+deposit:
+ private_api_url: https://swh-deposit:5006/1/private/
+ private_api_user: swhworker
+ private_api_password: ''
+
+allowed_hosts:
+ - "*"
+
+debug: yes
+
+serve_assets: yes
+
+development_db: /tmp/db.sqlite3
+production_db: /tmp/db.sqlite3
+
+throttling:
+ cache_uri: 127.0.0.1:11211
+ scopes:
+ swh_api:
+ limiter_rate:
+ default: 120/h
+ swh_api_origin_search:
+ limiter_rate:
+ default: 70/m
+ swh_api_origin_visit_latest:
+ limiter_rate:
+ default: 700/m
+ swh_vault_cooking:
+ limiter_rate:
+ default: 120/h
+ swh_save_origin:
+ limiter_rate:
+ default: 120/h
+
+search: {}
+
+keycloak:
+ server_url: http://keycloak:8080/keycloak/auth/
+ realm_name: SoftwareHeritage
diff --git a/docker/docker-compose.keycloak.yml b/docker/docker-compose.keycloak.yml
new file mode 100644
--- /dev/null
+++ b/docker/docker-compose.keycloak.yml
@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+
+ keycloak-db:
+ image: postgres:11
+ env_file:
+ - ./env/keycloak-db.env
+ environment:
+ # unset PGHOST as db service crashes otherwise
+ PGHOST:
+
+ keycloak:
+ build: services/keycloak
+ env_file:
+ - ./env/keycloak.env
+ entrypoint: /entrypoint.sh
+ environment:
+ JAVA_TOOL_OPTIONS: "-Dkeycloak.profile=preview
+ -Dkeycloak.profile.feature.token_exchange=enabled"
+ volumes:
+ - "./services/keycloak/entrypoint.sh:/entrypoint.sh:ro"
+ - "./services/keycloak/keycloak_swh_setup.py:/keycloak_swh_setup.py:ro"
+ expose:
+ - "8080"
+ depends_on:
+ - keycloak-db
+
+ swh-web:
+ depends_on:
+ - keycloak
+ environment:
+ SWH_CONFIG_FILENAME: /web-keycloak.yml
+ volumes:
+ - "./conf/web-keycloak.yml:/web-keycloak.yml:ro"
\ No newline at end of file
diff --git a/docker/env/keycloak-db.env b/docker/env/keycloak-db.env
new file mode 100644
--- /dev/null
+++ b/docker/env/keycloak-db.env
@@ -0,0 +1,4 @@
+PGHOST=keycloak-db
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=testpassword
+POSTGRES_DB=keycloak
\ No newline at end of file
diff --git a/docker/env/keycloak.env b/docker/env/keycloak.env
new file mode 100644
--- /dev/null
+++ b/docker/env/keycloak.env
@@ -0,0 +1,9 @@
+DB_VENDOR=POSTGRES
+DB_ADDR=keycloak-db
+DB_DATABASE=keycloak
+DB_USER=keycloak
+DB_SCHEMA=public
+DB_PASSWORD=testpassword
+KEYCLOAK_USER=admin
+KEYCLOAK_PASSWORD=admin
+PROXY_ADDRESS_FORWARDING=true
diff --git a/docker/services/keycloak/Dockerfile b/docker/services/keycloak/Dockerfile
new file mode 100644
--- /dev/null
+++ b/docker/services/keycloak/Dockerfile
@@ -0,0 +1,19 @@
+FROM jboss/keycloak
+
+USER root
+# install python3 and python-keycloak, this is needed to execute a
+# custom config script after keycloak server is up
+RUN microdnf install -y python3 && microdnf clean all
+RUN pip3 install python-keycloak
+# install wait-for-it script
+RUN curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh > /wait-for-it.sh
+RUN chmod +x /wait-for-it.sh
+
+USER jboss
+# Modify some config files for keycloak to work properly behind a reverse proxy
+# https://stackoverflow.com/questions/44624844/configure-reverse-proxy-for-keycloak-docker-with-custom-base-url
+RUN sed -i -e 's/<web-context>auth<\/web-context>/<web-context>keycloak\/auth<\/web-context>/' $JBOSS_HOME/standalone/configuration/standalone.xml
+RUN sed -i -e 's/<web-context>auth<\/web-context>/<web-context>keycloak\/auth<\/web-context>/' $JBOSS_HOME/standalone/configuration/standalone-ha.xml
+RUN sed -i -e 's/name="\/"/name="\/keycloak\/"/' $JBOSS_HOME/standalone/configuration/standalone.xml
+RUN sed -i -e 's/name="\/"/name="\/keycloak\/"/' $JBOSS_HOME/standalone/configuration/standalone-ha.xml
+RUN sed -i -e 's/\/auth/\/keycloak\/auth"/' $JBOSS_HOME/welcome-content/index.html
diff --git a/docker/services/keycloak/entrypoint.sh b/docker/services/keycloak/entrypoint.sh
new file mode 100755
--- /dev/null
+++ b/docker/services/keycloak/entrypoint.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# turn on bash's job control
+set -m
+
+echo "Starting Keycloak"
+/opt/jboss/tools/docker-entrypoint.sh -b 0.0.0.0&
+echo "Waiting for Keycloak server to be up"
+/wait-for-it.sh localhost:8080 -s --timeout=0
+echo "Configuring Keycloak to be used in docker environment"
+echo "and creating some test users in the SoftwareHeritage realm"
+/keycloak_swh_setup.py
+fg %1
+
diff --git a/docker/services/keycloak/keycloak_swh_setup.py b/docker/services/keycloak/keycloak_swh_setup.py
new file mode 100755
--- /dev/null
+++ b/docker/services/keycloak/keycloak_swh_setup.py
@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from keycloak import KeycloakAdmin
+
+
+server_url = 'http://localhost:8080/keycloak/auth/'
+realm_name = 'SoftwareHeritage'
+
+admin = {
+ 'username': 'admin',
+ 'password': 'admin'
+}
+
+
+def assign_client_base_url(keycloak_admin, client_name, base_url):
+ client_data = {
+ 'baseUrl': base_url,
+ 'clientId': client_name
+ }
+ client_id = keycloak_admin.get_client_id(client_name)
+ keycloak_admin.update_client(client_id, client_data)
+
+
+def assign_client_role_to_user(keycloak_admin, client_name, client_role,
+ username):
+ client_id = keycloak_admin.get_client_id(client_name)
+ staff_user_role = keycloak_admin.get_client_role(client_id, client_role)
+ user_id = keycloak_admin.get_user_id(username)
+ keycloak_admin.assign_client_role(user_id, client_id, staff_user_role)
+
+
+def assign_client_roles_to_user(keycloak_admin, client_name, client_roles,
+ username):
+ for client_role in client_roles:
+ assign_client_role_to_user(keycloak_admin, client_name, client_role,
+ username)
+
+
+def create_user(keycloak_admin, user_data):
+ try:
+ keycloak_admin.create_user(user_data)
+ except Exception:
+ # user already created
+ pass
+
+
+# login as admin in master realm
+keycloak_admin = KeycloakAdmin(server_url,
+ admin['username'],
+ admin['password'])
+
+# update master realm clients base urls as we use a reverse proxy
+assign_client_base_url(
+ keycloak_admin,
+ 'account',
+ '/keycloak/auth/realms/master/account'
+)
+
+assign_client_base_url(
+ keycloak_admin,
+ 'security-admin-console',
+ '/keycloak/auth/admin/master/console/index.html'
+)
+
+# create swh realm
+keycloak_admin.create_realm(payload={
+ 'realm': realm_name,
+ 'rememberMe': True,
+ 'attributes': {
+ 'frontendUrl': 'http://localhost:5080/keycloak/auth/'
+ },
+ 'enabled': True,
+}, skip_exists=True)
+
+# set swh realm name in order to create users in it
+keycloak_admin.realm_name = realm_name
+
+# update swh realm clients base urls as we use a reverse proxy
+assign_client_base_url(
+ keycloak_admin,
+ 'account',
+ f'/keycloak/auth/realms/{realm_name}/account'
+)
+
+assign_client_base_url(
+ keycloak_admin,
+ 'security-admin-console',
+ f'/keycloak/auth/admin/{realm_name}/console/index.html'
+)
+
+# create an admin user in the swh realm
+user_data = {
+ 'email': 'admin@example.org',
+ 'username': admin['username'],
+ 'firstName': admin['username'],
+ 'lastName': admin['username'],
+ 'credentials': [{
+ 'value': admin['username'],
+ 'type': admin['password'],
+ 'temporary': False
+ }],
+ 'enabled': True,
+ 'emailVerified': False,
+}
+
+create_user(keycloak_admin, user_data)
+
+# assign realm admin roles to created user
+realm_management_roles = [
+ 'view-users',
+ 'view-events',
+ 'view-identity-providers',
+ 'manage-identity-providers',
+ 'create-client',
+ 'query-clients',
+ 'query-realms',
+ 'manage-events',
+ 'view-clients',
+ 'manage-realm',
+ 'impersonation',
+ 'manage-clients',
+ 'view-authorization',
+ 'query-users',
+ 'view-realm',
+ 'manage-authorization',
+ 'manage-users',
+ 'query-groups'
+]
+assign_client_roles_to_user(keycloak_admin, 'realm-management',
+ realm_management_roles, admin['username'])
+
+
+# login as admin in swh realm
+keycloak_admin = KeycloakAdmin(server_url,
+ admin['username'],
+ admin['password'],
+ realm_name)
+
+# create swh-web public client
+keycloak_admin.create_client(payload={
+ 'id': 'swh-web',
+ 'clientId': 'swh-web',
+ 'surrogateAuthRequired': False,
+ 'enabled': True,
+ 'redirectUris': [
+ 'http://localhost:5004/*',
+ ],
+ 'bearerOnly': False,
+ 'consentRequired': False,
+ 'standardFlowEnabled': True,
+ 'implicitFlowEnabled': False,
+ 'directAccessGrantsEnabled': True,
+ 'serviceAccountsEnabled': False,
+ 'publicClient': True,
+ 'frontchannelLogout': False,
+ 'protocol': 'openid-connect',
+ 'fullScopeAllowed': True,
+ 'protocolMappers': [
+ {
+ 'name': 'user groups',
+ 'protocol': 'openid-connect',
+ 'protocolMapper': 'oidc-group-membership-mapper',
+ 'consentRequired': False,
+ 'config': {
+ 'full.path': True,
+ 'userinfo.token.claim': True,
+ 'id.token.claim': True,
+ 'access.token.claim': True,
+ 'claim.name': 'groups',
+ 'jsonType.label': 'String'
+ }
+ },
+ {
+ 'name': 'audience',
+ 'protocol': 'openid-connect',
+ 'protocolMapper': 'oidc-audience-mapper',
+ 'consentRequired': False,
+ 'config': {
+ 'included.client.audience': 'swh-web',
+ 'id.token.claim': True,
+ 'access.token.claim': True
+ }
+ }
+ ],
+}, skip_exists=True)
+
+# create staff group
+keycloak_admin.create_group(payload={
+ 'name': 'staff',
+}, skip_exists=True)
+
+groups = keycloak_admin.get_groups()
+
+admin_user_id = keycloak_admin.get_user_id(username=admin['username'])
+
+for group in groups:
+ if group['name'] == 'staff':
+ keycloak_admin.group_user_add(admin_user_id, group['id'])
+ break
+
+# create some test users
+user_data = {
+ 'email': 'john.doe@example.org',
+ 'username': 'johndoe',
+ 'firstName': 'John',
+ 'lastName': 'Doe',
+ 'credentials': [{
+ 'value': 'johndoe-swh',
+ 'type': 'password',
+ 'temporary': False
+ }],
+ 'enabled': True,
+ 'emailVerified': False,
+}
+create_user(keycloak_admin, user_data)
+
+user_data = {
+ 'email': 'jane.doe@example.org',
+ 'username': 'janedoe',
+ 'firstName': 'Jane',
+ 'lastName': 'Doe',
+ 'credentials': [{
+ 'value': 'janedoe-swh',
+ 'type': 'password',
+ 'temporary': False
+ }],
+ 'enabled': True,
+ 'emailVerified': False,
+}
+create_user(keycloak_admin, user_data)
diff --git a/docker/services/swh-web/entrypoint.sh b/docker/services/swh-web/entrypoint.sh
--- a/docker/services/swh-web/entrypoint.sh
+++ b/docker/services/swh-web/entrypoint.sh
@@ -29,7 +29,7 @@
echo "Migrating db using ${DJANGO_SETTINGS_MODULE}"
django-admin migrate --settings=${DJANGO_SETTINGS_MODULE}
- echo "Creating admin user"
+ echo "Creating Django admin user"
echo "$create_admin_script" | python3 -m swh.web.manage shell
echo "starting the swh-web server"

File Metadata

Mime Type
text/plain
Expires
Nov 5 2024, 7:23 AM (8 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217929

Event Timeline

About · Developer information · Legal