Page MenuHomeSoftware Heritage
Differential D7036

switch to sha256 for hashing & better read performances
ClosedPublic
Actions

Authored by dachary on Jan 25 2022, 4:18 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Commits
rDOBJS0b6bee78ab5c: switch to sha256 for hashing & better read performances
Summary

And maintain a pool of readers to not re-initialize them on every
lookup which horribly hurts performances

Relates to: T3531

Diff Detail

Repository
rDOBJS Object storage
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 26336
Build 41179: Phabricator diff pipeline on jenkinsJenkins console · Jenkins
Build 41178: arc lint + arc unit

Event Timeline

dachary created this revision.Jan 25 2022, 4:18 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptJan 25 2022, 4:18 PM
swh-public-ci added a comment.Jan 25 2022, 4:20 PM

Build is green

Patch application report for D7036 (id=25505)

Rebasing onto 7b5538279b...

Current branch diff-target is up to date.
Changes applied before test
commit 6f8ee3ccb7222f5590f26c21e6983d4896a6d05f
Author: Loïc Dachary <loic@dachary.org>
Date:   Tue Jan 25 15:58:04 2022 +0100

    switch to sha256 for hashing & better read performances
    
    And maintain a pool of readers to not re-initialize them on every
    lookup which horribly hurts performances
    
    Relates to: T3531

See https://jenkins.softwareheritage.org/job/DOBJS/job/tests-on-diff/116/ for more details.

Harbormaster completed remote builds in B26336: Diff 25505.Jan 25 2022, 4:20 PM
dachary requested review of this revision.Jan 25 2022, 4:20 PM
dachary updated this revision to Diff 25516.Jan 25 2022, 6:10 PM

Adjust the disk names on grid5000 to reflect changes at https://www.grid5000.fr/w/Grenoble:Hardware#dahu

swh-public-ci added a comment.Jan 25 2022, 6:13 PM

Build is green

Patch application report for D7036 (id=25516)

Rebasing onto 7b5538279b...

Current branch diff-target is up to date.
Changes applied before test
commit 0b6bee78ab5c4b950fccc38650c22e1d6bc8105e
Author: Loïc Dachary <loic@dachary.org>
Date:   Tue Jan 25 15:58:04 2022 +0100

    switch to sha256 for hashing & better read performances
    
    Maintain a pool of readers to not re-initialize them on every
    lookup which horribly hurts performances.
    
    Adjust the disk names on grid5000 to reflect changes at
    https://www.grid5000.fr/w/Grenoble:Hardware#dahu
    
    Relates to: T3531

See https://jenkins.softwareheritage.org/job/DOBJS/job/tests-on-diff/117/ for more details.

Harbormaster completed remote builds in B26344: Diff 25516.Jan 25 2022, 6:13 PM
ardumont accepted this revision.Jan 26 2022, 9:34 AM
This revision is now accepted and ready to land.Jan 26 2022, 9:34 AM
Closed by commit rDOBJS0b6bee78ab5c: switch to sha256 for hashing & better read performances (authored by dachary). · Explain WhyJan 26 2022, 10:10 AM
This revision was automatically updated to reflect the committed changes.
dachary added a commit: rDOBJS0b6bee78ab5c: switch to sha256 for hashing & better read performances.

Revision Contents
Changeset List

PathSizeCoverage (All)Coverage (Touched)
2 lines--
swh/
objstorage/
backends/
winery/
20 lines73%
Loading...
tests/
10 lines77%
Loading...

Diff 25505

View Options

requirements-swh.txt

# Copyright (C) 2020 The Software Heritage developers # Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version # License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from datetime import datetime, timedelta from datetime import datetime, timedelta
from django.contrib.auth import authenticate, get_backends from django.contrib.auth import authenticate, get_backends
import pytest import pytest
from django.conf import settings from django.conf import settings
from rest_framework.exceptions import AuthenticationFailed
from swh.web.auth.backends import OIDCBearerTokenAuthentication
from swh.web.auth.models import OIDCUser from swh.web.auth.models import OIDCUser
from swh.web.common.utils import reverse from swh.web.common.utils import reverse
from . import sample_data from . import sample_data
from .keycloak_mock import mock_keycloak from .keycloak_mock import mock_keycloak
def _authenticate_user(request_factory): def _authenticate_user(request_factory):
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines
@pytest.mark.django_db @pytest.mark.django_db
def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory): def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory):
mock_keycloak(mocker, auth_success=False) mock_keycloak(mocker, auth_success=False)
user = _authenticate_user(request_factory) user = _authenticate_user(request_factory)
assert user is None assert user is None
@pytest.mark.django_db
def test_drf_oidc_bearer_token_auth_backend_success(mocker,
api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
kc_oidc_mock = mock_keycloak(mocker)
access_token = sample_data.oidc_profile['access_token']
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer {access_token}")
# first authentication
user, _ = drf_auth_backend.authenticate(request)
_check_authenticated_user(user)
# oidc_profile is not filled when authenticating through bearer token
assert hasattr(user, 'access_token') and user.access_token is None
# second authentication, should fetch userinfo from cache
# until token expires
user, _ = drf_auth_backend.authenticate(request)
_check_authenticated_user(user)
assert hasattr(user, 'access_token') and user.access_token is None
# check user request to keycloak has been sent only once
kc_oidc_mock.userinfo.assert_called_once_with(access_token)
@pytest.mark.django_db
def test_drf_oidc_bearer_token_auth_backend_failure(mocker,
api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
# simulate a failed authentication with a bearer token in expected format
mock_keycloak(mocker, auth_success=False)
access_token = sample_data.oidc_profile['access_token']
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer {access_token}")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
# simulate a failed authentication with an invalid bearer token format
mock_keycloak(mocker)
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer invalid-token-format")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
def test_drf_oidc_auth_invalid_or_missing_auth_type(api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
access_token = sample_data.oidc_profile['access_token']
# Invalid authorization type
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Foo token")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
# Missing authorization type
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"{access_token}")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
View Options

swh/objstorage/backends/winery/objstorage.py

# Copyright (C) 2020 The Software Heritage developers # Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version # License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from datetime import datetime, timedelta from datetime import datetime, timedelta
from django.contrib.auth import authenticate, get_backends from django.contrib.auth import authenticate, get_backends
import pytest import pytest
from django.conf import settings from django.conf import settings
from rest_framework.exceptions import AuthenticationFailed
from swh.web.auth.backends import OIDCBearerTokenAuthentication
from swh.web.auth.models import OIDCUser from swh.web.auth.models import OIDCUser
from swh.web.common.utils import reverse from swh.web.common.utils import reverse
from . import sample_data from . import sample_data
from .keycloak_mock import mock_keycloak from .keycloak_mock import mock_keycloak
def _authenticate_user(request_factory): def _authenticate_user(request_factory):
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines
@pytest.mark.django_db @pytest.mark.django_db
def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory): def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory):
mock_keycloak(mocker, auth_success=False) mock_keycloak(mocker, auth_success=False)
user = _authenticate_user(request_factory) user = _authenticate_user(request_factory)
assert user is None assert user is None
@pytest.mark.django_db
def test_drf_oidc_bearer_token_auth_backend_success(mocker,
api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
kc_oidc_mock = mock_keycloak(mocker)
access_token = sample_data.oidc_profile['access_token']
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer {access_token}")
# first authentication
user, _ = drf_auth_backend.authenticate(request)
_check_authenticated_user(user)
# oidc_profile is not filled when authenticating through bearer token
assert hasattr(user, 'access_token') and user.access_token is None
# second authentication, should fetch userinfo from cache
# until token expires
user, _ = drf_auth_backend.authenticate(request)
_check_authenticated_user(user)
assert hasattr(user, 'access_token') and user.access_token is None
# check user request to keycloak has been sent only once
kc_oidc_mock.userinfo.assert_called_once_with(access_token)
@pytest.mark.django_db
def test_drf_oidc_bearer_token_auth_backend_failure(mocker,
api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
# simulate a failed authentication with a bearer token in expected format
mock_keycloak(mocker, auth_success=False)
access_token = sample_data.oidc_profile['access_token']
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer {access_token}")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
# simulate a failed authentication with an invalid bearer token format
mock_keycloak(mocker)
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer invalid-token-format")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
def test_drf_oidc_auth_invalid_or_missing_auth_type(api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
access_token = sample_data.oidc_profile['access_token']
# Invalid authorization type
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Foo token")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
# Missing authorization type
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"{access_token}")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
View Options

swh/objstorage/tests/test_objstorage_winery.py

# Copyright (C) 2020 The Software Heritage developers # Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version # License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from datetime import datetime, timedelta from datetime import datetime, timedelta
from django.contrib.auth import authenticate, get_backends from django.contrib.auth import authenticate, get_backends
import pytest import pytest
from django.conf import settings from django.conf import settings
from rest_framework.exceptions import AuthenticationFailed
from swh.web.auth.backends import OIDCBearerTokenAuthentication
from swh.web.auth.models import OIDCUser from swh.web.auth.models import OIDCUser
from swh.web.common.utils import reverse from swh.web.common.utils import reverse
from . import sample_data from . import sample_data
from .keycloak_mock import mock_keycloak from .keycloak_mock import mock_keycloak
def _authenticate_user(request_factory): def _authenticate_user(request_factory):
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines
@pytest.mark.django_db @pytest.mark.django_db
def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory): def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory):
mock_keycloak(mocker, auth_success=False) mock_keycloak(mocker, auth_success=False)
user = _authenticate_user(request_factory) user = _authenticate_user(request_factory)
assert user is None assert user is None
@pytest.mark.django_db
def test_drf_oidc_bearer_token_auth_backend_success(mocker,
api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
kc_oidc_mock = mock_keycloak(mocker)
access_token = sample_data.oidc_profile['access_token']
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer {access_token}")
# first authentication
user, _ = drf_auth_backend.authenticate(request)
_check_authenticated_user(user)
# oidc_profile is not filled when authenticating through bearer token
assert hasattr(user, 'access_token') and user.access_token is None
# second authentication, should fetch userinfo from cache
# until token expires
user, _ = drf_auth_backend.authenticate(request)
_check_authenticated_user(user)
assert hasattr(user, 'access_token') and user.access_token is None
# check user request to keycloak has been sent only once
kc_oidc_mock.userinfo.assert_called_once_with(access_token)
@pytest.mark.django_db
def test_drf_oidc_bearer_token_auth_backend_failure(mocker,
api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
# simulate a failed authentication with a bearer token in expected format
mock_keycloak(mocker, auth_success=False)
access_token = sample_data.oidc_profile['access_token']
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer {access_token}")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
# simulate a failed authentication with an invalid bearer token format
mock_keycloak(mocker)
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Bearer invalid-token-format")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
def test_drf_oidc_auth_invalid_or_missing_auth_type(api_request_factory):
url = reverse('api-1-stat-counters')
drf_auth_backend = OIDCBearerTokenAuthentication()
access_token = sample_data.oidc_profile['access_token']
# Invalid authorization type
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"Foo token")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
# Missing authorization type
request = api_request_factory.get(
url, HTTP_AUTHORIZATION=f"{access_token}")
with pytest.raises(AuthenticationFailed):
drf_auth_backend.authenticate(request)
About · Developer information · Legal