diff --git a/swh/web/admin/deposit.py b/swh/web/admin/deposit.py --- a/swh/web/admin/deposit.py +++ b/swh/web/admin/deposit.py @@ -1,23 +1,18 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import requests -from requests.auth import HTTPBasicAuth import sentry_sdk from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required -from django.core.cache import cache from django.core.paginator import Paginator from django.http import JsonResponse from django.shortcuts import render from swh.web.admin.adminurls import admin_route -from swh.web.config import get_config - -config = get_config()["deposit"] +from swh.web.common.utils import get_deposits_list @admin_route(r"deposit/", view_name="admin-deposit") @@ -31,26 +26,9 @@ def _admin_deposit_list(request): table_data = {} table_data["draw"] = int(request.GET["draw"]) - deposits_list_url = config["private_api_url"] + "deposits" - deposits_list_auth = HTTPBasicAuth( - config["private_api_user"], config["private_api_password"] - ) try: - nb_deposits = requests.get( - "%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30 - ).json()["count"] - - deposits_data = cache.get("swh-deposit-list") - if not deposits_data or deposits_data["count"] != nb_deposits: - deposits_data = requests.get( - "%s?page_size=%s" % (deposits_list_url, nb_deposits), - auth=deposits_list_auth, - timeout=30, - ).json() - cache.set("swh-deposit-list", deposits_data) - - deposits = deposits_data["results"] - + deposits = get_deposits_list() + deposits_count = len(deposits) search_value = request.GET["search[value]"] if search_value: deposits = [ @@ -85,7 +63,7 @@ page = int(request.GET["start"]) / length + 1 paginator = Paginator(deposits, length) data = paginator.page(page).object_list - table_data["recordsTotal"] = deposits_data["count"] + table_data["recordsTotal"] = deposits_count table_data["recordsFiltered"] = len(deposits) table_data["data"] = [ { diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone import os import re -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from bs4 import BeautifulSoup from docutils.core import publish_parts @@ -16,7 +16,10 @@ from iso8601 import ParseError, parse_date from pkg_resources import get_distribution from prometheus_client.registry import CollectorRegistry +import requests +from requests.auth import HTTPBasicAuth +from django.core.cache import cache from django.http import HttpRequest, QueryDict from django.urls import reverse as django_reverse @@ -354,3 +357,28 @@ The prettified HTML document """ return BeautifulSoup(html, "lxml").prettify() + + +def get_deposits_list() -> List[Dict[str, Any]]: + """Return the list of software deposits using swh-deposit API + """ + config = get_config()["deposit"] + deposits_list_url = config["private_api_url"] + "deposits" + deposits_list_auth = HTTPBasicAuth( + config["private_api_user"], config["private_api_password"] + ) + + nb_deposits = requests.get( + "%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30 + ).json()["count"] + + deposits_data = cache.get("swh-deposit-list") + if not deposits_data or deposits_data["count"] != nb_deposits: + deposits_data = requests.get( + "%s?page_size=%s" % (deposits_list_url, nb_deposits), + auth=deposits_list_auth, + timeout=30, + ).json() + cache.set("swh-deposit-list", deposits_data) + + return deposits_data["results"] diff --git a/swh/web/config.py b/swh/web/config.py --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2020 The Software Heritage developers +# Copyright (C) 2017-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -105,7 +105,7 @@ { "private_api_url": "https://deposit.softwareheritage.org/1/private/", "private_api_user": "swhworker", - "private_api_password": "", + "private_api_password": "some-password", }, ), "coverage_count_origins": ("bool", False), diff --git a/swh/web/tests/common/test_utils.py b/swh/web/tests/common/test_utils.py --- a/swh/web/tests/common/test_utils.py +++ b/swh/web/tests/common/test_utils.py @@ -1,8 +1,9 @@ -# Copyright (C) 2017-2020 The Software Heritage developers +# Copyright (C) 2017-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from base64 import b64encode import datetime from urllib.parse import quote @@ -14,6 +15,7 @@ from swh.web.common import utils from swh.web.common.exc import BadInputExc +from swh.web.config import get_config def test_shorten_path_noop(): @@ -228,3 +230,58 @@ request = request_factory.get(utils.reverse("sample-test-view-no-url-args")) url = utils.reverse("sample-test-view-no-url-args", request=request) assert url == f"http://{request.META['SERVER_NAME']}/sample/test/view/no/url/args/" + + +def test_get_deposits_list(requests_mock): + deposits_data = { + "count": 2, + "results": [ + { + "check_task_id": "351820217", + "client": 2, + "collection": 1, + "complete_date": "2021-01-21T07:52:19.919312Z", + "external_id": "hal-03116143", + "id": 1412, + "load_task_id": "351820260", + "origin_url": "https://hal.archives-ouvertes.fr/hal-03116143", + "parent": None, + "reception_date": "2021-01-21T07:52:19.471019Z", + "status": "done", + "status_detail": None, + "swhid": "swh:1:dir:f25157ad1b13cb20ac3457d4f6756b49ac63d079", + }, + { + "check_task_id": "381576507", + "client": 2, + "collection": 1, + "complete_date": "2021-07-07T08:00:44.726676Z", + "external_id": "hal-03275052", + "id": 1693, + "load_task_id": "381576508", + "origin_url": "https://hal.archives-ouvertes.fr/hal-03275052", + "parent": None, + "reception_date": "2021-07-07T08:00:44.327661Z", + "status": "done", + "status_detail": None, + "swhid": "swh:1:dir:825fa96d1810177ec08a772ffa5bd34bbd08b89c", + }, + ], + } + + config = get_config()["deposit"] + deposits_list_url = config["private_api_url"] + "deposits" + + basic_auth_payload = ( + config["private_api_user"] + ":" + config["private_api_password"] + ).encode() + + requests_mock.get( + deposits_list_url, + json=deposits_data, + request_headers={ + "Authorization": f"Basic {b64encode(basic_auth_payload).decode('ascii')}" + }, + ) + + assert utils.get_deposits_list() == deposits_data["results"]