Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7437453
D6007.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
7 KB
Subscribers
None
D6007.diff
View Options
diff --git a/swh/web/admin/deposit.py b/swh/web/admin/deposit.py
--- a/swh/web/admin/deposit.py
+++ b/swh/web/admin/deposit.py
@@ -1,23 +1,18 @@
-# Copyright (C) 2018-2019 The Software Heritage developers
+# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import requests
-from requests.auth import HTTPBasicAuth
import sentry_sdk
from django.conf import settings
from django.contrib.admin.views.decorators import staff_member_required
-from django.core.cache import cache
from django.core.paginator import Paginator
from django.http import JsonResponse
from django.shortcuts import render
from swh.web.admin.adminurls import admin_route
-from swh.web.config import get_config
-
-config = get_config()["deposit"]
+from swh.web.common.utils import get_deposits_list
@admin_route(r"deposit/", view_name="admin-deposit")
@@ -31,26 +26,9 @@
def _admin_deposit_list(request):
table_data = {}
table_data["draw"] = int(request.GET["draw"])
- deposits_list_url = config["private_api_url"] + "deposits"
- deposits_list_auth = HTTPBasicAuth(
- config["private_api_user"], config["private_api_password"]
- )
try:
- nb_deposits = requests.get(
- "%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30
- ).json()["count"]
-
- deposits_data = cache.get("swh-deposit-list")
- if not deposits_data or deposits_data["count"] != nb_deposits:
- deposits_data = requests.get(
- "%s?page_size=%s" % (deposits_list_url, nb_deposits),
- auth=deposits_list_auth,
- timeout=30,
- ).json()
- cache.set("swh-deposit-list", deposits_data)
-
- deposits = deposits_data["results"]
-
+ deposits = get_deposits_list()
+ deposits_count = len(deposits)
search_value = request.GET["search[value]"]
if search_value:
deposits = [
@@ -85,7 +63,7 @@
page = int(request.GET["start"]) / length + 1
paginator = Paginator(deposits, length)
data = paginator.page(page).object_list
- table_data["recordsTotal"] = deposits_data["count"]
+ table_data["recordsTotal"] = deposits_count
table_data["recordsFiltered"] = len(deposits)
table_data["data"] = [
{
diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py
--- a/swh/web/common/utils.py
+++ b/swh/web/common/utils.py
@@ -6,7 +6,7 @@
from datetime import datetime, timezone
import os
import re
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
from bs4 import BeautifulSoup
from docutils.core import publish_parts
@@ -16,7 +16,10 @@
from iso8601 import ParseError, parse_date
from pkg_resources import get_distribution
from prometheus_client.registry import CollectorRegistry
+import requests
+from requests.auth import HTTPBasicAuth
+from django.core.cache import cache
from django.http import HttpRequest, QueryDict
from django.urls import reverse as django_reverse
@@ -354,3 +357,28 @@
The prettified HTML document
"""
return BeautifulSoup(html, "lxml").prettify()
+
+
+def get_deposits_list() -> List[Dict[str, Any]]:
+ """Return the list of software deposits using swh-deposit API
+ """
+ config = get_config()["deposit"]
+ deposits_list_url = config["private_api_url"] + "deposits"
+ deposits_list_auth = HTTPBasicAuth(
+ config["private_api_user"], config["private_api_password"]
+ )
+
+ nb_deposits = requests.get(
+ "%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30
+ ).json()["count"]
+
+ deposits_data = cache.get("swh-deposit-list")
+ if not deposits_data or deposits_data["count"] != nb_deposits:
+ deposits_data = requests.get(
+ "%s?page_size=%s" % (deposits_list_url, nb_deposits),
+ auth=deposits_list_auth,
+ timeout=30,
+ ).json()
+ cache.set("swh-deposit-list", deposits_data)
+
+ return deposits_data["results"]
diff --git a/swh/web/config.py b/swh/web/config.py
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2020 The Software Heritage developers
+# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -105,7 +105,7 @@
{
"private_api_url": "https://deposit.softwareheritage.org/1/private/",
"private_api_user": "swhworker",
- "private_api_password": "",
+ "private_api_password": "some-password",
},
),
"coverage_count_origins": ("bool", False),
diff --git a/swh/web/tests/common/test_utils.py b/swh/web/tests/common/test_utils.py
--- a/swh/web/tests/common/test_utils.py
+++ b/swh/web/tests/common/test_utils.py
@@ -1,8 +1,9 @@
-# Copyright (C) 2017-2020 The Software Heritage developers
+# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
+from base64 import b64encode
import datetime
from urllib.parse import quote
@@ -14,6 +15,7 @@
from swh.web.common import utils
from swh.web.common.exc import BadInputExc
+from swh.web.config import get_config
def test_shorten_path_noop():
@@ -228,3 +230,58 @@
request = request_factory.get(utils.reverse("sample-test-view-no-url-args"))
url = utils.reverse("sample-test-view-no-url-args", request=request)
assert url == f"http://{request.META['SERVER_NAME']}/sample/test/view/no/url/args/"
+
+
+def test_get_deposits_list(requests_mock):
+ deposits_data = {
+ "count": 2,
+ "results": [
+ {
+ "check_task_id": "351820217",
+ "client": 2,
+ "collection": 1,
+ "complete_date": "2021-01-21T07:52:19.919312Z",
+ "external_id": "hal-03116143",
+ "id": 1412,
+ "load_task_id": "351820260",
+ "origin_url": "https://hal.archives-ouvertes.fr/hal-03116143",
+ "parent": None,
+ "reception_date": "2021-01-21T07:52:19.471019Z",
+ "status": "done",
+ "status_detail": None,
+ "swhid": "swh:1:dir:f25157ad1b13cb20ac3457d4f6756b49ac63d079",
+ },
+ {
+ "check_task_id": "381576507",
+ "client": 2,
+ "collection": 1,
+ "complete_date": "2021-07-07T08:00:44.726676Z",
+ "external_id": "hal-03275052",
+ "id": 1693,
+ "load_task_id": "381576508",
+ "origin_url": "https://hal.archives-ouvertes.fr/hal-03275052",
+ "parent": None,
+ "reception_date": "2021-07-07T08:00:44.327661Z",
+ "status": "done",
+ "status_detail": None,
+ "swhid": "swh:1:dir:825fa96d1810177ec08a772ffa5bd34bbd08b89c",
+ },
+ ],
+ }
+
+ config = get_config()["deposit"]
+ deposits_list_url = config["private_api_url"] + "deposits"
+
+ basic_auth_payload = (
+ config["private_api_user"] + ":" + config["private_api_password"]
+ ).encode()
+
+ requests_mock.get(
+ deposits_list_url,
+ json=deposits_data,
+ request_headers={
+ "Authorization": f"Basic {b64encode(basic_auth_payload).decode('ascii')}"
+ },
+ )
+
+ assert utils.get_deposits_list() == deposits_data["results"]
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Mon, Apr 14, 12:11 AM (1 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3219506
Attached To
D6007: common/utils: Wrap deposits list retrieval in a function
Event Timeline
Log In to Comment