diff --git a/swh/loader/package/nixguix/loader.py b/swh/loader/package/nixguix/loader.py --- a/swh/loader/package/nixguix/loader.py +++ b/swh/loader/package/nixguix/loader.py @@ -5,7 +5,6 @@ import json import logging -import requests from typing import Any, Dict, Iterator, Mapping, Optional, Tuple import attr @@ -22,7 +21,7 @@ Sha1Git, ) -from swh.loader.package.utils import EMPTY_AUTHOR +from swh.loader.package.utils import EMPTY_AUTHOR, api_info from swh.loader.package.loader import ( BasePackageInfo, PackageLoader, @@ -207,11 +206,7 @@ def retrieve_sources(url: str) -> bytes: - response = requests.get(url, allow_redirects=True) - if response.status_code != 200: - raise ValueError("Got %d HTTP code on %s", response.status_code, url) - - return response.content + return api_info(url, allow_redirects=True) def parse_sources(raw_sources: bytes) -> Dict[str, Any]: diff --git a/swh/loader/package/npm/loader.py b/swh/loader/package/npm/loader.py --- a/swh/loader/package/npm/loader.py +++ b/swh/loader/package/npm/loader.py @@ -96,7 +96,7 @@ """ if not self._info: - self._info = api_info(self.provider_url) + self._info = json.loads(api_info(self.provider_url)) return self._info def get_versions(self) -> Sequence[str]: diff --git a/swh/loader/package/pypi/loader.py b/swh/loader/package/pypi/loader.py --- a/swh/loader/package/pypi/loader.py +++ b/swh/loader/package/pypi/loader.py @@ -1,10 +1,11 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import os import logging +import json +import os from typing import Any, Dict, Iterator, Optional, Sequence, Tuple from urllib.parse import urlparse @@ -63,7 +64,7 @@ """ if not self._info: - self._info = api_info(self.provider_url) + self._info = json.loads(api_info(self.provider_url)) return self._info def get_versions(self) -> Sequence[str]: diff --git a/swh/loader/package/tests/test_utils.py b/swh/loader/package/tests/test_utils.py --- a/swh/loader/package/tests/test_utils.py +++ b/swh/loader/package/tests/test_utils.py @@ -1,12 +1,13 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import json import os -import pytest +import pytest import swh.loader.package from swh.loader.package.utils import download, api_info, release_name @@ -144,7 +145,7 @@ """Fetching json info from pypi project should be ok""" url = "https://pypi.org/pypi/requests/json" requests_mock.get(url, text='{"version": "0.0.1"}') - actual_info = api_info(url) + actual_info = json.loads(api_info(url)) assert actual_info == { "version": "0.0.1", } diff --git a/swh/loader/package/utils.py b/swh/loader/package/utils.py --- a/swh/loader/package/utils.py +++ b/swh/loader/package/utils.py @@ -25,7 +25,7 @@ EMPTY_AUTHOR = Person(fullname=b"", name=None, email=None,) -def api_info(url: str) -> Dict: +def api_info(url: str, **extra_params) -> bytes: """Basic api client to retrieve information on project. This deals with fetching json metadata about pypi projects. @@ -36,13 +36,13 @@ ValueError in case of query failures (for some reasons: 404, ...) Returns: - The associated response's information dict + The associated response's information """ - response = requests.get(url, **DEFAULT_PARAMS) + response = requests.get(url, **{**DEFAULT_PARAMS, **extra_params}) if response.status_code != 200: raise ValueError("Fail to query '%s'. Reason: %s" % (url, response.status_code)) - return response.json() + return response.content def download(