Changeset View

Standalone View

# See the AUTHORS file at the top-level directory of this distribution

# License: GNU General Public License version 3, or any later version

# See top-level LICENSE file for more information

import copy

import functools

import logging

import os

from typing import Callable, Dict, Optional, Tuple, TypeVar

from urllib.parse import urlparse

import requests

from swh.loader.exception import NotFound

from swh.loader.package import DEFAULT_PARAMS

from swh.model.hashutil import HASH_BLOCK_SIZE, MultiHash

from swh.model.model import Person

) -> Tuple[str, Dict]:

"""

params = copy.deepcopy(DEFAULT_PARAMS)

if auth is not None:

params["auth"] = auth

if extra_request_headers is not None:

params["headers"].update(extra_request_headers)

# so the connection does not hang indefinitely (read/connection timeout)

timeout = params.get("timeout", 60)

if not urlparse(url).scheme:

url = "http://" + url

ardumontUnsubmitted

Not Done

if not urlparse(url).scheme:

- url = "http://" + url

+ url = f"http://{url}"

response = requests.get(url, **params, timeout=timeout, stream=True)

what about https?

ardumont: what about https?

response = requests.get(url, **params, timeout=timeout, stream=True)

if response.status_code != 200:

raise ValueError("Fail to query '%s'. Reason: %s" % (url, response.status_code))

filename = filename if filename else os.path.basename(url)

logger.debug("filename: %s", filename)

filepath = os.path.join(dest, filename)

logger.debug("filepath: %s", filepath)