Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/utils.py
| Show First 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | if url.startswith("ftp://"): | ||||
| chunks = (response.read(HASH_BLOCK_SIZE) for _ in itertools.count()) | chunks = (response.read(HASH_BLOCK_SIZE) for _ in itertools.count()) | ||||
| response_data = itertools.takewhile(bool, chunks) | response_data = itertools.takewhile(bool, chunks) | ||||
| else: | else: | ||||
| response = requests.get(url, **params, timeout=timeout, stream=True) | response = requests.get(url, **params, timeout=timeout, stream=True) | ||||
| if response.status_code != 200: | if response.status_code != 200: | ||||
| raise ValueError( | raise ValueError( | ||||
| "Fail to query '%s'. Reason: %s" % (url, response.status_code) | "Fail to query '%s'. Reason: %s" % (url, response.status_code) | ||||
| ) | ) | ||||
| # update URL to response one as requests follow redirection by default | |||||
| # on GET requests | |||||
| url = response.url | |||||
| response_data = response.iter_content(chunk_size=HASH_BLOCK_SIZE) | response_data = response.iter_content(chunk_size=HASH_BLOCK_SIZE) | ||||
| filename = filename if filename else os.path.basename(url) | filename = filename if filename else os.path.basename(url) | ||||
olasd: The first one will match until the next double quote. The second one will stop at the first… | |||||
| logger.debug("filename: %s", filename) | logger.debug("filename: %s", filename) | ||||
| filepath = os.path.join(dest, filename) | filepath = os.path.join(dest, filename) | ||||
| logger.debug("filepath: %s", filepath) | logger.debug("filepath: %s", filepath) | ||||
| h = MultiHash(hash_names=DOWNLOAD_HASHES) | h = MultiHash(hash_names=DOWNLOAD_HASHES) | ||||
| with open(filepath, "wb") as f: | with open(filepath, "wb") as f: | ||||
| for chunk in response_data: | for chunk in response_data: | ||||
| h.update(chunk) | h.update(chunk) | ||||
| ▲ Show 20 Lines • Show All 55 Lines • Show Last 20 Lines | |||||
The first one will match until the next double quote. The second one will stop at the first space in the header (which may happen depending how broken it is).