Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/tar/loader.py
| Show All 9 Lines | |||||
| import shutil | import shutil | ||||
| from urllib.parse import urlparse | from urllib.parse import urlparse | ||||
| from tempfile import mkdtemp | from tempfile import mkdtemp | ||||
| from swh.core import tarball | from swh.core import tarball | ||||
| from swh.loader.core.loader import BufferedLoader | from swh.loader.core.loader import BufferedLoader | ||||
| from swh.loader.dir.loader import revision_from, snapshot_from | from swh.loader.dir.loader import revision_from, snapshot_from | ||||
| from swh.model.hashutil import MultiHash | from swh.model.hashutil import MultiHash, HASH_BLOCK_SIZE | ||||
| from swh.model.from_disk import Directory | from swh.model.from_disk import Directory | ||||
| from .build import compute_revision | from .build import compute_revision | ||||
| try: | try: | ||||
| from _version import __version__ | from _version import __version__ | ||||
| except ImportError: | except ImportError: | ||||
| __version__ = 'devel' | __version__ = 'devel' | ||||
| ▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | def download(self, url): | ||||
| raise ValueError("Fail to query '%s'. Reason: %s" % ( | raise ValueError("Fail to query '%s'. Reason: %s" % ( | ||||
| url, response.status_code)) | url, response.status_code)) | ||||
| length = int(response.headers['content-length']) | length = int(response.headers['content-length']) | ||||
| filepath = os.path.join(self.temp_directory, os.path.basename(url)) | filepath = os.path.join(self.temp_directory, os.path.basename(url)) | ||||
| h = MultiHash(length=length) | h = MultiHash(length=length) | ||||
| with open(filepath, 'wb') as f: | with open(filepath, 'wb') as f: | ||||
| for chunk in response.iter_content(chunk_size=None): | for chunk in response.iter_content(chunk_size=HASH_BLOCK_SIZE): | ||||
| h.update(chunk) | h.update(chunk) | ||||
| f.write(chunk) | f.write(chunk) | ||||
| actual_length = os.path.getsize(filepath) | actual_length = os.path.getsize(filepath) | ||||
| if length != actual_length: | if length != actual_length: | ||||
| raise ValueError('Error when checking size: %s != %s' % ( | raise ValueError('Error when checking size: %s != %s' % ( | ||||
| length, actual_length)) | length, actual_length)) | ||||
| ▲ Show 20 Lines • Show All 247 Lines • Show Last 20 Lines | |||||