Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/dumb.py
Show First 20 Lines • Show All 110 Lines • ▼ Show 20 Lines | def iter_objects(self, object_type: bytes) -> Iterable[ShaFile]: | ||||
A generator fetching git objects on the fly. | A generator fetching git objects on the fly. | ||||
""" | """ | ||||
return map(self._get_git_object, self.objects[object_type]) | return map(self._get_git_object, self.objects[object_type]) | ||||
def _http_get(self, path: str) -> SpooledTemporaryFile: | def _http_get(self, path: str) -> SpooledTemporaryFile: | ||||
url = urllib.parse.urljoin(self.repo_url, path) | url = urllib.parse.urljoin(self.repo_url, path) | ||||
response = self._session.get(url, headers=HEADERS) | response = self._session.get(url, headers=HEADERS) | ||||
buffer = SpooledTemporaryFile(max_size=100 * 1024 * 1024) | buffer = SpooledTemporaryFile(max_size=100 * 1024 * 1024) | ||||
buffer.write(response.content) | for chunk in response.iter_content(chunk_size=10 * 1024 * 1024): | ||||
buffer.write(chunk) | |||||
buffer.flush() | buffer.flush() | ||||
buffer.seek(0) | buffer.seek(0) | ||||
return buffer | return buffer | ||||
def _get_refs(self) -> Dict[bytes, HexBytes]: | def _get_refs(self) -> Dict[bytes, HexBytes]: | ||||
refs = {} | refs = {} | ||||
refs_resp_bytes = self._http_get("info/refs") | refs_resp_bytes = self._http_get("info/refs") | ||||
for ref_line in refs_resp_bytes.readlines(): | for ref_line in refs_resp_bytes.readlines(): | ||||
▲ Show 20 Lines • Show All 69 Lines • Show Last 20 Lines |