Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/loader.py
Show First 20 Lines • Show All 228 Lines • ▼ Show 20 Lines | def fetch_pack_from_origin(self, origin_url, | ||||
if cur_size + would_write > size_limit: | if cur_size + would_write > size_limit: | ||||
raise IOError('Pack file too big for repository %s, ' | raise IOError('Pack file too big for repository %s, ' | ||||
'limit is %d bytes, current size is %d, ' | 'limit is %d bytes, current size is %d, ' | ||||
'would write %d' % | 'would write %d' % | ||||
(origin_url, size_limit, cur_size, would_write)) | (origin_url, size_limit, cur_size, would_write)) | ||||
pack_buffer.write(data) | pack_buffer.write(data) | ||||
remote_refs = client.fetch_pack(path, | pack_result = client.fetch_pack(path, | ||||
base_repo.determine_wants, | base_repo.determine_wants, | ||||
base_repo.graph_walker(), | base_repo.graph_walker(), | ||||
do_pack, | do_pack, | ||||
progress=do_activity).refs | progress=do_activity) | ||||
remote_refs = pack_result.refs | |||||
symbolic_refs = pack_result.symrefs | |||||
if remote_refs: | if remote_refs: | ||||
local_refs = base_repo.find_remote_ref_types_in_swh(remote_refs) | local_refs = base_repo.find_remote_ref_types_in_swh(remote_refs) | ||||
else: | else: | ||||
local_refs = remote_refs = {} | local_refs = remote_refs = {} | ||||
pack_buffer.flush() | pack_buffer.flush() | ||||
pack_size = pack_buffer.tell() | pack_size = pack_buffer.tell() | ||||
pack_buffer.seek(0) | pack_buffer.seek(0) | ||||
return { | return { | ||||
'remote_refs': base_repo.filter_unwanted_refs(remote_refs), | 'remote_refs': base_repo.filter_unwanted_refs(remote_refs), | ||||
'local_refs': local_refs, | 'local_refs': local_refs, | ||||
'symbolic_refs': symbolic_refs, | |||||
'pack_buffer': pack_buffer, | 'pack_buffer': pack_buffer, | ||||
'pack_size': pack_size, | 'pack_size': pack_size, | ||||
} | } | ||||
def list_pack(self, pack_data, pack_size): | def list_pack(self, pack_data, pack_size): | ||||
id_to_type = {} | id_to_type = {} | ||||
type_to_ids = defaultdict(set) | type_to_ids = defaultdict(set) | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | def fetch_data(self): | ||||
self.origin['url'], self.base_snapshot, | self.origin['url'], self.base_snapshot, | ||||
do_progress) | do_progress) | ||||
self.pack_buffer = fetch_info['pack_buffer'] | self.pack_buffer = fetch_info['pack_buffer'] | ||||
self.pack_size = fetch_info['pack_size'] | self.pack_size = fetch_info['pack_size'] | ||||
self.remote_refs = fetch_info['remote_refs'] | self.remote_refs = fetch_info['remote_refs'] | ||||
self.local_refs = fetch_info['local_refs'] | self.local_refs = fetch_info['local_refs'] | ||||
self.symbolic_refs = fetch_info['symbolic_refs'] | |||||
origin_url = self.origin['url'] | origin_url = self.origin['url'] | ||||
self.log.info('Listed %d refs for repo %s' % ( | self.log.info('Listed %d refs for repo %s' % ( | ||||
len(self.remote_refs), origin_url), extra={ | len(self.remote_refs), origin_url), extra={ | ||||
'swh_type': 'git_repo_list_refs', | 'swh_type': 'git_repo_list_refs', | ||||
'swh_repo': origin_url, | 'swh_repo': origin_url, | ||||
'swh_num_refs': len(self.remote_refs), | 'swh_num_refs': len(self.remote_refs), | ||||
▲ Show 20 Lines • Show All 137 Lines • ▼ Show 20 Lines | def get_snapshot(self): | ||||
if not ret_ref['target_type']: | if not ret_ref['target_type']: | ||||
target_type = self.id_to_type[ret_ref['target']] | target_type = self.id_to_type[ret_ref['target']] | ||||
ret_ref['target_type'] = converters.DULWICH_TYPES[target_type] | ret_ref['target_type'] = converters.DULWICH_TYPES[target_type] | ||||
ret_ref['target'] = hashutil.bytehex_to_hash(ret_ref['target']) | ret_ref['target'] = hashutil.bytehex_to_hash(ret_ref['target']) | ||||
branches[ref] = ret_ref | branches[ref] = ret_ref | ||||
for ref, target in self.symbolic_refs.items(): | |||||
branches[ref] = {'target_type': 'alias', 'target': target} | |||||
self.snapshot = converters.branches_to_snapshot(branches) | self.snapshot = converters.branches_to_snapshot(branches) | ||||
return self.snapshot | return self.snapshot | ||||
def get_fetch_history_result(self): | def get_fetch_history_result(self): | ||||
return { | return { | ||||
'contents': len(self.type_to_ids[b'blob']), | 'contents': len(self.type_to_ids[b'blob']), | ||||
'directories': len(self.type_to_ids[b'tree']), | 'directories': len(self.type_to_ids[b'tree']), | ||||
'revisions': len(self.type_to_ids[b'commit']), | 'revisions': len(self.type_to_ids[b'commit']), | ||||
Show All 37 Lines |