Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/mercurial/loader.py
| Show All 31 Lines | |||||
| import billiard | import billiard | ||||
| from swh.model import identifiers | from swh.model import identifiers | ||||
| from swh.model.hashutil import ( | from swh.model.hashutil import ( | ||||
| MultiHash, hash_to_hex, hash_to_bytehex, hash_to_bytes, | MultiHash, hash_to_hex, hash_to_bytehex, hash_to_bytes, | ||||
| DEFAULT_ALGORITHMS | DEFAULT_ALGORITHMS | ||||
| ) | ) | ||||
| from swh.loader.core.loader import DVCSLoader | from swh.loader.core.loader import DVCSLoader | ||||
| from swh.loader.core.converters import content_for_storage | |||||
| from swh.loader.core.utils import clean_dangling_folders | from swh.loader.core.utils import clean_dangling_folders | ||||
| from . import converters | from . import converters | ||||
| from .archive_extract import tmp_extract | from .archive_extract import tmp_extract | ||||
| from .bundle20_reader import Bundle20Reader | from .bundle20_reader import Bundle20Reader | ||||
| from .converters import PRIMARY_ALGO as ALGO | from .converters import PRIMARY_ALGO as ALGO | ||||
| from .objects import SelectiveCache, SimpleTree | from .objects import SelectiveCache, SimpleTree | ||||
| Show All 27 Lines | class HgBundle20Loader(DVCSLoader): | ||||
| visit_type = 'hg' | visit_type = 'hg' | ||||
| def __init__(self, url, visit_date=None, directory=None, | def __init__(self, url, visit_date=None, directory=None, | ||||
| logging_class='swh.loader.mercurial.Bundle20Loader'): | logging_class='swh.loader.mercurial.Bundle20Loader'): | ||||
| super().__init__(logging_class=logging_class) | super().__init__(logging_class=logging_class) | ||||
| self.origin_url = url | self.origin_url = url | ||||
| self.visit_date = visit_date | self.visit_date = visit_date | ||||
| self.directory = directory | self.directory = directory | ||||
| self.content_max_size_limit = self.config['max_content_size'] | |||||
| self.bundle_filename = self.config['bundle_filename'] | self.bundle_filename = self.config['bundle_filename'] | ||||
| self.reduce_effort_flag = self.config['reduce_effort'] | self.reduce_effort_flag = self.config['reduce_effort'] | ||||
| self.empty_repository = None | self.empty_repository = None | ||||
| self.temp_directory = self.config['temp_directory'] | self.temp_directory = self.config['temp_directory'] | ||||
| self.cache1_size = self.config['cache1_size'] | self.cache1_size = self.config['cache1_size'] | ||||
| self.cache2_size = self.config['cache2_size'] | self.cache2_size = self.config['cache2_size'] | ||||
| self.clone_timeout = self.config['clone_timeout_seconds'] | self.clone_timeout = self.config['clone_timeout_seconds'] | ||||
| self.working_directory = None | self.working_directory = None | ||||
| ▲ Show 20 Lines • Show All 243 Lines • ▼ Show 20 Lines | def get_contents(self): | ||||
| group_offset=offset | group_offset=offset | ||||
| ): | ): | ||||
| node = header['node'] | node = header['node'] | ||||
| if node in node_hashes: | if node in node_hashes: | ||||
| blob, meta = self.br.extract_meta_from_blob(data) | blob, meta = self.br.extract_meta_from_blob(data) | ||||
| content = contents.pop(node_hashes[node], None) | content = contents.pop(node_hashes[node], None) | ||||
| if content: | if content: | ||||
| content['data'] = blob | content['data'] = blob | ||||
| yield content_for_storage( | yield content | ||||
| content, | |||||
| log=self.log, | |||||
| max_content_size=self.content_max_size_limit, | |||||
| origin_url=self.origin_url | |||||
| ) | |||||
| def load_directories(self): | def load_directories(self): | ||||
| """This is where the work is done to convert manifest deltas from the | """This is where the work is done to convert manifest deltas from the | ||||
| repository bundle into SWH directories. | repository bundle into SWH directories. | ||||
| """ | """ | ||||
| self.mnode_to_tree_id = {} | self.mnode_to_tree_id = {} | ||||
| cache_hints = self.br.build_manifest_hints() | cache_hints = self.br.build_manifest_hints() | ||||
| ▲ Show 20 Lines • Show All 249 Lines • Show Last 20 Lines | |||||