Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 135 Lines • ▼ Show 20 Lines | DEFAULT_CONFIG = { | ||||
| 'directory_packet_size': ('int', 25000), | 'directory_packet_size': ('int', 25000), | ||||
| 'revision_packet_size': ('int', 100000), | 'revision_packet_size': ('int', 100000), | ||||
| 'release_packet_size': ('int', 100000), | 'release_packet_size': ('int', 100000), | ||||
| 'occurrence_packet_size': ('int', 100000), | 'occurrence_packet_size': ('int', 100000), | ||||
| } | } | ||||
| ADDITIONAL_CONFIG = {} | ADDITIONAL_CONFIG = {} | ||||
| def __init__(self, logging_class, config=None): | def __init__(self, logging_class=None, config=None): | ||||
| if config: | if config: | ||||
| self.config = config | self.config = config | ||||
| else: | else: | ||||
| self.config = self.parse_config_file( | self.config = self.parse_config_file( | ||||
| additional_configs=[self.ADDITIONAL_CONFIG]) | additional_configs=[self.ADDITIONAL_CONFIG]) | ||||
| self.storage = get_storage(**self.config['storage']) | self.storage = get_storage(**self.config['storage']) | ||||
| if logging_class is None: | |||||
| logging_class = '%s.%s' % (self.__class__.__module__, | |||||
| self.__class__.__name__) | |||||
| self.log = logging.getLogger(logging_class) | self.log = logging.getLogger(logging_class) | ||||
| self.contents = QueuePerSizeAndNbUniqueElements( | self.contents = QueuePerSizeAndNbUniqueElements( | ||||
| key='sha1', | key='sha1', | ||||
| max_nb_elements=self.config['content_packet_size'], | max_nb_elements=self.config['content_packet_size'], | ||||
| max_size=self.config['content_packet_size_bytes']) | max_size=self.config['content_packet_size_bytes']) | ||||
| self.contents_seen = set() | self.contents_seen = set() | ||||
| ▲ Show 20 Lines • Show All 846 Lines • Show Last 20 Lines | |||||