Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/indexer.py
Show First 20 Lines • Show All 435 Lines • ▼ Show 20 Lines | def _index_contents(self, start, end, indexed, **kwargs): | ||||
for sha1 in self._list_contents_to_index(start, end, indexed): | for sha1 in self._list_contents_to_index(start, end, indexed): | ||||
try: | try: | ||||
raw_content = self.objstorage.get(sha1) | raw_content = self.objstorage.get(sha1) | ||||
except ObjNotFoundError: | except ObjNotFoundError: | ||||
self.log.warning('Content %s not found in objstorage' % | self.log.warning('Content %s not found in objstorage' % | ||||
hashutil.hash_to_hex(sha1)) | hashutil.hash_to_hex(sha1)) | ||||
continue | continue | ||||
res = self.index(sha1, raw_content, **kwargs) | res = self.index(sha1, raw_content, **kwargs) | ||||
if res: | |||||
if not isinstance(res['id'], bytes): | if not isinstance(res['id'], bytes): | ||||
raise TypeError( | raise TypeError( | ||||
'%r.index should return ids as bytes, not %r' % | '%r.index should return ids as bytes, not %r' % | ||||
(self.__class__.__name__, res['id'])) | (self.__class__.__name__, res['id'])) | ||||
if res: | |||||
yield res | yield res | ||||
def _index_with_skipping_already_done(self, start, end): | def _index_with_skipping_already_done(self, start, end): | ||||
"""Index not already indexed contents in range [start, end]. | """Index not already indexed contents in range [start, end]. | ||||
Args: | Args: | ||||
start** (Union[bytes, str]): Starting range identifier | start** (Union[bytes, str]): Starting range identifier | ||||
end (Union[bytes, str]): Ending range identifier | end (Union[bytes, str]): Ending range identifier | ||||
▲ Show 20 Lines • Show All 165 Lines • Show Last 20 Lines |