Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/indexer.py
Show First 20 Lines • Show All 570 Lines • ▼ Show 20 Lines | def run(self, ids, policy_update='update-dups', parse_ids=True, | ||||
parse_ids (bool): Do we need to parse id or not (default) | parse_ids (bool): Do we need to parse id or not (default) | ||||
**kwargs: passed to the `index` method | **kwargs: passed to the `index` method | ||||
""" | """ | ||||
if parse_ids: | if parse_ids: | ||||
ids = [o.split('+', 1) if ':' in o else int(o) # type+url or id | ids = [o.split('+', 1) if ':' in o else int(o) # type+url or id | ||||
for o in ids] | for o in ids] | ||||
origins = [] | origins_filtered = [] | ||||
for id_ in ids: | origins = self.storage.origin_get( | ||||
origin = self.storage.origin_get(origin_get_params(id_)) | [origin_get_params(id_) for id_ in ids]) | ||||
ardumont: You do not need the zip call, you can use origin['id'].
that might also avoid any problem in… | |||||
Done Inline Actionsorigin can be None. vlorentz: `origin` can be `None`. | |||||
Done Inline Actionsok, so the zip call is for logging the missing origin id. ardumont: ok, so the zip call is for logging the missing origin id. | |||||
for (id_, origin) in zip(ids, origins): | |||||
if not origin: | if not origin: | ||||
self.log.warning('Origin %s not found in storage' % | self.log.warning('Origin %s not found in storage' % | ||||
id_) | id_) | ||||
continue | continue | ||||
origins.append(origin) | origins_filtered.append(origin) | ||||
results = self.index_list(origins, **kwargs) | results = self.index_list(origins_filtered, **kwargs) | ||||
self.persist_index_computations(results, policy_update) | self.persist_index_computations(results, policy_update) | ||||
self.results = results | self.results = results | ||||
return self.next_step(results, task=next_step) | return self.next_step(results, task=next_step) | ||||
def index_list(self, origins, **kwargs): | def index_list(self, origins, **kwargs): | ||||
results = [] | results = [] | ||||
for origin in origins: | for origin in origins: | ||||
▲ Show 20 Lines • Show All 54 Lines • Show Last 20 Lines |
You do not need the zip call, you can use origin['id'].
that might also avoid any problem in regards of inconsistency regarding the list's order