Differential D874 Diff 2792 swh/indexer/indexer.py

Changeset View

Standalone View

swh/indexer/indexer.py

Show First 20 Lines • Show All 330 Lines • ▼ Show 20 Lines	def run(self, ids, policy_update,
next_step=None, **kwargs):		next_step=None, **kwargs):
"""Given a list of ids:		"""Given a list of ids:

- retrieve the content from the storage		- retrieve the content from the storage
- execute the indexing computations		- execute the indexing computations
- store the results (according to policy_update)		- store the results (according to policy_update)

Args:		Args:
ids ([bytes]): sha1's identifier list		ids (Iterable[Union[bytes, str]]): sha1's identifier list
		olasdUnsubmitted Not Done Inline Actions needs to be fixed olasd: needs to be fixed
policy_update (str): either 'update-dups' or 'ignore-dups' to		policy_update (str): either 'update-dups' or 'ignore-dups' to
respectively update duplicates or ignore		respectively update duplicates or ignore
them		them
next_step (dict): a dict in the form expected by		next_step (dict): a dict in the form expected by
`scheduler.backend.SchedulerBackend.create_tasks`		`scheduler.backend.SchedulerBackend.create_tasks`
without `next_run`, plus an optional `result_name` key.		without `next_run`, plus an optional `result_name` key.
**kwargs: passed to the `index` method		**kwargs: passed to the `index` method

"""		"""
		ids = [hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_
		for id_ in ids]
results = []		results = []
try:		try:
for sha1 in ids:		for sha1 in ids:
try:		try:
raw_content = self.objstorage.get(sha1)		raw_content = self.objstorage.get(sha1)
except ObjNotFoundError:		except ObjNotFoundError:
self.log.warning('Content %s not found in objstorage' %		self.log.warning('Content %s not found in objstorage' %
hashutil.hash_to_hex(sha1))		hashutil.hash_to_hex(sha1))
▲ Show 20 Lines • Show All 263 Lines • Show Last 20 Lines