Changeset View
Changeset View
Standalone View
Standalone View
swh/journal/replay.py
Show First 20 Lines • Show All 199 Lines • ▼ Show 20 Lines | if object_type == 'revision': | ||||
objects = _fix_revisions(objects) | objects = _fix_revisions(objects) | ||||
elif object_type == 'origin_visit': | elif object_type == 'origin_visit': | ||||
objects = _fix_origin_visits(objects) | objects = _fix_origin_visits(objects) | ||||
return objects | return objects | ||||
def _insert_objects(object_type, objects, storage): | def _insert_objects(object_type, objects, storage): | ||||
objects = fix_objects(object_type, objects) | objects = fix_objects(object_type, objects) | ||||
if object_type == 'content': | if object_type == 'content': | ||||
# TODO: insert 'content' in batches | # TODO: insert 'content' in batches | ||||
for object_ in objects: | for object_ in objects: | ||||
try: | try: | ||||
storage.content_add_metadata([object_]) | storage.content_add_metadata([object_]) | ||||
except HashCollision as e: | except HashCollision as e: | ||||
logger.error('Hash collision: %s', e.args) | logger.error('Hash collision: %s', e.args) | ||||
elif object_type in ('directory', 'revision', 'release', | elif object_type in ('directory', 'revision', 'release', | ||||
'snapshot', 'origin'): | 'snapshot', 'origin'): | ||||
# TODO: split batches that are too large for the storage | # TODO: split batches that are too large for the storage | ||||
# to handle? | # to handle? | ||||
method = getattr(storage, object_type + '_add') | method = getattr(storage, object_type + '_add') | ||||
method(objects) | method(objects) | ||||
elif object_type == 'origin_visit': | elif object_type == 'origin_visit': | ||||
for visit in objects: | for visit in objects: | ||||
if 'type' in visit['origin']: | |||||
storage.origin_add_one(visit['origin']) | storage.origin_add_one(visit['origin']) | ||||
if 'metadata' not in visit: | if 'metadata' not in visit: | ||||
visit['metadata'] = None | visit['metadata'] = None | ||||
storage.origin_visit_upsert(objects) | storage.origin_visit_upsert(objects) | ||||
else: | else: | ||||
logger.warning('Received a series of %s, this should not happen', | logger.warning('Received a series of %s, this should not happen', | ||||
object_type) | object_type) | ||||
▲ Show 20 Lines • Show All 161 Lines • Show Last 20 Lines |