Changeset View
Changeset View
Standalone View
Standalone View
swh/journal/replay.py
# Copyright (C) 2019 The Software Heritage developers | # Copyright (C) 2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import logging | import logging | ||||
from swh.storage import HashCollision | from swh.storage import HashCollision | ||||
from swh.model.hashutil import hash_to_hex | |||||
from swh.objstorage.objstorage import ID_HASH_ALGO | from swh.objstorage.objstorage import ID_HASH_ALGO | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
def process_replay_objects(all_objects, *, storage): | def process_replay_objects(all_objects, *, storage): | ||||
for (object_type, objects) in all_objects.items(): | for (object_type, objects) in all_objects.items(): | ||||
Show All 27 Lines | |||||
def process_replay_objects_content(all_objects, *, src, dst): | def process_replay_objects_content(all_objects, *, src, dst): | ||||
for (object_type, objects) in all_objects.items(): | for (object_type, objects) in all_objects.items(): | ||||
if object_type != 'content': | if object_type != 'content': | ||||
logger.warning('Received a series of %s, this should not happen', | logger.warning('Received a series of %s, this should not happen', | ||||
object_type) | object_type) | ||||
continue | continue | ||||
logger.info('processing %s content objects', len(objects)) | |||||
for obj in objects: | for obj in objects: | ||||
if obj['status'] == 'visible': | |||||
obj_id = obj[ID_HASH_ALGO] | obj_id = obj[ID_HASH_ALGO] | ||||
if obj['status'] == 'visible': | |||||
try: | |||||
obj = src.get(obj_id) | obj = src.get(obj_id) | ||||
dst.add(obj, obj_id=obj_id) | dst.add(obj, obj_id=obj_id) | ||||
logger.debug('copied %s', hash_to_hex(obj_id)) | |||||
except Exception: | |||||
logger.exception('Failed to copy %s', hash_to_hex(obj_id)) | |||||
vlorentz: `logger.exception('Failed to copy %s', hash_to_hex(obj_id))` | |||||
else: | |||||
logger.debug('skipped %s (%s)', | |||||
hash_to_hex(obj_id), obj['status']) |
logger.exception('Failed to copy %s', hash_to_hex(obj_id))