Differential D3918 Diff 13849 swh/objstorage/replayer/cli.py

Changeset View

Standalone View

swh/objstorage/replayer/cli.py

# Copyright (C) 2016-2020 The Software Heritage developers		# Copyright (C) 2016-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import functools		# WARNING: do not import unnecessary things here to keep cli startup time under
		# control
import logging		import logging
import mmap

import click		import click

try:		try:
from systemd.daemon import notify		from systemd.daemon import notify
except ImportError:		except ImportError:
notify = None		notify = None

from swh.model.model import SHA1_SIZE		from swh.objstorage.cli import cli as objstorage_cli_group
from swh.journal.client import get_journal_client
from swh.objstorage.cli import cli
from swh.objstorage.factory import get_objstorage

from swh.objstorage.replayer.replay import is_hash_in_bytearray
from swh.objstorage.replayer.replay import process_replay_objects_content


@cli.command("replay")		@objstorage_cli_group.command("replay")
@click.option(		@click.option(
"--stop-after-objects",		"--stop-after-objects",
"-n",		"-n",
default=None,		default=None,
type=int,		type=int,
help="Stop after processing this many objects. Default is to run forever.",		help="Stop after processing this many objects. Default is to run forever.",
)		)
@click.option(		@click.option(
Show All 28 Lines	def content_replay(ctx, stop_after_objects, exclude_sha1_file, check_dst):
and it must be sorted.		and it must be sorted.
This file will not be fully loaded into memory at any given time,		This file will not be fully loaded into memory at any given time,
so it can be arbitrarily large.		so it can be arbitrarily large.

``--check-dst`` sets whether the replayer should check in the destination		``--check-dst`` sets whether the replayer should check in the destination
ObjStorage before copying an object. You can turn that off if you know		ObjStorage before copying an object. You can turn that off if you know
you're copying to an empty ObjStorage.		you're copying to an empty ObjStorage.
"""		"""
		import functools
		import mmap
		from swh.model.model import SHA1_SIZE
		from swh.journal.client import get_journal_client
		from swh.objstorage.factory import get_objstorage
		from swh.objstorage.replayer.replay import is_hash_in_bytearray
		from swh.objstorage.replayer.replay import process_replay_objects_content

conf = ctx.obj["config"]		conf = ctx.obj["config"]
try:		try:
objstorage_src = get_objstorage(**conf.pop("objstorage"))		objstorage_src = get_objstorage(**conf.pop("objstorage"))
except KeyError:		except KeyError:
ctx.fail("You must have a source objstorage configured in " "your config file.")		ctx.fail("You must have a source objstorage configured in " "your config file.")
try:		try:
objstorage_dst = get_objstorage(**conf.pop("objstorage_dst"))		objstorage_dst = get_objstorage(**conf.pop("objstorage_dst"))
except KeyError:		except KeyError:
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	def content_replay(ctx, stop_after_objects, exclude_sha1_file, check_dst):
finally:		finally:
if notify:		if notify:
notify("STOPPING=1")		notify("STOPPING=1")
client.close()		client.close()


def main():		def main():
logging.basicConfig()		logging.basicConfig()
return cli(auto_envvar_prefix="SWH_OBJSTORAGE")		return objstorage_cli_group(auto_envvar_prefix="SWH_OBJSTORAGE")


if __name__ == "__main__":		if __name__ == "__main__":
main()		main()