No OneTemporary
Actions

Size

9 KB

Subscribers

None

View Options

	diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py
	index bcaf77c..a840d55 100644
	--- a/swh/core/cli/__init__.py
	+++ b/swh/core/cli/__init__.py
	@@ -1,126 +1,126 @@
	# Copyright (C) 2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import logging
	import logging.config
	-import signal

	import click
	import pkg_resources
	-import yaml
	-
	-from ..sentry import init_sentry

	LOG_LEVEL_NAMES = ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]

	CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])

	logger = logging.getLogger(__name__)


	class AliasedGroup(click.Group):
	"""A simple Group that supports command aliases, as well as notes related to
	options"""

	def __init__(self, name=None, commands=None, **attrs):
	self.option_notes = attrs.pop("option_notes", None)
	self.aliases = {}
	super().__init__(name, commands, **attrs)

	def get_command(self, ctx, cmd_name):
	return super().get_command(ctx, self.aliases.get(cmd_name, cmd_name))

	def add_alias(self, name, alias):
	if not isinstance(name, str):
	name = name.name
	self.aliases[alias] = name

	def format_options(self, ctx, formatter):
	click.Command.format_options(self, ctx, formatter)
	if self.option_notes:
	with formatter.section("Notes"):
	formatter.write_text(self.option_notes)
	self.format_commands(ctx, formatter)


	def clean_exit_on_signal(signal, frame):
	"""Raise a SystemExit exception to let command-line clients wind themselves
	down on exit"""
	raise SystemExit(0)


	@click.group(
	context_settings=CONTEXT_SETTINGS,
	cls=AliasedGroup,
	option_notes="""\
	If both options are present, --log-level will override the root logger
	configuration set in --log-config.

	The --log-config YAML must conform to the logging.config.dictConfig schema
	documented at https://docs.python.org/3/library/logging.config.html.
	""",
	)
	@click.option(
	"--log-level",
	"-l",
	default=None,
	type=click.Choice(LOG_LEVEL_NAMES),
	help="Log level (defaults to INFO).",
	)
	@click.option(
	"--log-config",
	default=None,
	type=click.File("r"),
	help="Python yaml logging configuration file.",
	)
	@click.option(
	"--sentry-dsn", default=None, help="DSN of the Sentry instance to report to"
	)
	@click.option(
	"--sentry-debug/--no-sentry-debug",
	default=False,
	hidden=True,
	help="Enable debugging of sentry",
	)
	@click.pass_context
	def swh(ctx, log_level, log_config, sentry_dsn, sentry_debug):
	"""Command line interface for Software Heritage.
	"""
	+ import signal
	+ import yaml
	+ from ..sentry import init_sentry
	+
	signal.signal(signal.SIGTERM, clean_exit_on_signal)
	signal.signal(signal.SIGINT, clean_exit_on_signal)

	init_sentry(sentry_dsn, debug=sentry_debug)

	if log_level is None and log_config is None:
	log_level = "INFO"

	if log_config:
	logging.config.dictConfig(yaml.safe_load(log_config.read()))

	if log_level:
	log_level = logging.getLevelName(log_level)
	logging.root.setLevel(log_level)

	ctx.ensure_object(dict)
	ctx.obj["log_level"] = log_level


	def main():
	# Even though swh() sets up logging, we need an earlier basic logging setup
	# for the next few logging statements
	logging.basicConfig()
	# load plugins that define cli sub commands
	for entry_point in pkg_resources.iter_entry_points("swh.cli.subcommands"):
	try:
	cmd = entry_point.load()
	swh.add_command(cmd, name=entry_point.name)
	except Exception as e:
	logger.warning("Could not load subcommand %s: %s", entry_point.name, str(e))

	return swh(auto_envvar_prefix="SWH")


	if __name__ == "__main__":
	main()
	diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py
	index 78d3c81..2c89ca9 100755
	--- a/swh/core/cli/db.py
	+++ b/swh/core/cli/db.py
	@@ -1,190 +1,191 @@
	#!/usr/bin/env python3
	# Copyright (C) 2018-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-import glob
	import logging
	from os import path, environ
	-import subprocess
	import warnings

	import click

	from swh.core.cli import CONTEXT_SETTINGS
	-from swh.core.config import read as config_read


	warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t


	logger = logging.getLogger(__name__)


	@click.group(name="db", context_settings=CONTEXT_SETTINGS)
	@click.option(
	"--config-file",
	"-C",
	default=None,
	type=click.Path(exists=True, dir_okay=False),
	help="Configuration file.",
	)
	@click.pass_context
	def db(ctx, config_file):
	"""Software Heritage database generic tools.
	"""
	+ from swh.core.config import read as config_read
	+
	ctx.ensure_object(dict)
	if config_file is None:
	config_file = environ.get("SWH_CONFIG_FILENAME")
	cfg = config_read(config_file)
	ctx.obj["config"] = cfg


	@db.command(name="init", context_settings=CONTEXT_SETTINGS)
	@click.pass_context
	def init(ctx):
	"""Initialize the database for every Software Heritage module found in the
	configuration file. For every configuration section in the config file
	that:

	1. has the name of an existing swh package,
	2. has credentials for a local db access,

	it will run the initialization scripts from the swh package against the
	given database.

	Example for the config file::

	\b
	storage:
	cls: local
	args:
	db: postgresql:///?service=swh-storage
	objstorage:
	cls: remote
	args:
	url: http://swh-objstorage:5003/

	the command:

	swh db -C /path/to/config.yml init

	will initialize the database for the `storage` section using initialization
	scripts from the `swh.storage` package.
	"""
	+ import subprocess

	for modname, cfg in ctx.obj["config"].items():
	if cfg.get("cls") == "local" and cfg.get("args"):
	try:
	sqlfiles = get_sql_for_package(modname)
	except click.BadParameter:
	logger.info(
	"Failed to load/find sql initialization files for %s", modname
	)

	if sqlfiles:
	conninfo = cfg["args"]["db"]
	for sqlfile in sqlfiles:
	subprocess.check_call(
	[
	"psql",
	"--quiet",
	"--no-psqlrc",
	"-v",
	"ON_ERROR_STOP=1",
	"-d",
	conninfo,
	"-f",
	sqlfile,
	]
	)


	@click.command(context_settings=CONTEXT_SETTINGS)
	@click.argument("module", nargs=-1, required=True)
	@click.option(
	"--db-name",
	"-d",
	help="Database name.",
	default="softwareheritage-dev",
	show_default=True,
	)
	@click.option(
	"--create-db/--no-create-db",
	"-C",
	help="Attempt to create the database.",
	default=False,
	)
	def db_init(module, db_name, create_db):
	"""Initialise a database for the Software Heritage <module>. By
	default, does not attempt to create the database.

	Example:

	swh db-init -d swh-test storage

	If you want to specify non-default postgresql connection parameters,
	please provide them using standard environment variables.
	See psql(1) man page (section ENVIRONMENTS) for details.

	Example:

	PGPORT=5434 swh db-init indexer

	"""
	# put import statements here so we can keep startup time of the main swh
	# command as short as possible
	from swh.core.db.tests.db_testing import (
	pg_createdb,
	pg_restore,
	DB_DUMP_TYPES,
	swh_db_version,
	)

	logger.debug("db_init %s dn_name=%s", module, db_name)
	dump_files = []

	for modname in module:
	dump_files.extend(get_sql_for_package(modname))

	if create_db:
	# Create the db (or fail silently if already existing)
	pg_createdb(db_name, check=False)
	# Try to retrieve the db version if any
	db_version = swh_db_version(db_name)
	if not db_version: # Initialize the db
	dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) for x in dump_files]
	for dump, dtype in dump_files:
	click.secho("Loading {}".format(dump), fg="yellow")
	pg_restore(db_name, dump, dtype)

	db_version = swh_db_version(db_name)

	# TODO: Ideally migrate the version from db_version to the latest
	# db version

	click.secho(
	"DONE database is {} version {}".format(db_name, db_version),
	fg="green",
	bold=True,
	)


	def get_sql_for_package(modname):
	+ import glob
	from importlib import import_module
	from swh.core.utils import numfile_sortkey as sortkey

	if not modname.startswith("swh."):
	modname = "swh.{}".format(modname)
	try:
	m = import_module(modname)
	except ImportError:
	raise click.BadParameter("Unable to load module {}".format(modname))

	sqldir = path.join(path.dirname(m.__file__), "sql")
	if not path.isdir(sqldir):
	raise click.BadParameter(
	"Module {} does not provide a db schema " "(no sql/ dir)".format(modname)
	)
	return list(sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey))

File Metadata

Mime Type: text/x-diff
Expires: Jul 4 2025, 6:27 PM (5 w, 1 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3268529

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions