diff --git a/swh/lister/cli.py b/swh/lister/cli.py --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -63,8 +63,9 @@ if not config_file: config_file = os.environ.get('SWH_CONFIG_FILENAME') conf = config.read(config_file, override_conf) + if not conf: + conf = override_conf ctx.obj['config'] = conf - ctx.obj['override_conf'] = override_conf @lister.command(name='db-init', context_settings=CONTEXT_SETTINGS) diff --git a/swh/lister/core/models.py b/swh/lister/core/models.py --- a/swh/lister/core/models.py +++ b/swh/lister/core/models.py @@ -71,7 +71,6 @@ drop_tables (bool): if True, tables will be dropped before (re)creating them. """ - if drop_tables: logger.info('Dropping tables') SQLBase.metadata.drop_all(db_engine, checkfirst=True) diff --git a/swh/lister/debian/__init__.py b/swh/lister/debian/__init__.py --- a/swh/lister/debian/__init__.py +++ b/swh/lister/debian/__init__.py @@ -6,7 +6,7 @@ from typing import Any, List, Mapping -def debian_init(db_engine, lister=None, +def debian_init(db_engine, override_conf: Mapping[str, Any] = {}, distributions: List[str] = ['stretch', 'buster'], area_names: List[str] = ['main', 'contrib', 'non-free']): @@ -14,7 +14,6 @@ Args: db_engine: SQLAlchemy manipulation database object - lister: Debian lister instance. None by default. override_conf: Override conf to pass to instantiate a lister distributions: Default distribution to build @@ -22,32 +21,29 @@ """ distribution_name = 'Debian' from swh.lister.debian.models import Distribution, Area + from sqlalchemy.orm import sessionmaker + db_session = sessionmaker(bind=db_engine)() + + existing_distrib = db_session \ + .query(Distribution) \ + .filter(Distribution.name == distribution_name) \ + .one_or_none() + if not existing_distrib: + distrib = Distribution(name=distribution_name, + type='deb', + mirror_uri='http://deb.debian.org/debian/') + db_session.add(distrib) - if lister is None: - from .lister import DebianLister - lister = DebianLister(distribution=distribution_name, - override_config=override_conf) - - if not lister.db_session\ - .query(Distribution)\ - .filter(Distribution.name == distribution_name)\ - .one_or_none(): - - d = Distribution( - name=distribution_name, - type='deb', - mirror_uri='http://deb.debian.org/debian/') - lister.db_session.add(d) - - areas = [] for distribution_name in distributions: for area_name in area_names: - areas.append(Area( + area = Area( name='%s/%s' % (distribution_name, area_name), - distribution=d, - )) - lister.db_session.add_all(areas) - lister.db_session.commit() + distribution=distrib, + ) + db_session.add(area) + + db_session.commit() + db_session.close() def register() -> Mapping[str, Any]: diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -15,7 +15,7 @@ lister = swh_listers['debian'] # Initialize the debian data model - debian_init(lister.db_engine, lister=lister, + debian_init(lister.db_engine, distributions=['stretch'], area_names=['main', 'contrib']) diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -98,21 +98,22 @@ def test_task_types(swh_scheduler_config, tmp_path): - db_url = init_db().url() - configfile = tmp_path / 'config.yml' - configfile.write_text(yaml.dump({'scheduler': { - 'cls': 'local', - 'args': swh_scheduler_config}})) + config = { + 'scheduler': { + 'cls': 'local', + 'args': swh_scheduler_config + } + } + configfile.write_text(yaml.dump(config)) runner = CliRunner() result = runner.invoke(cli, [ - '--db-url', db_url, '--config-file', configfile.as_posix(), 'register-task-types']) assert result.exit_code == 0, traceback.print_exception(*result.exc_info) - scheduler = get_scheduler(cls='local', args=swh_scheduler_config) + scheduler = get_scheduler(**config['scheduler']) all_tasks = [ 'list-bitbucket-full', 'list-bitbucket-incremental', 'list-cran', @@ -125,7 +126,7 @@ 'list-phabricator-full', 'list-packagist', 'list-pypi', - ] + ] for task in all_tasks: task_type_desc = scheduler.get_task_type(task) assert task_type_desc