diff --git a/swh/lister/debian/__init__.py b/swh/lister/debian/__init__.py index 6dbb553..e07a179 100644 --- a/swh/lister/debian/__init__.py +++ b/swh/lister/debian/__init__.py @@ -1,54 +1,70 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging + from typing import Any, List, Mapping +logger = logging.getLogger(__name__) + + def debian_init(db_engine, override_conf: Mapping[str, Any] = {}, - distributions: List[str] = ['stretch', 'buster'], - area_names: List[str] = ['main', 'contrib', 'non-free']): + distribution_name: str = 'Debian', + suites: List[str] = ['stretch', 'buster', 'bullseye'], + components: List[str] = ['main', 'contrib', 'non-free']): """Initialize the debian data model. Args: db_engine: SQLAlchemy manipulation database object override_conf: Override conf to pass to instantiate a lister - distributions: Default distribution to build - + distribution_name: Distribution to initialize + suites: Default suites to register with the lister + components: Default components to register per suite """ - distribution_name = 'Debian' from swh.lister.debian.models import Distribution, Area from sqlalchemy.orm import sessionmaker db_session = sessionmaker(bind=db_engine)() - - existing_distrib = db_session \ - .query(Distribution) \ + distrib = db_session.query(Distribution) \ .filter(Distribution.name == distribution_name) \ .one_or_none() - if not existing_distrib: - distrib = Distribution(name=distribution_name, - type='deb', - mirror_uri='http://deb.debian.org/debian/') + + if distrib is None: + distrib = Distribution( + name=distribution_name, type='deb', + mirror_uri='http://deb.debian.org/debian/' + ) db_session.add(distrib) - for distribution_name in distributions: - for area_name in area_names: - area = Area( - name='%s/%s' % (distribution_name, area_name), - distribution=distrib, - ) - db_session.add(area) + # Check the existing + existing_area = db_session.query(Area) \ + .filter(Area.distribution == distrib) \ + .all() + existing_area = set([a.name for a in existing_area]) + + logger.debug('Area already known: %s', ', '.join(existing_area)) + + # Create only the new ones + for suite in suites: + for component in components: + area_name = f'{suite}/{component}' + if area_name in existing_area: + logger.debug("Area '%s' already set, skipping", area_name) + continue + area = Area(name=area_name, distribution=distrib) + db_session.add(area) - db_session.commit() + db_session.commit() db_session.close() def register() -> Mapping[str, Any]: from .lister import DebianLister return {'models': [DebianLister.MODEL], 'lister': DebianLister, 'task_modules': ['%s.tasks' % __name__], 'init': debian_init} diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py index 42a9ab3..8bbc443 100644 --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -1,60 +1,60 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from pytest_postgresql.janitor import DatabaseJanitor from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from swh.lister.core.tests.conftest import * # noqa from swh.lister.core.models import SQLBase from swh.lister.debian import debian_init @pytest.fixture def lister_debian(swh_listers): lister = swh_listers['debian'] # Initialize the debian data model - debian_init(lister.db_engine, - distributions=['stretch'], - area_names=['main', 'contrib']) + debian_init( + lister.db_engine, suites=['stretch'], components=['main', 'contrib'] + ) # Add the load-deb-package in the scheduler backend lister.scheduler.create_task_type({ 'type': 'load-deb-package', 'description': 'Load a Debian package', 'backend_name': 'swh.loader.debian.tasks.LoaderDebianPackage', 'default_interval': '1 day', }) return lister @pytest.fixture def sqlalchemy_engine(postgresql_proc): pg_host = postgresql_proc.host pg_port = postgresql_proc.port pg_user = postgresql_proc.user pg_db = 'sqlalchemy-tests' url = f'postgresql://{pg_user}@{pg_host}:{pg_port}/{pg_db}' with DatabaseJanitor( pg_user, pg_host, pg_port, pg_db, postgresql_proc.version ): engine = create_engine(url) yield engine engine.dispose() @pytest.fixture def session(sqlalchemy_engine): SQLBase.metadata.create_all(sqlalchemy_engine) Session = sessionmaker(bind=sqlalchemy_engine) session = Session() yield session session.close() diff --git a/swh/lister/debian/tests/test_init.py b/swh/lister/debian/tests/test_init.py new file mode 100644 index 0000000..928cfa6 --- /dev/null +++ b/swh/lister/debian/tests/test_init.py @@ -0,0 +1,77 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from swh.lister.debian import debian_init + +from swh.lister.debian.models import Distribution, Area + + +@pytest.fixture +def engine(session): + session.autoflush = False + return session.bind + + +def test_debian_init_step(engine, session): + distribution_name = 'KaliLinux' + + distrib = session.query(Distribution) \ + .filter(Distribution.name == distribution_name) \ + .one_or_none() + assert distrib is None + + all_area = session.query(Area).all() + assert all_area == [] + + suites = ['wheezy', 'jessie'] + components = ['main', 'contrib'] + + debian_init(engine, distribution_name=distribution_name, + suites=suites, components=components) + distrib = session.query(Distribution) \ + .filter(Distribution.name == distribution_name) \ + .one_or_none() + + assert distrib is not None + assert distrib.name == distribution_name + assert distrib.type == 'deb' + assert distrib.mirror_uri == 'http://deb.debian.org/debian/' + + all_area = session.query(Area).all() + assert len(all_area) == 2 * 2, "2 suites * 2 components per suite" + + expected_area_names = [] + for suite in suites: + for component in components: + expected_area_names.append(f'{suite}/{component}') + + for area in all_area: + area.id = None + assert area.distribution == distrib + assert area.name in expected_area_names + + # check idempotency (on exact same call) + + debian_init(engine, distribution_name=distribution_name, + suites=suites, components=components) + + distribs = session.query(Distribution) \ + .filter(Distribution.name == distribution_name) \ + .all() + + assert len(distribs) == 1 + distrib = distribs[0] + + all_area = session.query(Area).all() + assert len(all_area) == 2 * 2, "2 suites * 2 components per suite" + + # Add a new suite + debian_init(engine, distribution_name=distribution_name, + suites=['lenny'], components=components) + + all_area = [a.name for a in session.query(Area).all()] + assert len(all_area) == (2 + 1) * 2, "3 suites * 2 components per suite"