diff --git a/swh/lister/debian/__init__.py b/swh/lister/debian/__init__.py --- a/swh/lister/debian/__init__.py +++ b/swh/lister/debian/__init__.py @@ -1,14 +1,32 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Any, Mapping, Optional -def debian_init(db_engine, override_conf=None): + +def debian_init(db_engine, lister=None, + override_conf: Optional[Mapping[str, Any]] = None, + distributions: Optional[str] = ['stretch', 'buster'], + area_names: Optional[str] = ['main', 'contrib', 'non-free']): + """Initialize the debian data model. + + Args: + db_engine: SQLAlchemy manipulation database object + lister: Debian lister instance. None by default. + override_conf: Override conf to pass to instantiate a lister. + None by default + distributions: Default distribution to build + + + """ from swh.storage.schemata.distribution import ( Distribution, Area) - from .lister import DebianLister - lister = DebianLister(override_config=override_conf) + if lister is None: + from .lister import DebianLister + lister = DebianLister(override_config=override_conf) if not lister.db_session\ .query(Distribution)\ @@ -22,8 +40,8 @@ lister.db_session.add(d) areas = [] - for distribution_name in ['stretch', 'buster']: - for area_name in ['main', 'contrib', 'non-free']: + for distribution_name in distributions: + for area_name in area_names: areas.append(Area( name='%s/%s' % (distribution_name, area_name), distribution=d, @@ -32,7 +50,7 @@ lister.db_session.commit() -def register(): +def register() -> Mapping[str, Any]: from .lister import DebianLister return {'models': [DebianLister.MODEL], 'lister': DebianLister, diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py --- a/swh/lister/debian/lister.py +++ b/swh/lister/debian/lister.py @@ -1,4 +1,5 @@ -# Copyright (C) 2017 the Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -28,6 +29,9 @@ } +logger = logging.getLogger(__name__) + + class DebianLister(ListerHttpTransport, ListerBase): MODEL = Package PATH_TEMPLATE = None @@ -209,8 +213,8 @@ date = date or datetime.datetime.now(tz=datetime.timezone.utc) - logging.debug('Creating snapshot for distribution %s on date %s' % - (distribution, date)) + logger.debug('Creating snapshot for distribution %s on date %s' % + (distribution, date)) snapshot = DistributionSnapshot(date=date, distribution=distribution) @@ -222,7 +226,7 @@ self.area = area - logging.debug('Processing area %s' % area) + logger.debug('Processing area %s' % area) _, new_area_packages = self.ingest_data(None) area_snapshot = AreaSnapshot(snapshot=snapshot, area=area) diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -1 +1,30 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + from swh.lister.core.tests.conftest import * # noqa + +from swh.lister.debian import debian_init + + +@pytest.fixture +def lister_debian(swh_listers): + lister = swh_listers['debian'] + + # Initialize the debian data model + debian_init(lister.db_engine, lister=lister, + distributions=['stretch'], + area_names=['main', 'contrib']) + + # Add the load-deb-package in the scheduler backend + lister.scheduler.create_task_type({ + 'type': 'load-deb-package', + 'description': 'Load a Debian package', + 'backend_name': 'swh.loader.debian.tasks.LoaderDebianPackage', + 'default_interval': '1 day', + }) + + return lister diff --git a/swh/lister/debian/tests/data/http_deb.debian.org/debian__dists_stretch_contrib_source_Sources.xz b/swh/lister/debian/tests/data/http_deb.debian.org/debian__dists_stretch_contrib_source_Sources.xz new file mode 100644 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@