Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7066384
D2134.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
6 KB
Subscribers
None
D2134.diff
View Options
diff --git a/swh/lister/debian/__init__.py b/swh/lister/debian/__init__.py
--- a/swh/lister/debian/__init__.py
+++ b/swh/lister/debian/__init__.py
@@ -1,14 +1,32 @@
-# Copyright (C) 2019 the Software Heritage developers
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+from typing import Any, Mapping, Optional
-def debian_init(db_engine, override_conf=None):
+
+def debian_init(db_engine, lister=None,
+ override_conf: Optional[Mapping[str, Any]] = None,
+ distributions: Optional[str] = ['stretch', 'buster'],
+ area_names: Optional[str] = ['main', 'contrib', 'non-free']):
+ """Initialize the debian data model.
+
+ Args:
+ db_engine: SQLAlchemy manipulation database object
+ lister: Debian lister instance. None by default.
+ override_conf: Override conf to pass to instantiate a lister.
+ None by default
+ distributions: Default distribution to build
+
+
+ """
from swh.storage.schemata.distribution import (
Distribution, Area)
- from .lister import DebianLister
- lister = DebianLister(override_config=override_conf)
+ if lister is None:
+ from .lister import DebianLister
+ lister = DebianLister(override_config=override_conf)
if not lister.db_session\
.query(Distribution)\
@@ -22,8 +40,8 @@
lister.db_session.add(d)
areas = []
- for distribution_name in ['stretch', 'buster']:
- for area_name in ['main', 'contrib', 'non-free']:
+ for distribution_name in distributions:
+ for area_name in area_names:
areas.append(Area(
name='%s/%s' % (distribution_name, area_name),
distribution=d,
@@ -32,7 +50,7 @@
lister.db_session.commit()
-def register():
+def register() -> Mapping[str, Any]:
from .lister import DebianLister
return {'models': [DebianLister.MODEL],
'lister': DebianLister,
diff --git a/swh/lister/debian/lister.py b/swh/lister/debian/lister.py
--- a/swh/lister/debian/lister.py
+++ b/swh/lister/debian/lister.py
@@ -1,4 +1,5 @@
-# Copyright (C) 2017 the Software Heritage developers
+# Copyright (C) 2017-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -28,6 +29,9 @@
}
+logger = logging.getLogger(__name__)
+
+
class DebianLister(ListerHttpTransport, ListerBase):
MODEL = Package
PATH_TEMPLATE = None
@@ -209,8 +213,8 @@
date = date or datetime.datetime.now(tz=datetime.timezone.utc)
- logging.debug('Creating snapshot for distribution %s on date %s' %
- (distribution, date))
+ logger.debug('Creating snapshot for distribution %s on date %s' %
+ (distribution, date))
snapshot = DistributionSnapshot(date=date, distribution=distribution)
@@ -222,7 +226,7 @@
self.area = area
- logging.debug('Processing area %s' % area)
+ logger.debug('Processing area %s' % area)
_, new_area_packages = self.ingest_data(None)
area_snapshot = AreaSnapshot(snapshot=snapshot, area=area)
diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py
--- a/swh/lister/debian/tests/conftest.py
+++ b/swh/lister/debian/tests/conftest.py
@@ -1 +1,30 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
from swh.lister.core.tests.conftest import * # noqa
+
+from swh.lister.debian import debian_init
+
+
+@pytest.fixture
+def lister_debian(swh_listers):
+ lister = swh_listers['debian']
+
+ # Initialize the debian data model
+ debian_init(lister.db_engine, lister=lister,
+ distributions=['stretch'],
+ area_names=['main', 'contrib'])
+
+ # Add the load-deb-package in the scheduler backend
+ lister.scheduler.create_task_type({
+ 'type': 'load-deb-package',
+ 'description': 'Load a Debian package',
+ 'backend_name': 'swh.loader.debian.tasks.LoaderDebianPackage',
+ 'default_interval': '1 day',
+ })
+
+ return lister
diff --git a/swh/lister/debian/tests/data/http_deb.debian.org/debian__dists_stretch_contrib_source_Sources.xz b/swh/lister/debian/tests/data/http_deb.debian.org/debian__dists_stretch_contrib_source_Sources.xz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/lister/debian/tests/data/http_deb.debian.org/debian__dists_stretch_main_source_Sources.xz b/swh/lister/debian/tests/data/http_deb.debian.org/debian__dists_stretch_main_source_Sources.xz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/lister/debian/tests/test_lister.py b/swh/lister/debian/tests/test_lister.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/debian/tests/test_lister.py
@@ -0,0 +1,35 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+def test_lister_debian(lister_debian, datadir, requests_mock_datadir):
+ """Simple debian listing should create scheduled tasks
+
+ """
+ # Run the lister
+ lister_debian.run(distribution="Debian")
+
+ r = lister_debian.scheduler.search_tasks(task_type='load-deb-package')
+ assert len(r) == 151
+
+ for row in r:
+ assert row['type'] == 'load-deb-package'
+ # arguments check
+ args = row['arguments']['args']
+ assert len(args) == 0
+
+ # kwargs
+ kwargs = row['arguments']['kwargs']
+ assert set(kwargs.keys()) == {'origin', 'date', 'packages'}
+
+ logger.debug('kwargs: %s', kwargs)
+
+ assert row['policy'] == 'oneshot'
+ assert row['priority'] is None
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Nov 5 2024, 7:25 AM (8 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3215809
Attached To
D2134: debian.lister: Add integration test which checks scheduled tasks
Event Timeline
Log In to Comment