diff --git a/debian/control b/debian/control index 492cc59..3a27e84 100644 --- a/debian/control +++ b/debian/control @@ -1,33 +1,34 @@ Source: swh-loader-debian Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python (>= 2), dpkg-dev, python3-all, python3-debian, python3-nose, + python3-requests-mock, python3-setuptools, python3-sqlalchemy, python3-swh.core, python3-swh.loader.core (>= 0.0.37), python3-swh.model (>= 0.0.27), python3-swh.scheduler (>= 0.0.14), python3-swh.storage (>= 0.0.31), python3-swh.storage.schemata, python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLDDEB/ Package: python3-swh.loader.debian Architecture: all Depends: dpkg-dev, python3-swh.loader.core (>= 0.0.37), python3-swh.model (>= 0.0.27), python3-swh.scheduler (>= 0.0.14), python3-swh.storage (>= 0.0.31), python3-swh.storage.schemata, ${misc:Depends}, ${python3:Depends} Description: Software Heritage Debian Loader diff --git a/requirements-test.txt b/requirements-test.txt index e079f8a..05a082c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,3 @@ pytest +requests-mock +sqlalchemy diff --git a/swh/loader/debian/tests/__init__.py b/swh/loader/debian/tests/__init__.py index e69de29..9ef4c60 100644 --- a/swh/loader/debian/tests/__init__.py +++ b/swh/loader/debian/tests/__init__.py @@ -0,0 +1,27 @@ +import os + +TEST_LOADER_CONFIG = { + 'storage': { + 'cls': 'memory', + 'args': { + } + }, + 'send_contents': True, + 'send_directories': True, + 'send_revisions': True, + 'send_releases': True, + 'send_snapshot': True, + + 'content_size_limit': 100 * 1024 * 1024, + 'content_packet_size': 10, + 'content_packet_size_bytes': 100 * 1024 * 1024, + 'directory_packet_size': 10, + 'revision_packet_size': 10, + 'release_packet_size': 10, + + 'save_data': False, + + 'lister_db_url': + 'postgresql+psycopg2:///test-lister-debian?host={PGHOST}'.format( + **os.environ) +} diff --git a/swh/loader/debian/tests/resources/hello_2.10-1+deb9u1.debian.tar.xz b/swh/loader/debian/tests/resources/hello_2.10-1+deb9u1.debian.tar.xz new file mode 100644 index 0000000..54655dd Binary files /dev/null and b/swh/loader/debian/tests/resources/hello_2.10-1+deb9u1.debian.tar.xz differ diff --git a/swh/loader/debian/tests/resources/hello_2.10-1+deb9u1.dsc b/swh/loader/debian/tests/resources/hello_2.10-1+deb9u1.dsc new file mode 100644 index 0000000..484dc60 --- /dev/null +++ b/swh/loader/debian/tests/resources/hello_2.10-1+deb9u1.dsc @@ -0,0 +1,43 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +Format: 3.0 (quilt) +Source: hello +Binary: hello +Architecture: any +Version: 2.10-1+deb9u1 +Maintainer: Santiago Vila +Homepage: http://www.gnu.org/software/hello/ +Standards-Version: 3.9.6 +Build-Depends: debhelper (>= 9.20120311) +Package-List: + hello deb devel optional arch=any +Checksums-Sha1: + f7bebf6f9c62a2295e889f66e05ce9bfaed9ace3 725946 hello_2.10.orig.tar.gz + c29e15965f370fb9e3c742113b6ce5704ad2761c 6156 hello_2.10-1+deb9u1.debian.tar.xz +Checksums-Sha256: + 31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b 725946 hello_2.10.orig.tar.gz + 8f8031f001112e1c869907e77fdc730733cf591c4d66b74252eb0d669d1dfb01 6156 hello_2.10-1+deb9u1.debian.tar.xz +Files: + 6cd0ffea3884a4e79330338dcc2987d6 725946 hello_2.10.orig.tar.gz + 8b0ca540355860aaea1cfc96f2aaa18f 6156 hello_2.10-1+deb9u1.debian.tar.xz + +-----BEGIN PGP SIGNATURE----- + +iQKmBAEBCgCQFiEERkRAmAjBceBVMd3uBUy48xNDz0QFAlj3NzBfFIAAAAAALgAo +aXNzdWVyLWZwckBub3RhdGlvbnMub3BlbnBncC5maWZ0aGhvcnNlbWFuLm5ldDQ2 +NDQ0MDk4MDhDMTcxRTA1NTMxRERFRTA1NENCOEYzMTM0M0NGNDQSHGNhcm5pbEBk +ZWJpYW4ub3JnAAoJEAVMuPMTQ89EacUP/jTn7xkMv9grOsGVVRWnvzQG+qBxl8qB +M3KzKS5Qug88GiCs0poJb0atA5GA9AQo9KznT5OcKDMAlPtx+AZuWTaP5GakKgBl +KJKeT2Evi7T+hQaI60OvGpLIeOEipnrBS3JH37rwn/OMcYbgL0IevfTN34oL45HX +YG0Y3DsRQzEDkLQZEm3cHieWXvjLsl2yWMWoB60dBSxHUH354BfdUOVWczfoVWYk +5U7/G4reQE3dIkYqvnFhbswli474WifnN93AObxN06QojNzYihNzHRhys5YEFAmU +lfmK6FdsCruCpsX8M6wOSTFZSEpPqvwDKAP89O4p3hmRGjmjq+1TqfeA1lfe5rKk +EA7cDb1Je4TrMtqq4WaY2uhcfoke0XkjufpYNqwMx6N2kn5nTN8f7xafrnCwOdWl +XxZx+EkOUNq7yBGNzgm9gSBgvpOdm7goug9n6bLq0i2RqaofymrT4NdFejPTfy0d +gFLs92LrYO/rZhY4/AJp1h5YCOJXay8A6hS2gI1ndd+tv7CooB3hNi922M6C5dnk +HIAcSEl+OOrcV7OU2RbBydJUbxqGiuXjWESkeWCEbtU+YAlBdtTORNMNXz9o0TS4 +AfrYvrc/1PnzsUmPM3gP5BXli5MExgIUTdPk3XD36xdrZQG9tI9VUBfPWaIkOQip +9ApzdObqiPZW +=Sh9M +-----END PGP SIGNATURE----- diff --git a/swh/loader/debian/tests/resources/hello_2.10.orig.tar.gz b/swh/loader/debian/tests/resources/hello_2.10.orig.tar.gz new file mode 100644 index 0000000..cae6b33 Binary files /dev/null and b/swh/loader/debian/tests/resources/hello_2.10.orig.tar.gz differ diff --git a/swh/loader/debian/tests/test_loader.py b/swh/loader/debian/tests/test_loader.py index 9bbd2c4..e13fcb8 100644 --- a/swh/loader/debian/tests/test_loader.py +++ b/swh/loader/debian/tests/test_loader.py @@ -1,32 +1,148 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os +from unittest import TestCase + import pytest +import requests_mock -from unittest import TestCase +from swh.core.tests.db_testing import SingleDbTestFixture +from swh.model.hashutil import hash_to_bytes +from swh.storage.schemata.distribution import SQLBase +from swh.loader.core.tests import BaseLoaderTest +from swh.loader.debian.loader import get_file_info, DebianLoader + +from . import TEST_LOADER_CONFIG + +RESOURCES_PATH = os.path.join(os.path.dirname(__file__), 'resources') -from swh.loader.debian.loader import get_file_info -RESOURCES_PATH = './swh/loader/debian/tests/resources' +class DebianLoaderTest(DebianLoader): + def parse_config_file(self, *args, **kwargs): + return TEST_LOADER_CONFIG @pytest.mark.fs class TestFileInfo(TestCase): def test_get_file_info(self): path = '%s/%s' % (RESOURCES_PATH, 'onefile.txt') actual_info = get_file_info(path) expected_info = { 'name': 'onefile.txt', 'length': 62, 'sha1': '135572f4ac013f49e624612301f9076af1eacef2', 'sha1_git': '1d62cd247ef251d52d98bbd931d44ad1f967ea99', 'sha256': '40f1a3cbe9355879319759bae1a6ba09cbf34056e79e951cd2dc0adbff169b9f', # noqa 'blake2s256': '4072cf9a0017ad7705a9995bbfbbc098276e6a3afea8d84ab54bff6381c897ab', # noqa } self.assertEqual(actual_info, expected_info) + + +@pytest.mark.fs +class TestDebianLoader(SingleDbTestFixture, BaseLoaderTest): + TEST_DB_NAME = 'test-lister-debian' + TEST_DB_DUMP = [] + + def setUp(self): + super().setUp(archive_name='', + start_path=os.path.dirname(__file__), + uncompress_archive=False) + self.loader = DebianLoaderTest() + SQLBase.metadata.create_all(self.loader.db_engine) + self.storage = self.loader.storage + make_uri = ('file://' + RESOURCES_PATH + '/').__add__ + self.debian_src_name = 'hello_2.10-1+deb9u1.debian.tar.xz' + self.orig_src_name = 'hello_2.10.orig.tar.gz' + self.dsc_name = 'hello_2.10-1+deb9u1.dsc' + self.files = { + self.dsc_name: { + 'name': self.dsc_name, + 'uri': make_uri(self.dsc_name), + 'size': 1866, + }, + self.debian_src_name: { + 'name': self.debian_src_name, + 'uri': make_uri(self.debian_src_name), + 'size': 6156, + }, + self.orig_src_name: { + 'name': self.orig_src_name, + 'uri': make_uri(self.orig_src_name), + 'size': 725946, + }, + } + self._fill_db() + + def tearDown(self): + SQLBase.metadata.drop_all(self.loader.db_engine) + self.loader.db_session.close() + self.loader.db_engine.dispose() # close the connection pool + super().tearDown() + + def _fill_db(self): + from swh.storage.schemata.distribution import \ + Area, Distribution, Package + dist = Distribution(name='Debian', type='deb', mirror_uri='devnull://') + area = Area(distribution=dist, name='main') + pkg = Package(area=area, name='hello', version='2.10-1+deb9u1', + directory='dir', files=self.files) + self.loader.db_session.add_all([area, pkg]) + self.loader.db_session.commit() + self.pkg_id = pkg.id + + def _load(self): + self.loader.load( + origin={'url': self.repo_url, 'type': 'deb'}, + date='2018-12-14 16:45:00+00', + packages={ + 'stretch/main/2.10-1+deb9u1': { + 'id': self.pkg_id, + 'name': 'hello', + 'version': '2.10-1+deb9u1', + 'revision_id': None, + 'files': self.files, + } + } + ) + + def test_load(self): + with requests_mock.Mocker() as m: + for file_ in self.files.values(): + path = os.path.join(RESOURCES_PATH, file_['name']) + with open(path, 'rb') as fd: + m.get(file_['uri'], content=fd.read()) + self._load() + + self.assertCountSnapshots(1) + self.assertCountReleases(0) # FIXME: Why not 1? + self.assertCountRevisions(1) + self.assertCountDirectories(14) + self.assertCountContents(315) + + # Check the root dir was loaded, and contains 'src/' + root_hash = 'c906789049d2327a69b81cca6a1c1737321c836f' + ls_root = list(self.storage.directory_ls( + hash_to_bytes(root_hash))) + src_dirs = [x for x in ls_root if x['name'] == b'src'] + self.assertEqual(len(src_dirs), 1, src_dirs) + + # Check 'src/hello.c' exists + src_dir_hash = src_dirs[0]['target'] + ls_src = list(self.storage.directory_ls(src_dir_hash)) + hello_c = [x for x in ls_src if x['name'] == b'hello.c'] + self.assertEqual(len(hello_c), 1, hello_c) + + # Check 'src.hello.c' was loaded and has the right hash + hello_c_hash = 'b60a061ac9dd25b29d57b756b5959aadc1fe6386' + self.assertEqual(hello_c[0]['sha1'], hash_to_bytes(hello_c_hash)) + + missing = list(self.storage.content_missing( + [{'sha1': hash_to_bytes(hello_c_hash)}])) + self.assertEqual(missing, []) diff --git a/tox.ini b/tox.ini index 0fb07c6..70265ee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,17 @@ [tox] envlist=flake8,py3 [testenv:py3] deps = .[testing] pytest-cov + pifpaf commands = - pytest --cov=swh --cov-branch {posargs} + pifpaf run postgresql -- pytest --cov=swh --cov-branch {posargs} [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8