diff --git a/MANIFEST.in b/MANIFEST.in index c31438f..2f90a77 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,8 @@ include Makefile include README include requirements*.txt include version.txt +include conftest.py include swh/lister/cran/list_all_packages.R recursive-include swh/lister/*/tests/data/ * recursive-include swh py.typed diff --git a/conftest.py b/conftest.py index 1759d04..e709cbe 100644 --- a/conftest.py +++ b/conftest.py @@ -1,28 +1,28 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import pytest -pytest_plugins = ["swh.scheduler.pytest_plugin"] +pytest_plugins = ["swh.scheduler.pytest_plugin", "swh.lister.pytest_plugin"] os.environ["LC_ALL"] = "C.UTF-8" @pytest.fixture def mock_get_scheduler(monkeypatch, swh_scheduler): """Override the get_scheduler function in swh.lister.core.lister_base, to return the swh_scheduler fixture. """ from swh.lister.core import lister_base # Match the signature from swh.scheduler.get_scheduler def get_scheduler(cls, args={}): return swh_scheduler monkeypatch.setattr(lister_base, "get_scheduler", get_scheduler) yield monkeypatch diff --git a/swh/lister/bitbucket/tests/conftest.py b/swh/lister/bitbucket/tests/conftest.py deleted file mode 100644 index 507fef9..0000000 --- a/swh/lister/bitbucket/tests/conftest.py +++ /dev/null @@ -1 +0,0 @@ -from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/cgit/tests/conftest.py b/swh/lister/cgit/tests/conftest.py deleted file mode 100644 index 507fef9..0000000 --- a/swh/lister/cgit/tests/conftest.py +++ /dev/null @@ -1 +0,0 @@ -from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py index 30d88c3..7b7777a 100644 --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -1,25 +1,23 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest -from swh.lister.core.tests.conftest import * # noqa - @pytest.fixture def lister_cran(swh_listers): lister = swh_listers["cran"] # Add the load-deb-package in the scheduler backend lister.scheduler.create_task_type( { "type": "load-cran", "description": "Load a CRAN package", "backend_name": "swh.loader.package.cran.tasks.LoaderCRAN", "default_interval": "1 day", } ) return lister diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py index 4b2ab4c..bea3f46 100644 --- a/swh/lister/debian/tests/conftest.py +++ b/swh/lister/debian/tests/conftest.py @@ -1,58 +1,57 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from pytest_postgresql.janitor import DatabaseJanitor from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from swh.lister.core.tests.conftest import * # noqa from swh.lister.core.models import SQLBase from swh.lister.debian import debian_init @pytest.fixture def lister_debian(swh_listers): lister = swh_listers["debian"] # Initialize the debian data model debian_init(lister.db_engine, suites=["stretch"], components=["main", "contrib"]) # Add the load-deb-package in the scheduler backend lister.scheduler.create_task_type( { "type": "load-deb-package", "description": "Load a Debian package", "backend_name": "swh.loader.debian.tasks.LoaderDebianPackage", "default_interval": "1 day", } ) return lister @pytest.fixture def sqlalchemy_engine(postgresql_proc): pg_host = postgresql_proc.host pg_port = postgresql_proc.port pg_user = postgresql_proc.user pg_db = "sqlalchemy-tests" url = f"postgresql://{pg_user}@{pg_host}:{pg_port}/{pg_db}" with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, postgresql_proc.version): engine = create_engine(url) yield engine engine.dispose() @pytest.fixture def session(sqlalchemy_engine): SQLBase.metadata.create_all(sqlalchemy_engine) Session = sessionmaker(bind=sqlalchemy_engine) session = Session() yield session session.close() diff --git a/swh/lister/gitea/tests/conftest.py b/swh/lister/gitea/tests/conftest.py deleted file mode 100644 index c18796b..0000000 --- a/swh/lister/gitea/tests/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2020 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/github/tests/conftest.py b/swh/lister/github/tests/conftest.py deleted file mode 100644 index 507fef9..0000000 --- a/swh/lister/github/tests/conftest.py +++ /dev/null @@ -1 +0,0 @@ -from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/gitlab/tests/conftest.py b/swh/lister/gitlab/tests/conftest.py deleted file mode 100644 index 2426987..0000000 --- a/swh/lister/gitlab/tests/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2019 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/gnu/tests/conftest.py b/swh/lister/gnu/tests/conftest.py deleted file mode 100644 index a4b2b26..0000000 --- a/swh/lister/gnu/tests/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2019 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/launchpad/tests/conftest.py b/swh/lister/launchpad/tests/conftest.py index d1dd73e..655d858 100644 --- a/swh/lister/launchpad/tests/conftest.py +++ b/swh/lister/launchpad/tests/conftest.py @@ -1,51 +1,50 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.lister.core.tests.conftest import * # noqa from datetime import datetime import json import os from unittest.mock import patch import pytest from swh.lister import get_lister from swh.lister.core.models import initialize from sqlalchemy.engine import create_engine @pytest.fixture def lister_launchpad(datadir, lister_db_url, swh_scheduler): class Collection: entries = [] def __init__(self, file): self.entries = [Repo(r) for r in file] def __getitem__(self, key): return self.entries[key] class Repo: def __init__(self, d: dict): for key in d.keys(): if key == "date_last_modified": setattr(self, key, datetime.fromisoformat(d[key])) else: setattr(self, key, d[key]) def mock_lp_response(page) -> Collection: response_filepath = os.path.join(datadir, f"response{page}.json") with open(response_filepath, "r", encoding="utf-8") as f: return Collection(json.load(f)) with patch("launchpadlib.launchpad.Launchpad.login_anonymously"): lister = get_lister("launchpad", db_url=lister_db_url) lister.scheduler = swh_scheduler # inject scheduler fixture lister.launchpad.git_repositories.getRepositories.side_effect = [ mock_lp_response(i) for i in range(3) ] initialize(create_engine(lister_db_url), drop_tables=True) return lister diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py index bfa555f..507064d 100644 --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -1,25 +1,23 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest -from swh.lister.core.tests.conftest import * # noqa - @pytest.fixture def lister_npm(swh_listers): lister = swh_listers["npm"] # Add the load-deb-package in the scheduler backend lister.scheduler.create_task_type( { "type": "load-npm", "description": "Load npm package", "backend_name": "swh.loader.package.tasks.LoadNpm", "default_interval": "1 day", } ) return lister diff --git a/swh/lister/packagist/tests/conftest.py b/swh/lister/packagist/tests/conftest.py index 1eafc36..81f8e44 100644 --- a/swh/lister/packagist/tests/conftest.py +++ b/swh/lister/packagist/tests/conftest.py @@ -1,25 +1,23 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest -from swh.lister.core.tests.conftest import * # noqa - @pytest.fixture def lister_packagist(swh_listers): lister = swh_listers["packagist"] # Amend the scheduler with an unknown yet load-packagist task type lister.scheduler.create_task_type( { "type": "load-packagist", "description": "Load packagist origin", "backend_name": "swh.loader.package.tasks.LoaderPackagist", "default_interval": "1 day", } ) return lister diff --git a/swh/lister/phabricator/tests/conftest.py b/swh/lister/phabricator/tests/conftest.py index 2713ce9..1926122 100644 --- a/swh/lister/phabricator/tests/conftest.py +++ b/swh/lister/phabricator/tests/conftest.py @@ -1,21 +1,19 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest -from swh.lister.core.tests.conftest import * # noqa - @pytest.fixture def lister_phabricator(swh_listers): lister = swh_listers["phabricator"] # Amend the credentials lister.config = { "cache_responses": False, "credentials": {"phabricator": {lister.instance: [{"password": "foo"}]}}, } return lister diff --git a/swh/lister/pypi/tests/conftest.py b/swh/lister/pypi/tests/conftest.py index 658fdcb..5d235f7 100644 --- a/swh/lister/pypi/tests/conftest.py +++ b/swh/lister/pypi/tests/conftest.py @@ -1,25 +1,23 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest -from swh.lister.core.tests.conftest import * # noqa - @pytest.fixture def lister_pypi(swh_listers): lister = swh_listers["pypi"] # Add the load-deb-package in the scheduler backend lister.scheduler.create_task_type( { "type": "load-pypi", "description": "Load PyPI package", "backend_name": "swh.loader.package.tasks.LoadPyPI", "default_interval": "1 day", } ) return lister diff --git a/swh/lister/core/tests/conftest.py b/swh/lister/pytest_plugin.py similarity index 99% rename from swh/lister/core/tests/conftest.py rename to swh/lister/pytest_plugin.py index 647fd46..d58195c 100644 --- a/swh/lister/core/tests/conftest.py +++ b/swh/lister/pytest_plugin.py @@ -1,49 +1,51 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging + import pytest from sqlalchemy import create_engine from swh.lister import get_lister, SUPPORTED_LISTERS from swh.lister.core.models import initialize + logger = logging.getLogger(__name__) @pytest.fixture def lister_db_url(postgresql_proc, postgresql): db_url = "postgresql://{user}@{host}:{port}/{dbname}".format( host=postgresql_proc.host, port=postgresql_proc.port, user="postgres", dbname="tests", ) logger.debug("lister db_url: %s", db_url) return db_url @pytest.fixture def swh_listers(mock_get_scheduler, lister_db_url, swh_scheduler): listers = {} # Prepare schema for all listers for lister_name in SUPPORTED_LISTERS: lister = get_lister(lister_name, db_url=lister_db_url) listers[lister_name] = lister initialize(create_engine(lister_db_url), drop_tables=True) # Add the load-archive-files expected by some listers (gnu, cran, ...) swh_scheduler.create_task_type( { "type": "load-archive-files", "description": "Load archive files.", "backend_name": "swh.loader.package.tasks.LoadArchive", "default_interval": "1 day", } ) return listers