diff --git a/swh/lister/bitbucket/tests/conftest.py b/swh/lister/bitbucket/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/bitbucket/tests/conftest.py @@ -0,0 +1,32 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + + +@pytest.fixture +def lister_under_test(): + return "bitbucket" + + +@pytest.fixture +def lister_bitbucket(swh_lister): + for task_type in [ + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + }, + { + "type": "load-hg", + "description": "Load hg repository", + "backend_name": "swh.loader.mercurial.tasks.LoadMercurial", + "default_interval": "1 day", + }, + ]: + swh_lister.scheduler.create_task_type(task_type) + + return swh_lister diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -9,18 +9,12 @@ from urllib.parse import unquote import iso8601 -import pytest import requests_mock from swh.lister.bitbucket.lister import BitBucketLister from swh.lister.core.tests.test_lister import HttpListerTester -@pytest.fixture -def lister_under_test(): - return "bitbucket" - - def _convert_type(req_index): """Convert the req_index to its right type according to the model's "indexable" column. @@ -85,15 +79,13 @@ ) -def test_lister_bitbucket(swh_lister, requests_mock_datadir): +def test_lister_bitbucket(lister_bitbucket, requests_mock_datadir): """Simple bitbucket listing should create scheduled tasks (git, hg) """ - lister = swh_lister - - lister.run() + lister_bitbucket.run() - r = lister.scheduler.search_tasks(task_type="load-hg") + r = lister_bitbucket.scheduler.search_tasks(task_type="load-hg") assert len(r) == 9 for row in r: @@ -109,7 +101,7 @@ assert row["policy"] == "recurring" assert row["priority"] is None - r = lister.scheduler.search_tasks(task_type="load-git") + r = lister_bitbucket.scheduler.search_tasks(task_type="load-git") assert len(r) == 1 for row in r: diff --git a/swh/lister/cgit/tests/conftest.py b/swh/lister/cgit/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/cgit/tests/conftest.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + + +@pytest.fixture +def lister_under_test(): + return "cgit" + + +@pytest.fixture +def lister_cgit(swh_lister): + for task_type in [ + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + }, + ]: + swh_lister.scheduler.create_task_type(task_type) + + return swh_lister diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -2,22 +2,13 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pytest - from swh.lister import __version__ -@pytest.fixture -def lister_under_test(): - return "cgit" - - -def test_lister_no_page(requests_mock_datadir, swh_lister): - lister = swh_lister +def test_lister_cgit_no_page(requests_mock_datadir, lister_cgit): + assert lister_cgit.url == "https://git.savannah.gnu.org/cgit/" - assert lister.url == "https://git.savannah.gnu.org/cgit/" - - repos = list(lister.get_repos()) + repos = list(lister_cgit.get_repos()) assert len(repos) == 977 assert repos[0] == "https://git.savannah.gnu.org/cgit/elisp-es.git/" @@ -27,12 +18,10 @@ assert repos[-2] == "http://example.org/cgit/xstarcastle.git/" -def test_lister_model(requests_mock_datadir, swh_lister): - lister = swh_lister - - repo = next(lister.get_repos()) +def test_lister_cgit_model(requests_mock_datadir, lister_cgit): + repo = next(lister_cgit.get_repos()) - model = lister.build_model(repo) + model = lister_cgit.build_model(repo) assert model == { "uid": "https://git.savannah.gnu.org/cgit/elisp-es.git/", "name": "elisp-es.git", @@ -42,21 +31,19 @@ } -def test_lister_with_pages(requests_mock_datadir, swh_lister): - lister = swh_lister - lister.url = "https://git.tizen/cgit/" +def test_lister_cgit_with_pages(requests_mock_datadir, lister_cgit): + lister_cgit.url = "https://git.tizen/cgit/" - repos = list(lister.get_repos()) + repos = list(lister_cgit.get_repos()) # we should have 16 repos (listed on 3 pages) assert len(repos) == 16 -def test_lister_run(requests_mock_datadir, swh_lister): - lister = swh_lister - lister.url = "https://git.tizen/cgit/" - lister.run() +def test_lister_cgit_run(requests_mock_datadir, lister_cgit): + lister_cgit.url = "https://git.tizen/cgit/" + lister_cgit.run() - r = lister.scheduler.search_tasks(task_type="load-git") + r = lister_cgit.scheduler.search_tasks(task_type="load-git") assert len(r) == 16 for row in r: @@ -75,10 +62,9 @@ assert row["priority"] is None -def test_lister_requests(requests_mock_datadir, swh_lister): - lister = swh_lister - lister.url = "https://git.tizen/cgit/" - lister.run() +def test_lister_cgit_requests(requests_mock_datadir, lister_cgit): + lister_cgit.url = "https://git.tizen/cgit/" + lister_cgit.run() assert len(requests_mock_datadir.request_history) != 0 for request in requests_mock_datadir.request_history: diff --git a/swh/lister/cran/tests/conftest.py b/swh/lister/cran/tests/conftest.py --- a/swh/lister/cran/tests/conftest.py +++ b/swh/lister/cran/tests/conftest.py @@ -13,7 +13,6 @@ @pytest.fixture def lister_cran(swh_lister): - # Add the load-cran in the scheduler backend swh_lister.scheduler.create_task_type( { "type": "load-cran", diff --git a/swh/lister/cran/tests/test_lister.py b/swh/lister/cran/tests/test_lister.py --- a/swh/lister/cran/tests/test_lister.py +++ b/swh/lister/cran/tests/test_lister.py @@ -29,17 +29,15 @@ @patch("swh.lister.cran.lister.read_cran_data") def test_cran_lister_cran(mock_cran, datadir, lister_cran): - lister = lister_cran - with open(path.join(datadir, "list-r-packages.json")) as f: data = json.loads(f.read()) mock_cran.return_value = data assert len(data) == 6 - lister.run() + lister_cran.run() - r = lister.scheduler.search_tasks(task_type="load-cran") + r = lister_cran.scheduler.search_tasks(task_type="load-cran") assert len(r) == 6 for row in r: @@ -63,7 +61,7 @@ origin_url = kwargs["url"] record = ( - lister.db_session.query(lister.MODEL) + lister_cran.db_session.query(lister_cran.MODEL) .filter(origin_url == origin_url) .first() ) diff --git a/swh/lister/gitea/tests/conftest.py b/swh/lister/gitea/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/gitea/tests/conftest.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + + +@pytest.fixture +def lister_under_test(): + return "gitea" + + +@pytest.fixture +def lister_gitea(swh_lister): + for task_type in [ + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + }, + ]: + swh_lister.scheduler.create_task_type(task_type) + + return swh_lister diff --git a/swh/lister/gitea/tests/test_lister.py b/swh/lister/gitea/tests/test_lister.py --- a/swh/lister/gitea/tests/test_lister.py +++ b/swh/lister/gitea/tests/test_lister.py @@ -7,8 +7,6 @@ import re import unittest -import pytest - from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.gitea.lister import GiteaLister @@ -40,16 +38,9 @@ return headers -@pytest.fixture -def lister_under_test(): - return "gitea" - - -def test_lister_gitea(swh_lister, requests_mock_datadir): - lister: GiteaLister = swh_lister - - lister.run() - r = lister.scheduler.search_tasks(task_type="load-git") +def test_lister_gitea(lister_gitea, requests_mock_datadir): + lister_gitea.run() + r = lister_gitea.scheduler.search_tasks(task_type="load-git") assert len(r) == 3 for row in r: diff --git a/swh/lister/github/tests/conftest.py b/swh/lister/github/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/github/tests/conftest.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + + +@pytest.fixture +def lister_under_test(): + return "github" + + +@pytest.fixture +def lister_github(swh_lister): + for task_type in [ + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + }, + ]: + swh_lister.scheduler.create_task_type(task_type) + + return swh_lister diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -6,7 +6,6 @@ import re import unittest -import pytest import requests_mock from swh.lister.core.tests.test_lister import HttpListerTester @@ -58,18 +57,13 @@ ) -@pytest.fixture -def lister_under_test(): - return "github" - - -def test_lister_github(swh_lister, requests_mock_datadir): +def test_lister_github(lister_github, requests_mock_datadir): """Simple github listing should create scheduled tasks """ - swh_lister.run() + lister_github.run() - r = swh_lister.scheduler.search_tasks(task_type="load-git") + r = lister_github.scheduler.search_tasks(task_type="load-git") assert len(r) == 100 for row in r: diff --git a/swh/lister/gitlab/tests/conftest.py b/swh/lister/gitlab/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/gitlab/tests/conftest.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + + +@pytest.fixture +def lister_under_test(): + return "gitlab" + + +@pytest.fixture +def lister_gitlab(swh_lister): + for task_type in [ + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + }, + ]: + swh_lister.scheduler.create_task_type(task_type) + + return swh_lister diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -49,10 +49,10 @@ return "gitlab" -def test_lister_gitlab(swh_lister, requests_mock_datadir): - swh_lister.run() +def test_lister_gitlab(lister_gitlab, requests_mock_datadir): + lister_gitlab.run() - r = swh_lister.scheduler.search_tasks(task_type="load-git") + r = lister_gitlab.scheduler.search_tasks(task_type="load-git") assert len(r) == 10 for row in r: diff --git a/swh/lister/gnu/tests/conftest.py b/swh/lister/gnu/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/gnu/tests/conftest.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + + +@pytest.fixture +def lister_under_test(): + return "gnu" + + +@pytest.fixture +def lister_gnu(swh_lister): + for task_type in [ + { + "type": "load-archive-files", + "description": "Load archive repository", + "backend_name": "swh.loader.packages.tasks.LoadArchive", + "default_interval": "1 day", + }, + ]: + swh_lister.scheduler.create_task_type(task_type) + + return swh_lister diff --git a/swh/lister/gnu/tests/test_lister.py b/swh/lister/gnu/tests/test_lister.py --- a/swh/lister/gnu/tests/test_lister.py +++ b/swh/lister/gnu/tests/test_lister.py @@ -5,21 +5,13 @@ import logging -import pytest - logger = logging.getLogger(__name__) -@pytest.fixture -def lister_under_test(): - return "gnu" - - -@pytest.fixture -def test_gnu_lister(swh_lister, requests_mock_datadir): - swh_lister.run() +def test_gnu_lister(lister_gnu, requests_mock_datadir): + lister_gnu.run() - r = swh_lister.scheduler.search_tasks(task_type="load-archive-files") + r = lister_gnu.scheduler.search_tasks(task_type="load-archive-files") assert len(r) == 383 for row in r: diff --git a/swh/lister/launchpad/tests/conftest.py b/swh/lister/launchpad/tests/conftest.py --- a/swh/lister/launchpad/tests/conftest.py +++ b/swh/lister/launchpad/tests/conftest.py @@ -44,4 +44,13 @@ mock_lp_response(i) for i in range(3) ] + lister.scheduler.create_task_type( + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + } + ) + return lister diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -73,7 +73,7 @@ assert row["priority"] is None -def test_lister_npm_basic_listing(lister_npm, requests_mock_datadir): +def test_npm_lister_basic_listing(lister_npm, requests_mock_datadir): lister_npm.run() tasks = lister_npm.scheduler.search_tasks(task_type="load-npm") @@ -82,7 +82,7 @@ check_tasks(tasks) -def test_lister_npm_listing_pagination(lister_npm, requests_mock_datadir): +def test_npm_lister_listing_pagination(lister_npm, requests_mock_datadir): lister = lister_npm # Patch per page pagination lister.per_page = 10 + 1 diff --git a/swh/lister/phabricator/tests/conftest.py b/swh/lister/phabricator/tests/conftest.py --- a/swh/lister/phabricator/tests/conftest.py +++ b/swh/lister/phabricator/tests/conftest.py @@ -18,5 +18,13 @@ "cache_responses": False, "credentials": {"phabricator": {swh_lister.instance: [{"password": "foo"}]}}, } + swh_lister.scheduler.create_task_type( + { + "type": "load-git", + "description": "Load git repository", + "backend_name": "swh.loader.git.tasks.UpdateGitRepository", + "default_interval": "1 day", + } + ) return swh_lister