Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/cgit/tests/test_lister.py
| # Copyright (C) 2019 the Software Heritage developers | # Copyright (C) 2019 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from os.path import join, dirname | |||||
| import re | |||||
| from urllib.parse import urlparse | |||||
| from unittest.mock import Mock | |||||
| import requests_mock | def test_lister_no_page(requests_mock_datadir, swh_listers): | ||||
| from sqlalchemy import create_engine | lister = swh_listers['cgit'] | ||||
| from swh.lister.cgit.lister import CGitLister | assert lister.url == 'https://git.savannah.gnu.org/cgit/' | ||||
| from swh.lister.tests.test_utils import init_db | |||||
| DATADIR = join(dirname(__file__), 'data') | |||||
| def get_response_cb(request, context): | |||||
| url = urlparse(request.url) | |||||
| dirname = url.hostname | |||||
| filename = url.path[1:-1].replace('/', '_') | |||||
| if url.query: | |||||
| filename += ',' + url.query | |||||
| resp = open(join(DATADIR, dirname, filename), 'rb').read() | |||||
| return resp.decode('ascii', 'ignore') | |||||
| def test_lister_no_page(): | |||||
| with requests_mock.Mocker() as m: | |||||
| m.get(re.compile('http://git.savannah.gnu.org'), text=get_response_cb) | |||||
| lister = CGitLister() | |||||
| assert lister.url == 'http://git.savannah.gnu.org/cgit/' | |||||
| repos = list(lister.get_repos()) | repos = list(lister.get_repos()) | ||||
| assert len(repos) == 977 | assert len(repos) == 977 | ||||
| assert repos[0] == 'http://git.savannah.gnu.org/cgit/elisp-es.git/' | assert repos[0] == 'https://git.savannah.gnu.org/cgit/elisp-es.git/' | ||||
| # note the url below is NOT a subpath of /cgit/ | # note the url below is NOT a subpath of /cgit/ | ||||
| assert repos[-1] == 'http://git.savannah.gnu.org/path/to/yetris.git/' # noqa | assert repos[-1] == 'https://git.savannah.gnu.org/path/to/yetris.git/' # noqa | ||||
| # note the url below is NOT on the same server | # note the url below is NOT on the same server | ||||
| assert repos[-2] == 'http://example.org/cgit/xstarcastle.git/' | assert repos[-2] == 'http://example.org/cgit/xstarcastle.git/' | ||||
| def test_lister_model(): | def test_lister_model(requests_mock_datadir, swh_listers): | ||||
| with requests_mock.Mocker() as m: | lister = swh_listers['cgit'] | ||||
| m.get(re.compile('http://git.savannah.gnu.org'), text=get_response_cb) | |||||
| lister = CGitLister() | |||||
| repo = next(lister.get_repos()) | repo = next(lister.get_repos()) | ||||
| model = lister.build_model(repo) | model = lister.build_model(repo) | ||||
| assert model == { | assert model == { | ||||
| 'uid': 'http://git.savannah.gnu.org/cgit/elisp-es.git/', | 'uid': 'https://git.savannah.gnu.org/cgit/elisp-es.git/', | ||||
| 'name': 'elisp-es.git', | 'name': 'elisp-es.git', | ||||
| 'origin_type': 'git', | 'origin_type': 'git', | ||||
| 'instance': 'git.savannah.gnu.org', | 'instance': 'git.savannah.gnu.org', | ||||
| 'origin_url': 'https://git.savannah.gnu.org/git/elisp-es.git' | 'origin_url': 'https://git.savannah.gnu.org/git/elisp-es.git' | ||||
| } | } | ||||
| def test_lister_with_pages(): | def test_lister_with_pages(requests_mock_datadir, swh_listers): | ||||
| with requests_mock.Mocker() as m: | lister = swh_listers['cgit'] | ||||
| m.get(re.compile('http://git.tizen/cgit/'), text=get_response_cb) | lister.url = 'https://git.tizen/cgit/' | ||||
| lister = CGitLister(url='http://git.tizen/cgit/') | |||||
| assert lister.url == 'http://git.tizen/cgit/' | |||||
| repos = list(lister.get_repos()) | repos = list(lister.get_repos()) | ||||
| # we should have 16 repos (listed on 3 pages) | # we should have 16 repos (listed on 3 pages) | ||||
| assert len(repos) == 16 | assert len(repos) == 16 | ||||
| def test_lister_run(): | def test_lister_run(requests_mock_datadir, swh_listers): | ||||
| with requests_mock.Mocker() as m: | lister = swh_listers['cgit'] | ||||
| m.get(re.compile('http://git.tizen/cgit/'), text=get_response_cb) | lister.url = 'https://git.tizen/cgit/' | ||||
| db = init_db() | |||||
| conf = {'lister': {'cls': 'local', 'args': {'db': db.url()}}} | |||||
| lister = CGitLister(url='http://git.tizen/cgit/', | |||||
| override_config=conf) | |||||
| engine = create_engine(db.url()) | |||||
| lister.MODEL.metadata.create_all(engine) | |||||
| lister.schedule_missing_tasks = Mock(return_value=None) | |||||
| lister.run() | lister.run() | ||||
| r = lister.scheduler.search_tasks(task_type='load-git') | |||||
| assert len(r) == 16 | |||||
| for row in r: | |||||
| assert row['type'] == 'load-git' | |||||
| # arguments check | |||||
| args = row['arguments']['args'] | |||||
| assert len(args) == 1 | |||||
| url = args[0] | |||||
| assert url.startswith('https://git.tizen') | |||||
| # kwargs | |||||
| kwargs = row['arguments']['kwargs'] | |||||
| assert kwargs == {} | |||||
| assert row['policy'] == 'recurring' | |||||
| assert row['priority'] is None | |||||