Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/cgit/tests/test_lister.py
# Copyright (C) 2019 the Software Heritage developers | # Copyright (C) 2019 the Software Heritage developers | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from os.path import join, dirname | |||||
import re | |||||
from urllib.parse import urlparse | |||||
from unittest.mock import Mock | |||||
import requests_mock | def test_lister_no_page(requests_mock_datadir, swh_listers): | ||||
from sqlalchemy import create_engine | lister = swh_listers['cgit'] | ||||
from swh.lister.cgit.lister import CGitLister | assert lister.url == 'https://git.savannah.gnu.org/cgit/' | ||||
from swh.lister.tests.test_utils import init_db | |||||
DATADIR = join(dirname(__file__), 'data') | |||||
def get_response_cb(request, context): | |||||
url = urlparse(request.url) | |||||
dirname = url.hostname | |||||
filename = url.path[1:-1].replace('/', '_') | |||||
if url.query: | |||||
filename += ',' + url.query | |||||
resp = open(join(DATADIR, dirname, filename), 'rb').read() | |||||
return resp.decode('ascii', 'ignore') | |||||
def test_lister_no_page(): | |||||
with requests_mock.Mocker() as m: | |||||
m.get(re.compile('http://git.savannah.gnu.org'), text=get_response_cb) | |||||
lister = CGitLister() | |||||
assert lister.url == 'http://git.savannah.gnu.org/cgit/' | |||||
repos = list(lister.get_repos()) | repos = list(lister.get_repos()) | ||||
assert len(repos) == 977 | assert len(repos) == 977 | ||||
assert repos[0] == 'http://git.savannah.gnu.org/cgit/elisp-es.git/' | assert repos[0] == 'https://git.savannah.gnu.org/cgit/elisp-es.git/' | ||||
# note the url below is NOT a subpath of /cgit/ | # note the url below is NOT a subpath of /cgit/ | ||||
assert repos[-1] == 'http://git.savannah.gnu.org/path/to/yetris.git/' # noqa | assert repos[-1] == 'https://git.savannah.gnu.org/path/to/yetris.git/' # noqa | ||||
# note the url below is NOT on the same server | # note the url below is NOT on the same server | ||||
assert repos[-2] == 'http://example.org/cgit/xstarcastle.git/' | assert repos[-2] == 'http://example.org/cgit/xstarcastle.git/' | ||||
def test_lister_model(): | def test_lister_model(requests_mock_datadir, swh_listers): | ||||
with requests_mock.Mocker() as m: | lister = swh_listers['cgit'] | ||||
m.get(re.compile('http://git.savannah.gnu.org'), text=get_response_cb) | |||||
lister = CGitLister() | |||||
repo = next(lister.get_repos()) | repo = next(lister.get_repos()) | ||||
model = lister.build_model(repo) | model = lister.build_model(repo) | ||||
assert model == { | assert model == { | ||||
'uid': 'http://git.savannah.gnu.org/cgit/elisp-es.git/', | 'uid': 'https://git.savannah.gnu.org/cgit/elisp-es.git/', | ||||
'name': 'elisp-es.git', | 'name': 'elisp-es.git', | ||||
'origin_type': 'git', | 'origin_type': 'git', | ||||
'instance': 'git.savannah.gnu.org', | 'instance': 'git.savannah.gnu.org', | ||||
'origin_url': 'https://git.savannah.gnu.org/git/elisp-es.git' | 'origin_url': 'https://git.savannah.gnu.org/git/elisp-es.git' | ||||
} | } | ||||
def test_lister_with_pages(): | def test_lister_with_pages(requests_mock_datadir, swh_listers): | ||||
with requests_mock.Mocker() as m: | lister = swh_listers['cgit'] | ||||
m.get(re.compile('http://git.tizen/cgit/'), text=get_response_cb) | lister.url = 'https://git.tizen/cgit/' | ||||
lister = CGitLister(url='http://git.tizen/cgit/') | |||||
assert lister.url == 'http://git.tizen/cgit/' | |||||
repos = list(lister.get_repos()) | repos = list(lister.get_repos()) | ||||
# we should have 16 repos (listed on 3 pages) | # we should have 16 repos (listed on 3 pages) | ||||
assert len(repos) == 16 | assert len(repos) == 16 | ||||
def test_lister_run(): | def test_lister_run(requests_mock_datadir, swh_listers): | ||||
with requests_mock.Mocker() as m: | lister = swh_listers['cgit'] | ||||
m.get(re.compile('http://git.tizen/cgit/'), text=get_response_cb) | lister.url = 'https://git.tizen/cgit/' | ||||
db = init_db() | |||||
conf = {'lister': {'cls': 'local', 'args': {'db': db.url()}}} | |||||
lister = CGitLister(url='http://git.tizen/cgit/', | |||||
override_config=conf) | |||||
engine = create_engine(db.url()) | |||||
lister.MODEL.metadata.create_all(engine) | |||||
lister.schedule_missing_tasks = Mock(return_value=None) | |||||
lister.run() | lister.run() | ||||
r = lister.scheduler.search_tasks(task_type='load-git') | |||||
assert len(r) == 16 | |||||
for row in r: | |||||
assert row['type'] == 'load-git' | |||||
# arguments check | |||||
args = row['arguments']['args'] | |||||
assert len(args) == 1 | |||||
url = args[0] | |||||
assert url.startswith('https://git.tizen') | |||||
# kwargs | |||||
kwargs = row['arguments']['kwargs'] | |||||
assert kwargs == {} | |||||
assert row['policy'] == 'recurring' | |||||
assert row['priority'] is None |