diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -10,6 +10,7 @@ from typing import Any from urllib import parse + from swh.lister.bitbucket.models import BitBucketModel from swh.lister.core.indexing_lister import IndexingHttpLister diff --git a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 --- a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 +++ b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 @@ -82,7 +82,7 @@ "description": "Basic files and directory structure for a C++ project. Intended as a starting point for a new project. Includes a basic cross platform core library." }, { - "scm": "hg", + "scm": "git", "website": "", "has_wiki": true, "name": "mercurialeclipse", diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -74,7 +74,7 @@ def test_lister_bitbucket(swh_listers, requests_mock_datadir): - """Simple bitbucket listing should create scheduled tasks + """Simple bitbucket listing should create scheduled tasks (git, hg) """ lister = swh_listers['bitbucket'] @@ -82,20 +82,32 @@ lister.run() r = lister.scheduler.search_tasks(task_type='load-hg') - assert len(r) == 10 + assert len(r) == 9 for row in r: - assert row['type'] == 'load-hg' - # arguments check args = row['arguments']['args'] - assert len(args) == 1 + kwargs = row['arguments']['kwargs'] + + assert len(args) == 0 + assert len(kwargs) == 1 + url = kwargs['url'] - url = args[0] assert url.startswith('https://bitbucket.org') - # kwargs + assert row['policy'] == 'recurring' + assert row['priority'] is None + + r = lister.scheduler.search_tasks(task_type='load-git') + assert len(r) == 1 + + for row in r: + args = row['arguments']['args'] kwargs = row['arguments']['kwargs'] - assert kwargs == {} + assert len(args) == 0 + assert len(kwargs) == 1 + url = kwargs['url'] + + assert url.startswith('https://bitbucket.org') assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -57,14 +57,14 @@ assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://git.tizen') + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + assert len(kwargs) == 1 + url = kwargs['url'] + assert url.startswith('https://git.tizen') + assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py --- a/swh/lister/core/lister_base.py +++ b/swh/lister/core/lister_base.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 the Software Heritage developers +# Copyright (C) 2015-2019 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -13,7 +13,7 @@ from sqlalchemy import create_engine, func from sqlalchemy.orm import sessionmaker -from typing import Any, Type, Union +from typing import Any, Dict, Type, Union from swh.core import config from swh.scheduler import get_scheduler, utils @@ -376,7 +376,8 @@ return sql_repo - def task_dict(self, origin_type, origin_url, **kwargs): + def task_dict(self, origin_type: str, + origin_url: str, **kwargs) -> Dict[str, Any]: """Return special dict format for the tasks list Args: @@ -390,7 +391,7 @@ _policy = kwargs.get('policy', 'recurring') priority = kwargs.get('priority') kw = {'priority': priority} if priority else {} - return utils.create_task_dict(_type, _policy, origin_url, **kw) + return utils.create_task_dict(_type, _policy, url=origin_url, **kw) def string_pattern_check(self, a, b, c=None): """When comparing indexable types in is_within_bounds, complex strings diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -70,14 +70,12 @@ assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://github.com') + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + url = kwargs['url'] + assert url.startswith('https://github.com') assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py --- a/swh/lister/gitlab/tests/test_lister.py +++ b/swh/lister/gitlab/tests/test_lister.py @@ -55,14 +55,12 @@ assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert url.startswith('https://gitlab.com') + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + url = kwargs['url'] + assert url.startswith('https://gitlab.com') assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py --- a/swh/lister/packagist/lister.py +++ b/swh/lister/packagist/lister.py @@ -7,7 +7,7 @@ import logging import random -from typing import Any, List, Mapping +from typing import Any, Dict, List, Mapping from swh.scheduler import utils from swh.lister.core.simple_lister import SimpleLister @@ -60,7 +60,7 @@ SimpleLister.__init__(self, override_config=override_config) def task_dict(self, origin_type: str, origin_url: str, - **kwargs: Mapping[str, str]) -> Mapping[str, str]: + **kwargs: Mapping[str, str]) -> Dict[str, Any]: """Return task format dict This is overridden from the lister_base as more information is diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -129,14 +129,12 @@ assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] - assert len(args) == 1 - - url = args[0] - assert lister.instance in url + assert len(args) == 0 # kwargs kwargs = row['arguments']['kwargs'] - assert kwargs == {} + url = kwargs['url'] + assert lister.instance in url assert row['policy'] == 'recurring' assert row['priority'] is None