Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7066459
D2408.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
7 KB
Subscribers
None
D2408.diff
View Options
diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py
--- a/swh/lister/bitbucket/lister.py
+++ b/swh/lister/bitbucket/lister.py
@@ -10,6 +10,7 @@
from typing import Any
from urllib import parse
+
from swh.lister.bitbucket.models import BitBucketModel
from swh.lister.core.indexing_lister import IndexingHttpLister
diff --git a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100
--- a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100
+++ b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100
@@ -82,7 +82,7 @@
"description": "Basic files and directory structure for a C++ project. Intended as a starting point for a new project. Includes a basic cross platform core library."
},
{
- "scm": "hg",
+ "scm": "git",
"website": "",
"has_wiki": true,
"name": "mercurialeclipse",
diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py
--- a/swh/lister/bitbucket/tests/test_lister.py
+++ b/swh/lister/bitbucket/tests/test_lister.py
@@ -74,7 +74,7 @@
def test_lister_bitbucket(swh_listers, requests_mock_datadir):
- """Simple bitbucket listing should create scheduled tasks
+ """Simple bitbucket listing should create scheduled tasks (git, hg)
"""
lister = swh_listers['bitbucket']
@@ -82,20 +82,32 @@
lister.run()
r = lister.scheduler.search_tasks(task_type='load-hg')
- assert len(r) == 10
+ assert len(r) == 9
for row in r:
- assert row['type'] == 'load-hg'
- # arguments check
args = row['arguments']['args']
- assert len(args) == 1
+ kwargs = row['arguments']['kwargs']
+
+ assert len(args) == 0
+ assert len(kwargs) == 1
+ url = kwargs['url']
- url = args[0]
assert url.startswith('https://bitbucket.org')
- # kwargs
+ assert row['policy'] == 'recurring'
+ assert row['priority'] is None
+
+ r = lister.scheduler.search_tasks(task_type='load-git')
+ assert len(r) == 1
+
+ for row in r:
+ args = row['arguments']['args']
kwargs = row['arguments']['kwargs']
- assert kwargs == {}
+ assert len(args) == 0
+ assert len(kwargs) == 1
+ url = kwargs['url']
+
+ assert url.startswith('https://bitbucket.org')
assert row['policy'] == 'recurring'
assert row['priority'] is None
diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py
--- a/swh/lister/cgit/tests/test_lister.py
+++ b/swh/lister/cgit/tests/test_lister.py
@@ -57,14 +57,14 @@
assert row['type'] == 'load-git'
# arguments check
args = row['arguments']['args']
- assert len(args) == 1
-
- url = args[0]
- assert url.startswith('https://git.tizen')
+ assert len(args) == 0
# kwargs
kwargs = row['arguments']['kwargs']
- assert kwargs == {}
+ assert len(kwargs) == 1
+ url = kwargs['url']
+ assert url.startswith('https://git.tizen')
+
assert row['policy'] == 'recurring'
assert row['priority'] is None
diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py
--- a/swh/lister/core/lister_base.py
+++ b/swh/lister/core/lister_base.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 the Software Heritage developers
+# Copyright (C) 2015-2019 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -13,7 +13,7 @@
from sqlalchemy import create_engine, func
from sqlalchemy.orm import sessionmaker
-from typing import Any, Type, Union
+from typing import Any, Dict, Type, Union
from swh.core import config
from swh.scheduler import get_scheduler, utils
@@ -376,7 +376,8 @@
return sql_repo
- def task_dict(self, origin_type, origin_url, **kwargs):
+ def task_dict(self, origin_type: str,
+ origin_url: str, **kwargs) -> Dict[str, Any]:
"""Return special dict format for the tasks list
Args:
@@ -390,7 +391,7 @@
_policy = kwargs.get('policy', 'recurring')
priority = kwargs.get('priority')
kw = {'priority': priority} if priority else {}
- return utils.create_task_dict(_type, _policy, origin_url, **kw)
+ return utils.create_task_dict(_type, _policy, url=origin_url, **kw)
def string_pattern_check(self, a, b, c=None):
"""When comparing indexable types in is_within_bounds, complex strings
diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py
--- a/swh/lister/github/tests/test_lister.py
+++ b/swh/lister/github/tests/test_lister.py
@@ -70,14 +70,12 @@
assert row['type'] == 'load-git'
# arguments check
args = row['arguments']['args']
- assert len(args) == 1
-
- url = args[0]
- assert url.startswith('https://github.com')
+ assert len(args) == 0
# kwargs
kwargs = row['arguments']['kwargs']
- assert kwargs == {}
+ url = kwargs['url']
+ assert url.startswith('https://github.com')
assert row['policy'] == 'recurring'
assert row['priority'] is None
diff --git a/swh/lister/gitlab/tests/test_lister.py b/swh/lister/gitlab/tests/test_lister.py
--- a/swh/lister/gitlab/tests/test_lister.py
+++ b/swh/lister/gitlab/tests/test_lister.py
@@ -55,14 +55,12 @@
assert row['type'] == 'load-git'
# arguments check
args = row['arguments']['args']
- assert len(args) == 1
-
- url = args[0]
- assert url.startswith('https://gitlab.com')
+ assert len(args) == 0
# kwargs
kwargs = row['arguments']['kwargs']
- assert kwargs == {}
+ url = kwargs['url']
+ assert url.startswith('https://gitlab.com')
assert row['policy'] == 'recurring'
assert row['priority'] is None
diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py
--- a/swh/lister/packagist/lister.py
+++ b/swh/lister/packagist/lister.py
@@ -7,7 +7,7 @@
import logging
import random
-from typing import Any, List, Mapping
+from typing import Any, Dict, List, Mapping
from swh.scheduler import utils
from swh.lister.core.simple_lister import SimpleLister
@@ -60,7 +60,7 @@
SimpleLister.__init__(self, override_config=override_config)
def task_dict(self, origin_type: str, origin_url: str,
- **kwargs: Mapping[str, str]) -> Mapping[str, str]:
+ **kwargs: Mapping[str, str]) -> Dict[str, Any]:
"""Return task format dict
This is overridden from the lister_base as more information is
diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py
--- a/swh/lister/phabricator/tests/test_lister.py
+++ b/swh/lister/phabricator/tests/test_lister.py
@@ -129,14 +129,12 @@
assert row['type'] == 'load-git'
# arguments check
args = row['arguments']['args']
- assert len(args) == 1
-
- url = args[0]
- assert lister.instance in url
+ assert len(args) == 0
# kwargs
kwargs = row['arguments']['kwargs']
- assert kwargs == {}
+ url = kwargs['url']
+ assert lister.instance in url
assert row['policy'] == 'recurring'
assert row['priority'] is None
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Nov 5 2024, 10:06 AM (11 w, 16 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3223252
Attached To
D2408: hg/git: Align lister's output (load-hg/git tasks) with loader's new format
Event Timeline
Log In to Comment