Page MenuHomeSoftware Heritage
Files F9697209

D374.id1157.diff
No OneTemporary
Actions

D374.id1157.diff
View Options

diff --git a/swh/lister/bitbucket/tests/test_bb_lister.py b/swh/lister/bitbucket/tests/test_bb_lister.py
--- a/swh/lister/bitbucket/tests/test_bb_lister.py
+++ b/swh/lister/bitbucket/tests/test_bb_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017 the Software Heritage developers
+# Copyright (C) 2017-2018 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -6,10 +6,10 @@
import unittest
from swh.lister.bitbucket.lister import BitBucketLister
-from swh.lister.core.tests.test_lister import IndexingHttpListerTesterBase
+from swh.lister.core.tests.test_lister import HttpListerTesterBase
-class BitBucketListerTester(IndexingHttpListerTesterBase, unittest.TestCase):
+class BitBucketListerTester(HttpListerTesterBase, unittest.TestCase):
Lister = BitBucketLister
test_re = re.compile(r'/repositories\?after=([^?&]+)')
lister_subdir = 'bitbucket'
diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py
--- a/swh/lister/core/tests/test_lister.py
+++ b/swh/lister/core/tests/test_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017 the Software Heritage developers
+# Copyright (C) 2017-2018 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -20,12 +20,15 @@
@requests_mock.Mocker()
-class IndexingHttpListerTesterBase(abc.ABC):
+class HttpListerTesterBase(abc.ABC):
"""Base testing class for subclasses of
- swh.lister.core.indexing_lister.SWHIndexingHttpLister.
- See swh.lister.github.tests.test_gh_lister for an example of how to
- customize for a specific listing service.
+ swh.lister.core.indexing_lister.SWHIndexingHttpLister.
+ swh.lister.core.paging_lister.SWHPagingHttpLister
+
+ See swh.lister.github.tests.test_gh_lister for an example of how
+ to customize for a specific listing service.
+
"""
Lister = AbstractAttribute('The lister class to test')
test_re = AbstractAttribute('Compiled regex matching the server url. Must'
@@ -56,7 +59,7 @@
self.response = None
self.fl = None
self.helper = None
- if self.__class__ != IndexingHttpListerTesterBase:
+ if self.__class__ != HttpListerTesterBase:
self.run = TestCase.run.__get__(self, self.__class__)
else:
self.run = noop
@@ -99,6 +102,9 @@
return self.mock_limit_n_response(2, request, context)
def get_fl(self, override_config=None):
+ """Retrieve an instance of fake lister (fl).
+
+ """
if override_config or self.fl is None:
with patch(
'swh.scheduler.backend.SchedulerBackend.reconnect', noop
@@ -164,7 +170,7 @@
self.assertIsInstance(di, dict)
pubs = [k for k in vars(fl.MODEL).keys() if not k.startswith('_')]
for k in pubs:
- if k not in ['last_seen', 'task_id', 'origin_id']:
+ if k not in ['last_seen', 'task_id', 'origin_id', 'id']:
self.assertIn(k, di)
def disable_storage_and_scheduler(self, fl):
@@ -221,6 +227,10 @@
self.disable_storage_and_scheduler(fl)
+ # FIXME: Separate the tests properly for the gitlab lister
+ # did not succeed yet
+ if not hasattr(fl, 'db_last_index'): # gitlab lister cannot pass here
+ return
fl.run(min_index=self.first_index)
self.assertEqual(fl.db_last_index(), self.last_index)
diff --git a/swh/lister/github/tests/test_gh_lister.py b/swh/lister/github/tests/test_gh_lister.py
--- a/swh/lister/github/tests/test_gh_lister.py
+++ b/swh/lister/github/tests/test_gh_lister.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017 the Software Heritage developers
+# Copyright (C) 2017-2018 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -6,11 +6,11 @@
import unittest
from datetime import datetime, timedelta
-from swh.lister.core.tests.test_lister import IndexingHttpListerTesterBase
+from swh.lister.core.tests.test_lister import HttpListerTesterBase
from swh.lister.github.lister import GitHubLister
-class GitHubListerTester(IndexingHttpListerTesterBase, unittest.TestCase):
+class GitHubListerTester(HttpListerTesterBase, unittest.TestCase):
Lister = GitHubLister
test_re = re.compile(r'/repositories\?since=([^?&]+)')
lister_subdir = 'github'
diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py
--- a/swh/lister/gitlab/lister.py
+++ b/swh/lister/gitlab/lister.py
@@ -13,7 +13,7 @@
class GitLabLister(SWHPagingHttpLister):
# Template path expecting an integer that represents the page id
PATH_TEMPLATE = '/projects?page=%d&order_by=id&sort=asc&simple=true'
- API_URL_INDEX_RE = re.compile(r'^.*/projects.*\&page=(\d+).*')
+ API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*')
MODEL = GitLabModel
@property
diff --git a/swh/lister/gitlab/tests/__init__.py b/swh/lister/gitlab/tests/__init__.py
new file mode 100644
diff --git a/swh/lister/gitlab/tests/api_empty_response.json b/swh/lister/gitlab/tests/api_empty_response.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/gitlab/tests/api_empty_response.json
@@ -0,0 +1 @@
+[]
diff --git a/swh/lister/gitlab/tests/api_response.json b/swh/lister/gitlab/tests/api_response.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/gitlab/tests/api_response.json
@@ -0,0 +1,170 @@
+[{"avatar_url": null,
+ "created_at": "2012-10-15T17:26:53.000Z",
+ "default_branch": "master",
+ "description": null,
+ "forks_count": 3,
+ "http_url_to_repo": "https://gitlab.com/leberwurscht/teardownwalls.git",
+ "id": 143,
+ "last_activity_at": "2013-10-03T08:08:46.000Z",
+ "name": "TearDownWalls",
+ "name_with_namespace": "Leberwurscht / TearDownWalls",
+ "path": "teardownwalls",
+ "path_with_namespace": "leberwurscht/teardownwalls",
+ "readme_url": "https://gitlab.com/leberwurscht/teardownwalls/blob/master/README.md",
+ "ssh_url_to_repo": "git@gitlab.com:leberwurscht/teardownwalls.git",
+ "star_count": 1,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/leberwurscht/teardownwalls"},
+ {"avatar_url": null,
+ "created_at": "2012-12-12T21:30:14.000Z",
+ "default_branch": "master",
+ "description": "",
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/technomancy/leiningen.git",
+ "id": 450,
+ "last_activity_at": "2018-06-24T00:07:06.666Z",
+ "name": "Leiningen",
+ "name_with_namespace": "Phil Hagelberg / Leiningen",
+ "path": "leiningen",
+ "path_with_namespace": "technomancy/leiningen",
+ "readme_url": "https://gitlab.com/technomancy/leiningen/blob/master/README.md",
+ "ssh_url_to_repo": "git@gitlab.com:technomancy/leiningen.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/technomancy/leiningen"},
+ {"avatar_url": null,
+ "created_at": "2012-12-18T17:25:39.000Z",
+ "default_branch": "master",
+ "description": null,
+ "forks_count": 4,
+ "http_url_to_repo": "https://gitlab.com/jonan/heroes-of-wesnoth.git",
+ "id": 526,
+ "last_activity_at": "2015-04-09T14:43:49.363Z",
+ "name": "Heroes of Wesnoth",
+ "name_with_namespace": "Jonan / Heroes of Wesnoth",
+ "path": "heroes-of-wesnoth",
+ "path_with_namespace": "jonan/heroes-of-wesnoth",
+ "readme_url": null,
+ "ssh_url_to_repo": "git@gitlab.com:jonan/heroes-of-wesnoth.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/jonan/heroes-of-wesnoth"},
+ {"avatar_url": null,
+ "created_at": "2012-12-18T17:33:03.000Z",
+ "default_branch": "master",
+ "description": null,
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/jonan/k.git",
+ "id": 527,
+ "last_activity_at": "2014-10-11T22:29:04.138Z",
+ "name": "K",
+ "name_with_namespace": "Jonan / K",
+ "path": "k",
+ "path_with_namespace": "jonan/k",
+ "readme_url": "https://gitlab.com/jonan/k/blob/master/README",
+ "ssh_url_to_repo": "git@gitlab.com:jonan/k.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/jonan/k"},
+ {"avatar_url": null,
+ "created_at": "2013-01-06T20:35:42.000Z",
+ "default_branch": "master",
+ "description": "",
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/hcs/hcs_utils.git",
+ "id": 1025,
+ "last_activity_at": "2015-09-14T12:01:11.151Z",
+ "name": "hcs_utils",
+ "name_with_namespace": "Christer Sjöholm / hcs_utils",
+ "path": "hcs_utils",
+ "path_with_namespace": "hcs/hcs_utils",
+ "readme_url": "https://gitlab.com/hcs/hcs_utils/blob/master/README.txt",
+ "ssh_url_to_repo": "git@gitlab.com:hcs/hcs_utils.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/hcs/hcs_utils"},
+ {"avatar_url": null,
+ "created_at": "2013-01-24T08:41:56.000Z",
+ "default_branch": null,
+ "description": null,
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/soeren/sspssptest.git",
+ "id": 1702,
+ "last_activity_at": "2013-10-03T08:31:54.000Z",
+ "name": "sspssptest",
+ "name_with_namespace": "kruemel / sspssptest",
+ "path": "sspssptest",
+ "path_with_namespace": "soeren/sspssptest",
+ "readme_url": null,
+ "ssh_url_to_repo": "git@gitlab.com:soeren/sspssptest.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/soeren/sspssptest"},
+ {"avatar_url": null,
+ "created_at": "2013-01-28T22:59:31.000Z",
+ "default_branch": "master",
+ "description": null,
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/dpp/slothbeast.git",
+ "id": 1865,
+ "last_activity_at": "2013-05-05T09:44:57.000Z",
+ "name": "slothbeast",
+ "name_with_namespace": "David Pollak / slothbeast",
+ "path": "slothbeast",
+ "path_with_namespace": "dpp/slothbeast",
+ "readme_url": "https://gitlab.com/dpp/slothbeast/blob/master/README.md",
+ "ssh_url_to_repo": "git@gitlab.com:dpp/slothbeast.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/dpp/slothbeast"},
+ {"avatar_url": null,
+ "created_at": "2013-02-07T20:50:20.000Z",
+ "default_branch": "master",
+ "description": null,
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/rocksoniko/easy.git",
+ "id": 2227,
+ "last_activity_at": "2013-05-05T09:45:00.000Z",
+ "name": "easy",
+ "name_with_namespace": "Hugo / easy",
+ "path": "easy",
+ "path_with_namespace": "rocksoniko/easy",
+ "readme_url": "https://gitlab.com/rocksoniko/easy/blob/master/README",
+ "ssh_url_to_repo": "git@gitlab.com:rocksoniko/easy.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/rocksoniko/easy"},
+ {"avatar_url": null,
+ "created_at": "2013-02-10T17:21:24.000Z",
+ "default_branch": null,
+ "description": null,
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/grup/grup.git",
+ "id": 2294,
+ "last_activity_at": "2013-05-05T09:45:01.000Z",
+ "name": "grup",
+ "name_with_namespace": "grup / grup",
+ "path": "grup",
+ "path_with_namespace": "grup/grup",
+ "readme_url": null,
+ "ssh_url_to_repo": "git@gitlab.com:grup/grup.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/grup/grup"},
+ {"avatar_url": null,
+ "created_at": "2013-02-14T09:31:50.000Z",
+ "default_branch": "master",
+ "description": "",
+ "forks_count": 0,
+ "http_url_to_repo": "https://gitlab.com/varac/test.git",
+ "id": 2390,
+ "last_activity_at": "2016-02-11T13:51:47.463Z",
+ "name": "test",
+ "name_with_namespace": "varac / test",
+ "path": "test",
+ "path_with_namespace": "varac/test",
+ "readme_url": null,
+ "ssh_url_to_repo": "git@gitlab.com:varac/test.git",
+ "star_count": 0,
+ "tag_list": [],
+ "web_url": "https://gitlab.com/varac/test"}]
diff --git a/swh/lister/github/tests/test_gh_lister.py b/swh/lister/gitlab/tests/test_gitlab_lister.py
copy from swh/lister/github/tests/test_gh_lister.py
copy to swh/lister/gitlab/tests/test_gitlab_lister.py
--- a/swh/lister/github/tests/test_gh_lister.py
+++ b/swh/lister/gitlab/tests/test_gitlab_lister.py
@@ -1,37 +1,37 @@
-# Copyright (C) 2017 the Software Heritage developers
+# Copyright (C) 2017-2018 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import re
import unittest
+
from datetime import datetime, timedelta
-from swh.lister.core.tests.test_lister import IndexingHttpListerTesterBase
-from swh.lister.github.lister import GitHubLister
+from swh.lister.gitlab.lister import GitLabLister
+from swh.lister.core.tests.test_lister import HttpListerTesterBase
-class GitHubListerTester(IndexingHttpListerTesterBase, unittest.TestCase):
- Lister = GitHubLister
- test_re = re.compile(r'/repositories\?since=([^?&]+)')
- lister_subdir = 'github'
+class GitLabListerTester(HttpListerTesterBase, unittest.TestCase):
+ Lister = GitLabLister
+ test_re = GitLabLister.API_URL_INDEX_RE
+ lister_subdir = 'gitlab'
good_api_response_file = 'api_response.json'
bad_api_response_file = 'api_empty_response.json'
- first_index = 26
- last_index = 368
- entries_per_page = 100
+ first_index = 1
+ last_index = 2
+ entries_per_page = 10
def response_headers(self, request):
- headers = {'X-RateLimit-Remaining': '1'}
+ headers = {'RateLimit-Remaining': '1'}
if self.request_index(request) == str(self.first_index):
headers.update({
- 'Link': '<https://api.github.com/repositories?since=367>;'
+ 'Link': '<https://gitlab.com/v4/projects?page=2>;'
' rel="next",'
- '<https://api.github.com/repositories{?since}>;'
+ '<https://gitlab.com/v4/projects{?page}>;'
' rel="first"'
})
else:
headers.update({
- 'Link': '<https://api.github.com/repositories{?since}>;'
+ 'Link': '<https://gitlab.com/v4/projects{?page}>;'
' rel="first"'
})
@@ -40,7 +40,7 @@
def mock_rate_quota(self, n, request, context):
self.rate_limit += 1
context.status_code = 403
- context.headers['X-RateLimit-Remaining'] = '0'
+ context.headers['RateLimit-Remaining'] = '0'
one_second = int((datetime.now() + timedelta(seconds=1.5)).timestamp())
- context.headers['X-RateLimit-Reset'] = str(one_second)
+ context.headers['RateLimit-Reset'] = str(one_second)
return '{"error":"dummy"}'

File Metadata

Mime Type
text/plain
Expires
Sun, Aug 17, 11:07 PM (1 w, 15 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3221678

Event Timeline

About · Developer information · Legal