diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -5,7 +5,7 @@ import logging import iso8601 -from datetime import datetime +from datetime import datetime, timezone from urllib import parse from swh.lister.bitbucket.models import BitBucketModel @@ -21,7 +21,7 @@ LISTER_NAME = 'bitbucket' DEFAULT_URL = 'https://api.bitbucket.org/2.0' instance = 'bitbucket' - default_min_bound = datetime.utcfromtimestamp(0) + default_min_bound = datetime.fromtimestamp(0, timezone.utc) def __init__(self, url=None, override_config=None, per_page=100): super().__init__(url=url, override_config=override_config) diff --git a/swh/lister/bitbucket/tests/test_bb_lister.py b/swh/lister/bitbucket/tests/test_bb_lister.py --- a/swh/lister/bitbucket/tests/test_bb_lister.py +++ b/swh/lister/bitbucket/tests/test_bb_lister.py @@ -35,6 +35,16 @@ entries_per_page = 10 convert_type = staticmethod(convert_type) + def request_index(self, request): + """(Override) This is needed to emulate the listing bootstrap + when no min_bound is provided to run + """ + m = self.test_re.search(request.path_url) + idx = convert_type(m.group(1)) + if idx == self.Lister.default_min_bound: + idx = self.first_index + return idx + @requests_mock.Mocker() def test_fetch_none_nodb(self, http_mocker): """Overridden because index is not an integer nor a string diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py --- a/swh/lister/core/tests/test_lister.py +++ b/swh/lister/core/tests/test_lister.py @@ -209,6 +209,17 @@ return fl @requests_mock.Mocker() + def test_fetch_no_bounds_yesdb(self, http_mocker): + fl = self.create_fl_with_db(http_mocker) + + fl.run() + + self.assertEqual(fl.db_last_index(), self.last_index) + ingested_repos = list(fl.db_query_range(self.first_index, + self.last_index)) + self.assertEqual(len(ingested_repos), self.entries_per_page) + + @requests_mock.Mocker() def test_fetch_multiple_pages_yesdb(self, http_mocker): fl = self.create_fl_with_db(http_mocker) diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py --- a/swh/lister/github/lister.py +++ b/swh/lister/github/lister.py @@ -16,6 +16,7 @@ API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)') LISTER_NAME = 'github' instance = 'github' # There is only 1 instance of such lister + default_min_bound = 0 def get_model_from_repo(self, repo): return { diff --git a/swh/lister/github/tests/test_gh_lister.py b/swh/lister/github/tests/test_gh_lister.py --- a/swh/lister/github/tests/test_gh_lister.py +++ b/swh/lister/github/tests/test_gh_lister.py @@ -16,7 +16,7 @@ lister_subdir = 'github' good_api_response_file = 'api_response.json' bad_api_response_file = 'api_empty_response.json' - first_index = 26 + first_index = 0 last_index = 368 entries_per_page = 100 convert_type = int diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -75,17 +75,6 @@ get_repo_url(repo['attachments']['uris']['uris'])) @requests_mock.Mocker() - def test_full_listing(self, http_mocker): - fl = self.create_fl_with_db(http_mocker) - - fl.run() - - self.assertEqual(fl.db_last_index(), self.last_index) - ingested_repos = list(fl.db_query_range(self.first_index, - self.last_index)) - self.assertEqual(len(ingested_repos), self.entries_per_page) - - @requests_mock.Mocker() def test_scheduled_tasks(self, http_mocker): fl = self.create_fl_with_db(http_mocker)