Page MenuHomeSoftware Heritage
Files F9341786

D1982.id6695.diff
No OneTemporary
Actions

D1982.id6695.diff
View Options

diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py
--- a/swh/lister/core/tests/test_lister.py
+++ b/swh/lister/core/tests/test_lister.py
@@ -163,8 +163,7 @@
if m and (len(m.groups()) > 0):
return self.convert_type(m.group(1))
- @requests_mock.Mocker()
- def test_fetch_multiple_pages_yesdb(self, http_mocker):
+ def create_fl_with_db(self, http_mocker):
http_mocker.get(self.test_re, text=self.mock_response)
db = init_db()
@@ -174,10 +173,16 @@
'args': {'db': db.url()}
}
})
+ fl.db = db
self.init_db(db, fl.MODEL)
self.disable_scheduler(fl)
+ return fl
+
+ @requests_mock.Mocker()
+ def test_fetch_multiple_pages_yesdb(self, http_mocker):
+ fl = self.create_fl_with_db(http_mocker)
fl.run(min_bound=self.first_index)
self.assertEqual(fl.db_last_index(), self.last_index)
diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py
--- a/swh/lister/packagist/lister.py
+++ b/swh/lister/packagist/lister.py
@@ -12,11 +12,11 @@
class PackagistLister(ListerOnePageApiTransport, SimpleLister):
- """List packages available in the Packagist package manger.
+ """List packages available in the Packagist package manager.
The lister sends the request to the url present in the class
variable `PAGE`, to receive a list of all the package names
- present in the Packagist package manger. Iterates over all the
+ present in the Packagist package manager. Iterates over all the
packages and constructs the metadata url of the package from
the name of the package and creates a loading task.
diff --git a/swh/lister/phabricator/lister.py b/swh/lister/phabricator/lister.py
--- a/swh/lister/phabricator/lister.py
+++ b/swh/lister/phabricator/lister.py
@@ -119,11 +119,10 @@
params = '&order=oldest&limit=1'
response = self.safely_issue_request(params)
models_list = self.transport_response_simplified(response)
- self.max_index = models_list[0]['indexable']
models_list = self.filter_before_inject(models_list)
injected = self.inject_repo_data_into_db(models_list)
self.schedule_missing_tasks(models_list, injected)
- return self.max_index
+ return models_list[0]['indexable']
def get_repo_url(attachments):
diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py
--- a/swh/lister/phabricator/tests/test_lister.py
+++ b/swh/lister/phabricator/tests/test_lister.py
@@ -5,6 +5,9 @@
import re
import json
import unittest
+
+import requests_mock
+
from swh.lister.core.tests.test_lister import HttpListerTester
from swh.lister.phabricator.lister import PhabricatorLister
from swh.lister.phabricator.lister import get_repo_url
@@ -12,17 +15,29 @@
class PhabricatorListerTester(HttpListerTester, unittest.TestCase):
Lister = PhabricatorLister
- test_re = re.compile(r'\&after=([^?&]+)')
+ # first request will have the after parameter empty
+ test_re = re.compile(r'\&after=([^?&]*)')
lister_subdir = 'phabricator'
good_api_response_file = 'api_response.json'
good_api_response_undefined_protocol = 'api_response_undefined_'\
'protocol.json'
bad_api_response_file = 'api_empty_response.json'
- first_index = 1
+ # first_index must be retrieved through a bootstrap process for Phabricator
+ first_index = None
last_index = 12
entries_per_page = 10
+
convert_type = int
+ def request_index(self, request):
+ """(Override) This is needed to emulate the listing bootstrap
+ when no min_bound is provided to run
+ """
+ m = self.test_re.search(request.path_url)
+ idx = m.group(1)
+ if idx == str(self.last_index):
+ return int(idx)
+
def get_fl(self, override_config=None):
"""(Override) Retrieve an instance of fake lister (fl).
@@ -58,3 +73,14 @@
self.assertEqual(
'https://svn.blender.org/svnroot/bf-blender/',
get_repo_url(repo['attachments']['uris']['uris']))
+
+ @requests_mock.Mocker()
+ def test_full_listing(self, http_mocker):
+ fl = self.create_fl_with_db(http_mocker)
+
+ fl.run()
+
+ self.assertEqual(fl.db_last_index(), self.last_index)
+ ingested_repos = list(fl.db_query_range(self.first_index,
+ self.last_index))
+ self.assertEqual(len(ingested_repos), self.entries_per_page)

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 12:18 PM (2 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3223122

Event Timeline

About · Developer information · Legal