diff --git a/swh/storage/db.py b/swh/storage/db.py --- a/swh/storage/db.py +++ b/swh/storage/db.py @@ -621,6 +621,26 @@ yield from execute_values_generator( cur, query, ((id,) for id in ids)) + def origin_get_range(self, origin_from=1, origin_count=100, cur=None): + """Retrieve ``origin_count`` origins whose ids are greater + or equal than ``origin_from``. + + Origins are sorted by id before retrieving them. + + Args: + origin_from (int): the minimum id of origins to retrieve + origin_count (int): the maximum number of origins to retrieve + """ + cur = self._cursor(cur) + + query = """SELECT %s + FROM origin WHERE id >= %%s + ORDER BY id LIMIT %%s + """ % ','.join(self.origin_cols) + + cur.execute(query, (origin_from, origin_count)) + yield from cur + def _origin_query(self, url_pattern, count=False, offset=0, limit=50, regexp=False, with_visit=False, cur=None): """ @@ -696,26 +716,6 @@ person_cols = ['fullname', 'name', 'email'] person_get_cols = person_cols + ['id'] - def origin_get_range(self, origin_from=1, origin_count=100, cur=None): - """Retrieve ``origin_count`` origins whose ids are greater - or equal than ``origin_from``. - - Origins are sorted by id before retrieving them. - - Args: - origin_from (int): the minimum id of origins to retrieve - origin_count (int): the maximum number of origins to retrieve - """ - cur = self._cursor(cur) - - query = """SELECT %s - FROM origin WHERE id >= %%s - ORDER BY id LIMIT %%s - """ % ','.join(self.origin_cols) - - cur.execute(query, (origin_from, origin_count)) - yield from cur - def person_get(self, ids, cur=None): """Retrieve the persons identified by the list of ids. diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -1150,6 +1150,25 @@ return [None if res['id'] is None else res for res in results] @db_transaction_generator() + def origin_get_range(self, origin_from=1, origin_count=100, + db=None, cur=None): + """Retrieve ``origin_count`` origins whose ids are greater + or equal than ``origin_from``. + + Origins are sorted by id before retrieving them. + + Args: + origin_from (int): the minimum id of origins to retrieve + origin_count (int): the maximum number of origins to retrieve + + Yields: + dicts containing origin information as returned + by :meth:`swh.storage.storage.Storage.origin_get`. + """ + for origin in db.origin_get_range(origin_from, origin_count, cur): + yield dict(zip(self.origin_keys, origin)) + + @db_transaction_generator() def origin_search(self, url_pattern, offset=0, limit=50, regexp=False, with_visit=False, db=None, cur=None): """Search for origins whose urls contain a provided string pattern @@ -1192,25 +1211,6 @@ """ return db.origin_count(url_pattern, regexp, with_visit, cur) - @db_transaction_generator() - def origin_get_range(self, origin_from=1, origin_count=100, - db=None, cur=None): - """Retrieve ``origin_count`` origins whose ids are greater - or equal than ``origin_from``. - - Origins are sorted by id before retrieving them. - - Args: - origin_from (int): the minimum id of origins to retrieve - origin_count (int): the maximum number of origins to retrieve - - Yields: - dicts containing origin information as returned - by :meth:`swh.storage.storage.Storage.origin_get`. - """ - for origin in db.origin_get_range(origin_from, origin_count, cur): - yield dict(zip(self.origin_keys, origin)) - @db_transaction_generator(statement_timeout=500) def person_get(self, person, db=None, cur=None): """Return the persons identified by their ids.