Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/postgresql/storage.py
| Show First 20 Lines • Show All 1,141 Lines • ▼ Show 20 Lines | |||||
| @db_transaction() | @db_transaction() | ||||
| def origin_search( | def origin_search( | ||||
| self, | self, | ||||
| url_pattern: str, | url_pattern: str, | ||||
| page_token: Optional[str] = None, | page_token: Optional[str] = None, | ||||
| limit: int = 50, | limit: int = 50, | ||||
| regexp: bool = False, | regexp: bool = False, | ||||
| with_visit: bool = False, | with_visit: bool = False, | ||||
| visit_types: Optional[List[str]] = None, | |||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> PagedResult[Origin]: | ) -> PagedResult[Origin]: | ||||
| next_page_token = None | next_page_token = None | ||||
| offset = int(page_token) if page_token else 0 | offset = int(page_token) if page_token else 0 | ||||
| origins = [] | origins = [] | ||||
| # Take one more origin so we can reuse it as the next page token if any | # Take one more origin so we can reuse it as the next page token if any | ||||
| for origin in db.origin_search( | for origin in db.origin_search( | ||||
| url_pattern, offset, limit + 1, regexp, with_visit, cur | url_pattern, offset, limit + 1, regexp, with_visit, visit_types, cur | ||||
| ): | ): | ||||
| row_d = dict(zip(db.origin_cols, origin)) | row_d = dict(zip(db.origin_cols, origin)) | ||||
| origins.append(Origin(url=row_d["url"])) | origins.append(Origin(url=row_d["url"])) | ||||
| if len(origins) > limit: | if len(origins) > limit: | ||||
| # next offset | # next offset | ||||
| next_page_token = str(offset + limit) | next_page_token = str(offset + limit) | ||||
| # excluding that origin from the result to respect the limit size | # excluding that origin from the result to respect the limit size | ||||
| ▲ Show 20 Lines • Show All 245 Lines • Show Last 20 Lines | |||||