Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/npm/lister.py
| Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]): | ||||
| PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}" | PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}" | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| scheduler: SchedulerInterface, | scheduler: SchedulerInterface, | ||||
| page_size: int = 1000, | page_size: int = 1000, | ||||
| incremental: bool = False, | incremental: bool = False, | ||||
| credentials: CredentialsType = None, | credentials: CredentialsType = None, | ||||
| max_origins_per_page: Optional[int] = None, | |||||
| max_pages: Optional[int] = None, | |||||
| enable_origins: bool = True, | |||||
| ): | ): | ||||
| super().__init__( | super().__init__( | ||||
| scheduler=scheduler, | scheduler=scheduler, | ||||
| credentials=credentials, | credentials=credentials, | ||||
| url=self.API_INCREMENTAL_LISTING_URL | url=self.API_INCREMENTAL_LISTING_URL | ||||
| if incremental | if incremental | ||||
| else self.API_FULL_LISTING_URL, | else self.API_FULL_LISTING_URL, | ||||
| instance=self.INSTANCE, | instance=self.INSTANCE, | ||||
| max_origins_per_page=max_origins_per_page, | |||||
| max_pages=max_pages, | |||||
| enable_origins=enable_origins, | |||||
| ) | ) | ||||
| self.page_size = page_size | self.page_size = page_size | ||||
| if not incremental: | if not incremental: | ||||
| # in full listing mode, first package in each page corresponds to the one | # in full listing mode, first package in each page corresponds to the one | ||||
| # provided as the startkey query parameter value, so we increment the page | # provided as the startkey query parameter value, so we increment the page | ||||
| # size by one to avoid double package processing | # size by one to avoid double package processing | ||||
| self.page_size += 1 | self.page_size += 1 | ||||
| ▲ Show 20 Lines • Show All 99 Lines • Show Last 20 Lines | |||||