diff --git a/swh/lister/npm/__init__.py b/swh/lister/npm/__init__.py --- a/swh/lister/npm/__init__.py +++ b/swh/lister/npm/__init__.py @@ -1,14 +1,13 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def register(): from .lister import NpmLister - from .models import NpmModel, NpmVisitModel return { - "models": [NpmVisitModel, NpmModel], + "models": [], "lister": NpmLister, "task_modules": ["%s.tasks" % __name__], "task_types": { diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -1,154 +1,190 @@ -# Copyright (C) 2018-2019 the Software Heritage developers +# Copyright (C) 2018-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Any, Dict, List, Optional +from dataclasses import asdict, dataclass +import logging +from typing import Any, Dict, Iterator, List, Optional -from requests import Response +import iso8601 +import requests +from tenacity.before_sleep import before_sleep_log -from swh.core import config -from swh.lister.core.indexing_lister import IndexingHttpLister -from swh.lister.npm.models import NpmModel -from swh.scheduler.utils import create_task_dict +from swh.lister import USER_AGENT +from swh.lister.pattern import CredentialsType, Lister +from swh.lister.utils import throttling_retry +from swh.scheduler.interface import SchedulerInterface +from swh.scheduler.model import ListedOrigin -DEFAULT_CONFIG = { - "loading_task_policy": "recurring", -} +logger = logging.getLogger(__name__) -class NpmListerBase(IndexingHttpLister): - """List packages available in the npm registry in a paginated way +@dataclass +class NpmListerState: + """State of npm lister""" - """ - - MODEL = NpmModel - LISTER_NAME = "npm" - instance = "npm" + last_seq: Optional[int] = None - def __init__( - self, url="https://replicate.npmjs.com", per_page=1000, override_config=None - ): - super().__init__(url=url, override_config=override_config) - self.config = config.merge_configs(DEFAULT_CONFIG, self.config) - self.per_page = per_page + 1 - self.PATH_TEMPLATE += "&limit=%s" % self.per_page - - def get_model_from_repo(self, repo_name: str) -> Dict[str, str]: - """(Override) Transform from npm package name to model - - """ - package_url = "https://www.npmjs.com/package/%s" % repo_name - return { - "uid": repo_name, - "indexable": repo_name, - "name": repo_name, - "full_name": repo_name, - "html_url": package_url, - "origin_url": package_url, - "origin_type": "npm", - } - - def task_dict(self, origin_type: str, origin_url: str, **kwargs): - """(Override) Return task dict for loading a npm package into the - archive. - - This is overridden from the lister_base as more information is - needed for the ingestion task creation. - - """ - task_type = "load-%s" % origin_type - task_policy = self.config["loading_task_policy"] - return create_task_dict(task_type, task_policy, url=origin_url) - - def request_headers(self) -> Dict[str, Any]: - """(Override) Set requests headers to send when querying the npm - registry. - - """ - headers = super().request_headers() - headers["Accept"] = "application/json" - return headers - - def string_pattern_check(self, inner: int, lower: int, upper: int = None): - """ (Override) Inhibit the effect of that method as packages indices - correspond to package names and thus do not respect any kind - of fixed length string pattern - - """ - pass - - -class NpmLister(NpmListerBase): - """List all packages available in the npm registry in a paginated way +class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]): """ + List all packages hosted on the npm registry. - PATH_TEMPLATE = '/_all_docs?startkey="%s"' - - def get_next_target_from_response(self, response: Response) -> Optional[str]: - """(Override) Get next npm package name to continue the listing - - """ - repos = response.json()["rows"] - return repos[-1]["id"] if len(repos) == self.per_page else None - - def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]: - """(Override) Transform npm registry response to list for model manipulation - - """ - repos = response.json()["rows"] - if len(repos) == self.per_page: - repos = repos[:-1] - return [self.get_model_from_repo(repo["id"]) for repo in repos] - + The lister is based on the npm replication API powered by a + CouchDB database (https://docs.couchdb.org/en/stable/api/database/). -class NpmIncrementalLister(NpmListerBase): - """List packages in the npm registry, updated since a specific - update_seq value of the underlying CouchDB database, in a paginated way. + Args: + scheduler: a scheduler instance + page_size: number of packages info to return per page when querying npm API + incremental: defines if incremental listing should be used, in that case + only modified or new packages since last incremental listing operation + will be returned, otherwise all packages will be listed in lexicographical + order """ - PATH_TEMPLATE = "/_changes?since=%s" - - @property - def CONFIG_BASE_FILENAME(self): # noqa: N802 - return "lister_npm_incremental" - - def get_next_target_from_response(self, response: Response) -> Optional[str]: - """(Override) Get next npm package name to continue the listing. - - """ - repos = response.json()["results"] - return repos[-1]["seq"] if len(repos) == self.per_page else None - - def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]: - """(Override) Transform npm registry response to list for model - manipulation. - - """ - repos = response.json()["results"] - if len(repos) == self.per_page: - repos = repos[:-1] - return [self.get_model_from_repo(repo["id"]) for repo in repos] + LISTER_NAME = "npm" + INSTANCE = "npm" - def filter_before_inject(self, models_list: List[Dict[str, Any]]): - """(Override) Filter out documents in the CouchDB database - not related to a npm package. + API_BASE_URL = "https://replicate.npmjs.com" + API_INCREMENTAL_LISTING_URL = f"{API_BASE_URL}/_changes" + API_FULL_LISTING_URL = f"{API_BASE_URL}/_all_docs" + PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}" - """ - models_filtered = [] - for model in models_list: - package_name = model["name"] - # document related to CouchDB internals - if package_name.startswith("_design/"): + def __init__( + self, + scheduler: SchedulerInterface, + page_size: int = 1000, + incremental: bool = False, + credentials: CredentialsType = None, + ): + super().__init__( + scheduler=scheduler, + credentials=credentials, + url=self.API_INCREMENTAL_LISTING_URL + if incremental + else self.API_FULL_LISTING_URL, + instance=self.INSTANCE, + ) + + self.page_size = page_size + if not incremental: + # in full listing mode, first package in each page corresponds to the one + # provided as the startkey query parameter value, so we increment the page + # size by one to avoid double package processing + self.page_size += 1 + self.incremental = incremental + + self.session = requests.Session() + self.session.headers.update( + {"Accept": "application/json", "User-Agent": USER_AGENT} + ) + + def state_from_dict(self, d: Dict[str, Any]) -> NpmListerState: + return NpmListerState(**d) + + def state_to_dict(self, state: NpmListerState) -> Dict[str, Any]: + return asdict(state) + + def request_params(self, last_package_id: str) -> Dict[str, Any]: + # include package JSON document to get its last update date + params = {"limit": self.page_size, "include_docs": "true"} + if self.incremental: + params["since"] = last_package_id + else: + params["startkey"] = last_package_id + return params + + @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) + def page_request(self, last_package_id: str) -> requests.Response: + params = self.request_params(last_package_id) + logger.debug("Fetching URL %s with params %s", self.url, params) + response = self.session.get(self.url, params=params) + if response.status_code != 200: + logger.warning( + "Unexpected HTTP status code %s on %s: %s", + response.status_code, + response.url, + response.content, + ) + response.raise_for_status() + return response + + def get_pages(self) -> Iterator[List[Dict[str, Any]]]: + last_package_id: str = "0" if self.incremental else '""' + if ( + self.incremental + and self.state is not None + and self.state.last_seq is not None + ): + last_package_id = str(self.state.last_seq) + + while True: + + response = self.page_request(last_package_id) + + data = response.json() + page = data["results"] if self.incremental else data["rows"] + + if not page: + break + + if self.incremental or len(page) < self.page_size: + yield page + else: + yield page[:-1] + + if len(page) < self.page_size: + break + + last_package_id = ( + str(page[-1]["seq"]) if self.incremental else f'"{page[-1]["id"]}"' + ) + + def get_origins_from_page( + self, page: List[Dict[str, Any]] + ) -> Iterator[ListedOrigin]: + """Convert a page of Npm repositories into a list of ListedOrigin.""" + assert self.lister_obj.id is not None + + for package in page: + # no source code to archive here + if not package["doc"].get("versions", {}): continue - models_filtered.append(model) - return models_filtered - - def disable_deleted_repo_tasks(self, start, end, keep_these): - """(Override) Disable the processing performed by that method as it is - not relevant in this incremental lister context. It also raises an - exception due to a different index type (int instead of str). - """ - pass + package_name = package["doc"]["name"] + package_latest_version = ( + package["doc"].get("dist-tags", {}).get("latest", "") + ) + + last_update = None + if package_latest_version in package["doc"].get("time", {}): + last_update = iso8601.parse_date( + package["doc"]["time"][package_latest_version] + ) + + yield ListedOrigin( + lister_id=self.lister_obj.id, + url=self.PACKAGE_URL_TEMPLATE.format(package_name=package_name), + visit_type="npm", + last_update=last_update, + ) + + def commit_page(self, page: List[Dict[str, Any]]): + """Update the currently stored state using the latest listed page.""" + if self.incremental: + last_package = page[-1] + last_seq = last_package["seq"] + + if self.state.last_seq is None or last_seq > self.state.last_seq: + self.state.last_seq = last_seq + + def finalize(self): + if self.incremental and self.state.last_seq is not None: + scheduler_state = self.get_state_from_scheduler() + + if ( + scheduler_state.last_seq is None + or self.state.last_seq > scheduler_state.last_seq + ): + self.updated = True diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py deleted file mode 100644 --- a/swh/lister/npm/models.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2018 the Software Heritage developers -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from sqlalchemy import BigInteger, Column, DateTime, Integer, Sequence, String - -from swh.lister.core.models import ABCSQLMeta, IndexingModelBase, SQLBase - - -class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta): - """Table to store the npm registry state at the time of a - content listing by Software Heritage - """ - - __tablename__ = "npm_visit" - - uid = Column(Integer, Sequence("npm_visit_id_seq"), primary_key=True) - visit_date = Column(DateTime, nullable=False) - doc_count = Column(BigInteger) - doc_del_count = Column(BigInteger) - update_seq = Column(BigInteger) - purge_seq = Column(BigInteger) - disk_size = Column(BigInteger) - data_size = Column(BigInteger) - committed_update_seq = Column(BigInteger) - compacted_seq = Column(BigInteger) - - -class NpmModel(IndexingModelBase): - """A npm package representation - - """ - - __tablename__ = "npm_repo" - - uid = Column(String, primary_key=True) - indexable = Column(String, index=True) diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py --- a/swh/lister/npm/tasks.py +++ b/swh/lister/npm/tasks.py @@ -1,69 +1,24 @@ -# Copyright (C) 2018 the Software Heritage developers +# Copyright (C) 2018-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from contextlib import contextmanager -from datetime import datetime - from celery import shared_task -from swh.lister.npm.lister import NpmIncrementalLister, NpmLister -from swh.lister.npm.models import NpmVisitModel - - -@contextmanager -def save_registry_state(lister): - params = {"headers": lister.request_headers()} - registry_state = lister.session.get(lister.url, **params) - registry_state = registry_state.json() - keys = ( - "doc_count", - "doc_del_count", - "update_seq", - "purge_seq", - "disk_size", - "data_size", - "committed_update_seq", - "compacted_seq", - ) - - state = {key: registry_state[key] for key in keys} - state["visit_date"] = datetime.now() - yield - npm_visit = NpmVisitModel(**state) - lister.db_session.add(npm_visit) - lister.db_session.commit() - - -def get_last_update_seq(lister): - """Get latest ``update_seq`` value for listing only updated packages. - """ - query = lister.db_session.query(NpmVisitModel.update_seq) - row = query.order_by(NpmVisitModel.uid.desc()).first() - if not row: - raise ValueError( - "No npm registry listing previously performed ! " - "This is required prior to the execution of an " - "incremental listing." - ) - return row[0] +from swh.lister.npm.lister import NpmLister @shared_task(name=__name__ + ".NpmListerTask") def list_npm_full(**lister_args): "Full lister for the npm (javascript) registry" - lister = NpmLister(**lister_args) - with save_registry_state(lister): - return lister.run() + lister = NpmLister.from_configfile(incremental=False, **lister_args) + return lister.run().dict() @shared_task(name=__name__ + ".NpmIncrementalListerTask") def list_npm_incremental(**lister_args): "Incremental lister for the npm (javascript) registry" - lister = NpmIncrementalLister(**lister_args) - update_seq_start = get_last_update_seq(lister) - with save_registry_state(lister): - return lister.run(min_bound=update_seq_start) + lister = NpmLister.from_configfile(incremental=True, **lister_args) + return lister.run().dict() @shared_task(name=__name__ + ".ping") diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,18 +9,3 @@ @pytest.fixture def lister_under_test(): return "npm" - - -@pytest.fixture -def lister_npm(swh_lister): - # Add the load-npm in the scheduler backend - swh_lister.scheduler.create_task_type( - { - "type": "load-npm", - "description": "Load npm package", - "backend_name": "swh.loader.package.tasks.LoadNpm", - "default_interval": "1 day", - } - ) - - return swh_lister diff --git a/swh/lister/npm/tests/data/api_empty_response.json b/swh/lister/npm/tests/data/api_empty_response.json deleted file mode 100644 --- a/swh/lister/npm/tests/data/api_empty_response.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "total_rows": 839080, - "offset": 839080, - "rows": [] -} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/api_inc_empty_response.json b/swh/lister/npm/tests/data/api_inc_empty_response.json deleted file mode 100644 --- a/swh/lister/npm/tests/data/api_inc_empty_response.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "results": [], - "last_seq": 6927821 -} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/api_inc_response.json b/swh/lister/npm/tests/data/api_inc_response.json deleted file mode 100644 --- a/swh/lister/npm/tests/data/api_inc_response.json +++ /dev/null @@ -1,906 +0,0 @@ -{ - "results": [ - { - "seq": 6920644, - "id": "electron-scripts", - "changes": [ - { - "rev": "3-a19944df5a3636bb225af9e0c8f9eedc" - } - ] - }, - { - "seq": 6920649, - "id": "@crexi-dev/schematics", - "changes": [ - { - "rev": "3-00188360eeca1f9123b2d7cd4b468c50" - } - ] - }, - { - "seq": 6920651, - "id": "botfactory-conversation", - "changes": [ - { - "rev": "50-f3667cde87637505528c46adc87f44e3" - } - ] - }, - { - "seq": 6920667, - "id": "castle", - "changes": [ - { - "rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0" - } - ] - }, - { - "seq": 6920671, - "id": "rbc-wm-framework-vuejs", - "changes": [ - { - "rev": "111-32bed479afacdd88aed9ac16dd135843" - } - ] - }, - { - "seq": 6920678, - "id": "bitcoinfiles", - "changes": [ - { - "rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0" - } - ] - }, - { - "seq": 6920679, - "id": "jovo-core", - "changes": [ - { - "rev": "2-d7440f1d17823e1a0760d9b3d4537c6e" - } - ] - }, - { - "seq": 6920687, - "id": "jovo-framework", - "changes": [ - { - "rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9" - } - ] - }, - { - "seq": 6920690, - "id": "smart-form-lib", - "changes": [ - { - "rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade" - } - ] - }, - { - "seq": 6920694, - "id": "bokehjs", - "changes": [ - { - "rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2" - } - ] - }, - { - "seq": 6920701, - "id": "guijarro", - "changes": [ - { - "rev": "14-82ece581d6a35d4e1d78e5292ca245c0" - } - ] - }, - { - "seq": 6920702, - "id": "@kava-labs/crypto-rate-utils", - "changes": [ - { - "rev": "3-cecc6a6c226a0590b1a685e3041028c6" - } - ] - }, - { - "seq": 6920703, - "id": "@riouxjean/test", - "changes": [ - { - "rev": "10-01e97dc7d0241dc49ea93b3468ec7b29" - } - ] - }, - { - "seq": 6920704, - "id": "react-scrabblefy", - "changes": [ - { - "rev": "7-970c8206f3b8744204f7dcb106f8462b" - } - ] - }, - { - "seq": 6920706, - "id": "molart", - "changes": [ - { - "rev": "14-416cd3cec62dd46f9b59a3bbe35308f6" - } - ] - }, - { - "seq": 6920707, - "id": "@universal-material/angular", - "changes": [ - { - "rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3" - } - ] - }, - { - "seq": 6920708, - "id": "cozy-doctypes", - "changes": [ - { - "rev": "68-8e90cc26e25da6c9430d373e43ac3c25" - } - ] - }, - { - "seq": 6920710, - "id": "2o3t-ui", - "changes": [ - { - "rev": "96-1e65d5320ea7c78525aba5daf328bd4b" - } - ] - }, - { - "seq": 6920712, - "id": "ark-ts", - "changes": [ - { - "rev": "24-033183c2f7f9cbb6e44d553213e525b6" - } - ] - }, - { - "seq": 6920715, - "id": "mysqlconnector", - "changes": [ - { - "rev": "19-f09bc0b82281ca486db5ebe83843679e" - } - ] - }, - { - "seq": 6920716, - "id": "@innovexa/ng-form-creator-lib", - "changes": [ - { - "rev": "147-480665ee17fa889dfec1aee75b907ff2" - } - ] - }, - { - "seq": 6920717, - "id": "k-routes-example-basic", - "changes": [ - { - "rev": "1-35142059e1c63cc724da71a9eebf229c" - } - ] - }, - { - "seq": 6920718, - "id": "wloggertojs", - "changes": [ - { - "rev": "29-5b5aa74bd30ff0fc86b39fba799befe2" - } - ] - }, - { - "seq": 6920720, - "id": "wloggertofile", - "changes": [ - { - "rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5" - } - ] - }, - { - "seq": 6920721, - "id": "@brightcove/flashls", - "changes": [ - { - "rev": "62-fbadb49476a58e98f0f136c86b614734" - } - ] - }, - { - "seq": 6920722, - "id": "@brightcove/hls-fetcher", - "changes": [ - { - "rev": "76-3341ed8ade38f3251a97c94c3a7af5ac" - } - ] - }, - { - "seq": 6920723, - "id": "@brightcove/kacl", - "changes": [ - { - "rev": "33-d0bc6b639cccb301086114d548ecfdbf" - } - ] - }, - { - "seq": 6920724, - "id": "just-in-types", - "changes": [ - { - "rev": "2-fc329aa885dc795aee340f36ec60f333" - } - ] - }, - { - "seq": 6920725, - "id": "@brightcove/player-loader", - "changes": [ - { - "rev": "56-9ff5aebc9743a44d46c182746313877d" - } - ] - }, - { - "seq": 6920726, - "id": "@brightcove/player-loader-webpack-plugin", - "changes": [ - { - "rev": "33-db8b4d6765f19e475e1c1d16843824cb" - } - ] - }, - { - "seq": 6920727, - "id": "@brightcove/player-url", - "changes": [ - { - "rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc" - } - ] - }, - { - "seq": 6920728, - "id": "@brightcove/react-player-loader", - "changes": [ - { - "rev": "39-b7bf609de666ec7e71f517db53ab9c0a" - } - ] - }, - { - "seq": 6920729, - "id": "vscode-theme-generator", - "changes": [ - { - "rev": "21-bcb92281d6f7e37548bb18113681df88" - } - ] - }, - { - "seq": 6920733, - "id": "@brightcove/typed-immutable-extensions", - "changes": [ - { - "rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe" - } - ] - }, - { - "seq": 6920734, - "id": "@brightcove/typed-immutable-proptypes", - "changes": [ - { - "rev": "27-e4802afc947c55d34f778864476c17e4" - } - ] - }, - { - "seq": 6920737, - "id": "@brightcove/videojs-flashls-source-handler", - "changes": [ - { - "rev": "59-faf69c49be866b2ab7faa7be9972e7a5" - } - ] - }, - { - "seq": 6920738, - "id": "@brightcove/videojs-flashls-swf", - "changes": [ - { - "rev": "60-04908466eaac2194bc3061e91f463dab" - } - ] - }, - { - "seq": 6920739, - "id": "@noqcks/generated", - "changes": [ - { - "rev": "2-e07d07614182d4beccc507ca199e612d" - } - ] - }, - { - "seq": 6920740, - "id": "pkcs7", - "changes": [ - { - "rev": "60-65ba116f3b6b705f472971b5c6a8f8d2" - } - ] - }, - { - "seq": 6920741, - "id": "videojs-errors", - "changes": [ - { - "rev": "57-c999abd162ca4b93412e363443aa688a" - } - ] - }, - { - "seq": 6920742, - "id": "videojs-flashls-source-handler", - "changes": [ - { - "rev": "59-46d62e18971a8c800710a8fbf985c1c5" - } - ] - }, - { - "seq": 6920743, - "id": "videojs-playlist", - "changes": [ - { - "rev": "97-d4b3492a94c1084c272162dd51901188" - } - ] - }, - { - "seq": 6920745, - "id": "videojs-playlist-ui", - "changes": [ - { - "rev": "95-ba97c44c354b2262e639f8c515bed9bc" - } - ] - }, - { - "seq": 6920746, - "id": "fusion-apollo-universal-client", - "changes": [ - { - "rev": "25-7123042a477cec67c7d5fc702254c7a3" - } - ] - }, - { - "seq": 6920749, - "id": "msg-fabric-core", - "changes": [ - { - "rev": "20-17c33e06faca357526c7395aca1113d2" - } - ] - }, - { - "seq": 6920750, - "id": "@expo/schemer", - "changes": [ - { - "rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d" - } - ] - }, - { - "seq": 6920752, - "id": "mathjs", - "changes": [ - { - "rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18" - } - ] - }, - { - "seq": 6920758, - "id": "statesauce-ui", - "changes": [ - { - "rev": "6-db9a39366c1a082c56a2212e368e3ae2" - } - ] - }, - { - "seq": 6920782, - "id": "@catchandrelease/arbor", - "changes": [ - { - "rev": "19-925648432b398ecadc98993e6fba2353" - } - ] - }, - { - "seq": 6920784, - "id": "discover-shared-ebsco-ui-core", - "changes": [ - { - "rev": "4-277063cbc6b71f969e5f0db8c371db65" - } - ] - }, - { - "seq": 6920807, - "id": "react-apexcharts", - "changes": [ - { - "rev": "13-18505be8026a50390c1ff1ba522cb9bd" - } - ] - }, - { - "seq": 6920819, - "id": "zigbee-shepherd-converters", - "changes": [ - { - "rev": "90-5819692a5a9679ff8669fb410e190515" - } - ] - }, - { - "seq": 6920835, - "id": "honeycomb-grid", - "changes": [ - { - "rev": "36-edd6733c80b04a72600558dc55348c73" - } - ] - }, - { - "seq": 6920838, - "id": "pixl-config", - "changes": [ - { - "rev": "7-5dd2b68d04fefb4039b3965b3497eda2" - } - ] - }, - { - "seq": 6920842, - "id": "discover-shared-ebsco-ui-theming", - "changes": [ - { - "rev": "4-e9d083825b1eae46f28c4def2d0db79f" - } - ] - }, - { - "seq": 6920843, - "id": "common-oxgalaxy-lengua-app", - "changes": [ - { - "rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f" - } - ] - }, - { - "seq": 6920845, - "id": "discover-shared-ebsco-ui-grid", - "changes": [ - { - "rev": "2-6f71cf625a5232075071952b2adaa8f2" - } - ] - }, - { - "seq": 6920847, - "id": "@auth0/cosmos-tokens", - "changes": [ - { - "rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb" - } - ] - }, - { - "seq": 6920848, - "id": "@auth0/babel-preset-cosmos", - "changes": [ - { - "rev": "43-d05d3779db08f08726ba048da298e046" - } - ] - }, - { - "seq": 6920849, - "id": "jsrender", - "changes": [ - { - "rev": "11-c949091592b3329d73ae564e45a3472d" - } - ] - }, - { - "seq": 6920850, - "id": "discover-shared-ebsco-ui-container", - "changes": [ - { - "rev": "2-c32089f76b7f253bc0d765da8b9f670d" - } - ] - }, - { - "seq": 6920852, - "id": "@auth0/cosmos", - "changes": [ - { - "rev": "42-5fdaf3d9063c20dac13dcf455c42773c" - } - ] - }, - { - "seq": 6920853, - "id": "discover-shared-ebsco-ui-checkbox", - "changes": [ - { - "rev": "2-06d9521b86f0dbf4a398726faead1212" - } - ] - }, - { - "seq": 6920854, - "id": "@adunigan/toggles", - "changes": [ - { - "rev": "1-c2a830cf814a9fe2d72084339c9c5d28" - } - ] - }, - { - "seq": 6920855, - "id": "@spriteful/spriteful-lazy-carousel", - "changes": [ - { - "rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837" - } - ] - }, - { - "seq": 6920856, - "id": "react-modal-hook", - "changes": [ - { - "rev": "2-364b39d6559364c41d5b852ccad4ce31" - } - ], - "deleted": true - }, - { - "seq": 6920859, - "id": "@bellese/angular-design-system", - "changes": [ - { - "rev": "39-3e297f85ce2d6a6b6d15fc26420fc471" - } - ] - }, - { - "seq": 6920861, - "id": "@uifabric/styling", - "changes": [ - { - "rev": "229-addf6cc0e74a335125c04d60047353f5" - } - ] - }, - { - "seq": 6920862, - "id": "@uifabric/file-type-icons", - "changes": [ - { - "rev": "37-8a7e43399d1bb9f17334b10995f78df4" - } - ] - }, - { - "seq": 6920864, - "id": "throttlewrap", - "changes": [ - { - "rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa" - } - ] - }, - { - "seq": 6920865, - "id": "airtable", - "changes": [ - { - "rev": "16-d8aee935f6fa4c88057d75a0542bc58c" - } - ] - }, - { - "seq": 6920866, - "id": "@csmart/ngc-smart-address", - "changes": [ - { - "rev": "19-66a6ea868aae1912952f232d2c699f3a" - } - ] - }, - { - "seq": 6920868, - "id": "office-ui-fabric-react", - "changes": [ - { - "rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461" - } - ] - }, - { - "seq": 6920869, - "id": "@fuelrats/eslint-config", - "changes": [ - { - "rev": "12-1b4c71b78fd078e3c1cba535e8541bed" - } - ] - }, - { - "seq": 6920870, - "id": "@uifabric/date-time", - "changes": [ - { - "rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed" - } - ] - }, - { - "seq": 6920872, - "id": "dark-client", - "changes": [ - { - "rev": "11-a954c2a89a130ae73f064233d9b3bce2" - } - ] - }, - { - "seq": 6920873, - "id": "@uifabric/variants", - "changes": [ - { - "rev": "59-391c720194c663b9a5c59fe2c10a1535" - } - ] - }, - { - "seq": 6920875, - "id": "discover-shared-ebsco-ui-header", - "changes": [ - { - "rev": "2-efd8f0426a83422a6c8b7bff11054c72" - } - ] - }, - { - "seq": 6920876, - "id": "react-responsive-picture", - "changes": [ - { - "rev": "14-32a6d0850c8af33412cfdb23afd2ecfa" - } - ] - }, - { - "seq": 6920877, - "id": "@uifabric/fluent-theme", - "changes": [ - { - "rev": "16-39c29e00b81a0b654213a5a50d7e7f42" - } - ] - }, - { - "seq": 6920878, - "id": "@uifabric/dashboard", - "changes": [ - { - "rev": "82-04d6dc25b33e811c1d8c24566127b09c" - } - ] - }, - { - "seq": 6920879, - "id": "ids-enterprise", - "changes": [ - { - "rev": "201-dd709a3912f9832440320d448850b61a" - } - ] - }, - { - "seq": 6920880, - "id": "@uifabric/experiments", - "changes": [ - { - "rev": "224-efd1ef07f7640952c286488eae282367" - } - ] - }, - { - "seq": 6920881, - "id": "@fuelrats/eslint-config-react", - "changes": [ - { - "rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc" - } - ] - }, - { - "seq": 6920883, - "id": "jsviews", - "changes": [ - { - "rev": "11-44d8bedffc98cf6ac4aa669ba8844746" - } - ] - }, - { - "seq": 6920885, - "id": "pixl-server", - "changes": [ - { - "rev": "15-823f4598c3354500d8d2a266dd062502" - } - ] - }, - { - "seq": 6920887, - "id": "@rrpm/netlify-cms-core", - "changes": [ - { - "rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa" - } - ] - }, - { - "seq": 6920889, - "id": "lodash-a", - "changes": [ - { - "rev": "2-6ee66153dbe611a080b40775387d2d45" - } - ] - }, - { - "seq": 6920891, - "id": "meshcentral", - "changes": [ - { - "rev": "499-6677ca74525ed2aa77644c68001382fe" - } - ] - }, - { - "seq": 6920892, - "id": "vue-transition-collection", - "changes": [ - { - "rev": "2-0510ee52c014c0d3b1e65f24376d76f0" - } - ] - }, - { - "seq": 6920894, - "id": "fury-adapter-swagger", - "changes": [ - { - "rev": "47-09f0c55d8574d654c67f9244c21d7ef7" - } - ] - }, - { - "seq": 6920895, - "id": "@isobar-us/redux-form-gen", - "changes": [ - { - "rev": "30-70d7d9210264a321092c832063934648" - } - ] - }, - { - "seq": 6920896, - "id": "atomizer", - "changes": [ - { - "rev": "19-129774900cb2a67a46871cc2c40c34d3" - } - ] - }, - { - "seq": 6920904, - "id": "boom-js-client", - "changes": [ - { - "rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9" - } - ] - }, - { - "seq": 6920905, - "id": "@ts-common/json-parser", - "changes": [ - { - "rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3" - } - ] - }, - { - "seq": 6920906, - "id": "rutt", - "changes": [ - { - "rev": "13-78aab849cb00a6ef7ebc8165770b7d33" - } - ] - }, - { - "seq": 6920907, - "id": "linear-react-components-ui", - "changes": [ - { - "rev": "171-0307f1d69843b270e687371c67cbd1b0" - } - ] - }, - { - "seq": 6920908, - "id": "@earnest/eslint-config", - "changes": [ - { - "rev": "180-b5250dd803102cf7dbac8da9c1a403fd" - } - ] - }, - { - "seq": 6920909, - "id": "@earnest/eslint-config-es7", - "changes": [ - { - "rev": "181-da26885e0baacaea95814857f459572d" - } - ] - }, - { - "seq": 6920910, - "id": "fuse-design", - "changes": [ - { - "rev": "10-e2b78592872f680c05e55eb5b81a0cab" - } - ] - } - ], - "last_seq": 6920912 -} \ No newline at end of file diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" deleted file mode 100644 --- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" +++ /dev/null @@ -1,807 +0,0 @@ -{ - "total_rows": 839080, - "offset": 422482, - "rows": [ - - { - "id": "jquery", - "key": "jquery", - "value": { - "rev": "212-2eac7c93af4c8bccdf7317739f0319b6" - } - }, - - { - "id": "jquery-1.8", - "key": "jquery-1.8", - "value": { - "rev": "1-711ded49a7453adce85ce7a51c2157de" - } - }, - - { - "id": "jquery-1x", - "key": "jquery-1x", - "value": { - "rev": "1-c53fa04d9c8fb231336704508732c287" - } - }, - - { - "id": "jquery-2-typescript-async-await-adapter", - "key": "jquery-2-typescript-async-await-adapter", - "value": { - "rev": "8-5cfb484e9afaa6e326a97240fccd8f93" - } - }, - - { - "id": "jquery-accessible-accordion-aria", - "key": "jquery-accessible-accordion-aria", - "value": { - "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e" - } - }, - - { - "id": "jquery-accessible-autocomplete-list-aria", - "key": "jquery-accessible-autocomplete-list-aria", - "value": { - "rev": "8-961b382442c1a5bafe58f0e05424701d" - } - }, - - { - "id": "jquery-accessible-carrousel-aria", - "key": "jquery-accessible-carrousel-aria", - "value": { - "rev": "9-f33f59d7f601bafe023bd711b551282b" - } - }, - - { - "id": "jquery-accessible-dialog-tooltip-aria", - "key": "jquery-accessible-dialog-tooltip-aria", - "value": { - "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba" - } - }, - - { - "id": "jquery-accessible-hide-show-aria", - "key": "jquery-accessible-hide-show-aria", - "value": { - "rev": "10-5a03c47a8995b08246e4bc103782dafa" - } - }, - - { - "id": "jquery-accessible-modal-window-aria", - "key": "jquery-accessible-modal-window-aria", - "value": { - "rev": "18-50266e260f6b807019cfcfcd3a3685ab" - } - }, - - { - "id": "jquery-accessible-simple-tooltip-aria", - "key": "jquery-accessible-simple-tooltip-aria", - "value": { - "rev": "6-ea71aa37760790dc603b56117f054e1b" - } - }, - - { - "id": "jquery-accessible-subnav-dropdown", - "key": "jquery-accessible-subnav-dropdown", - "value": { - "rev": "2-496f017a9ac243655225e43b5697b09b" - } - }, - - { - "id": "jquery-accessible-tabpanel-aria", - "key": "jquery-accessible-tabpanel-aria", - "value": { - "rev": "11-659971471e6ac0fbb3b2f78ad208722a" - } - }, - - { - "id": "jquery-accessible-tabs-umd", - "key": "jquery-accessible-tabs-umd", - "value": { - "rev": "1-f92015de5bb36e411d8c0940cca2883f" - } - }, - - { - "id": "jquery-active-descendant", - "key": "jquery-active-descendant", - "value": { - "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de" - } - }, - - { - "id": "jquery-ada-validation", - "key": "jquery-ada-validation", - "value": { - "rev": "1-9aab9629027c29fbece90485dd9d3112" - } - }, - - { - "id": "jquery-adaptText", - "key": "jquery-adaptText", - "value": { - "rev": "3-2e15fc801ea8235b9180a3defc782ed0" - } - }, - - { - "id": "jquery-adapttr", - "key": "jquery-adapttr", - "value": { - "rev": "6-74585f2d4be60b3f493585a6d28b90bc" - } - }, - - { - "id": "jquery-add-prefixed-class", - "key": "jquery-add-prefixed-class", - "value": { - "rev": "1-9e43aee9758504b3f5271e9804a95f20" - } - }, - - { - "id": "jquery-address", - "key": "jquery-address", - "value": { - "rev": "1-64173ede32157b26f4de910ad0f49590" - } - }, - - { - "id": "jquery-address-suggestion", - "key": "jquery-address-suggestion", - "value": { - "rev": "6-18d9df51d472c365bcd84a61c9105774" - } - }, - - { - "id": "jquery-advscrollevent", - "key": "jquery-advscrollevent", - "value": { - "rev": "1-f6033de9ba0f8e364c42826441d93119" - } - }, - - { - "id": "jquery-affix", - "key": "jquery-affix", - "value": { - "rev": "6-777371f67df59abf18ec1fe326df3b82" - } - }, - - { - "id": "jquery-airload", - "key": "jquery-airload", - "value": { - "rev": "7-136d513d2604a25238eb88709d6d9003" - } - }, - - { - "id": "jquery-ajax", - "key": "jquery-ajax", - "value": { - "rev": "1-ee358f630d4c928b52c968c7667d0d31" - } - }, - - { - "id": "jquery-ajax-cache", - "key": "jquery-ajax-cache", - "value": { - "rev": "2-ca31e0d43ae28e9cea968f1f538f06d3" - } - }, - - { - "id": "jquery-ajax-chain", - "key": "jquery-ajax-chain", - "value": { - "rev": "1-dc0e5aee651c0128b7f411aac96132a2" - } - }, - - { - "id": "jquery-ajax-file-upload", - "key": "jquery-ajax-file-upload", - "value": { - "rev": "1-96147d8bf69245c622e76583bb615d49" - } - }, - - { - "id": "jquery-ajax-json", - "key": "jquery-ajax-json", - "value": { - "rev": "1-b47eec12168e4cb39b45f1523d7cd397" - } - }, - - { - "id": "jquery-ajax-markup", - "key": "jquery-ajax-markup", - "value": { - "rev": "1-8e65dc822cb63be76c62a1323666265e" - } - }, - - { - "id": "jquery-ajax-native", - "key": "jquery-ajax-native", - "value": { - "rev": "2-9d67b8d43713e3546ad50f817c040139" - } - }, - - { - "id": "jquery-ajax-request", - "key": "jquery-ajax-request", - "value": { - "rev": "1-fdc0960ec73667bc2b46adf493c05db4" - } - }, - - { - "id": "jquery-ajax-retry", - "key": "jquery-ajax-retry", - "value": { - "rev": "1-27ca186953e346aa9c0ca2310c732751" - } - }, - - { - "id": "jquery-ajax-tracking", - "key": "jquery-ajax-tracking", - "value": { - "rev": "3-d48876f3c115ee4743a6a94bb65bb01d" - } - }, - - { - "id": "jquery-ajax-transport-xdomainrequest", - "key": "jquery-ajax-transport-xdomainrequest", - "value": { - "rev": "1-ece69aa5b9f0c950a1fa2806cf74392d" - } - }, - - { - "id": "jquery-ajax-unobtrusive", - "key": "jquery-ajax-unobtrusive", - "value": { - "rev": "3-fb0daab8480b9a2cc9c6876e1c4874f4" - } - }, - - { - "id": "jquery-ajax-unobtrusive-multi", - "key": "jquery-ajax-unobtrusive-multi", - "value": { - "rev": "1-0a2ffdabaf5708d4ae3d9e29a3a9ef11" - } - }, - - { - "id": "jquery-ajaxreadystate", - "key": "jquery-ajaxreadystate", - "value": { - "rev": "1-5e618474fe2e77ad5869c206164f82bf" - } - }, - - { - "id": "jquery-albe-timeline", - "key": "jquery-albe-timeline", - "value": { - "rev": "2-3db2b43778b5c50db873e724d9940eb6" - } - }, - - { - "id": "jquery-all-attributes", - "key": "jquery-all-attributes", - "value": { - "rev": "1-89bb7e01ee312ad5d36d78a3aa2327e4" - } - }, - - { - "id": "jquery-alphaindex", - "key": "jquery-alphaindex", - "value": { - "rev": "4-7f61cde9cfb70617a6fbe992dfcbc10a" - } - }, - - { - "id": "jquery-always", - "key": "jquery-always", - "value": { - "rev": "1-0ad944881bbc39c67df0a694d80bebef" - } - }, - - { - "id": "jquery-amd", - "key": "jquery-amd", - "value": { - "rev": "1-931646c751bef740c361dd0f6e68653c" - } - }, - - { - "id": "jquery-anaglyph-image-effect", - "key": "jquery-anaglyph-image-effect", - "value": { - "rev": "1-9bf7afce2e1bc73747ef22abc859b22b" - } - }, - - { - "id": "jquery-analytics", - "key": "jquery-analytics", - "value": { - "rev": "1-d84b0c8ce886b9f01d2c5c1cf0a7317f" - } - }, - - { - "id": "jquery-ancestors", - "key": "jquery-ancestors", - "value": { - "rev": "1-49b30817a03558f1f585c8c0cd4b8afb" - } - }, - - { - "id": "jquery-angry-loader", - "key": "jquery-angry-loader", - "value": { - "rev": "1-31c9fd950d32b9d3a73829cde1dae577" - } - }, - - { - "id": "jquery-angular-shim", - "key": "jquery-angular-shim", - "value": { - "rev": "1-723e72b2981f02dd3abcfe6d2395d636" - } - }, - - { - "id": "jquery-animate-gradient", - "key": "jquery-animate-gradient", - "value": { - "rev": "5-a3e0fc89699237e7e7241cd608a0dcf7" - } - }, - - { - "id": "jquery-animate-scroll", - "key": "jquery-animate-scroll", - "value": { - "rev": "1-37d49d89fe99aa599540e6ff83b15888" - } - }, - - { - "id": "jquery-animated-headlines", - "key": "jquery-animated-headlines", - "value": { - "rev": "1-adf1d149bc83fa8445e141e3c900759e" - } - }, - - { - "id": "jquery-animation", - "key": "jquery-animation", - "value": { - "rev": "4-f51d0559010bbe9d74d70e58de9bd733" - } - }, - - { - "id": "jquery-animation-support", - "key": "jquery-animation-support", - "value": { - "rev": "1-9013bc4bdeb2bd70bedcc988a811fcc0" - } - }, - - { - "id": "jquery-aniview", - "key": "jquery-aniview", - "value": { - "rev": "3-5754524da237693458bcff19b626b875" - } - }, - - { - "id": "jquery-anything-clickable", - "key": "jquery-anything-clickable", - "value": { - "rev": "2-e1aaaf1a369f7796c438a3efbf05bcce" - } - }, - - { - "id": "jquery-app", - "key": "jquery-app", - "value": { - "rev": "6-4e0bf5abd71c72ced3c4cf3035116f70" - } - }, - - { - "id": "jquery-app-banner", - "key": "jquery-app-banner", - "value": { - "rev": "2-8a5b530eaab94315eb00c77acd13f2dd" - } - }, - - { - "id": "jquery-appear-poetic", - "key": "jquery-appear-poetic", - "value": { - "rev": "1-368094b72ed36d42cf2fca438fa4b344" - } - }, - - { - "id": "jquery-applyonscreen", - "key": "jquery-applyonscreen", - "value": { - "rev": "4-d76c18a6e66fffba01a9a774b40663f8" - } - }, - - { - "id": "jquery-apta", - "key": "jquery-apta", - "value": { - "rev": "1-c486380fedefd887e6293a00c3b6a222" - } - }, - - { - "id": "jquery-arrow-navigate", - "key": "jquery-arrow-navigate", - "value": { - "rev": "3-0efe881e01ef0eac24a92baf1eb6d8d1" - } - }, - - { - "id": "jquery-asAccordion", - "key": "jquery-asAccordion", - "value": { - "rev": "2-2d18d3fe9089dcf67de5f29d1763b4ce" - } - }, - - { - "id": "jquery-asBgPicker", - "key": "jquery-asBgPicker", - "value": { - "rev": "2-d1403cd306d5764ee0f5aa852c2bed8e" - } - }, - - { - "id": "jquery-asBreadcrumbs", - "key": "jquery-asBreadcrumbs", - "value": { - "rev": "2-77e566a07680005ce1cb322f2a733fe4" - } - }, - - { - "id": "jquery-asCheck", - "key": "jquery-asCheck", - "value": { - "rev": "2-d0b2741b70616c7d563419cc125d193d" - } - }, - - { - "id": "jquery-asChoice", - "key": "jquery-asChoice", - "value": { - "rev": "2-0eda5269cbd59976ee904b74da209389" - } - }, - - { - "id": "jquery-asColor", - "key": "jquery-asColor", - "value": { - "rev": "3-aa730d81322561c7a3174d5c7bb6b3b8" - } - }, - - { - "id": "jquery-asColorPicker", - "key": "jquery-asColorPicker", - "value": { - "rev": "2-6bbaecaf94a324331a3d1f5d3aad3b3d" - } - }, - - { - "id": "jquery-asDropdown", - "key": "jquery-asDropdown", - "value": { - "rev": "2-b29b187cdd0bdce502d11855415e6887" - } - }, - - { - "id": "jquery-asFontEditor", - "key": "jquery-asFontEditor", - "value": { - "rev": "2-132882375101062896413afdc93b4c8c" - } - }, - - { - "id": "jquery-asGalleryPicker", - "key": "jquery-asGalleryPicker", - "value": { - "rev": "1-864a80930d72c6150aa74969a28617e4" - } - }, - - { - "id": "jquery-asGmap", - "key": "jquery-asGmap", - "value": { - "rev": "2-b0c4330774137b2f1b91bd4686880f2a" - } - }, - - { - "id": "jquery-asGradient", - "key": "jquery-asGradient", - "value": { - "rev": "2-5184670a313d5e161cb62659de3db55c" - } - }, - - { - "id": "jquery-asHoverScroll", - "key": "jquery-asHoverScroll", - "value": { - "rev": "7-3f6efebf248bd27520d03eaac33d8ca2" - } - }, - - { - "id": "jquery-asIconPicker", - "key": "jquery-asIconPicker", - "value": { - "rev": "2-9070adda148ea75247c7cee810ae91e2" - } - }, - - { - "id": "jquery-asImagePicker", - "key": "jquery-asImagePicker", - "value": { - "rev": "2-fb3115c2296b0b07ed9e379176626e01" - } - }, - - { - "id": "jquery-asItemList", - "key": "jquery-asItemList", - "value": { - "rev": "2-88a7d2900f47c785c2a6cb764ac467d6" - } - }, - - { - "id": "jquery-asModal", - "key": "jquery-asModal", - "value": { - "rev": "2-1719b8e6a489e03cc3e22bd329148366" - } - }, - - { - "id": "jquery-asOffset", - "key": "jquery-asOffset", - "value": { - "rev": "2-e45a0077e5bc0bbf91b32dc76387c945" - } - }, - - { - "id": "jquery-asPaginator", - "key": "jquery-asPaginator", - "value": { - "rev": "2-0d279d2748fc5e875f5fb2a8d3d48377" - } - }, - - { - "id": "jquery-asPieProgress", - "key": "jquery-asPieProgress", - "value": { - "rev": "2-14dc464a19e9d3feaa532f62e45bbd26" - } - }, - - { - "id": "jquery-asProgress", - "key": "jquery-asProgress", - "value": { - "rev": "2-a58d7100f1a78f7753efcf0e34dfaf0e" - } - }, - - { - "id": "jquery-asRange", - "key": "jquery-asRange", - "value": { - "rev": "3-aa3d2f348a933161868ba6b6fd9eb881" - } - }, - - { - "id": "jquery-asScroll", - "key": "jquery-asScroll", - "value": { - "rev": "1-f4880ea057adbfebb912ba0157575ca1" - } - }, - - { - "id": "jquery-asScrollable", - "key": "jquery-asScrollable", - "value": { - "rev": "7-5c18eb2180d8aa85f0b5e940667c8344" - } - }, - - { - "id": "jquery-asScrollbar", - "key": "jquery-asScrollbar", - "value": { - "rev": "4-89420658c355a5584825b45ee4ef0beb" - } - }, - - { - "id": "jquery-asSelect", - "key": "jquery-asSelect", - "value": { - "rev": "2-caf3dc516665009b654236b876fe02bb" - } - }, - - { - "id": "jquery-asSpinner", - "key": "jquery-asSpinner", - "value": { - "rev": "2-bf26b5d9c77eb4b63acbf16019407834" - } - }, - - { - "id": "jquery-asSwitch", - "key": "jquery-asSwitch", - "value": { - "rev": "2-f738586946b432caa73297568b5f38ad" - } - }, - - { - "id": "jquery-asTooltip", - "key": "jquery-asTooltip", - "value": { - "rev": "2-80d3fe5cdae70d9310969723e7045384" - } - }, - - { - "id": "jquery-asTree", - "key": "jquery-asTree", - "value": { - "rev": "2-353063a563c0322cbc317af385f71b27" - } - }, - - { - "id": "jquery-ascolorpicker-flat", - "key": "jquery-ascolorpicker-flat", - "value": { - "rev": "11-1681d53cd475e7b6b9564baa51a79611" - } - }, - - { - "id": "jquery-aslider", - "key": "jquery-aslider", - "value": { - "rev": "1-2b3dd953493eeaa4dc329cbf0d81116a" - } - }, - - { - "id": "jquery-aspect-ratio-keeper", - "key": "jquery-aspect-ratio-keeper", - "value": { - "rev": "1-1ad8e5588218e1d38fff351858655eda" - } - }, - - { - "id": "jquery-assinadordigitaldiscus", - "key": "jquery-assinadordigitaldiscus", - "value": { - "rev": "1-897cd68ef3699551630bd3454dceb6f0" - } - }, - - { - "id": "jquery-async-gravatar", - "key": "jquery-async-gravatar", - "value": { - "rev": "3-a3192e741d14d57635f4ebfb41a904db" - } - }, - - { - "id": "jquery-asynclink", - "key": "jquery-asynclink", - "value": { - "rev": "1-2159a3c49e3c8fe9280c592770e83522" - } - }, - - { - "id": "jquery-atlas", - "key": "jquery-atlas", - "value": { - "rev": "1-6142c5a0af67a0470daf36151d3f9d8c" - } - }, - - { - "id": "jquery-atomic-nav", - "key": "jquery-atomic-nav", - "value": { - "rev": "1-18e4ef14be83a907cbee0cd0adee25d4" - } - }, - - { - "id": "jquery-attach", - "key": "jquery-attach", - "value": { - "rev": "8-da4f17596c25a02b0cce266e59706d5f" - } - } - - ] -} diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" deleted file mode 100644 --- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" +++ /dev/null @@ -1,83 +0,0 @@ -{ - "total_rows": 839080, - "offset": 422482, - "rows": [ - { - "id": "jquery", - "key": "jquery", - "value": { - "rev": "212-2eac7c93af4c8bccdf7317739f0319b6" - } - }, - { - "id": "jquery-1.8", - "key": "jquery-1.8", - "value": { - "rev": "1-711ded49a7453adce85ce7a51c2157de" - } - }, - { - "id": "jquery-1x", - "key": "jquery-1x", - "value": { - "rev": "1-c53fa04d9c8fb231336704508732c287" - } - }, - { - "id": "jquery-2-typescript-async-await-adapter", - "key": "jquery-2-typescript-async-await-adapter", - "value": { - "rev": "8-5cfb484e9afaa6e326a97240fccd8f93" - } - }, - { - "id": "jquery-accessible-accordion-aria", - "key": "jquery-accessible-accordion-aria", - "value": { - "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e" - } - }, - { - "id": "jquery-accessible-autocomplete-list-aria", - "key": "jquery-accessible-autocomplete-list-aria", - "value": { - "rev": "8-961b382442c1a5bafe58f0e05424701d" - } - }, - { - "id": "jquery-accessible-carrousel-aria", - "key": "jquery-accessible-carrousel-aria", - "value": { - "rev": "9-f33f59d7f601bafe023bd711b551282b" - } - }, - { - "id": "jquery-accessible-dialog-tooltip-aria", - "key": "jquery-accessible-dialog-tooltip-aria", - "value": { - "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba" - } - }, - { - "id": "jquery-accessible-hide-show-aria", - "key": "jquery-accessible-hide-show-aria", - "value": { - "rev": "10-5a03c47a8995b08246e4bc103782dafa" - } - }, - { - "id": "jquery-accessible-modal-window-aria", - "key": "jquery-accessible-modal-window-aria", - "value": { - "rev": "18-50266e260f6b807019cfcfcd3a3685ab" - } - }, - { - "id": "jquery-accessible-simple-tooltip-aria", - "key": "jquery-accessible-simple-tooltip-aria", - "value": { - "rev": "6-ea71aa37760790dc603b56117f054e1b" - } - } - ] -} diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" deleted file mode 100644 --- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" +++ /dev/null @@ -1,83 +0,0 @@ -{ - "total_rows": 839080, - "offset": 422482, - "rows": [ - { - "id": "jquery-accessible-simple-tooltip-aria", - "key": "jquery-accessible-simple-tooltip-aria", - "value": { - "rev": "6-ea71aa37760790dc603b56117f054e1b" - } - }, - { - "id": "jquery-accessible-subnav-dropdown", - "key": "jquery-accessible-subnav-dropdown", - "value": { - "rev": "2-496f017a9ac243655225e43b5697b09b" - } - }, - { - "id": "jquery-accessible-tabpanel-aria", - "key": "jquery-accessible-tabpanel-aria", - "value": { - "rev": "11-659971471e6ac0fbb3b2f78ad208722a" - } - }, - { - "id": "jquery-accessible-tabs-umd", - "key": "jquery-accessible-tabs-umd", - "value": { - "rev": "1-f92015de5bb36e411d8c0940cca2883f" - } - }, - { - "id": "jquery-active-descendant", - "key": "jquery-active-descendant", - "value": { - "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de" - } - }, - { - "id": "jquery-ada-validation", - "key": "jquery-ada-validation", - "value": { - "rev": "1-9aab9629027c29fbece90485dd9d3112" - } - }, - { - "id": "jquery-adaptText", - "key": "jquery-adaptText", - "value": { - "rev": "3-2e15fc801ea8235b9180a3defc782ed0" - } - }, - { - "id": "jquery-adapttr", - "key": "jquery-adapttr", - "value": { - "rev": "6-74585f2d4be60b3f493585a6d28b90bc" - } - }, - { - "id": "jquery-add-prefixed-class", - "key": "jquery-add-prefixed-class", - "value": { - "rev": "1-9e43aee9758504b3f5271e9804a95f20" - } - }, - { - "id": "jquery-address", - "key": "jquery-address", - "value": { - "rev": "1-64173ede32157b26f4de910ad0f49590" - } - }, - { - "id": "jquery-address-suggestion", - "key": "jquery-address-suggestion", - "value": { - "rev": "6-18d9df51d472c365bcd84a61c9105774" - } - } - ] -} diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json deleted file mode 120000 --- a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json +++ /dev/null @@ -1 +0,0 @@ -_all_docs,startkey="",limit=1001 \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_full_page1.json b/swh/lister/npm/tests/data/npm_full_page1.json new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/data/npm_full_page1.json @@ -0,0 +1,235 @@ +{ + "total_rows": 1496481, + "offset": 0, + "rows": [{ + "id": "-", + "key": "-", + "value": { + "rev": "1-e340e64b090ea8c6b4e14edc0460c751" + }, + "doc": { + "_id": "-", + "_rev": "1-e340e64b090ea8c6b4e14edc0460c751", + "name": "-", + "dist-tags": { + "latest": "0.0.1" + }, + "versions": { + "0.0.1": { + "name": "-", + "version": "0.0.1", + "license": "UNLICENSED", + "keywords": [], + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node --require ts-node/register src", + "build": "tsc" + }, + "devDependencies": { + "@types/node": "13.9.0", + "ts-node": "8.6.2", + "typescript": "3.8.3" + }, + "description": "> Created using https://github.com/parzh/create-package-typescript", + "_id": "-@0.0.1", + "_nodeVersion": "13.3.0", + "_npmVersion": "6.14.4", + "dist": { + "integrity": "sha512-3HfneK3DGAm05fpyj20sT3apkNcvPpCuccOThOPdzz8sY7GgQGe0l93XH9bt+YzibcTIgUAIMoyVJI740RtgyQ==", + "shasum": "db6db7cd866142880dd03e5b8781d1b4fac0e5bd", + "tarball": "https://registry.npmjs.org/-/-/--0.0.1.tgz", + "fileCount": 3, + "unpackedSize": 600, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJehdEkCRA9TVsSAnZWagAAqcEQAIUCCWB2RzNV2p5CTeWf\nntknmFRbGh3K/gptyBnTC3MaNtk54Vf36XK1BfKgKz2rd/FOq1xcSQ8XUWt6\n7ZL9zSR3K/EBEgNr6p00fJAJ7ZdYu2kqq+uiZDKpSHLd6/bKaSdt0bAL78co\nmnzKOn9WULS1UC/ac8txNX9QdPg/aJezzxQadtKQR+SX6Fd/tTHKIsWv3ytI\nUtH/Lg9AM/tdTc94m2Ivo3+JxmkU+p8z7hjR97ym+AaZE8g1a47begVsSeZ/\nrIu1ZFCQ/XYFp0amNZ131qnW6FGnzpPlQTfkuJ1sPIORM9+o+6Ql6qa0mtde\nSJyAMKFfemmwkp56aZkL5NFX1snyJE4itfg99gsUrogHZIW3Jd1LqUe9yFZc\nVik1NZxqG4x6ktoog9IzG8ElOBWauKQAT+FIgNGea6IBv4wPQ8VoNOd4JKW/\nadvzus0hAdEZLyVsCyDw4wDCAK9JJXnSlJmJFc/vPtgPrbAj8xcBW6XIn6Nw\n0EVm993zlKmQ6/VmQUbHx8EP1zBm7M0vTcoI8dPIJkjGCZgWU571JARCetb2\npob+siYon3ndpoHG2OOWOuoRhwi4YeDW1ilc9isPve+S/iygKnfY8LTqHN1P\nOntz9Cu9YxHQ3WEiBuun7KP7FECbiSkozmzKWLHUhs2qXFSFYWeBLEI/HS1y\nJ9QO\r\n=lB+Q\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "parzhitsky", + "email": "parzhitsky@gmail.com" + }], + "_npmUser": { + "name": "parzhitsky", + "email": "parzhitsky@gmail.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-_0.0.1_1585828132159_0.6185838126718755" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-04-02T11:48:52.159Z", + "0.0.1": "2020-04-02T11:48:52.339Z", + "modified": "2020-04-02T11:49:00.248Z" + }, + "maintainers": [{ + "name": "parzhitsky", + "email": "parzhitsky@gmail.com" + }], + "description": "> Created using https://github.com/parzh/create-package-typescript", + "keywords": [], + "license": "UNLICENSED", + "readme": "# `-`\r\n\r\n> Created using https://github.com/parzh/create-package-typescript\r\n\r\n```\r\nnpm i -\r\n```\r\n", + "readmeFilename": "README.md" + } + }, + { + "id": "--file--1", + "key": "--file--1", + "value": { + "rev": "1-b23c0a584f925336378c29df0b858aff" + }, + "doc": { + "_id": "--file--1", + "_rev": "1-b23c0a584f925336378c29df0b858aff", + "name": "--file--1", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "--file--1", + "version": "1.0.0", + "description": "", + "main": "file.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "_id": "--file--1@1.0.0", + "_nodeVersion": "15.3.0", + "_npmVersion": "7.0.14", + "dist": { + "integrity": "sha512-OtGGxxqxDl1KCCUts3fN0gL4vW28u8WNIo+eqQON78IVxKJS3gYchaEZDrRubC4SyQOhDDKDtvswwLOkApuicQ==", + "shasum": "58ed9d9169197ded653d8df7153968d22c14ea59", + "tarball": "https://registry.npmjs.org/--file--1/-/--file--1-1.0.0.tgz", + "fileCount": 6, + "unpackedSize": 1327, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf2AGJCRA9TVsSAnZWagAALqcP+wZ9tzFsdQy4cEb5HGV6\nxV0LTnW+wxFYBUm2IfOwK7UH/Pl5Nym4yxlIjMEBinb9efepUAsucCyAka/v\n0OBGn+8I8kHDKvfhxiUVbi3CFAYcQTaaPffK5xJZi77netdsaLyS0cdx4RBm\nEWRIVK4axJBSUT6UVpmRqB73OlAv+JmNzBmL+CXHn9gksyvnm74pbT8tukLV\nX0TwkXNBtzn4lmydPRyecEZRDFydzp9UBQJyJACtpYxgTAwTT5ENtSymx4Ql\nlEqC9cqT9Q3n7z8lvntaCb6jgNN1aSWP3hOKCYQzezk5uSacJDWJOGAK7DE5\ndKUqsWa0HG84H+MMLxuepbrNYM5Fm7ulykl1yMrAf1G8T5pU/fUpq5uEHbqW\naVbW3GlCxJuUvlaTTFjGqDJx4tepY+Uel+TQigJoN2DGXTWiB+JeAAWLP+cu\nqcoZwkqoISmgru2g1+6TR//hod3CnCK4eQl/ph3VthtKyh4d1zUQAPcCGax6\nocJSWQsLRNW0QUmvpHeHQQOUGEb0XDTYkl9i8s8DY6JCqfgl9UIDZzNl2nK8\n385JseJKKLQR3tEk+HLZM2UGF6XgACBFTHYxlOrpzGgX+PJ1ns3noMrKo2jO\nbnvMN6Mp0GtoSzQj8FUcoU6BgOFHhHkB/i4Ke2VVeMYVj/CrFO13pZJOPR4O\n6uts\r\n=W7vP\r\n-----END PGP SIGNATURE-----\r\n" + }, + "_npmUser": { + "name": "guosuzhen", + "email": "1077655229@qq.com" + }, + "directories": {}, + "maintainers": [{ + "name": "guosuzhen", + "email": "1077655229@qq.com" + }], + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/--file--1_1.0.0_1607991689533_0.6331916469772794" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-12-15T00:21:29.533Z", + "1.0.0": "2020-12-15T00:21:29.657Z", + "modified": "2020-12-15T00:21:32.116Z" + }, + "maintainers": [{ + "name": "guosuzhen", + "email": "1077655229@qq.com" + }], + "keywords": [], + "license": "ISC", + "readme": "ERROR: No README data found!", + "readmeFilename": "" + } + }, + { + "id": "-keyboardevent", + "key": "-keyboardevent", + "value": { + "rev": "1-0fdff278618c4f8c1026889f1b525483" + }, + "doc": { + "_id": "-keyboardevent", + "_rev": "1-0fdff278618c4f8c1026889f1b525483", + "name": "-keyboardevent", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "-keyboardevent", + "version": "1.0.0", + "description": "监听键盘事件", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "license": "ISO", + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3", + "_id": "-keyboardevent@1.0.0", + "_nodeVersion": "12.10.0", + "_npmVersion": "6.10.3", + "dist": { + "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==", + "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5", + "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz", + "fileCount": 3, + "unpackedSize": 2714, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "_npmUser": { + "name": "yanggw", + "email": "2993065178@qq.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-09-25T06:10:56.357Z", + "1.0.0": "2020-09-25T06:10:56.513Z", + "modified": "2020-09-25T06:10:58.774Z" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "description": "监听键盘事件", + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "license": "ISO", + "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n", + "readmeFilename": "README.md" + } + } + ] +} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_full_page2.json b/swh/lister/npm/tests/data/npm_full_page2.json new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/data/npm_full_page2.json @@ -0,0 +1,168 @@ +{ + "total_rows": 1496482, + "offset": 2, + "rows": [{ + "id": "-keyboardevent", + "key": "-keyboardevent", + "value": { + "rev": "1-0fdff278618c4f8c1026889f1b525483" + }, + "doc": { + "_id": "-keyboardevent", + "_rev": "1-0fdff278618c4f8c1026889f1b525483", + "name": "-keyboardevent", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "-keyboardevent", + "version": "1.0.0", + "description": "监听键盘事件", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "license": "ISO", + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3", + "_id": "-keyboardevent@1.0.0", + "_nodeVersion": "12.10.0", + "_npmVersion": "6.10.3", + "dist": { + "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==", + "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5", + "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz", + "fileCount": 3, + "unpackedSize": 2714, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "_npmUser": { + "name": "yanggw", + "email": "2993065178@qq.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-09-25T06:10:56.357Z", + "1.0.0": "2020-09-25T06:10:56.513Z", + "modified": "2020-09-25T06:10:58.774Z" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "description": "监听键盘事件", + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "license": "ISO", + "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n", + "readmeFilename": "README.md" + } + }, + { + "id": "-lidonghui", + "key": "-lidonghui", + "value": { + "rev": "1-1aa03a85878671cdb74b789c2b124578" + }, + "doc": { + "_id": "-lidonghui", + "_rev": "1-1aa03a85878671cdb74b789c2b124578", + "name": "-lidonghui", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "-lidonghui", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "bin": { + "test1-cli": "bin/index.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^0.19.2", + "commander": "^5.0.0", + "inquirer": "^7.1.0" + }, + "_id": "-lidonghui@1.0.0", + "_nodeVersion": "12.14.0", + "_npmVersion": "6.13.4", + "dist": { + "integrity": "sha512-1eUTEDU1kgAAsQ85uCEAp9tDsEIh3WNa5dB/ks8Uhf9Nh2RR37EeTGyxszn2A1B+MN+Q0fWrpCfBFzrrFyLFnA==", + "shasum": "be7ffedc328bd123ba13c468a89ebcf0cfe76777", + "tarball": "https://registry.npmjs.org/-lidonghui/-/-lidonghui-1.0.0.tgz", + "fileCount": 2, + "unpackedSize": 1024, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJenVH+CRA9TVsSAnZWagAAEIMP/ipex7Yo+T2LqfWtFuZz\n5NuXhcnaNyANXyJ7iQCYzq1ynq+tkwBOTM7FNLkTbIiIUsn8d6JlAt5CfIFZ\nrfaYDK7BhU4E8KfLoSC0VqyaprTSvAeqeQDEAguYM9Nbt4TWa7W2HoorgQd7\njUAuGpRqvkw8Omh3eOvhqEa0Y5mkRf853ndJHD425bAww2R0KCA7RuOBO1MW\nnPZqBe3XOgJmsyEW+yl2tSGVq2hEEeD3C0G9x5K2ZfxZKVU7lGfqRlUdo7na\nTLJU6YZPTB9AhDTM9b8XFUMWlEtvHcwvGvuB+gLupXqWwEfwSqWg/2TJ11AQ\nMBFeC1xSav0BCv6kFxFBRwz1k7IAdVYatLnmqwNsIGwjWsyfmIThSNzYTIq6\nE/E0iD2gSAsLW5r7U0t+rk4sWLmu4xbLDvT1xpEY7C00Zj4NXJhdXSo5BR45\n1IBiU96LqEA2bIw+TsyaFO7w/chgpvd8mU0g34tYQCcAts56ZhCr43x5kUSY\ns7hm1b3MB5qpOBbom1Ta24xjvoJwxv2Et/T7zfZoXzHDyX9BgUCiVIvupxd9\nzR5LdyXIh6JdiVBAgEftbm59FEVrxVfAAi/sFMNJKp8pE1ELrSp/gulzxDd2\nC5Hq1z5RNFRQUlbkMJHLIDOZLAvdkPKieE+uFJ7WTYCPEL9nvQoIJiO9UFH1\njKtU\r\n=Opv1\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "tenderli", + "email": "l2369660482@163.com" + }], + "_npmUser": { + "name": "tenderli", + "email": "l2369660482@163.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-lidonghui_1.0.0_1587368445998_0.028137547296155807" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-04-20T07:40:45.997Z", + "1.0.0": "2020-04-20T07:40:46.087Z", + "modified": "2020-04-20T07:40:48.525Z" + }, + "maintainers": [{ + "name": "tenderli", + "email": "l2369660482@163.com" + }], + "keywords": [], + "license": "ISC", + "readme": "ERROR: No README data found!", + "readmeFilename": "" + } + } + ] +} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_incremental_page1.json b/swh/lister/npm/tests/data/npm_incremental_page1.json new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/data/npm_incremental_page1.json @@ -0,0 +1,175 @@ +{ + "results": [{ + "seq": 1, + "id": "overlay-fishnet", + "changes": [{ + "rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f" + }], + "doc": { + "_id": "overlay-fishnet", + "_rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f", + "name": "overlay-fishnet", + "description": "Overlay polygon with equal sized tiles (like a fishnet)", + "dist-tags": { + "latest": "0.0.0" + }, + "versions": { + "0.0.0": { + "name": "overlay-fishnet", + "version": "0.0.0", + "description": "Overlay polygon with equal sized tiles (like a fishnet)", + "main": "dist/index.js", + "scripts": { + "build": "rm -rf dist && mkdir dist && babel src --out-dir dist", + "prepublish": "npm run test && npm run build", + "test": "standard" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/yongjun21/fishnet.git" + }, + "keywords": [ + "geospatial", + "polygon", + "tiles", + "choropleth" + ], + "author": { + "name": "Yong Jun" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/yongjun21/fishnet/issues" + }, + "homepage": "https://github.com/yongjun21/fishnet#readme", + "dependencies": { + "babel-cli": "^6.18.0", + "babel-preset-es2015": "^6.18.0", + "babel-preset-stage-3": "^6.17.0", + "standard": "^8.6.0" + }, + "gitHead": "bc3d598445a767efefd094260c83d6f57f06e86d", + "_id": "overlay-fishnet@0.0.0", + "_shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0", + "_from": ".", + "_npmVersion": "3.10.7", + "_nodeVersion": "6.2.2", + "_npmUser": { + "name": "yongjun21", + "email": "yjthong.2009@smu.edu.sg" + }, + "dist": { + "shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0", + "tarball": "https://registry.npmjs.org/overlay-fishnet/-/overlay-fishnet-0.0.0.tgz" + }, + "maintainers": [{ + "name": "yongjun21", + "email": "yjthong.2009@smu.edu.sg" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/overlay-fishnet-0.0.0.tgz_1484192243466_0.4572450101841241" + } + } + }, + "readme": "# fishnet\nOverlay polygon with equal sized tiles (like a fishnet)\n", + "maintainers": [{ + "name": "yongjun21", + "email": "yjthong.2009@smu.edu.sg" + }], + "time": { + "modified": "2017-01-12T03:37:23.696Z", + "created": "2017-01-12T03:37:23.696Z", + "0.0.0": "2017-01-12T03:37:23.696Z" + }, + "homepage": "https://github.com/yongjun21/fishnet#readme", + "keywords": [ + "geospatial", + "polygon", + "tiles", + "choropleth" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/yongjun21/fishnet.git" + }, + "author": { + "name": "Yong Jun" + }, + "bugs": { + "url": "https://github.com/yongjun21/fishnet/issues" + }, + "license": "ISC", + "readmeFilename": "README.md" + } + }, + { + "seq": 2, + "id": "shopping_site_detect", + "changes": [{ + "rev": "1-c62a780cdd9865b06b1e4eb0e6552560" + }], + "doc": { + "_id": "shopping_site_detect", + "_rev": "1-c62a780cdd9865b06b1e4eb0e6552560", + "name": "shopping_site_detect", + "description": "shopping site detect", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "shopping_site_detect", + "version": "1.0.0", + "description": "shopping site detect", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": { + "name": "Jayin Ton" + }, + "license": "MIT", + "gitHead": "223c4a7e12acf28310a106780652d1e7d0629d68", + "_id": "shopping_site_detect@1.0.0", + "_shasum": "0408f054053d7581869d42934e42d50794d20b8c", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.4.0", + "_npmUser": { + "name": "jayinton", + "email": "tonjayin@gmail.com" + }, + "dist": { + "shasum": "0408f054053d7581869d42934e42d50794d20b8c", + "tarball": "https://registry.npmjs.org/shopping_site_detect/-/shopping_site_detect-1.0.0.tgz" + }, + "maintainers": [{ + "name": "jayinton", + "email": "tonjayin@gmail.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/shopping_site_detect-1.0.0.tgz_1484193398924_0.3402463826350868" + } + } + }, + "maintainers": [{ + "name": "jayinton", + "email": "tonjayin@gmail.com" + }], + "time": { + "modified": "2017-01-12T03:56:39.161Z", + "created": "2017-01-12T03:56:39.161Z", + "1.0.0": "2017-01-12T03:56:39.161Z" + }, + "author": { + "name": "Jayin Ton" + }, + "license": "MIT", + "readmeFilename": "readme.md" + } + } + ], + "last_seq": 2 +} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_incremental_page2.json b/swh/lister/npm/tests/data/npm_incremental_page2.json new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/data/npm_incremental_page2.json @@ -0,0 +1,632 @@ +{ + "results": [{ + "seq": 3, + "id": "tinyviewpager", + "changes": [{ + "rev": "1-01e65a5602e9c43ed3482657f1309ef0" + }], + "doc": { + "_id": "tinyviewpager", + "_rev": "1-01e65a5602e9c43ed3482657f1309ef0", + "name": "tinyviewpager", + "time": { + "modified": "2017-01-12T04:05:34.847Z", + "created": "2017-01-11T10:52:41.047Z", + "1.0.0": "2017-01-11T10:52:41.047Z", + "1.0.1": "2017-01-11T14:02:03.232Z", + "1.0.2": "2017-01-11T14:31:04.820Z", + "1.0.3": "2017-01-11T14:38:41.688Z", + "1.0.4": "2017-01-11T14:45:57.970Z", + "1.0.5": "2017-01-11T14:49:04.778Z", + "1.0.6": "2017-01-11T14:54:06.876Z", + "1.0.7": "2017-01-11T15:07:13.271Z", + "1.0.8": "2017-01-12T04:05:34.847Z" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "dist-tags": { + "latest": "1.0.8" + }, + "description": "上下左右无限循环viewpager", + "versions": { + "1.0.1": { + "name": "tinyviewpager", + "version": "1.0.1", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "fdb455a393c7d6b6d43d01f73362fa6e3ed0dbb9", + "_id": "tinyviewpager@1.0.1", + "_shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.1.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.1.tgz_1484143322968_0.5517262546345592" + } + }, + "1.0.2": { + "name": "tinyviewpager", + "version": "1.0.2", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "daae30e6ce7bea30a14619b5cef5f52a86cc5528", + "_id": "tinyviewpager@1.0.2", + "_shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.2.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.2.tgz_1484145064595_0.49263988458551466" + } + }, + "1.0.3": { + "name": "tinyviewpager", + "version": "1.0.3", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "d08aeab765b537d71ed21e7e37ae6546e9bdbef2", + "_id": "tinyviewpager@1.0.3", + "_shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.3.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.3.tgz_1484145521456_0.8925717421807349" + } + }, + "1.0.4": { + "name": "tinyviewpager", + "version": "1.0.4", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.4", + "_shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.4.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.4.tgz_1484145957736_0.7274783558677882" + } + }, + "1.0.5": { + "name": "tinyviewpager", + "version": "1.0.5", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.5", + "_shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.5.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.5.tgz_1484146144523_0.9442351886536926" + } + }, + "1.0.6": { + "name": "tinyviewpager", + "version": "1.0.6", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.6", + "_shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.6.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.6.tgz_1484146444932_0.7940177812706679" + } + }, + "1.0.7": { + "name": "tinyviewpager", + "version": "1.0.7", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.7", + "_shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.7.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.7.tgz_1484147231500_0.17695364635437727" + } + }, + "1.0.8": { + "name": "tinyviewpager", + "version": "1.0.8", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "54b9e9f1517a0385307065f96ef85e9b7b13868e", + "_id": "tinyviewpager@1.0.8", + "_shasum": "67234eebbae3a8205319c950fab62c8f84840061", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "67234eebbae3a8205319c950fab62c8f84840061", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.8.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.8.tgz_1484193932800_0.371786616044119" + } + } + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "keywords": [ + "viewpager" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "author": { + "name": "tinylpc" + }, + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "license": "ISC", + "readmeFilename": "README.md" + } + }, + { + "seq": 4, + "id": "ubase-ext-wecloud", + "changes": [{ + "rev": "1-0a540144594976690ff540d57ed3ab3b" + }], + "doc": { + "_id": "ubase-ext-wecloud", + "_rev": "1-0a540144594976690ff540d57ed3ab3b", + "name": "ubase-ext-wecloud", + "dist-tags": { + "latest": "1.0.3" + }, + "versions": { + "1.0.0": { + "name": "ubase-ext-wecloud", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "_id": "ubase-ext-wecloud@1.0.0", + "_shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.0.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.0.tgz_1484191505909_0.5115192488301545" + } + }, + "1.0.1": { + "name": "ubase-ext-wecloud", + "version": "1.0.1", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "_id": "ubase-ext-wecloud@1.0.1", + "_shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.1.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.1.tgz_1484191845449_0.817801987985149" + } + }, + "1.0.2": { + "name": "ubase-ext-wecloud", + "version": "1.0.2", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "_id": "ubase-ext-wecloud@1.0.2", + "_shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.2.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.2.tgz_1484192705755_0.1207962513435632" + } + }, + "1.0.3": { + "name": "ubase-ext-wecloud", + "version": "1.0.3", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "gitHead": "a5b60823b706c4fd98ae798dbe76f18a7841eead", + "_id": "ubase-ext-wecloud@1.0.3", + "_shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.3.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.3.tgz_1484193826093_0.8929786814842373" + } + } + }, + "readme": "# ubase-ext-wecloud\n", + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "time": { + "modified": "2017-01-12T04:03:46.329Z", + "created": "2017-01-12T03:25:07.858Z", + "1.0.0": "2017-01-12T03:25:07.858Z", + "1.0.1": "2017-01-12T03:30:45.704Z", + "1.0.2": "2017-01-12T03:45:07.635Z", + "1.0.3": "2017-01-12T04:03:46.329Z" + }, + "license": "ISC", + "readmeFilename": "README.md" + } + } + ], + "last_seq": 4 +} \ No newline at end of file diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -1,97 +1,200 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import logging -import re -from typing import Any, List -import unittest +from itertools import chain +import json +from pathlib import Path -import requests_mock +import iso8601 +import pytest +from requests.exceptions import HTTPError -from swh.lister.core.tests.test_lister import HttpListerTesterBase -from swh.lister.npm.lister import NpmIncrementalLister, NpmLister +from swh.lister import USER_AGENT +from swh.lister.npm.lister import NpmLister, NpmListerState -logger = logging.getLogger(__name__) +@pytest.fixture +def npm_full_listing_page1(datadir): + return json.loads(Path(datadir, "npm_full_page1.json").read_text()) -class NpmListerTester(HttpListerTesterBase, unittest.TestCase): - Lister = NpmLister - test_re = re.compile(r'^.*/_all_docs\?startkey="(.+)".*') - lister_subdir = "npm" - good_api_response_file = "data/replicate.npmjs.com/api_response.json" - bad_api_response_file = "data/api_empty_response.json" - first_index = "jquery" - entries_per_page = 100 - @requests_mock.Mocker() - def test_is_within_bounds(self, http_mocker): - # disable this test from HttpListerTesterBase as - # it can not succeed for the npm lister due to the - # overriding of the string_pattern_check method - pass +@pytest.fixture +def npm_full_listing_page2(datadir): + return json.loads(Path(datadir, "npm_full_page2.json").read_text()) -class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase): - Lister = NpmIncrementalLister - test_re = re.compile(r"^.*/_changes\?since=([0-9]+).*") - lister_subdir = "npm" - good_api_response_file = "data/api_inc_response.json" - bad_api_response_file = "data/api_inc_empty_response.json" - first_index = "6920642" - entries_per_page = 100 +@pytest.fixture +def npm_incremental_listing_page1(datadir): + return json.loads(Path(datadir, "npm_incremental_page1.json").read_text()) - @requests_mock.Mocker() - def test_is_within_bounds(self, http_mocker): - # disable this test from HttpListerTesterBase as - # it can not succeed for the npm lister due to the - # overriding of the string_pattern_check method - pass +@pytest.fixture +def npm_incremental_listing_page2(datadir): + return json.loads(Path(datadir, "npm_incremental_page2.json").read_text()) -def check_tasks(tasks: List[Any]): - """Ensure scheduled tasks are in the expected format. +def _check_listed_npm_packages(lister, packages, scheduler_origins): + for package in packages: + package_name = package["doc"]["name"] + latest_version = package["doc"]["dist-tags"]["latest"] + package_last_update = iso8601.parse_date(package["doc"]["time"][latest_version]) + origin_url = lister.PACKAGE_URL_TEMPLATE.format(package_name=package_name) - """ - for row in tasks: - logger.debug("row: %s", row) - assert row["type"] == "load-npm" - # arguments check - args = row["arguments"]["args"] - assert len(args) == 0 + scheduler_origin = [o for o in scheduler_origins if o.url == origin_url] + assert scheduler_origin + assert scheduler_origin[0].last_update == package_last_update - # kwargs - kwargs = row["arguments"]["kwargs"] - assert len(kwargs) == 1 - package_url = kwargs["url"] - package_name = package_url.split("/")[-1] - assert package_url == f"https://www.npmjs.com/package/{package_name}" - assert row["policy"] == "recurring" - assert row["priority"] is None +def _match_request(request): + return request.headers.get("User-Agent") == USER_AGENT -def test_npm_lister_basic_listing(lister_npm, requests_mock_datadir): - lister_npm.run() +def _url_params(page_size, **kwargs): + params = {"limit": page_size, "include_docs": "true"} + params.update(**kwargs) + return params - tasks = lister_npm.scheduler.search_tasks(task_type="load-npm") - assert len(tasks) == 100 - check_tasks(tasks) +def test_npm_lister_full( + swh_scheduler, requests_mock, mocker, npm_full_listing_page1, npm_full_listing_page2 +): + """Simulate a full listing of four npm packages in two pages""" + page_size = 2 + lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=False) + requests_mock.get( + lister.API_FULL_LISTING_URL, + [{"json": npm_full_listing_page1}, {"json": npm_full_listing_page2},], + additional_matcher=_match_request, + ) + + spy_get = mocker.spy(lister.session, "get") + + stats = lister.run() + assert stats.pages == 2 + assert stats.origins == page_size * stats.pages + + spy_get.assert_has_calls( + [ + mocker.call( + lister.API_FULL_LISTING_URL, + params=_url_params(page_size + 1, startkey='""'), + ), + mocker.call( + lister.API_FULL_LISTING_URL, + params=_url_params( + page_size + 1, + startkey=f'"{npm_full_listing_page1["rows"][-1]["id"]}"', + ), + ), + ] + ) + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins + + _check_listed_npm_packages( + lister, + chain(npm_full_listing_page1["rows"][:-1], npm_full_listing_page2["rows"]), + scheduler_origins, + ) + + assert lister.get_state_from_scheduler() == NpmListerState() + + +def test_npm_lister_incremental( + swh_scheduler, + requests_mock, + mocker, + npm_incremental_listing_page1, + npm_incremental_listing_page2, +): + """Simulate an incremental listing of four npm packages in two pages""" + page_size = 2 + lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True) + + requests_mock.get( + lister.API_INCREMENTAL_LISTING_URL, + [ + {"json": npm_incremental_listing_page1}, + {"json": npm_incremental_listing_page2}, + {"json": {"results": []}}, + ], + additional_matcher=_match_request, + ) + + spy_get = mocker.spy(lister.session, "get") + + assert lister.get_state_from_scheduler() == NpmListerState() + + stats = lister.run() + assert stats.pages == 2 + assert stats.origins == page_size * stats.pages + + last_seq = npm_incremental_listing_page2["results"][-1]["seq"] + + spy_get.assert_has_calls( + [ + mocker.call( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params(page_size, since="0"), + ), + mocker.call( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params( + page_size, + since=str(npm_incremental_listing_page1["results"][-1]["seq"]), + ), + ), + mocker.call( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params(page_size, since=str(last_seq)), + ), + ] + ) + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins -def test_npm_lister_listing_pagination(lister_npm, requests_mock_datadir): - lister = lister_npm - # Patch per page pagination - lister.per_page = 10 + 1 - lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace( - "&limit=1001", "&limit=%s" % lister.per_page + _check_listed_npm_packages( + lister, + chain( + npm_incremental_listing_page1["results"], + npm_incremental_listing_page2["results"], + ), + scheduler_origins, ) + + assert lister.get_state_from_scheduler() == NpmListerState(last_seq=last_seq) + + +def test_npm_lister_incremental_restart( + swh_scheduler, requests_mock, mocker, +): + """Check incremental npm listing will restart from saved state""" + page_size = 2 + last_seq = 67 + lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True) + lister.state = NpmListerState(last_seq=last_seq) + + requests_mock.get(lister.API_INCREMENTAL_LISTING_URL, json={"results": []}) + + spy_get = mocker.spy(lister.session, "get") + lister.run() - tasks = lister.scheduler.search_tasks(task_type="load-npm") - assert len(tasks) == 2 * 10 # only 2 files with 10 results each + spy_get.assert_called_with( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params(page_size, since=str(last_seq)), + ) + + +def test_npm_lister_http_error( + swh_scheduler, requests_mock, mocker, +): + lister = NpmLister(scheduler=swh_scheduler) + + requests_mock.get(lister.API_FULL_LISTING_URL, status_code=500) - check_tasks(tasks) + with pytest.raises(HTTPError): + lister.run() diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py --- a/swh/lister/npm/tests/test_tasks.py +++ b/swh/lister/npm/tests/test_tasks.py @@ -1,15 +1,9 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from contextlib import contextmanager -from unittest.mock import patch - - -@contextmanager -def mock_save(lister): - yield +from swh.lister.pattern import ListerStats def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): @@ -20,34 +14,31 @@ assert res.result == "OK" -@patch("swh.lister.npm.tasks.save_registry_state") -@patch("swh.lister.npm.tasks.NpmLister") -def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker): - # setup the mocked NpmLister - lister.return_value = lister - lister.run.return_value = None - save.side_effect = mock_save +def test_full_lister_task( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker +): + stats = ListerStats(pages=10, origins=900) + mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister") + mock_lister.from_configfile.return_value = mock_lister + mock_lister.run.return_value = stats res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask") assert res res.wait() assert res.successful() - lister.assert_called_once_with() - lister.run.assert_called_once_with() + mock_lister.from_configfile.assert_called_once_with(incremental=False) + mock_lister.run.assert_called_once_with() + assert res.result == stats.dict() -@patch("swh.lister.npm.tasks.save_registry_state") -@patch("swh.lister.npm.tasks.get_last_update_seq") -@patch("swh.lister.npm.tasks.NpmIncrementalLister") -def test_incremental( - lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker +def test_incremental_lister_task( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker ): - # setup the mocked NpmLister - lister.return_value = lister - lister.run.return_value = None - seq.return_value = 42 - save.side_effect = mock_save + stats = ListerStats(pages=10, origins=900) + mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister") + mock_lister.from_configfile.return_value = mock_lister + mock_lister.run.return_value = stats res = swh_scheduler_celery_app.send_task( "swh.lister.npm.tasks.NpmIncrementalListerTask" @@ -56,6 +47,6 @@ res.wait() assert res.successful() - lister.assert_called_once_with() - seq.assert_called_once_with(lister) - lister.run.assert_called_once_with(min_bound=42) + mock_lister.from_configfile.assert_called_once_with(incremental=True) + mock_lister.run.assert_called_once_with() + assert res.result == stats.dict()