Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9346081
D4877.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
117 KB
Subscribers
None
D4877.diff
View Options
diff --git a/swh/lister/npm/__init__.py b/swh/lister/npm/__init__.py
--- a/swh/lister/npm/__init__.py
+++ b/swh/lister/npm/__init__.py
@@ -1,14 +1,13 @@
-# Copyright (C) 2019 the Software Heritage developers
+# Copyright (C) 2019-2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
def register():
from .lister import NpmLister
- from .models import NpmModel, NpmVisitModel
return {
- "models": [NpmVisitModel, NpmModel],
+ "models": [],
"lister": NpmLister,
"task_modules": ["%s.tasks" % __name__],
"task_types": {
diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py
--- a/swh/lister/npm/lister.py
+++ b/swh/lister/npm/lister.py
@@ -1,154 +1,190 @@
-# Copyright (C) 2018-2019 the Software Heritage developers
+# Copyright (C) 2018-2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from typing import Any, Dict, List, Optional
+from dataclasses import asdict, dataclass
+import logging
+from typing import Any, Dict, Iterator, List, Optional
-from requests import Response
+import iso8601
+import requests
+from tenacity.before_sleep import before_sleep_log
-from swh.core import config
-from swh.lister.core.indexing_lister import IndexingHttpLister
-from swh.lister.npm.models import NpmModel
-from swh.scheduler.utils import create_task_dict
+from swh.lister import USER_AGENT
+from swh.lister.pattern import CredentialsType, Lister
+from swh.lister.utils import throttling_retry
+from swh.scheduler.interface import SchedulerInterface
+from swh.scheduler.model import ListedOrigin
-DEFAULT_CONFIG = {
- "loading_task_policy": "recurring",
-}
+logger = logging.getLogger(__name__)
-class NpmListerBase(IndexingHttpLister):
- """List packages available in the npm registry in a paginated way
+@dataclass
+class NpmListerState:
+ """State of npm lister"""
- """
-
- MODEL = NpmModel
- LISTER_NAME = "npm"
- instance = "npm"
+ last_seq: Optional[int] = None
- def __init__(
- self, url="https://replicate.npmjs.com", per_page=1000, override_config=None
- ):
- super().__init__(url=url, override_config=override_config)
- self.config = config.merge_configs(DEFAULT_CONFIG, self.config)
- self.per_page = per_page + 1
- self.PATH_TEMPLATE += "&limit=%s" % self.per_page
-
- def get_model_from_repo(self, repo_name: str) -> Dict[str, str]:
- """(Override) Transform from npm package name to model
-
- """
- package_url = "https://www.npmjs.com/package/%s" % repo_name
- return {
- "uid": repo_name,
- "indexable": repo_name,
- "name": repo_name,
- "full_name": repo_name,
- "html_url": package_url,
- "origin_url": package_url,
- "origin_type": "npm",
- }
-
- def task_dict(self, origin_type: str, origin_url: str, **kwargs):
- """(Override) Return task dict for loading a npm package into the
- archive.
-
- This is overridden from the lister_base as more information is
- needed for the ingestion task creation.
-
- """
- task_type = "load-%s" % origin_type
- task_policy = self.config["loading_task_policy"]
- return create_task_dict(task_type, task_policy, url=origin_url)
-
- def request_headers(self) -> Dict[str, Any]:
- """(Override) Set requests headers to send when querying the npm
- registry.
-
- """
- headers = super().request_headers()
- headers["Accept"] = "application/json"
- return headers
-
- def string_pattern_check(self, inner: int, lower: int, upper: int = None):
- """ (Override) Inhibit the effect of that method as packages indices
- correspond to package names and thus do not respect any kind
- of fixed length string pattern
-
- """
- pass
-
-
-class NpmLister(NpmListerBase):
- """List all packages available in the npm registry in a paginated way
+class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]):
"""
+ List all packages hosted on the npm registry.
- PATH_TEMPLATE = '/_all_docs?startkey="%s"'
-
- def get_next_target_from_response(self, response: Response) -> Optional[str]:
- """(Override) Get next npm package name to continue the listing
-
- """
- repos = response.json()["rows"]
- return repos[-1]["id"] if len(repos) == self.per_page else None
-
- def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]:
- """(Override) Transform npm registry response to list for model manipulation
-
- """
- repos = response.json()["rows"]
- if len(repos) == self.per_page:
- repos = repos[:-1]
- return [self.get_model_from_repo(repo["id"]) for repo in repos]
-
+ The lister is based on the npm replication API powered by a
+ CouchDB database (https://docs.couchdb.org/en/stable/api/database/).
-class NpmIncrementalLister(NpmListerBase):
- """List packages in the npm registry, updated since a specific
- update_seq value of the underlying CouchDB database, in a paginated way.
+ Args:
+ scheduler: a scheduler instance
+ page_size: number of packages info to return per page when querying npm API
+ incremental: defines if incremental listing should be used, in that case
+ only modified or new packages since last incremental listing operation
+ will be returned, otherwise all packages will be listed in lexicographical
+ order
"""
- PATH_TEMPLATE = "/_changes?since=%s"
-
- @property
- def CONFIG_BASE_FILENAME(self): # noqa: N802
- return "lister_npm_incremental"
-
- def get_next_target_from_response(self, response: Response) -> Optional[str]:
- """(Override) Get next npm package name to continue the listing.
-
- """
- repos = response.json()["results"]
- return repos[-1]["seq"] if len(repos) == self.per_page else None
-
- def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]:
- """(Override) Transform npm registry response to list for model
- manipulation.
-
- """
- repos = response.json()["results"]
- if len(repos) == self.per_page:
- repos = repos[:-1]
- return [self.get_model_from_repo(repo["id"]) for repo in repos]
+ LISTER_NAME = "npm"
+ INSTANCE = "npm"
- def filter_before_inject(self, models_list: List[Dict[str, Any]]):
- """(Override) Filter out documents in the CouchDB database
- not related to a npm package.
+ API_BASE_URL = "https://replicate.npmjs.com"
+ API_INCREMENTAL_LISTING_URL = f"{API_BASE_URL}/_changes"
+ API_FULL_LISTING_URL = f"{API_BASE_URL}/_all_docs"
+ PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}"
- """
- models_filtered = []
- for model in models_list:
- package_name = model["name"]
- # document related to CouchDB internals
- if package_name.startswith("_design/"):
+ def __init__(
+ self,
+ scheduler: SchedulerInterface,
+ page_size: int = 1000,
+ incremental: bool = False,
+ credentials: CredentialsType = None,
+ ):
+ super().__init__(
+ scheduler=scheduler,
+ credentials=credentials,
+ url=self.API_INCREMENTAL_LISTING_URL
+ if incremental
+ else self.API_FULL_LISTING_URL,
+ instance=self.INSTANCE,
+ )
+
+ self.page_size = page_size
+ if not incremental:
+ # in full listing mode, first package in each page corresponds to the one
+ # provided as the startkey query parameter value, so we increment the page
+ # size by one to avoid double package processing
+ self.page_size += 1
+ self.incremental = incremental
+
+ self.session = requests.Session()
+ self.session.headers.update(
+ {"Accept": "application/json", "User-Agent": USER_AGENT}
+ )
+
+ def state_from_dict(self, d: Dict[str, Any]) -> NpmListerState:
+ return NpmListerState(**d)
+
+ def state_to_dict(self, state: NpmListerState) -> Dict[str, Any]:
+ return asdict(state)
+
+ def request_params(self, last_package_id: str) -> Dict[str, Any]:
+ # include package JSON document to get its last update date
+ params = {"limit": self.page_size, "include_docs": "true"}
+ if self.incremental:
+ params["since"] = last_package_id
+ else:
+ params["startkey"] = last_package_id
+ return params
+
+ @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
+ def page_request(self, last_package_id: str) -> requests.Response:
+ params = self.request_params(last_package_id)
+ logger.debug("Fetching URL %s with params %s", self.url, params)
+ response = self.session.get(self.url, params=params)
+ if response.status_code != 200:
+ logger.warning(
+ "Unexpected HTTP status code %s on %s: %s",
+ response.status_code,
+ response.url,
+ response.content,
+ )
+ response.raise_for_status()
+ return response
+
+ def get_pages(self) -> Iterator[List[Dict[str, Any]]]:
+ last_package_id: str = "0" if self.incremental else '""'
+ if (
+ self.incremental
+ and self.state is not None
+ and self.state.last_seq is not None
+ ):
+ last_package_id = str(self.state.last_seq)
+
+ while True:
+
+ response = self.page_request(last_package_id)
+
+ data = response.json()
+ page = data["results"] if self.incremental else data["rows"]
+
+ if not page:
+ break
+
+ if self.incremental or len(page) < self.page_size:
+ yield page
+ else:
+ yield page[:-1]
+
+ if len(page) < self.page_size:
+ break
+
+ last_package_id = (
+ str(page[-1]["seq"]) if self.incremental else f'"{page[-1]["id"]}"'
+ )
+
+ def get_origins_from_page(
+ self, page: List[Dict[str, Any]]
+ ) -> Iterator[ListedOrigin]:
+ """Convert a page of Npm repositories into a list of ListedOrigin."""
+ assert self.lister_obj.id is not None
+
+ for package in page:
+ # no source code to archive here
+ if not package["doc"].get("versions", {}):
continue
- models_filtered.append(model)
- return models_filtered
-
- def disable_deleted_repo_tasks(self, start, end, keep_these):
- """(Override) Disable the processing performed by that method as it is
- not relevant in this incremental lister context. It also raises an
- exception due to a different index type (int instead of str).
- """
- pass
+ package_name = package["doc"]["name"]
+ package_latest_version = (
+ package["doc"].get("dist-tags", {}).get("latest", "")
+ )
+
+ last_update = None
+ if package_latest_version in package["doc"].get("time", {}):
+ last_update = iso8601.parse_date(
+ package["doc"]["time"][package_latest_version]
+ )
+
+ yield ListedOrigin(
+ lister_id=self.lister_obj.id,
+ url=self.PACKAGE_URL_TEMPLATE.format(package_name=package_name),
+ visit_type="npm",
+ last_update=last_update,
+ )
+
+ def commit_page(self, page: List[Dict[str, Any]]):
+ """Update the currently stored state using the latest listed page."""
+ if self.incremental:
+ last_package = page[-1]
+ last_seq = last_package["seq"]
+
+ if self.state.last_seq is None or last_seq > self.state.last_seq:
+ self.state.last_seq = last_seq
+
+ def finalize(self):
+ if self.incremental and self.state.last_seq is not None:
+ scheduler_state = self.get_state_from_scheduler()
+
+ if (
+ scheduler_state.last_seq is None
+ or self.state.last_seq > scheduler_state.last_seq
+ ):
+ self.updated = True
diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py
deleted file mode 100644
--- a/swh/lister/npm/models.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (C) 2018 the Software Heritage developers
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-from sqlalchemy import BigInteger, Column, DateTime, Integer, Sequence, String
-
-from swh.lister.core.models import ABCSQLMeta, IndexingModelBase, SQLBase
-
-
-class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta):
- """Table to store the npm registry state at the time of a
- content listing by Software Heritage
- """
-
- __tablename__ = "npm_visit"
-
- uid = Column(Integer, Sequence("npm_visit_id_seq"), primary_key=True)
- visit_date = Column(DateTime, nullable=False)
- doc_count = Column(BigInteger)
- doc_del_count = Column(BigInteger)
- update_seq = Column(BigInteger)
- purge_seq = Column(BigInteger)
- disk_size = Column(BigInteger)
- data_size = Column(BigInteger)
- committed_update_seq = Column(BigInteger)
- compacted_seq = Column(BigInteger)
-
-
-class NpmModel(IndexingModelBase):
- """A npm package representation
-
- """
-
- __tablename__ = "npm_repo"
-
- uid = Column(String, primary_key=True)
- indexable = Column(String, index=True)
diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py
--- a/swh/lister/npm/tasks.py
+++ b/swh/lister/npm/tasks.py
@@ -1,69 +1,24 @@
-# Copyright (C) 2018 the Software Heritage developers
+# Copyright (C) 2018-2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from contextlib import contextmanager
-from datetime import datetime
-
from celery import shared_task
-from swh.lister.npm.lister import NpmIncrementalLister, NpmLister
-from swh.lister.npm.models import NpmVisitModel
-
-
-@contextmanager
-def save_registry_state(lister):
- params = {"headers": lister.request_headers()}
- registry_state = lister.session.get(lister.url, **params)
- registry_state = registry_state.json()
- keys = (
- "doc_count",
- "doc_del_count",
- "update_seq",
- "purge_seq",
- "disk_size",
- "data_size",
- "committed_update_seq",
- "compacted_seq",
- )
-
- state = {key: registry_state[key] for key in keys}
- state["visit_date"] = datetime.now()
- yield
- npm_visit = NpmVisitModel(**state)
- lister.db_session.add(npm_visit)
- lister.db_session.commit()
-
-
-def get_last_update_seq(lister):
- """Get latest ``update_seq`` value for listing only updated packages.
- """
- query = lister.db_session.query(NpmVisitModel.update_seq)
- row = query.order_by(NpmVisitModel.uid.desc()).first()
- if not row:
- raise ValueError(
- "No npm registry listing previously performed ! "
- "This is required prior to the execution of an "
- "incremental listing."
- )
- return row[0]
+from swh.lister.npm.lister import NpmLister
@shared_task(name=__name__ + ".NpmListerTask")
def list_npm_full(**lister_args):
"Full lister for the npm (javascript) registry"
- lister = NpmLister(**lister_args)
- with save_registry_state(lister):
- return lister.run()
+ lister = NpmLister.from_configfile(incremental=False, **lister_args)
+ return lister.run().dict()
@shared_task(name=__name__ + ".NpmIncrementalListerTask")
def list_npm_incremental(**lister_args):
"Incremental lister for the npm (javascript) registry"
- lister = NpmIncrementalLister(**lister_args)
- update_seq_start = get_last_update_seq(lister)
- with save_registry_state(lister):
- return lister.run(min_bound=update_seq_start)
+ lister = NpmLister.from_configfile(incremental=True, **lister_args)
+ return lister.run().dict()
@shared_task(name=__name__ + ".ping")
diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py
--- a/swh/lister/npm/tests/conftest.py
+++ b/swh/lister/npm/tests/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,18 +9,3 @@
@pytest.fixture
def lister_under_test():
return "npm"
-
-
-@pytest.fixture
-def lister_npm(swh_lister):
- # Add the load-npm in the scheduler backend
- swh_lister.scheduler.create_task_type(
- {
- "type": "load-npm",
- "description": "Load npm package",
- "backend_name": "swh.loader.package.tasks.LoadNpm",
- "default_interval": "1 day",
- }
- )
-
- return swh_lister
diff --git a/swh/lister/npm/tests/data/api_empty_response.json b/swh/lister/npm/tests/data/api_empty_response.json
deleted file mode 100644
--- a/swh/lister/npm/tests/data/api_empty_response.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 839080,
- "rows": []
-}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/api_inc_empty_response.json b/swh/lister/npm/tests/data/api_inc_empty_response.json
deleted file mode 100644
--- a/swh/lister/npm/tests/data/api_inc_empty_response.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "results": [],
- "last_seq": 6927821
-}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/api_inc_response.json b/swh/lister/npm/tests/data/api_inc_response.json
deleted file mode 100644
--- a/swh/lister/npm/tests/data/api_inc_response.json
+++ /dev/null
@@ -1,906 +0,0 @@
-{
- "results": [
- {
- "seq": 6920644,
- "id": "electron-scripts",
- "changes": [
- {
- "rev": "3-a19944df5a3636bb225af9e0c8f9eedc"
- }
- ]
- },
- {
- "seq": 6920649,
- "id": "@crexi-dev/schematics",
- "changes": [
- {
- "rev": "3-00188360eeca1f9123b2d7cd4b468c50"
- }
- ]
- },
- {
- "seq": 6920651,
- "id": "botfactory-conversation",
- "changes": [
- {
- "rev": "50-f3667cde87637505528c46adc87f44e3"
- }
- ]
- },
- {
- "seq": 6920667,
- "id": "castle",
- "changes": [
- {
- "rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0"
- }
- ]
- },
- {
- "seq": 6920671,
- "id": "rbc-wm-framework-vuejs",
- "changes": [
- {
- "rev": "111-32bed479afacdd88aed9ac16dd135843"
- }
- ]
- },
- {
- "seq": 6920678,
- "id": "bitcoinfiles",
- "changes": [
- {
- "rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0"
- }
- ]
- },
- {
- "seq": 6920679,
- "id": "jovo-core",
- "changes": [
- {
- "rev": "2-d7440f1d17823e1a0760d9b3d4537c6e"
- }
- ]
- },
- {
- "seq": 6920687,
- "id": "jovo-framework",
- "changes": [
- {
- "rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9"
- }
- ]
- },
- {
- "seq": 6920690,
- "id": "smart-form-lib",
- "changes": [
- {
- "rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade"
- }
- ]
- },
- {
- "seq": 6920694,
- "id": "bokehjs",
- "changes": [
- {
- "rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2"
- }
- ]
- },
- {
- "seq": 6920701,
- "id": "guijarro",
- "changes": [
- {
- "rev": "14-82ece581d6a35d4e1d78e5292ca245c0"
- }
- ]
- },
- {
- "seq": 6920702,
- "id": "@kava-labs/crypto-rate-utils",
- "changes": [
- {
- "rev": "3-cecc6a6c226a0590b1a685e3041028c6"
- }
- ]
- },
- {
- "seq": 6920703,
- "id": "@riouxjean/test",
- "changes": [
- {
- "rev": "10-01e97dc7d0241dc49ea93b3468ec7b29"
- }
- ]
- },
- {
- "seq": 6920704,
- "id": "react-scrabblefy",
- "changes": [
- {
- "rev": "7-970c8206f3b8744204f7dcb106f8462b"
- }
- ]
- },
- {
- "seq": 6920706,
- "id": "molart",
- "changes": [
- {
- "rev": "14-416cd3cec62dd46f9b59a3bbe35308f6"
- }
- ]
- },
- {
- "seq": 6920707,
- "id": "@universal-material/angular",
- "changes": [
- {
- "rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3"
- }
- ]
- },
- {
- "seq": 6920708,
- "id": "cozy-doctypes",
- "changes": [
- {
- "rev": "68-8e90cc26e25da6c9430d373e43ac3c25"
- }
- ]
- },
- {
- "seq": 6920710,
- "id": "2o3t-ui",
- "changes": [
- {
- "rev": "96-1e65d5320ea7c78525aba5daf328bd4b"
- }
- ]
- },
- {
- "seq": 6920712,
- "id": "ark-ts",
- "changes": [
- {
- "rev": "24-033183c2f7f9cbb6e44d553213e525b6"
- }
- ]
- },
- {
- "seq": 6920715,
- "id": "mysqlconnector",
- "changes": [
- {
- "rev": "19-f09bc0b82281ca486db5ebe83843679e"
- }
- ]
- },
- {
- "seq": 6920716,
- "id": "@innovexa/ng-form-creator-lib",
- "changes": [
- {
- "rev": "147-480665ee17fa889dfec1aee75b907ff2"
- }
- ]
- },
- {
- "seq": 6920717,
- "id": "k-routes-example-basic",
- "changes": [
- {
- "rev": "1-35142059e1c63cc724da71a9eebf229c"
- }
- ]
- },
- {
- "seq": 6920718,
- "id": "wloggertojs",
- "changes": [
- {
- "rev": "29-5b5aa74bd30ff0fc86b39fba799befe2"
- }
- ]
- },
- {
- "seq": 6920720,
- "id": "wloggertofile",
- "changes": [
- {
- "rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5"
- }
- ]
- },
- {
- "seq": 6920721,
- "id": "@brightcove/flashls",
- "changes": [
- {
- "rev": "62-fbadb49476a58e98f0f136c86b614734"
- }
- ]
- },
- {
- "seq": 6920722,
- "id": "@brightcove/hls-fetcher",
- "changes": [
- {
- "rev": "76-3341ed8ade38f3251a97c94c3a7af5ac"
- }
- ]
- },
- {
- "seq": 6920723,
- "id": "@brightcove/kacl",
- "changes": [
- {
- "rev": "33-d0bc6b639cccb301086114d548ecfdbf"
- }
- ]
- },
- {
- "seq": 6920724,
- "id": "just-in-types",
- "changes": [
- {
- "rev": "2-fc329aa885dc795aee340f36ec60f333"
- }
- ]
- },
- {
- "seq": 6920725,
- "id": "@brightcove/player-loader",
- "changes": [
- {
- "rev": "56-9ff5aebc9743a44d46c182746313877d"
- }
- ]
- },
- {
- "seq": 6920726,
- "id": "@brightcove/player-loader-webpack-plugin",
- "changes": [
- {
- "rev": "33-db8b4d6765f19e475e1c1d16843824cb"
- }
- ]
- },
- {
- "seq": 6920727,
- "id": "@brightcove/player-url",
- "changes": [
- {
- "rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc"
- }
- ]
- },
- {
- "seq": 6920728,
- "id": "@brightcove/react-player-loader",
- "changes": [
- {
- "rev": "39-b7bf609de666ec7e71f517db53ab9c0a"
- }
- ]
- },
- {
- "seq": 6920729,
- "id": "vscode-theme-generator",
- "changes": [
- {
- "rev": "21-bcb92281d6f7e37548bb18113681df88"
- }
- ]
- },
- {
- "seq": 6920733,
- "id": "@brightcove/typed-immutable-extensions",
- "changes": [
- {
- "rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe"
- }
- ]
- },
- {
- "seq": 6920734,
- "id": "@brightcove/typed-immutable-proptypes",
- "changes": [
- {
- "rev": "27-e4802afc947c55d34f778864476c17e4"
- }
- ]
- },
- {
- "seq": 6920737,
- "id": "@brightcove/videojs-flashls-source-handler",
- "changes": [
- {
- "rev": "59-faf69c49be866b2ab7faa7be9972e7a5"
- }
- ]
- },
- {
- "seq": 6920738,
- "id": "@brightcove/videojs-flashls-swf",
- "changes": [
- {
- "rev": "60-04908466eaac2194bc3061e91f463dab"
- }
- ]
- },
- {
- "seq": 6920739,
- "id": "@noqcks/generated",
- "changes": [
- {
- "rev": "2-e07d07614182d4beccc507ca199e612d"
- }
- ]
- },
- {
- "seq": 6920740,
- "id": "pkcs7",
- "changes": [
- {
- "rev": "60-65ba116f3b6b705f472971b5c6a8f8d2"
- }
- ]
- },
- {
- "seq": 6920741,
- "id": "videojs-errors",
- "changes": [
- {
- "rev": "57-c999abd162ca4b93412e363443aa688a"
- }
- ]
- },
- {
- "seq": 6920742,
- "id": "videojs-flashls-source-handler",
- "changes": [
- {
- "rev": "59-46d62e18971a8c800710a8fbf985c1c5"
- }
- ]
- },
- {
- "seq": 6920743,
- "id": "videojs-playlist",
- "changes": [
- {
- "rev": "97-d4b3492a94c1084c272162dd51901188"
- }
- ]
- },
- {
- "seq": 6920745,
- "id": "videojs-playlist-ui",
- "changes": [
- {
- "rev": "95-ba97c44c354b2262e639f8c515bed9bc"
- }
- ]
- },
- {
- "seq": 6920746,
- "id": "fusion-apollo-universal-client",
- "changes": [
- {
- "rev": "25-7123042a477cec67c7d5fc702254c7a3"
- }
- ]
- },
- {
- "seq": 6920749,
- "id": "msg-fabric-core",
- "changes": [
- {
- "rev": "20-17c33e06faca357526c7395aca1113d2"
- }
- ]
- },
- {
- "seq": 6920750,
- "id": "@expo/schemer",
- "changes": [
- {
- "rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d"
- }
- ]
- },
- {
- "seq": 6920752,
- "id": "mathjs",
- "changes": [
- {
- "rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18"
- }
- ]
- },
- {
- "seq": 6920758,
- "id": "statesauce-ui",
- "changes": [
- {
- "rev": "6-db9a39366c1a082c56a2212e368e3ae2"
- }
- ]
- },
- {
- "seq": 6920782,
- "id": "@catchandrelease/arbor",
- "changes": [
- {
- "rev": "19-925648432b398ecadc98993e6fba2353"
- }
- ]
- },
- {
- "seq": 6920784,
- "id": "discover-shared-ebsco-ui-core",
- "changes": [
- {
- "rev": "4-277063cbc6b71f969e5f0db8c371db65"
- }
- ]
- },
- {
- "seq": 6920807,
- "id": "react-apexcharts",
- "changes": [
- {
- "rev": "13-18505be8026a50390c1ff1ba522cb9bd"
- }
- ]
- },
- {
- "seq": 6920819,
- "id": "zigbee-shepherd-converters",
- "changes": [
- {
- "rev": "90-5819692a5a9679ff8669fb410e190515"
- }
- ]
- },
- {
- "seq": 6920835,
- "id": "honeycomb-grid",
- "changes": [
- {
- "rev": "36-edd6733c80b04a72600558dc55348c73"
- }
- ]
- },
- {
- "seq": 6920838,
- "id": "pixl-config",
- "changes": [
- {
- "rev": "7-5dd2b68d04fefb4039b3965b3497eda2"
- }
- ]
- },
- {
- "seq": 6920842,
- "id": "discover-shared-ebsco-ui-theming",
- "changes": [
- {
- "rev": "4-e9d083825b1eae46f28c4def2d0db79f"
- }
- ]
- },
- {
- "seq": 6920843,
- "id": "common-oxgalaxy-lengua-app",
- "changes": [
- {
- "rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f"
- }
- ]
- },
- {
- "seq": 6920845,
- "id": "discover-shared-ebsco-ui-grid",
- "changes": [
- {
- "rev": "2-6f71cf625a5232075071952b2adaa8f2"
- }
- ]
- },
- {
- "seq": 6920847,
- "id": "@auth0/cosmos-tokens",
- "changes": [
- {
- "rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb"
- }
- ]
- },
- {
- "seq": 6920848,
- "id": "@auth0/babel-preset-cosmos",
- "changes": [
- {
- "rev": "43-d05d3779db08f08726ba048da298e046"
- }
- ]
- },
- {
- "seq": 6920849,
- "id": "jsrender",
- "changes": [
- {
- "rev": "11-c949091592b3329d73ae564e45a3472d"
- }
- ]
- },
- {
- "seq": 6920850,
- "id": "discover-shared-ebsco-ui-container",
- "changes": [
- {
- "rev": "2-c32089f76b7f253bc0d765da8b9f670d"
- }
- ]
- },
- {
- "seq": 6920852,
- "id": "@auth0/cosmos",
- "changes": [
- {
- "rev": "42-5fdaf3d9063c20dac13dcf455c42773c"
- }
- ]
- },
- {
- "seq": 6920853,
- "id": "discover-shared-ebsco-ui-checkbox",
- "changes": [
- {
- "rev": "2-06d9521b86f0dbf4a398726faead1212"
- }
- ]
- },
- {
- "seq": 6920854,
- "id": "@adunigan/toggles",
- "changes": [
- {
- "rev": "1-c2a830cf814a9fe2d72084339c9c5d28"
- }
- ]
- },
- {
- "seq": 6920855,
- "id": "@spriteful/spriteful-lazy-carousel",
- "changes": [
- {
- "rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837"
- }
- ]
- },
- {
- "seq": 6920856,
- "id": "react-modal-hook",
- "changes": [
- {
- "rev": "2-364b39d6559364c41d5b852ccad4ce31"
- }
- ],
- "deleted": true
- },
- {
- "seq": 6920859,
- "id": "@bellese/angular-design-system",
- "changes": [
- {
- "rev": "39-3e297f85ce2d6a6b6d15fc26420fc471"
- }
- ]
- },
- {
- "seq": 6920861,
- "id": "@uifabric/styling",
- "changes": [
- {
- "rev": "229-addf6cc0e74a335125c04d60047353f5"
- }
- ]
- },
- {
- "seq": 6920862,
- "id": "@uifabric/file-type-icons",
- "changes": [
- {
- "rev": "37-8a7e43399d1bb9f17334b10995f78df4"
- }
- ]
- },
- {
- "seq": 6920864,
- "id": "throttlewrap",
- "changes": [
- {
- "rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa"
- }
- ]
- },
- {
- "seq": 6920865,
- "id": "airtable",
- "changes": [
- {
- "rev": "16-d8aee935f6fa4c88057d75a0542bc58c"
- }
- ]
- },
- {
- "seq": 6920866,
- "id": "@csmart/ngc-smart-address",
- "changes": [
- {
- "rev": "19-66a6ea868aae1912952f232d2c699f3a"
- }
- ]
- },
- {
- "seq": 6920868,
- "id": "office-ui-fabric-react",
- "changes": [
- {
- "rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461"
- }
- ]
- },
- {
- "seq": 6920869,
- "id": "@fuelrats/eslint-config",
- "changes": [
- {
- "rev": "12-1b4c71b78fd078e3c1cba535e8541bed"
- }
- ]
- },
- {
- "seq": 6920870,
- "id": "@uifabric/date-time",
- "changes": [
- {
- "rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed"
- }
- ]
- },
- {
- "seq": 6920872,
- "id": "dark-client",
- "changes": [
- {
- "rev": "11-a954c2a89a130ae73f064233d9b3bce2"
- }
- ]
- },
- {
- "seq": 6920873,
- "id": "@uifabric/variants",
- "changes": [
- {
- "rev": "59-391c720194c663b9a5c59fe2c10a1535"
- }
- ]
- },
- {
- "seq": 6920875,
- "id": "discover-shared-ebsco-ui-header",
- "changes": [
- {
- "rev": "2-efd8f0426a83422a6c8b7bff11054c72"
- }
- ]
- },
- {
- "seq": 6920876,
- "id": "react-responsive-picture",
- "changes": [
- {
- "rev": "14-32a6d0850c8af33412cfdb23afd2ecfa"
- }
- ]
- },
- {
- "seq": 6920877,
- "id": "@uifabric/fluent-theme",
- "changes": [
- {
- "rev": "16-39c29e00b81a0b654213a5a50d7e7f42"
- }
- ]
- },
- {
- "seq": 6920878,
- "id": "@uifabric/dashboard",
- "changes": [
- {
- "rev": "82-04d6dc25b33e811c1d8c24566127b09c"
- }
- ]
- },
- {
- "seq": 6920879,
- "id": "ids-enterprise",
- "changes": [
- {
- "rev": "201-dd709a3912f9832440320d448850b61a"
- }
- ]
- },
- {
- "seq": 6920880,
- "id": "@uifabric/experiments",
- "changes": [
- {
- "rev": "224-efd1ef07f7640952c286488eae282367"
- }
- ]
- },
- {
- "seq": 6920881,
- "id": "@fuelrats/eslint-config-react",
- "changes": [
- {
- "rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc"
- }
- ]
- },
- {
- "seq": 6920883,
- "id": "jsviews",
- "changes": [
- {
- "rev": "11-44d8bedffc98cf6ac4aa669ba8844746"
- }
- ]
- },
- {
- "seq": 6920885,
- "id": "pixl-server",
- "changes": [
- {
- "rev": "15-823f4598c3354500d8d2a266dd062502"
- }
- ]
- },
- {
- "seq": 6920887,
- "id": "@rrpm/netlify-cms-core",
- "changes": [
- {
- "rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa"
- }
- ]
- },
- {
- "seq": 6920889,
- "id": "lodash-a",
- "changes": [
- {
- "rev": "2-6ee66153dbe611a080b40775387d2d45"
- }
- ]
- },
- {
- "seq": 6920891,
- "id": "meshcentral",
- "changes": [
- {
- "rev": "499-6677ca74525ed2aa77644c68001382fe"
- }
- ]
- },
- {
- "seq": 6920892,
- "id": "vue-transition-collection",
- "changes": [
- {
- "rev": "2-0510ee52c014c0d3b1e65f24376d76f0"
- }
- ]
- },
- {
- "seq": 6920894,
- "id": "fury-adapter-swagger",
- "changes": [
- {
- "rev": "47-09f0c55d8574d654c67f9244c21d7ef7"
- }
- ]
- },
- {
- "seq": 6920895,
- "id": "@isobar-us/redux-form-gen",
- "changes": [
- {
- "rev": "30-70d7d9210264a321092c832063934648"
- }
- ]
- },
- {
- "seq": 6920896,
- "id": "atomizer",
- "changes": [
- {
- "rev": "19-129774900cb2a67a46871cc2c40c34d3"
- }
- ]
- },
- {
- "seq": 6920904,
- "id": "boom-js-client",
- "changes": [
- {
- "rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9"
- }
- ]
- },
- {
- "seq": 6920905,
- "id": "@ts-common/json-parser",
- "changes": [
- {
- "rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3"
- }
- ]
- },
- {
- "seq": 6920906,
- "id": "rutt",
- "changes": [
- {
- "rev": "13-78aab849cb00a6ef7ebc8165770b7d33"
- }
- ]
- },
- {
- "seq": 6920907,
- "id": "linear-react-components-ui",
- "changes": [
- {
- "rev": "171-0307f1d69843b270e687371c67cbd1b0"
- }
- ]
- },
- {
- "seq": 6920908,
- "id": "@earnest/eslint-config",
- "changes": [
- {
- "rev": "180-b5250dd803102cf7dbac8da9c1a403fd"
- }
- ]
- },
- {
- "seq": 6920909,
- "id": "@earnest/eslint-config-es7",
- "changes": [
- {
- "rev": "181-da26885e0baacaea95814857f459572d"
- }
- ]
- },
- {
- "seq": 6920910,
- "id": "fuse-design",
- "changes": [
- {
- "rev": "10-e2b78592872f680c05e55eb5b81a0cab"
- }
- ]
- }
- ],
- "last_seq": 6920912
-}
\ No newline at end of file
diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001"
deleted file mode 100644
--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001"
+++ /dev/null
@@ -1,807 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 422482,
- "rows": [
-
- {
- "id": "jquery",
- "key": "jquery",
- "value": {
- "rev": "212-2eac7c93af4c8bccdf7317739f0319b6"
- }
- },
-
- {
- "id": "jquery-1.8",
- "key": "jquery-1.8",
- "value": {
- "rev": "1-711ded49a7453adce85ce7a51c2157de"
- }
- },
-
- {
- "id": "jquery-1x",
- "key": "jquery-1x",
- "value": {
- "rev": "1-c53fa04d9c8fb231336704508732c287"
- }
- },
-
- {
- "id": "jquery-2-typescript-async-await-adapter",
- "key": "jquery-2-typescript-async-await-adapter",
- "value": {
- "rev": "8-5cfb484e9afaa6e326a97240fccd8f93"
- }
- },
-
- {
- "id": "jquery-accessible-accordion-aria",
- "key": "jquery-accessible-accordion-aria",
- "value": {
- "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e"
- }
- },
-
- {
- "id": "jquery-accessible-autocomplete-list-aria",
- "key": "jquery-accessible-autocomplete-list-aria",
- "value": {
- "rev": "8-961b382442c1a5bafe58f0e05424701d"
- }
- },
-
- {
- "id": "jquery-accessible-carrousel-aria",
- "key": "jquery-accessible-carrousel-aria",
- "value": {
- "rev": "9-f33f59d7f601bafe023bd711b551282b"
- }
- },
-
- {
- "id": "jquery-accessible-dialog-tooltip-aria",
- "key": "jquery-accessible-dialog-tooltip-aria",
- "value": {
- "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba"
- }
- },
-
- {
- "id": "jquery-accessible-hide-show-aria",
- "key": "jquery-accessible-hide-show-aria",
- "value": {
- "rev": "10-5a03c47a8995b08246e4bc103782dafa"
- }
- },
-
- {
- "id": "jquery-accessible-modal-window-aria",
- "key": "jquery-accessible-modal-window-aria",
- "value": {
- "rev": "18-50266e260f6b807019cfcfcd3a3685ab"
- }
- },
-
- {
- "id": "jquery-accessible-simple-tooltip-aria",
- "key": "jquery-accessible-simple-tooltip-aria",
- "value": {
- "rev": "6-ea71aa37760790dc603b56117f054e1b"
- }
- },
-
- {
- "id": "jquery-accessible-subnav-dropdown",
- "key": "jquery-accessible-subnav-dropdown",
- "value": {
- "rev": "2-496f017a9ac243655225e43b5697b09b"
- }
- },
-
- {
- "id": "jquery-accessible-tabpanel-aria",
- "key": "jquery-accessible-tabpanel-aria",
- "value": {
- "rev": "11-659971471e6ac0fbb3b2f78ad208722a"
- }
- },
-
- {
- "id": "jquery-accessible-tabs-umd",
- "key": "jquery-accessible-tabs-umd",
- "value": {
- "rev": "1-f92015de5bb36e411d8c0940cca2883f"
- }
- },
-
- {
- "id": "jquery-active-descendant",
- "key": "jquery-active-descendant",
- "value": {
- "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de"
- }
- },
-
- {
- "id": "jquery-ada-validation",
- "key": "jquery-ada-validation",
- "value": {
- "rev": "1-9aab9629027c29fbece90485dd9d3112"
- }
- },
-
- {
- "id": "jquery-adaptText",
- "key": "jquery-adaptText",
- "value": {
- "rev": "3-2e15fc801ea8235b9180a3defc782ed0"
- }
- },
-
- {
- "id": "jquery-adapttr",
- "key": "jquery-adapttr",
- "value": {
- "rev": "6-74585f2d4be60b3f493585a6d28b90bc"
- }
- },
-
- {
- "id": "jquery-add-prefixed-class",
- "key": "jquery-add-prefixed-class",
- "value": {
- "rev": "1-9e43aee9758504b3f5271e9804a95f20"
- }
- },
-
- {
- "id": "jquery-address",
- "key": "jquery-address",
- "value": {
- "rev": "1-64173ede32157b26f4de910ad0f49590"
- }
- },
-
- {
- "id": "jquery-address-suggestion",
- "key": "jquery-address-suggestion",
- "value": {
- "rev": "6-18d9df51d472c365bcd84a61c9105774"
- }
- },
-
- {
- "id": "jquery-advscrollevent",
- "key": "jquery-advscrollevent",
- "value": {
- "rev": "1-f6033de9ba0f8e364c42826441d93119"
- }
- },
-
- {
- "id": "jquery-affix",
- "key": "jquery-affix",
- "value": {
- "rev": "6-777371f67df59abf18ec1fe326df3b82"
- }
- },
-
- {
- "id": "jquery-airload",
- "key": "jquery-airload",
- "value": {
- "rev": "7-136d513d2604a25238eb88709d6d9003"
- }
- },
-
- {
- "id": "jquery-ajax",
- "key": "jquery-ajax",
- "value": {
- "rev": "1-ee358f630d4c928b52c968c7667d0d31"
- }
- },
-
- {
- "id": "jquery-ajax-cache",
- "key": "jquery-ajax-cache",
- "value": {
- "rev": "2-ca31e0d43ae28e9cea968f1f538f06d3"
- }
- },
-
- {
- "id": "jquery-ajax-chain",
- "key": "jquery-ajax-chain",
- "value": {
- "rev": "1-dc0e5aee651c0128b7f411aac96132a2"
- }
- },
-
- {
- "id": "jquery-ajax-file-upload",
- "key": "jquery-ajax-file-upload",
- "value": {
- "rev": "1-96147d8bf69245c622e76583bb615d49"
- }
- },
-
- {
- "id": "jquery-ajax-json",
- "key": "jquery-ajax-json",
- "value": {
- "rev": "1-b47eec12168e4cb39b45f1523d7cd397"
- }
- },
-
- {
- "id": "jquery-ajax-markup",
- "key": "jquery-ajax-markup",
- "value": {
- "rev": "1-8e65dc822cb63be76c62a1323666265e"
- }
- },
-
- {
- "id": "jquery-ajax-native",
- "key": "jquery-ajax-native",
- "value": {
- "rev": "2-9d67b8d43713e3546ad50f817c040139"
- }
- },
-
- {
- "id": "jquery-ajax-request",
- "key": "jquery-ajax-request",
- "value": {
- "rev": "1-fdc0960ec73667bc2b46adf493c05db4"
- }
- },
-
- {
- "id": "jquery-ajax-retry",
- "key": "jquery-ajax-retry",
- "value": {
- "rev": "1-27ca186953e346aa9c0ca2310c732751"
- }
- },
-
- {
- "id": "jquery-ajax-tracking",
- "key": "jquery-ajax-tracking",
- "value": {
- "rev": "3-d48876f3c115ee4743a6a94bb65bb01d"
- }
- },
-
- {
- "id": "jquery-ajax-transport-xdomainrequest",
- "key": "jquery-ajax-transport-xdomainrequest",
- "value": {
- "rev": "1-ece69aa5b9f0c950a1fa2806cf74392d"
- }
- },
-
- {
- "id": "jquery-ajax-unobtrusive",
- "key": "jquery-ajax-unobtrusive",
- "value": {
- "rev": "3-fb0daab8480b9a2cc9c6876e1c4874f4"
- }
- },
-
- {
- "id": "jquery-ajax-unobtrusive-multi",
- "key": "jquery-ajax-unobtrusive-multi",
- "value": {
- "rev": "1-0a2ffdabaf5708d4ae3d9e29a3a9ef11"
- }
- },
-
- {
- "id": "jquery-ajaxreadystate",
- "key": "jquery-ajaxreadystate",
- "value": {
- "rev": "1-5e618474fe2e77ad5869c206164f82bf"
- }
- },
-
- {
- "id": "jquery-albe-timeline",
- "key": "jquery-albe-timeline",
- "value": {
- "rev": "2-3db2b43778b5c50db873e724d9940eb6"
- }
- },
-
- {
- "id": "jquery-all-attributes",
- "key": "jquery-all-attributes",
- "value": {
- "rev": "1-89bb7e01ee312ad5d36d78a3aa2327e4"
- }
- },
-
- {
- "id": "jquery-alphaindex",
- "key": "jquery-alphaindex",
- "value": {
- "rev": "4-7f61cde9cfb70617a6fbe992dfcbc10a"
- }
- },
-
- {
- "id": "jquery-always",
- "key": "jquery-always",
- "value": {
- "rev": "1-0ad944881bbc39c67df0a694d80bebef"
- }
- },
-
- {
- "id": "jquery-amd",
- "key": "jquery-amd",
- "value": {
- "rev": "1-931646c751bef740c361dd0f6e68653c"
- }
- },
-
- {
- "id": "jquery-anaglyph-image-effect",
- "key": "jquery-anaglyph-image-effect",
- "value": {
- "rev": "1-9bf7afce2e1bc73747ef22abc859b22b"
- }
- },
-
- {
- "id": "jquery-analytics",
- "key": "jquery-analytics",
- "value": {
- "rev": "1-d84b0c8ce886b9f01d2c5c1cf0a7317f"
- }
- },
-
- {
- "id": "jquery-ancestors",
- "key": "jquery-ancestors",
- "value": {
- "rev": "1-49b30817a03558f1f585c8c0cd4b8afb"
- }
- },
-
- {
- "id": "jquery-angry-loader",
- "key": "jquery-angry-loader",
- "value": {
- "rev": "1-31c9fd950d32b9d3a73829cde1dae577"
- }
- },
-
- {
- "id": "jquery-angular-shim",
- "key": "jquery-angular-shim",
- "value": {
- "rev": "1-723e72b2981f02dd3abcfe6d2395d636"
- }
- },
-
- {
- "id": "jquery-animate-gradient",
- "key": "jquery-animate-gradient",
- "value": {
- "rev": "5-a3e0fc89699237e7e7241cd608a0dcf7"
- }
- },
-
- {
- "id": "jquery-animate-scroll",
- "key": "jquery-animate-scroll",
- "value": {
- "rev": "1-37d49d89fe99aa599540e6ff83b15888"
- }
- },
-
- {
- "id": "jquery-animated-headlines",
- "key": "jquery-animated-headlines",
- "value": {
- "rev": "1-adf1d149bc83fa8445e141e3c900759e"
- }
- },
-
- {
- "id": "jquery-animation",
- "key": "jquery-animation",
- "value": {
- "rev": "4-f51d0559010bbe9d74d70e58de9bd733"
- }
- },
-
- {
- "id": "jquery-animation-support",
- "key": "jquery-animation-support",
- "value": {
- "rev": "1-9013bc4bdeb2bd70bedcc988a811fcc0"
- }
- },
-
- {
- "id": "jquery-aniview",
- "key": "jquery-aniview",
- "value": {
- "rev": "3-5754524da237693458bcff19b626b875"
- }
- },
-
- {
- "id": "jquery-anything-clickable",
- "key": "jquery-anything-clickable",
- "value": {
- "rev": "2-e1aaaf1a369f7796c438a3efbf05bcce"
- }
- },
-
- {
- "id": "jquery-app",
- "key": "jquery-app",
- "value": {
- "rev": "6-4e0bf5abd71c72ced3c4cf3035116f70"
- }
- },
-
- {
- "id": "jquery-app-banner",
- "key": "jquery-app-banner",
- "value": {
- "rev": "2-8a5b530eaab94315eb00c77acd13f2dd"
- }
- },
-
- {
- "id": "jquery-appear-poetic",
- "key": "jquery-appear-poetic",
- "value": {
- "rev": "1-368094b72ed36d42cf2fca438fa4b344"
- }
- },
-
- {
- "id": "jquery-applyonscreen",
- "key": "jquery-applyonscreen",
- "value": {
- "rev": "4-d76c18a6e66fffba01a9a774b40663f8"
- }
- },
-
- {
- "id": "jquery-apta",
- "key": "jquery-apta",
- "value": {
- "rev": "1-c486380fedefd887e6293a00c3b6a222"
- }
- },
-
- {
- "id": "jquery-arrow-navigate",
- "key": "jquery-arrow-navigate",
- "value": {
- "rev": "3-0efe881e01ef0eac24a92baf1eb6d8d1"
- }
- },
-
- {
- "id": "jquery-asAccordion",
- "key": "jquery-asAccordion",
- "value": {
- "rev": "2-2d18d3fe9089dcf67de5f29d1763b4ce"
- }
- },
-
- {
- "id": "jquery-asBgPicker",
- "key": "jquery-asBgPicker",
- "value": {
- "rev": "2-d1403cd306d5764ee0f5aa852c2bed8e"
- }
- },
-
- {
- "id": "jquery-asBreadcrumbs",
- "key": "jquery-asBreadcrumbs",
- "value": {
- "rev": "2-77e566a07680005ce1cb322f2a733fe4"
- }
- },
-
- {
- "id": "jquery-asCheck",
- "key": "jquery-asCheck",
- "value": {
- "rev": "2-d0b2741b70616c7d563419cc125d193d"
- }
- },
-
- {
- "id": "jquery-asChoice",
- "key": "jquery-asChoice",
- "value": {
- "rev": "2-0eda5269cbd59976ee904b74da209389"
- }
- },
-
- {
- "id": "jquery-asColor",
- "key": "jquery-asColor",
- "value": {
- "rev": "3-aa730d81322561c7a3174d5c7bb6b3b8"
- }
- },
-
- {
- "id": "jquery-asColorPicker",
- "key": "jquery-asColorPicker",
- "value": {
- "rev": "2-6bbaecaf94a324331a3d1f5d3aad3b3d"
- }
- },
-
- {
- "id": "jquery-asDropdown",
- "key": "jquery-asDropdown",
- "value": {
- "rev": "2-b29b187cdd0bdce502d11855415e6887"
- }
- },
-
- {
- "id": "jquery-asFontEditor",
- "key": "jquery-asFontEditor",
- "value": {
- "rev": "2-132882375101062896413afdc93b4c8c"
- }
- },
-
- {
- "id": "jquery-asGalleryPicker",
- "key": "jquery-asGalleryPicker",
- "value": {
- "rev": "1-864a80930d72c6150aa74969a28617e4"
- }
- },
-
- {
- "id": "jquery-asGmap",
- "key": "jquery-asGmap",
- "value": {
- "rev": "2-b0c4330774137b2f1b91bd4686880f2a"
- }
- },
-
- {
- "id": "jquery-asGradient",
- "key": "jquery-asGradient",
- "value": {
- "rev": "2-5184670a313d5e161cb62659de3db55c"
- }
- },
-
- {
- "id": "jquery-asHoverScroll",
- "key": "jquery-asHoverScroll",
- "value": {
- "rev": "7-3f6efebf248bd27520d03eaac33d8ca2"
- }
- },
-
- {
- "id": "jquery-asIconPicker",
- "key": "jquery-asIconPicker",
- "value": {
- "rev": "2-9070adda148ea75247c7cee810ae91e2"
- }
- },
-
- {
- "id": "jquery-asImagePicker",
- "key": "jquery-asImagePicker",
- "value": {
- "rev": "2-fb3115c2296b0b07ed9e379176626e01"
- }
- },
-
- {
- "id": "jquery-asItemList",
- "key": "jquery-asItemList",
- "value": {
- "rev": "2-88a7d2900f47c785c2a6cb764ac467d6"
- }
- },
-
- {
- "id": "jquery-asModal",
- "key": "jquery-asModal",
- "value": {
- "rev": "2-1719b8e6a489e03cc3e22bd329148366"
- }
- },
-
- {
- "id": "jquery-asOffset",
- "key": "jquery-asOffset",
- "value": {
- "rev": "2-e45a0077e5bc0bbf91b32dc76387c945"
- }
- },
-
- {
- "id": "jquery-asPaginator",
- "key": "jquery-asPaginator",
- "value": {
- "rev": "2-0d279d2748fc5e875f5fb2a8d3d48377"
- }
- },
-
- {
- "id": "jquery-asPieProgress",
- "key": "jquery-asPieProgress",
- "value": {
- "rev": "2-14dc464a19e9d3feaa532f62e45bbd26"
- }
- },
-
- {
- "id": "jquery-asProgress",
- "key": "jquery-asProgress",
- "value": {
- "rev": "2-a58d7100f1a78f7753efcf0e34dfaf0e"
- }
- },
-
- {
- "id": "jquery-asRange",
- "key": "jquery-asRange",
- "value": {
- "rev": "3-aa3d2f348a933161868ba6b6fd9eb881"
- }
- },
-
- {
- "id": "jquery-asScroll",
- "key": "jquery-asScroll",
- "value": {
- "rev": "1-f4880ea057adbfebb912ba0157575ca1"
- }
- },
-
- {
- "id": "jquery-asScrollable",
- "key": "jquery-asScrollable",
- "value": {
- "rev": "7-5c18eb2180d8aa85f0b5e940667c8344"
- }
- },
-
- {
- "id": "jquery-asScrollbar",
- "key": "jquery-asScrollbar",
- "value": {
- "rev": "4-89420658c355a5584825b45ee4ef0beb"
- }
- },
-
- {
- "id": "jquery-asSelect",
- "key": "jquery-asSelect",
- "value": {
- "rev": "2-caf3dc516665009b654236b876fe02bb"
- }
- },
-
- {
- "id": "jquery-asSpinner",
- "key": "jquery-asSpinner",
- "value": {
- "rev": "2-bf26b5d9c77eb4b63acbf16019407834"
- }
- },
-
- {
- "id": "jquery-asSwitch",
- "key": "jquery-asSwitch",
- "value": {
- "rev": "2-f738586946b432caa73297568b5f38ad"
- }
- },
-
- {
- "id": "jquery-asTooltip",
- "key": "jquery-asTooltip",
- "value": {
- "rev": "2-80d3fe5cdae70d9310969723e7045384"
- }
- },
-
- {
- "id": "jquery-asTree",
- "key": "jquery-asTree",
- "value": {
- "rev": "2-353063a563c0322cbc317af385f71b27"
- }
- },
-
- {
- "id": "jquery-ascolorpicker-flat",
- "key": "jquery-ascolorpicker-flat",
- "value": {
- "rev": "11-1681d53cd475e7b6b9564baa51a79611"
- }
- },
-
- {
- "id": "jquery-aslider",
- "key": "jquery-aslider",
- "value": {
- "rev": "1-2b3dd953493eeaa4dc329cbf0d81116a"
- }
- },
-
- {
- "id": "jquery-aspect-ratio-keeper",
- "key": "jquery-aspect-ratio-keeper",
- "value": {
- "rev": "1-1ad8e5588218e1d38fff351858655eda"
- }
- },
-
- {
- "id": "jquery-assinadordigitaldiscus",
- "key": "jquery-assinadordigitaldiscus",
- "value": {
- "rev": "1-897cd68ef3699551630bd3454dceb6f0"
- }
- },
-
- {
- "id": "jquery-async-gravatar",
- "key": "jquery-async-gravatar",
- "value": {
- "rev": "3-a3192e741d14d57635f4ebfb41a904db"
- }
- },
-
- {
- "id": "jquery-asynclink",
- "key": "jquery-asynclink",
- "value": {
- "rev": "1-2159a3c49e3c8fe9280c592770e83522"
- }
- },
-
- {
- "id": "jquery-atlas",
- "key": "jquery-atlas",
- "value": {
- "rev": "1-6142c5a0af67a0470daf36151d3f9d8c"
- }
- },
-
- {
- "id": "jquery-atomic-nav",
- "key": "jquery-atomic-nav",
- "value": {
- "rev": "1-18e4ef14be83a907cbee0cd0adee25d4"
- }
- },
-
- {
- "id": "jquery-attach",
- "key": "jquery-attach",
- "value": {
- "rev": "8-da4f17596c25a02b0cce266e59706d5f"
- }
- }
-
- ]
-}
diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11"
deleted file mode 100644
--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11"
+++ /dev/null
@@ -1,83 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 422482,
- "rows": [
- {
- "id": "jquery",
- "key": "jquery",
- "value": {
- "rev": "212-2eac7c93af4c8bccdf7317739f0319b6"
- }
- },
- {
- "id": "jquery-1.8",
- "key": "jquery-1.8",
- "value": {
- "rev": "1-711ded49a7453adce85ce7a51c2157de"
- }
- },
- {
- "id": "jquery-1x",
- "key": "jquery-1x",
- "value": {
- "rev": "1-c53fa04d9c8fb231336704508732c287"
- }
- },
- {
- "id": "jquery-2-typescript-async-await-adapter",
- "key": "jquery-2-typescript-async-await-adapter",
- "value": {
- "rev": "8-5cfb484e9afaa6e326a97240fccd8f93"
- }
- },
- {
- "id": "jquery-accessible-accordion-aria",
- "key": "jquery-accessible-accordion-aria",
- "value": {
- "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e"
- }
- },
- {
- "id": "jquery-accessible-autocomplete-list-aria",
- "key": "jquery-accessible-autocomplete-list-aria",
- "value": {
- "rev": "8-961b382442c1a5bafe58f0e05424701d"
- }
- },
- {
- "id": "jquery-accessible-carrousel-aria",
- "key": "jquery-accessible-carrousel-aria",
- "value": {
- "rev": "9-f33f59d7f601bafe023bd711b551282b"
- }
- },
- {
- "id": "jquery-accessible-dialog-tooltip-aria",
- "key": "jquery-accessible-dialog-tooltip-aria",
- "value": {
- "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba"
- }
- },
- {
- "id": "jquery-accessible-hide-show-aria",
- "key": "jquery-accessible-hide-show-aria",
- "value": {
- "rev": "10-5a03c47a8995b08246e4bc103782dafa"
- }
- },
- {
- "id": "jquery-accessible-modal-window-aria",
- "key": "jquery-accessible-modal-window-aria",
- "value": {
- "rev": "18-50266e260f6b807019cfcfcd3a3685ab"
- }
- },
- {
- "id": "jquery-accessible-simple-tooltip-aria",
- "key": "jquery-accessible-simple-tooltip-aria",
- "value": {
- "rev": "6-ea71aa37760790dc603b56117f054e1b"
- }
- }
- ]
-}
diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11"
deleted file mode 100644
--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11"
+++ /dev/null
@@ -1,83 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 422482,
- "rows": [
- {
- "id": "jquery-accessible-simple-tooltip-aria",
- "key": "jquery-accessible-simple-tooltip-aria",
- "value": {
- "rev": "6-ea71aa37760790dc603b56117f054e1b"
- }
- },
- {
- "id": "jquery-accessible-subnav-dropdown",
- "key": "jquery-accessible-subnav-dropdown",
- "value": {
- "rev": "2-496f017a9ac243655225e43b5697b09b"
- }
- },
- {
- "id": "jquery-accessible-tabpanel-aria",
- "key": "jquery-accessible-tabpanel-aria",
- "value": {
- "rev": "11-659971471e6ac0fbb3b2f78ad208722a"
- }
- },
- {
- "id": "jquery-accessible-tabs-umd",
- "key": "jquery-accessible-tabs-umd",
- "value": {
- "rev": "1-f92015de5bb36e411d8c0940cca2883f"
- }
- },
- {
- "id": "jquery-active-descendant",
- "key": "jquery-active-descendant",
- "value": {
- "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de"
- }
- },
- {
- "id": "jquery-ada-validation",
- "key": "jquery-ada-validation",
- "value": {
- "rev": "1-9aab9629027c29fbece90485dd9d3112"
- }
- },
- {
- "id": "jquery-adaptText",
- "key": "jquery-adaptText",
- "value": {
- "rev": "3-2e15fc801ea8235b9180a3defc782ed0"
- }
- },
- {
- "id": "jquery-adapttr",
- "key": "jquery-adapttr",
- "value": {
- "rev": "6-74585f2d4be60b3f493585a6d28b90bc"
- }
- },
- {
- "id": "jquery-add-prefixed-class",
- "key": "jquery-add-prefixed-class",
- "value": {
- "rev": "1-9e43aee9758504b3f5271e9804a95f20"
- }
- },
- {
- "id": "jquery-address",
- "key": "jquery-address",
- "value": {
- "rev": "1-64173ede32157b26f4de910ad0f49590"
- }
- },
- {
- "id": "jquery-address-suggestion",
- "key": "jquery-address-suggestion",
- "value": {
- "rev": "6-18d9df51d472c365bcd84a61c9105774"
- }
- }
- ]
-}
diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json
deleted file mode 120000
--- a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json
+++ /dev/null
@@ -1 +0,0 @@
-_all_docs,startkey="",limit=1001
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_full_page1.json b/swh/lister/npm/tests/data/npm_full_page1.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_full_page1.json
@@ -0,0 +1,235 @@
+{
+ "total_rows": 1496481,
+ "offset": 0,
+ "rows": [{
+ "id": "-",
+ "key": "-",
+ "value": {
+ "rev": "1-e340e64b090ea8c6b4e14edc0460c751"
+ },
+ "doc": {
+ "_id": "-",
+ "_rev": "1-e340e64b090ea8c6b4e14edc0460c751",
+ "name": "-",
+ "dist-tags": {
+ "latest": "0.0.1"
+ },
+ "versions": {
+ "0.0.1": {
+ "name": "-",
+ "version": "0.0.1",
+ "license": "UNLICENSED",
+ "keywords": [],
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1",
+ "start": "node --require ts-node/register src",
+ "build": "tsc"
+ },
+ "devDependencies": {
+ "@types/node": "13.9.0",
+ "ts-node": "8.6.2",
+ "typescript": "3.8.3"
+ },
+ "description": "> Created using https://github.com/parzh/create-package-typescript",
+ "_id": "-@0.0.1",
+ "_nodeVersion": "13.3.0",
+ "_npmVersion": "6.14.4",
+ "dist": {
+ "integrity": "sha512-3HfneK3DGAm05fpyj20sT3apkNcvPpCuccOThOPdzz8sY7GgQGe0l93XH9bt+YzibcTIgUAIMoyVJI740RtgyQ==",
+ "shasum": "db6db7cd866142880dd03e5b8781d1b4fac0e5bd",
+ "tarball": "https://registry.npmjs.org/-/-/--0.0.1.tgz",
+ "fileCount": 3,
+ "unpackedSize": 600,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJehdEkCRA9TVsSAnZWagAAqcEQAIUCCWB2RzNV2p5CTeWf\nntknmFRbGh3K/gptyBnTC3MaNtk54Vf36XK1BfKgKz2rd/FOq1xcSQ8XUWt6\n7ZL9zSR3K/EBEgNr6p00fJAJ7ZdYu2kqq+uiZDKpSHLd6/bKaSdt0bAL78co\nmnzKOn9WULS1UC/ac8txNX9QdPg/aJezzxQadtKQR+SX6Fd/tTHKIsWv3ytI\nUtH/Lg9AM/tdTc94m2Ivo3+JxmkU+p8z7hjR97ym+AaZE8g1a47begVsSeZ/\nrIu1ZFCQ/XYFp0amNZ131qnW6FGnzpPlQTfkuJ1sPIORM9+o+6Ql6qa0mtde\nSJyAMKFfemmwkp56aZkL5NFX1snyJE4itfg99gsUrogHZIW3Jd1LqUe9yFZc\nVik1NZxqG4x6ktoog9IzG8ElOBWauKQAT+FIgNGea6IBv4wPQ8VoNOd4JKW/\nadvzus0hAdEZLyVsCyDw4wDCAK9JJXnSlJmJFc/vPtgPrbAj8xcBW6XIn6Nw\n0EVm993zlKmQ6/VmQUbHx8EP1zBm7M0vTcoI8dPIJkjGCZgWU571JARCetb2\npob+siYon3ndpoHG2OOWOuoRhwi4YeDW1ilc9isPve+S/iygKnfY8LTqHN1P\nOntz9Cu9YxHQ3WEiBuun7KP7FECbiSkozmzKWLHUhs2qXFSFYWeBLEI/HS1y\nJ9QO\r\n=lB+Q\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "parzhitsky",
+ "email": "parzhitsky@gmail.com"
+ }],
+ "_npmUser": {
+ "name": "parzhitsky",
+ "email": "parzhitsky@gmail.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-_0.0.1_1585828132159_0.6185838126718755"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-04-02T11:48:52.159Z",
+ "0.0.1": "2020-04-02T11:48:52.339Z",
+ "modified": "2020-04-02T11:49:00.248Z"
+ },
+ "maintainers": [{
+ "name": "parzhitsky",
+ "email": "parzhitsky@gmail.com"
+ }],
+ "description": "> Created using https://github.com/parzh/create-package-typescript",
+ "keywords": [],
+ "license": "UNLICENSED",
+ "readme": "# `-`\r\n\r\n> Created using https://github.com/parzh/create-package-typescript\r\n\r\n```\r\nnpm i -\r\n```\r\n",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "id": "--file--1",
+ "key": "--file--1",
+ "value": {
+ "rev": "1-b23c0a584f925336378c29df0b858aff"
+ },
+ "doc": {
+ "_id": "--file--1",
+ "_rev": "1-b23c0a584f925336378c29df0b858aff",
+ "name": "--file--1",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "--file--1",
+ "version": "1.0.0",
+ "description": "",
+ "main": "file.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "_id": "--file--1@1.0.0",
+ "_nodeVersion": "15.3.0",
+ "_npmVersion": "7.0.14",
+ "dist": {
+ "integrity": "sha512-OtGGxxqxDl1KCCUts3fN0gL4vW28u8WNIo+eqQON78IVxKJS3gYchaEZDrRubC4SyQOhDDKDtvswwLOkApuicQ==",
+ "shasum": "58ed9d9169197ded653d8df7153968d22c14ea59",
+ "tarball": "https://registry.npmjs.org/--file--1/-/--file--1-1.0.0.tgz",
+ "fileCount": 6,
+ "unpackedSize": 1327,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf2AGJCRA9TVsSAnZWagAALqcP+wZ9tzFsdQy4cEb5HGV6\nxV0LTnW+wxFYBUm2IfOwK7UH/Pl5Nym4yxlIjMEBinb9efepUAsucCyAka/v\n0OBGn+8I8kHDKvfhxiUVbi3CFAYcQTaaPffK5xJZi77netdsaLyS0cdx4RBm\nEWRIVK4axJBSUT6UVpmRqB73OlAv+JmNzBmL+CXHn9gksyvnm74pbT8tukLV\nX0TwkXNBtzn4lmydPRyecEZRDFydzp9UBQJyJACtpYxgTAwTT5ENtSymx4Ql\nlEqC9cqT9Q3n7z8lvntaCb6jgNN1aSWP3hOKCYQzezk5uSacJDWJOGAK7DE5\ndKUqsWa0HG84H+MMLxuepbrNYM5Fm7ulykl1yMrAf1G8T5pU/fUpq5uEHbqW\naVbW3GlCxJuUvlaTTFjGqDJx4tepY+Uel+TQigJoN2DGXTWiB+JeAAWLP+cu\nqcoZwkqoISmgru2g1+6TR//hod3CnCK4eQl/ph3VthtKyh4d1zUQAPcCGax6\nocJSWQsLRNW0QUmvpHeHQQOUGEb0XDTYkl9i8s8DY6JCqfgl9UIDZzNl2nK8\n385JseJKKLQR3tEk+HLZM2UGF6XgACBFTHYxlOrpzGgX+PJ1ns3noMrKo2jO\nbnvMN6Mp0GtoSzQj8FUcoU6BgOFHhHkB/i4Ke2VVeMYVj/CrFO13pZJOPR4O\n6uts\r\n=W7vP\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "_npmUser": {
+ "name": "guosuzhen",
+ "email": "1077655229@qq.com"
+ },
+ "directories": {},
+ "maintainers": [{
+ "name": "guosuzhen",
+ "email": "1077655229@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/--file--1_1.0.0_1607991689533_0.6331916469772794"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-12-15T00:21:29.533Z",
+ "1.0.0": "2020-12-15T00:21:29.657Z",
+ "modified": "2020-12-15T00:21:32.116Z"
+ },
+ "maintainers": [{
+ "name": "guosuzhen",
+ "email": "1077655229@qq.com"
+ }],
+ "keywords": [],
+ "license": "ISC",
+ "readme": "ERROR: No README data found!",
+ "readmeFilename": ""
+ }
+ },
+ {
+ "id": "-keyboardevent",
+ "key": "-keyboardevent",
+ "value": {
+ "rev": "1-0fdff278618c4f8c1026889f1b525483"
+ },
+ "doc": {
+ "_id": "-keyboardevent",
+ "_rev": "1-0fdff278618c4f8c1026889f1b525483",
+ "name": "-keyboardevent",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "-keyboardevent",
+ "version": "1.0.0",
+ "description": "监听键盘事件",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "license": "ISO",
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3",
+ "_id": "-keyboardevent@1.0.0",
+ "_nodeVersion": "12.10.0",
+ "_npmVersion": "6.10.3",
+ "dist": {
+ "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==",
+ "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5",
+ "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz",
+ "fileCount": 3,
+ "unpackedSize": 2714,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "_npmUser": {
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-09-25T06:10:56.357Z",
+ "1.0.0": "2020-09-25T06:10:56.513Z",
+ "modified": "2020-09-25T06:10:58.774Z"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "description": "监听键盘事件",
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "license": "ISO",
+ "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n",
+ "readmeFilename": "README.md"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_full_page2.json b/swh/lister/npm/tests/data/npm_full_page2.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_full_page2.json
@@ -0,0 +1,168 @@
+{
+ "total_rows": 1496482,
+ "offset": 2,
+ "rows": [{
+ "id": "-keyboardevent",
+ "key": "-keyboardevent",
+ "value": {
+ "rev": "1-0fdff278618c4f8c1026889f1b525483"
+ },
+ "doc": {
+ "_id": "-keyboardevent",
+ "_rev": "1-0fdff278618c4f8c1026889f1b525483",
+ "name": "-keyboardevent",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "-keyboardevent",
+ "version": "1.0.0",
+ "description": "监听键盘事件",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "license": "ISO",
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3",
+ "_id": "-keyboardevent@1.0.0",
+ "_nodeVersion": "12.10.0",
+ "_npmVersion": "6.10.3",
+ "dist": {
+ "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==",
+ "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5",
+ "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz",
+ "fileCount": 3,
+ "unpackedSize": 2714,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "_npmUser": {
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-09-25T06:10:56.357Z",
+ "1.0.0": "2020-09-25T06:10:56.513Z",
+ "modified": "2020-09-25T06:10:58.774Z"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "description": "监听键盘事件",
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "license": "ISO",
+ "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "id": "-lidonghui",
+ "key": "-lidonghui",
+ "value": {
+ "rev": "1-1aa03a85878671cdb74b789c2b124578"
+ },
+ "doc": {
+ "_id": "-lidonghui",
+ "_rev": "1-1aa03a85878671cdb74b789c2b124578",
+ "name": "-lidonghui",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "-lidonghui",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "bin": {
+ "test1-cli": "bin/index.js"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "axios": "^0.19.2",
+ "commander": "^5.0.0",
+ "inquirer": "^7.1.0"
+ },
+ "_id": "-lidonghui@1.0.0",
+ "_nodeVersion": "12.14.0",
+ "_npmVersion": "6.13.4",
+ "dist": {
+ "integrity": "sha512-1eUTEDU1kgAAsQ85uCEAp9tDsEIh3WNa5dB/ks8Uhf9Nh2RR37EeTGyxszn2A1B+MN+Q0fWrpCfBFzrrFyLFnA==",
+ "shasum": "be7ffedc328bd123ba13c468a89ebcf0cfe76777",
+ "tarball": "https://registry.npmjs.org/-lidonghui/-/-lidonghui-1.0.0.tgz",
+ "fileCount": 2,
+ "unpackedSize": 1024,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJenVH+CRA9TVsSAnZWagAAEIMP/ipex7Yo+T2LqfWtFuZz\n5NuXhcnaNyANXyJ7iQCYzq1ynq+tkwBOTM7FNLkTbIiIUsn8d6JlAt5CfIFZ\nrfaYDK7BhU4E8KfLoSC0VqyaprTSvAeqeQDEAguYM9Nbt4TWa7W2HoorgQd7\njUAuGpRqvkw8Omh3eOvhqEa0Y5mkRf853ndJHD425bAww2R0KCA7RuOBO1MW\nnPZqBe3XOgJmsyEW+yl2tSGVq2hEEeD3C0G9x5K2ZfxZKVU7lGfqRlUdo7na\nTLJU6YZPTB9AhDTM9b8XFUMWlEtvHcwvGvuB+gLupXqWwEfwSqWg/2TJ11AQ\nMBFeC1xSav0BCv6kFxFBRwz1k7IAdVYatLnmqwNsIGwjWsyfmIThSNzYTIq6\nE/E0iD2gSAsLW5r7U0t+rk4sWLmu4xbLDvT1xpEY7C00Zj4NXJhdXSo5BR45\n1IBiU96LqEA2bIw+TsyaFO7w/chgpvd8mU0g34tYQCcAts56ZhCr43x5kUSY\ns7hm1b3MB5qpOBbom1Ta24xjvoJwxv2Et/T7zfZoXzHDyX9BgUCiVIvupxd9\nzR5LdyXIh6JdiVBAgEftbm59FEVrxVfAAi/sFMNJKp8pE1ELrSp/gulzxDd2\nC5Hq1z5RNFRQUlbkMJHLIDOZLAvdkPKieE+uFJ7WTYCPEL9nvQoIJiO9UFH1\njKtU\r\n=Opv1\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "tenderli",
+ "email": "l2369660482@163.com"
+ }],
+ "_npmUser": {
+ "name": "tenderli",
+ "email": "l2369660482@163.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-lidonghui_1.0.0_1587368445998_0.028137547296155807"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-04-20T07:40:45.997Z",
+ "1.0.0": "2020-04-20T07:40:46.087Z",
+ "modified": "2020-04-20T07:40:48.525Z"
+ },
+ "maintainers": [{
+ "name": "tenderli",
+ "email": "l2369660482@163.com"
+ }],
+ "keywords": [],
+ "license": "ISC",
+ "readme": "ERROR: No README data found!",
+ "readmeFilename": ""
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_incremental_page1.json b/swh/lister/npm/tests/data/npm_incremental_page1.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_incremental_page1.json
@@ -0,0 +1,175 @@
+{
+ "results": [{
+ "seq": 1,
+ "id": "overlay-fishnet",
+ "changes": [{
+ "rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f"
+ }],
+ "doc": {
+ "_id": "overlay-fishnet",
+ "_rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f",
+ "name": "overlay-fishnet",
+ "description": "Overlay polygon with equal sized tiles (like a fishnet)",
+ "dist-tags": {
+ "latest": "0.0.0"
+ },
+ "versions": {
+ "0.0.0": {
+ "name": "overlay-fishnet",
+ "version": "0.0.0",
+ "description": "Overlay polygon with equal sized tiles (like a fishnet)",
+ "main": "dist/index.js",
+ "scripts": {
+ "build": "rm -rf dist && mkdir dist && babel src --out-dir dist",
+ "prepublish": "npm run test && npm run build",
+ "test": "standard"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/yongjun21/fishnet.git"
+ },
+ "keywords": [
+ "geospatial",
+ "polygon",
+ "tiles",
+ "choropleth"
+ ],
+ "author": {
+ "name": "Yong Jun"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/yongjun21/fishnet/issues"
+ },
+ "homepage": "https://github.com/yongjun21/fishnet#readme",
+ "dependencies": {
+ "babel-cli": "^6.18.0",
+ "babel-preset-es2015": "^6.18.0",
+ "babel-preset-stage-3": "^6.17.0",
+ "standard": "^8.6.0"
+ },
+ "gitHead": "bc3d598445a767efefd094260c83d6f57f06e86d",
+ "_id": "overlay-fishnet@0.0.0",
+ "_shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0",
+ "_from": ".",
+ "_npmVersion": "3.10.7",
+ "_nodeVersion": "6.2.2",
+ "_npmUser": {
+ "name": "yongjun21",
+ "email": "yjthong.2009@smu.edu.sg"
+ },
+ "dist": {
+ "shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0",
+ "tarball": "https://registry.npmjs.org/overlay-fishnet/-/overlay-fishnet-0.0.0.tgz"
+ },
+ "maintainers": [{
+ "name": "yongjun21",
+ "email": "yjthong.2009@smu.edu.sg"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/overlay-fishnet-0.0.0.tgz_1484192243466_0.4572450101841241"
+ }
+ }
+ },
+ "readme": "# fishnet\nOverlay polygon with equal sized tiles (like a fishnet)\n",
+ "maintainers": [{
+ "name": "yongjun21",
+ "email": "yjthong.2009@smu.edu.sg"
+ }],
+ "time": {
+ "modified": "2017-01-12T03:37:23.696Z",
+ "created": "2017-01-12T03:37:23.696Z",
+ "0.0.0": "2017-01-12T03:37:23.696Z"
+ },
+ "homepage": "https://github.com/yongjun21/fishnet#readme",
+ "keywords": [
+ "geospatial",
+ "polygon",
+ "tiles",
+ "choropleth"
+ ],
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/yongjun21/fishnet.git"
+ },
+ "author": {
+ "name": "Yong Jun"
+ },
+ "bugs": {
+ "url": "https://github.com/yongjun21/fishnet/issues"
+ },
+ "license": "ISC",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "seq": 2,
+ "id": "shopping_site_detect",
+ "changes": [{
+ "rev": "1-c62a780cdd9865b06b1e4eb0e6552560"
+ }],
+ "doc": {
+ "_id": "shopping_site_detect",
+ "_rev": "1-c62a780cdd9865b06b1e4eb0e6552560",
+ "name": "shopping_site_detect",
+ "description": "shopping site detect",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "shopping_site_detect",
+ "version": "1.0.0",
+ "description": "shopping site detect",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": {
+ "name": "Jayin Ton"
+ },
+ "license": "MIT",
+ "gitHead": "223c4a7e12acf28310a106780652d1e7d0629d68",
+ "_id": "shopping_site_detect@1.0.0",
+ "_shasum": "0408f054053d7581869d42934e42d50794d20b8c",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.4.0",
+ "_npmUser": {
+ "name": "jayinton",
+ "email": "tonjayin@gmail.com"
+ },
+ "dist": {
+ "shasum": "0408f054053d7581869d42934e42d50794d20b8c",
+ "tarball": "https://registry.npmjs.org/shopping_site_detect/-/shopping_site_detect-1.0.0.tgz"
+ },
+ "maintainers": [{
+ "name": "jayinton",
+ "email": "tonjayin@gmail.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/shopping_site_detect-1.0.0.tgz_1484193398924_0.3402463826350868"
+ }
+ }
+ },
+ "maintainers": [{
+ "name": "jayinton",
+ "email": "tonjayin@gmail.com"
+ }],
+ "time": {
+ "modified": "2017-01-12T03:56:39.161Z",
+ "created": "2017-01-12T03:56:39.161Z",
+ "1.0.0": "2017-01-12T03:56:39.161Z"
+ },
+ "author": {
+ "name": "Jayin Ton"
+ },
+ "license": "MIT",
+ "readmeFilename": "readme.md"
+ }
+ }
+ ],
+ "last_seq": 2
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_incremental_page2.json b/swh/lister/npm/tests/data/npm_incremental_page2.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_incremental_page2.json
@@ -0,0 +1,632 @@
+{
+ "results": [{
+ "seq": 3,
+ "id": "tinyviewpager",
+ "changes": [{
+ "rev": "1-01e65a5602e9c43ed3482657f1309ef0"
+ }],
+ "doc": {
+ "_id": "tinyviewpager",
+ "_rev": "1-01e65a5602e9c43ed3482657f1309ef0",
+ "name": "tinyviewpager",
+ "time": {
+ "modified": "2017-01-12T04:05:34.847Z",
+ "created": "2017-01-11T10:52:41.047Z",
+ "1.0.0": "2017-01-11T10:52:41.047Z",
+ "1.0.1": "2017-01-11T14:02:03.232Z",
+ "1.0.2": "2017-01-11T14:31:04.820Z",
+ "1.0.3": "2017-01-11T14:38:41.688Z",
+ "1.0.4": "2017-01-11T14:45:57.970Z",
+ "1.0.5": "2017-01-11T14:49:04.778Z",
+ "1.0.6": "2017-01-11T14:54:06.876Z",
+ "1.0.7": "2017-01-11T15:07:13.271Z",
+ "1.0.8": "2017-01-12T04:05:34.847Z"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "dist-tags": {
+ "latest": "1.0.8"
+ },
+ "description": "上下左右无限循环viewpager",
+ "versions": {
+ "1.0.1": {
+ "name": "tinyviewpager",
+ "version": "1.0.1",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "fdb455a393c7d6b6d43d01f73362fa6e3ed0dbb9",
+ "_id": "tinyviewpager@1.0.1",
+ "_shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.1.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.1.tgz_1484143322968_0.5517262546345592"
+ }
+ },
+ "1.0.2": {
+ "name": "tinyviewpager",
+ "version": "1.0.2",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "daae30e6ce7bea30a14619b5cef5f52a86cc5528",
+ "_id": "tinyviewpager@1.0.2",
+ "_shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.2.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.2.tgz_1484145064595_0.49263988458551466"
+ }
+ },
+ "1.0.3": {
+ "name": "tinyviewpager",
+ "version": "1.0.3",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "d08aeab765b537d71ed21e7e37ae6546e9bdbef2",
+ "_id": "tinyviewpager@1.0.3",
+ "_shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.3.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.3.tgz_1484145521456_0.8925717421807349"
+ }
+ },
+ "1.0.4": {
+ "name": "tinyviewpager",
+ "version": "1.0.4",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.4",
+ "_shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.4.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.4.tgz_1484145957736_0.7274783558677882"
+ }
+ },
+ "1.0.5": {
+ "name": "tinyviewpager",
+ "version": "1.0.5",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.5",
+ "_shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.5.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.5.tgz_1484146144523_0.9442351886536926"
+ }
+ },
+ "1.0.6": {
+ "name": "tinyviewpager",
+ "version": "1.0.6",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.6",
+ "_shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.6.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.6.tgz_1484146444932_0.7940177812706679"
+ }
+ },
+ "1.0.7": {
+ "name": "tinyviewpager",
+ "version": "1.0.7",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.7",
+ "_shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.7.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.7.tgz_1484147231500_0.17695364635437727"
+ }
+ },
+ "1.0.8": {
+ "name": "tinyviewpager",
+ "version": "1.0.8",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "54b9e9f1517a0385307065f96ef85e9b7b13868e",
+ "_id": "tinyviewpager@1.0.8",
+ "_shasum": "67234eebbae3a8205319c950fab62c8f84840061",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "67234eebbae3a8205319c950fab62c8f84840061",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.8.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.8.tgz_1484193932800_0.371786616044119"
+ }
+ }
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "keywords": [
+ "viewpager"
+ ],
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "author": {
+ "name": "tinylpc"
+ },
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "license": "ISC",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "seq": 4,
+ "id": "ubase-ext-wecloud",
+ "changes": [{
+ "rev": "1-0a540144594976690ff540d57ed3ab3b"
+ }],
+ "doc": {
+ "_id": "ubase-ext-wecloud",
+ "_rev": "1-0a540144594976690ff540d57ed3ab3b",
+ "name": "ubase-ext-wecloud",
+ "dist-tags": {
+ "latest": "1.0.3"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "_id": "ubase-ext-wecloud@1.0.0",
+ "_shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.0.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.0.tgz_1484191505909_0.5115192488301545"
+ }
+ },
+ "1.0.1": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.1",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "_id": "ubase-ext-wecloud@1.0.1",
+ "_shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.1.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.1.tgz_1484191845449_0.817801987985149"
+ }
+ },
+ "1.0.2": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.2",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "_id": "ubase-ext-wecloud@1.0.2",
+ "_shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.2.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.2.tgz_1484192705755_0.1207962513435632"
+ }
+ },
+ "1.0.3": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.3",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "gitHead": "a5b60823b706c4fd98ae798dbe76f18a7841eead",
+ "_id": "ubase-ext-wecloud@1.0.3",
+ "_shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.3.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.3.tgz_1484193826093_0.8929786814842373"
+ }
+ }
+ },
+ "readme": "# ubase-ext-wecloud\n",
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "time": {
+ "modified": "2017-01-12T04:03:46.329Z",
+ "created": "2017-01-12T03:25:07.858Z",
+ "1.0.0": "2017-01-12T03:25:07.858Z",
+ "1.0.1": "2017-01-12T03:30:45.704Z",
+ "1.0.2": "2017-01-12T03:45:07.635Z",
+ "1.0.3": "2017-01-12T04:03:46.329Z"
+ },
+ "license": "ISC",
+ "readmeFilename": "README.md"
+ }
+ }
+ ],
+ "last_seq": 4
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py
--- a/swh/lister/npm/tests/test_lister.py
+++ b/swh/lister/npm/tests/test_lister.py
@@ -1,97 +1,200 @@
-# Copyright (C) 2018-2019 The Software Heritage developers
+# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import logging
-import re
-from typing import Any, List
-import unittest
+from itertools import chain
+import json
+from pathlib import Path
-import requests_mock
+import iso8601
+import pytest
+from requests.exceptions import HTTPError
-from swh.lister.core.tests.test_lister import HttpListerTesterBase
-from swh.lister.npm.lister import NpmIncrementalLister, NpmLister
+from swh.lister import USER_AGENT
+from swh.lister.npm.lister import NpmLister, NpmListerState
-logger = logging.getLogger(__name__)
+@pytest.fixture
+def npm_full_listing_page1(datadir):
+ return json.loads(Path(datadir, "npm_full_page1.json").read_text())
-class NpmListerTester(HttpListerTesterBase, unittest.TestCase):
- Lister = NpmLister
- test_re = re.compile(r'^.*/_all_docs\?startkey="(.+)".*')
- lister_subdir = "npm"
- good_api_response_file = "data/replicate.npmjs.com/api_response.json"
- bad_api_response_file = "data/api_empty_response.json"
- first_index = "jquery"
- entries_per_page = 100
- @requests_mock.Mocker()
- def test_is_within_bounds(self, http_mocker):
- # disable this test from HttpListerTesterBase as
- # it can not succeed for the npm lister due to the
- # overriding of the string_pattern_check method
- pass
+@pytest.fixture
+def npm_full_listing_page2(datadir):
+ return json.loads(Path(datadir, "npm_full_page2.json").read_text())
-class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase):
- Lister = NpmIncrementalLister
- test_re = re.compile(r"^.*/_changes\?since=([0-9]+).*")
- lister_subdir = "npm"
- good_api_response_file = "data/api_inc_response.json"
- bad_api_response_file = "data/api_inc_empty_response.json"
- first_index = "6920642"
- entries_per_page = 100
+@pytest.fixture
+def npm_incremental_listing_page1(datadir):
+ return json.loads(Path(datadir, "npm_incremental_page1.json").read_text())
- @requests_mock.Mocker()
- def test_is_within_bounds(self, http_mocker):
- # disable this test from HttpListerTesterBase as
- # it can not succeed for the npm lister due to the
- # overriding of the string_pattern_check method
- pass
+@pytest.fixture
+def npm_incremental_listing_page2(datadir):
+ return json.loads(Path(datadir, "npm_incremental_page2.json").read_text())
-def check_tasks(tasks: List[Any]):
- """Ensure scheduled tasks are in the expected format.
+def _check_listed_npm_packages(lister, packages, scheduler_origins):
+ for package in packages:
+ package_name = package["doc"]["name"]
+ latest_version = package["doc"]["dist-tags"]["latest"]
+ package_last_update = iso8601.parse_date(package["doc"]["time"][latest_version])
+ origin_url = lister.PACKAGE_URL_TEMPLATE.format(package_name=package_name)
- """
- for row in tasks:
- logger.debug("row: %s", row)
- assert row["type"] == "load-npm"
- # arguments check
- args = row["arguments"]["args"]
- assert len(args) == 0
+ scheduler_origin = [o for o in scheduler_origins if o.url == origin_url]
+ assert scheduler_origin
+ assert scheduler_origin[0].last_update == package_last_update
- # kwargs
- kwargs = row["arguments"]["kwargs"]
- assert len(kwargs) == 1
- package_url = kwargs["url"]
- package_name = package_url.split("/")[-1]
- assert package_url == f"https://www.npmjs.com/package/{package_name}"
- assert row["policy"] == "recurring"
- assert row["priority"] is None
+def _match_request(request):
+ return request.headers.get("User-Agent") == USER_AGENT
-def test_npm_lister_basic_listing(lister_npm, requests_mock_datadir):
- lister_npm.run()
+def _url_params(page_size, **kwargs):
+ params = {"limit": page_size, "include_docs": "true"}
+ params.update(**kwargs)
+ return params
- tasks = lister_npm.scheduler.search_tasks(task_type="load-npm")
- assert len(tasks) == 100
- check_tasks(tasks)
+def test_npm_lister_full(
+ swh_scheduler, requests_mock, mocker, npm_full_listing_page1, npm_full_listing_page2
+):
+ """Simulate a full listing of four npm packages in two pages"""
+ page_size = 2
+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=False)
+ requests_mock.get(
+ lister.API_FULL_LISTING_URL,
+ [{"json": npm_full_listing_page1}, {"json": npm_full_listing_page2},],
+ additional_matcher=_match_request,
+ )
+
+ spy_get = mocker.spy(lister.session, "get")
+
+ stats = lister.run()
+ assert stats.pages == 2
+ assert stats.origins == page_size * stats.pages
+
+ spy_get.assert_has_calls(
+ [
+ mocker.call(
+ lister.API_FULL_LISTING_URL,
+ params=_url_params(page_size + 1, startkey='""'),
+ ),
+ mocker.call(
+ lister.API_FULL_LISTING_URL,
+ params=_url_params(
+ page_size + 1,
+ startkey=f'"{npm_full_listing_page1["rows"][-1]["id"]}"',
+ ),
+ ),
+ ]
+ )
+
+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
+
+ _check_listed_npm_packages(
+ lister,
+ chain(npm_full_listing_page1["rows"][:-1], npm_full_listing_page2["rows"]),
+ scheduler_origins,
+ )
+
+ assert lister.get_state_from_scheduler() == NpmListerState()
+
+
+def test_npm_lister_incremental(
+ swh_scheduler,
+ requests_mock,
+ mocker,
+ npm_incremental_listing_page1,
+ npm_incremental_listing_page2,
+):
+ """Simulate an incremental listing of four npm packages in two pages"""
+ page_size = 2
+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True)
+
+ requests_mock.get(
+ lister.API_INCREMENTAL_LISTING_URL,
+ [
+ {"json": npm_incremental_listing_page1},
+ {"json": npm_incremental_listing_page2},
+ {"json": {"results": []}},
+ ],
+ additional_matcher=_match_request,
+ )
+
+ spy_get = mocker.spy(lister.session, "get")
+
+ assert lister.get_state_from_scheduler() == NpmListerState()
+
+ stats = lister.run()
+ assert stats.pages == 2
+ assert stats.origins == page_size * stats.pages
+
+ last_seq = npm_incremental_listing_page2["results"][-1]["seq"]
+
+ spy_get.assert_has_calls(
+ [
+ mocker.call(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(page_size, since="0"),
+ ),
+ mocker.call(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(
+ page_size,
+ since=str(npm_incremental_listing_page1["results"][-1]["seq"]),
+ ),
+ ),
+ mocker.call(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(page_size, since=str(last_seq)),
+ ),
+ ]
+ )
+
+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
-def test_npm_lister_listing_pagination(lister_npm, requests_mock_datadir):
- lister = lister_npm
- # Patch per page pagination
- lister.per_page = 10 + 1
- lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace(
- "&limit=1001", "&limit=%s" % lister.per_page
+ _check_listed_npm_packages(
+ lister,
+ chain(
+ npm_incremental_listing_page1["results"],
+ npm_incremental_listing_page2["results"],
+ ),
+ scheduler_origins,
)
+
+ assert lister.get_state_from_scheduler() == NpmListerState(last_seq=last_seq)
+
+
+def test_npm_lister_incremental_restart(
+ swh_scheduler, requests_mock, mocker,
+):
+ """Check incremental npm listing will restart from saved state"""
+ page_size = 2
+ last_seq = 67
+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True)
+ lister.state = NpmListerState(last_seq=last_seq)
+
+ requests_mock.get(lister.API_INCREMENTAL_LISTING_URL, json={"results": []})
+
+ spy_get = mocker.spy(lister.session, "get")
+
lister.run()
- tasks = lister.scheduler.search_tasks(task_type="load-npm")
- assert len(tasks) == 2 * 10 # only 2 files with 10 results each
+ spy_get.assert_called_with(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(page_size, since=str(last_seq)),
+ )
+
+
+def test_npm_lister_http_error(
+ swh_scheduler, requests_mock, mocker,
+):
+ lister = NpmLister(scheduler=swh_scheduler)
+
+ requests_mock.get(lister.API_FULL_LISTING_URL, status_code=500)
- check_tasks(tasks)
+ with pytest.raises(HTTPError):
+ lister.run()
diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py
--- a/swh/lister/npm/tests/test_tasks.py
+++ b/swh/lister/npm/tests/test_tasks.py
@@ -1,15 +1,9 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from contextlib import contextmanager
-from unittest.mock import patch
-
-
-@contextmanager
-def mock_save(lister):
- yield
+from swh.lister.pattern import ListerStats
def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
@@ -20,34 +14,31 @@
assert res.result == "OK"
-@patch("swh.lister.npm.tasks.save_registry_state")
-@patch("swh.lister.npm.tasks.NpmLister")
-def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker):
- # setup the mocked NpmLister
- lister.return_value = lister
- lister.run.return_value = None
- save.side_effect = mock_save
+def test_full_lister_task(
+ swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
+):
+ stats = ListerStats(pages=10, origins=900)
+ mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
+ mock_lister.from_configfile.return_value = mock_lister
+ mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask")
assert res
res.wait()
assert res.successful()
- lister.assert_called_once_with()
- lister.run.assert_called_once_with()
+ mock_lister.from_configfile.assert_called_once_with(incremental=False)
+ mock_lister.run.assert_called_once_with()
+ assert res.result == stats.dict()
-@patch("swh.lister.npm.tasks.save_registry_state")
-@patch("swh.lister.npm.tasks.get_last_update_seq")
-@patch("swh.lister.npm.tasks.NpmIncrementalLister")
-def test_incremental(
- lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker
+def test_incremental_lister_task(
+ swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
):
- # setup the mocked NpmLister
- lister.return_value = lister
- lister.run.return_value = None
- seq.return_value = 42
- save.side_effect = mock_save
+ stats = ListerStats(pages=10, origins=900)
+ mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
+ mock_lister.from_configfile.return_value = mock_lister
+ mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task(
"swh.lister.npm.tasks.NpmIncrementalListerTask"
@@ -56,6 +47,6 @@
res.wait()
assert res.successful()
- lister.assert_called_once_with()
- seq.assert_called_once_with(lister)
- lister.run.assert_called_once_with(min_bound=42)
+ mock_lister.from_configfile.assert_called_once_with(incremental=True)
+ mock_lister.run.assert_called_once_with()
+ assert res.result == stats.dict()
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Jul 3, 3:43 PM (2 w, 8 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220514
Attached To
D4877: npm: Reimplement lister using new Lister API
Event Timeline
Log In to Comment