D4877.diff
No OneTemporary
Actions

Size

117 KB

Subscribers

None

D4877.diff
View Options

	diff --git a/swh/lister/npm/__init__.py b/swh/lister/npm/__init__.py
	--- a/swh/lister/npm/__init__.py
	+++ b/swh/lister/npm/__init__.py
	@@ -1,14 +1,13 @@
	-# Copyright (C) 2019 the Software Heritage developers
	+# Copyright (C) 2019-2021 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information


	def register():
	from .lister import NpmLister
	- from .models import NpmModel, NpmVisitModel

	return {
	- "models": [NpmVisitModel, NpmModel],
	+ "models": [],
	"lister": NpmLister,
	"task_modules": ["%s.tasks" % __name__],
	"task_types": {
	diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py
	--- a/swh/lister/npm/lister.py
	+++ b/swh/lister/npm/lister.py
	@@ -1,154 +1,190 @@
	-# Copyright (C) 2018-2019 the Software Heritage developers
	+# Copyright (C) 2018-2021 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-from typing import Any, Dict, List, Optional
	+from dataclasses import asdict, dataclass
	+import logging
	+from typing import Any, Dict, Iterator, List, Optional

	-from requests import Response
	+import iso8601
	+import requests
	+from tenacity.before_sleep import before_sleep_log

	-from swh.core import config
	-from swh.lister.core.indexing_lister import IndexingHttpLister
	-from swh.lister.npm.models import NpmModel
	-from swh.scheduler.utils import create_task_dict
	+from swh.lister import USER_AGENT
	+from swh.lister.pattern import CredentialsType, Lister
	+from swh.lister.utils import throttling_retry
	+from swh.scheduler.interface import SchedulerInterface
	+from swh.scheduler.model import ListedOrigin

	-DEFAULT_CONFIG = {
	- "loading_task_policy": "recurring",
	-}
	+logger = logging.getLogger(__name__)


	-class NpmListerBase(IndexingHttpLister):
	- """List packages available in the npm registry in a paginated way
	+@dataclass
	+class NpmListerState:
	+ """State of npm lister"""

	- """
	-
	- MODEL = NpmModel
	- LISTER_NAME = "npm"
	- instance = "npm"
	+ last_seq: Optional[int] = None

	- def __init__(
	- self, url="https://replicate.npmjs.com", per_page=1000, override_config=None
	- ):
	- super().__init__(url=url, override_config=override_config)
	- self.config = config.merge_configs(DEFAULT_CONFIG, self.config)
	- self.per_page = per_page + 1
	- self.PATH_TEMPLATE += "&limit=%s" % self.per_page
	-
	- def get_model_from_repo(self, repo_name: str) -> Dict[str, str]:
	- """(Override) Transform from npm package name to model
	-
	- """
	- package_url = "https://www.npmjs.com/package/%s" % repo_name
	- return {
	- "uid": repo_name,
	- "indexable": repo_name,
	- "name": repo_name,
	- "full_name": repo_name,
	- "html_url": package_url,
	- "origin_url": package_url,
	- "origin_type": "npm",
	- }
	-
	- def task_dict(self, origin_type: str, origin_url: str, **kwargs):
	- """(Override) Return task dict for loading a npm package into the
	- archive.
	-
	- This is overridden from the lister_base as more information is
	- needed for the ingestion task creation.
	-
	- """
	- task_type = "load-%s" % origin_type
	- task_policy = self.config["loading_task_policy"]
	- return create_task_dict(task_type, task_policy, url=origin_url)
	-
	- def request_headers(self) -> Dict[str, Any]:
	- """(Override) Set requests headers to send when querying the npm
	- registry.
	-
	- """
	- headers = super().request_headers()
	- headers["Accept"] = "application/json"
	- return headers
	-
	- def string_pattern_check(self, inner: int, lower: int, upper: int = None):
	- """ (Override) Inhibit the effect of that method as packages indices
	- correspond to package names and thus do not respect any kind
	- of fixed length string pattern
	-
	- """
	- pass
	-
	-
	-class NpmLister(NpmListerBase):
	- """List all packages available in the npm registry in a paginated way

	+class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]):
	"""
	+ List all packages hosted on the npm registry.

	- PATH_TEMPLATE = '/_all_docs?startkey="%s"'
	-
	- def get_next_target_from_response(self, response: Response) -> Optional[str]:
	- """(Override) Get next npm package name to continue the listing
	-
	- """
	- repos = response.json()["rows"]
	- return repos[-1]["id"] if len(repos) == self.per_page else None
	-
	- def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]:
	- """(Override) Transform npm registry response to list for model manipulation
	-
	- """
	- repos = response.json()["rows"]
	- if len(repos) == self.per_page:
	- repos = repos[:-1]
	- return [self.get_model_from_repo(repo["id"]) for repo in repos]
	-
	+ The lister is based on the npm replication API powered by a
	+ CouchDB database (https://docs.couchdb.org/en/stable/api/database/).

	-class NpmIncrementalLister(NpmListerBase):
	- """List packages in the npm registry, updated since a specific
	- update_seq value of the underlying CouchDB database, in a paginated way.
	+ Args:
	+ scheduler: a scheduler instance
	+ page_size: number of packages info to return per page when querying npm API
	+ incremental: defines if incremental listing should be used, in that case
	+ only modified or new packages since last incremental listing operation
	+ will be returned, otherwise all packages will be listed in lexicographical
	+ order

	"""

	- PATH_TEMPLATE = "/_changes?since=%s"
	-
	- @property
	- def CONFIG_BASE_FILENAME(self): # noqa: N802
	- return "lister_npm_incremental"
	-
	- def get_next_target_from_response(self, response: Response) -> Optional[str]:
	- """(Override) Get next npm package name to continue the listing.
	-
	- """
	- repos = response.json()["results"]
	- return repos[-1]["seq"] if len(repos) == self.per_page else None
	-
	- def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]:
	- """(Override) Transform npm registry response to list for model
	- manipulation.
	-
	- """
	- repos = response.json()["results"]
	- if len(repos) == self.per_page:
	- repos = repos[:-1]
	- return [self.get_model_from_repo(repo["id"]) for repo in repos]
	+ LISTER_NAME = "npm"
	+ INSTANCE = "npm"

	- def filter_before_inject(self, models_list: List[Dict[str, Any]]):
	- """(Override) Filter out documents in the CouchDB database
	- not related to a npm package.
	+ API_BASE_URL = "https://replicate.npmjs.com"
	+ API_INCREMENTAL_LISTING_URL = f"{API_BASE_URL}/_changes"
	+ API_FULL_LISTING_URL = f"{API_BASE_URL}/_all_docs"
	+ PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}"

	- """
	- models_filtered = []
	- for model in models_list:
	- package_name = model["name"]
	- # document related to CouchDB internals
	- if package_name.startswith("_design/"):
	+ def __init__(
	+ self,
	+ scheduler: SchedulerInterface,
	+ page_size: int = 1000,
	+ incremental: bool = False,
	+ credentials: CredentialsType = None,
	+ ):
	+ super().__init__(
	+ scheduler=scheduler,
	+ credentials=credentials,
	+ url=self.API_INCREMENTAL_LISTING_URL
	+ if incremental
	+ else self.API_FULL_LISTING_URL,
	+ instance=self.INSTANCE,
	+ )
	+
	+ self.page_size = page_size
	+ if not incremental:
	+ # in full listing mode, first package in each page corresponds to the one
	+ # provided as the startkey query parameter value, so we increment the page
	+ # size by one to avoid double package processing
	+ self.page_size += 1
	+ self.incremental = incremental
	+
	+ self.session = requests.Session()
	+ self.session.headers.update(
	+ {"Accept": "application/json", "User-Agent": USER_AGENT}
	+ )
	+
	+ def state_from_dict(self, d: Dict[str, Any]) -> NpmListerState:
	+ return NpmListerState(**d)
	+
	+ def state_to_dict(self, state: NpmListerState) -> Dict[str, Any]:
	+ return asdict(state)
	+
	+ def request_params(self, last_package_id: str) -> Dict[str, Any]:
	+ # include package JSON document to get its last update date
	+ params = {"limit": self.page_size, "include_docs": "true"}
	+ if self.incremental:
	+ params["since"] = last_package_id
	+ else:
	+ params["startkey"] = last_package_id
	+ return params
	+
	+ @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
	+ def page_request(self, last_package_id: str) -> requests.Response:
	+ params = self.request_params(last_package_id)
	+ logger.debug("Fetching URL %s with params %s", self.url, params)
	+ response = self.session.get(self.url, params=params)
	+ if response.status_code != 200:
	+ logger.warning(
	+ "Unexpected HTTP status code %s on %s: %s",
	+ response.status_code,
	+ response.url,
	+ response.content,
	+ )
	+ response.raise_for_status()
	+ return response
	+
	+ def get_pages(self) -> Iterator[List[Dict[str, Any]]]:
	+ last_package_id: str = "0" if self.incremental else '""'
	+ if (
	+ self.incremental
	+ and self.state is not None
	+ and self.state.last_seq is not None
	+ ):
	+ last_package_id = str(self.state.last_seq)
	+
	+ while True:
	+
	+ response = self.page_request(last_package_id)
	+
	+ data = response.json()
	+ page = data["results"] if self.incremental else data["rows"]
	+
	+ if not page:
	+ break
	+
	+ if self.incremental or len(page) < self.page_size:
	+ yield page
	+ else:
	+ yield page[:-1]
	+
	+ if len(page) < self.page_size:
	+ break
	+
	+ last_package_id = (
	+ str(page[-1]["seq"]) if self.incremental else f'"{page[-1]["id"]}"'
	+ )
	+
	+ def get_origins_from_page(
	+ self, page: List[Dict[str, Any]]
	+ ) -> Iterator[ListedOrigin]:
	+ """Convert a page of Npm repositories into a list of ListedOrigin."""
	+ assert self.lister_obj.id is not None
	+
	+ for package in page:
	+ # no source code to archive here
	+ if not package["doc"].get("versions", {}):
	continue
	- models_filtered.append(model)
	- return models_filtered
	-
	- def disable_deleted_repo_tasks(self, start, end, keep_these):
	- """(Override) Disable the processing performed by that method as it is
	- not relevant in this incremental lister context. It also raises an
	- exception due to a different index type (int instead of str).

	- """
	- pass
	+ package_name = package["doc"]["name"]
	+ package_latest_version = (
	+ package["doc"].get("dist-tags", {}).get("latest", "")
	+ )
	+
	+ last_update = None
	+ if package_latest_version in package["doc"].get("time", {}):
	+ last_update = iso8601.parse_date(
	+ package["doc"]["time"][package_latest_version]
	+ )
	+
	+ yield ListedOrigin(
	+ lister_id=self.lister_obj.id,
	+ url=self.PACKAGE_URL_TEMPLATE.format(package_name=package_name),
	+ visit_type="npm",
	+ last_update=last_update,
	+ )
	+
	+ def commit_page(self, page: List[Dict[str, Any]]):
	+ """Update the currently stored state using the latest listed page."""
	+ if self.incremental:
	+ last_package = page[-1]
	+ last_seq = last_package["seq"]
	+
	+ if self.state.last_seq is None or last_seq > self.state.last_seq:
	+ self.state.last_seq = last_seq
	+
	+ def finalize(self):
	+ if self.incremental and self.state.last_seq is not None:
	+ scheduler_state = self.get_state_from_scheduler()
	+
	+ if (
	+ scheduler_state.last_seq is None
	+ or self.state.last_seq > scheduler_state.last_seq
	+ ):
	+ self.updated = True
	diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py
	deleted file mode 100644
	--- a/swh/lister/npm/models.py
	+++ /dev/null
	@@ -1,37 +0,0 @@
	-# Copyright (C) 2018 the Software Heritage developers
	-# License: GNU General Public License version 3, or any later version
	-# See top-level LICENSE file for more information
	-
	-from sqlalchemy import BigInteger, Column, DateTime, Integer, Sequence, String
	-
	-from swh.lister.core.models import ABCSQLMeta, IndexingModelBase, SQLBase
	-
	-
	-class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta):
	- """Table to store the npm registry state at the time of a
	- content listing by Software Heritage
	- """
	-
	- __tablename__ = "npm_visit"
	-
	- uid = Column(Integer, Sequence("npm_visit_id_seq"), primary_key=True)
	- visit_date = Column(DateTime, nullable=False)
	- doc_count = Column(BigInteger)
	- doc_del_count = Column(BigInteger)
	- update_seq = Column(BigInteger)
	- purge_seq = Column(BigInteger)
	- disk_size = Column(BigInteger)
	- data_size = Column(BigInteger)
	- committed_update_seq = Column(BigInteger)
	- compacted_seq = Column(BigInteger)
	-
	-
	-class NpmModel(IndexingModelBase):
	- """A npm package representation
	-
	- """
	-
	- __tablename__ = "npm_repo"
	-
	- uid = Column(String, primary_key=True)
	- indexable = Column(String, index=True)
	diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py
	--- a/swh/lister/npm/tasks.py
	+++ b/swh/lister/npm/tasks.py
	@@ -1,69 +1,24 @@
	-# Copyright (C) 2018 the Software Heritage developers
	+# Copyright (C) 2018-2021 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-from contextlib import contextmanager
	-from datetime import datetime
	-
	from celery import shared_task

	-from swh.lister.npm.lister import NpmIncrementalLister, NpmLister
	-from swh.lister.npm.models import NpmVisitModel
	-
	-
	-@contextmanager
	-def save_registry_state(lister):
	- params = {"headers": lister.request_headers()}
	- registry_state = lister.session.get(lister.url, **params)
	- registry_state = registry_state.json()
	- keys = (
	- "doc_count",
	- "doc_del_count",
	- "update_seq",
	- "purge_seq",
	- "disk_size",
	- "data_size",
	- "committed_update_seq",
	- "compacted_seq",
	- )
	-
	- state = {key: registry_state[key] for key in keys}
	- state["visit_date"] = datetime.now()
	- yield
	- npm_visit = NpmVisitModel(**state)
	- lister.db_session.add(npm_visit)
	- lister.db_session.commit()
	-
	-
	-def get_last_update_seq(lister):
	- """Get latest ``update_seq`` value for listing only updated packages.
	- """
	- query = lister.db_session.query(NpmVisitModel.update_seq)
	- row = query.order_by(NpmVisitModel.uid.desc()).first()
	- if not row:
	- raise ValueError(
	- "No npm registry listing previously performed ! "
	- "This is required prior to the execution of an "
	- "incremental listing."
	- )
	- return row[0]
	+from swh.lister.npm.lister import NpmLister


	@shared_task(name=__name__ + ".NpmListerTask")
	def list_npm_full(**lister_args):
	"Full lister for the npm (javascript) registry"
	- lister = NpmLister(**lister_args)
	- with save_registry_state(lister):
	- return lister.run()
	+ lister = NpmLister.from_configfile(incremental=False, **lister_args)
	+ return lister.run().dict()


	@shared_task(name=__name__ + ".NpmIncrementalListerTask")
	def list_npm_incremental(**lister_args):
	"Incremental lister for the npm (javascript) registry"
	- lister = NpmIncrementalLister(**lister_args)
	- update_seq_start = get_last_update_seq(lister)
	- with save_registry_state(lister):
	- return lister.run(min_bound=update_seq_start)
	+ lister = NpmLister.from_configfile(incremental=True, **lister_args)
	+ return lister.run().dict()


	@shared_task(name=__name__ + ".ping")
	diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py
	--- a/swh/lister/npm/tests/conftest.py
	+++ b/swh/lister/npm/tests/conftest.py
	@@ -1,4 +1,4 @@
	-# Copyright (C) 2019-2020 The Software Heritage developers
	+# Copyright (C) 2019-2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information
	@@ -9,18 +9,3 @@
	@pytest.fixture
	def lister_under_test():
	return "npm"
	-
	-
	-@pytest.fixture
	-def lister_npm(swh_lister):
	- # Add the load-npm in the scheduler backend
	- swh_lister.scheduler.create_task_type(
	- {
	- "type": "load-npm",
	- "description": "Load npm package",
	- "backend_name": "swh.loader.package.tasks.LoadNpm",
	- "default_interval": "1 day",
	- }
	- )
	-
	- return swh_lister
	diff --git a/swh/lister/npm/tests/data/api_empty_response.json b/swh/lister/npm/tests/data/api_empty_response.json
	deleted file mode 100644
	--- a/swh/lister/npm/tests/data/api_empty_response.json
	+++ /dev/null
	@@ -1,5 +0,0 @@
	-{
	- "total_rows": 839080,
	- "offset": 839080,
	- "rows": []
	-}
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/data/api_inc_empty_response.json b/swh/lister/npm/tests/data/api_inc_empty_response.json
	deleted file mode 100644
	--- a/swh/lister/npm/tests/data/api_inc_empty_response.json
	+++ /dev/null
	@@ -1,4 +0,0 @@
	-{
	- "results": [],
	- "last_seq": 6927821
	-}
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/data/api_inc_response.json b/swh/lister/npm/tests/data/api_inc_response.json
	deleted file mode 100644
	--- a/swh/lister/npm/tests/data/api_inc_response.json
	+++ /dev/null
	@@ -1,906 +0,0 @@
	-{
	- "results": [
	- {
	- "seq": 6920644,
	- "id": "electron-scripts",
	- "changes": [
	- {
	- "rev": "3-a19944df5a3636bb225af9e0c8f9eedc"
	- }
	- ]
	- },
	- {
	- "seq": 6920649,
	- "id": "@crexi-dev/schematics",
	- "changes": [
	- {
	- "rev": "3-00188360eeca1f9123b2d7cd4b468c50"
	- }
	- ]
	- },
	- {
	- "seq": 6920651,
	- "id": "botfactory-conversation",
	- "changes": [
	- {
	- "rev": "50-f3667cde87637505528c46adc87f44e3"
	- }
	- ]
	- },
	- {
	- "seq": 6920667,
	- "id": "castle",
	- "changes": [
	- {
	- "rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0"
	- }
	- ]
	- },
	- {
	- "seq": 6920671,
	- "id": "rbc-wm-framework-vuejs",
	- "changes": [
	- {
	- "rev": "111-32bed479afacdd88aed9ac16dd135843"
	- }
	- ]
	- },
	- {
	- "seq": 6920678,
	- "id": "bitcoinfiles",
	- "changes": [
	- {
	- "rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0"
	- }
	- ]
	- },
	- {
	- "seq": 6920679,
	- "id": "jovo-core",
	- "changes": [
	- {
	- "rev": "2-d7440f1d17823e1a0760d9b3d4537c6e"
	- }
	- ]
	- },
	- {
	- "seq": 6920687,
	- "id": "jovo-framework",
	- "changes": [
	- {
	- "rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9"
	- }
	- ]
	- },
	- {
	- "seq": 6920690,
	- "id": "smart-form-lib",
	- "changes": [
	- {
	- "rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade"
	- }
	- ]
	- },
	- {
	- "seq": 6920694,
	- "id": "bokehjs",
	- "changes": [
	- {
	- "rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2"
	- }
	- ]
	- },
	- {
	- "seq": 6920701,
	- "id": "guijarro",
	- "changes": [
	- {
	- "rev": "14-82ece581d6a35d4e1d78e5292ca245c0"
	- }
	- ]
	- },
	- {
	- "seq": 6920702,
	- "id": "@kava-labs/crypto-rate-utils",
	- "changes": [
	- {
	- "rev": "3-cecc6a6c226a0590b1a685e3041028c6"
	- }
	- ]
	- },
	- {
	- "seq": 6920703,
	- "id": "@riouxjean/test",
	- "changes": [
	- {
	- "rev": "10-01e97dc7d0241dc49ea93b3468ec7b29"
	- }
	- ]
	- },
	- {
	- "seq": 6920704,
	- "id": "react-scrabblefy",
	- "changes": [
	- {
	- "rev": "7-970c8206f3b8744204f7dcb106f8462b"
	- }
	- ]
	- },
	- {
	- "seq": 6920706,
	- "id": "molart",
	- "changes": [
	- {
	- "rev": "14-416cd3cec62dd46f9b59a3bbe35308f6"
	- }
	- ]
	- },
	- {
	- "seq": 6920707,
	- "id": "@universal-material/angular",
	- "changes": [
	- {
	- "rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3"
	- }
	- ]
	- },
	- {
	- "seq": 6920708,
	- "id": "cozy-doctypes",
	- "changes": [
	- {
	- "rev": "68-8e90cc26e25da6c9430d373e43ac3c25"
	- }
	- ]
	- },
	- {
	- "seq": 6920710,
	- "id": "2o3t-ui",
	- "changes": [
	- {
	- "rev": "96-1e65d5320ea7c78525aba5daf328bd4b"
	- }
	- ]
	- },
	- {
	- "seq": 6920712,
	- "id": "ark-ts",
	- "changes": [
	- {
	- "rev": "24-033183c2f7f9cbb6e44d553213e525b6"
	- }
	- ]
	- },
	- {
	- "seq": 6920715,
	- "id": "mysqlconnector",
	- "changes": [
	- {
	- "rev": "19-f09bc0b82281ca486db5ebe83843679e"
	- }
	- ]
	- },
	- {
	- "seq": 6920716,
	- "id": "@innovexa/ng-form-creator-lib",
	- "changes": [
	- {
	- "rev": "147-480665ee17fa889dfec1aee75b907ff2"
	- }
	- ]
	- },
	- {
	- "seq": 6920717,
	- "id": "k-routes-example-basic",
	- "changes": [
	- {
	- "rev": "1-35142059e1c63cc724da71a9eebf229c"
	- }
	- ]
	- },
	- {
	- "seq": 6920718,
	- "id": "wloggertojs",
	- "changes": [
	- {
	- "rev": "29-5b5aa74bd30ff0fc86b39fba799befe2"
	- }
	- ]
	- },
	- {
	- "seq": 6920720,
	- "id": "wloggertofile",
	- "changes": [
	- {
	- "rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5"
	- }
	- ]
	- },
	- {
	- "seq": 6920721,
	- "id": "@brightcove/flashls",
	- "changes": [
	- {
	- "rev": "62-fbadb49476a58e98f0f136c86b614734"
	- }
	- ]
	- },
	- {
	- "seq": 6920722,
	- "id": "@brightcove/hls-fetcher",
	- "changes": [
	- {
	- "rev": "76-3341ed8ade38f3251a97c94c3a7af5ac"
	- }
	- ]
	- },
	- {
	- "seq": 6920723,
	- "id": "@brightcove/kacl",
	- "changes": [
	- {
	- "rev": "33-d0bc6b639cccb301086114d548ecfdbf"
	- }
	- ]
	- },
	- {
	- "seq": 6920724,
	- "id": "just-in-types",
	- "changes": [
	- {
	- "rev": "2-fc329aa885dc795aee340f36ec60f333"
	- }
	- ]
	- },
	- {
	- "seq": 6920725,
	- "id": "@brightcove/player-loader",
	- "changes": [
	- {
	- "rev": "56-9ff5aebc9743a44d46c182746313877d"
	- }
	- ]
	- },
	- {
	- "seq": 6920726,
	- "id": "@brightcove/player-loader-webpack-plugin",
	- "changes": [
	- {
	- "rev": "33-db8b4d6765f19e475e1c1d16843824cb"
	- }
	- ]
	- },
	- {
	- "seq": 6920727,
	- "id": "@brightcove/player-url",
	- "changes": [
	- {
	- "rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc"
	- }
	- ]
	- },
	- {
	- "seq": 6920728,
	- "id": "@brightcove/react-player-loader",
	- "changes": [
	- {
	- "rev": "39-b7bf609de666ec7e71f517db53ab9c0a"
	- }
	- ]
	- },
	- {
	- "seq": 6920729,
	- "id": "vscode-theme-generator",
	- "changes": [
	- {
	- "rev": "21-bcb92281d6f7e37548bb18113681df88"
	- }
	- ]
	- },
	- {
	- "seq": 6920733,
	- "id": "@brightcove/typed-immutable-extensions",
	- "changes": [
	- {
	- "rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe"
	- }
	- ]
	- },
	- {
	- "seq": 6920734,
	- "id": "@brightcove/typed-immutable-proptypes",
	- "changes": [
	- {
	- "rev": "27-e4802afc947c55d34f778864476c17e4"
	- }
	- ]
	- },
	- {
	- "seq": 6920737,
	- "id": "@brightcove/videojs-flashls-source-handler",
	- "changes": [
	- {
	- "rev": "59-faf69c49be866b2ab7faa7be9972e7a5"
	- }
	- ]
	- },
	- {
	- "seq": 6920738,
	- "id": "@brightcove/videojs-flashls-swf",
	- "changes": [
	- {
	- "rev": "60-04908466eaac2194bc3061e91f463dab"
	- }
	- ]
	- },
	- {
	- "seq": 6920739,
	- "id": "@noqcks/generated",
	- "changes": [
	- {
	- "rev": "2-e07d07614182d4beccc507ca199e612d"
	- }
	- ]
	- },
	- {
	- "seq": 6920740,
	- "id": "pkcs7",
	- "changes": [
	- {
	- "rev": "60-65ba116f3b6b705f472971b5c6a8f8d2"
	- }
	- ]
	- },
	- {
	- "seq": 6920741,
	- "id": "videojs-errors",
	- "changes": [
	- {
	- "rev": "57-c999abd162ca4b93412e363443aa688a"
	- }
	- ]
	- },
	- {
	- "seq": 6920742,
	- "id": "videojs-flashls-source-handler",
	- "changes": [
	- {
	- "rev": "59-46d62e18971a8c800710a8fbf985c1c5"
	- }
	- ]
	- },
	- {
	- "seq": 6920743,
	- "id": "videojs-playlist",
	- "changes": [
	- {
	- "rev": "97-d4b3492a94c1084c272162dd51901188"
	- }
	- ]
	- },
	- {
	- "seq": 6920745,
	- "id": "videojs-playlist-ui",
	- "changes": [
	- {
	- "rev": "95-ba97c44c354b2262e639f8c515bed9bc"
	- }
	- ]
	- },
	- {
	- "seq": 6920746,
	- "id": "fusion-apollo-universal-client",
	- "changes": [
	- {
	- "rev": "25-7123042a477cec67c7d5fc702254c7a3"
	- }
	- ]
	- },
	- {
	- "seq": 6920749,
	- "id": "msg-fabric-core",
	- "changes": [
	- {
	- "rev": "20-17c33e06faca357526c7395aca1113d2"
	- }
	- ]
	- },
	- {
	- "seq": 6920750,
	- "id": "@expo/schemer",
	- "changes": [
	- {
	- "rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d"
	- }
	- ]
	- },
	- {
	- "seq": 6920752,
	- "id": "mathjs",
	- "changes": [
	- {
	- "rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18"
	- }
	- ]
	- },
	- {
	- "seq": 6920758,
	- "id": "statesauce-ui",
	- "changes": [
	- {
	- "rev": "6-db9a39366c1a082c56a2212e368e3ae2"
	- }
	- ]
	- },
	- {
	- "seq": 6920782,
	- "id": "@catchandrelease/arbor",
	- "changes": [
	- {
	- "rev": "19-925648432b398ecadc98993e6fba2353"
	- }
	- ]
	- },
	- {
	- "seq": 6920784,
	- "id": "discover-shared-ebsco-ui-core",
	- "changes": [
	- {
	- "rev": "4-277063cbc6b71f969e5f0db8c371db65"
	- }
	- ]
	- },
	- {
	- "seq": 6920807,
	- "id": "react-apexcharts",
	- "changes": [
	- {
	- "rev": "13-18505be8026a50390c1ff1ba522cb9bd"
	- }
	- ]
	- },
	- {
	- "seq": 6920819,
	- "id": "zigbee-shepherd-converters",
	- "changes": [
	- {
	- "rev": "90-5819692a5a9679ff8669fb410e190515"
	- }
	- ]
	- },
	- {
	- "seq": 6920835,
	- "id": "honeycomb-grid",
	- "changes": [
	- {
	- "rev": "36-edd6733c80b04a72600558dc55348c73"
	- }
	- ]
	- },
	- {
	- "seq": 6920838,
	- "id": "pixl-config",
	- "changes": [
	- {
	- "rev": "7-5dd2b68d04fefb4039b3965b3497eda2"
	- }
	- ]
	- },
	- {
	- "seq": 6920842,
	- "id": "discover-shared-ebsco-ui-theming",
	- "changes": [
	- {
	- "rev": "4-e9d083825b1eae46f28c4def2d0db79f"
	- }
	- ]
	- },
	- {
	- "seq": 6920843,
	- "id": "common-oxgalaxy-lengua-app",
	- "changes": [
	- {
	- "rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f"
	- }
	- ]
	- },
	- {
	- "seq": 6920845,
	- "id": "discover-shared-ebsco-ui-grid",
	- "changes": [
	- {
	- "rev": "2-6f71cf625a5232075071952b2adaa8f2"
	- }
	- ]
	- },
	- {
	- "seq": 6920847,
	- "id": "@auth0/cosmos-tokens",
	- "changes": [
	- {
	- "rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb"
	- }
	- ]
	- },
	- {
	- "seq": 6920848,
	- "id": "@auth0/babel-preset-cosmos",
	- "changes": [
	- {
	- "rev": "43-d05d3779db08f08726ba048da298e046"
	- }
	- ]
	- },
	- {
	- "seq": 6920849,
	- "id": "jsrender",
	- "changes": [
	- {
	- "rev": "11-c949091592b3329d73ae564e45a3472d"
	- }
	- ]
	- },
	- {
	- "seq": 6920850,
	- "id": "discover-shared-ebsco-ui-container",
	- "changes": [
	- {
	- "rev": "2-c32089f76b7f253bc0d765da8b9f670d"
	- }
	- ]
	- },
	- {
	- "seq": 6920852,
	- "id": "@auth0/cosmos",
	- "changes": [
	- {
	- "rev": "42-5fdaf3d9063c20dac13dcf455c42773c"
	- }
	- ]
	- },
	- {
	- "seq": 6920853,
	- "id": "discover-shared-ebsco-ui-checkbox",
	- "changes": [
	- {
	- "rev": "2-06d9521b86f0dbf4a398726faead1212"
	- }
	- ]
	- },
	- {
	- "seq": 6920854,
	- "id": "@adunigan/toggles",
	- "changes": [
	- {
	- "rev": "1-c2a830cf814a9fe2d72084339c9c5d28"
	- }
	- ]
	- },
	- {
	- "seq": 6920855,
	- "id": "@spriteful/spriteful-lazy-carousel",
	- "changes": [
	- {
	- "rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837"
	- }
	- ]
	- },
	- {
	- "seq": 6920856,
	- "id": "react-modal-hook",
	- "changes": [
	- {
	- "rev": "2-364b39d6559364c41d5b852ccad4ce31"
	- }
	- ],
	- "deleted": true
	- },
	- {
	- "seq": 6920859,
	- "id": "@bellese/angular-design-system",
	- "changes": [
	- {
	- "rev": "39-3e297f85ce2d6a6b6d15fc26420fc471"
	- }
	- ]
	- },
	- {
	- "seq": 6920861,
	- "id": "@uifabric/styling",
	- "changes": [
	- {
	- "rev": "229-addf6cc0e74a335125c04d60047353f5"
	- }
	- ]
	- },
	- {
	- "seq": 6920862,
	- "id": "@uifabric/file-type-icons",
	- "changes": [
	- {
	- "rev": "37-8a7e43399d1bb9f17334b10995f78df4"
	- }
	- ]
	- },
	- {
	- "seq": 6920864,
	- "id": "throttlewrap",
	- "changes": [
	- {
	- "rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa"
	- }
	- ]
	- },
	- {
	- "seq": 6920865,
	- "id": "airtable",
	- "changes": [
	- {
	- "rev": "16-d8aee935f6fa4c88057d75a0542bc58c"
	- }
	- ]
	- },
	- {
	- "seq": 6920866,
	- "id": "@csmart/ngc-smart-address",
	- "changes": [
	- {
	- "rev": "19-66a6ea868aae1912952f232d2c699f3a"
	- }
	- ]
	- },
	- {
	- "seq": 6920868,
	- "id": "office-ui-fabric-react",
	- "changes": [
	- {
	- "rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461"
	- }
	- ]
	- },
	- {
	- "seq": 6920869,
	- "id": "@fuelrats/eslint-config",
	- "changes": [
	- {
	- "rev": "12-1b4c71b78fd078e3c1cba535e8541bed"
	- }
	- ]
	- },
	- {
	- "seq": 6920870,
	- "id": "@uifabric/date-time",
	- "changes": [
	- {
	- "rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed"
	- }
	- ]
	- },
	- {
	- "seq": 6920872,
	- "id": "dark-client",
	- "changes": [
	- {
	- "rev": "11-a954c2a89a130ae73f064233d9b3bce2"
	- }
	- ]
	- },
	- {
	- "seq": 6920873,
	- "id": "@uifabric/variants",
	- "changes": [
	- {
	- "rev": "59-391c720194c663b9a5c59fe2c10a1535"
	- }
	- ]
	- },
	- {
	- "seq": 6920875,
	- "id": "discover-shared-ebsco-ui-header",
	- "changes": [
	- {
	- "rev": "2-efd8f0426a83422a6c8b7bff11054c72"
	- }
	- ]
	- },
	- {
	- "seq": 6920876,
	- "id": "react-responsive-picture",
	- "changes": [
	- {
	- "rev": "14-32a6d0850c8af33412cfdb23afd2ecfa"
	- }
	- ]
	- },
	- {
	- "seq": 6920877,
	- "id": "@uifabric/fluent-theme",
	- "changes": [
	- {
	- "rev": "16-39c29e00b81a0b654213a5a50d7e7f42"
	- }
	- ]
	- },
	- {
	- "seq": 6920878,
	- "id": "@uifabric/dashboard",
	- "changes": [
	- {
	- "rev": "82-04d6dc25b33e811c1d8c24566127b09c"
	- }
	- ]
	- },
	- {
	- "seq": 6920879,
	- "id": "ids-enterprise",
	- "changes": [
	- {
	- "rev": "201-dd709a3912f9832440320d448850b61a"
	- }
	- ]
	- },
	- {
	- "seq": 6920880,
	- "id": "@uifabric/experiments",
	- "changes": [
	- {
	- "rev": "224-efd1ef07f7640952c286488eae282367"
	- }
	- ]
	- },
	- {
	- "seq": 6920881,
	- "id": "@fuelrats/eslint-config-react",
	- "changes": [
	- {
	- "rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc"
	- }
	- ]
	- },
	- {
	- "seq": 6920883,
	- "id": "jsviews",
	- "changes": [
	- {
	- "rev": "11-44d8bedffc98cf6ac4aa669ba8844746"
	- }
	- ]
	- },
	- {
	- "seq": 6920885,
	- "id": "pixl-server",
	- "changes": [
	- {
	- "rev": "15-823f4598c3354500d8d2a266dd062502"
	- }
	- ]
	- },
	- {
	- "seq": 6920887,
	- "id": "@rrpm/netlify-cms-core",
	- "changes": [
	- {
	- "rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa"
	- }
	- ]
	- },
	- {
	- "seq": 6920889,
	- "id": "lodash-a",
	- "changes": [
	- {
	- "rev": "2-6ee66153dbe611a080b40775387d2d45"
	- }
	- ]
	- },
	- {
	- "seq": 6920891,
	- "id": "meshcentral",
	- "changes": [
	- {
	- "rev": "499-6677ca74525ed2aa77644c68001382fe"
	- }
	- ]
	- },
	- {
	- "seq": 6920892,
	- "id": "vue-transition-collection",
	- "changes": [
	- {
	- "rev": "2-0510ee52c014c0d3b1e65f24376d76f0"
	- }
	- ]
	- },
	- {
	- "seq": 6920894,
	- "id": "fury-adapter-swagger",
	- "changes": [
	- {
	- "rev": "47-09f0c55d8574d654c67f9244c21d7ef7"
	- }
	- ]
	- },
	- {
	- "seq": 6920895,
	- "id": "@isobar-us/redux-form-gen",
	- "changes": [
	- {
	- "rev": "30-70d7d9210264a321092c832063934648"
	- }
	- ]
	- },
	- {
	- "seq": 6920896,
	- "id": "atomizer",
	- "changes": [
	- {
	- "rev": "19-129774900cb2a67a46871cc2c40c34d3"
	- }
	- ]
	- },
	- {
	- "seq": 6920904,
	- "id": "boom-js-client",
	- "changes": [
	- {
	- "rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9"
	- }
	- ]
	- },
	- {
	- "seq": 6920905,
	- "id": "@ts-common/json-parser",
	- "changes": [
	- {
	- "rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3"
	- }
	- ]
	- },
	- {
	- "seq": 6920906,
	- "id": "rutt",
	- "changes": [
	- {
	- "rev": "13-78aab849cb00a6ef7ebc8165770b7d33"
	- }
	- ]
	- },
	- {
	- "seq": 6920907,
	- "id": "linear-react-components-ui",
	- "changes": [
	- {
	- "rev": "171-0307f1d69843b270e687371c67cbd1b0"
	- }
	- ]
	- },
	- {
	- "seq": 6920908,
	- "id": "@earnest/eslint-config",
	- "changes": [
	- {
	- "rev": "180-b5250dd803102cf7dbac8da9c1a403fd"
	- }
	- ]
	- },
	- {
	- "seq": 6920909,
	- "id": "@earnest/eslint-config-es7",
	- "changes": [
	- {
	- "rev": "181-da26885e0baacaea95814857f459572d"
	- }
	- ]
	- },
	- {
	- "seq": 6920910,
	- "id": "fuse-design",
	- "changes": [
	- {
	- "rev": "10-e2b78592872f680c05e55eb5b81a0cab"
	- }
	- ]
	- }
	- ],
	- "last_seq": 6920912
	-}
	\ No newline at end of file
	diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001"
	deleted file mode 100644
	--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001"
	+++ /dev/null
	@@ -1,807 +0,0 @@
	-{
	- "total_rows": 839080,
	- "offset": 422482,
	- "rows": [
	-
	- {
	- "id": "jquery",
	- "key": "jquery",
	- "value": {
	- "rev": "212-2eac7c93af4c8bccdf7317739f0319b6"
	- }
	- },
	-
	- {
	- "id": "jquery-1.8",
	- "key": "jquery-1.8",
	- "value": {
	- "rev": "1-711ded49a7453adce85ce7a51c2157de"
	- }
	- },
	-
	- {
	- "id": "jquery-1x",
	- "key": "jquery-1x",
	- "value": {
	- "rev": "1-c53fa04d9c8fb231336704508732c287"
	- }
	- },
	-
	- {
	- "id": "jquery-2-typescript-async-await-adapter",
	- "key": "jquery-2-typescript-async-await-adapter",
	- "value": {
	- "rev": "8-5cfb484e9afaa6e326a97240fccd8f93"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-accordion-aria",
	- "key": "jquery-accessible-accordion-aria",
	- "value": {
	- "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-autocomplete-list-aria",
	- "key": "jquery-accessible-autocomplete-list-aria",
	- "value": {
	- "rev": "8-961b382442c1a5bafe58f0e05424701d"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-carrousel-aria",
	- "key": "jquery-accessible-carrousel-aria",
	- "value": {
	- "rev": "9-f33f59d7f601bafe023bd711b551282b"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-dialog-tooltip-aria",
	- "key": "jquery-accessible-dialog-tooltip-aria",
	- "value": {
	- "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-hide-show-aria",
	- "key": "jquery-accessible-hide-show-aria",
	- "value": {
	- "rev": "10-5a03c47a8995b08246e4bc103782dafa"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-modal-window-aria",
	- "key": "jquery-accessible-modal-window-aria",
	- "value": {
	- "rev": "18-50266e260f6b807019cfcfcd3a3685ab"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-simple-tooltip-aria",
	- "key": "jquery-accessible-simple-tooltip-aria",
	- "value": {
	- "rev": "6-ea71aa37760790dc603b56117f054e1b"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-subnav-dropdown",
	- "key": "jquery-accessible-subnav-dropdown",
	- "value": {
	- "rev": "2-496f017a9ac243655225e43b5697b09b"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-tabpanel-aria",
	- "key": "jquery-accessible-tabpanel-aria",
	- "value": {
	- "rev": "11-659971471e6ac0fbb3b2f78ad208722a"
	- }
	- },
	-
	- {
	- "id": "jquery-accessible-tabs-umd",
	- "key": "jquery-accessible-tabs-umd",
	- "value": {
	- "rev": "1-f92015de5bb36e411d8c0940cca2883f"
	- }
	- },
	-
	- {
	- "id": "jquery-active-descendant",
	- "key": "jquery-active-descendant",
	- "value": {
	- "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de"
	- }
	- },
	-
	- {
	- "id": "jquery-ada-validation",
	- "key": "jquery-ada-validation",
	- "value": {
	- "rev": "1-9aab9629027c29fbece90485dd9d3112"
	- }
	- },
	-
	- {
	- "id": "jquery-adaptText",
	- "key": "jquery-adaptText",
	- "value": {
	- "rev": "3-2e15fc801ea8235b9180a3defc782ed0"
	- }
	- },
	-
	- {
	- "id": "jquery-adapttr",
	- "key": "jquery-adapttr",
	- "value": {
	- "rev": "6-74585f2d4be60b3f493585a6d28b90bc"
	- }
	- },
	-
	- {
	- "id": "jquery-add-prefixed-class",
	- "key": "jquery-add-prefixed-class",
	- "value": {
	- "rev": "1-9e43aee9758504b3f5271e9804a95f20"
	- }
	- },
	-
	- {
	- "id": "jquery-address",
	- "key": "jquery-address",
	- "value": {
	- "rev": "1-64173ede32157b26f4de910ad0f49590"
	- }
	- },
	-
	- {
	- "id": "jquery-address-suggestion",
	- "key": "jquery-address-suggestion",
	- "value": {
	- "rev": "6-18d9df51d472c365bcd84a61c9105774"
	- }
	- },
	-
	- {
	- "id": "jquery-advscrollevent",
	- "key": "jquery-advscrollevent",
	- "value": {
	- "rev": "1-f6033de9ba0f8e364c42826441d93119"
	- }
	- },
	-
	- {
	- "id": "jquery-affix",
	- "key": "jquery-affix",
	- "value": {
	- "rev": "6-777371f67df59abf18ec1fe326df3b82"
	- }
	- },
	-
	- {
	- "id": "jquery-airload",
	- "key": "jquery-airload",
	- "value": {
	- "rev": "7-136d513d2604a25238eb88709d6d9003"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax",
	- "key": "jquery-ajax",
	- "value": {
	- "rev": "1-ee358f630d4c928b52c968c7667d0d31"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-cache",
	- "key": "jquery-ajax-cache",
	- "value": {
	- "rev": "2-ca31e0d43ae28e9cea968f1f538f06d3"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-chain",
	- "key": "jquery-ajax-chain",
	- "value": {
	- "rev": "1-dc0e5aee651c0128b7f411aac96132a2"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-file-upload",
	- "key": "jquery-ajax-file-upload",
	- "value": {
	- "rev": "1-96147d8bf69245c622e76583bb615d49"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-json",
	- "key": "jquery-ajax-json",
	- "value": {
	- "rev": "1-b47eec12168e4cb39b45f1523d7cd397"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-markup",
	- "key": "jquery-ajax-markup",
	- "value": {
	- "rev": "1-8e65dc822cb63be76c62a1323666265e"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-native",
	- "key": "jquery-ajax-native",
	- "value": {
	- "rev": "2-9d67b8d43713e3546ad50f817c040139"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-request",
	- "key": "jquery-ajax-request",
	- "value": {
	- "rev": "1-fdc0960ec73667bc2b46adf493c05db4"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-retry",
	- "key": "jquery-ajax-retry",
	- "value": {
	- "rev": "1-27ca186953e346aa9c0ca2310c732751"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-tracking",
	- "key": "jquery-ajax-tracking",
	- "value": {
	- "rev": "3-d48876f3c115ee4743a6a94bb65bb01d"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-transport-xdomainrequest",
	- "key": "jquery-ajax-transport-xdomainrequest",
	- "value": {
	- "rev": "1-ece69aa5b9f0c950a1fa2806cf74392d"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-unobtrusive",
	- "key": "jquery-ajax-unobtrusive",
	- "value": {
	- "rev": "3-fb0daab8480b9a2cc9c6876e1c4874f4"
	- }
	- },
	-
	- {
	- "id": "jquery-ajax-unobtrusive-multi",
	- "key": "jquery-ajax-unobtrusive-multi",
	- "value": {
	- "rev": "1-0a2ffdabaf5708d4ae3d9e29a3a9ef11"
	- }
	- },
	-
	- {
	- "id": "jquery-ajaxreadystate",
	- "key": "jquery-ajaxreadystate",
	- "value": {
	- "rev": "1-5e618474fe2e77ad5869c206164f82bf"
	- }
	- },
	-
	- {
	- "id": "jquery-albe-timeline",
	- "key": "jquery-albe-timeline",
	- "value": {
	- "rev": "2-3db2b43778b5c50db873e724d9940eb6"
	- }
	- },
	-
	- {
	- "id": "jquery-all-attributes",
	- "key": "jquery-all-attributes",
	- "value": {
	- "rev": "1-89bb7e01ee312ad5d36d78a3aa2327e4"
	- }
	- },
	-
	- {
	- "id": "jquery-alphaindex",
	- "key": "jquery-alphaindex",
	- "value": {
	- "rev": "4-7f61cde9cfb70617a6fbe992dfcbc10a"
	- }
	- },
	-
	- {
	- "id": "jquery-always",
	- "key": "jquery-always",
	- "value": {
	- "rev": "1-0ad944881bbc39c67df0a694d80bebef"
	- }
	- },
	-
	- {
	- "id": "jquery-amd",
	- "key": "jquery-amd",
	- "value": {
	- "rev": "1-931646c751bef740c361dd0f6e68653c"
	- }
	- },
	-
	- {
	- "id": "jquery-anaglyph-image-effect",
	- "key": "jquery-anaglyph-image-effect",
	- "value": {
	- "rev": "1-9bf7afce2e1bc73747ef22abc859b22b"
	- }
	- },
	-
	- {
	- "id": "jquery-analytics",
	- "key": "jquery-analytics",
	- "value": {
	- "rev": "1-d84b0c8ce886b9f01d2c5c1cf0a7317f"
	- }
	- },
	-
	- {
	- "id": "jquery-ancestors",
	- "key": "jquery-ancestors",
	- "value": {
	- "rev": "1-49b30817a03558f1f585c8c0cd4b8afb"
	- }
	- },
	-
	- {
	- "id": "jquery-angry-loader",
	- "key": "jquery-angry-loader",
	- "value": {
	- "rev": "1-31c9fd950d32b9d3a73829cde1dae577"
	- }
	- },
	-
	- {
	- "id": "jquery-angular-shim",
	- "key": "jquery-angular-shim",
	- "value": {
	- "rev": "1-723e72b2981f02dd3abcfe6d2395d636"
	- }
	- },
	-
	- {
	- "id": "jquery-animate-gradient",
	- "key": "jquery-animate-gradient",
	- "value": {
	- "rev": "5-a3e0fc89699237e7e7241cd608a0dcf7"
	- }
	- },
	-
	- {
	- "id": "jquery-animate-scroll",
	- "key": "jquery-animate-scroll",
	- "value": {
	- "rev": "1-37d49d89fe99aa599540e6ff83b15888"
	- }
	- },
	-
	- {
	- "id": "jquery-animated-headlines",
	- "key": "jquery-animated-headlines",
	- "value": {
	- "rev": "1-adf1d149bc83fa8445e141e3c900759e"
	- }
	- },
	-
	- {
	- "id": "jquery-animation",
	- "key": "jquery-animation",
	- "value": {
	- "rev": "4-f51d0559010bbe9d74d70e58de9bd733"
	- }
	- },
	-
	- {
	- "id": "jquery-animation-support",
	- "key": "jquery-animation-support",
	- "value": {
	- "rev": "1-9013bc4bdeb2bd70bedcc988a811fcc0"
	- }
	- },
	-
	- {
	- "id": "jquery-aniview",
	- "key": "jquery-aniview",
	- "value": {
	- "rev": "3-5754524da237693458bcff19b626b875"
	- }
	- },
	-
	- {
	- "id": "jquery-anything-clickable",
	- "key": "jquery-anything-clickable",
	- "value": {
	- "rev": "2-e1aaaf1a369f7796c438a3efbf05bcce"
	- }
	- },
	-
	- {
	- "id": "jquery-app",
	- "key": "jquery-app",
	- "value": {
	- "rev": "6-4e0bf5abd71c72ced3c4cf3035116f70"
	- }
	- },
	-
	- {
	- "id": "jquery-app-banner",
	- "key": "jquery-app-banner",
	- "value": {
	- "rev": "2-8a5b530eaab94315eb00c77acd13f2dd"
	- }
	- },
	-
	- {
	- "id": "jquery-appear-poetic",
	- "key": "jquery-appear-poetic",
	- "value": {
	- "rev": "1-368094b72ed36d42cf2fca438fa4b344"
	- }
	- },
	-
	- {
	- "id": "jquery-applyonscreen",
	- "key": "jquery-applyonscreen",
	- "value": {
	- "rev": "4-d76c18a6e66fffba01a9a774b40663f8"
	- }
	- },
	-
	- {
	- "id": "jquery-apta",
	- "key": "jquery-apta",
	- "value": {
	- "rev": "1-c486380fedefd887e6293a00c3b6a222"
	- }
	- },
	-
	- {
	- "id": "jquery-arrow-navigate",
	- "key": "jquery-arrow-navigate",
	- "value": {
	- "rev": "3-0efe881e01ef0eac24a92baf1eb6d8d1"
	- }
	- },
	-
	- {
	- "id": "jquery-asAccordion",
	- "key": "jquery-asAccordion",
	- "value": {
	- "rev": "2-2d18d3fe9089dcf67de5f29d1763b4ce"
	- }
	- },
	-
	- {
	- "id": "jquery-asBgPicker",
	- "key": "jquery-asBgPicker",
	- "value": {
	- "rev": "2-d1403cd306d5764ee0f5aa852c2bed8e"
	- }
	- },
	-
	- {
	- "id": "jquery-asBreadcrumbs",
	- "key": "jquery-asBreadcrumbs",
	- "value": {
	- "rev": "2-77e566a07680005ce1cb322f2a733fe4"
	- }
	- },
	-
	- {
	- "id": "jquery-asCheck",
	- "key": "jquery-asCheck",
	- "value": {
	- "rev": "2-d0b2741b70616c7d563419cc125d193d"
	- }
	- },
	-
	- {
	- "id": "jquery-asChoice",
	- "key": "jquery-asChoice",
	- "value": {
	- "rev": "2-0eda5269cbd59976ee904b74da209389"
	- }
	- },
	-
	- {
	- "id": "jquery-asColor",
	- "key": "jquery-asColor",
	- "value": {
	- "rev": "3-aa730d81322561c7a3174d5c7bb6b3b8"
	- }
	- },
	-
	- {
	- "id": "jquery-asColorPicker",
	- "key": "jquery-asColorPicker",
	- "value": {
	- "rev": "2-6bbaecaf94a324331a3d1f5d3aad3b3d"
	- }
	- },
	-
	- {
	- "id": "jquery-asDropdown",
	- "key": "jquery-asDropdown",
	- "value": {
	- "rev": "2-b29b187cdd0bdce502d11855415e6887"
	- }
	- },
	-
	- {
	- "id": "jquery-asFontEditor",
	- "key": "jquery-asFontEditor",
	- "value": {
	- "rev": "2-132882375101062896413afdc93b4c8c"
	- }
	- },
	-
	- {
	- "id": "jquery-asGalleryPicker",
	- "key": "jquery-asGalleryPicker",
	- "value": {
	- "rev": "1-864a80930d72c6150aa74969a28617e4"
	- }
	- },
	-
	- {
	- "id": "jquery-asGmap",
	- "key": "jquery-asGmap",
	- "value": {
	- "rev": "2-b0c4330774137b2f1b91bd4686880f2a"
	- }
	- },
	-
	- {
	- "id": "jquery-asGradient",
	- "key": "jquery-asGradient",
	- "value": {
	- "rev": "2-5184670a313d5e161cb62659de3db55c"
	- }
	- },
	-
	- {
	- "id": "jquery-asHoverScroll",
	- "key": "jquery-asHoverScroll",
	- "value": {
	- "rev": "7-3f6efebf248bd27520d03eaac33d8ca2"
	- }
	- },
	-
	- {
	- "id": "jquery-asIconPicker",
	- "key": "jquery-asIconPicker",
	- "value": {
	- "rev": "2-9070adda148ea75247c7cee810ae91e2"
	- }
	- },
	-
	- {
	- "id": "jquery-asImagePicker",
	- "key": "jquery-asImagePicker",
	- "value": {
	- "rev": "2-fb3115c2296b0b07ed9e379176626e01"
	- }
	- },
	-
	- {
	- "id": "jquery-asItemList",
	- "key": "jquery-asItemList",
	- "value": {
	- "rev": "2-88a7d2900f47c785c2a6cb764ac467d6"
	- }
	- },
	-
	- {
	- "id": "jquery-asModal",
	- "key": "jquery-asModal",
	- "value": {
	- "rev": "2-1719b8e6a489e03cc3e22bd329148366"
	- }
	- },
	-
	- {
	- "id": "jquery-asOffset",
	- "key": "jquery-asOffset",
	- "value": {
	- "rev": "2-e45a0077e5bc0bbf91b32dc76387c945"
	- }
	- },
	-
	- {
	- "id": "jquery-asPaginator",
	- "key": "jquery-asPaginator",
	- "value": {
	- "rev": "2-0d279d2748fc5e875f5fb2a8d3d48377"
	- }
	- },
	-
	- {
	- "id": "jquery-asPieProgress",
	- "key": "jquery-asPieProgress",
	- "value": {
	- "rev": "2-14dc464a19e9d3feaa532f62e45bbd26"
	- }
	- },
	-
	- {
	- "id": "jquery-asProgress",
	- "key": "jquery-asProgress",
	- "value": {
	- "rev": "2-a58d7100f1a78f7753efcf0e34dfaf0e"
	- }
	- },
	-
	- {
	- "id": "jquery-asRange",
	- "key": "jquery-asRange",
	- "value": {
	- "rev": "3-aa3d2f348a933161868ba6b6fd9eb881"
	- }
	- },
	-
	- {
	- "id": "jquery-asScroll",
	- "key": "jquery-asScroll",
	- "value": {
	- "rev": "1-f4880ea057adbfebb912ba0157575ca1"
	- }
	- },
	-
	- {
	- "id": "jquery-asScrollable",
	- "key": "jquery-asScrollable",
	- "value": {
	- "rev": "7-5c18eb2180d8aa85f0b5e940667c8344"
	- }
	- },
	-
	- {
	- "id": "jquery-asScrollbar",
	- "key": "jquery-asScrollbar",
	- "value": {
	- "rev": "4-89420658c355a5584825b45ee4ef0beb"
	- }
	- },
	-
	- {
	- "id": "jquery-asSelect",
	- "key": "jquery-asSelect",
	- "value": {
	- "rev": "2-caf3dc516665009b654236b876fe02bb"
	- }
	- },
	-
	- {
	- "id": "jquery-asSpinner",
	- "key": "jquery-asSpinner",
	- "value": {
	- "rev": "2-bf26b5d9c77eb4b63acbf16019407834"
	- }
	- },
	-
	- {
	- "id": "jquery-asSwitch",
	- "key": "jquery-asSwitch",
	- "value": {
	- "rev": "2-f738586946b432caa73297568b5f38ad"
	- }
	- },
	-
	- {
	- "id": "jquery-asTooltip",
	- "key": "jquery-asTooltip",
	- "value": {
	- "rev": "2-80d3fe5cdae70d9310969723e7045384"
	- }
	- },
	-
	- {
	- "id": "jquery-asTree",
	- "key": "jquery-asTree",
	- "value": {
	- "rev": "2-353063a563c0322cbc317af385f71b27"
	- }
	- },
	-
	- {
	- "id": "jquery-ascolorpicker-flat",
	- "key": "jquery-ascolorpicker-flat",
	- "value": {
	- "rev": "11-1681d53cd475e7b6b9564baa51a79611"
	- }
	- },
	-
	- {
	- "id": "jquery-aslider",
	- "key": "jquery-aslider",
	- "value": {
	- "rev": "1-2b3dd953493eeaa4dc329cbf0d81116a"
	- }
	- },
	-
	- {
	- "id": "jquery-aspect-ratio-keeper",
	- "key": "jquery-aspect-ratio-keeper",
	- "value": {
	- "rev": "1-1ad8e5588218e1d38fff351858655eda"
	- }
	- },
	-
	- {
	- "id": "jquery-assinadordigitaldiscus",
	- "key": "jquery-assinadordigitaldiscus",
	- "value": {
	- "rev": "1-897cd68ef3699551630bd3454dceb6f0"
	- }
	- },
	-
	- {
	- "id": "jquery-async-gravatar",
	- "key": "jquery-async-gravatar",
	- "value": {
	- "rev": "3-a3192e741d14d57635f4ebfb41a904db"
	- }
	- },
	-
	- {
	- "id": "jquery-asynclink",
	- "key": "jquery-asynclink",
	- "value": {
	- "rev": "1-2159a3c49e3c8fe9280c592770e83522"
	- }
	- },
	-
	- {
	- "id": "jquery-atlas",
	- "key": "jquery-atlas",
	- "value": {
	- "rev": "1-6142c5a0af67a0470daf36151d3f9d8c"
	- }
	- },
	-
	- {
	- "id": "jquery-atomic-nav",
	- "key": "jquery-atomic-nav",
	- "value": {
	- "rev": "1-18e4ef14be83a907cbee0cd0adee25d4"
	- }
	- },
	-
	- {
	- "id": "jquery-attach",
	- "key": "jquery-attach",
	- "value": {
	- "rev": "8-da4f17596c25a02b0cce266e59706d5f"
	- }
	- }
	-
	- ]
	-}
	diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11"
	deleted file mode 100644
	--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11"
	+++ /dev/null
	@@ -1,83 +0,0 @@
	-{
	- "total_rows": 839080,
	- "offset": 422482,
	- "rows": [
	- {
	- "id": "jquery",
	- "key": "jquery",
	- "value": {
	- "rev": "212-2eac7c93af4c8bccdf7317739f0319b6"
	- }
	- },
	- {
	- "id": "jquery-1.8",
	- "key": "jquery-1.8",
	- "value": {
	- "rev": "1-711ded49a7453adce85ce7a51c2157de"
	- }
	- },
	- {
	- "id": "jquery-1x",
	- "key": "jquery-1x",
	- "value": {
	- "rev": "1-c53fa04d9c8fb231336704508732c287"
	- }
	- },
	- {
	- "id": "jquery-2-typescript-async-await-adapter",
	- "key": "jquery-2-typescript-async-await-adapter",
	- "value": {
	- "rev": "8-5cfb484e9afaa6e326a97240fccd8f93"
	- }
	- },
	- {
	- "id": "jquery-accessible-accordion-aria",
	- "key": "jquery-accessible-accordion-aria",
	- "value": {
	- "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e"
	- }
	- },
	- {
	- "id": "jquery-accessible-autocomplete-list-aria",
	- "key": "jquery-accessible-autocomplete-list-aria",
	- "value": {
	- "rev": "8-961b382442c1a5bafe58f0e05424701d"
	- }
	- },
	- {
	- "id": "jquery-accessible-carrousel-aria",
	- "key": "jquery-accessible-carrousel-aria",
	- "value": {
	- "rev": "9-f33f59d7f601bafe023bd711b551282b"
	- }
	- },
	- {
	- "id": "jquery-accessible-dialog-tooltip-aria",
	- "key": "jquery-accessible-dialog-tooltip-aria",
	- "value": {
	- "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba"
	- }
	- },
	- {
	- "id": "jquery-accessible-hide-show-aria",
	- "key": "jquery-accessible-hide-show-aria",
	- "value": {
	- "rev": "10-5a03c47a8995b08246e4bc103782dafa"
	- }
	- },
	- {
	- "id": "jquery-accessible-modal-window-aria",
	- "key": "jquery-accessible-modal-window-aria",
	- "value": {
	- "rev": "18-50266e260f6b807019cfcfcd3a3685ab"
	- }
	- },
	- {
	- "id": "jquery-accessible-simple-tooltip-aria",
	- "key": "jquery-accessible-simple-tooltip-aria",
	- "value": {
	- "rev": "6-ea71aa37760790dc603b56117f054e1b"
	- }
	- }
	- ]
	-}
	diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11"
	deleted file mode 100644
	--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11"
	+++ /dev/null
	@@ -1,83 +0,0 @@
	-{
	- "total_rows": 839080,
	- "offset": 422482,
	- "rows": [
	- {
	- "id": "jquery-accessible-simple-tooltip-aria",
	- "key": "jquery-accessible-simple-tooltip-aria",
	- "value": {
	- "rev": "6-ea71aa37760790dc603b56117f054e1b"
	- }
	- },
	- {
	- "id": "jquery-accessible-subnav-dropdown",
	- "key": "jquery-accessible-subnav-dropdown",
	- "value": {
	- "rev": "2-496f017a9ac243655225e43b5697b09b"
	- }
	- },
	- {
	- "id": "jquery-accessible-tabpanel-aria",
	- "key": "jquery-accessible-tabpanel-aria",
	- "value": {
	- "rev": "11-659971471e6ac0fbb3b2f78ad208722a"
	- }
	- },
	- {
	- "id": "jquery-accessible-tabs-umd",
	- "key": "jquery-accessible-tabs-umd",
	- "value": {
	- "rev": "1-f92015de5bb36e411d8c0940cca2883f"
	- }
	- },
	- {
	- "id": "jquery-active-descendant",
	- "key": "jquery-active-descendant",
	- "value": {
	- "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de"
	- }
	- },
	- {
	- "id": "jquery-ada-validation",
	- "key": "jquery-ada-validation",
	- "value": {
	- "rev": "1-9aab9629027c29fbece90485dd9d3112"
	- }
	- },
	- {
	- "id": "jquery-adaptText",
	- "key": "jquery-adaptText",
	- "value": {
	- "rev": "3-2e15fc801ea8235b9180a3defc782ed0"
	- }
	- },
	- {
	- "id": "jquery-adapttr",
	- "key": "jquery-adapttr",
	- "value": {
	- "rev": "6-74585f2d4be60b3f493585a6d28b90bc"
	- }
	- },
	- {
	- "id": "jquery-add-prefixed-class",
	- "key": "jquery-add-prefixed-class",
	- "value": {
	- "rev": "1-9e43aee9758504b3f5271e9804a95f20"
	- }
	- },
	- {
	- "id": "jquery-address",
	- "key": "jquery-address",
	- "value": {
	- "rev": "1-64173ede32157b26f4de910ad0f49590"
	- }
	- },
	- {
	- "id": "jquery-address-suggestion",
	- "key": "jquery-address-suggestion",
	- "value": {
	- "rev": "6-18d9df51d472c365bcd84a61c9105774"
	- }
	- }
	- ]
	-}
	diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json
	deleted file mode 120000
	--- a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json
	+++ /dev/null
	@@ -1 +0,0 @@
	-_all_docs,startkey="",limit=1001
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/data/npm_full_page1.json b/swh/lister/npm/tests/data/npm_full_page1.json
	new file mode 100644
	--- /dev/null
	+++ b/swh/lister/npm/tests/data/npm_full_page1.json
	@@ -0,0 +1,235 @@
	+{
	+ "total_rows": 1496481,
	+ "offset": 0,
	+ "rows": [{
	+ "id": "-",
	+ "key": "-",
	+ "value": {
	+ "rev": "1-e340e64b090ea8c6b4e14edc0460c751"
	+ },
	+ "doc": {
	+ "_id": "-",
	+ "_rev": "1-e340e64b090ea8c6b4e14edc0460c751",
	+ "name": "-",
	+ "dist-tags": {
	+ "latest": "0.0.1"
	+ },
	+ "versions": {
	+ "0.0.1": {
	+ "name": "-",
	+ "version": "0.0.1",
	+ "license": "UNLICENSED",
	+ "keywords": [],
	+ "main": "dist/index.js",
	+ "types": "dist/index.d.ts",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1",
	+ "start": "node --require ts-node/register src",
	+ "build": "tsc"
	+ },
	+ "devDependencies": {
	+ "@types/node": "13.9.0",
	+ "ts-node": "8.6.2",
	+ "typescript": "3.8.3"
	+ },
	+ "description": "> Created using https://github.com/parzh/create-package-typescript",
	+ "_id": "-@0.0.1",
	+ "_nodeVersion": "13.3.0",
	+ "_npmVersion": "6.14.4",
	+ "dist": {
	+ "integrity": "sha512-3HfneK3DGAm05fpyj20sT3apkNcvPpCuccOThOPdzz8sY7GgQGe0l93XH9bt+YzibcTIgUAIMoyVJI740RtgyQ==",
	+ "shasum": "db6db7cd866142880dd03e5b8781d1b4fac0e5bd",
	+ "tarball": "https://registry.npmjs.org/-/-/--0.0.1.tgz",
	+ "fileCount": 3,
	+ "unpackedSize": 600,
	+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJehdEkCRA9TVsSAnZWagAAqcEQAIUCCWB2RzNV2p5CTeWf\nntknmFRbGh3K/gptyBnTC3MaNtk54Vf36XK1BfKgKz2rd/FOq1xcSQ8XUWt6\n7ZL9zSR3K/EBEgNr6p00fJAJ7ZdYu2kqq+uiZDKpSHLd6/bKaSdt0bAL78co\nmnzKOn9WULS1UC/ac8txNX9QdPg/aJezzxQadtKQR+SX6Fd/tTHKIsWv3ytI\nUtH/Lg9AM/tdTc94m2Ivo3+JxmkU+p8z7hjR97ym+AaZE8g1a47begVsSeZ/\nrIu1ZFCQ/XYFp0amNZ131qnW6FGnzpPlQTfkuJ1sPIORM9+o+6Ql6qa0mtde\nSJyAMKFfemmwkp56aZkL5NFX1snyJE4itfg99gsUrogHZIW3Jd1LqUe9yFZc\nVik1NZxqG4x6ktoog9IzG8ElOBWauKQAT+FIgNGea6IBv4wPQ8VoNOd4JKW/\nadvzus0hAdEZLyVsCyDw4wDCAK9JJXnSlJmJFc/vPtgPrbAj8xcBW6XIn6Nw\n0EVm993zlKmQ6/VmQUbHx8EP1zBm7M0vTcoI8dPIJkjGCZgWU571JARCetb2\npob+siYon3ndpoHG2OOWOuoRhwi4YeDW1ilc9isPve+S/iygKnfY8LTqHN1P\nOntz9Cu9YxHQ3WEiBuun7KP7FECbiSkozmzKWLHUhs2qXFSFYWeBLEI/HS1y\nJ9QO\r\n=lB+Q\r\n-----END PGP SIGNATURE-----\r\n"
	+ },
	+ "maintainers": [{
	+ "name": "parzhitsky",
	+ "email": "parzhitsky@gmail.com"
	+ }],
	+ "_npmUser": {
	+ "name": "parzhitsky",
	+ "email": "parzhitsky@gmail.com"
	+ },
	+ "directories": {},
	+ "_npmOperationalInternal": {
	+ "host": "s3://npm-registry-packages",
	+ "tmp": "tmp/-_0.0.1_1585828132159_0.6185838126718755"
	+ },
	+ "_hasShrinkwrap": false
	+ }
	+ },
	+ "time": {
	+ "created": "2020-04-02T11:48:52.159Z",
	+ "0.0.1": "2020-04-02T11:48:52.339Z",
	+ "modified": "2020-04-02T11:49:00.248Z"
	+ },
	+ "maintainers": [{
	+ "name": "parzhitsky",
	+ "email": "parzhitsky@gmail.com"
	+ }],
	+ "description": "> Created using https://github.com/parzh/create-package-typescript",
	+ "keywords": [],
	+ "license": "UNLICENSED",
	+ "readme": "# `-`\r\n\r\n> Created using https://github.com/parzh/create-package-typescript\r\n\r\n```\r\nnpm i -\r\n```\r\n",
	+ "readmeFilename": "README.md"
	+ }
	+ },
	+ {
	+ "id": "--file--1",
	+ "key": "--file--1",
	+ "value": {
	+ "rev": "1-b23c0a584f925336378c29df0b858aff"
	+ },
	+ "doc": {
	+ "_id": "--file--1",
	+ "_rev": "1-b23c0a584f925336378c29df0b858aff",
	+ "name": "--file--1",
	+ "dist-tags": {
	+ "latest": "1.0.0"
	+ },
	+ "versions": {
	+ "1.0.0": {
	+ "name": "--file--1",
	+ "version": "1.0.0",
	+ "description": "",
	+ "main": "file.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "keywords": [],
	+ "author": "",
	+ "license": "ISC",
	+ "_id": "--file--1@1.0.0",
	+ "_nodeVersion": "15.3.0",
	+ "_npmVersion": "7.0.14",
	+ "dist": {
	+ "integrity": "sha512-OtGGxxqxDl1KCCUts3fN0gL4vW28u8WNIo+eqQON78IVxKJS3gYchaEZDrRubC4SyQOhDDKDtvswwLOkApuicQ==",
	+ "shasum": "58ed9d9169197ded653d8df7153968d22c14ea59",
	+ "tarball": "https://registry.npmjs.org/--file--1/-/--file--1-1.0.0.tgz",
	+ "fileCount": 6,
	+ "unpackedSize": 1327,
	+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf2AGJCRA9TVsSAnZWagAALqcP+wZ9tzFsdQy4cEb5HGV6\nxV0LTnW+wxFYBUm2IfOwK7UH/Pl5Nym4yxlIjMEBinb9efepUAsucCyAka/v\n0OBGn+8I8kHDKvfhxiUVbi3CFAYcQTaaPffK5xJZi77netdsaLyS0cdx4RBm\nEWRIVK4axJBSUT6UVpmRqB73OlAv+JmNzBmL+CXHn9gksyvnm74pbT8tukLV\nX0TwkXNBtzn4lmydPRyecEZRDFydzp9UBQJyJACtpYxgTAwTT5ENtSymx4Ql\nlEqC9cqT9Q3n7z8lvntaCb6jgNN1aSWP3hOKCYQzezk5uSacJDWJOGAK7DE5\ndKUqsWa0HG84H+MMLxuepbrNYM5Fm7ulykl1yMrAf1G8T5pU/fUpq5uEHbqW\naVbW3GlCxJuUvlaTTFjGqDJx4tepY+Uel+TQigJoN2DGXTWiB+JeAAWLP+cu\nqcoZwkqoISmgru2g1+6TR//hod3CnCK4eQl/ph3VthtKyh4d1zUQAPcCGax6\nocJSWQsLRNW0QUmvpHeHQQOUGEb0XDTYkl9i8s8DY6JCqfgl9UIDZzNl2nK8\n385JseJKKLQR3tEk+HLZM2UGF6XgACBFTHYxlOrpzGgX+PJ1ns3noMrKo2jO\nbnvMN6Mp0GtoSzQj8FUcoU6BgOFHhHkB/i4Ke2VVeMYVj/CrFO13pZJOPR4O\n6uts\r\n=W7vP\r\n-----END PGP SIGNATURE-----\r\n"
	+ },
	+ "_npmUser": {
	+ "name": "guosuzhen",
	+ "email": "1077655229@qq.com"
	+ },
	+ "directories": {},
	+ "maintainers": [{
	+ "name": "guosuzhen",
	+ "email": "1077655229@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "s3://npm-registry-packages",
	+ "tmp": "tmp/--file--1_1.0.0_1607991689533_0.6331916469772794"
	+ },
	+ "_hasShrinkwrap": false
	+ }
	+ },
	+ "time": {
	+ "created": "2020-12-15T00:21:29.533Z",
	+ "1.0.0": "2020-12-15T00:21:29.657Z",
	+ "modified": "2020-12-15T00:21:32.116Z"
	+ },
	+ "maintainers": [{
	+ "name": "guosuzhen",
	+ "email": "1077655229@qq.com"
	+ }],
	+ "keywords": [],
	+ "license": "ISC",
	+ "readme": "ERROR: No README data found!",
	+ "readmeFilename": ""
	+ }
	+ },
	+ {
	+ "id": "-keyboardevent",
	+ "key": "-keyboardevent",
	+ "value": {
	+ "rev": "1-0fdff278618c4f8c1026889f1b525483"
	+ },
	+ "doc": {
	+ "_id": "-keyboardevent",
	+ "_rev": "1-0fdff278618c4f8c1026889f1b525483",
	+ "name": "-keyboardevent",
	+ "dist-tags": {
	+ "latest": "1.0.0"
	+ },
	+ "versions": {
	+ "1.0.0": {
	+ "name": "-keyboardevent",
	+ "version": "1.0.0",
	+ "description": "监听键盘事件",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
	+ },
	+ "author": {
	+ "name": "yanggw"
	+ },
	+ "license": "ISO",
	+ "bugs": {
	+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
	+ },
	+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
	+ "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3",
	+ "_id": "-keyboardevent@1.0.0",
	+ "_nodeVersion": "12.10.0",
	+ "_npmVersion": "6.10.3",
	+ "dist": {
	+ "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==",
	+ "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5",
	+ "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz",
	+ "fileCount": 3,
	+ "unpackedSize": 2714,
	+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n"
	+ },
	+ "maintainers": [{
	+ "name": "yanggw",
	+ "email": "2993065178@qq.com"
	+ }],
	+ "_npmUser": {
	+ "name": "yanggw",
	+ "email": "2993065178@qq.com"
	+ },
	+ "directories": {},
	+ "_npmOperationalInternal": {
	+ "host": "s3://npm-registry-packages",
	+ "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927"
	+ },
	+ "_hasShrinkwrap": false
	+ }
	+ },
	+ "time": {
	+ "created": "2020-09-25T06:10:56.357Z",
	+ "1.0.0": "2020-09-25T06:10:56.513Z",
	+ "modified": "2020-09-25T06:10:58.774Z"
	+ },
	+ "maintainers": [{
	+ "name": "yanggw",
	+ "email": "2993065178@qq.com"
	+ }],
	+ "description": "监听键盘事件",
	+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
	+ },
	+ "author": {
	+ "name": "yanggw"
	+ },
	+ "bugs": {
	+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
	+ },
	+ "license": "ISO",
	+ "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n",
	+ "readmeFilename": "README.md"
	+ }
	+ }
	+ ]
	+}
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/data/npm_full_page2.json b/swh/lister/npm/tests/data/npm_full_page2.json
	new file mode 100644
	--- /dev/null
	+++ b/swh/lister/npm/tests/data/npm_full_page2.json
	@@ -0,0 +1,168 @@
	+{
	+ "total_rows": 1496482,
	+ "offset": 2,
	+ "rows": [{
	+ "id": "-keyboardevent",
	+ "key": "-keyboardevent",
	+ "value": {
	+ "rev": "1-0fdff278618c4f8c1026889f1b525483"
	+ },
	+ "doc": {
	+ "_id": "-keyboardevent",
	+ "_rev": "1-0fdff278618c4f8c1026889f1b525483",
	+ "name": "-keyboardevent",
	+ "dist-tags": {
	+ "latest": "1.0.0"
	+ },
	+ "versions": {
	+ "1.0.0": {
	+ "name": "-keyboardevent",
	+ "version": "1.0.0",
	+ "description": "监听键盘事件",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
	+ },
	+ "author": {
	+ "name": "yanggw"
	+ },
	+ "license": "ISO",
	+ "bugs": {
	+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
	+ },
	+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
	+ "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3",
	+ "_id": "-keyboardevent@1.0.0",
	+ "_nodeVersion": "12.10.0",
	+ "_npmVersion": "6.10.3",
	+ "dist": {
	+ "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==",
	+ "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5",
	+ "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz",
	+ "fileCount": 3,
	+ "unpackedSize": 2714,
	+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n"
	+ },
	+ "maintainers": [{
	+ "name": "yanggw",
	+ "email": "2993065178@qq.com"
	+ }],
	+ "_npmUser": {
	+ "name": "yanggw",
	+ "email": "2993065178@qq.com"
	+ },
	+ "directories": {},
	+ "_npmOperationalInternal": {
	+ "host": "s3://npm-registry-packages",
	+ "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927"
	+ },
	+ "_hasShrinkwrap": false
	+ }
	+ },
	+ "time": {
	+ "created": "2020-09-25T06:10:56.357Z",
	+ "1.0.0": "2020-09-25T06:10:56.513Z",
	+ "modified": "2020-09-25T06:10:58.774Z"
	+ },
	+ "maintainers": [{
	+ "name": "yanggw",
	+ "email": "2993065178@qq.com"
	+ }],
	+ "description": "监听键盘事件",
	+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
	+ },
	+ "author": {
	+ "name": "yanggw"
	+ },
	+ "bugs": {
	+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
	+ },
	+ "license": "ISO",
	+ "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n",
	+ "readmeFilename": "README.md"
	+ }
	+ },
	+ {
	+ "id": "-lidonghui",
	+ "key": "-lidonghui",
	+ "value": {
	+ "rev": "1-1aa03a85878671cdb74b789c2b124578"
	+ },
	+ "doc": {
	+ "_id": "-lidonghui",
	+ "_rev": "1-1aa03a85878671cdb74b789c2b124578",
	+ "name": "-lidonghui",
	+ "dist-tags": {
	+ "latest": "1.0.0"
	+ },
	+ "versions": {
	+ "1.0.0": {
	+ "name": "-lidonghui",
	+ "version": "1.0.0",
	+ "description": "",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "bin": {
	+ "test1-cli": "bin/index.js"
	+ },
	+ "keywords": [],
	+ "author": "",
	+ "license": "ISC",
	+ "dependencies": {
	+ "axios": "^0.19.2",
	+ "commander": "^5.0.0",
	+ "inquirer": "^7.1.0"
	+ },
	+ "_id": "-lidonghui@1.0.0",
	+ "_nodeVersion": "12.14.0",
	+ "_npmVersion": "6.13.4",
	+ "dist": {
	+ "integrity": "sha512-1eUTEDU1kgAAsQ85uCEAp9tDsEIh3WNa5dB/ks8Uhf9Nh2RR37EeTGyxszn2A1B+MN+Q0fWrpCfBFzrrFyLFnA==",
	+ "shasum": "be7ffedc328bd123ba13c468a89ebcf0cfe76777",
	+ "tarball": "https://registry.npmjs.org/-lidonghui/-/-lidonghui-1.0.0.tgz",
	+ "fileCount": 2,
	+ "unpackedSize": 1024,
	+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJenVH+CRA9TVsSAnZWagAAEIMP/ipex7Yo+T2LqfWtFuZz\n5NuXhcnaNyANXyJ7iQCYzq1ynq+tkwBOTM7FNLkTbIiIUsn8d6JlAt5CfIFZ\nrfaYDK7BhU4E8KfLoSC0VqyaprTSvAeqeQDEAguYM9Nbt4TWa7W2HoorgQd7\njUAuGpRqvkw8Omh3eOvhqEa0Y5mkRf853ndJHD425bAww2R0KCA7RuOBO1MW\nnPZqBe3XOgJmsyEW+yl2tSGVq2hEEeD3C0G9x5K2ZfxZKVU7lGfqRlUdo7na\nTLJU6YZPTB9AhDTM9b8XFUMWlEtvHcwvGvuB+gLupXqWwEfwSqWg/2TJ11AQ\nMBFeC1xSav0BCv6kFxFBRwz1k7IAdVYatLnmqwNsIGwjWsyfmIThSNzYTIq6\nE/E0iD2gSAsLW5r7U0t+rk4sWLmu4xbLDvT1xpEY7C00Zj4NXJhdXSo5BR45\n1IBiU96LqEA2bIw+TsyaFO7w/chgpvd8mU0g34tYQCcAts56ZhCr43x5kUSY\ns7hm1b3MB5qpOBbom1Ta24xjvoJwxv2Et/T7zfZoXzHDyX9BgUCiVIvupxd9\nzR5LdyXIh6JdiVBAgEftbm59FEVrxVfAAi/sFMNJKp8pE1ELrSp/gulzxDd2\nC5Hq1z5RNFRQUlbkMJHLIDOZLAvdkPKieE+uFJ7WTYCPEL9nvQoIJiO9UFH1\njKtU\r\n=Opv1\r\n-----END PGP SIGNATURE-----\r\n"
	+ },
	+ "maintainers": [{
	+ "name": "tenderli",
	+ "email": "l2369660482@163.com"
	+ }],
	+ "_npmUser": {
	+ "name": "tenderli",
	+ "email": "l2369660482@163.com"
	+ },
	+ "directories": {},
	+ "_npmOperationalInternal": {
	+ "host": "s3://npm-registry-packages",
	+ "tmp": "tmp/-lidonghui_1.0.0_1587368445998_0.028137547296155807"
	+ },
	+ "_hasShrinkwrap": false
	+ }
	+ },
	+ "time": {
	+ "created": "2020-04-20T07:40:45.997Z",
	+ "1.0.0": "2020-04-20T07:40:46.087Z",
	+ "modified": "2020-04-20T07:40:48.525Z"
	+ },
	+ "maintainers": [{
	+ "name": "tenderli",
	+ "email": "l2369660482@163.com"
	+ }],
	+ "keywords": [],
	+ "license": "ISC",
	+ "readme": "ERROR: No README data found!",
	+ "readmeFilename": ""
	+ }
	+ }
	+ ]
	+}
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/data/npm_incremental_page1.json b/swh/lister/npm/tests/data/npm_incremental_page1.json
	new file mode 100644
	--- /dev/null
	+++ b/swh/lister/npm/tests/data/npm_incremental_page1.json
	@@ -0,0 +1,175 @@
	+{
	+ "results": [{
	+ "seq": 1,
	+ "id": "overlay-fishnet",
	+ "changes": [{
	+ "rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f"
	+ }],
	+ "doc": {
	+ "_id": "overlay-fishnet",
	+ "_rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f",
	+ "name": "overlay-fishnet",
	+ "description": "Overlay polygon with equal sized tiles (like a fishnet)",
	+ "dist-tags": {
	+ "latest": "0.0.0"
	+ },
	+ "versions": {
	+ "0.0.0": {
	+ "name": "overlay-fishnet",
	+ "version": "0.0.0",
	+ "description": "Overlay polygon with equal sized tiles (like a fishnet)",
	+ "main": "dist/index.js",
	+ "scripts": {
	+ "build": "rm -rf dist && mkdir dist && babel src --out-dir dist",
	+ "prepublish": "npm run test && npm run build",
	+ "test": "standard"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/yongjun21/fishnet.git"
	+ },
	+ "keywords": [
	+ "geospatial",
	+ "polygon",
	+ "tiles",
	+ "choropleth"
	+ ],
	+ "author": {
	+ "name": "Yong Jun"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/yongjun21/fishnet/issues"
	+ },
	+ "homepage": "https://github.com/yongjun21/fishnet#readme",
	+ "dependencies": {
	+ "babel-cli": "^6.18.0",
	+ "babel-preset-es2015": "^6.18.0",
	+ "babel-preset-stage-3": "^6.17.0",
	+ "standard": "^8.6.0"
	+ },
	+ "gitHead": "bc3d598445a767efefd094260c83d6f57f06e86d",
	+ "_id": "overlay-fishnet@0.0.0",
	+ "_shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0",
	+ "_from": ".",
	+ "_npmVersion": "3.10.7",
	+ "_nodeVersion": "6.2.2",
	+ "_npmUser": {
	+ "name": "yongjun21",
	+ "email": "yjthong.2009@smu.edu.sg"
	+ },
	+ "dist": {
	+ "shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0",
	+ "tarball": "https://registry.npmjs.org/overlay-fishnet/-/overlay-fishnet-0.0.0.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "yongjun21",
	+ "email": "yjthong.2009@smu.edu.sg"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/overlay-fishnet-0.0.0.tgz_1484192243466_0.4572450101841241"
	+ }
	+ }
	+ },
	+ "readme": "# fishnet\nOverlay polygon with equal sized tiles (like a fishnet)\n",
	+ "maintainers": [{
	+ "name": "yongjun21",
	+ "email": "yjthong.2009@smu.edu.sg"
	+ }],
	+ "time": {
	+ "modified": "2017-01-12T03:37:23.696Z",
	+ "created": "2017-01-12T03:37:23.696Z",
	+ "0.0.0": "2017-01-12T03:37:23.696Z"
	+ },
	+ "homepage": "https://github.com/yongjun21/fishnet#readme",
	+ "keywords": [
	+ "geospatial",
	+ "polygon",
	+ "tiles",
	+ "choropleth"
	+ ],
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/yongjun21/fishnet.git"
	+ },
	+ "author": {
	+ "name": "Yong Jun"
	+ },
	+ "bugs": {
	+ "url": "https://github.com/yongjun21/fishnet/issues"
	+ },
	+ "license": "ISC",
	+ "readmeFilename": "README.md"
	+ }
	+ },
	+ {
	+ "seq": 2,
	+ "id": "shopping_site_detect",
	+ "changes": [{
	+ "rev": "1-c62a780cdd9865b06b1e4eb0e6552560"
	+ }],
	+ "doc": {
	+ "_id": "shopping_site_detect",
	+ "_rev": "1-c62a780cdd9865b06b1e4eb0e6552560",
	+ "name": "shopping_site_detect",
	+ "description": "shopping site detect",
	+ "dist-tags": {
	+ "latest": "1.0.0"
	+ },
	+ "versions": {
	+ "1.0.0": {
	+ "name": "shopping_site_detect",
	+ "version": "1.0.0",
	+ "description": "shopping site detect",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "author": {
	+ "name": "Jayin Ton"
	+ },
	+ "license": "MIT",
	+ "gitHead": "223c4a7e12acf28310a106780652d1e7d0629d68",
	+ "_id": "shopping_site_detect@1.0.0",
	+ "_shasum": "0408f054053d7581869d42934e42d50794d20b8c",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.4.0",
	+ "_npmUser": {
	+ "name": "jayinton",
	+ "email": "tonjayin@gmail.com"
	+ },
	+ "dist": {
	+ "shasum": "0408f054053d7581869d42934e42d50794d20b8c",
	+ "tarball": "https://registry.npmjs.org/shopping_site_detect/-/shopping_site_detect-1.0.0.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "jayinton",
	+ "email": "tonjayin@gmail.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/shopping_site_detect-1.0.0.tgz_1484193398924_0.3402463826350868"
	+ }
	+ }
	+ },
	+ "maintainers": [{
	+ "name": "jayinton",
	+ "email": "tonjayin@gmail.com"
	+ }],
	+ "time": {
	+ "modified": "2017-01-12T03:56:39.161Z",
	+ "created": "2017-01-12T03:56:39.161Z",
	+ "1.0.0": "2017-01-12T03:56:39.161Z"
	+ },
	+ "author": {
	+ "name": "Jayin Ton"
	+ },
	+ "license": "MIT",
	+ "readmeFilename": "readme.md"
	+ }
	+ }
	+ ],
	+ "last_seq": 2
	+}
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/data/npm_incremental_page2.json b/swh/lister/npm/tests/data/npm_incremental_page2.json
	new file mode 100644
	--- /dev/null
	+++ b/swh/lister/npm/tests/data/npm_incremental_page2.json
	@@ -0,0 +1,632 @@
	+{
	+ "results": [{
	+ "seq": 3,
	+ "id": "tinyviewpager",
	+ "changes": [{
	+ "rev": "1-01e65a5602e9c43ed3482657f1309ef0"
	+ }],
	+ "doc": {
	+ "_id": "tinyviewpager",
	+ "_rev": "1-01e65a5602e9c43ed3482657f1309ef0",
	+ "name": "tinyviewpager",
	+ "time": {
	+ "modified": "2017-01-12T04:05:34.847Z",
	+ "created": "2017-01-11T10:52:41.047Z",
	+ "1.0.0": "2017-01-11T10:52:41.047Z",
	+ "1.0.1": "2017-01-11T14:02:03.232Z",
	+ "1.0.2": "2017-01-11T14:31:04.820Z",
	+ "1.0.3": "2017-01-11T14:38:41.688Z",
	+ "1.0.4": "2017-01-11T14:45:57.970Z",
	+ "1.0.5": "2017-01-11T14:49:04.778Z",
	+ "1.0.6": "2017-01-11T14:54:06.876Z",
	+ "1.0.7": "2017-01-11T15:07:13.271Z",
	+ "1.0.8": "2017-01-12T04:05:34.847Z"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "dist-tags": {
	+ "latest": "1.0.8"
	+ },
	+ "description": "上下左右无限循环viewpager",
	+ "versions": {
	+ "1.0.1": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.1",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react": "^15.4.2",
	+ "react-native": "^0.40.0",
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "fdb455a393c7d6b6d43d01f73362fa6e3ed0dbb9",
	+ "_id": "tinyviewpager@1.0.1",
	+ "_shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.1.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.1.tgz_1484143322968_0.5517262546345592"
	+ }
	+ },
	+ "1.0.2": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.2",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react": "^15.4.2",
	+ "react-native": "^0.40.0",
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "daae30e6ce7bea30a14619b5cef5f52a86cc5528",
	+ "_id": "tinyviewpager@1.0.2",
	+ "_shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.2.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.2.tgz_1484145064595_0.49263988458551466"
	+ }
	+ },
	+ "1.0.3": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.3",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react": "^15.4.2",
	+ "react-native": "^0.40.0",
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "d08aeab765b537d71ed21e7e37ae6546e9bdbef2",
	+ "_id": "tinyviewpager@1.0.3",
	+ "_shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.3.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.3.tgz_1484145521456_0.8925717421807349"
	+ }
	+ },
	+ "1.0.4": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.4",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react": "^15.4.2",
	+ "react-native": "^0.40.0",
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
	+ "_id": "tinyviewpager@1.0.4",
	+ "_shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.4.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.4.tgz_1484145957736_0.7274783558677882"
	+ }
	+ },
	+ "1.0.5": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.5",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
	+ "_id": "tinyviewpager@1.0.5",
	+ "_shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.5.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.5.tgz_1484146144523_0.9442351886536926"
	+ }
	+ },
	+ "1.0.6": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.6",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
	+ "_id": "tinyviewpager@1.0.6",
	+ "_shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.6.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-18-east.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.6.tgz_1484146444932_0.7940177812706679"
	+ }
	+ },
	+ "1.0.7": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.7",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
	+ "_id": "tinyviewpager@1.0.7",
	+ "_shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.7.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-18-east.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.7.tgz_1484147231500_0.17695364635437727"
	+ }
	+ },
	+ "1.0.8": {
	+ "name": "tinyviewpager",
	+ "version": "1.0.8",
	+ "description": "上下左右无限循环viewpager",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "license": "ISC",
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "dependencies": {
	+ "react-timer-mixin": "^0.13.3"
	+ },
	+ "gitHead": "54b9e9f1517a0385307065f96ef85e9b7b13868e",
	+ "_id": "tinyviewpager@1.0.8",
	+ "_shasum": "67234eebbae3a8205319c950fab62c8f84840061",
	+ "_from": ".",
	+ "_npmVersion": "3.10.3",
	+ "_nodeVersion": "6.3.1",
	+ "_npmUser": {
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "67234eebbae3a8205319c950fab62c8f84840061",
	+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.8.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "tinylpc",
	+ "email": "1150247879@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-18-east.internal.npmjs.com",
	+ "tmp": "tmp/tinyviewpager-1.0.8.tgz_1484193932800_0.371786616044119"
	+ }
	+ }
	+ },
	+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
	+ "keywords": [
	+ "viewpager"
	+ ],
	+ "repository": {
	+ "type": "git",
	+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
	+ },
	+ "author": {
	+ "name": "tinylpc"
	+ },
	+ "bugs": {
	+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
	+ },
	+ "license": "ISC",
	+ "readmeFilename": "README.md"
	+ }
	+ },
	+ {
	+ "seq": 4,
	+ "id": "ubase-ext-wecloud",
	+ "changes": [{
	+ "rev": "1-0a540144594976690ff540d57ed3ab3b"
	+ }],
	+ "doc": {
	+ "_id": "ubase-ext-wecloud",
	+ "_rev": "1-0a540144594976690ff540d57ed3ab3b",
	+ "name": "ubase-ext-wecloud",
	+ "dist-tags": {
	+ "latest": "1.0.3"
	+ },
	+ "versions": {
	+ "1.0.0": {
	+ "name": "ubase-ext-wecloud",
	+ "version": "1.0.0",
	+ "description": "",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "author": "",
	+ "license": "ISC",
	+ "dependencies": {
	+ "copy-webpack-plugin": "^4.0.1",
	+ "ubase-vue": "^1.1.14"
	+ },
	+ "_id": "ubase-ext-wecloud@1.0.0",
	+ "_shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897",
	+ "_from": ".",
	+ "_npmVersion": "3.8.6",
	+ "_nodeVersion": "6.1.0",
	+ "_npmUser": {
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897",
	+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.0.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-18-east.internal.npmjs.com",
	+ "tmp": "tmp/ubase-ext-wecloud-1.0.0.tgz_1484191505909_0.5115192488301545"
	+ }
	+ },
	+ "1.0.1": {
	+ "name": "ubase-ext-wecloud",
	+ "version": "1.0.1",
	+ "description": "",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "author": "",
	+ "license": "ISC",
	+ "dependencies": {
	+ "copy-webpack-plugin": "^4.0.1",
	+ "ubase-vue": "^1.1.14"
	+ },
	+ "_id": "ubase-ext-wecloud@1.0.1",
	+ "_shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8",
	+ "_from": ".",
	+ "_npmVersion": "3.8.6",
	+ "_nodeVersion": "6.1.0",
	+ "_npmUser": {
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8",
	+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.1.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/ubase-ext-wecloud-1.0.1.tgz_1484191845449_0.817801987985149"
	+ }
	+ },
	+ "1.0.2": {
	+ "name": "ubase-ext-wecloud",
	+ "version": "1.0.2",
	+ "description": "",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "author": "",
	+ "license": "ISC",
	+ "dependencies": {
	+ "copy-webpack-plugin": "^4.0.1",
	+ "ubase-vue": "^1.1.14"
	+ },
	+ "_id": "ubase-ext-wecloud@1.0.2",
	+ "_shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73",
	+ "_from": ".",
	+ "_npmVersion": "3.8.6",
	+ "_nodeVersion": "6.1.0",
	+ "_npmUser": {
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73",
	+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.2.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-18-east.internal.npmjs.com",
	+ "tmp": "tmp/ubase-ext-wecloud-1.0.2.tgz_1484192705755_0.1207962513435632"
	+ }
	+ },
	+ "1.0.3": {
	+ "name": "ubase-ext-wecloud",
	+ "version": "1.0.3",
	+ "description": "",
	+ "main": "index.js",
	+ "scripts": {
	+ "test": "echo \"Error: no test specified\" && exit 1"
	+ },
	+ "author": "",
	+ "license": "ISC",
	+ "dependencies": {
	+ "copy-webpack-plugin": "^4.0.1",
	+ "ubase-vue": "^1.1.14"
	+ },
	+ "gitHead": "a5b60823b706c4fd98ae798dbe76f18a7841eead",
	+ "_id": "ubase-ext-wecloud@1.0.3",
	+ "_shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b",
	+ "_from": ".",
	+ "_npmVersion": "3.8.6",
	+ "_nodeVersion": "6.1.0",
	+ "_npmUser": {
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ },
	+ "dist": {
	+ "shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b",
	+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.3.tgz"
	+ },
	+ "maintainers": [{
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ }],
	+ "_npmOperationalInternal": {
	+ "host": "packages-12-west.internal.npmjs.com",
	+ "tmp": "tmp/ubase-ext-wecloud-1.0.3.tgz_1484193826093_0.8929786814842373"
	+ }
	+ }
	+ },
	+ "readme": "# ubase-ext-wecloud\n",
	+ "maintainers": [{
	+ "name": "askliujun",
	+ "email": "547794529@qq.com"
	+ }],
	+ "time": {
	+ "modified": "2017-01-12T04:03:46.329Z",
	+ "created": "2017-01-12T03:25:07.858Z",
	+ "1.0.0": "2017-01-12T03:25:07.858Z",
	+ "1.0.1": "2017-01-12T03:30:45.704Z",
	+ "1.0.2": "2017-01-12T03:45:07.635Z",
	+ "1.0.3": "2017-01-12T04:03:46.329Z"
	+ },
	+ "license": "ISC",
	+ "readmeFilename": "README.md"
	+ }
	+ }
	+ ],
	+ "last_seq": 4
	+}
	\ No newline at end of file
	diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py
	--- a/swh/lister/npm/tests/test_lister.py
	+++ b/swh/lister/npm/tests/test_lister.py
	@@ -1,97 +1,200 @@
	-# Copyright (C) 2018-2019 The Software Heritage developers
	+# Copyright (C) 2018-2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-import logging
	-import re
	-from typing import Any, List
	-import unittest
	+from itertools import chain
	+import json
	+from pathlib import Path

	-import requests_mock
	+import iso8601
	+import pytest
	+from requests.exceptions import HTTPError

	-from swh.lister.core.tests.test_lister import HttpListerTesterBase
	-from swh.lister.npm.lister import NpmIncrementalLister, NpmLister
	+from swh.lister import USER_AGENT
	+from swh.lister.npm.lister import NpmLister, NpmListerState

	-logger = logging.getLogger(__name__)

	+@pytest.fixture
	+def npm_full_listing_page1(datadir):
	+ return json.loads(Path(datadir, "npm_full_page1.json").read_text())

	-class NpmListerTester(HttpListerTesterBase, unittest.TestCase):
	- Lister = NpmLister
	- test_re = re.compile(r'^./_all_docs\?startkey="(.+)".')
	- lister_subdir = "npm"
	- good_api_response_file = "data/replicate.npmjs.com/api_response.json"
	- bad_api_response_file = "data/api_empty_response.json"
	- first_index = "jquery"
	- entries_per_page = 100

	- @requests_mock.Mocker()
	- def test_is_within_bounds(self, http_mocker):
	- # disable this test from HttpListerTesterBase as
	- # it can not succeed for the npm lister due to the
	- # overriding of the string_pattern_check method
	- pass
	+@pytest.fixture
	+def npm_full_listing_page2(datadir):
	+ return json.loads(Path(datadir, "npm_full_page2.json").read_text())


	-class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase):
	- Lister = NpmIncrementalLister
	- test_re = re.compile(r"^./_changes\?since=([0-9]+).")
	- lister_subdir = "npm"
	- good_api_response_file = "data/api_inc_response.json"
	- bad_api_response_file = "data/api_inc_empty_response.json"
	- first_index = "6920642"
	- entries_per_page = 100
	+@pytest.fixture
	+def npm_incremental_listing_page1(datadir):
	+ return json.loads(Path(datadir, "npm_incremental_page1.json").read_text())

	- @requests_mock.Mocker()
	- def test_is_within_bounds(self, http_mocker):
	- # disable this test from HttpListerTesterBase as
	- # it can not succeed for the npm lister due to the
	- # overriding of the string_pattern_check method
	- pass

	+@pytest.fixture
	+def npm_incremental_listing_page2(datadir):
	+ return json.loads(Path(datadir, "npm_incremental_page2.json").read_text())

	-def check_tasks(tasks: List[Any]):
	- """Ensure scheduled tasks are in the expected format.

	+def _check_listed_npm_packages(lister, packages, scheduler_origins):
	+ for package in packages:
	+ package_name = package["doc"]["name"]
	+ latest_version = package["doc"]["dist-tags"]["latest"]
	+ package_last_update = iso8601.parse_date(package["doc"]["time"][latest_version])
	+ origin_url = lister.PACKAGE_URL_TEMPLATE.format(package_name=package_name)

	- """
	- for row in tasks:
	- logger.debug("row: %s", row)
	- assert row["type"] == "load-npm"
	- # arguments check
	- args = row["arguments"]["args"]
	- assert len(args) == 0
	+ scheduler_origin = [o for o in scheduler_origins if o.url == origin_url]
	+ assert scheduler_origin
	+ assert scheduler_origin[0].last_update == package_last_update

	- # kwargs
	- kwargs = row["arguments"]["kwargs"]
	- assert len(kwargs) == 1
	- package_url = kwargs["url"]
	- package_name = package_url.split("/")[-1]
	- assert package_url == f"https://www.npmjs.com/package/{package_name}"

	- assert row["policy"] == "recurring"
	- assert row["priority"] is None
	+def _match_request(request):
	+ return request.headers.get("User-Agent") == USER_AGENT


	-def test_npm_lister_basic_listing(lister_npm, requests_mock_datadir):
	- lister_npm.run()
	+def _url_params(page_size, **kwargs):
	+ params = {"limit": page_size, "include_docs": "true"}
	+ params.update(**kwargs)
	+ return params

	- tasks = lister_npm.scheduler.search_tasks(task_type="load-npm")
	- assert len(tasks) == 100

	- check_tasks(tasks)
	+def test_npm_lister_full(
	+ swh_scheduler, requests_mock, mocker, npm_full_listing_page1, npm_full_listing_page2
	+):
	+ """Simulate a full listing of four npm packages in two pages"""
	+ page_size = 2
	+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=False)

	+ requests_mock.get(
	+ lister.API_FULL_LISTING_URL,
	+ [{"json": npm_full_listing_page1}, {"json": npm_full_listing_page2},],
	+ additional_matcher=_match_request,
	+ )
	+
	+ spy_get = mocker.spy(lister.session, "get")
	+
	+ stats = lister.run()
	+ assert stats.pages == 2
	+ assert stats.origins == page_size * stats.pages
	+
	+ spy_get.assert_has_calls(
	+ [
	+ mocker.call(
	+ lister.API_FULL_LISTING_URL,
	+ params=_url_params(page_size + 1, startkey='""'),
	+ ),
	+ mocker.call(
	+ lister.API_FULL_LISTING_URL,
	+ params=_url_params(
	+ page_size + 1,
	+ startkey=f'"{npm_full_listing_page1["rows"][-1]["id"]}"',
	+ ),
	+ ),
	+ ]
	+ )
	+
	+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
	+
	+ _check_listed_npm_packages(
	+ lister,
	+ chain(npm_full_listing_page1["rows"][:-1], npm_full_listing_page2["rows"]),
	+ scheduler_origins,
	+ )
	+
	+ assert lister.get_state_from_scheduler() == NpmListerState()
	+
	+
	+def test_npm_lister_incremental(
	+ swh_scheduler,
	+ requests_mock,
	+ mocker,
	+ npm_incremental_listing_page1,
	+ npm_incremental_listing_page2,
	+):
	+ """Simulate an incremental listing of four npm packages in two pages"""
	+ page_size = 2
	+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True)
	+
	+ requests_mock.get(
	+ lister.API_INCREMENTAL_LISTING_URL,
	+ [
	+ {"json": npm_incremental_listing_page1},
	+ {"json": npm_incremental_listing_page2},
	+ {"json": {"results": []}},
	+ ],
	+ additional_matcher=_match_request,
	+ )
	+
	+ spy_get = mocker.spy(lister.session, "get")
	+
	+ assert lister.get_state_from_scheduler() == NpmListerState()
	+
	+ stats = lister.run()
	+ assert stats.pages == 2
	+ assert stats.origins == page_size * stats.pages
	+
	+ last_seq = npm_incremental_listing_page2["results"][-1]["seq"]
	+
	+ spy_get.assert_has_calls(
	+ [
	+ mocker.call(
	+ lister.API_INCREMENTAL_LISTING_URL,
	+ params=_url_params(page_size, since="0"),
	+ ),
	+ mocker.call(
	+ lister.API_INCREMENTAL_LISTING_URL,
	+ params=_url_params(
	+ page_size,
	+ since=str(npm_incremental_listing_page1["results"][-1]["seq"]),
	+ ),
	+ ),
	+ mocker.call(
	+ lister.API_INCREMENTAL_LISTING_URL,
	+ params=_url_params(page_size, since=str(last_seq)),
	+ ),
	+ ]
	+ )
	+
	+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins

	-def test_npm_lister_listing_pagination(lister_npm, requests_mock_datadir):
	- lister = lister_npm
	- # Patch per page pagination
	- lister.per_page = 10 + 1
	- lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace(
	- "&limit=1001", "&limit=%s" % lister.per_page
	+ _check_listed_npm_packages(
	+ lister,
	+ chain(
	+ npm_incremental_listing_page1["results"],
	+ npm_incremental_listing_page2["results"],
	+ ),
	+ scheduler_origins,
	)
	+
	+ assert lister.get_state_from_scheduler() == NpmListerState(last_seq=last_seq)
	+
	+
	+def test_npm_lister_incremental_restart(
	+ swh_scheduler, requests_mock, mocker,
	+):
	+ """Check incremental npm listing will restart from saved state"""
	+ page_size = 2
	+ last_seq = 67
	+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True)
	+ lister.state = NpmListerState(last_seq=last_seq)
	+
	+ requests_mock.get(lister.API_INCREMENTAL_LISTING_URL, json={"results": []})
	+
	+ spy_get = mocker.spy(lister.session, "get")
	+
	lister.run()

	- tasks = lister.scheduler.search_tasks(task_type="load-npm")
	- assert len(tasks) == 2 * 10 # only 2 files with 10 results each
	+ spy_get.assert_called_with(
	+ lister.API_INCREMENTAL_LISTING_URL,
	+ params=_url_params(page_size, since=str(last_seq)),
	+ )
	+
	+
	+def test_npm_lister_http_error(
	+ swh_scheduler, requests_mock, mocker,
	+):
	+ lister = NpmLister(scheduler=swh_scheduler)
	+
	+ requests_mock.get(lister.API_FULL_LISTING_URL, status_code=500)

	- check_tasks(tasks)
	+ with pytest.raises(HTTPError):
	+ lister.run()
	diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py
	--- a/swh/lister/npm/tests/test_tasks.py
	+++ b/swh/lister/npm/tests/test_tasks.py
	@@ -1,15 +1,9 @@
	-# Copyright (C) 2019-2020 The Software Heritage developers
	+# Copyright (C) 2019-2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-from contextlib import contextmanager
	-from unittest.mock import patch
	-
	-
	-@contextmanager
	-def mock_save(lister):
	- yield
	+from swh.lister.pattern import ListerStats


	def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
	@@ -20,34 +14,31 @@
	assert res.result == "OK"


	-@patch("swh.lister.npm.tasks.save_registry_state")
	-@patch("swh.lister.npm.tasks.NpmLister")
	-def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker):
	- # setup the mocked NpmLister
	- lister.return_value = lister
	- lister.run.return_value = None
	- save.side_effect = mock_save
	+def test_full_lister_task(
	+ swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
	+):
	+ stats = ListerStats(pages=10, origins=900)
	+ mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
	+ mock_lister.from_configfile.return_value = mock_lister
	+ mock_lister.run.return_value = stats

	res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask")
	assert res
	res.wait()
	assert res.successful()

	- lister.assert_called_once_with()
	- lister.run.assert_called_once_with()
	+ mock_lister.from_configfile.assert_called_once_with(incremental=False)
	+ mock_lister.run.assert_called_once_with()
	+ assert res.result == stats.dict()


	-@patch("swh.lister.npm.tasks.save_registry_state")
	-@patch("swh.lister.npm.tasks.get_last_update_seq")
	-@patch("swh.lister.npm.tasks.NpmIncrementalLister")
	-def test_incremental(
	- lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker
	+def test_incremental_lister_task(
	+ swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
	):
	- # setup the mocked NpmLister
	- lister.return_value = lister
	- lister.run.return_value = None
	- seq.return_value = 42
	- save.side_effect = mock_save
	+ stats = ListerStats(pages=10, origins=900)
	+ mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
	+ mock_lister.from_configfile.return_value = mock_lister
	+ mock_lister.run.return_value = stats

	res = swh_scheduler_celery_app.send_task(
	"swh.lister.npm.tasks.NpmIncrementalListerTask"
	@@ -56,6 +47,6 @@
	res.wait()
	assert res.successful()

	- lister.assert_called_once_with()
	- seq.assert_called_once_with(lister)
	- lister.run.assert_called_once_with(min_bound=42)
	+ mock_lister.from_configfile.assert_called_once_with(incremental=True)
	+ mock_lister.run.assert_called_once_with()
	+ assert res.result == stats.dict()

File Metadata

Mime Type: text/plain
Expires: Thu, Jul 3, 3:43 PM (2 w, 3 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3220514

D4877.diffNo OneTemporaryActions

D4877.diffView Options

File Metadata

Event Timeline

D4877.diff
No OneTemporary
Actions

D4877.diff
View Options