Page MenuHomeSoftware Heritage
Files F9346081

D4877.diff
No OneTemporary
Actions

D4877.diff
View Options

diff --git a/swh/lister/npm/__init__.py b/swh/lister/npm/__init__.py
--- a/swh/lister/npm/__init__.py
+++ b/swh/lister/npm/__init__.py
@@ -1,14 +1,13 @@
-# Copyright (C) 2019 the Software Heritage developers
+# Copyright (C) 2019-2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
def register():
from .lister import NpmLister
- from .models import NpmModel, NpmVisitModel
return {
- "models": [NpmVisitModel, NpmModel],
+ "models": [],
"lister": NpmLister,
"task_modules": ["%s.tasks" % __name__],
"task_types": {
diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py
--- a/swh/lister/npm/lister.py
+++ b/swh/lister/npm/lister.py
@@ -1,154 +1,190 @@
-# Copyright (C) 2018-2019 the Software Heritage developers
+# Copyright (C) 2018-2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from typing import Any, Dict, List, Optional
+from dataclasses import asdict, dataclass
+import logging
+from typing import Any, Dict, Iterator, List, Optional
-from requests import Response
+import iso8601
+import requests
+from tenacity.before_sleep import before_sleep_log
-from swh.core import config
-from swh.lister.core.indexing_lister import IndexingHttpLister
-from swh.lister.npm.models import NpmModel
-from swh.scheduler.utils import create_task_dict
+from swh.lister import USER_AGENT
+from swh.lister.pattern import CredentialsType, Lister
+from swh.lister.utils import throttling_retry
+from swh.scheduler.interface import SchedulerInterface
+from swh.scheduler.model import ListedOrigin
-DEFAULT_CONFIG = {
- "loading_task_policy": "recurring",
-}
+logger = logging.getLogger(__name__)
-class NpmListerBase(IndexingHttpLister):
- """List packages available in the npm registry in a paginated way
+@dataclass
+class NpmListerState:
+ """State of npm lister"""
- """
-
- MODEL = NpmModel
- LISTER_NAME = "npm"
- instance = "npm"
+ last_seq: Optional[int] = None
- def __init__(
- self, url="https://replicate.npmjs.com", per_page=1000, override_config=None
- ):
- super().__init__(url=url, override_config=override_config)
- self.config = config.merge_configs(DEFAULT_CONFIG, self.config)
- self.per_page = per_page + 1
- self.PATH_TEMPLATE += "&limit=%s" % self.per_page
-
- def get_model_from_repo(self, repo_name: str) -> Dict[str, str]:
- """(Override) Transform from npm package name to model
-
- """
- package_url = "https://www.npmjs.com/package/%s" % repo_name
- return {
- "uid": repo_name,
- "indexable": repo_name,
- "name": repo_name,
- "full_name": repo_name,
- "html_url": package_url,
- "origin_url": package_url,
- "origin_type": "npm",
- }
-
- def task_dict(self, origin_type: str, origin_url: str, **kwargs):
- """(Override) Return task dict for loading a npm package into the
- archive.
-
- This is overridden from the lister_base as more information is
- needed for the ingestion task creation.
-
- """
- task_type = "load-%s" % origin_type
- task_policy = self.config["loading_task_policy"]
- return create_task_dict(task_type, task_policy, url=origin_url)
-
- def request_headers(self) -> Dict[str, Any]:
- """(Override) Set requests headers to send when querying the npm
- registry.
-
- """
- headers = super().request_headers()
- headers["Accept"] = "application/json"
- return headers
-
- def string_pattern_check(self, inner: int, lower: int, upper: int = None):
- """ (Override) Inhibit the effect of that method as packages indices
- correspond to package names and thus do not respect any kind
- of fixed length string pattern
-
- """
- pass
-
-
-class NpmLister(NpmListerBase):
- """List all packages available in the npm registry in a paginated way
+class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]):
"""
+ List all packages hosted on the npm registry.
- PATH_TEMPLATE = '/_all_docs?startkey="%s"'
-
- def get_next_target_from_response(self, response: Response) -> Optional[str]:
- """(Override) Get next npm package name to continue the listing
-
- """
- repos = response.json()["rows"]
- return repos[-1]["id"] if len(repos) == self.per_page else None
-
- def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]:
- """(Override) Transform npm registry response to list for model manipulation
-
- """
- repos = response.json()["rows"]
- if len(repos) == self.per_page:
- repos = repos[:-1]
- return [self.get_model_from_repo(repo["id"]) for repo in repos]
-
+ The lister is based on the npm replication API powered by a
+ CouchDB database (https://docs.couchdb.org/en/stable/api/database/).
-class NpmIncrementalLister(NpmListerBase):
- """List packages in the npm registry, updated since a specific
- update_seq value of the underlying CouchDB database, in a paginated way.
+ Args:
+ scheduler: a scheduler instance
+ page_size: number of packages info to return per page when querying npm API
+ incremental: defines if incremental listing should be used, in that case
+ only modified or new packages since last incremental listing operation
+ will be returned, otherwise all packages will be listed in lexicographical
+ order
"""
- PATH_TEMPLATE = "/_changes?since=%s"
-
- @property
- def CONFIG_BASE_FILENAME(self): # noqa: N802
- return "lister_npm_incremental"
-
- def get_next_target_from_response(self, response: Response) -> Optional[str]:
- """(Override) Get next npm package name to continue the listing.
-
- """
- repos = response.json()["results"]
- return repos[-1]["seq"] if len(repos) == self.per_page else None
-
- def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]:
- """(Override) Transform npm registry response to list for model
- manipulation.
-
- """
- repos = response.json()["results"]
- if len(repos) == self.per_page:
- repos = repos[:-1]
- return [self.get_model_from_repo(repo["id"]) for repo in repos]
+ LISTER_NAME = "npm"
+ INSTANCE = "npm"
- def filter_before_inject(self, models_list: List[Dict[str, Any]]):
- """(Override) Filter out documents in the CouchDB database
- not related to a npm package.
+ API_BASE_URL = "https://replicate.npmjs.com"
+ API_INCREMENTAL_LISTING_URL = f"{API_BASE_URL}/_changes"
+ API_FULL_LISTING_URL = f"{API_BASE_URL}/_all_docs"
+ PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}"
- """
- models_filtered = []
- for model in models_list:
- package_name = model["name"]
- # document related to CouchDB internals
- if package_name.startswith("_design/"):
+ def __init__(
+ self,
+ scheduler: SchedulerInterface,
+ page_size: int = 1000,
+ incremental: bool = False,
+ credentials: CredentialsType = None,
+ ):
+ super().__init__(
+ scheduler=scheduler,
+ credentials=credentials,
+ url=self.API_INCREMENTAL_LISTING_URL
+ if incremental
+ else self.API_FULL_LISTING_URL,
+ instance=self.INSTANCE,
+ )
+
+ self.page_size = page_size
+ if not incremental:
+ # in full listing mode, first package in each page corresponds to the one
+ # provided as the startkey query parameter value, so we increment the page
+ # size by one to avoid double package processing
+ self.page_size += 1
+ self.incremental = incremental
+
+ self.session = requests.Session()
+ self.session.headers.update(
+ {"Accept": "application/json", "User-Agent": USER_AGENT}
+ )
+
+ def state_from_dict(self, d: Dict[str, Any]) -> NpmListerState:
+ return NpmListerState(**d)
+
+ def state_to_dict(self, state: NpmListerState) -> Dict[str, Any]:
+ return asdict(state)
+
+ def request_params(self, last_package_id: str) -> Dict[str, Any]:
+ # include package JSON document to get its last update date
+ params = {"limit": self.page_size, "include_docs": "true"}
+ if self.incremental:
+ params["since"] = last_package_id
+ else:
+ params["startkey"] = last_package_id
+ return params
+
+ @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
+ def page_request(self, last_package_id: str) -> requests.Response:
+ params = self.request_params(last_package_id)
+ logger.debug("Fetching URL %s with params %s", self.url, params)
+ response = self.session.get(self.url, params=params)
+ if response.status_code != 200:
+ logger.warning(
+ "Unexpected HTTP status code %s on %s: %s",
+ response.status_code,
+ response.url,
+ response.content,
+ )
+ response.raise_for_status()
+ return response
+
+ def get_pages(self) -> Iterator[List[Dict[str, Any]]]:
+ last_package_id: str = "0" if self.incremental else '""'
+ if (
+ self.incremental
+ and self.state is not None
+ and self.state.last_seq is not None
+ ):
+ last_package_id = str(self.state.last_seq)
+
+ while True:
+
+ response = self.page_request(last_package_id)
+
+ data = response.json()
+ page = data["results"] if self.incremental else data["rows"]
+
+ if not page:
+ break
+
+ if self.incremental or len(page) < self.page_size:
+ yield page
+ else:
+ yield page[:-1]
+
+ if len(page) < self.page_size:
+ break
+
+ last_package_id = (
+ str(page[-1]["seq"]) if self.incremental else f'"{page[-1]["id"]}"'
+ )
+
+ def get_origins_from_page(
+ self, page: List[Dict[str, Any]]
+ ) -> Iterator[ListedOrigin]:
+ """Convert a page of Npm repositories into a list of ListedOrigin."""
+ assert self.lister_obj.id is not None
+
+ for package in page:
+ # no source code to archive here
+ if not package["doc"].get("versions", {}):
continue
- models_filtered.append(model)
- return models_filtered
-
- def disable_deleted_repo_tasks(self, start, end, keep_these):
- """(Override) Disable the processing performed by that method as it is
- not relevant in this incremental lister context. It also raises an
- exception due to a different index type (int instead of str).
- """
- pass
+ package_name = package["doc"]["name"]
+ package_latest_version = (
+ package["doc"].get("dist-tags", {}).get("latest", "")
+ )
+
+ last_update = None
+ if package_latest_version in package["doc"].get("time", {}):
+ last_update = iso8601.parse_date(
+ package["doc"]["time"][package_latest_version]
+ )
+
+ yield ListedOrigin(
+ lister_id=self.lister_obj.id,
+ url=self.PACKAGE_URL_TEMPLATE.format(package_name=package_name),
+ visit_type="npm",
+ last_update=last_update,
+ )
+
+ def commit_page(self, page: List[Dict[str, Any]]):
+ """Update the currently stored state using the latest listed page."""
+ if self.incremental:
+ last_package = page[-1]
+ last_seq = last_package["seq"]
+
+ if self.state.last_seq is None or last_seq > self.state.last_seq:
+ self.state.last_seq = last_seq
+
+ def finalize(self):
+ if self.incremental and self.state.last_seq is not None:
+ scheduler_state = self.get_state_from_scheduler()
+
+ if (
+ scheduler_state.last_seq is None
+ or self.state.last_seq > scheduler_state.last_seq
+ ):
+ self.updated = True
diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py
deleted file mode 100644
--- a/swh/lister/npm/models.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (C) 2018 the Software Heritage developers
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-from sqlalchemy import BigInteger, Column, DateTime, Integer, Sequence, String
-
-from swh.lister.core.models import ABCSQLMeta, IndexingModelBase, SQLBase
-
-
-class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta):
- """Table to store the npm registry state at the time of a
- content listing by Software Heritage
- """
-
- __tablename__ = "npm_visit"
-
- uid = Column(Integer, Sequence("npm_visit_id_seq"), primary_key=True)
- visit_date = Column(DateTime, nullable=False)
- doc_count = Column(BigInteger)
- doc_del_count = Column(BigInteger)
- update_seq = Column(BigInteger)
- purge_seq = Column(BigInteger)
- disk_size = Column(BigInteger)
- data_size = Column(BigInteger)
- committed_update_seq = Column(BigInteger)
- compacted_seq = Column(BigInteger)
-
-
-class NpmModel(IndexingModelBase):
- """A npm package representation
-
- """
-
- __tablename__ = "npm_repo"
-
- uid = Column(String, primary_key=True)
- indexable = Column(String, index=True)
diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py
--- a/swh/lister/npm/tasks.py
+++ b/swh/lister/npm/tasks.py
@@ -1,69 +1,24 @@
-# Copyright (C) 2018 the Software Heritage developers
+# Copyright (C) 2018-2021 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from contextlib import contextmanager
-from datetime import datetime
-
from celery import shared_task
-from swh.lister.npm.lister import NpmIncrementalLister, NpmLister
-from swh.lister.npm.models import NpmVisitModel
-
-
-@contextmanager
-def save_registry_state(lister):
- params = {"headers": lister.request_headers()}
- registry_state = lister.session.get(lister.url, **params)
- registry_state = registry_state.json()
- keys = (
- "doc_count",
- "doc_del_count",
- "update_seq",
- "purge_seq",
- "disk_size",
- "data_size",
- "committed_update_seq",
- "compacted_seq",
- )
-
- state = {key: registry_state[key] for key in keys}
- state["visit_date"] = datetime.now()
- yield
- npm_visit = NpmVisitModel(**state)
- lister.db_session.add(npm_visit)
- lister.db_session.commit()
-
-
-def get_last_update_seq(lister):
- """Get latest ``update_seq`` value for listing only updated packages.
- """
- query = lister.db_session.query(NpmVisitModel.update_seq)
- row = query.order_by(NpmVisitModel.uid.desc()).first()
- if not row:
- raise ValueError(
- "No npm registry listing previously performed ! "
- "This is required prior to the execution of an "
- "incremental listing."
- )
- return row[0]
+from swh.lister.npm.lister import NpmLister
@shared_task(name=__name__ + ".NpmListerTask")
def list_npm_full(**lister_args):
"Full lister for the npm (javascript) registry"
- lister = NpmLister(**lister_args)
- with save_registry_state(lister):
- return lister.run()
+ lister = NpmLister.from_configfile(incremental=False, **lister_args)
+ return lister.run().dict()
@shared_task(name=__name__ + ".NpmIncrementalListerTask")
def list_npm_incremental(**lister_args):
"Incremental lister for the npm (javascript) registry"
- lister = NpmIncrementalLister(**lister_args)
- update_seq_start = get_last_update_seq(lister)
- with save_registry_state(lister):
- return lister.run(min_bound=update_seq_start)
+ lister = NpmLister.from_configfile(incremental=True, **lister_args)
+ return lister.run().dict()
@shared_task(name=__name__ + ".ping")
diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py
--- a/swh/lister/npm/tests/conftest.py
+++ b/swh/lister/npm/tests/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,18 +9,3 @@
@pytest.fixture
def lister_under_test():
return "npm"
-
-
-@pytest.fixture
-def lister_npm(swh_lister):
- # Add the load-npm in the scheduler backend
- swh_lister.scheduler.create_task_type(
- {
- "type": "load-npm",
- "description": "Load npm package",
- "backend_name": "swh.loader.package.tasks.LoadNpm",
- "default_interval": "1 day",
- }
- )
-
- return swh_lister
diff --git a/swh/lister/npm/tests/data/api_empty_response.json b/swh/lister/npm/tests/data/api_empty_response.json
deleted file mode 100644
--- a/swh/lister/npm/tests/data/api_empty_response.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 839080,
- "rows": []
-}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/api_inc_empty_response.json b/swh/lister/npm/tests/data/api_inc_empty_response.json
deleted file mode 100644
--- a/swh/lister/npm/tests/data/api_inc_empty_response.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "results": [],
- "last_seq": 6927821
-}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/api_inc_response.json b/swh/lister/npm/tests/data/api_inc_response.json
deleted file mode 100644
--- a/swh/lister/npm/tests/data/api_inc_response.json
+++ /dev/null
@@ -1,906 +0,0 @@
-{
- "results": [
- {
- "seq": 6920644,
- "id": "electron-scripts",
- "changes": [
- {
- "rev": "3-a19944df5a3636bb225af9e0c8f9eedc"
- }
- ]
- },
- {
- "seq": 6920649,
- "id": "@crexi-dev/schematics",
- "changes": [
- {
- "rev": "3-00188360eeca1f9123b2d7cd4b468c50"
- }
- ]
- },
- {
- "seq": 6920651,
- "id": "botfactory-conversation",
- "changes": [
- {
- "rev": "50-f3667cde87637505528c46adc87f44e3"
- }
- ]
- },
- {
- "seq": 6920667,
- "id": "castle",
- "changes": [
- {
- "rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0"
- }
- ]
- },
- {
- "seq": 6920671,
- "id": "rbc-wm-framework-vuejs",
- "changes": [
- {
- "rev": "111-32bed479afacdd88aed9ac16dd135843"
- }
- ]
- },
- {
- "seq": 6920678,
- "id": "bitcoinfiles",
- "changes": [
- {
- "rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0"
- }
- ]
- },
- {
- "seq": 6920679,
- "id": "jovo-core",
- "changes": [
- {
- "rev": "2-d7440f1d17823e1a0760d9b3d4537c6e"
- }
- ]
- },
- {
- "seq": 6920687,
- "id": "jovo-framework",
- "changes": [
- {
- "rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9"
- }
- ]
- },
- {
- "seq": 6920690,
- "id": "smart-form-lib",
- "changes": [
- {
- "rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade"
- }
- ]
- },
- {
- "seq": 6920694,
- "id": "bokehjs",
- "changes": [
- {
- "rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2"
- }
- ]
- },
- {
- "seq": 6920701,
- "id": "guijarro",
- "changes": [
- {
- "rev": "14-82ece581d6a35d4e1d78e5292ca245c0"
- }
- ]
- },
- {
- "seq": 6920702,
- "id": "@kava-labs/crypto-rate-utils",
- "changes": [
- {
- "rev": "3-cecc6a6c226a0590b1a685e3041028c6"
- }
- ]
- },
- {
- "seq": 6920703,
- "id": "@riouxjean/test",
- "changes": [
- {
- "rev": "10-01e97dc7d0241dc49ea93b3468ec7b29"
- }
- ]
- },
- {
- "seq": 6920704,
- "id": "react-scrabblefy",
- "changes": [
- {
- "rev": "7-970c8206f3b8744204f7dcb106f8462b"
- }
- ]
- },
- {
- "seq": 6920706,
- "id": "molart",
- "changes": [
- {
- "rev": "14-416cd3cec62dd46f9b59a3bbe35308f6"
- }
- ]
- },
- {
- "seq": 6920707,
- "id": "@universal-material/angular",
- "changes": [
- {
- "rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3"
- }
- ]
- },
- {
- "seq": 6920708,
- "id": "cozy-doctypes",
- "changes": [
- {
- "rev": "68-8e90cc26e25da6c9430d373e43ac3c25"
- }
- ]
- },
- {
- "seq": 6920710,
- "id": "2o3t-ui",
- "changes": [
- {
- "rev": "96-1e65d5320ea7c78525aba5daf328bd4b"
- }
- ]
- },
- {
- "seq": 6920712,
- "id": "ark-ts",
- "changes": [
- {
- "rev": "24-033183c2f7f9cbb6e44d553213e525b6"
- }
- ]
- },
- {
- "seq": 6920715,
- "id": "mysqlconnector",
- "changes": [
- {
- "rev": "19-f09bc0b82281ca486db5ebe83843679e"
- }
- ]
- },
- {
- "seq": 6920716,
- "id": "@innovexa/ng-form-creator-lib",
- "changes": [
- {
- "rev": "147-480665ee17fa889dfec1aee75b907ff2"
- }
- ]
- },
- {
- "seq": 6920717,
- "id": "k-routes-example-basic",
- "changes": [
- {
- "rev": "1-35142059e1c63cc724da71a9eebf229c"
- }
- ]
- },
- {
- "seq": 6920718,
- "id": "wloggertojs",
- "changes": [
- {
- "rev": "29-5b5aa74bd30ff0fc86b39fba799befe2"
- }
- ]
- },
- {
- "seq": 6920720,
- "id": "wloggertofile",
- "changes": [
- {
- "rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5"
- }
- ]
- },
- {
- "seq": 6920721,
- "id": "@brightcove/flashls",
- "changes": [
- {
- "rev": "62-fbadb49476a58e98f0f136c86b614734"
- }
- ]
- },
- {
- "seq": 6920722,
- "id": "@brightcove/hls-fetcher",
- "changes": [
- {
- "rev": "76-3341ed8ade38f3251a97c94c3a7af5ac"
- }
- ]
- },
- {
- "seq": 6920723,
- "id": "@brightcove/kacl",
- "changes": [
- {
- "rev": "33-d0bc6b639cccb301086114d548ecfdbf"
- }
- ]
- },
- {
- "seq": 6920724,
- "id": "just-in-types",
- "changes": [
- {
- "rev": "2-fc329aa885dc795aee340f36ec60f333"
- }
- ]
- },
- {
- "seq": 6920725,
- "id": "@brightcove/player-loader",
- "changes": [
- {
- "rev": "56-9ff5aebc9743a44d46c182746313877d"
- }
- ]
- },
- {
- "seq": 6920726,
- "id": "@brightcove/player-loader-webpack-plugin",
- "changes": [
- {
- "rev": "33-db8b4d6765f19e475e1c1d16843824cb"
- }
- ]
- },
- {
- "seq": 6920727,
- "id": "@brightcove/player-url",
- "changes": [
- {
- "rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc"
- }
- ]
- },
- {
- "seq": 6920728,
- "id": "@brightcove/react-player-loader",
- "changes": [
- {
- "rev": "39-b7bf609de666ec7e71f517db53ab9c0a"
- }
- ]
- },
- {
- "seq": 6920729,
- "id": "vscode-theme-generator",
- "changes": [
- {
- "rev": "21-bcb92281d6f7e37548bb18113681df88"
- }
- ]
- },
- {
- "seq": 6920733,
- "id": "@brightcove/typed-immutable-extensions",
- "changes": [
- {
- "rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe"
- }
- ]
- },
- {
- "seq": 6920734,
- "id": "@brightcove/typed-immutable-proptypes",
- "changes": [
- {
- "rev": "27-e4802afc947c55d34f778864476c17e4"
- }
- ]
- },
- {
- "seq": 6920737,
- "id": "@brightcove/videojs-flashls-source-handler",
- "changes": [
- {
- "rev": "59-faf69c49be866b2ab7faa7be9972e7a5"
- }
- ]
- },
- {
- "seq": 6920738,
- "id": "@brightcove/videojs-flashls-swf",
- "changes": [
- {
- "rev": "60-04908466eaac2194bc3061e91f463dab"
- }
- ]
- },
- {
- "seq": 6920739,
- "id": "@noqcks/generated",
- "changes": [
- {
- "rev": "2-e07d07614182d4beccc507ca199e612d"
- }
- ]
- },
- {
- "seq": 6920740,
- "id": "pkcs7",
- "changes": [
- {
- "rev": "60-65ba116f3b6b705f472971b5c6a8f8d2"
- }
- ]
- },
- {
- "seq": 6920741,
- "id": "videojs-errors",
- "changes": [
- {
- "rev": "57-c999abd162ca4b93412e363443aa688a"
- }
- ]
- },
- {
- "seq": 6920742,
- "id": "videojs-flashls-source-handler",
- "changes": [
- {
- "rev": "59-46d62e18971a8c800710a8fbf985c1c5"
- }
- ]
- },
- {
- "seq": 6920743,
- "id": "videojs-playlist",
- "changes": [
- {
- "rev": "97-d4b3492a94c1084c272162dd51901188"
- }
- ]
- },
- {
- "seq": 6920745,
- "id": "videojs-playlist-ui",
- "changes": [
- {
- "rev": "95-ba97c44c354b2262e639f8c515bed9bc"
- }
- ]
- },
- {
- "seq": 6920746,
- "id": "fusion-apollo-universal-client",
- "changes": [
- {
- "rev": "25-7123042a477cec67c7d5fc702254c7a3"
- }
- ]
- },
- {
- "seq": 6920749,
- "id": "msg-fabric-core",
- "changes": [
- {
- "rev": "20-17c33e06faca357526c7395aca1113d2"
- }
- ]
- },
- {
- "seq": 6920750,
- "id": "@expo/schemer",
- "changes": [
- {
- "rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d"
- }
- ]
- },
- {
- "seq": 6920752,
- "id": "mathjs",
- "changes": [
- {
- "rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18"
- }
- ]
- },
- {
- "seq": 6920758,
- "id": "statesauce-ui",
- "changes": [
- {
- "rev": "6-db9a39366c1a082c56a2212e368e3ae2"
- }
- ]
- },
- {
- "seq": 6920782,
- "id": "@catchandrelease/arbor",
- "changes": [
- {
- "rev": "19-925648432b398ecadc98993e6fba2353"
- }
- ]
- },
- {
- "seq": 6920784,
- "id": "discover-shared-ebsco-ui-core",
- "changes": [
- {
- "rev": "4-277063cbc6b71f969e5f0db8c371db65"
- }
- ]
- },
- {
- "seq": 6920807,
- "id": "react-apexcharts",
- "changes": [
- {
- "rev": "13-18505be8026a50390c1ff1ba522cb9bd"
- }
- ]
- },
- {
- "seq": 6920819,
- "id": "zigbee-shepherd-converters",
- "changes": [
- {
- "rev": "90-5819692a5a9679ff8669fb410e190515"
- }
- ]
- },
- {
- "seq": 6920835,
- "id": "honeycomb-grid",
- "changes": [
- {
- "rev": "36-edd6733c80b04a72600558dc55348c73"
- }
- ]
- },
- {
- "seq": 6920838,
- "id": "pixl-config",
- "changes": [
- {
- "rev": "7-5dd2b68d04fefb4039b3965b3497eda2"
- }
- ]
- },
- {
- "seq": 6920842,
- "id": "discover-shared-ebsco-ui-theming",
- "changes": [
- {
- "rev": "4-e9d083825b1eae46f28c4def2d0db79f"
- }
- ]
- },
- {
- "seq": 6920843,
- "id": "common-oxgalaxy-lengua-app",
- "changes": [
- {
- "rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f"
- }
- ]
- },
- {
- "seq": 6920845,
- "id": "discover-shared-ebsco-ui-grid",
- "changes": [
- {
- "rev": "2-6f71cf625a5232075071952b2adaa8f2"
- }
- ]
- },
- {
- "seq": 6920847,
- "id": "@auth0/cosmos-tokens",
- "changes": [
- {
- "rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb"
- }
- ]
- },
- {
- "seq": 6920848,
- "id": "@auth0/babel-preset-cosmos",
- "changes": [
- {
- "rev": "43-d05d3779db08f08726ba048da298e046"
- }
- ]
- },
- {
- "seq": 6920849,
- "id": "jsrender",
- "changes": [
- {
- "rev": "11-c949091592b3329d73ae564e45a3472d"
- }
- ]
- },
- {
- "seq": 6920850,
- "id": "discover-shared-ebsco-ui-container",
- "changes": [
- {
- "rev": "2-c32089f76b7f253bc0d765da8b9f670d"
- }
- ]
- },
- {
- "seq": 6920852,
- "id": "@auth0/cosmos",
- "changes": [
- {
- "rev": "42-5fdaf3d9063c20dac13dcf455c42773c"
- }
- ]
- },
- {
- "seq": 6920853,
- "id": "discover-shared-ebsco-ui-checkbox",
- "changes": [
- {
- "rev": "2-06d9521b86f0dbf4a398726faead1212"
- }
- ]
- },
- {
- "seq": 6920854,
- "id": "@adunigan/toggles",
- "changes": [
- {
- "rev": "1-c2a830cf814a9fe2d72084339c9c5d28"
- }
- ]
- },
- {
- "seq": 6920855,
- "id": "@spriteful/spriteful-lazy-carousel",
- "changes": [
- {
- "rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837"
- }
- ]
- },
- {
- "seq": 6920856,
- "id": "react-modal-hook",
- "changes": [
- {
- "rev": "2-364b39d6559364c41d5b852ccad4ce31"
- }
- ],
- "deleted": true
- },
- {
- "seq": 6920859,
- "id": "@bellese/angular-design-system",
- "changes": [
- {
- "rev": "39-3e297f85ce2d6a6b6d15fc26420fc471"
- }
- ]
- },
- {
- "seq": 6920861,
- "id": "@uifabric/styling",
- "changes": [
- {
- "rev": "229-addf6cc0e74a335125c04d60047353f5"
- }
- ]
- },
- {
- "seq": 6920862,
- "id": "@uifabric/file-type-icons",
- "changes": [
- {
- "rev": "37-8a7e43399d1bb9f17334b10995f78df4"
- }
- ]
- },
- {
- "seq": 6920864,
- "id": "throttlewrap",
- "changes": [
- {
- "rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa"
- }
- ]
- },
- {
- "seq": 6920865,
- "id": "airtable",
- "changes": [
- {
- "rev": "16-d8aee935f6fa4c88057d75a0542bc58c"
- }
- ]
- },
- {
- "seq": 6920866,
- "id": "@csmart/ngc-smart-address",
- "changes": [
- {
- "rev": "19-66a6ea868aae1912952f232d2c699f3a"
- }
- ]
- },
- {
- "seq": 6920868,
- "id": "office-ui-fabric-react",
- "changes": [
- {
- "rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461"
- }
- ]
- },
- {
- "seq": 6920869,
- "id": "@fuelrats/eslint-config",
- "changes": [
- {
- "rev": "12-1b4c71b78fd078e3c1cba535e8541bed"
- }
- ]
- },
- {
- "seq": 6920870,
- "id": "@uifabric/date-time",
- "changes": [
- {
- "rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed"
- }
- ]
- },
- {
- "seq": 6920872,
- "id": "dark-client",
- "changes": [
- {
- "rev": "11-a954c2a89a130ae73f064233d9b3bce2"
- }
- ]
- },
- {
- "seq": 6920873,
- "id": "@uifabric/variants",
- "changes": [
- {
- "rev": "59-391c720194c663b9a5c59fe2c10a1535"
- }
- ]
- },
- {
- "seq": 6920875,
- "id": "discover-shared-ebsco-ui-header",
- "changes": [
- {
- "rev": "2-efd8f0426a83422a6c8b7bff11054c72"
- }
- ]
- },
- {
- "seq": 6920876,
- "id": "react-responsive-picture",
- "changes": [
- {
- "rev": "14-32a6d0850c8af33412cfdb23afd2ecfa"
- }
- ]
- },
- {
- "seq": 6920877,
- "id": "@uifabric/fluent-theme",
- "changes": [
- {
- "rev": "16-39c29e00b81a0b654213a5a50d7e7f42"
- }
- ]
- },
- {
- "seq": 6920878,
- "id": "@uifabric/dashboard",
- "changes": [
- {
- "rev": "82-04d6dc25b33e811c1d8c24566127b09c"
- }
- ]
- },
- {
- "seq": 6920879,
- "id": "ids-enterprise",
- "changes": [
- {
- "rev": "201-dd709a3912f9832440320d448850b61a"
- }
- ]
- },
- {
- "seq": 6920880,
- "id": "@uifabric/experiments",
- "changes": [
- {
- "rev": "224-efd1ef07f7640952c286488eae282367"
- }
- ]
- },
- {
- "seq": 6920881,
- "id": "@fuelrats/eslint-config-react",
- "changes": [
- {
- "rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc"
- }
- ]
- },
- {
- "seq": 6920883,
- "id": "jsviews",
- "changes": [
- {
- "rev": "11-44d8bedffc98cf6ac4aa669ba8844746"
- }
- ]
- },
- {
- "seq": 6920885,
- "id": "pixl-server",
- "changes": [
- {
- "rev": "15-823f4598c3354500d8d2a266dd062502"
- }
- ]
- },
- {
- "seq": 6920887,
- "id": "@rrpm/netlify-cms-core",
- "changes": [
- {
- "rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa"
- }
- ]
- },
- {
- "seq": 6920889,
- "id": "lodash-a",
- "changes": [
- {
- "rev": "2-6ee66153dbe611a080b40775387d2d45"
- }
- ]
- },
- {
- "seq": 6920891,
- "id": "meshcentral",
- "changes": [
- {
- "rev": "499-6677ca74525ed2aa77644c68001382fe"
- }
- ]
- },
- {
- "seq": 6920892,
- "id": "vue-transition-collection",
- "changes": [
- {
- "rev": "2-0510ee52c014c0d3b1e65f24376d76f0"
- }
- ]
- },
- {
- "seq": 6920894,
- "id": "fury-adapter-swagger",
- "changes": [
- {
- "rev": "47-09f0c55d8574d654c67f9244c21d7ef7"
- }
- ]
- },
- {
- "seq": 6920895,
- "id": "@isobar-us/redux-form-gen",
- "changes": [
- {
- "rev": "30-70d7d9210264a321092c832063934648"
- }
- ]
- },
- {
- "seq": 6920896,
- "id": "atomizer",
- "changes": [
- {
- "rev": "19-129774900cb2a67a46871cc2c40c34d3"
- }
- ]
- },
- {
- "seq": 6920904,
- "id": "boom-js-client",
- "changes": [
- {
- "rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9"
- }
- ]
- },
- {
- "seq": 6920905,
- "id": "@ts-common/json-parser",
- "changes": [
- {
- "rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3"
- }
- ]
- },
- {
- "seq": 6920906,
- "id": "rutt",
- "changes": [
- {
- "rev": "13-78aab849cb00a6ef7ebc8165770b7d33"
- }
- ]
- },
- {
- "seq": 6920907,
- "id": "linear-react-components-ui",
- "changes": [
- {
- "rev": "171-0307f1d69843b270e687371c67cbd1b0"
- }
- ]
- },
- {
- "seq": 6920908,
- "id": "@earnest/eslint-config",
- "changes": [
- {
- "rev": "180-b5250dd803102cf7dbac8da9c1a403fd"
- }
- ]
- },
- {
- "seq": 6920909,
- "id": "@earnest/eslint-config-es7",
- "changes": [
- {
- "rev": "181-da26885e0baacaea95814857f459572d"
- }
- ]
- },
- {
- "seq": 6920910,
- "id": "fuse-design",
- "changes": [
- {
- "rev": "10-e2b78592872f680c05e55eb5b81a0cab"
- }
- ]
- }
- ],
- "last_seq": 6920912
-}
\ No newline at end of file
diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001"
deleted file mode 100644
--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001"
+++ /dev/null
@@ -1,807 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 422482,
- "rows": [
-
- {
- "id": "jquery",
- "key": "jquery",
- "value": {
- "rev": "212-2eac7c93af4c8bccdf7317739f0319b6"
- }
- },
-
- {
- "id": "jquery-1.8",
- "key": "jquery-1.8",
- "value": {
- "rev": "1-711ded49a7453adce85ce7a51c2157de"
- }
- },
-
- {
- "id": "jquery-1x",
- "key": "jquery-1x",
- "value": {
- "rev": "1-c53fa04d9c8fb231336704508732c287"
- }
- },
-
- {
- "id": "jquery-2-typescript-async-await-adapter",
- "key": "jquery-2-typescript-async-await-adapter",
- "value": {
- "rev": "8-5cfb484e9afaa6e326a97240fccd8f93"
- }
- },
-
- {
- "id": "jquery-accessible-accordion-aria",
- "key": "jquery-accessible-accordion-aria",
- "value": {
- "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e"
- }
- },
-
- {
- "id": "jquery-accessible-autocomplete-list-aria",
- "key": "jquery-accessible-autocomplete-list-aria",
- "value": {
- "rev": "8-961b382442c1a5bafe58f0e05424701d"
- }
- },
-
- {
- "id": "jquery-accessible-carrousel-aria",
- "key": "jquery-accessible-carrousel-aria",
- "value": {
- "rev": "9-f33f59d7f601bafe023bd711b551282b"
- }
- },
-
- {
- "id": "jquery-accessible-dialog-tooltip-aria",
- "key": "jquery-accessible-dialog-tooltip-aria",
- "value": {
- "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba"
- }
- },
-
- {
- "id": "jquery-accessible-hide-show-aria",
- "key": "jquery-accessible-hide-show-aria",
- "value": {
- "rev": "10-5a03c47a8995b08246e4bc103782dafa"
- }
- },
-
- {
- "id": "jquery-accessible-modal-window-aria",
- "key": "jquery-accessible-modal-window-aria",
- "value": {
- "rev": "18-50266e260f6b807019cfcfcd3a3685ab"
- }
- },
-
- {
- "id": "jquery-accessible-simple-tooltip-aria",
- "key": "jquery-accessible-simple-tooltip-aria",
- "value": {
- "rev": "6-ea71aa37760790dc603b56117f054e1b"
- }
- },
-
- {
- "id": "jquery-accessible-subnav-dropdown",
- "key": "jquery-accessible-subnav-dropdown",
- "value": {
- "rev": "2-496f017a9ac243655225e43b5697b09b"
- }
- },
-
- {
- "id": "jquery-accessible-tabpanel-aria",
- "key": "jquery-accessible-tabpanel-aria",
- "value": {
- "rev": "11-659971471e6ac0fbb3b2f78ad208722a"
- }
- },
-
- {
- "id": "jquery-accessible-tabs-umd",
- "key": "jquery-accessible-tabs-umd",
- "value": {
- "rev": "1-f92015de5bb36e411d8c0940cca2883f"
- }
- },
-
- {
- "id": "jquery-active-descendant",
- "key": "jquery-active-descendant",
- "value": {
- "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de"
- }
- },
-
- {
- "id": "jquery-ada-validation",
- "key": "jquery-ada-validation",
- "value": {
- "rev": "1-9aab9629027c29fbece90485dd9d3112"
- }
- },
-
- {
- "id": "jquery-adaptText",
- "key": "jquery-adaptText",
- "value": {
- "rev": "3-2e15fc801ea8235b9180a3defc782ed0"
- }
- },
-
- {
- "id": "jquery-adapttr",
- "key": "jquery-adapttr",
- "value": {
- "rev": "6-74585f2d4be60b3f493585a6d28b90bc"
- }
- },
-
- {
- "id": "jquery-add-prefixed-class",
- "key": "jquery-add-prefixed-class",
- "value": {
- "rev": "1-9e43aee9758504b3f5271e9804a95f20"
- }
- },
-
- {
- "id": "jquery-address",
- "key": "jquery-address",
- "value": {
- "rev": "1-64173ede32157b26f4de910ad0f49590"
- }
- },
-
- {
- "id": "jquery-address-suggestion",
- "key": "jquery-address-suggestion",
- "value": {
- "rev": "6-18d9df51d472c365bcd84a61c9105774"
- }
- },
-
- {
- "id": "jquery-advscrollevent",
- "key": "jquery-advscrollevent",
- "value": {
- "rev": "1-f6033de9ba0f8e364c42826441d93119"
- }
- },
-
- {
- "id": "jquery-affix",
- "key": "jquery-affix",
- "value": {
- "rev": "6-777371f67df59abf18ec1fe326df3b82"
- }
- },
-
- {
- "id": "jquery-airload",
- "key": "jquery-airload",
- "value": {
- "rev": "7-136d513d2604a25238eb88709d6d9003"
- }
- },
-
- {
- "id": "jquery-ajax",
- "key": "jquery-ajax",
- "value": {
- "rev": "1-ee358f630d4c928b52c968c7667d0d31"
- }
- },
-
- {
- "id": "jquery-ajax-cache",
- "key": "jquery-ajax-cache",
- "value": {
- "rev": "2-ca31e0d43ae28e9cea968f1f538f06d3"
- }
- },
-
- {
- "id": "jquery-ajax-chain",
- "key": "jquery-ajax-chain",
- "value": {
- "rev": "1-dc0e5aee651c0128b7f411aac96132a2"
- }
- },
-
- {
- "id": "jquery-ajax-file-upload",
- "key": "jquery-ajax-file-upload",
- "value": {
- "rev": "1-96147d8bf69245c622e76583bb615d49"
- }
- },
-
- {
- "id": "jquery-ajax-json",
- "key": "jquery-ajax-json",
- "value": {
- "rev": "1-b47eec12168e4cb39b45f1523d7cd397"
- }
- },
-
- {
- "id": "jquery-ajax-markup",
- "key": "jquery-ajax-markup",
- "value": {
- "rev": "1-8e65dc822cb63be76c62a1323666265e"
- }
- },
-
- {
- "id": "jquery-ajax-native",
- "key": "jquery-ajax-native",
- "value": {
- "rev": "2-9d67b8d43713e3546ad50f817c040139"
- }
- },
-
- {
- "id": "jquery-ajax-request",
- "key": "jquery-ajax-request",
- "value": {
- "rev": "1-fdc0960ec73667bc2b46adf493c05db4"
- }
- },
-
- {
- "id": "jquery-ajax-retry",
- "key": "jquery-ajax-retry",
- "value": {
- "rev": "1-27ca186953e346aa9c0ca2310c732751"
- }
- },
-
- {
- "id": "jquery-ajax-tracking",
- "key": "jquery-ajax-tracking",
- "value": {
- "rev": "3-d48876f3c115ee4743a6a94bb65bb01d"
- }
- },
-
- {
- "id": "jquery-ajax-transport-xdomainrequest",
- "key": "jquery-ajax-transport-xdomainrequest",
- "value": {
- "rev": "1-ece69aa5b9f0c950a1fa2806cf74392d"
- }
- },
-
- {
- "id": "jquery-ajax-unobtrusive",
- "key": "jquery-ajax-unobtrusive",
- "value": {
- "rev": "3-fb0daab8480b9a2cc9c6876e1c4874f4"
- }
- },
-
- {
- "id": "jquery-ajax-unobtrusive-multi",
- "key": "jquery-ajax-unobtrusive-multi",
- "value": {
- "rev": "1-0a2ffdabaf5708d4ae3d9e29a3a9ef11"
- }
- },
-
- {
- "id": "jquery-ajaxreadystate",
- "key": "jquery-ajaxreadystate",
- "value": {
- "rev": "1-5e618474fe2e77ad5869c206164f82bf"
- }
- },
-
- {
- "id": "jquery-albe-timeline",
- "key": "jquery-albe-timeline",
- "value": {
- "rev": "2-3db2b43778b5c50db873e724d9940eb6"
- }
- },
-
- {
- "id": "jquery-all-attributes",
- "key": "jquery-all-attributes",
- "value": {
- "rev": "1-89bb7e01ee312ad5d36d78a3aa2327e4"
- }
- },
-
- {
- "id": "jquery-alphaindex",
- "key": "jquery-alphaindex",
- "value": {
- "rev": "4-7f61cde9cfb70617a6fbe992dfcbc10a"
- }
- },
-
- {
- "id": "jquery-always",
- "key": "jquery-always",
- "value": {
- "rev": "1-0ad944881bbc39c67df0a694d80bebef"
- }
- },
-
- {
- "id": "jquery-amd",
- "key": "jquery-amd",
- "value": {
- "rev": "1-931646c751bef740c361dd0f6e68653c"
- }
- },
-
- {
- "id": "jquery-anaglyph-image-effect",
- "key": "jquery-anaglyph-image-effect",
- "value": {
- "rev": "1-9bf7afce2e1bc73747ef22abc859b22b"
- }
- },
-
- {
- "id": "jquery-analytics",
- "key": "jquery-analytics",
- "value": {
- "rev": "1-d84b0c8ce886b9f01d2c5c1cf0a7317f"
- }
- },
-
- {
- "id": "jquery-ancestors",
- "key": "jquery-ancestors",
- "value": {
- "rev": "1-49b30817a03558f1f585c8c0cd4b8afb"
- }
- },
-
- {
- "id": "jquery-angry-loader",
- "key": "jquery-angry-loader",
- "value": {
- "rev": "1-31c9fd950d32b9d3a73829cde1dae577"
- }
- },
-
- {
- "id": "jquery-angular-shim",
- "key": "jquery-angular-shim",
- "value": {
- "rev": "1-723e72b2981f02dd3abcfe6d2395d636"
- }
- },
-
- {
- "id": "jquery-animate-gradient",
- "key": "jquery-animate-gradient",
- "value": {
- "rev": "5-a3e0fc89699237e7e7241cd608a0dcf7"
- }
- },
-
- {
- "id": "jquery-animate-scroll",
- "key": "jquery-animate-scroll",
- "value": {
- "rev": "1-37d49d89fe99aa599540e6ff83b15888"
- }
- },
-
- {
- "id": "jquery-animated-headlines",
- "key": "jquery-animated-headlines",
- "value": {
- "rev": "1-adf1d149bc83fa8445e141e3c900759e"
- }
- },
-
- {
- "id": "jquery-animation",
- "key": "jquery-animation",
- "value": {
- "rev": "4-f51d0559010bbe9d74d70e58de9bd733"
- }
- },
-
- {
- "id": "jquery-animation-support",
- "key": "jquery-animation-support",
- "value": {
- "rev": "1-9013bc4bdeb2bd70bedcc988a811fcc0"
- }
- },
-
- {
- "id": "jquery-aniview",
- "key": "jquery-aniview",
- "value": {
- "rev": "3-5754524da237693458bcff19b626b875"
- }
- },
-
- {
- "id": "jquery-anything-clickable",
- "key": "jquery-anything-clickable",
- "value": {
- "rev": "2-e1aaaf1a369f7796c438a3efbf05bcce"
- }
- },
-
- {
- "id": "jquery-app",
- "key": "jquery-app",
- "value": {
- "rev": "6-4e0bf5abd71c72ced3c4cf3035116f70"
- }
- },
-
- {
- "id": "jquery-app-banner",
- "key": "jquery-app-banner",
- "value": {
- "rev": "2-8a5b530eaab94315eb00c77acd13f2dd"
- }
- },
-
- {
- "id": "jquery-appear-poetic",
- "key": "jquery-appear-poetic",
- "value": {
- "rev": "1-368094b72ed36d42cf2fca438fa4b344"
- }
- },
-
- {
- "id": "jquery-applyonscreen",
- "key": "jquery-applyonscreen",
- "value": {
- "rev": "4-d76c18a6e66fffba01a9a774b40663f8"
- }
- },
-
- {
- "id": "jquery-apta",
- "key": "jquery-apta",
- "value": {
- "rev": "1-c486380fedefd887e6293a00c3b6a222"
- }
- },
-
- {
- "id": "jquery-arrow-navigate",
- "key": "jquery-arrow-navigate",
- "value": {
- "rev": "3-0efe881e01ef0eac24a92baf1eb6d8d1"
- }
- },
-
- {
- "id": "jquery-asAccordion",
- "key": "jquery-asAccordion",
- "value": {
- "rev": "2-2d18d3fe9089dcf67de5f29d1763b4ce"
- }
- },
-
- {
- "id": "jquery-asBgPicker",
- "key": "jquery-asBgPicker",
- "value": {
- "rev": "2-d1403cd306d5764ee0f5aa852c2bed8e"
- }
- },
-
- {
- "id": "jquery-asBreadcrumbs",
- "key": "jquery-asBreadcrumbs",
- "value": {
- "rev": "2-77e566a07680005ce1cb322f2a733fe4"
- }
- },
-
- {
- "id": "jquery-asCheck",
- "key": "jquery-asCheck",
- "value": {
- "rev": "2-d0b2741b70616c7d563419cc125d193d"
- }
- },
-
- {
- "id": "jquery-asChoice",
- "key": "jquery-asChoice",
- "value": {
- "rev": "2-0eda5269cbd59976ee904b74da209389"
- }
- },
-
- {
- "id": "jquery-asColor",
- "key": "jquery-asColor",
- "value": {
- "rev": "3-aa730d81322561c7a3174d5c7bb6b3b8"
- }
- },
-
- {
- "id": "jquery-asColorPicker",
- "key": "jquery-asColorPicker",
- "value": {
- "rev": "2-6bbaecaf94a324331a3d1f5d3aad3b3d"
- }
- },
-
- {
- "id": "jquery-asDropdown",
- "key": "jquery-asDropdown",
- "value": {
- "rev": "2-b29b187cdd0bdce502d11855415e6887"
- }
- },
-
- {
- "id": "jquery-asFontEditor",
- "key": "jquery-asFontEditor",
- "value": {
- "rev": "2-132882375101062896413afdc93b4c8c"
- }
- },
-
- {
- "id": "jquery-asGalleryPicker",
- "key": "jquery-asGalleryPicker",
- "value": {
- "rev": "1-864a80930d72c6150aa74969a28617e4"
- }
- },
-
- {
- "id": "jquery-asGmap",
- "key": "jquery-asGmap",
- "value": {
- "rev": "2-b0c4330774137b2f1b91bd4686880f2a"
- }
- },
-
- {
- "id": "jquery-asGradient",
- "key": "jquery-asGradient",
- "value": {
- "rev": "2-5184670a313d5e161cb62659de3db55c"
- }
- },
-
- {
- "id": "jquery-asHoverScroll",
- "key": "jquery-asHoverScroll",
- "value": {
- "rev": "7-3f6efebf248bd27520d03eaac33d8ca2"
- }
- },
-
- {
- "id": "jquery-asIconPicker",
- "key": "jquery-asIconPicker",
- "value": {
- "rev": "2-9070adda148ea75247c7cee810ae91e2"
- }
- },
-
- {
- "id": "jquery-asImagePicker",
- "key": "jquery-asImagePicker",
- "value": {
- "rev": "2-fb3115c2296b0b07ed9e379176626e01"
- }
- },
-
- {
- "id": "jquery-asItemList",
- "key": "jquery-asItemList",
- "value": {
- "rev": "2-88a7d2900f47c785c2a6cb764ac467d6"
- }
- },
-
- {
- "id": "jquery-asModal",
- "key": "jquery-asModal",
- "value": {
- "rev": "2-1719b8e6a489e03cc3e22bd329148366"
- }
- },
-
- {
- "id": "jquery-asOffset",
- "key": "jquery-asOffset",
- "value": {
- "rev": "2-e45a0077e5bc0bbf91b32dc76387c945"
- }
- },
-
- {
- "id": "jquery-asPaginator",
- "key": "jquery-asPaginator",
- "value": {
- "rev": "2-0d279d2748fc5e875f5fb2a8d3d48377"
- }
- },
-
- {
- "id": "jquery-asPieProgress",
- "key": "jquery-asPieProgress",
- "value": {
- "rev": "2-14dc464a19e9d3feaa532f62e45bbd26"
- }
- },
-
- {
- "id": "jquery-asProgress",
- "key": "jquery-asProgress",
- "value": {
- "rev": "2-a58d7100f1a78f7753efcf0e34dfaf0e"
- }
- },
-
- {
- "id": "jquery-asRange",
- "key": "jquery-asRange",
- "value": {
- "rev": "3-aa3d2f348a933161868ba6b6fd9eb881"
- }
- },
-
- {
- "id": "jquery-asScroll",
- "key": "jquery-asScroll",
- "value": {
- "rev": "1-f4880ea057adbfebb912ba0157575ca1"
- }
- },
-
- {
- "id": "jquery-asScrollable",
- "key": "jquery-asScrollable",
- "value": {
- "rev": "7-5c18eb2180d8aa85f0b5e940667c8344"
- }
- },
-
- {
- "id": "jquery-asScrollbar",
- "key": "jquery-asScrollbar",
- "value": {
- "rev": "4-89420658c355a5584825b45ee4ef0beb"
- }
- },
-
- {
- "id": "jquery-asSelect",
- "key": "jquery-asSelect",
- "value": {
- "rev": "2-caf3dc516665009b654236b876fe02bb"
- }
- },
-
- {
- "id": "jquery-asSpinner",
- "key": "jquery-asSpinner",
- "value": {
- "rev": "2-bf26b5d9c77eb4b63acbf16019407834"
- }
- },
-
- {
- "id": "jquery-asSwitch",
- "key": "jquery-asSwitch",
- "value": {
- "rev": "2-f738586946b432caa73297568b5f38ad"
- }
- },
-
- {
- "id": "jquery-asTooltip",
- "key": "jquery-asTooltip",
- "value": {
- "rev": "2-80d3fe5cdae70d9310969723e7045384"
- }
- },
-
- {
- "id": "jquery-asTree",
- "key": "jquery-asTree",
- "value": {
- "rev": "2-353063a563c0322cbc317af385f71b27"
- }
- },
-
- {
- "id": "jquery-ascolorpicker-flat",
- "key": "jquery-ascolorpicker-flat",
- "value": {
- "rev": "11-1681d53cd475e7b6b9564baa51a79611"
- }
- },
-
- {
- "id": "jquery-aslider",
- "key": "jquery-aslider",
- "value": {
- "rev": "1-2b3dd953493eeaa4dc329cbf0d81116a"
- }
- },
-
- {
- "id": "jquery-aspect-ratio-keeper",
- "key": "jquery-aspect-ratio-keeper",
- "value": {
- "rev": "1-1ad8e5588218e1d38fff351858655eda"
- }
- },
-
- {
- "id": "jquery-assinadordigitaldiscus",
- "key": "jquery-assinadordigitaldiscus",
- "value": {
- "rev": "1-897cd68ef3699551630bd3454dceb6f0"
- }
- },
-
- {
- "id": "jquery-async-gravatar",
- "key": "jquery-async-gravatar",
- "value": {
- "rev": "3-a3192e741d14d57635f4ebfb41a904db"
- }
- },
-
- {
- "id": "jquery-asynclink",
- "key": "jquery-asynclink",
- "value": {
- "rev": "1-2159a3c49e3c8fe9280c592770e83522"
- }
- },
-
- {
- "id": "jquery-atlas",
- "key": "jquery-atlas",
- "value": {
- "rev": "1-6142c5a0af67a0470daf36151d3f9d8c"
- }
- },
-
- {
- "id": "jquery-atomic-nav",
- "key": "jquery-atomic-nav",
- "value": {
- "rev": "1-18e4ef14be83a907cbee0cd0adee25d4"
- }
- },
-
- {
- "id": "jquery-attach",
- "key": "jquery-attach",
- "value": {
- "rev": "8-da4f17596c25a02b0cce266e59706d5f"
- }
- }
-
- ]
-}
diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11"
deleted file mode 100644
--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11"
+++ /dev/null
@@ -1,83 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 422482,
- "rows": [
- {
- "id": "jquery",
- "key": "jquery",
- "value": {
- "rev": "212-2eac7c93af4c8bccdf7317739f0319b6"
- }
- },
- {
- "id": "jquery-1.8",
- "key": "jquery-1.8",
- "value": {
- "rev": "1-711ded49a7453adce85ce7a51c2157de"
- }
- },
- {
- "id": "jquery-1x",
- "key": "jquery-1x",
- "value": {
- "rev": "1-c53fa04d9c8fb231336704508732c287"
- }
- },
- {
- "id": "jquery-2-typescript-async-await-adapter",
- "key": "jquery-2-typescript-async-await-adapter",
- "value": {
- "rev": "8-5cfb484e9afaa6e326a97240fccd8f93"
- }
- },
- {
- "id": "jquery-accessible-accordion-aria",
- "key": "jquery-accessible-accordion-aria",
- "value": {
- "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e"
- }
- },
- {
- "id": "jquery-accessible-autocomplete-list-aria",
- "key": "jquery-accessible-autocomplete-list-aria",
- "value": {
- "rev": "8-961b382442c1a5bafe58f0e05424701d"
- }
- },
- {
- "id": "jquery-accessible-carrousel-aria",
- "key": "jquery-accessible-carrousel-aria",
- "value": {
- "rev": "9-f33f59d7f601bafe023bd711b551282b"
- }
- },
- {
- "id": "jquery-accessible-dialog-tooltip-aria",
- "key": "jquery-accessible-dialog-tooltip-aria",
- "value": {
- "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba"
- }
- },
- {
- "id": "jquery-accessible-hide-show-aria",
- "key": "jquery-accessible-hide-show-aria",
- "value": {
- "rev": "10-5a03c47a8995b08246e4bc103782dafa"
- }
- },
- {
- "id": "jquery-accessible-modal-window-aria",
- "key": "jquery-accessible-modal-window-aria",
- "value": {
- "rev": "18-50266e260f6b807019cfcfcd3a3685ab"
- }
- },
- {
- "id": "jquery-accessible-simple-tooltip-aria",
- "key": "jquery-accessible-simple-tooltip-aria",
- "value": {
- "rev": "6-ea71aa37760790dc603b56117f054e1b"
- }
- }
- ]
-}
diff --git a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" b/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11"
deleted file mode 100644
--- a/"a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11"
+++ /dev/null
@@ -1,83 +0,0 @@
-{
- "total_rows": 839080,
- "offset": 422482,
- "rows": [
- {
- "id": "jquery-accessible-simple-tooltip-aria",
- "key": "jquery-accessible-simple-tooltip-aria",
- "value": {
- "rev": "6-ea71aa37760790dc603b56117f054e1b"
- }
- },
- {
- "id": "jquery-accessible-subnav-dropdown",
- "key": "jquery-accessible-subnav-dropdown",
- "value": {
- "rev": "2-496f017a9ac243655225e43b5697b09b"
- }
- },
- {
- "id": "jquery-accessible-tabpanel-aria",
- "key": "jquery-accessible-tabpanel-aria",
- "value": {
- "rev": "11-659971471e6ac0fbb3b2f78ad208722a"
- }
- },
- {
- "id": "jquery-accessible-tabs-umd",
- "key": "jquery-accessible-tabs-umd",
- "value": {
- "rev": "1-f92015de5bb36e411d8c0940cca2883f"
- }
- },
- {
- "id": "jquery-active-descendant",
- "key": "jquery-active-descendant",
- "value": {
- "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de"
- }
- },
- {
- "id": "jquery-ada-validation",
- "key": "jquery-ada-validation",
- "value": {
- "rev": "1-9aab9629027c29fbece90485dd9d3112"
- }
- },
- {
- "id": "jquery-adaptText",
- "key": "jquery-adaptText",
- "value": {
- "rev": "3-2e15fc801ea8235b9180a3defc782ed0"
- }
- },
- {
- "id": "jquery-adapttr",
- "key": "jquery-adapttr",
- "value": {
- "rev": "6-74585f2d4be60b3f493585a6d28b90bc"
- }
- },
- {
- "id": "jquery-add-prefixed-class",
- "key": "jquery-add-prefixed-class",
- "value": {
- "rev": "1-9e43aee9758504b3f5271e9804a95f20"
- }
- },
- {
- "id": "jquery-address",
- "key": "jquery-address",
- "value": {
- "rev": "1-64173ede32157b26f4de910ad0f49590"
- }
- },
- {
- "id": "jquery-address-suggestion",
- "key": "jquery-address-suggestion",
- "value": {
- "rev": "6-18d9df51d472c365bcd84a61c9105774"
- }
- }
- ]
-}
diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json
deleted file mode 120000
--- a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json
+++ /dev/null
@@ -1 +0,0 @@
-_all_docs,startkey="",limit=1001
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_full_page1.json b/swh/lister/npm/tests/data/npm_full_page1.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_full_page1.json
@@ -0,0 +1,235 @@
+{
+ "total_rows": 1496481,
+ "offset": 0,
+ "rows": [{
+ "id": "-",
+ "key": "-",
+ "value": {
+ "rev": "1-e340e64b090ea8c6b4e14edc0460c751"
+ },
+ "doc": {
+ "_id": "-",
+ "_rev": "1-e340e64b090ea8c6b4e14edc0460c751",
+ "name": "-",
+ "dist-tags": {
+ "latest": "0.0.1"
+ },
+ "versions": {
+ "0.0.1": {
+ "name": "-",
+ "version": "0.0.1",
+ "license": "UNLICENSED",
+ "keywords": [],
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1",
+ "start": "node --require ts-node/register src",
+ "build": "tsc"
+ },
+ "devDependencies": {
+ "@types/node": "13.9.0",
+ "ts-node": "8.6.2",
+ "typescript": "3.8.3"
+ },
+ "description": "> Created using https://github.com/parzh/create-package-typescript",
+ "_id": "-@0.0.1",
+ "_nodeVersion": "13.3.0",
+ "_npmVersion": "6.14.4",
+ "dist": {
+ "integrity": "sha512-3HfneK3DGAm05fpyj20sT3apkNcvPpCuccOThOPdzz8sY7GgQGe0l93XH9bt+YzibcTIgUAIMoyVJI740RtgyQ==",
+ "shasum": "db6db7cd866142880dd03e5b8781d1b4fac0e5bd",
+ "tarball": "https://registry.npmjs.org/-/-/--0.0.1.tgz",
+ "fileCount": 3,
+ "unpackedSize": 600,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJehdEkCRA9TVsSAnZWagAAqcEQAIUCCWB2RzNV2p5CTeWf\nntknmFRbGh3K/gptyBnTC3MaNtk54Vf36XK1BfKgKz2rd/FOq1xcSQ8XUWt6\n7ZL9zSR3K/EBEgNr6p00fJAJ7ZdYu2kqq+uiZDKpSHLd6/bKaSdt0bAL78co\nmnzKOn9WULS1UC/ac8txNX9QdPg/aJezzxQadtKQR+SX6Fd/tTHKIsWv3ytI\nUtH/Lg9AM/tdTc94m2Ivo3+JxmkU+p8z7hjR97ym+AaZE8g1a47begVsSeZ/\nrIu1ZFCQ/XYFp0amNZ131qnW6FGnzpPlQTfkuJ1sPIORM9+o+6Ql6qa0mtde\nSJyAMKFfemmwkp56aZkL5NFX1snyJE4itfg99gsUrogHZIW3Jd1LqUe9yFZc\nVik1NZxqG4x6ktoog9IzG8ElOBWauKQAT+FIgNGea6IBv4wPQ8VoNOd4JKW/\nadvzus0hAdEZLyVsCyDw4wDCAK9JJXnSlJmJFc/vPtgPrbAj8xcBW6XIn6Nw\n0EVm993zlKmQ6/VmQUbHx8EP1zBm7M0vTcoI8dPIJkjGCZgWU571JARCetb2\npob+siYon3ndpoHG2OOWOuoRhwi4YeDW1ilc9isPve+S/iygKnfY8LTqHN1P\nOntz9Cu9YxHQ3WEiBuun7KP7FECbiSkozmzKWLHUhs2qXFSFYWeBLEI/HS1y\nJ9QO\r\n=lB+Q\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "parzhitsky",
+ "email": "parzhitsky@gmail.com"
+ }],
+ "_npmUser": {
+ "name": "parzhitsky",
+ "email": "parzhitsky@gmail.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-_0.0.1_1585828132159_0.6185838126718755"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-04-02T11:48:52.159Z",
+ "0.0.1": "2020-04-02T11:48:52.339Z",
+ "modified": "2020-04-02T11:49:00.248Z"
+ },
+ "maintainers": [{
+ "name": "parzhitsky",
+ "email": "parzhitsky@gmail.com"
+ }],
+ "description": "> Created using https://github.com/parzh/create-package-typescript",
+ "keywords": [],
+ "license": "UNLICENSED",
+ "readme": "# `-`\r\n\r\n> Created using https://github.com/parzh/create-package-typescript\r\n\r\n```\r\nnpm i -\r\n```\r\n",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "id": "--file--1",
+ "key": "--file--1",
+ "value": {
+ "rev": "1-b23c0a584f925336378c29df0b858aff"
+ },
+ "doc": {
+ "_id": "--file--1",
+ "_rev": "1-b23c0a584f925336378c29df0b858aff",
+ "name": "--file--1",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "--file--1",
+ "version": "1.0.0",
+ "description": "",
+ "main": "file.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "_id": "--file--1@1.0.0",
+ "_nodeVersion": "15.3.0",
+ "_npmVersion": "7.0.14",
+ "dist": {
+ "integrity": "sha512-OtGGxxqxDl1KCCUts3fN0gL4vW28u8WNIo+eqQON78IVxKJS3gYchaEZDrRubC4SyQOhDDKDtvswwLOkApuicQ==",
+ "shasum": "58ed9d9169197ded653d8df7153968d22c14ea59",
+ "tarball": "https://registry.npmjs.org/--file--1/-/--file--1-1.0.0.tgz",
+ "fileCount": 6,
+ "unpackedSize": 1327,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf2AGJCRA9TVsSAnZWagAALqcP+wZ9tzFsdQy4cEb5HGV6\nxV0LTnW+wxFYBUm2IfOwK7UH/Pl5Nym4yxlIjMEBinb9efepUAsucCyAka/v\n0OBGn+8I8kHDKvfhxiUVbi3CFAYcQTaaPffK5xJZi77netdsaLyS0cdx4RBm\nEWRIVK4axJBSUT6UVpmRqB73OlAv+JmNzBmL+CXHn9gksyvnm74pbT8tukLV\nX0TwkXNBtzn4lmydPRyecEZRDFydzp9UBQJyJACtpYxgTAwTT5ENtSymx4Ql\nlEqC9cqT9Q3n7z8lvntaCb6jgNN1aSWP3hOKCYQzezk5uSacJDWJOGAK7DE5\ndKUqsWa0HG84H+MMLxuepbrNYM5Fm7ulykl1yMrAf1G8T5pU/fUpq5uEHbqW\naVbW3GlCxJuUvlaTTFjGqDJx4tepY+Uel+TQigJoN2DGXTWiB+JeAAWLP+cu\nqcoZwkqoISmgru2g1+6TR//hod3CnCK4eQl/ph3VthtKyh4d1zUQAPcCGax6\nocJSWQsLRNW0QUmvpHeHQQOUGEb0XDTYkl9i8s8DY6JCqfgl9UIDZzNl2nK8\n385JseJKKLQR3tEk+HLZM2UGF6XgACBFTHYxlOrpzGgX+PJ1ns3noMrKo2jO\nbnvMN6Mp0GtoSzQj8FUcoU6BgOFHhHkB/i4Ke2VVeMYVj/CrFO13pZJOPR4O\n6uts\r\n=W7vP\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "_npmUser": {
+ "name": "guosuzhen",
+ "email": "1077655229@qq.com"
+ },
+ "directories": {},
+ "maintainers": [{
+ "name": "guosuzhen",
+ "email": "1077655229@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/--file--1_1.0.0_1607991689533_0.6331916469772794"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-12-15T00:21:29.533Z",
+ "1.0.0": "2020-12-15T00:21:29.657Z",
+ "modified": "2020-12-15T00:21:32.116Z"
+ },
+ "maintainers": [{
+ "name": "guosuzhen",
+ "email": "1077655229@qq.com"
+ }],
+ "keywords": [],
+ "license": "ISC",
+ "readme": "ERROR: No README data found!",
+ "readmeFilename": ""
+ }
+ },
+ {
+ "id": "-keyboardevent",
+ "key": "-keyboardevent",
+ "value": {
+ "rev": "1-0fdff278618c4f8c1026889f1b525483"
+ },
+ "doc": {
+ "_id": "-keyboardevent",
+ "_rev": "1-0fdff278618c4f8c1026889f1b525483",
+ "name": "-keyboardevent",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "-keyboardevent",
+ "version": "1.0.0",
+ "description": "监听键盘事件",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "license": "ISO",
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3",
+ "_id": "-keyboardevent@1.0.0",
+ "_nodeVersion": "12.10.0",
+ "_npmVersion": "6.10.3",
+ "dist": {
+ "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==",
+ "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5",
+ "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz",
+ "fileCount": 3,
+ "unpackedSize": 2714,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "_npmUser": {
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-09-25T06:10:56.357Z",
+ "1.0.0": "2020-09-25T06:10:56.513Z",
+ "modified": "2020-09-25T06:10:58.774Z"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "description": "监听键盘事件",
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "license": "ISO",
+ "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n",
+ "readmeFilename": "README.md"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_full_page2.json b/swh/lister/npm/tests/data/npm_full_page2.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_full_page2.json
@@ -0,0 +1,168 @@
+{
+ "total_rows": 1496482,
+ "offset": 2,
+ "rows": [{
+ "id": "-keyboardevent",
+ "key": "-keyboardevent",
+ "value": {
+ "rev": "1-0fdff278618c4f8c1026889f1b525483"
+ },
+ "doc": {
+ "_id": "-keyboardevent",
+ "_rev": "1-0fdff278618c4f8c1026889f1b525483",
+ "name": "-keyboardevent",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "-keyboardevent",
+ "version": "1.0.0",
+ "description": "监听键盘事件",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "license": "ISO",
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3",
+ "_id": "-keyboardevent@1.0.0",
+ "_nodeVersion": "12.10.0",
+ "_npmVersion": "6.10.3",
+ "dist": {
+ "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==",
+ "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5",
+ "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz",
+ "fileCount": 3,
+ "unpackedSize": 2714,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "_npmUser": {
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-09-25T06:10:56.357Z",
+ "1.0.0": "2020-09-25T06:10:56.513Z",
+ "modified": "2020-09-25T06:10:58.774Z"
+ },
+ "maintainers": [{
+ "name": "yanggw",
+ "email": "2993065178@qq.com"
+ }],
+ "description": "监听键盘事件",
+ "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git"
+ },
+ "author": {
+ "name": "yanggw"
+ },
+ "bugs": {
+ "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues"
+ },
+ "license": "ISO",
+ "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "id": "-lidonghui",
+ "key": "-lidonghui",
+ "value": {
+ "rev": "1-1aa03a85878671cdb74b789c2b124578"
+ },
+ "doc": {
+ "_id": "-lidonghui",
+ "_rev": "1-1aa03a85878671cdb74b789c2b124578",
+ "name": "-lidonghui",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "-lidonghui",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "bin": {
+ "test1-cli": "bin/index.js"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "axios": "^0.19.2",
+ "commander": "^5.0.0",
+ "inquirer": "^7.1.0"
+ },
+ "_id": "-lidonghui@1.0.0",
+ "_nodeVersion": "12.14.0",
+ "_npmVersion": "6.13.4",
+ "dist": {
+ "integrity": "sha512-1eUTEDU1kgAAsQ85uCEAp9tDsEIh3WNa5dB/ks8Uhf9Nh2RR37EeTGyxszn2A1B+MN+Q0fWrpCfBFzrrFyLFnA==",
+ "shasum": "be7ffedc328bd123ba13c468a89ebcf0cfe76777",
+ "tarball": "https://registry.npmjs.org/-lidonghui/-/-lidonghui-1.0.0.tgz",
+ "fileCount": 2,
+ "unpackedSize": 1024,
+ "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJenVH+CRA9TVsSAnZWagAAEIMP/ipex7Yo+T2LqfWtFuZz\n5NuXhcnaNyANXyJ7iQCYzq1ynq+tkwBOTM7FNLkTbIiIUsn8d6JlAt5CfIFZ\nrfaYDK7BhU4E8KfLoSC0VqyaprTSvAeqeQDEAguYM9Nbt4TWa7W2HoorgQd7\njUAuGpRqvkw8Omh3eOvhqEa0Y5mkRf853ndJHD425bAww2R0KCA7RuOBO1MW\nnPZqBe3XOgJmsyEW+yl2tSGVq2hEEeD3C0G9x5K2ZfxZKVU7lGfqRlUdo7na\nTLJU6YZPTB9AhDTM9b8XFUMWlEtvHcwvGvuB+gLupXqWwEfwSqWg/2TJ11AQ\nMBFeC1xSav0BCv6kFxFBRwz1k7IAdVYatLnmqwNsIGwjWsyfmIThSNzYTIq6\nE/E0iD2gSAsLW5r7U0t+rk4sWLmu4xbLDvT1xpEY7C00Zj4NXJhdXSo5BR45\n1IBiU96LqEA2bIw+TsyaFO7w/chgpvd8mU0g34tYQCcAts56ZhCr43x5kUSY\ns7hm1b3MB5qpOBbom1Ta24xjvoJwxv2Et/T7zfZoXzHDyX9BgUCiVIvupxd9\nzR5LdyXIh6JdiVBAgEftbm59FEVrxVfAAi/sFMNJKp8pE1ELrSp/gulzxDd2\nC5Hq1z5RNFRQUlbkMJHLIDOZLAvdkPKieE+uFJ7WTYCPEL9nvQoIJiO9UFH1\njKtU\r\n=Opv1\r\n-----END PGP SIGNATURE-----\r\n"
+ },
+ "maintainers": [{
+ "name": "tenderli",
+ "email": "l2369660482@163.com"
+ }],
+ "_npmUser": {
+ "name": "tenderli",
+ "email": "l2369660482@163.com"
+ },
+ "directories": {},
+ "_npmOperationalInternal": {
+ "host": "s3://npm-registry-packages",
+ "tmp": "tmp/-lidonghui_1.0.0_1587368445998_0.028137547296155807"
+ },
+ "_hasShrinkwrap": false
+ }
+ },
+ "time": {
+ "created": "2020-04-20T07:40:45.997Z",
+ "1.0.0": "2020-04-20T07:40:46.087Z",
+ "modified": "2020-04-20T07:40:48.525Z"
+ },
+ "maintainers": [{
+ "name": "tenderli",
+ "email": "l2369660482@163.com"
+ }],
+ "keywords": [],
+ "license": "ISC",
+ "readme": "ERROR: No README data found!",
+ "readmeFilename": ""
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_incremental_page1.json b/swh/lister/npm/tests/data/npm_incremental_page1.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_incremental_page1.json
@@ -0,0 +1,175 @@
+{
+ "results": [{
+ "seq": 1,
+ "id": "overlay-fishnet",
+ "changes": [{
+ "rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f"
+ }],
+ "doc": {
+ "_id": "overlay-fishnet",
+ "_rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f",
+ "name": "overlay-fishnet",
+ "description": "Overlay polygon with equal sized tiles (like a fishnet)",
+ "dist-tags": {
+ "latest": "0.0.0"
+ },
+ "versions": {
+ "0.0.0": {
+ "name": "overlay-fishnet",
+ "version": "0.0.0",
+ "description": "Overlay polygon with equal sized tiles (like a fishnet)",
+ "main": "dist/index.js",
+ "scripts": {
+ "build": "rm -rf dist && mkdir dist && babel src --out-dir dist",
+ "prepublish": "npm run test && npm run build",
+ "test": "standard"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/yongjun21/fishnet.git"
+ },
+ "keywords": [
+ "geospatial",
+ "polygon",
+ "tiles",
+ "choropleth"
+ ],
+ "author": {
+ "name": "Yong Jun"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/yongjun21/fishnet/issues"
+ },
+ "homepage": "https://github.com/yongjun21/fishnet#readme",
+ "dependencies": {
+ "babel-cli": "^6.18.0",
+ "babel-preset-es2015": "^6.18.0",
+ "babel-preset-stage-3": "^6.17.0",
+ "standard": "^8.6.0"
+ },
+ "gitHead": "bc3d598445a767efefd094260c83d6f57f06e86d",
+ "_id": "overlay-fishnet@0.0.0",
+ "_shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0",
+ "_from": ".",
+ "_npmVersion": "3.10.7",
+ "_nodeVersion": "6.2.2",
+ "_npmUser": {
+ "name": "yongjun21",
+ "email": "yjthong.2009@smu.edu.sg"
+ },
+ "dist": {
+ "shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0",
+ "tarball": "https://registry.npmjs.org/overlay-fishnet/-/overlay-fishnet-0.0.0.tgz"
+ },
+ "maintainers": [{
+ "name": "yongjun21",
+ "email": "yjthong.2009@smu.edu.sg"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/overlay-fishnet-0.0.0.tgz_1484192243466_0.4572450101841241"
+ }
+ }
+ },
+ "readme": "# fishnet\nOverlay polygon with equal sized tiles (like a fishnet)\n",
+ "maintainers": [{
+ "name": "yongjun21",
+ "email": "yjthong.2009@smu.edu.sg"
+ }],
+ "time": {
+ "modified": "2017-01-12T03:37:23.696Z",
+ "created": "2017-01-12T03:37:23.696Z",
+ "0.0.0": "2017-01-12T03:37:23.696Z"
+ },
+ "homepage": "https://github.com/yongjun21/fishnet#readme",
+ "keywords": [
+ "geospatial",
+ "polygon",
+ "tiles",
+ "choropleth"
+ ],
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/yongjun21/fishnet.git"
+ },
+ "author": {
+ "name": "Yong Jun"
+ },
+ "bugs": {
+ "url": "https://github.com/yongjun21/fishnet/issues"
+ },
+ "license": "ISC",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "seq": 2,
+ "id": "shopping_site_detect",
+ "changes": [{
+ "rev": "1-c62a780cdd9865b06b1e4eb0e6552560"
+ }],
+ "doc": {
+ "_id": "shopping_site_detect",
+ "_rev": "1-c62a780cdd9865b06b1e4eb0e6552560",
+ "name": "shopping_site_detect",
+ "description": "shopping site detect",
+ "dist-tags": {
+ "latest": "1.0.0"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "shopping_site_detect",
+ "version": "1.0.0",
+ "description": "shopping site detect",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": {
+ "name": "Jayin Ton"
+ },
+ "license": "MIT",
+ "gitHead": "223c4a7e12acf28310a106780652d1e7d0629d68",
+ "_id": "shopping_site_detect@1.0.0",
+ "_shasum": "0408f054053d7581869d42934e42d50794d20b8c",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.4.0",
+ "_npmUser": {
+ "name": "jayinton",
+ "email": "tonjayin@gmail.com"
+ },
+ "dist": {
+ "shasum": "0408f054053d7581869d42934e42d50794d20b8c",
+ "tarball": "https://registry.npmjs.org/shopping_site_detect/-/shopping_site_detect-1.0.0.tgz"
+ },
+ "maintainers": [{
+ "name": "jayinton",
+ "email": "tonjayin@gmail.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/shopping_site_detect-1.0.0.tgz_1484193398924_0.3402463826350868"
+ }
+ }
+ },
+ "maintainers": [{
+ "name": "jayinton",
+ "email": "tonjayin@gmail.com"
+ }],
+ "time": {
+ "modified": "2017-01-12T03:56:39.161Z",
+ "created": "2017-01-12T03:56:39.161Z",
+ "1.0.0": "2017-01-12T03:56:39.161Z"
+ },
+ "author": {
+ "name": "Jayin Ton"
+ },
+ "license": "MIT",
+ "readmeFilename": "readme.md"
+ }
+ }
+ ],
+ "last_seq": 2
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/data/npm_incremental_page2.json b/swh/lister/npm/tests/data/npm_incremental_page2.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/npm/tests/data/npm_incremental_page2.json
@@ -0,0 +1,632 @@
+{
+ "results": [{
+ "seq": 3,
+ "id": "tinyviewpager",
+ "changes": [{
+ "rev": "1-01e65a5602e9c43ed3482657f1309ef0"
+ }],
+ "doc": {
+ "_id": "tinyviewpager",
+ "_rev": "1-01e65a5602e9c43ed3482657f1309ef0",
+ "name": "tinyviewpager",
+ "time": {
+ "modified": "2017-01-12T04:05:34.847Z",
+ "created": "2017-01-11T10:52:41.047Z",
+ "1.0.0": "2017-01-11T10:52:41.047Z",
+ "1.0.1": "2017-01-11T14:02:03.232Z",
+ "1.0.2": "2017-01-11T14:31:04.820Z",
+ "1.0.3": "2017-01-11T14:38:41.688Z",
+ "1.0.4": "2017-01-11T14:45:57.970Z",
+ "1.0.5": "2017-01-11T14:49:04.778Z",
+ "1.0.6": "2017-01-11T14:54:06.876Z",
+ "1.0.7": "2017-01-11T15:07:13.271Z",
+ "1.0.8": "2017-01-12T04:05:34.847Z"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "dist-tags": {
+ "latest": "1.0.8"
+ },
+ "description": "上下左右无限循环viewpager",
+ "versions": {
+ "1.0.1": {
+ "name": "tinyviewpager",
+ "version": "1.0.1",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "fdb455a393c7d6b6d43d01f73362fa6e3ed0dbb9",
+ "_id": "tinyviewpager@1.0.1",
+ "_shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.1.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.1.tgz_1484143322968_0.5517262546345592"
+ }
+ },
+ "1.0.2": {
+ "name": "tinyviewpager",
+ "version": "1.0.2",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "daae30e6ce7bea30a14619b5cef5f52a86cc5528",
+ "_id": "tinyviewpager@1.0.2",
+ "_shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.2.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.2.tgz_1484145064595_0.49263988458551466"
+ }
+ },
+ "1.0.3": {
+ "name": "tinyviewpager",
+ "version": "1.0.3",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "d08aeab765b537d71ed21e7e37ae6546e9bdbef2",
+ "_id": "tinyviewpager@1.0.3",
+ "_shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.3.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.3.tgz_1484145521456_0.8925717421807349"
+ }
+ },
+ "1.0.4": {
+ "name": "tinyviewpager",
+ "version": "1.0.4",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react": "^15.4.2",
+ "react-native": "^0.40.0",
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.4",
+ "_shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.4.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.4.tgz_1484145957736_0.7274783558677882"
+ }
+ },
+ "1.0.5": {
+ "name": "tinyviewpager",
+ "version": "1.0.5",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.5",
+ "_shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.5.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.5.tgz_1484146144523_0.9442351886536926"
+ }
+ },
+ "1.0.6": {
+ "name": "tinyviewpager",
+ "version": "1.0.6",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.6",
+ "_shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.6.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.6.tgz_1484146444932_0.7940177812706679"
+ }
+ },
+ "1.0.7": {
+ "name": "tinyviewpager",
+ "version": "1.0.7",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "cde72131c7adac616b79032a588c09d814816266",
+ "_id": "tinyviewpager@1.0.7",
+ "_shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.7.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.7.tgz_1484147231500_0.17695364635437727"
+ }
+ },
+ "1.0.8": {
+ "name": "tinyviewpager",
+ "version": "1.0.8",
+ "description": "上下左右无限循环viewpager",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "keywords": [
+ "viewpager"
+ ],
+ "author": {
+ "name": "tinylpc"
+ },
+ "license": "ISC",
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "dependencies": {
+ "react-timer-mixin": "^0.13.3"
+ },
+ "gitHead": "54b9e9f1517a0385307065f96ef85e9b7b13868e",
+ "_id": "tinyviewpager@1.0.8",
+ "_shasum": "67234eebbae3a8205319c950fab62c8f84840061",
+ "_from": ".",
+ "_npmVersion": "3.10.3",
+ "_nodeVersion": "6.3.1",
+ "_npmUser": {
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ },
+ "dist": {
+ "shasum": "67234eebbae3a8205319c950fab62c8f84840061",
+ "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.8.tgz"
+ },
+ "maintainers": [{
+ "name": "tinylpc",
+ "email": "1150247879@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/tinyviewpager-1.0.8.tgz_1484193932800_0.371786616044119"
+ }
+ }
+ },
+ "homepage": "https://github.com/tinylpc/TinyViewPager#readme",
+ "keywords": [
+ "viewpager"
+ ],
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tinylpc/TinyViewPager.git"
+ },
+ "author": {
+ "name": "tinylpc"
+ },
+ "bugs": {
+ "url": "https://github.com/tinylpc/TinyViewPager/issues"
+ },
+ "license": "ISC",
+ "readmeFilename": "README.md"
+ }
+ },
+ {
+ "seq": 4,
+ "id": "ubase-ext-wecloud",
+ "changes": [{
+ "rev": "1-0a540144594976690ff540d57ed3ab3b"
+ }],
+ "doc": {
+ "_id": "ubase-ext-wecloud",
+ "_rev": "1-0a540144594976690ff540d57ed3ab3b",
+ "name": "ubase-ext-wecloud",
+ "dist-tags": {
+ "latest": "1.0.3"
+ },
+ "versions": {
+ "1.0.0": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "_id": "ubase-ext-wecloud@1.0.0",
+ "_shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.0.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.0.tgz_1484191505909_0.5115192488301545"
+ }
+ },
+ "1.0.1": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.1",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "_id": "ubase-ext-wecloud@1.0.1",
+ "_shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.1.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.1.tgz_1484191845449_0.817801987985149"
+ }
+ },
+ "1.0.2": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.2",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "_id": "ubase-ext-wecloud@1.0.2",
+ "_shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.2.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-18-east.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.2.tgz_1484192705755_0.1207962513435632"
+ }
+ },
+ "1.0.3": {
+ "name": "ubase-ext-wecloud",
+ "version": "1.0.3",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "copy-webpack-plugin": "^4.0.1",
+ "ubase-vue": "^1.1.14"
+ },
+ "gitHead": "a5b60823b706c4fd98ae798dbe76f18a7841eead",
+ "_id": "ubase-ext-wecloud@1.0.3",
+ "_shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b",
+ "_from": ".",
+ "_npmVersion": "3.8.6",
+ "_nodeVersion": "6.1.0",
+ "_npmUser": {
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ },
+ "dist": {
+ "shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b",
+ "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.3.tgz"
+ },
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "_npmOperationalInternal": {
+ "host": "packages-12-west.internal.npmjs.com",
+ "tmp": "tmp/ubase-ext-wecloud-1.0.3.tgz_1484193826093_0.8929786814842373"
+ }
+ }
+ },
+ "readme": "# ubase-ext-wecloud\n",
+ "maintainers": [{
+ "name": "askliujun",
+ "email": "547794529@qq.com"
+ }],
+ "time": {
+ "modified": "2017-01-12T04:03:46.329Z",
+ "created": "2017-01-12T03:25:07.858Z",
+ "1.0.0": "2017-01-12T03:25:07.858Z",
+ "1.0.1": "2017-01-12T03:30:45.704Z",
+ "1.0.2": "2017-01-12T03:45:07.635Z",
+ "1.0.3": "2017-01-12T04:03:46.329Z"
+ },
+ "license": "ISC",
+ "readmeFilename": "README.md"
+ }
+ }
+ ],
+ "last_seq": 4
+}
\ No newline at end of file
diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py
--- a/swh/lister/npm/tests/test_lister.py
+++ b/swh/lister/npm/tests/test_lister.py
@@ -1,97 +1,200 @@
-# Copyright (C) 2018-2019 The Software Heritage developers
+# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-import logging
-import re
-from typing import Any, List
-import unittest
+from itertools import chain
+import json
+from pathlib import Path
-import requests_mock
+import iso8601
+import pytest
+from requests.exceptions import HTTPError
-from swh.lister.core.tests.test_lister import HttpListerTesterBase
-from swh.lister.npm.lister import NpmIncrementalLister, NpmLister
+from swh.lister import USER_AGENT
+from swh.lister.npm.lister import NpmLister, NpmListerState
-logger = logging.getLogger(__name__)
+@pytest.fixture
+def npm_full_listing_page1(datadir):
+ return json.loads(Path(datadir, "npm_full_page1.json").read_text())
-class NpmListerTester(HttpListerTesterBase, unittest.TestCase):
- Lister = NpmLister
- test_re = re.compile(r'^.*/_all_docs\?startkey="(.+)".*')
- lister_subdir = "npm"
- good_api_response_file = "data/replicate.npmjs.com/api_response.json"
- bad_api_response_file = "data/api_empty_response.json"
- first_index = "jquery"
- entries_per_page = 100
- @requests_mock.Mocker()
- def test_is_within_bounds(self, http_mocker):
- # disable this test from HttpListerTesterBase as
- # it can not succeed for the npm lister due to the
- # overriding of the string_pattern_check method
- pass
+@pytest.fixture
+def npm_full_listing_page2(datadir):
+ return json.loads(Path(datadir, "npm_full_page2.json").read_text())
-class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase):
- Lister = NpmIncrementalLister
- test_re = re.compile(r"^.*/_changes\?since=([0-9]+).*")
- lister_subdir = "npm"
- good_api_response_file = "data/api_inc_response.json"
- bad_api_response_file = "data/api_inc_empty_response.json"
- first_index = "6920642"
- entries_per_page = 100
+@pytest.fixture
+def npm_incremental_listing_page1(datadir):
+ return json.loads(Path(datadir, "npm_incremental_page1.json").read_text())
- @requests_mock.Mocker()
- def test_is_within_bounds(self, http_mocker):
- # disable this test from HttpListerTesterBase as
- # it can not succeed for the npm lister due to the
- # overriding of the string_pattern_check method
- pass
+@pytest.fixture
+def npm_incremental_listing_page2(datadir):
+ return json.loads(Path(datadir, "npm_incremental_page2.json").read_text())
-def check_tasks(tasks: List[Any]):
- """Ensure scheduled tasks are in the expected format.
+def _check_listed_npm_packages(lister, packages, scheduler_origins):
+ for package in packages:
+ package_name = package["doc"]["name"]
+ latest_version = package["doc"]["dist-tags"]["latest"]
+ package_last_update = iso8601.parse_date(package["doc"]["time"][latest_version])
+ origin_url = lister.PACKAGE_URL_TEMPLATE.format(package_name=package_name)
- """
- for row in tasks:
- logger.debug("row: %s", row)
- assert row["type"] == "load-npm"
- # arguments check
- args = row["arguments"]["args"]
- assert len(args) == 0
+ scheduler_origin = [o for o in scheduler_origins if o.url == origin_url]
+ assert scheduler_origin
+ assert scheduler_origin[0].last_update == package_last_update
- # kwargs
- kwargs = row["arguments"]["kwargs"]
- assert len(kwargs) == 1
- package_url = kwargs["url"]
- package_name = package_url.split("/")[-1]
- assert package_url == f"https://www.npmjs.com/package/{package_name}"
- assert row["policy"] == "recurring"
- assert row["priority"] is None
+def _match_request(request):
+ return request.headers.get("User-Agent") == USER_AGENT
-def test_npm_lister_basic_listing(lister_npm, requests_mock_datadir):
- lister_npm.run()
+def _url_params(page_size, **kwargs):
+ params = {"limit": page_size, "include_docs": "true"}
+ params.update(**kwargs)
+ return params
- tasks = lister_npm.scheduler.search_tasks(task_type="load-npm")
- assert len(tasks) == 100
- check_tasks(tasks)
+def test_npm_lister_full(
+ swh_scheduler, requests_mock, mocker, npm_full_listing_page1, npm_full_listing_page2
+):
+ """Simulate a full listing of four npm packages in two pages"""
+ page_size = 2
+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=False)
+ requests_mock.get(
+ lister.API_FULL_LISTING_URL,
+ [{"json": npm_full_listing_page1}, {"json": npm_full_listing_page2},],
+ additional_matcher=_match_request,
+ )
+
+ spy_get = mocker.spy(lister.session, "get")
+
+ stats = lister.run()
+ assert stats.pages == 2
+ assert stats.origins == page_size * stats.pages
+
+ spy_get.assert_has_calls(
+ [
+ mocker.call(
+ lister.API_FULL_LISTING_URL,
+ params=_url_params(page_size + 1, startkey='""'),
+ ),
+ mocker.call(
+ lister.API_FULL_LISTING_URL,
+ params=_url_params(
+ page_size + 1,
+ startkey=f'"{npm_full_listing_page1["rows"][-1]["id"]}"',
+ ),
+ ),
+ ]
+ )
+
+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
+
+ _check_listed_npm_packages(
+ lister,
+ chain(npm_full_listing_page1["rows"][:-1], npm_full_listing_page2["rows"]),
+ scheduler_origins,
+ )
+
+ assert lister.get_state_from_scheduler() == NpmListerState()
+
+
+def test_npm_lister_incremental(
+ swh_scheduler,
+ requests_mock,
+ mocker,
+ npm_incremental_listing_page1,
+ npm_incremental_listing_page2,
+):
+ """Simulate an incremental listing of four npm packages in two pages"""
+ page_size = 2
+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True)
+
+ requests_mock.get(
+ lister.API_INCREMENTAL_LISTING_URL,
+ [
+ {"json": npm_incremental_listing_page1},
+ {"json": npm_incremental_listing_page2},
+ {"json": {"results": []}},
+ ],
+ additional_matcher=_match_request,
+ )
+
+ spy_get = mocker.spy(lister.session, "get")
+
+ assert lister.get_state_from_scheduler() == NpmListerState()
+
+ stats = lister.run()
+ assert stats.pages == 2
+ assert stats.origins == page_size * stats.pages
+
+ last_seq = npm_incremental_listing_page2["results"][-1]["seq"]
+
+ spy_get.assert_has_calls(
+ [
+ mocker.call(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(page_size, since="0"),
+ ),
+ mocker.call(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(
+ page_size,
+ since=str(npm_incremental_listing_page1["results"][-1]["seq"]),
+ ),
+ ),
+ mocker.call(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(page_size, since=str(last_seq)),
+ ),
+ ]
+ )
+
+ scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
-def test_npm_lister_listing_pagination(lister_npm, requests_mock_datadir):
- lister = lister_npm
- # Patch per page pagination
- lister.per_page = 10 + 1
- lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace(
- "&limit=1001", "&limit=%s" % lister.per_page
+ _check_listed_npm_packages(
+ lister,
+ chain(
+ npm_incremental_listing_page1["results"],
+ npm_incremental_listing_page2["results"],
+ ),
+ scheduler_origins,
)
+
+ assert lister.get_state_from_scheduler() == NpmListerState(last_seq=last_seq)
+
+
+def test_npm_lister_incremental_restart(
+ swh_scheduler, requests_mock, mocker,
+):
+ """Check incremental npm listing will restart from saved state"""
+ page_size = 2
+ last_seq = 67
+ lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True)
+ lister.state = NpmListerState(last_seq=last_seq)
+
+ requests_mock.get(lister.API_INCREMENTAL_LISTING_URL, json={"results": []})
+
+ spy_get = mocker.spy(lister.session, "get")
+
lister.run()
- tasks = lister.scheduler.search_tasks(task_type="load-npm")
- assert len(tasks) == 2 * 10 # only 2 files with 10 results each
+ spy_get.assert_called_with(
+ lister.API_INCREMENTAL_LISTING_URL,
+ params=_url_params(page_size, since=str(last_seq)),
+ )
+
+
+def test_npm_lister_http_error(
+ swh_scheduler, requests_mock, mocker,
+):
+ lister = NpmLister(scheduler=swh_scheduler)
+
+ requests_mock.get(lister.API_FULL_LISTING_URL, status_code=500)
- check_tasks(tasks)
+ with pytest.raises(HTTPError):
+ lister.run()
diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py
--- a/swh/lister/npm/tests/test_tasks.py
+++ b/swh/lister/npm/tests/test_tasks.py
@@ -1,15 +1,9 @@
-# Copyright (C) 2019-2020 The Software Heritage developers
+# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from contextlib import contextmanager
-from unittest.mock import patch
-
-
-@contextmanager
-def mock_save(lister):
- yield
+from swh.lister.pattern import ListerStats
def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
@@ -20,34 +14,31 @@
assert res.result == "OK"
-@patch("swh.lister.npm.tasks.save_registry_state")
-@patch("swh.lister.npm.tasks.NpmLister")
-def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker):
- # setup the mocked NpmLister
- lister.return_value = lister
- lister.run.return_value = None
- save.side_effect = mock_save
+def test_full_lister_task(
+ swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
+):
+ stats = ListerStats(pages=10, origins=900)
+ mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
+ mock_lister.from_configfile.return_value = mock_lister
+ mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask")
assert res
res.wait()
assert res.successful()
- lister.assert_called_once_with()
- lister.run.assert_called_once_with()
+ mock_lister.from_configfile.assert_called_once_with(incremental=False)
+ mock_lister.run.assert_called_once_with()
+ assert res.result == stats.dict()
-@patch("swh.lister.npm.tasks.save_registry_state")
-@patch("swh.lister.npm.tasks.get_last_update_seq")
-@patch("swh.lister.npm.tasks.NpmIncrementalLister")
-def test_incremental(
- lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker
+def test_incremental_lister_task(
+ swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
):
- # setup the mocked NpmLister
- lister.return_value = lister
- lister.run.return_value = None
- seq.return_value = 42
- save.side_effect = mock_save
+ stats = ListerStats(pages=10, origins=900)
+ mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister")
+ mock_lister.from_configfile.return_value = mock_lister
+ mock_lister.run.return_value = stats
res = swh_scheduler_celery_app.send_task(
"swh.lister.npm.tasks.NpmIncrementalListerTask"
@@ -56,6 +47,6 @@
res.wait()
assert res.successful()
- lister.assert_called_once_with()
- seq.assert_called_once_with(lister)
- lister.run.assert_called_once_with(min_bound=42)
+ mock_lister.from_configfile.assert_called_once_with(incremental=True)
+ mock_lister.run.assert_called_once_with()
+ assert res.result == stats.dict()

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 3:43 PM (2 w, 8 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220514

Event Timeline

About · Developer information · Legal