diff --git a/swh/lister/cli.py b/swh/lister/cli.py --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -78,8 +78,13 @@ elif lister == 'npm': from .npm.models import IndexingModelBase as ModelBase + from .npm.models import NpmVisitModel from .npm.lister import NpmLister _lister = NpmLister(override_config=override_conf) + if drop_tables: + NpmVisitModel.metadata.drop_all(_lister.db_engine) + if create_tables: + NpmVisitModel.metadata.create_all(_lister.db_engine) else: raise ValueError('Only supported listers are %s' % SUPPORTED_LISTERS) diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -9,20 +9,28 @@ from swh.scheduler.utils import create_task_dict -class NpmLister(SWHIndexingHttpLister): - """List all packages available in the npm registry in a paginated way. +class NpmListerBase(SWHIndexingHttpLister): + """List packages available in the npm registry in a paginated way """ - PATH_TEMPLATE = '/_all_docs?startkey="%s"' MODEL = NpmModel LISTER_NAME = 'npm' def __init__(self, api_baseurl='https://replicate.npmjs.com', - per_page=10000, override_config=None): + per_page=1000, override_config=None): super().__init__(api_baseurl=api_baseurl, override_config=override_config) self.per_page = per_page + 1 self.PATH_TEMPLATE += '&limit=%s' % self.per_page + @property + def ADDITIONAL_CONFIG(self): + """(Override) Add extra configuration + + """ + default_config = super().ADDITIONAL_CONFIG + default_config['loading_task_policy'] = ('str', 'oneshot') + return default_config + def get_model_from_repo(self, repo_name): """(Override) Transform from npm package name to model @@ -46,29 +54,14 @@ needed for the ingestion task creation. """ - _type = 'origin-update-%s' % origin_type - _policy = 'recurring' + task_type = 'origin-update-%s' % origin_type + task_policy = self.config['loading_task_policy'] package_name = kwargs.get('name') package_metadata_url = kwargs.get('html_url') - return create_task_dict(_type, _policy, package_name, origin_url, + return create_task_dict(task_type, task_policy, + package_name, origin_url, package_metadata_url=package_metadata_url) - def get_next_target_from_response(self, response): - """(Override) Get next npm package name to continue the listing - - """ - repos = response.json()['rows'] - return repos[-1]['id'] if len(repos) == self.per_page else None - - def transport_response_simplified(self, response): - """(Override) Transform npm registry response to list for model manipulation - - """ - repos = response.json()['rows'] - if len(repos) == self.per_page: - repos = repos[:-1] - return [self.get_model_from_repo(repo['id']) for repo in repos] - def request_headers(self): """(Override) Set requests headers to send when querying the npm registry @@ -92,3 +85,69 @@ of fixed length string pattern """ pass + + +class NpmLister(NpmListerBase): + """List all packages available in the npm registry in a paginated way + """ + PATH_TEMPLATE = '/_all_docs?startkey="%s"' + + def get_next_target_from_response(self, response): + """(Override) Get next npm package name to continue the listing + + """ + repos = response.json()['rows'] + return repos[-1]['id'] if len(repos) == self.per_page else None + + def transport_response_simplified(self, response): + """(Override) Transform npm registry response to list for model manipulation + + """ + repos = response.json()['rows'] + if len(repos) == self.per_page: + repos = repos[:-1] + return [self.get_model_from_repo(repo['id']) for repo in repos] + + +class NpmIncrementalLister(NpmListerBase): + """List packages in the npm registry, updated since a specific + update_seq value of the underlying CouchDB database, in a paginated way + """ + PATH_TEMPLATE = '/_changes?since=%s' + + def get_next_target_from_response(self, response): + """(Override) Get next npm package name to continue the listing + + """ + repos = response.json()['results'] + return repos[-1]['seq'] if len(repos) == self.per_page else None + + def transport_response_simplified(self, response): + """(Override) Transform npm registry response to list for model manipulation + + """ + repos = response.json()['results'] + if len(repos) == self.per_page: + repos = repos[:-1] + return [self.get_model_from_repo(repo['id']) for repo in repos] + + def filter_before_inject(self, models_list): + """(Override) Filter out documents in the CouchDB database + not related to a npm package + """ + models_filtered = [] + for model in models_list: + package_name = model['name'] + # document related to CouchDB internals + if package_name.startswith('_design/'): + continue + models_filtered.append(model) + return models_filtered + + def disable_deleted_repo_tasks(self, start, end, keep_these): + """(Override) Disable the processing performed by that method + as it is not relevant in this incremental lister context + and it raises and exception due to a different index type + (int instead of str) + """ + pass diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py --- a/swh/lister/npm/models.py +++ b/swh/lister/npm/models.py @@ -2,13 +2,34 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, String +from sqlalchemy import Column, String, DateTime, Integer, BigInteger, Sequence +from sqlalchemy.ext.declarative import declarative_base -from swh.lister.core.models import IndexingModelBase +from swh.lister.core.models import IndexingModelBase, ABCSQLMeta + +SQLBase = declarative_base() + + +class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta): + """Table to store the npm registry state at the time of a + content listing by Software Heritage + """ + __tablename__ = 'npm_visit' + + uid = Column(Integer, Sequence('npm_visit_id_seq'), primary_key=True) + visit_date = Column(DateTime, nullable=False) + doc_count = Column(BigInteger) + doc_del_count = Column(BigInteger) + update_seq = Column(BigInteger) + purge_seq = Column(BigInteger) + disk_size = Column(BigInteger) + data_size = Column(BigInteger) + committed_update_seq = Column(BigInteger) + compacted_seq = Column(BigInteger) class NpmModel(IndexingModelBase): - """a npm repository representation + """A npm package representation """ __tablename__ = 'npm_repo' diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py --- a/swh/lister/npm/tasks.py +++ b/swh/lister/npm/tasks.py @@ -2,19 +2,76 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from datetime import datetime + from swh.lister.core.tasks import ListerTaskBase -from swh.lister.npm.lister import NpmLister +from swh.lister.npm.lister import NpmLister, NpmIncrementalLister +from swh.lister.npm.models import NpmVisitModel -class NpmListerTask(ListerTaskBase): - """Full npm lister (list all available packages from the npm registry). +class _NpmListerTaskBase(ListerTaskBase): - """ task_queue = 'swh_lister_npm_refresh' + def _save_registry_state(self): + """Query the root endpoint from the npm registry and + backup values of interest for future listing + """ + params = {'headers': self.lister.request_headers()} + registry_state = \ + self.lister.session.get(self.lister.api_baseurl, **params) + registry_state = registry_state.json() + self.registry_state = { + 'visit_date': datetime.now(), + } + for key in ('doc_count', 'doc_del_count', 'update_seq', 'purge_seq', + 'disk_size', 'data_size', 'committed_update_seq', + 'compacted_seq'): + self.registry_state[key] = registry_state[key] + + def _store_registry_state(self): + """Store the backup npm registry state to database. + """ + npm_visit = NpmVisitModel(**self.registry_state) + self.lister.db_session.add(npm_visit) + self.lister.db_session.commit() + + +class NpmListerTask(_NpmListerTaskBase): + """Full npm lister (list all available packages from the npm registry) + """ + def new_lister(self): return NpmLister() def run_task(self): - lister = self.new_lister() - lister.run() + self.lister = self.new_lister() + self._save_registry_state() + self.lister.run() + self._store_registry_state() + + +class NpmIncrementalListerTask(_NpmListerTaskBase): + """Incremental npm lister (list all updated packages since the last listing) + """ + + def new_lister(self): + return NpmIncrementalLister() + + def run_task(self): + self.lister = self.new_lister() + update_seq_start = self._get_last_update_seq() + self._save_registry_state() + self.lister.run(min_bound=update_seq_start) + self._store_registry_state() + + def _get_last_update_seq(self): + """Get latest ``update_seq`` value for listing only updated packages. + """ + query = self.lister.db_session.query(NpmVisitModel.update_seq) + row = query.order_by(NpmVisitModel.uid.desc()).first() + if not row: + raise ValueError('No npm registry listing previously performed ! ' + 'This is required prior to the execution of an ' + 'incremental listing.') + return row[0] diff --git a/swh/lister/npm/tests/api_inc_empty_response.json b/swh/lister/npm/tests/api_inc_empty_response.json new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/api_inc_empty_response.json @@ -0,0 +1,4 @@ +{ + "results": [], + "last_seq": 6927821 +} \ No newline at end of file diff --git a/swh/lister/npm/tests/api_inc_response.json b/swh/lister/npm/tests/api_inc_response.json new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/api_inc_response.json @@ -0,0 +1,906 @@ +{ + "results": [ + { + "seq": 6920644, + "id": "electron-scripts", + "changes": [ + { + "rev": "3-a19944df5a3636bb225af9e0c8f9eedc" + } + ] + }, + { + "seq": 6920649, + "id": "@crexi-dev/schematics", + "changes": [ + { + "rev": "3-00188360eeca1f9123b2d7cd4b468c50" + } + ] + }, + { + "seq": 6920651, + "id": "botfactory-conversation", + "changes": [ + { + "rev": "50-f3667cde87637505528c46adc87f44e3" + } + ] + }, + { + "seq": 6920667, + "id": "castle", + "changes": [ + { + "rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0" + } + ] + }, + { + "seq": 6920671, + "id": "rbc-wm-framework-vuejs", + "changes": [ + { + "rev": "111-32bed479afacdd88aed9ac16dd135843" + } + ] + }, + { + "seq": 6920678, + "id": "bitcoinfiles", + "changes": [ + { + "rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0" + } + ] + }, + { + "seq": 6920679, + "id": "jovo-core", + "changes": [ + { + "rev": "2-d7440f1d17823e1a0760d9b3d4537c6e" + } + ] + }, + { + "seq": 6920687, + "id": "jovo-framework", + "changes": [ + { + "rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9" + } + ] + }, + { + "seq": 6920690, + "id": "smart-form-lib", + "changes": [ + { + "rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade" + } + ] + }, + { + "seq": 6920694, + "id": "bokehjs", + "changes": [ + { + "rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2" + } + ] + }, + { + "seq": 6920701, + "id": "guijarro", + "changes": [ + { + "rev": "14-82ece581d6a35d4e1d78e5292ca245c0" + } + ] + }, + { + "seq": 6920702, + "id": "@kava-labs/crypto-rate-utils", + "changes": [ + { + "rev": "3-cecc6a6c226a0590b1a685e3041028c6" + } + ] + }, + { + "seq": 6920703, + "id": "@riouxjean/test", + "changes": [ + { + "rev": "10-01e97dc7d0241dc49ea93b3468ec7b29" + } + ] + }, + { + "seq": 6920704, + "id": "react-scrabblefy", + "changes": [ + { + "rev": "7-970c8206f3b8744204f7dcb106f8462b" + } + ] + }, + { + "seq": 6920706, + "id": "molart", + "changes": [ + { + "rev": "14-416cd3cec62dd46f9b59a3bbe35308f6" + } + ] + }, + { + "seq": 6920707, + "id": "@universal-material/angular", + "changes": [ + { + "rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3" + } + ] + }, + { + "seq": 6920708, + "id": "cozy-doctypes", + "changes": [ + { + "rev": "68-8e90cc26e25da6c9430d373e43ac3c25" + } + ] + }, + { + "seq": 6920710, + "id": "2o3t-ui", + "changes": [ + { + "rev": "96-1e65d5320ea7c78525aba5daf328bd4b" + } + ] + }, + { + "seq": 6920712, + "id": "ark-ts", + "changes": [ + { + "rev": "24-033183c2f7f9cbb6e44d553213e525b6" + } + ] + }, + { + "seq": 6920715, + "id": "mysqlconnector", + "changes": [ + { + "rev": "19-f09bc0b82281ca486db5ebe83843679e" + } + ] + }, + { + "seq": 6920716, + "id": "@innovexa/ng-form-creator-lib", + "changes": [ + { + "rev": "147-480665ee17fa889dfec1aee75b907ff2" + } + ] + }, + { + "seq": 6920717, + "id": "k-routes-example-basic", + "changes": [ + { + "rev": "1-35142059e1c63cc724da71a9eebf229c" + } + ] + }, + { + "seq": 6920718, + "id": "wloggertojs", + "changes": [ + { + "rev": "29-5b5aa74bd30ff0fc86b39fba799befe2" + } + ] + }, + { + "seq": 6920720, + "id": "wloggertofile", + "changes": [ + { + "rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5" + } + ] + }, + { + "seq": 6920721, + "id": "@brightcove/flashls", + "changes": [ + { + "rev": "62-fbadb49476a58e98f0f136c86b614734" + } + ] + }, + { + "seq": 6920722, + "id": "@brightcove/hls-fetcher", + "changes": [ + { + "rev": "76-3341ed8ade38f3251a97c94c3a7af5ac" + } + ] + }, + { + "seq": 6920723, + "id": "@brightcove/kacl", + "changes": [ + { + "rev": "33-d0bc6b639cccb301086114d548ecfdbf" + } + ] + }, + { + "seq": 6920724, + "id": "just-in-types", + "changes": [ + { + "rev": "2-fc329aa885dc795aee340f36ec60f333" + } + ] + }, + { + "seq": 6920725, + "id": "@brightcove/player-loader", + "changes": [ + { + "rev": "56-9ff5aebc9743a44d46c182746313877d" + } + ] + }, + { + "seq": 6920726, + "id": "@brightcove/player-loader-webpack-plugin", + "changes": [ + { + "rev": "33-db8b4d6765f19e475e1c1d16843824cb" + } + ] + }, + { + "seq": 6920727, + "id": "@brightcove/player-url", + "changes": [ + { + "rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc" + } + ] + }, + { + "seq": 6920728, + "id": "@brightcove/react-player-loader", + "changes": [ + { + "rev": "39-b7bf609de666ec7e71f517db53ab9c0a" + } + ] + }, + { + "seq": 6920729, + "id": "vscode-theme-generator", + "changes": [ + { + "rev": "21-bcb92281d6f7e37548bb18113681df88" + } + ] + }, + { + "seq": 6920733, + "id": "@brightcove/typed-immutable-extensions", + "changes": [ + { + "rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe" + } + ] + }, + { + "seq": 6920734, + "id": "@brightcove/typed-immutable-proptypes", + "changes": [ + { + "rev": "27-e4802afc947c55d34f778864476c17e4" + } + ] + }, + { + "seq": 6920737, + "id": "@brightcove/videojs-flashls-source-handler", + "changes": [ + { + "rev": "59-faf69c49be866b2ab7faa7be9972e7a5" + } + ] + }, + { + "seq": 6920738, + "id": "@brightcove/videojs-flashls-swf", + "changes": [ + { + "rev": "60-04908466eaac2194bc3061e91f463dab" + } + ] + }, + { + "seq": 6920739, + "id": "@noqcks/generated", + "changes": [ + { + "rev": "2-e07d07614182d4beccc507ca199e612d" + } + ] + }, + { + "seq": 6920740, + "id": "pkcs7", + "changes": [ + { + "rev": "60-65ba116f3b6b705f472971b5c6a8f8d2" + } + ] + }, + { + "seq": 6920741, + "id": "videojs-errors", + "changes": [ + { + "rev": "57-c999abd162ca4b93412e363443aa688a" + } + ] + }, + { + "seq": 6920742, + "id": "videojs-flashls-source-handler", + "changes": [ + { + "rev": "59-46d62e18971a8c800710a8fbf985c1c5" + } + ] + }, + { + "seq": 6920743, + "id": "videojs-playlist", + "changes": [ + { + "rev": "97-d4b3492a94c1084c272162dd51901188" + } + ] + }, + { + "seq": 6920745, + "id": "videojs-playlist-ui", + "changes": [ + { + "rev": "95-ba97c44c354b2262e639f8c515bed9bc" + } + ] + }, + { + "seq": 6920746, + "id": "fusion-apollo-universal-client", + "changes": [ + { + "rev": "25-7123042a477cec67c7d5fc702254c7a3" + } + ] + }, + { + "seq": 6920749, + "id": "msg-fabric-core", + "changes": [ + { + "rev": "20-17c33e06faca357526c7395aca1113d2" + } + ] + }, + { + "seq": 6920750, + "id": "@expo/schemer", + "changes": [ + { + "rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d" + } + ] + }, + { + "seq": 6920752, + "id": "mathjs", + "changes": [ + { + "rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18" + } + ] + }, + { + "seq": 6920758, + "id": "statesauce-ui", + "changes": [ + { + "rev": "6-db9a39366c1a082c56a2212e368e3ae2" + } + ] + }, + { + "seq": 6920782, + "id": "@catchandrelease/arbor", + "changes": [ + { + "rev": "19-925648432b398ecadc98993e6fba2353" + } + ] + }, + { + "seq": 6920784, + "id": "discover-shared-ebsco-ui-core", + "changes": [ + { + "rev": "4-277063cbc6b71f969e5f0db8c371db65" + } + ] + }, + { + "seq": 6920807, + "id": "react-apexcharts", + "changes": [ + { + "rev": "13-18505be8026a50390c1ff1ba522cb9bd" + } + ] + }, + { + "seq": 6920819, + "id": "zigbee-shepherd-converters", + "changes": [ + { + "rev": "90-5819692a5a9679ff8669fb410e190515" + } + ] + }, + { + "seq": 6920835, + "id": "honeycomb-grid", + "changes": [ + { + "rev": "36-edd6733c80b04a72600558dc55348c73" + } + ] + }, + { + "seq": 6920838, + "id": "pixl-config", + "changes": [ + { + "rev": "7-5dd2b68d04fefb4039b3965b3497eda2" + } + ] + }, + { + "seq": 6920842, + "id": "discover-shared-ebsco-ui-theming", + "changes": [ + { + "rev": "4-e9d083825b1eae46f28c4def2d0db79f" + } + ] + }, + { + "seq": 6920843, + "id": "common-oxgalaxy-lengua-app", + "changes": [ + { + "rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f" + } + ] + }, + { + "seq": 6920845, + "id": "discover-shared-ebsco-ui-grid", + "changes": [ + { + "rev": "2-6f71cf625a5232075071952b2adaa8f2" + } + ] + }, + { + "seq": 6920847, + "id": "@auth0/cosmos-tokens", + "changes": [ + { + "rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb" + } + ] + }, + { + "seq": 6920848, + "id": "@auth0/babel-preset-cosmos", + "changes": [ + { + "rev": "43-d05d3779db08f08726ba048da298e046" + } + ] + }, + { + "seq": 6920849, + "id": "jsrender", + "changes": [ + { + "rev": "11-c949091592b3329d73ae564e45a3472d" + } + ] + }, + { + "seq": 6920850, + "id": "discover-shared-ebsco-ui-container", + "changes": [ + { + "rev": "2-c32089f76b7f253bc0d765da8b9f670d" + } + ] + }, + { + "seq": 6920852, + "id": "@auth0/cosmos", + "changes": [ + { + "rev": "42-5fdaf3d9063c20dac13dcf455c42773c" + } + ] + }, + { + "seq": 6920853, + "id": "discover-shared-ebsco-ui-checkbox", + "changes": [ + { + "rev": "2-06d9521b86f0dbf4a398726faead1212" + } + ] + }, + { + "seq": 6920854, + "id": "@adunigan/toggles", + "changes": [ + { + "rev": "1-c2a830cf814a9fe2d72084339c9c5d28" + } + ] + }, + { + "seq": 6920855, + "id": "@spriteful/spriteful-lazy-carousel", + "changes": [ + { + "rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837" + } + ] + }, + { + "seq": 6920856, + "id": "react-modal-hook", + "changes": [ + { + "rev": "2-364b39d6559364c41d5b852ccad4ce31" + } + ], + "deleted": true + }, + { + "seq": 6920859, + "id": "@bellese/angular-design-system", + "changes": [ + { + "rev": "39-3e297f85ce2d6a6b6d15fc26420fc471" + } + ] + }, + { + "seq": 6920861, + "id": "@uifabric/styling", + "changes": [ + { + "rev": "229-addf6cc0e74a335125c04d60047353f5" + } + ] + }, + { + "seq": 6920862, + "id": "@uifabric/file-type-icons", + "changes": [ + { + "rev": "37-8a7e43399d1bb9f17334b10995f78df4" + } + ] + }, + { + "seq": 6920864, + "id": "throttlewrap", + "changes": [ + { + "rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa" + } + ] + }, + { + "seq": 6920865, + "id": "airtable", + "changes": [ + { + "rev": "16-d8aee935f6fa4c88057d75a0542bc58c" + } + ] + }, + { + "seq": 6920866, + "id": "@csmart/ngc-smart-address", + "changes": [ + { + "rev": "19-66a6ea868aae1912952f232d2c699f3a" + } + ] + }, + { + "seq": 6920868, + "id": "office-ui-fabric-react", + "changes": [ + { + "rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461" + } + ] + }, + { + "seq": 6920869, + "id": "@fuelrats/eslint-config", + "changes": [ + { + "rev": "12-1b4c71b78fd078e3c1cba535e8541bed" + } + ] + }, + { + "seq": 6920870, + "id": "@uifabric/date-time", + "changes": [ + { + "rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed" + } + ] + }, + { + "seq": 6920872, + "id": "dark-client", + "changes": [ + { + "rev": "11-a954c2a89a130ae73f064233d9b3bce2" + } + ] + }, + { + "seq": 6920873, + "id": "@uifabric/variants", + "changes": [ + { + "rev": "59-391c720194c663b9a5c59fe2c10a1535" + } + ] + }, + { + "seq": 6920875, + "id": "discover-shared-ebsco-ui-header", + "changes": [ + { + "rev": "2-efd8f0426a83422a6c8b7bff11054c72" + } + ] + }, + { + "seq": 6920876, + "id": "react-responsive-picture", + "changes": [ + { + "rev": "14-32a6d0850c8af33412cfdb23afd2ecfa" + } + ] + }, + { + "seq": 6920877, + "id": "@uifabric/fluent-theme", + "changes": [ + { + "rev": "16-39c29e00b81a0b654213a5a50d7e7f42" + } + ] + }, + { + "seq": 6920878, + "id": "@uifabric/dashboard", + "changes": [ + { + "rev": "82-04d6dc25b33e811c1d8c24566127b09c" + } + ] + }, + { + "seq": 6920879, + "id": "ids-enterprise", + "changes": [ + { + "rev": "201-dd709a3912f9832440320d448850b61a" + } + ] + }, + { + "seq": 6920880, + "id": "@uifabric/experiments", + "changes": [ + { + "rev": "224-efd1ef07f7640952c286488eae282367" + } + ] + }, + { + "seq": 6920881, + "id": "@fuelrats/eslint-config-react", + "changes": [ + { + "rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc" + } + ] + }, + { + "seq": 6920883, + "id": "jsviews", + "changes": [ + { + "rev": "11-44d8bedffc98cf6ac4aa669ba8844746" + } + ] + }, + { + "seq": 6920885, + "id": "pixl-server", + "changes": [ + { + "rev": "15-823f4598c3354500d8d2a266dd062502" + } + ] + }, + { + "seq": 6920887, + "id": "@rrpm/netlify-cms-core", + "changes": [ + { + "rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa" + } + ] + }, + { + "seq": 6920889, + "id": "lodash-a", + "changes": [ + { + "rev": "2-6ee66153dbe611a080b40775387d2d45" + } + ] + }, + { + "seq": 6920891, + "id": "meshcentral", + "changes": [ + { + "rev": "499-6677ca74525ed2aa77644c68001382fe" + } + ] + }, + { + "seq": 6920892, + "id": "vue-transition-collection", + "changes": [ + { + "rev": "2-0510ee52c014c0d3b1e65f24376d76f0" + } + ] + }, + { + "seq": 6920894, + "id": "fury-adapter-swagger", + "changes": [ + { + "rev": "47-09f0c55d8574d654c67f9244c21d7ef7" + } + ] + }, + { + "seq": 6920895, + "id": "@isobar-us/redux-form-gen", + "changes": [ + { + "rev": "30-70d7d9210264a321092c832063934648" + } + ] + }, + { + "seq": 6920896, + "id": "atomizer", + "changes": [ + { + "rev": "19-129774900cb2a67a46871cc2c40c34d3" + } + ] + }, + { + "seq": 6920904, + "id": "boom-js-client", + "changes": [ + { + "rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9" + } + ] + }, + { + "seq": 6920905, + "id": "@ts-common/json-parser", + "changes": [ + { + "rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3" + } + ] + }, + { + "seq": 6920906, + "id": "rutt", + "changes": [ + { + "rev": "13-78aab849cb00a6ef7ebc8165770b7d33" + } + ] + }, + { + "seq": 6920907, + "id": "linear-react-components-ui", + "changes": [ + { + "rev": "171-0307f1d69843b270e687371c67cbd1b0" + } + ] + }, + { + "seq": 6920908, + "id": "@earnest/eslint-config", + "changes": [ + { + "rev": "180-b5250dd803102cf7dbac8da9c1a403fd" + } + ] + }, + { + "seq": 6920909, + "id": "@earnest/eslint-config-es7", + "changes": [ + { + "rev": "181-da26885e0baacaea95814857f459572d" + } + ] + }, + { + "seq": 6920910, + "id": "fuse-design", + "changes": [ + { + "rev": "10-e2b78592872f680c05e55eb5b81a0cab" + } + ] + } + ], + "last_seq": 6920912 +} \ No newline at end of file diff --git a/swh/lister/npm/tests/test_npm_lister.py b/swh/lister/npm/tests/test_npm_lister.py --- a/swh/lister/npm/tests/test_npm_lister.py +++ b/swh/lister/npm/tests/test_npm_lister.py @@ -7,7 +7,7 @@ import unittest from swh.lister.core.tests.test_lister import HttpListerTesterBase -from swh.lister.npm.lister import NpmLister +from swh.lister.npm.lister import NpmLister, NpmIncrementalLister class NpmListerTester(HttpListerTesterBase, unittest.TestCase): @@ -25,3 +25,20 @@ # it can not succeed for the npm lister due to the # overriding of the string_pattern_check method pass + + +class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase): + Lister = NpmIncrementalLister + test_re = re.compile(r'^.*/_changes\?since=([0-9]+).*') + lister_subdir = 'npm' + good_api_response_file = 'api_inc_response.json' + bad_api_response_file = 'api_inc_empty_response.json' + first_index = 6920642 + entries_per_page = 100 + + @requests_mock.Mocker() + def test_is_within_bounds(self, http_mocker): + # disable this test from HttpListerTesterBase as + # it can not succeed for the npm lister due to the + # overriding of the string_pattern_check method + pass