diff --git a/swh/lister/npm/tests/api_empty_response.json b/swh/lister/npm/tests/data/api_empty_response.json similarity index 100% rename from swh/lister/npm/tests/api_empty_response.json rename to swh/lister/npm/tests/data/api_empty_response.json diff --git a/swh/lister/npm/tests/api_inc_empty_response.json b/swh/lister/npm/tests/data/api_inc_empty_response.json similarity index 100% rename from swh/lister/npm/tests/api_inc_empty_response.json rename to swh/lister/npm/tests/data/api_inc_empty_response.json diff --git a/swh/lister/npm/tests/api_inc_response.json b/swh/lister/npm/tests/data/api_inc_response.json similarity index 100% rename from swh/lister/npm/tests/api_inc_response.json rename to swh/lister/npm/tests/data/api_inc_response.json diff --git a/swh/lister/npm/tests/api_response.json b/swh/lister/npm/tests/data/replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 similarity index 100% rename from swh/lister/npm/tests/api_response.json rename to swh/lister/npm/tests/data/replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 diff --git a/swh/lister/npm/tests/data/replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/replicate.npmjs.com/api_response.json new file mode 120000 index 0000000..855fe60 --- /dev/null +++ b/swh/lister/npm/tests/data/replicate.npmjs.com/api_response.json @@ -0,0 +1 @@ +_all_docs,startkey=%22%22,limit=1001 \ No newline at end of file diff --git a/swh/lister/npm/tests/test_npm_lister.py b/swh/lister/npm/tests/test_lister.py similarity index 51% rename from swh/lister/npm/tests/test_npm_lister.py rename to swh/lister/npm/tests/test_lister.py index 01fe595..c4f393d 100644 --- a/swh/lister/npm/tests/test_npm_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -1,44 +1,78 @@ -# Copyright (C) 2018-2019 the Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging + import re import requests_mock import unittest from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.npm.lister import NpmLister, NpmIncrementalLister +logger = logging.getLogger(__name__) + + class NpmListerTester(HttpListerTesterBase, unittest.TestCase): Lister = NpmLister test_re = re.compile(r'^.*/_all_docs\?startkey=%22(.+)%22.*') lister_subdir = 'npm' - good_api_response_file = 'api_response.json' - bad_api_response_file = 'api_empty_response.json' + good_api_response_file = 'data/replicate.npmjs.com/api_response.json' + bad_api_response_file = 'data/api_empty_response.json' first_index = 'jquery' entries_per_page = 100 @requests_mock.Mocker() def test_is_within_bounds(self, http_mocker): # disable this test from HttpListerTesterBase as # it can not succeed for the npm lister due to the # overriding of the string_pattern_check method pass class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase): Lister = NpmIncrementalLister test_re = re.compile(r'^.*/_changes\?since=([0-9]+).*') lister_subdir = 'npm' - good_api_response_file = 'api_inc_response.json' - bad_api_response_file = 'api_inc_empty_response.json' + good_api_response_file = 'data/api_inc_response.json' + bad_api_response_file = 'data/api_inc_empty_response.json' first_index = '6920642' entries_per_page = 100 @requests_mock.Mocker() def test_is_within_bounds(self, http_mocker): # disable this test from HttpListerTesterBase as # it can not succeed for the npm lister due to the # overriding of the string_pattern_check method pass + + +def test_lister_npm_basic_listing(swh_listers, requests_mock_datadir): + lister = swh_listers['npm'] + + lister.run() + + r = lister.scheduler.search_tasks(task_type='load-npm') + assert len(r) == 100 + + for row in r: + logger.debug('row: %s', row) + assert row['type'] == 'load-npm' + # arguments check + args = row['arguments']['args'] + assert len(args) == 2 + + package = args[0] + url = args[1] + assert url == 'https://www.npmjs.com/package/%s' % package + + # kwargs + kwargs = row['arguments']['kwargs'] + meta_url = kwargs['package_metadata_url'] + assert meta_url == 'https://replicate.npmjs.com/%s' % package + + assert row['policy'] == 'recurring' + assert row['priority'] is None