diff --git a/requirements-swh.txt b/requirements-swh.txt index 01f1bd4..6410dfe 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core >= 0.0.75 +swh.core >= 0.0.77 swh.scheduler >= 0.0.58 diff --git a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 similarity index 100% rename from swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 rename to swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 diff --git a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/response.json b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/response.json index f420e5f..2540fa7 120000 --- a/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/response.json +++ b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/response.json @@ -1 +1 @@ -2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 \ No newline at end of file +2.0_repositories,after=1970-01-01T00:00:00+00:00,pagelen=100 \ No newline at end of file diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 "b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" similarity index 100% rename from swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 rename to "swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22%22,limit=11 "b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" similarity index 100% rename from swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22%22,limit=11 rename to "swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22jquery-accessible-simple-tooltip-aria%22,limit=11 "b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" similarity index 100% rename from swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22jquery-accessible-simple-tooltip-aria%22,limit=11 rename to "swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json index 855fe60..21ff340 120000 --- a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json +++ b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json @@ -1 +1 @@ -_all_docs,startkey=%22%22,limit=1001 \ No newline at end of file +_all_docs,startkey="",limit=1001 \ No newline at end of file diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py index 2458d86..281631a 100644 --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -1,102 +1,102 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import re import requests_mock import unittest from swh.lister.core.tests.test_lister import HttpListerTesterBase from swh.lister.npm.lister import NpmLister, NpmIncrementalLister from typing import Any, List logger = logging.getLogger(__name__) class NpmListerTester(HttpListerTesterBase, unittest.TestCase): Lister = NpmLister - test_re = re.compile(r'^.*/_all_docs\?startkey=%22(.+)%22.*') + test_re = re.compile(r'^.*/_all_docs\?startkey="(.+)".*') lister_subdir = 'npm' good_api_response_file = 'data/replicate.npmjs.com/api_response.json' bad_api_response_file = 'data/api_empty_response.json' first_index = 'jquery' entries_per_page = 100 @requests_mock.Mocker() def test_is_within_bounds(self, http_mocker): # disable this test from HttpListerTesterBase as # it can not succeed for the npm lister due to the # overriding of the string_pattern_check method pass class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase): Lister = NpmIncrementalLister test_re = re.compile(r'^.*/_changes\?since=([0-9]+).*') lister_subdir = 'npm' good_api_response_file = 'data/api_inc_response.json' bad_api_response_file = 'data/api_inc_empty_response.json' first_index = '6920642' entries_per_page = 100 @requests_mock.Mocker() def test_is_within_bounds(self, http_mocker): # disable this test from HttpListerTesterBase as # it can not succeed for the npm lister due to the # overriding of the string_pattern_check method pass def check_tasks(tasks: List[Any]): """Ensure scheduled tasks are in the expected format. """ for row in tasks: logger.debug('row: %s', row) assert row['type'] == 'load-npm' # arguments check args = row['arguments']['args'] assert len(args) == 2 package = args[0] url = args[1] assert url == 'https://www.npmjs.com/package/%s' % package # kwargs kwargs = row['arguments']['kwargs'] meta_url = kwargs['package_metadata_url'] assert meta_url == 'https://replicate.npmjs.com/%s' % package assert row['policy'] == 'recurring' assert row['priority'] is None def test_lister_npm_basic_listing(swh_listers, requests_mock_datadir): lister = swh_listers['npm'] lister.run() tasks = lister.scheduler.search_tasks(task_type='load-npm') assert len(tasks) == 100 check_tasks(tasks) def test_lister_npm_listing_pagination(swh_listers, requests_mock_datadir): lister = swh_listers['npm'] # Patch per page pagination lister.per_page = 10 + 1 lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace( '&limit=1001', '&limit=%s' % lister.per_page) lister.run() tasks = lister.scheduler.search_tasks(task_type='load-npm') assert len(tasks) == 2 * 10 # only 2 files with 10 results each check_tasks(tasks) diff --git a/swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments%5Buris%5D=1,after=,api.token=foo b/swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments[uris]=1,after=,api.token=foo similarity index 100% rename from swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments%5Buris%5D=1,after=,api.token=foo rename to swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments[uris]=1,after=,api.token=foo