diff --git a/requirements-swh.txt b/requirements-swh.txt index 7247006..d7661c0 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,3 +1,3 @@ -swh.core >= 0.0.73 +swh.core >= 0.0.75 swh.storage[schemata] >= 0.0.122 swh.scheduler >= 0.0.58 diff --git a/swh/lister/bitbucket/tests/data/api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 similarity index 100% rename from swh/lister/bitbucket/tests/data/api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 rename to swh/lister/bitbucket/tests/data/https_api.bitbucket.org/2.0_repositories,after=1970-01-01T00%3A00%3A00%2B00%3A00,pagelen=100 diff --git a/swh/lister/bitbucket/tests/data/api.bitbucket.org/empty_response.json b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/empty_response.json similarity index 100% rename from swh/lister/bitbucket/tests/data/api.bitbucket.org/empty_response.json rename to swh/lister/bitbucket/tests/data/https_api.bitbucket.org/empty_response.json diff --git a/swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json b/swh/lister/bitbucket/tests/data/https_api.bitbucket.org/response.json similarity index 100% rename from swh/lister/bitbucket/tests/data/api.bitbucket.org/response.json rename to swh/lister/bitbucket/tests/data/https_api.bitbucket.org/response.json diff --git a/swh/lister/bitbucket/tests/test_lister.py b/swh/lister/bitbucket/tests/test_lister.py index a3cbbe6..ce03b32 100644 --- a/swh/lister/bitbucket/tests/test_lister.py +++ b/swh/lister/bitbucket/tests/test_lister.py @@ -1,102 +1,102 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re import unittest from datetime import timedelta from urllib.parse import unquote import iso8601 import requests_mock from swh.lister.bitbucket.lister import BitBucketLister from swh.lister.core.tests.test_lister import HttpListerTester def convert_type(req_index): """Convert the req_index to its right type according to the model's "indexable" column. """ return iso8601.parse_date(unquote(req_index)) class BitBucketListerTester(HttpListerTester, unittest.TestCase): Lister = BitBucketLister test_re = re.compile(r'/repositories\?after=([^?&]+)') lister_subdir = 'bitbucket' - good_api_response_file = 'data/api.bitbucket.org/response.json' - bad_api_response_file = 'data/api.bitbucket.org/empty_response.json' + good_api_response_file = 'data/https_api.bitbucket.org/response.json' + bad_api_response_file = 'data/https_api.bitbucket.org/empty_response.json' first_index = convert_type('2008-07-12T07:44:01.476818+00:00') last_index = convert_type('2008-07-19T06:16:43.044743+00:00') entries_per_page = 10 convert_type = staticmethod(convert_type) def request_index(self, request): """(Override) This is needed to emulate the listing bootstrap when no min_bound is provided to run """ m = self.test_re.search(request.path_url) idx = convert_type(m.group(1)) if idx == self.Lister.default_min_bound: idx = self.first_index return idx @requests_mock.Mocker() def test_fetch_none_nodb(self, http_mocker): """Overridden because index is not an integer nor a string """ http_mocker.get(self.test_re, text=self.mock_response) fl = self.get_fl() self.disable_scheduler(fl) self.disable_db(fl) # stores no results fl.run(min_bound=self.first_index - timedelta(days=3), max_bound=self.first_index) def test_is_within_bounds(self): fl = self.get_fl() self.assertTrue(fl.is_within_bounds( iso8601.parse_date('2008-07-15'), self.first_index, self.last_index)) self.assertFalse(fl.is_within_bounds( iso8601.parse_date('2008-07-20'), self.first_index, self.last_index)) self.assertFalse(fl.is_within_bounds( iso8601.parse_date('2008-07-11'), self.first_index, self.last_index)) def test_lister_bitbucket(swh_listers, requests_mock_datadir): """Simple bitbucket listing should create scheduled tasks """ lister = swh_listers['bitbucket'] lister.run() r = lister.scheduler.search_tasks(task_type='load-hg') assert len(r) == 10 for row in r: assert row['type'] == 'load-hg' # arguments check args = row['arguments']['args'] assert len(args) == 1 url = args[0] assert url.startswith('https://bitbucket.org') # kwargs kwargs = row['arguments']['kwargs'] assert kwargs == {} assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/cgit/tests/data/git.savannah.gnu.org/README b/swh/lister/cgit/tests/data/https_git.savannah.gnu.org/README similarity index 100% rename from swh/lister/cgit/tests/data/git.savannah.gnu.org/README rename to swh/lister/cgit/tests/data/https_git.savannah.gnu.org/README diff --git a/swh/lister/cgit/tests/data/git.savannah.gnu.org/cgit b/swh/lister/cgit/tests/data/https_git.savannah.gnu.org/cgit similarity index 100% rename from swh/lister/cgit/tests/data/git.savannah.gnu.org/cgit rename to swh/lister/cgit/tests/data/https_git.savannah.gnu.org/cgit diff --git a/swh/lister/cgit/tests/data/git.savannah.gnu.org/cgit_elisp-es.git b/swh/lister/cgit/tests/data/https_git.savannah.gnu.org/cgit_elisp-es.git similarity index 100% rename from swh/lister/cgit/tests/data/git.savannah.gnu.org/cgit_elisp-es.git rename to swh/lister/cgit/tests/data/https_git.savannah.gnu.org/cgit_elisp-es.git diff --git a/swh/lister/cgit/tests/data/git.tizen/README b/swh/lister/cgit/tests/data/https_git.tizen/README similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/README rename to swh/lister/cgit/tests/data/https_git.tizen/README diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit b/swh/lister/cgit/tests/data/https_git.tizen/cgit similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit rename to swh/lister/cgit/tests/data/https_git.tizen/cgit diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit,ofs=100 b/swh/lister/cgit/tests/data/https_git.tizen/cgit,ofs=100 similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit,ofs=100 rename to swh/lister/cgit/tests/data/https_git.tizen/cgit,ofs=100 diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit,ofs=50 b/swh/lister/cgit/tests/data/https_git.tizen/cgit,ofs=50 similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit,ofs=50 rename to swh/lister/cgit/tests/data/https_git.tizen/cgit,ofs=50 diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_All-Projects b/swh/lister/cgit/tests/data/https_git.tizen/cgit_All-Projects similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_All-Projects rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_All-Projects diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_All-Users b/swh/lister/cgit/tests/data/https_git.tizen/cgit_All-Users similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_All-Users rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_All-Users diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_Lock-Projects b/swh/lister/cgit/tests/data/https_git.tizen/cgit_Lock-Projects similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_Lock-Projects rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_Lock-Projects diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-base b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-base similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-base rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-base diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-mc1n2 b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-mc1n2 similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-mc1n2 rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_alsa-scenario-scn-data-0-mc1n2 diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_ap_samsung_audio-hal-e3250 b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_ap_samsung_audio-hal-e3250 similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_ap_samsung_audio-hal-e3250 rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_ap_samsung_audio-hal-e3250 diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_ap_samsung_audio-hal-e4x12 b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_ap_samsung_audio-hal-e4x12 similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_ap_samsung_audio-hal-e4x12 rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_ap_samsung_audio-hal-e4x12 diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_devices_nfc-plugin-nxp b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_devices_nfc-plugin-nxp similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_devices_nfc-plugin-nxp rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_devices_nfc-plugin-nxp diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_intel_mfld_bootstub-mfld-blackbay b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_intel_mfld_bootstub-mfld-blackbay similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_intel_mfld_bootstub-mfld-blackbay rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_intel_mfld_bootstub-mfld-blackbay diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_mtdev b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_mtdev similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_mtdev rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_mtdev diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_opengl-es-virtual-drv b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_opengl-es-virtual-drv similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_opengl-es-virtual-drv rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_opengl-es-virtual-drv diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_panda_libdrm b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_panda_libdrm similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_panda_libdrm rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_panda_libdrm diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_panda_libnl b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_panda_libnl similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_panda_libnl rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_panda_libnl diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_xorg_driver_xserver-xorg-misc b/swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_xorg_driver_xserver-xorg-misc similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_adaptation_xorg_driver_xserver-xorg-misc rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_adaptation_xorg_driver_xserver-xorg-misc diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_apps_core_preloaded_ug-setting-gallery-efl b/swh/lister/cgit/tests/data/https_git.tizen/cgit_apps_core_preloaded_ug-setting-gallery-efl similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_apps_core_preloaded_ug-setting-gallery-efl rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_apps_core_preloaded_ug-setting-gallery-efl diff --git a/swh/lister/cgit/tests/data/git.tizen/cgit_apps_core_preloaded_ug-setting-homescreen-efl b/swh/lister/cgit/tests/data/https_git.tizen/cgit_apps_core_preloaded_ug-setting-homescreen-efl similarity index 100% rename from swh/lister/cgit/tests/data/git.tizen/cgit_apps_core_preloaded_ug-setting-homescreen-efl rename to swh/lister/cgit/tests/data/https_git.tizen/cgit_apps_core_preloaded_ug-setting-homescreen-efl diff --git a/swh/lister/github/tests/data/api.github.com/empty_response.json b/swh/lister/github/tests/data/https_api.github.com/empty_response.json similarity index 100% rename from swh/lister/github/tests/data/api.github.com/empty_response.json rename to swh/lister/github/tests/data/https_api.github.com/empty_response.json diff --git a/swh/lister/github/tests/data/api.github.com/first_response.json b/swh/lister/github/tests/data/https_api.github.com/first_response.json similarity index 100% rename from swh/lister/github/tests/data/api.github.com/first_response.json rename to swh/lister/github/tests/data/https_api.github.com/first_response.json diff --git a/swh/lister/github/tests/data/api.github.com/next_response.json b/swh/lister/github/tests/data/https_api.github.com/next_response.json similarity index 100% rename from swh/lister/github/tests/data/api.github.com/next_response.json rename to swh/lister/github/tests/data/https_api.github.com/next_response.json diff --git a/swh/lister/github/tests/data/api.github.com/repositories,since=0 b/swh/lister/github/tests/data/https_api.github.com/repositories,since=0 similarity index 100% rename from swh/lister/github/tests/data/api.github.com/repositories,since=0 rename to swh/lister/github/tests/data/https_api.github.com/repositories,since=0 diff --git a/swh/lister/github/tests/test_lister.py b/swh/lister/github/tests/test_lister.py index cc61bc0..db11c35 100644 --- a/swh/lister/github/tests/test_lister.py +++ b/swh/lister/github/tests/test_lister.py @@ -1,83 +1,83 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re import unittest import requests_mock from datetime import datetime, timedelta from swh.lister.core.tests.test_lister import HttpListerTester from swh.lister.github.lister import GitHubLister class GitHubListerTester(HttpListerTester, unittest.TestCase): Lister = GitHubLister test_re = re.compile(r'/repositories\?since=([^?&]+)') lister_subdir = 'github' - good_api_response_file = 'data/api.github.com/first_response.json' - bad_api_response_file = 'data/api.github.com/empty_response.json' + good_api_response_file = 'data/https_api.github.com/first_response.json' + bad_api_response_file = 'data/https_api.github.com/empty_response.json' first_index = 0 last_index = 369 entries_per_page = 100 convert_type = int def response_headers(self, request): headers = {'X-RateLimit-Remaining': '1'} if self.request_index(request) == self.first_index: headers.update({ 'Link': ';' ' rel="next",' ';' ' rel="first"' % self.last_index }) else: headers.update({ 'Link': ';' ' rel="first"' }) return headers def mock_rate_quota(self, n, request, context): self.rate_limit += 1 context.status_code = 403 context.headers['X-RateLimit-Remaining'] = '0' one_second = int((datetime.now() + timedelta(seconds=1.5)).timestamp()) context.headers['X-RateLimit-Reset'] = str(one_second) return '{"error":"dummy"}' @requests_mock.Mocker() def test_scheduled_tasks(self, http_mocker): self.scheduled_tasks_test( - 'data/api.github.com/next_response.json', 876, http_mocker) + 'data/https_api.github.com/next_response.json', 876, http_mocker) def test_lister_github(swh_listers, requests_mock_datadir): """Simple github listing should create scheduled tasks """ lister = swh_listers['github'] lister.run() r = lister.scheduler.search_tasks(task_type='load-git') assert len(r) == 100 for row in r: assert row['type'] == 'load-git' # arguments check args = row['arguments']['args'] assert len(args) == 1 url = args[0] assert url.startswith('https://github.com') # kwargs kwargs = row['arguments']['kwargs'] assert kwargs == {} assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/gitlab/tests/data/gitlab.com/api_empty_response.json b/swh/lister/gitlab/tests/data/https_gitlab.com/api_empty_response.json similarity index 100% rename from swh/lister/gitlab/tests/data/gitlab.com/api_empty_response.json rename to swh/lister/gitlab/tests/data/https_gitlab.com/api_empty_response.json diff --git a/swh/lister/gitlab/tests/data/gitlab.com/api_response.json b/swh/lister/gitlab/tests/data/https_gitlab.com/api_response.json similarity index 100% rename from swh/lister/gitlab/tests/data/gitlab.com/api_response.json rename to swh/lister/gitlab/tests/data/https_gitlab.com/api_response.json diff --git a/swh/lister/gitlab/tests/data/gitlab.com/api_v4__projects,page=0,order_by=id,sort=asc,per_page=20 b/swh/lister/gitlab/tests/data/https_gitlab.com/api_v4__projects,page=0,order_by=id,sort=asc,per_page=20 similarity index 100% rename from swh/lister/gitlab/tests/data/gitlab.com/api_v4__projects,page=0,order_by=id,sort=asc,per_page=20 rename to swh/lister/gitlab/tests/data/https_gitlab.com/api_v4__projects,page=0,order_by=id,sort=asc,per_page=20 diff --git a/swh/lister/gnu/tests/data/ftp.gnu.org/tree.json.gz b/swh/lister/gnu/tests/data/https_ftp.gnu.org/tree.json.gz similarity index 100% rename from swh/lister/gnu/tests/data/ftp.gnu.org/tree.json.gz rename to swh/lister/gnu/tests/data/https_ftp.gnu.org/tree.json.gz diff --git a/swh/lister/gnu/tests/test_tree.py b/swh/lister/gnu/tests/test_tree.py index 5df141b..5c6bf85 100644 --- a/swh/lister/gnu/tests/test_tree.py +++ b/swh/lister/gnu/tests/test_tree.py @@ -1,135 +1,135 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import pytest from os import path from swh.lister.gnu.tree import ( GNUTree, find_artifacts, check_filename_is_archive, load_raw_data ) def test_load_raw_data_from_query(requests_mock_datadir): actual_json = load_raw_data('https://ftp.gnu.org/tree.json.gz') assert actual_json is not None assert isinstance(actual_json, list) assert len(actual_json) == 2 def test_load_raw_data_from_query_failure(requests_mock_datadir): inexistant_url = 'https://ftp2.gnu.org/tree.unknown.gz' with pytest.raises(ValueError, match='Error during query'): load_raw_data(inexistant_url) def test_load_raw_data_from_file(datadir): - filepath = path.join(datadir, 'ftp.gnu.org', 'tree.json.gz') + filepath = path.join(datadir, 'https_ftp.gnu.org', 'tree.json.gz') actual_json = load_raw_data(filepath) assert actual_json is not None assert isinstance(actual_json, list) assert len(actual_json) == 2 def test_load_raw_data_from_file_failure(datadir): unknown_path = path.join(datadir, 'ftp.gnu.org2', 'tree.json.gz') with pytest.raises(FileNotFoundError): load_raw_data(unknown_path) def test_tree_json(requests_mock_datadir): tree_json = GNUTree('https://ftp.gnu.org/tree.json.gz') assert tree_json.projects['https://ftp.gnu.org/gnu/8sync/'] == { 'name': '8sync', 'time_modified': '1489817408', 'url': 'https://ftp.gnu.org/gnu/8sync/' } assert tree_json.projects['https://ftp.gnu.org/gnu/3dldf/'] == { 'name': '3dldf', 'time_modified': '1386961236', 'url': 'https://ftp.gnu.org/gnu/3dldf/' } assert tree_json.projects['https://ftp.gnu.org/gnu/a2ps/'] == { 'name': 'a2ps', 'time_modified': '1198900505', 'url': 'https://ftp.gnu.org/gnu/a2ps/' } assert tree_json.projects['https://ftp.gnu.org/old-gnu/xshogi/'] == { 'name': 'xshogi', 'time_modified': '1059822922', 'url': 'https://ftp.gnu.org/old-gnu/xshogi/' } assert tree_json.artifacts['https://ftp.gnu.org/old-gnu/zlibc/'] == [ { 'archive': 'https://ftp.gnu.org/old-gnu/zlibc/zlibc-0.9b.tar.gz', # noqa 'length': 90106, 'time': 857980800 }, { 'archive': 'https://ftp.gnu.org/old-gnu/zlibc/zlibc-0.9e.tar.gz', # noqa 'length': 89625, 'time': 860396400 } ] def test_tree_json_failures(requests_mock_datadir): url = 'https://unknown/tree.json.gz' tree_json = GNUTree(url) with pytest.raises(ValueError, match='Error during query to %s' % url): tree_json.artifacts['https://ftp.gnu.org/gnu/3dldf/'] with pytest.raises(ValueError, match='Error during query to %s' % url): tree_json.projects['https://ftp.gnu.org/old-gnu/xshogi/'] def test_find_artifacts_small_sample(datadir): expected_tarballs = [ { 'archive': '/root/artanis/artanis-0.2.1.tar.bz2', 'time': 1495205979, 'length': 424081, }, { 'archive': '/root/xboard/winboard/winboard-4_0_0-src.zip', # noqa 'time': 898422900, 'length': 1514448 }, { 'archive': '/root/xboard/xboard-3.6.2.tar.gz', # noqa 'time': 869814000, 'length': 450164, }, { 'archive': '/root/xboard/xboard-4.0.0.tar.gz', # noqa 'time': 898422900, 'length': 514951, }, ] file_structure = json.load(open(path.join(datadir, 'tree.min.json'))) actual_tarballs = find_artifacts(file_structure, '/root/') assert actual_tarballs == expected_tarballs def test_find_artifacts(datadir): file_structure = json.load(open(path.join(datadir, 'tree.json'))) actual_tarballs = find_artifacts(file_structure, '/root/') assert len(actual_tarballs) == 42 + 3 # tar + zip def test_check_filename_is_archive(): for ext in ['abc.xy.zip', 'cvb.zip', 'abc.tar.bz2', 'something.tar']: assert check_filename_is_archive(ext) is True for ext in ['abc.tar.gz.sig', 'abc', 'something.zip2', 'foo.tar.']: assert check_filename_is_archive(ext) is False diff --git a/swh/lister/npm/tests/data/replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 similarity index 100% rename from swh/lister/npm/tests/data/replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 rename to swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=%22%22,limit=1001 diff --git a/swh/lister/npm/tests/data/replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json similarity index 100% rename from swh/lister/npm/tests/data/replicate.npmjs.com/api_response.json rename to swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json diff --git a/swh/lister/packagist/tests/data/packagist.org/packages_list.json b/swh/lister/packagist/tests/data/https_packagist.org/packages_list.json similarity index 100% rename from swh/lister/packagist/tests/data/packagist.org/packages_list.json rename to swh/lister/packagist/tests/data/https_packagist.org/packages_list.json diff --git a/swh/lister/packagist/tests/test_lister.py b/swh/lister/packagist/tests/test_lister.py index f8b3a2a..869e6c2 100644 --- a/swh/lister/packagist/tests/test_lister.py +++ b/swh/lister/packagist/tests/test_lister.py @@ -1,102 +1,102 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest import requests_mock from unittest.mock import patch from swh.lister.packagist.lister import PackagistLister, compute_package_url from swh.lister.core.tests.test_lister import HttpSimpleListerTester expected_packages = ['0.0.0/composer-include-files', '0.0.0/laravel-env-shim', '0.0.1/try-make-package', '0099ff/dialogflowphp', '00f100/array_dot'] expected_model = { 'uid': '0099ff/dialogflowphp', 'name': '0099ff/dialogflowphp', 'full_name': '0099ff/dialogflowphp', 'html_url': 'https://repo.packagist.org/p/0099ff/dialogflowphp.json', 'origin_url': 'https://repo.packagist.org/p/0099ff/dialogflowphp.json', 'origin_type': 'packagist', } class PackagistListerTester(HttpSimpleListerTester, unittest.TestCase): Lister = PackagistLister PAGE = 'https://packagist.org/packages/list.json' lister_subdir = 'packagist' - good_api_response_file = 'data/packagist.org/packages_list.json' + good_api_response_file = 'data/https_packagist.org/packages_list.json' entries = 5 @requests_mock.Mocker() def test_list_packages(self, http_mocker): """List packages from simple api page should retrieve all packages within """ http_mocker.get(self.PAGE, text=self.mock_response) fl = self.get_fl() packages = fl.list_packages(self.get_api_response(0)) for package in expected_packages: assert package in packages def test_transport_response_simplified(self): """Test model created by the lister """ fl = self.get_fl() model = fl.transport_response_simplified(['0099ff/dialogflowphp']) assert len(model) == 1 for key, values in model[0].items(): assert values == expected_model[key] @patch('swh.lister.packagist.lister.utils.create_task_dict') def test_task_dict(self, mock_create_tasks): """Test the task creation of lister """ fl = self.get_fl() fl.task_dict(origin_type='packagist', origin_url='https://abc', name='test_pack') mock_create_tasks.assert_called_once_with( 'load-packagist', 'recurring', 'test_pack', 'https://abc', retries_left=3) def test_compute_package_url(): expected_url = 'https://repo.packagist.org/p/hello.json' actual_url = compute_package_url('hello') assert actual_url == expected_url def test_packagist_lister(lister_packagist, requests_mock_datadir): lister_packagist.run() r = lister_packagist.scheduler.search_tasks(task_type='load-packagist') assert len(r) == 5 for row in r: assert row['type'] == 'load-packagist' # arguments check args = row['arguments']['args'] assert len(args) == 2 package = args[0] url = args[1] expected_url = compute_package_url(package) assert url == expected_url # kwargs kwargs = row['arguments']['kwargs'] assert kwargs == {} assert row['policy'] == 'recurring' assert row['priority'] is None diff --git a/swh/lister/phabricator/tests/data/forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments%5Buris%5D=1,after=,api.token=foo b/swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments%5Buris%5D=1,after=,api.token=foo similarity index 100% rename from swh/lister/phabricator/tests/data/forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments%5Buris%5D=1,after=,api.token=foo rename to swh/lister/phabricator/tests/data/https_forge.softwareheritage.org/api_diffusion.repository.search,order=oldest,attachments%5Buris%5D=1,after=,api.token=foo diff --git a/swh/lister/pypi/tests/data/pypi.org/simple b/swh/lister/pypi/tests/data/https_pypi.org/simple similarity index 100% rename from swh/lister/pypi/tests/data/pypi.org/simple rename to swh/lister/pypi/tests/data/https_pypi.org/simple