diff --git a/swh/loader/package/tests/conftest.py b/swh/loader/package/tests/conftest.py index 8e781a9..1c14a7f 100644 --- a/swh/loader/package/tests/conftest.py +++ b/swh/loader/package/tests/conftest.py @@ -1,140 +1,148 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os import re import pytest from functools import partial from os import path from urllib.parse import urlparse from .common import DATADIR import swh.storage from swh.storage import get_storage as initial_get_storage logger = logging.getLogger(__name__) def get_storage(cls, args): if cls == 'filter': from swh.loader.package.storage import FilteringProxyStorage return FilteringProxyStorage(**args) if cls == 'buffer': from swh.loader.package.storage import BufferingProxyStorage return BufferingProxyStorage(**args) return initial_get_storage(cls, args) swh.storage.get_storage = get_storage # Check get_local_factory function # Maximum number of iteration checks to generate requests responses MAX_VISIT_FILES = 10 @pytest.fixture def swh_config(monkeypatch): conffile = os.path.join(DATADIR, 'loader.yml') monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile) return conffile -def get_response_cb(request, context, ignore_urls=[], visits=None): +def get_response_cb(request, context, datadir, ignore_urls=[], visits=None): """Mount point callback to fetch on disk the content of a request This is meant to be used as 'body' argument of the requests_mock.get() method. It will look for files on the local filesystem based on the requested URL, using the following rules: - files are searched in the DATADIR/ directory - the local file name is the path part of the URL with path hierarchy markers (aka '/') replaced by '_' Eg. if you use the requests_mock fixture in your test file as: requests_mock.get('https://nowhere.com', body=get_response_cb) # or even requests_mock.get(re.compile('https://'), body=get_response_cb) then a call requests.get like: requests.get('https://nowhere.com/path/to/resource') will look the content of the response in: DATADIR/resources/nowhere.com/path_to_resource Args: request (requests.Request): Object requests context (requests.Context): Object holding response metadata information (status_code, headers, etc...) ignore_urls (List): urls whose status response should be 404 even if the local file exists visits (Optional[Dict]): Map of url, number of visits. If None, disable multi visit support (default) Returns: Optional[FileDescriptor] on the on disk file to read from the test context """ logger.debug('get_response_cb(%s, %s)', request, context) logger.debug('url: %s', request.url) logger.debug('ignore_urls: %s', ignore_urls) if request.url in ignore_urls: context.status_code = 404 return None url = urlparse(request.url) dirname = url.hostname # pypi.org | files.pythonhosted.org # url.path: pypi//json -> local file: pypi__json filename = url.path[1:] if filename.endswith('/'): filename = filename[:-1] filename = filename.replace('/', '_') - filepath = path.join(DATADIR, dirname, filename) + filepath = path.join(datadir, dirname, filename) if visits is not None: visit = visits.get(url, 0) visits[url] = visit + 1 if visit: filepath = filepath + '_visit%s' % visit if not path.isfile(filepath): logger.debug('not found filepath: %s', filepath) context.status_code = 404 return None fd = open(filepath, 'rb') context.headers['content-length'] = str(path.getsize(filepath)) return fd +@pytest.fixture +def datadir(): + return DATADIR + + def local_get_factory(ignore_urls=[], has_multi_visit=False): @pytest.fixture - def local_get(requests_mock): + def local_get(requests_mock, datadir): if not has_multi_visit: cb = partial(get_response_cb, - ignore_urls=ignore_urls) + ignore_urls=ignore_urls, + datadir=datadir) requests_mock.get(re.compile('https://'), body=cb) else: visits = {} requests_mock.get(re.compile('https://'), body=partial( - get_response_cb, ignore_urls=ignore_urls, visits=visits) + get_response_cb, ignore_urls=ignore_urls, visits=visits, + datadir=datadir) ) return requests_mock return local_get local_get = local_get_factory([]) + local_get_visits = local_get_factory(has_multi_visit=True) diff --git a/swh/loader/package/tests/fixture/conftest.py b/swh/loader/package/tests/fixture/conftest.py new file mode 100644 index 0000000..399adac --- /dev/null +++ b/swh/loader/package/tests/fixture/conftest.py @@ -0,0 +1,16 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from os import path + + +DATADIR = path.join(path.abspath(path.dirname(__file__)), 'data') + + +@pytest.fixture +def datadir(): + return DATADIR diff --git a/swh/loader/package/tests/fixture/data/example.com/file.json b/swh/loader/package/tests/fixture/data/example.com/file.json new file mode 100644 index 0000000..000a8dd --- /dev/null +++ b/swh/loader/package/tests/fixture/data/example.com/file.json @@ -0,0 +1,3 @@ +{ + "welcome": "you" +} diff --git a/swh/loader/package/tests/fixture/test_conftest.py b/swh/loader/package/tests/fixture/test_conftest.py new file mode 100644 index 0000000..17361e1 --- /dev/null +++ b/swh/loader/package/tests/fixture/test_conftest.py @@ -0,0 +1,19 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import requests + + +# In this arborescence, we override in the local conftest.py module the +# "datadir" fixture to specify where to retrieve the data files from. + + +def test_local_get_with_datadir_fixture_override(local_get): + """Override datadir fixture should retrieve data from elsewhere + + """ + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'welcome': 'you'}