diff --git a/swh/loader/package/storage.py b/swh/loader/package/storage.py new file mode 100644 index 0000000..79ed7d8 --- /dev/null +++ b/swh/loader/package/storage.py @@ -0,0 +1,58 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.storage import get_storage + + +class ProxyStorage: + def __init__(self, **storage): + self.storage = get_storage(**storage) + + def origin_add(self, origins): + return self.storage.origin_add(origins) + + def origin_visit_add(self, origin, date, type=None): + return self.storage.origin_visit_add(origin, date, type=type) + + def content_add(self, content): + return self.storage.content_add(content) + + def directory_add(self, directories): + return self.storage.directory_add(directories) + + def revision_add(self, revisions): + return self.storage.revision_add(revisions) + + def snapshot_add(self, snapshots): + return self.storage.snapshot_add(snapshots) + + def origin_visit_update(self, origin, visit_id, status=None, + metadata=None, snapshot=None): + return self.storage.origin_visit_update( + origin, visit_id, status=status, + metadata=metadata, snapshot=snapshot + ) + + def stat_counters(self): + return self.storage.stat_counters() + + def origin_visit_get(self, origin, last_visit=None, limit=None): + return self.storage.origin_visit_get( + origin, last_visit=last_visit, limit=limit) + + def content_missing_per_sha1(self, contents): + return self.storage.content_missing_per_sha1(contents) + + def directory_missing(self, directories): + return self.storage.directory_missing(directories) + + def revision_missing(self, revisions): + return self.storage.revision_missing(revisions) + + def snapshot_get(self, snapshot_id): + return self.storage.snapshot_get(snapshot_id) + + def content_get(self, content): + return self.storage.content_get(content) diff --git a/swh/loader/package/tests/conftest.py b/swh/loader/package/tests/conftest.py index 0e4c7bc..714b97f 100644 --- a/swh/loader/package/tests/conftest.py +++ b/swh/loader/package/tests/conftest.py @@ -1,124 +1,137 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os import re import pytest from functools import partial from os import path from urllib.parse import urlparse from .common import DATADIR +import swh.storage +from swh.storage import get_storage as initial_get_storage + logger = logging.getLogger(__name__) +def get_storage(cls, args): + if cls == 'proxy': + from swh.loader.package.storage import ProxyStorage + return ProxyStorage(**args) + return initial_get_storage(cls, args) + + +swh.storage.get_storage = get_storage + + # Check get_local_factory function # Maximum number of iteration checks to generate requests responses MAX_VISIT_FILES = 10 @pytest.fixture def swh_config(monkeypatch): conffile = os.path.join(DATADIR, 'loader.yml') monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile) return conffile def get_response_cb(request, context, ignore_urls=[], visits=None): """Mount point callback to fetch on disk the content of a request This is meant to be used as 'body' argument of the requests_mock.get() method. It will look for files on the local filesystem based on the requested URL, using the following rules: - files are searched in the DATADIR/ directory - the local file name is the path part of the URL with path hierarchy markers (aka '/') replaced by '_' Eg. if you use the requests_mock fixture in your test file as: requests_mock.get('https://nowhere.com', body=get_response_cb) # or even requests_mock.get(re.compile('https://'), body=get_response_cb) then a call requests.get like: requests.get('https://nowhere.com/path/to/resource') will look the content of the response in: DATADIR/resources/nowhere.com/path_to_resource Args: request (requests.Request): Object requests context (requests.Context): Object holding response metadata information (status_code, headers, etc...) ignore_urls (List): urls whose status response should be 404 even if the local file exists visits (Optional[Dict]): Map of url, number of visits. If None, disable multi visit support (default) Returns: Optional[FileDescriptor] on the on disk file to read from the test context """ logger.debug('get_response_cb(%s, %s)', request, context) logger.debug('url: %s', request.url) logger.debug('ignore_urls: %s', ignore_urls) if request.url in ignore_urls: context.status_code = 404 return None url = urlparse(request.url) dirname = url.hostname # pypi.org | files.pythonhosted.org # url.path: pypi//json -> local file: pypi__json filename = url.path[1:] if filename.endswith('/'): filename = filename[:-1] filename = filename.replace('/', '_') filepath = path.join(DATADIR, dirname, filename) if visits is not None: visit = visits.get(url, 0) visits[url] = visit + 1 if visit: filepath = filepath + '_visit%s' % visit if not path.isfile(filepath): logger.debug('not found filepath: %s', filepath) context.status_code = 404 return None fd = open(filepath, 'rb') context.headers['content-length'] = str(path.getsize(filepath)) return fd def local_get_factory(ignore_urls=[], has_multi_visit=False): @pytest.fixture def local_get(requests_mock): if not has_multi_visit: cb = partial(get_response_cb, ignore_urls=ignore_urls) requests_mock.get(re.compile('https://'), body=cb) else: visits = {} requests_mock.get(re.compile('https'), body=partial( get_response_cb, ignore_urls=ignore_urls, visits=visits) ) return requests_mock return local_get local_get = local_get_factory([]) local_get_visits = local_get_factory(has_multi_visit=True) diff --git a/swh/loader/package/tests/resources/loader.yml b/swh/loader/package/tests/resources/loader.yml index 0c9a3fe..cba3f83 100644 --- a/swh/loader/package/tests/resources/loader.yml +++ b/swh/loader/package/tests/resources/loader.yml @@ -1,3 +1,5 @@ storage: - cls: memory - args: {} + cls: proxy + args: + cls: memory + args: {}