diff --git a/MANIFEST.in b/MANIFEST.in index 59629e9..23ad7f5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,10 @@ include Makefile include requirements.txt include requirements-swh.txt include requirements-http.txt include requirements-db.txt include version.txt recursive-include swh/core/sql *.sql recursive-include swh py.typed +recursive-include swh/core/tests/data/ * +recursive-include swh/core/tests/fixture/data/ * diff --git a/PKG-INFO b/PKG-INFO index 85b217c..b103c2d 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,88 +1,88 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.70 +Version: 0.0.72 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Funding, https://www.softwareheritage.org/donate Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - database connection - http-based RPC client/server Development ----------- We strongly recommend you to use a [virtualenv][1] if you want to run tests or hack the code. To set up your development environment: ``` (swh) user@host:~/swh-environment/swh-core$ pip install -e .[testing] ``` This will install every Python package needed to run this package's tests. Unit tests can be executed using [pytest][2] or [tox][3]. ``` (swh) user@host:~/swh-environment/swh-core$ pytest ============================== test session starts ============================== platform linux -- Python 3.7.3, pytest-3.10.1, py-1.8.0, pluggy-0.12.0 hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/ddouard/src/swh-environment/swh-core/.hypothesis/examples') rootdir: /home/ddouard/src/swh-environment/swh-core, inifile: pytest.ini plugins: requests-mock-1.6.0, hypothesis-4.26.4, celery-4.3.0, postgresql-1.4.1 collected 89 items swh/core/api/tests/test_api.py .. [ 2%] swh/core/api/tests/test_async.py .... [ 6%] swh/core/api/tests/test_serializers.py ..... [ 12%] swh/core/db/tests/test_db.py .... [ 16%] swh/core/tests/test_cli.py ...... [ 23%] swh/core/tests/test_config.py .............. [ 39%] swh/core/tests/test_statsd.py ........................................... [ 87%] .... [ 92%] swh/core/tests/test_utils.py ....... [100%] ===================== 89 passed, 9 warnings in 6.94 seconds ===================== ``` Note: this git repository uses [pre-commit][4] hooks to ensure better and more consistent code. It should already be installed in your virtualenv (if not, just type `pip install pre-commit`). Make sure to activate it in your local copy of the git repository: ``` (swh) user@host:~/swh-environment/swh-core$ pre-commit install pre-commit installed at .git/hooks/pre-commit ``` Please read the [developer setup manual][5] for more information on how to hack on Software Heritage. [1]: https://virtualenv.pypa.io [2]: https://docs.pytest.org [3]: https://tox.readthedocs.io [4]: https://pre-commit.com [5]: https://docs.softwareheritage.org/devel/developer-setup.html Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown -Provides-Extra: testing Provides-Extra: http Provides-Extra: db +Provides-Extra: testing diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 66235fe..0d06fc6 --- a/setup.py +++ b/setup.py @@ -1,79 +1,82 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(*names): requirements = [] for name in names: if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' if not os.path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.core', description='Software Heritage core utilities', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), + py_modules=['pytest_swh_core'], scripts=[], install_requires=parse_requirements(None, 'swh'), setup_requires=['vcversioner'], extras_require={ 'testing': parse_requirements('test', 'db', 'http'), 'db': parse_requirements('db'), 'http': parse_requirements('http'), }, vcversioner={}, include_package_data=True, entry_points=''' [console_scripts] swh=swh.core.cli:main swh-db-init=swh.core.cli.db:db_init [swh.cli.subcommands] db=swh.core.cli.db:db db-init=swh.core.cli.db:db_init + [pytest11] + pytest_swh_core = swh.core.pytest_plugin ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-core', }, ) diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 85b217c..b103c2d 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,88 +1,88 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.70 +Version: 0.0.72 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Funding, https://www.softwareheritage.org/donate Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - database connection - http-based RPC client/server Development ----------- We strongly recommend you to use a [virtualenv][1] if you want to run tests or hack the code. To set up your development environment: ``` (swh) user@host:~/swh-environment/swh-core$ pip install -e .[testing] ``` This will install every Python package needed to run this package's tests. Unit tests can be executed using [pytest][2] or [tox][3]. ``` (swh) user@host:~/swh-environment/swh-core$ pytest ============================== test session starts ============================== platform linux -- Python 3.7.3, pytest-3.10.1, py-1.8.0, pluggy-0.12.0 hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/ddouard/src/swh-environment/swh-core/.hypothesis/examples') rootdir: /home/ddouard/src/swh-environment/swh-core, inifile: pytest.ini plugins: requests-mock-1.6.0, hypothesis-4.26.4, celery-4.3.0, postgresql-1.4.1 collected 89 items swh/core/api/tests/test_api.py .. [ 2%] swh/core/api/tests/test_async.py .... [ 6%] swh/core/api/tests/test_serializers.py ..... [ 12%] swh/core/db/tests/test_db.py .... [ 16%] swh/core/tests/test_cli.py ...... [ 23%] swh/core/tests/test_config.py .............. [ 39%] swh/core/tests/test_statsd.py ........................................... [ 87%] .... [ 92%] swh/core/tests/test_utils.py ....... [100%] ===================== 89 passed, 9 warnings in 6.94 seconds ===================== ``` Note: this git repository uses [pre-commit][4] hooks to ensure better and more consistent code. It should already be installed in your virtualenv (if not, just type `pip install pre-commit`). Make sure to activate it in your local copy of the git repository: ``` (swh) user@host:~/swh-environment/swh-core$ pre-commit install pre-commit installed at .git/hooks/pre-commit ``` Please read the [developer setup manual][5] for more information on how to hack on Software Heritage. [1]: https://virtualenv.pypa.io [2]: https://docs.pytest.org [3]: https://tox.readthedocs.io [4]: https://pre-commit.com [5]: https://docs.softwareheritage.org/devel/developer-setup.html Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown -Provides-Extra: testing Provides-Extra: http Provides-Extra: db +Provides-Extra: testing diff --git a/swh.core.egg-info/SOURCES.txt b/swh.core.egg-info/SOURCES.txt index e9cb2f9..79cb3c3 100644 --- a/swh.core.egg-info/SOURCES.txt +++ b/swh.core.egg-info/SOURCES.txt @@ -1,51 +1,60 @@ MANIFEST.in Makefile README.md requirements-db.txt requirements-http.txt requirements-swh.txt requirements.txt setup.py version.txt swh/__init__.py swh.core.egg-info/PKG-INFO swh.core.egg-info/SOURCES.txt swh.core.egg-info/dependency_links.txt swh.core.egg-info/entry_points.txt swh.core.egg-info/requires.txt swh.core.egg-info/top_level.txt swh/core/__init__.py swh/core/api_async.py swh/core/config.py swh/core/logger.py swh/core/py.typed +swh/core/pytest_plugin.py swh/core/statsd.py swh/core/tarball.py swh/core/utils.py swh/core/api/__init__.py swh/core/api/asynchronous.py swh/core/api/negotiation.py swh/core/api/serializers.py swh/core/api/tests/__init__.py swh/core/api/tests/server_testing.py swh/core/api/tests/test_api.py swh/core/api/tests/test_async.py swh/core/api/tests/test_serializers.py swh/core/cli/__init__.py swh/core/cli/db.py swh/core/db/__init__.py swh/core/db/common.py swh/core/db/db_utils.py swh/core/db/tests/__init__.py swh/core/db/tests/conftest.py swh/core/db/tests/db_testing.py swh/core/db/tests/test_cli.py swh/core/db/tests/test_db.py swh/core/sql/log-schema.sql swh/core/tests/__init__.py swh/core/tests/test_cli.py swh/core/tests/test_config.py swh/core/tests/test_logger.py +swh/core/tests/test_pytest_plugin.py swh/core/tests/test_statsd.py swh/core/tests/test_tarball.py -swh/core/tests/test_utils.py \ No newline at end of file +swh/core/tests/test_utils.py +swh/core/tests/data/example.com/file.json +swh/core/tests/data/example.com/file.json_visit1 +swh/core/tests/data/example.com/other.json +swh/core/tests/fixture/__init__.py +swh/core/tests/fixture/conftest.py +swh/core/tests/fixture/test_pytest_plugin.py +swh/core/tests/fixture/data/example.com/file.json \ No newline at end of file diff --git a/swh.core.egg-info/entry_points.txt b/swh.core.egg-info/entry_points.txt index 403254e..57fce23 100644 --- a/swh.core.egg-info/entry_points.txt +++ b/swh.core.egg-info/entry_points.txt @@ -1,8 +1,10 @@ [console_scripts] swh=swh.core.cli:main swh-db-init=swh.core.cli.db:db_init [swh.cli.subcommands] db=swh.core.cli.db:db db-init=swh.core.cli.db:db_init + [pytest11] + pytest_swh_core = swh.core.pytest_plugin \ No newline at end of file diff --git a/swh.core.egg-info/top_level.txt b/swh.core.egg-info/top_level.txt index 0cb0f8f..18e4556 100644 --- a/swh.core.egg-info/top_level.txt +++ b/swh.core.egg-info/top_level.txt @@ -1 +1,2 @@ +pytest_swh_core swh diff --git a/swh/core/pytest_plugin.py b/swh/core/pytest_plugin.py new file mode 100644 index 0000000..f24e3d7 --- /dev/null +++ b/swh/core/pytest_plugin.py @@ -0,0 +1,175 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import logging +import re +import pytest + +from functools import partial +from os import path +from typing import Dict, List, Optional +from urllib.parse import urlparse + + +logger = logging.getLogger(__name__) + + +# Check get_local_factory function +# Maximum number of iteration checks to generate requests responses +MAX_VISIT_FILES = 10 + + +def get_response_cb(request, context, datadir, + ignore_urls: List[str] = [], + visits: Optional[Dict] = None): + """Mount point callback to fetch on disk the request's content. + + This is meant to be used as 'body' argument of the requests_mock.get() + method. + + It will look for files on the local filesystem based on the requested URL, + using the following rules: + + - files are searched in the datadir/ directory + + - the local file name is the path part of the URL with path hierarchy + markers (aka '/') replaced by '_' + + Eg. if you use the requests_mock fixture in your test file as: + + requests_mock.get('https://nowhere.com', body=get_response_cb) + # or even + requests_mock.get(re.compile('https://'), body=get_response_cb) + + then a call requests.get like: + + requests.get('https://nowhere.com/path/to/resource') + + will look the content of the response in: + + datadir/nowhere.com/path_to_resource + + Args: + request (requests.Request): Object requests + context (requests.Context): Object holding response metadata + information (status_code, headers, etc...) + ignore_urls: urls whose status response should be 404 even if the local + file exists + visits: Dict of url, number of visits. If None, disable multi visit + support (default) + + Returns: + Optional[FileDescriptor] on disk file to read from the test context + + """ + logger.debug('get_response_cb(%s, %s)', request, context) + logger.debug('url: %s', request.url) + logger.debug('ignore_urls: %s', ignore_urls) + if request.url in ignore_urls: + context.status_code = 404 + return None + url = urlparse(request.url) + dirname = url.hostname # pypi.org | files.pythonhosted.org + # url.path: pypi//json -> local file: pypi__json + filename = url.path[1:] + if filename.endswith('/'): + filename = filename[:-1] + filename = filename.replace('/', '_') + filepath = path.join(datadir, dirname, filename) + if visits is not None: + visit = visits.get(url, 0) + visits[url] = visit + 1 + if visit: + filepath = filepath + '_visit%s' % visit + + if not path.isfile(filepath): + logger.debug('not found filepath: %s', filepath) + context.status_code = 404 + return None + fd = open(filepath, 'rb') + context.headers['content-length'] = str(path.getsize(filepath)) + return fd + + +@pytest.fixture +def datadir(request): + """By default, returns the test directory's data directory. + + This can be overriden on a per arborescence basis. Add an override + definition in the local conftest, for example: + + import pytest + + from os import path + + @pytest.fixture + def datadir(): + return path.join(path.abspath(path.dirname(__file__)), 'resources') + + + """ + return path.join(path.dirname(str(request.fspath)), 'data') + + +def requests_mock_datadir_factory(ignore_urls: List[str] = [], + has_multi_visit: bool = False): + """This factory generates fixture which allow to look for files on the + local filesystem based on the requested URL, using the following rules: + + - files are searched in the datadir/ directory + + - the local file name is the path part of the URL with path hierarchy + markers (aka '/') replaced by '_' + + Multiple implementations are possible, for example: + + - requests_mock_datadir_factory([]): + This computes the file name from the query and always returns the same + result. + + - requests_mock_datadir_factory(has_multi_visit=True): + This computes the file name from the query and returns the content of + the filename the first time, the next call returning the content of + files suffixed with _visit1 and so on and so forth. If the file is not + found, returns a 404. + + - requests_mock_datadir_factory(ignore_urls=['url1', 'url2']): + This will ignore any files corresponding to url1 and url2, always + returning 404. + + Args: + ignore_urls: List of urls to always returns 404 (whether file + exists or not) + has_multi_visit: Activate or not the multiple visits behavior + + """ + @pytest.fixture + def requests_mock_datadir(requests_mock, datadir): + if not has_multi_visit: + cb = partial(get_response_cb, + ignore_urls=ignore_urls, + datadir=datadir) + requests_mock.get(re.compile('https://'), body=cb) + else: + visits = {} + requests_mock.get(re.compile('https://'), body=partial( + get_response_cb, ignore_urls=ignore_urls, visits=visits, + datadir=datadir) + ) + + return requests_mock + + return requests_mock_datadir + + +# Default `requests_mock_datadir` implementation +requests_mock_datadir = requests_mock_datadir_factory([]) + +# Implementation for multiple visits behavior: +# - first time, it checks for a file named `filename` +# - second time, it checks for a file named `filename`_visit1 +# etc... +requests_mock_datadir_visits = requests_mock_datadir_factory( + has_multi_visit=True) diff --git a/swh/core/tests/data/example.com/file.json b/swh/core/tests/data/example.com/file.json new file mode 100644 index 0000000..776c3e2 --- /dev/null +++ b/swh/core/tests/data/example.com/file.json @@ -0,0 +1,3 @@ +{ + "hello": "you" +} diff --git a/swh/core/tests/data/example.com/file.json_visit1 b/swh/core/tests/data/example.com/file.json_visit1 new file mode 100644 index 0000000..ea50c5d --- /dev/null +++ b/swh/core/tests/data/example.com/file.json_visit1 @@ -0,0 +1,3 @@ +{ + "hello": "world" +} diff --git a/swh/core/tests/data/example.com/other.json b/swh/core/tests/data/example.com/other.json new file mode 100644 index 0000000..14e24d4 --- /dev/null +++ b/swh/core/tests/data/example.com/other.json @@ -0,0 +1 @@ +"foobar" diff --git a/swh/core/tests/fixture/__init__.py b/swh/core/tests/fixture/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/core/tests/fixture/conftest.py b/swh/core/tests/fixture/conftest.py new file mode 100644 index 0000000..399adac --- /dev/null +++ b/swh/core/tests/fixture/conftest.py @@ -0,0 +1,16 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from os import path + + +DATADIR = path.join(path.abspath(path.dirname(__file__)), 'data') + + +@pytest.fixture +def datadir(): + return DATADIR diff --git a/swh/core/tests/fixture/data/example.com/file.json b/swh/core/tests/fixture/data/example.com/file.json new file mode 100644 index 0000000..000a8dd --- /dev/null +++ b/swh/core/tests/fixture/data/example.com/file.json @@ -0,0 +1,3 @@ +{ + "welcome": "you" +} diff --git a/swh/core/tests/fixture/test_pytest_plugin.py b/swh/core/tests/fixture/test_pytest_plugin.py new file mode 100644 index 0000000..534a7b7 --- /dev/null +++ b/swh/core/tests/fixture/test_pytest_plugin.py @@ -0,0 +1,25 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import requests + +from .conftest import DATADIR + +# In this arborescence, we override in the local conftest.py module the +# "datadir" fixture to specify where to retrieve the data files from. + + +def test_requests_mock_datadir_with_datadir_fixture_override( + requests_mock_datadir): + """Override datadir fixture should retrieve data from elsewhere + + """ + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'welcome': 'you'} + + +def test_data_dir_override(datadir): + assert datadir == DATADIR diff --git a/swh/core/tests/test_pytest_plugin.py b/swh/core/tests/test_pytest_plugin.py new file mode 100644 index 0000000..151059e --- /dev/null +++ b/swh/core/tests/test_pytest_plugin.py @@ -0,0 +1,90 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import requests + +from os import path + +from swh.core.pytest_plugin import requests_mock_datadir_factory + + +def test_get_response_cb_with_visits_nominal(requests_mock_datadir_visits): + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'hello': 'you'} + + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'hello': 'world'} + + response = requests.get('https://example.com/file.json') + assert not response.ok + assert response.status_code == 404 + + +def test_get_response_cb_with_visits(requests_mock_datadir_visits): + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'hello': 'you'} + + response = requests.get('https://example.com/other.json') + assert response.ok + assert response.json() == "foobar" + + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'hello': 'world'} + + response = requests.get('https://example.com/other.json') + assert not response.ok + assert response.status_code == 404 + + response = requests.get('https://example.com/file.json') + assert not response.ok + assert response.status_code == 404 + + +def test_get_response_cb_no_visit(requests_mock_datadir): + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'hello': 'you'} + + response = requests.get('https://example.com/file.json') + assert response.ok + assert response.json() == {'hello': 'you'} + + +requests_mock_datadir_ignore = requests_mock_datadir_factory( + ignore_urls=['https://example.com/file.json'], + has_multi_visit=False, +) + + +def test_get_response_cb_ignore_url(requests_mock_datadir_ignore): + response = requests.get('https://example.com/file.json') + assert not response.ok + assert response.status_code == 404 + + +requests_mock_datadir_ignore_and_visit = requests_mock_datadir_factory( + ignore_urls=['https://example.com/file.json'], + has_multi_visit=True, +) + + +def test_get_response_cb_ignore_url_with_visit( + requests_mock_datadir_ignore_and_visit): + response = requests.get('https://example.com/file.json') + assert not response.ok + assert response.status_code == 404 + + response = requests.get('https://example.com/file.json') + assert not response.ok + assert response.status_code == 404 + + +def test_data_dir(datadir): + expected_datadir = path.join(path.abspath(path.dirname(__file__)), 'data') + assert datadir == expected_datadir diff --git a/version.txt b/version.txt index 7a3c374..2faa5b6 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.70-0-g7a7d06a \ No newline at end of file +v0.0.72-0-g576e21c \ No newline at end of file