diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include version.txt include README.md recursive-include swh py.typed +recursive-include swh/scanner/tests/data/ * diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] norecursedirs = docs +addopts = --no-start-live-server --live-server-port 6600 -p no:django diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,7 @@ pytest +aioresponses +pytest_asyncio +pytest_flask swh.core[testing-core] swh.model[testing] +swh.storage[testing] diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/conftest.py @@ -0,0 +1,68 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest +import asyncio +import aiohttp +import os +from pathlib import PosixPath +from aioresponses import aioresponses # type: ignore + +from swh.model.cli import pid_of_file, pid_of_dir +from .flask_api import create_app + + +@pytest.fixture +def mock_aioresponse(): + with aioresponses() as m: + yield m + + +@pytest.fixture +def event_loop(): + """Fixture that generate an asyncio event loop.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + yield loop + loop.close() + + +@pytest.fixture +async def aiosession(): + """Fixture that generate an aiohttp Client Session.""" + session = aiohttp.ClientSession() + yield session + session.detach() + + +@pytest.fixture(scope='session') +def temp_paths(tmp_path_factory): + """Fixture to generate temporary paths""" + subpath = tmp_path_factory.getbasetemp() + subdir = tmp_path_factory.mktemp('subdir') + subdir2 = tmp_path_factory.mktemp('subdir2') + subfile = subpath.joinpath(PosixPath('./subfile.txt')) + subfile.touch() + + avail_path = [subdir, subdir2, subfile] + avail_pid = [pid_of_dir(bytes(subdir)), pid_of_dir(bytes(subdir2)), + pid_of_file(bytes(subfile))] + + return {'paths': avail_path, 'pids': avail_pid} + + +@pytest.fixture(scope='session') +def app(): + """Flask backend API (used by live_server).""" + app = create_app() + return app + + +@pytest.fixture +def test_folder(): + tests_path = PosixPath(os.path.abspath(__file__)).parent + tests_data_folder = tests_path.joinpath('data') + assert tests_data_folder.exists() + return tests_data_folder diff --git a/swh/scanner/tests/data.py b/swh/scanner/tests/data.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/data.py @@ -0,0 +1,19 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +correct_api_response = { + "swh:1:dir:17d207da3804cc60a77cba58e76c3b2f767cb112": + {"known": False}, + "swh:1:dir:01fa282bb80be5907505d44b4692d3fa40fad140": + {"known": True}, + "swh:1:dir:4b825dc642cb6eb9a060e54bf8d69288fbee4904": + {"known": True}} + +# present pids inside /data/sample-folder +present_pids = [ + "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a", # quotes.md + "swh:1:cnt:68769579c3eaadbe555379b9c3538e6628bae1eb", # some-binary + "swh:1:dir:9619a28687b2462efbb5be816bc1185b95753d93" # barfoo2/ + ] diff --git a/swh/scanner/tests/data/sample-folder-result.json b/swh/scanner/tests/data/sample-folder-result.json new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder-result.json @@ -0,0 +1 @@ +{"foo": {"quotes.md": "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a"}, "bar": {"barfoo2": "swh:1:dir:9619a28687b2462efbb5be816bc1185b95753d93"}, "link-to-foo": {"quotes.md": "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a"}, "some-binary": "swh:1:cnt:68769579c3eaadbe555379b9c3538e6628bae1eb"} diff --git a/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org b/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org @@ -0,0 +1,2 @@ +A Victory without danger is a triumph without glory. +-- Pierre Corneille \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt b/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt @@ -0,0 +1 @@ +this is a file. diff --git a/swh/scanner/tests/data/sample-folder/foo/barfoo b/swh/scanner/tests/data/sample-folder/foo/barfoo new file mode 120000 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/foo/barfoo @@ -0,0 +1 @@ +bar/barfoo \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/foo/quotes.md b/swh/scanner/tests/data/sample-folder/foo/quotes.md new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/foo/quotes.md @@ -0,0 +1 @@ +Shoot for the moon. Even if you miss, you'll land among the stars. \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo b/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo new file mode 120000 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo @@ -0,0 +1 @@ +../bar/barfoo \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/link-to-another-quote b/swh/scanner/tests/data/sample-folder/link-to-another-quote new file mode 120000 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/link-to-another-quote @@ -0,0 +1 @@ +bar/barfoo/another-quote.org \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/link-to-foo b/swh/scanner/tests/data/sample-folder/link-to-foo new file mode 120000 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/link-to-foo @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/some-binary b/swh/scanner/tests/data/sample-folder/some-binary new file mode 100755 --- /dev/null +++ b/swh/scanner/tests/data/sample-folder/some-binary @@ -0,0 +1 @@ +exec diff --git a/swh/scanner/tests/flask_api.py b/swh/scanner/tests/flask_api.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/flask_api.py @@ -0,0 +1,24 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from flask import Flask, request + +from .data import present_pids + + +def create_app(): + app = Flask(__name__) + + @app.route('/known/', methods=['POST']) + def known(): + pids = request.get_json() + res = {pid: {'known': False} for pid in pids} + for pid in pids: + if pid in present_pids: + res[pid]['known'] = True + + return res + + return app diff --git a/swh/scanner/tests/test_model.py b/swh/scanner/tests/test_model.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/test_model.py @@ -0,0 +1,4 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information diff --git a/swh/scanner/tests/test_scanner.py b/swh/scanner/tests/test_scanner.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/test_scanner.py @@ -0,0 +1,65 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest +import json +from pathlib import PosixPath + +from .data import correct_api_response + +from swh.scanner.scanner import pids_discovery, get_subpaths, run +from swh.scanner.model import Tree +from swh.scanner.exceptions import APIError + +aio_url = 'http://example.org/api/known/' + + +def test_scanner_correct_api_request(mock_aioresponse, event_loop, aiosession): + mock_aioresponse.post(aio_url, status=200, content_type='application/json', + body=json.dumps(correct_api_response)) + + actual_result = event_loop.run_until_complete( + pids_discovery([], aiosession, 'http://example.org/api/')) + + assert correct_api_response == actual_result + + +def test_scanner_raise_apierror(mock_aioresponse, event_loop, aiosession): + mock_aioresponse.post(aio_url, content_type='application/json', + status=413) + + with pytest.raises(APIError): + event_loop.run_until_complete( + pids_discovery([], aiosession, 'http://example.org/api/')) + + +def test_scanner_get_subpaths(tmp_path, temp_paths): + for subpath, pid in get_subpaths(tmp_path): + assert subpath in temp_paths['paths'] + assert pid in temp_paths['pids'] + + +@pytest.mark.options(debug=False) +def test_app(app): + assert not app.debug + + +def test_scanner_result(live_server, event_loop, test_folder): + live_server.start() + api_url = live_server.url() + '/' + + result_path = test_folder.joinpath(PosixPath('sample-folder-result.json')) + with open(result_path, 'r') as json_file: + expected_result = json.loads(json_file.read()) + + sample_folder = test_folder.joinpath(PosixPath('sample-folder')) + + source_tree = Tree(sample_folder) + event_loop.run_until_complete( + run(sample_folder, api_url, source_tree)) + + actual_result = source_tree.getJsonTree() + + assert actual_result == expected_result