diff --git a/MANIFEST.in b/MANIFEST.in index 1d00492..d2529de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include Makefile include requirements*.txt include version.txt include README.md recursive-include swh py.typed -recursive-include swh/scanner/tests/data/ * +recursive-include swh/scanner/tests/data/ *.tgz diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py index 91decd6..58b4c5a 100644 --- a/swh/scanner/tests/conftest.py +++ b/swh/scanner/tests/conftest.py @@ -1,136 +1,140 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest import asyncio import aiohttp import os +import shutil + from pathlib import PosixPath from aioresponses import aioresponses # type: ignore from swh.model.cli import swhid_of_file, swhid_of_dir from swh.scanner.model import Tree from .flask_api import create_app @pytest.fixture def mock_aioresponse(): with aioresponses() as m: yield m @pytest.fixture def event_loop(): """Fixture that generate an asyncio event loop.""" loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) yield loop loop.close() @pytest.fixture async def aiosession(): """Fixture that generate an aiohttp Client Session.""" session = aiohttp.ClientSession() yield session session.detach() @pytest.fixture(scope="session") def temp_folder(tmp_path_factory): """Fixture that generates a temporary folder with the following structure: .. code-block:: python root = { subdir: { subsubdir filesample.txt filesample2.txt } subdir2 subfile.txt } """ root = tmp_path_factory.getbasetemp() subdir = tmp_path_factory.mktemp("subdir") subsubdir = subdir.joinpath("subsubdir") subsubdir.mkdir() subdir2 = tmp_path_factory.mktemp("subdir2") subfile = root / "subfile.txt" subfile.touch() filesample = subdir / "filesample.txt" filesample.touch() filesample2 = subdir / "filesample2.txt" filesample2.touch() avail_path = { subdir: swhid_of_dir(bytes(subdir)), subsubdir: swhid_of_dir(bytes(subsubdir)), subdir2: swhid_of_dir(bytes(subdir2)), subfile: swhid_of_file(bytes(subfile)), filesample: swhid_of_file(bytes(filesample)), filesample2: swhid_of_file(bytes(filesample2)), } return { "root": root, "paths": avail_path, "filesample": filesample, "filesample2": filesample2, "subsubdir": subsubdir, "subdir": subdir, } @pytest.fixture(scope="function") def example_tree(temp_folder): """Fixture that generate a Tree with the root present in the session fixture "temp_folder". """ example_tree = Tree(temp_folder["root"]) assert example_tree.path == temp_folder["root"] return example_tree @pytest.fixture(scope="function") def example_dirs(example_tree, temp_folder): """ Fixture that fill the fixture example_tree with the values contained in the fixture temp_folder and returns the directories information of the filled example_tree. """ root = temp_folder["root"] filesample_path = temp_folder["filesample"] filesample2_path = temp_folder["filesample2"] subsubdir_path = temp_folder["subsubdir"] known_paths = [filesample_path, filesample2_path, subsubdir_path] for path, swhid in temp_folder["paths"].items(): if path in known_paths: example_tree.addNode(path, swhid, True) else: example_tree.addNode(path, swhid, False) return example_tree.getDirectoriesInfo(root) @pytest.fixture -def test_folder(): +def test_sample_folder(datadir, tmp_path): """Location of the "data" folder """ - tests_path = PosixPath(os.path.abspath(__file__)).parent - tests_data_folder = tests_path.joinpath("data") - assert tests_data_folder.exists() - return tests_data_folder + archive_path = PosixPath(os.path.join(datadir, "sample-folder.tgz")) + assert archive_path.exists() + shutil.unpack_archive(archive_path, extract_dir=tmp_path) + test_sample_folder = PosixPath(os.path.join(tmp_path, "sample-folder")) + assert test_sample_folder.exists() + return test_sample_folder @pytest.fixture(scope="session") def app(): """Flask backend API (used by live_server).""" app = create_app() return app diff --git a/swh/scanner/tests/data/sample-folder.tgz b/swh/scanner/tests/data/sample-folder.tgz new file mode 100644 index 0000000..da27921 Binary files /dev/null and b/swh/scanner/tests/data/sample-folder.tgz differ diff --git a/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org b/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org deleted file mode 100644 index 133693b..0000000 --- a/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org +++ /dev/null @@ -1,2 +0,0 @@ -A Victory without danger is a triumph without glory. --- Pierre Corneille \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt b/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt deleted file mode 100644 index 620d458..0000000 --- a/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt +++ /dev/null @@ -1 +0,0 @@ -this is a file. diff --git a/swh/scanner/tests/data/sample-folder/foo/barfoo b/swh/scanner/tests/data/sample-folder/foo/barfoo deleted file mode 120000 index 8185dfb..0000000 --- a/swh/scanner/tests/data/sample-folder/foo/barfoo +++ /dev/null @@ -1 +0,0 @@ -bar/barfoo \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/foo/quotes.md b/swh/scanner/tests/data/sample-folder/foo/quotes.md deleted file mode 100644 index 7c4c57b..0000000 --- a/swh/scanner/tests/data/sample-folder/foo/quotes.md +++ /dev/null @@ -1 +0,0 @@ -Shoot for the moon. Even if you miss, you'll land among the stars. \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo b/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo deleted file mode 120000 index acac326..0000000 --- a/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo +++ /dev/null @@ -1 +0,0 @@ -../bar/barfoo \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/link-to-another-quote b/swh/scanner/tests/data/sample-folder/link-to-another-quote deleted file mode 120000 index 7d5c081..0000000 --- a/swh/scanner/tests/data/sample-folder/link-to-another-quote +++ /dev/null @@ -1 +0,0 @@ -bar/barfoo/another-quote.org \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/link-to-foo b/swh/scanner/tests/data/sample-folder/link-to-foo deleted file mode 120000 index 1910281..0000000 --- a/swh/scanner/tests/data/sample-folder/link-to-foo +++ /dev/null @@ -1 +0,0 @@ -foo \ No newline at end of file diff --git a/swh/scanner/tests/data/sample-folder/some-binary b/swh/scanner/tests/data/sample-folder/some-binary deleted file mode 100755 index 6876957..0000000 --- a/swh/scanner/tests/data/sample-folder/some-binary +++ /dev/null @@ -1 +0,0 @@ -exec diff --git a/swh/scanner/tests/data/sample-folder/toexclude/example.txt b/swh/scanner/tests/data/sample-folder/toexclude/example.txt deleted file mode 100644 index 5f1cfce..0000000 --- a/swh/scanner/tests/data/sample-folder/toexclude/example.txt +++ /dev/null @@ -1 +0,0 @@ -example file diff --git a/swh/scanner/tests/test_scanner.py b/swh/scanner/tests/test_scanner.py index f580ef7..0e26780 100644 --- a/swh/scanner/tests/test_scanner.py +++ b/swh/scanner/tests/test_scanner.py @@ -1,106 +1,103 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest import json -from pathlib import PosixPath from .data import correct_api_response, present_swhids, to_exclude_swhid from swh.scanner.scanner import swhids_discovery, get_subpaths, run from swh.scanner.model import Tree from swh.scanner.cli import extract_regex_objs from swh.scanner.exceptions import APIError aio_url = "http://example.org/api/known/" def test_scanner_correct_api_request(mock_aioresponse, event_loop, aiosession): mock_aioresponse.post( aio_url, status=200, content_type="application/json", body=json.dumps(correct_api_response), ) actual_result = event_loop.run_until_complete( swhids_discovery([], aiosession, "http://example.org/api/") ) assert correct_api_response == actual_result def test_scanner_raise_apierror(mock_aioresponse, event_loop, aiosession): mock_aioresponse.post(aio_url, content_type="application/json", status=413) with pytest.raises(APIError): event_loop.run_until_complete( swhids_discovery([], aiosession, "http://example.org/api/") ) def test_scanner_raise_apierror_input_size_limit(event_loop, aiosession, live_server): api_url = live_server.url() + "/" request = [ "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a" for i in range(901) ] # /known/ is limited at 900 with pytest.raises(APIError): event_loop.run_until_complete(swhids_discovery(request, aiosession, api_url)) def test_scanner_get_subpaths(temp_folder): root = temp_folder["root"] actual_result = [] for subpath, swhid in get_subpaths(root, tuple()): # also check if it's a symlink since pytest tmp_dir fixture create # also a symlink to each directory inside the tmp_dir path if subpath.is_dir() and not subpath.is_symlink(): actual_result.append((subpath, swhid)) assert len(actual_result) == 2 @pytest.mark.options(debug=False) def test_app(app): assert not app.debug -def test_scanner_result(live_server, event_loop, test_folder): +def test_scanner_result(live_server, event_loop, test_sample_folder): api_url = live_server.url() + "/" - sample_folder = test_folder.joinpath(PosixPath("sample-folder")) - - source_tree = Tree(sample_folder) - event_loop.run_until_complete(run(sample_folder, api_url, source_tree, set())) + source_tree = Tree(test_sample_folder) + event_loop.run_until_complete(run(test_sample_folder, api_url, source_tree, set())) for child_node in source_tree.iterate(): node_info = list(child_node.attributes.values())[0] if node_info["swhid"] in present_swhids: assert node_info["known"] is True else: assert node_info["known"] is False -def test_scanner_result_with_exclude_patterns(live_server, event_loop, test_folder): +def test_scanner_result_with_exclude_patterns( + live_server, event_loop, test_sample_folder +): api_url = live_server.url() + "/" - sample_folder = test_folder.joinpath(PosixPath("sample-folder")) - - patterns = (str(sample_folder) + "/toexclude",) + patterns = (str(test_sample_folder) + "/toexclude",) exclude_pattern = { - reg_obj for reg_obj in extract_regex_objs(sample_folder, patterns) + reg_obj for reg_obj in extract_regex_objs(test_sample_folder, patterns) } - source_tree = Tree(sample_folder) + source_tree = Tree(test_sample_folder) event_loop.run_until_complete( - run(sample_folder, api_url, source_tree, exclude_pattern) + run(test_sample_folder, api_url, source_tree, exclude_pattern) ) for child_node in source_tree.iterate(): node_info = list(child_node.attributes.values())[0] assert node_info["swhid"] != to_exclude_swhid