Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9338980
D2791.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
9 KB
Subscribers
None
D2791.diff
View Options
diff --git a/MANIFEST.in b/MANIFEST.in
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,3 +3,4 @@
include version.txt
include README.md
recursive-include swh py.typed
+recursive-include swh/scanner/tests/data/ *
diff --git a/pytest.ini b/pytest.ini
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,2 +1,3 @@
[pytest]
norecursedirs = docs
+addopts = --no-start-live-server --live-server-port 6600 -p no:django
diff --git a/requirements-test.txt b/requirements-test.txt
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,3 +1,7 @@
pytest
+aioresponses
+pytest_asyncio
+pytest_flask
swh.core[testing-core]
swh.model[testing]
+swh.storage[testing]
diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/conftest.py
@@ -0,0 +1,68 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+import asyncio
+import aiohttp
+import os
+from pathlib import PosixPath
+from aioresponses import aioresponses # type: ignore
+
+from swh.model.cli import pid_of_file, pid_of_dir
+from .flask_api import create_app
+
+
+@pytest.fixture
+def mock_aioresponse():
+ with aioresponses() as m:
+ yield m
+
+
+@pytest.fixture
+def event_loop():
+ """Fixture that generate an asyncio event loop."""
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ yield loop
+ loop.close()
+
+
+@pytest.fixture
+async def aiosession():
+ """Fixture that generate an aiohttp Client Session."""
+ session = aiohttp.ClientSession()
+ yield session
+ session.detach()
+
+
+@pytest.fixture(scope='session')
+def temp_paths(tmp_path_factory):
+ """Fixture to generate temporary paths"""
+ subpath = tmp_path_factory.getbasetemp()
+ subdir = tmp_path_factory.mktemp('subdir')
+ subdir2 = tmp_path_factory.mktemp('subdir2')
+ subfile = subpath.joinpath(PosixPath('./subfile.txt'))
+ subfile.touch()
+
+ avail_path = [subdir, subdir2, subfile]
+ avail_pid = [pid_of_dir(bytes(subdir)), pid_of_dir(bytes(subdir2)),
+ pid_of_file(bytes(subfile))]
+
+ return {'paths': avail_path, 'pids': avail_pid}
+
+
+@pytest.fixture(scope='session')
+def app():
+ """Flask backend API (used by live_server)."""
+ app = create_app()
+ return app
+
+
+@pytest.fixture
+def test_folder():
+ tests_path = PosixPath(os.path.abspath(__file__)).parent
+ tests_data_folder = tests_path.joinpath('data')
+ assert tests_data_folder.exists()
+ return tests_data_folder
diff --git a/swh/scanner/tests/data.py b/swh/scanner/tests/data.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/data.py
@@ -0,0 +1,19 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+correct_api_response = {
+ "swh:1:dir:17d207da3804cc60a77cba58e76c3b2f767cb112":
+ {"known": False},
+ "swh:1:dir:01fa282bb80be5907505d44b4692d3fa40fad140":
+ {"known": True},
+ "swh:1:dir:4b825dc642cb6eb9a060e54bf8d69288fbee4904":
+ {"known": True}}
+
+# present pids inside /data/sample-folder
+present_pids = [
+ "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a", # quotes.md
+ "swh:1:cnt:68769579c3eaadbe555379b9c3538e6628bae1eb", # some-binary
+ "swh:1:dir:9619a28687b2462efbb5be816bc1185b95753d93" # barfoo2/
+ ]
diff --git a/swh/scanner/tests/data/sample-folder-result.json b/swh/scanner/tests/data/sample-folder-result.json
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder-result.json
@@ -0,0 +1 @@
+{"foo": {"quotes.md": "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a"}, "bar": {"barfoo2": "swh:1:dir:9619a28687b2462efbb5be816bc1185b95753d93"}, "link-to-foo": {"quotes.md": "swh:1:cnt:7c4c57ba9ff496ad179b8f65b1d286edbda34c9a"}, "some-binary": "swh:1:cnt:68769579c3eaadbe555379b9c3538e6628bae1eb"}
diff --git a/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org b/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/bar/barfoo/another-quote.org
@@ -0,0 +1,2 @@
+A Victory without danger is a triumph without glory.
+-- Pierre Corneille
\ No newline at end of file
diff --git a/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt b/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/bar/barfoo2/some-file.txt
@@ -0,0 +1 @@
+this is a file.
diff --git a/swh/scanner/tests/data/sample-folder/foo/barfoo b/swh/scanner/tests/data/sample-folder/foo/barfoo
new file mode 120000
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/foo/barfoo
@@ -0,0 +1 @@
+bar/barfoo
\ No newline at end of file
diff --git a/swh/scanner/tests/data/sample-folder/foo/quotes.md b/swh/scanner/tests/data/sample-folder/foo/quotes.md
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/foo/quotes.md
@@ -0,0 +1 @@
+Shoot for the moon. Even if you miss, you'll land among the stars.
\ No newline at end of file
diff --git a/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo b/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo
new file mode 120000
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/foo/rel-link-to-barfoo
@@ -0,0 +1 @@
+../bar/barfoo
\ No newline at end of file
diff --git a/swh/scanner/tests/data/sample-folder/link-to-another-quote b/swh/scanner/tests/data/sample-folder/link-to-another-quote
new file mode 120000
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/link-to-another-quote
@@ -0,0 +1 @@
+bar/barfoo/another-quote.org
\ No newline at end of file
diff --git a/swh/scanner/tests/data/sample-folder/link-to-foo b/swh/scanner/tests/data/sample-folder/link-to-foo
new file mode 120000
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/link-to-foo
@@ -0,0 +1 @@
+foo
\ No newline at end of file
diff --git a/swh/scanner/tests/data/sample-folder/some-binary b/swh/scanner/tests/data/sample-folder/some-binary
new file mode 100755
--- /dev/null
+++ b/swh/scanner/tests/data/sample-folder/some-binary
@@ -0,0 +1 @@
+exec
diff --git a/swh/scanner/tests/flask_api.py b/swh/scanner/tests/flask_api.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/flask_api.py
@@ -0,0 +1,24 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from flask import Flask, request
+
+from .data import present_pids
+
+
+def create_app():
+ app = Flask(__name__)
+
+ @app.route('/known/', methods=['POST'])
+ def known():
+ pids = request.get_json()
+ res = {pid: {'known': False} for pid in pids}
+ for pid in pids:
+ if pid in present_pids:
+ res[pid]['known'] = True
+
+ return res
+
+ return app
diff --git a/swh/scanner/tests/test_model.py b/swh/scanner/tests/test_model.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/test_model.py
@@ -0,0 +1,4 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
diff --git a/swh/scanner/tests/test_scanner.py b/swh/scanner/tests/test_scanner.py
new file mode 100644
--- /dev/null
+++ b/swh/scanner/tests/test_scanner.py
@@ -0,0 +1,65 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+import json
+from pathlib import PosixPath
+
+from .data import correct_api_response
+
+from swh.scanner.scanner import pids_discovery, get_subpaths, run
+from swh.scanner.model import Tree
+from swh.scanner.exceptions import APIError
+
+aio_url = 'http://example.org/api/known/'
+
+
+def test_scanner_correct_api_request(mock_aioresponse, event_loop, aiosession):
+ mock_aioresponse.post(aio_url, status=200, content_type='application/json',
+ body=json.dumps(correct_api_response))
+
+ actual_result = event_loop.run_until_complete(
+ pids_discovery([], aiosession, 'http://example.org/api/'))
+
+ assert correct_api_response == actual_result
+
+
+def test_scanner_raise_apierror(mock_aioresponse, event_loop, aiosession):
+ mock_aioresponse.post(aio_url, content_type='application/json',
+ status=413)
+
+ with pytest.raises(APIError):
+ event_loop.run_until_complete(
+ pids_discovery([], aiosession, 'http://example.org/api/'))
+
+
+def test_scanner_get_subpaths(tmp_path, temp_paths):
+ for subpath, pid in get_subpaths(tmp_path):
+ assert subpath in temp_paths['paths']
+ assert pid in temp_paths['pids']
+
+
+@pytest.mark.options(debug=False)
+def test_app(app):
+ assert not app.debug
+
+
+def test_scanner_result(live_server, event_loop, test_folder):
+ live_server.start()
+ api_url = live_server.url() + '/'
+
+ result_path = test_folder.joinpath(PosixPath('sample-folder-result.json'))
+ with open(result_path, 'r') as json_file:
+ expected_result = json.loads(json_file.read())
+
+ sample_folder = test_folder.joinpath(PosixPath('sample-folder'))
+
+ source_tree = Tree(sample_folder)
+ event_loop.run_until_complete(
+ run(sample_folder, api_url, source_tree))
+
+ actual_result = source_tree.getJsonTree()
+
+ assert actual_result == expected_result
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Jul 3 2025, 9:18 AM (6 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3221461
Attached To
D2791: swh-scanner: integration tests
Event Timeline
Log In to Comment