diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -p no:flask -p no:pytest_swh_storage +addopts = -p no:flask -p no:pytest_swh_storage --ignore-glob=*random_fixtures_test.py norecursedirs = docs node_modules .tox DJANGO_SETTINGS_MODULE = swh.web.settings.tests filterwarnings = diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -12,6 +12,7 @@ import shutil from subprocess import PIPE, run import sys +import time from typing import Any, Dict, List, Optional from _pytest.python import Function @@ -85,6 +86,10 @@ ) +def pytest_addoption(parser): + parser.addoption("--swh-web-random-seed", action="store", default=None) + + def pytest_configure(config): # Use fast hypothesis profile by default if none has been # explicitly specified in pytest option @@ -137,6 +142,47 @@ json.dump(mock_webpack_stats, outfile) +_swh_web_custom_section = "swh-web custom section" +_random_seed_cache_key = "swh-web/random-seed" + + +@pytest.fixture(scope="function", autouse=True) +def random_seed(pytestconfig): + state = random.getstate() + seed = pytestconfig.getoption("--swh-web-random-seed") + if seed is None: + seed = time.time() + seed = int(seed) + cache.set(_random_seed_cache_key, seed) + random.seed(seed) + yield seed + random.setstate(state) + + +def pytest_report_teststatus(report, config): + if report.when == "call" and report.outcome == "failed": + seed = cache.get(_random_seed_cache_key, None) + line = ( + f'FAILED {report.nodeid}: Use "pytest --swh-web-random-seed={seed} ' + f'{report.nodeid}" to reproduce that test failure with same inputs' + ) + report.sections.append((_swh_web_custom_section, line)) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + reports = terminalreporter.getreports("failed") + content = os.linesep.join( + text + for report in reports + for secname, text in report.sections + if secname == _swh_web_custom_section + ) + if content: + terminalreporter.ensure_newline() + terminalreporter.section(_swh_web_custom_section, sep="-", blue=True, bold=True) + terminalreporter.line(content) + + # Clear Django cache before each test @pytest.fixture(autouse=True) def django_cache_cleared(): diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -345,7 +345,7 @@ rev_id = rev_log[0] revisions.add(rev_id) - for rev in storage.revision_get(origin_revisions): + for rev in storage.revision_get(sorted(origin_revisions)): if rev is None: continue dir_id = rev.directory @@ -464,12 +464,12 @@ "idx_storage": idx_storage, "counters": counters, "origins": _TEST_ORIGINS, - "contents": contents, - "directories": list(directories), - "releases": list(releases), - "revisions": list(map(hash_to_hex, revisions)), - "snapshots": list(snapshots), - "swhids": swhids, + "contents": list(sorted(contents, key=lambda c: c["sha1"])), + "directories": list(sorted(directories)), + "releases": list(sorted(releases)), + "revisions": list(sorted(map(hash_to_hex, revisions))), + "snapshots": list(sorted(snapshots)), + "swhids": list(sorted(swhids, key=lambda swhid: swhid.object_id)), } diff --git a/swh/web/tests/random_fixtures_test.py b/swh/web/tests/random_fixtures_test.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/random_fixtures_test.py @@ -0,0 +1,99 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import sys + + +def test_random_fixture_values( + sha1, + invalid_sha1, + sha256, + content, + contents, + unknown_content, + unknown_contents, + content_text, + content_text_non_utf8, + content_application_no_highlight, + content_text_no_highlight, + content_image_type, + content_unsupported_image_type_rendering, + content_utf8_detected_as_binary, + directory, + directory_with_subdirs, + directory_with_files, + unknown_directory, + release, + releases, + unknown_release, + revision, + revisions, + revisions_list, + unknown_revision, + ancestor_revisions, + non_ancestor_revisions, + snapshot, + unknown_snapshot, + origin, + origin_with_multiple_visits, + origin_with_releases, + origin_with_pull_request_branches, + content_swhid, + directory_swhid, + release_swhid, + revision_swhid, + snapshot_swhid, +): + """Special test to print values of swh-web fixtures returning random data. + + It is not integrated in swh-web test suite but will be executed by explicitly + invoking pytest in tests located in swh/web/tests/test_random_fixtures.py. + """ + print( + "\n".join( + [ + sha1, + invalid_sha1, + sha256, + content["sha1"], + str([c["sha1"] for c in contents]), + unknown_content["sha1"], + str([c["sha1"] for c in unknown_contents]), + content_text["sha1"], + content_text_non_utf8["sha1"], + content_application_no_highlight["sha1"], + content_text_no_highlight["sha1"], + content_image_type["sha1"], + content_unsupported_image_type_rendering["sha1"], + content_utf8_detected_as_binary["sha1"], + directory, + directory_with_subdirs, + directory_with_files, + unknown_directory, + release, + str(releases), + unknown_release, + revision, + str(revisions), + str(revisions_list(size=3)), + unknown_revision, + str(ancestor_revisions), + str(non_ancestor_revisions), + snapshot, + unknown_snapshot, + origin["url"], + origin_with_multiple_visits["url"], + origin_with_releases["url"], + origin_with_pull_request_branches.url, + str(content_swhid), + str(directory_swhid), + str(release_swhid), + str(revision_swhid), + str(snapshot_swhid), + ] + ), + file=sys.stderr, + ) + assert False diff --git a/swh/web/tests/test_random_fixtures.py b/swh/web/tests/test_random_fixtures.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/test_random_fixtures.py @@ -0,0 +1,43 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os +import subprocess + + +def test_random_fixtures(): + """Check random fixture values will be different when random seed + is not explicitly provided. + """ + result_first = subprocess.run( + ["pytest", "-s", "random_fixtures_test.py"], + capture_output=True, + cwd=os.path.dirname(__file__), + ) + result_second = subprocess.run( + ["pytest", "-s", "random_fixtures_test.py"], + capture_output=True, + cwd=os.path.dirname(__file__), + ) + assert result_first.stderr != result_second.stderr + assert b'Use "pytest --swh-web-random-seed=' in result_first.stdout + + +def test_random_fixtures_with_seed(): + """Check random fixture values will be the same when random seed + is explicitly provided through a custom pytest option. + """ + result_first = subprocess.run( + ["pytest", "-s", "--swh-web-random-seed=2021", "random_fixtures_test.py"], + capture_output=True, + cwd=os.path.dirname(__file__), + ) + result_second = subprocess.run( + ["pytest", "-s", "--swh-web-random-seed=2021", "random_fixtures_test.py"], + capture_output=True, + cwd=os.path.dirname(__file__), + ) + assert result_first.stderr == result_second.stderr + assert b'Use "pytest --swh-web-random-seed=2021' in result_first.stdout