diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ include version.txt include README.md recursive-include swh py.typed -recursive-include swh/scanner/tests/data/ *.tgz +recursive-include swh/scanner/tests/data/ * diff --git a/swh/scanner/cli.py b/swh/scanner/cli.py --- a/swh/scanner/cli.py +++ b/swh/scanner/cli.py @@ -15,10 +15,8 @@ from swh.core.cli import swh as swh_cli_group # All generic config code should reside in swh.core.config -DEFAULT_CONFIG_PATH = os.environ.get( - "SWH_CONFIG_FILE", os.path.join(click.get_app_dir("swh"), "global.yml") -) - +CONFIG_ENVVAR = "SWH_CONFIG_FILE" +DEFAULT_CONFIG_PATH = os.path.join(click.get_app_dir("swh"), "global.yml") DEFAULT_CONFIG: Dict[str, Any] = { "web-api": { @@ -52,15 +50,21 @@ ) @click.pass_context def scanner(ctx, config_file: Optional[str]): - if config_file is None and config.config_exists(DEFAULT_CONFIG_PATH): - config_file = DEFAULT_CONFIG_PATH - - if config_file is None: - conf = DEFAULT_CONFIG - else: + env_config_path = os.environ.get(CONFIG_ENVVAR) + + if config_file and not config.config_exists(config_file): + raise FileNotFoundError(config_file) + if env_config_path and not config.config_exists(env_config_path): + raise FileNotFoundError(env_config_path) + if not config_file: + if env_config_path: + config_file = env_config_path + elif config.config_exists(DEFAULT_CONFIG_PATH): + config_file = DEFAULT_CONFIG_PATH + + conf = DEFAULT_CONFIG + if config_file is not None: # read_raw_config do not fail on ENOENT - if not config.config_exists(config_file): - raise FileNotFoundError(config_file) conf = config.read_raw_config(config.config_basepath(config_file)) conf = config.merge_configs(DEFAULT_CONFIG, conf) @@ -103,13 +107,13 @@ def scan(ctx, root_path, api_url, patterns, out_fmt, interactive): """Scan a source code project to discover files and directories already present in the archive""" - from .scanner import scan + import swh.scanner.scanner as scanner config = ctx.obj["config"] if api_url: config["web-api"]["url"] = api_url - scan(config, root_path, patterns, out_fmt, interactive) + scanner.do_scan(config, root_path, patterns, out_fmt, interactive) def main(): diff --git a/swh/scanner/scanner.py b/swh/scanner/scanner.py --- a/swh/scanner/scanner.py +++ b/swh/scanner/scanner.py @@ -217,7 +217,7 @@ yield re.compile(regex) -def scan( +def do_scan( config: Dict[str, Any], root_path: str, exclude_patterns: Iterable[str], diff --git a/swh/scanner/tests/conftest.py b/swh/scanner/tests/conftest.py --- a/swh/scanner/tests/conftest.py +++ b/swh/scanner/tests/conftest.py @@ -49,7 +49,7 @@ .. code-block:: python root = { - subdir: { + subdir0: { subsubdir filesample.txt filesample2.txt @@ -58,11 +58,14 @@ subfile.txt } """ - root = tmp_path_factory.getbasetemp() - subdir = tmp_path_factory.mktemp("subdir") - subsubdir = subdir.joinpath("subsubdir") + root = tmp_path_factory.mktemp("root") + subdir = root / "subdir0" + subdir.mkdir() + subsubdir = subdir / "subsubdir" subsubdir.mkdir() - subdir2 = tmp_path_factory.mktemp("subdir2") + subdir2 = root / "subdir2" + subdir2.mkdir() + subfile = root / "subfile.txt" subfile.touch() filesample = subdir / "filesample.txt" diff --git a/swh/scanner/tests/data/global.yml b/swh/scanner/tests/data/global.yml new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/data/global.yml @@ -0,0 +1,2 @@ +web-api: + auth-token: eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJqdGkiOiI0OGRhMGQwNS1iYjlmLTRlM2ItYTU5MS1kNTNmMDljOWFlNmIiLCJleHAiOjAsIm5iZiI6MCwiaWF0IjoxNTk5NTc1NTM3LCJpc3MiOiJodHRwczovL2F1dGguc29mdHdhcmVoZXJpdGFnZS5vcmcvYXV0aC9yZWFsbXMvU29mdHdhcmVIZXJpdGFnZSIsImF1ZCI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwic3ViIjoiYTY0MDhkNGUtNGI3Yy00OWIyLTg1ODctNzRmN2VhZjNhYWE1IiwidHlwIjoiT2ZmbGluZSIsImF6cCI6InN3aC13ZWIiLCJhdXRoX3RpbWUiOjAsInNlc3Npb25fc3RhdGUiOiJhYWE2NmQ0YS1lNDUxLTRiYTktODZkNi0wN2NhOTdhNmZjMTIiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIG9mZmxpbmVfYWNjZXNzIHByb2ZpbGUgZW1haWwifQ.4abJuiktMaY6FeCAMaZSSfYopeX-Lx3e7AoAxoBXXDM diff --git a/swh/scanner/tests/test_cli.py b/swh/scanner/tests/test_cli.py new file mode 100644 --- /dev/null +++ b/swh/scanner/tests/test_cli.py @@ -0,0 +1,158 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os.path +from pathlib import Path +from unittest.mock import Mock + +from click.testing import CliRunner +import pytest + +import swh.core.config as config +import swh.scanner.cli as cli +import swh.scanner.scanner as scanner + +DATADIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data") +TEST_CONFIG_PATH = os.path.join(DATADIR, "global.yml") + +ROOTPATH_GOOD = os.path.join(DATADIR, "") +ROOTPATH_BAD = "~/not_expanduser-ed" +CONFPATH_GOOD = TEST_CONFIG_PATH +CONFPATH_GOOD2 = TEST_CONFIG_PATH.replace("/swh/", "/swh/../swh/") # or a copy? +CONFPATH_BAD = "~/not_expanduser-ed" + +GLOBAL_CONFIG_DATA = Path(TEST_CONFIG_PATH).read_text() + + +@pytest.fixture(scope="function") +def m_scanner(monkeypatch): + """Returns a mock swh.core.config object whose do_scan is noop""" + # Customizable mock of scanner module + # Fortunately, noop is the default behavior + scanner_mock = Mock(scanner) + monkeypatch.setattr("swh.scanner.scanner", scanner_mock) + yield scanner_mock + + +@pytest.fixture(scope="function") +def m_config(monkeypatch): + """Returns a mock swh.core.config object that return default config data""" + import yaml + + # Customizable mock of config module, with some methods not mocked + # Whitelist approach, could be stricter with Mock.spec_set + config_mock = Mock(config) + config_mock.config_exists = config.config_exists + config_mock.read_raw_config = Mock(return_value=yaml.safe_load(GLOBAL_CONFIG_DATA)) + config_mock.merge_configs = config.merge_configs + monkeypatch.setattr("swh.scanner.cli.config", config_mock) + yield config_mock + + +@pytest.fixture(scope="function") +def cli_runner(): + """Return a CliRunner with default environment variable SWH_CONFIG_FILE unset""" + return CliRunner(env={"SWH_CONFIG_FILE": ""}) + + +@pytest.fixture(scope="function") +def good_default_config_path(monkeypatch): + """Patch default config path to a valid value""" + assert os.path.exists(TEST_CONFIG_PATH) + monkeypatch.setattr(cli, "DEFAULT_CONFIG_PATH", TEST_CONFIG_PATH) + + +@pytest.fixture(scope="function") +def bad_default_config_path(monkeypatch, tmpdir): + """Patch default config path to an invalid value""" + default_config_path = os.path.join(tmpdir, "missing") + monkeypatch.setattr(cli, "DEFAULT_CONFIG_PATH", default_config_path) + + +def cli_run_ok(res): + """Return whether cli command ran successfully, given its result object""" + return res.exit_code == 0 and res.exception is None and "Error:" not in res.stdout + + +# TEST BEGIN + +# For nominal code paths, check that the right config file is loaded with a mock +# mock scanner.scan to not run actual scan +# TODO give ROOTPATH_GOOD an existing package within the temp directory + + +def test_smoke(cli_runner): + """Break if basic functionality breaks""" + res = cli_runner.invoke(cli.scanner, ["scan", "-h"]) + assert cli_run_ok(res) + + +def test_config_path_option_bad(cli_runner, bad_default_config_path): + """Test bad option no envvar bad default""" + res = cli_runner.invoke(cli.scanner, ["-C", CONFPATH_BAD, "scan", ROOTPATH_GOOD]) + assert res.exit_code != 0 and res.exception.__class__ == FileNotFoundError + + +def test_default_config_path(cli_runner, good_default_config_path, m_scanner, m_config): + """Test no option no envvar good default""" + res = cli_runner.invoke(cli.scanner, ["scan", ROOTPATH_GOOD]) + assert cli_run_ok(res) + m_config.read_raw_config.assert_called_with( + m_config.config_basepath(TEST_CONFIG_PATH) + ) + m_scanner.do_scan.assert_called_once() + + +def test_root_no_config(cli_runner, bad_default_config_path, m_scanner, m_config): + """Test no config = no option no envvar bad default, good root""" + res = cli_runner.invoke(cli.scanner, ["scan", ROOTPATH_GOOD]) + assert cli_run_ok(res) + m_config.read_raw_config.assert_not_called() + m_scanner.do_scan.assert_called_once() + + +def test_root_bad(cli_runner, bad_default_config_path): + """Test no option no envvar bad default bad root""" + res = cli_runner.invoke(cli.scanner, ["scan", ROOTPATH_BAD]) + assert res.exit_code == 2 + + +def test_config_path_envvar_good( + cli_runner, bad_default_config_path, m_scanner, m_config +): + """Test no option good envvar bad default good root""" + cli_runner.env["SWH_CONFIG_FILE"] = CONFPATH_GOOD + res = cli_runner.invoke(cli.scanner, ["scan", ROOTPATH_GOOD]) + assert cli_run_ok(res) + m_config.read_raw_config.assert_called_with(m_config.config_basepath(CONFPATH_GOOD)) + m_scanner.do_scan.assert_called_once() + + +def test_config_path_envvar_bad(cli_runner, bad_default_config_path): + """Test no option bad envvar bad default good root""" + cli_runner.env["SWH_CONFIG_FILE"] = CONFPATH_BAD + res = cli_runner.invoke(cli.scanner, ["scan", ROOTPATH_GOOD]) + assert res.exception.__class__ == FileNotFoundError + + +def test_config_path_option_envvar( + cli_runner, bad_default_config_path, m_scanner, m_config +): + """Test good option good envvar bad default good root + Check that option has precedence over envvar""" + cli_runner.env["SWH_CONFIG_FILE"] = CONFPATH_GOOD2 + res = cli_runner.invoke(cli.scanner, ["-C", CONFPATH_GOOD, "scan", ROOTPATH_GOOD]) + assert cli_run_ok(res) + m_config.read_raw_config.assert_called_with(m_config.config_basepath(CONFPATH_GOOD)) + m_scanner.do_scan.assert_called_once() + + +def test_api_url_option(cli_runner, bad_default_config_path, m_scanner, m_config): + """Test no config good root good url""" + api_url = cli.DEFAULT_CONFIG["web-api"]["url"] + res = cli_runner.invoke(cli.scanner, ["scan", ROOTPATH_GOOD, "-u", api_url]) + assert cli_run_ok(res) + m_config.read_raw_config.assert_not_called() + m_scanner.do_scan.assert_called_once()