diff --git a/swh/core/config.py b/swh/core/config.py --- a/swh/core/config.py +++ b/swh/core/config.py @@ -1,16 +1,15 @@ -# Copyright (C) 2015 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import configparser import logging import os import yaml from itertools import chain from copy import deepcopy -from typing import Any, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple logger = logging.getLogger(__name__) @@ -22,27 +21,25 @@ "/etc/softwareheritage", ] -SWH_GLOBAL_CONFIG = "global.ini" +SWH_GLOBAL_CONFIG = "global.yml" SWH_DEFAULT_GLOBAL_CONFIG = { "max_content_size": ("int", 100 * 1024 * 1024), - "log_db": ("str", "dbname=softwareheritage-log"), } SWH_CONFIG_EXTENSIONS = [ ".yml", - ".ini", ] # conversion per type -_map_convert_fn = { +_map_convert_fn: Dict[str, Callable] = { "int": int, "bool": lambda x: x.lower() == "true", "list[str]": lambda x: [value.strip() for value in x.split(",")], "list[int]": lambda x: [int(value.strip()) for value in x.split(",")], } -_map_check_fn = { +_map_check_fn: Dict[str, Callable] = { "int": lambda x: isinstance(x, int), "bool": lambda x: isinstance(x, bool), "list[str]": lambda x: (isinstance(x, list) and all(isinstance(y, str) for y in x)), @@ -74,35 +71,25 @@ raise PermissionError("Permission denied: %r" % file) -def config_basepath(config_path): +def config_basepath(config_path: str) -> str: """Return the base path of a configuration file""" - if config_path.endswith((".ini", ".yml")): + if config_path.endswith(".yml"): return config_path[:-4] return config_path -def read_raw_config(base_config_path): +def read_raw_config(base_config_path: str) -> Dict[str, Any]: """Read the raw config corresponding to base_config_path. - Can read yml or ini files. + Can read yml files. """ - yml_file = base_config_path + ".yml" + yml_file = f"{base_config_path}.yml" if exists_accessible(yml_file): logger.info("Loading config file %s", yml_file) with open(yml_file) as f: return yaml.safe_load(f) - ini_file = base_config_path + ".ini" - if exists_accessible(ini_file): - config = configparser.ConfigParser() - config.read(ini_file) - if "main" in config._sections: - logger.info("Loading config file %s", ini_file) - return config._sections["main"] - else: - logger.warning("Ignoring config file %s (no [main] section)", ini_file) - return {} @@ -114,7 +101,10 @@ ) -def read(conf_file=None, default_conf=None): +def read( + conf_file: Optional[str] = None, + default_conf: Optional[Dict[str, Tuple[str, Any]]] = None, +) -> Dict[str, Any]: """Read the user's configuration file. Fill in the gap using `default_conf`. `default_conf` is similar to this:: @@ -130,19 +120,18 @@ If conf_file is None, return the default config. """ - conf = {} + conf: Dict[str, Any] = {} if conf_file: base_config_path = config_basepath(os.path.expanduser(conf_file)) - conf = read_raw_config(base_config_path) + conf = read_raw_config(base_config_path) or {} if not default_conf: - default_conf = {} + return conf # remaining missing default configuration key are set # also type conversion is enforced for underneath layer - for key in default_conf: - nature_type, default_value = default_conf[key] + for key, (nature_type, default_value) in default_conf.items(): val = conf.get(key, None) if val is None: # fallback to default value conf[key] = default_value diff --git a/swh/core/tests/test_config.py b/swh/core/tests/test_config.py --- a/swh/core/tests/test_config.py +++ b/swh/core/tests/test_config.py @@ -1,13 +1,15 @@ -# Copyright (C) 2015 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os -import shutil import pytest +import shutil import pkg_resources.extern.packaging.version +import yaml + from swh.core import config pytest_v = pkg_resources.get_distribution("pytest").parsed_version @@ -63,14 +65,14 @@ @pytest.fixture def swh_config(tmp_path): # create a temporary folder - conffile = tmp_path / "config.ini" - conf_contents = """[main] -a = 1 -b = this is a string -c = true -h = false -ls = list, of, strings -li = 1, 2, 3, 4 + conffile = tmp_path / "config.yml" + conf_contents = """ +a: 1 +b: this is a string +c: true +h: false +ls: list, of, strings +li: 1, 2, 3, 4 """ conffile.open("w").write(conf_contents) return conffile @@ -100,7 +102,7 @@ @pytest.fixture def swh_config_empty(tmp_path): # create a temporary folder - conffile = tmp_path / "config.ini" + conffile = tmp_path / "config.yml" conffile.touch() return conffile @@ -113,6 +115,18 @@ assert res == parsed_conffile +def test_read_no_default_conf(swh_config): + """If no default config if provided to read, this should directly parse the config file + yaml + + """ + config_path = str(swh_config) + actual_config = config.read(config_path) + with open(config_path) as f: + expected_config = yaml.safe_load(f) + assert actual_config == expected_config + + def test_read_empty_file(): # when res = config.read(None, default_conf) @@ -123,7 +137,7 @@ def test_support_non_existing_conffile(tmp_path): # when - res = config.read(str(tmp_path / "void.ini"), default_conf) + res = config.read(str(tmp_path / "void.yml"), default_conf) # then assert res == parsed_default_conf @@ -156,7 +170,7 @@ def test_priority_read_nonexist_conf(swh_config): - noexist = str(swh_config.parent / "void.ini") + noexist = str(swh_config.parent / "void.yml") # when res = config.priority_read([noexist, str(swh_config)], default_conf) @@ -165,8 +179,8 @@ def test_priority_read_conf_nonexist_empty(swh_config): - noexist = swh_config.parent / "void.ini" - empty = swh_config.parent / "empty.ini" + noexist = swh_config.parent / "void.yml" + empty = swh_config.parent / "empty.yml" empty.touch() # when @@ -179,8 +193,8 @@ def test_priority_read_empty_conf_nonexist(swh_config): - noexist = swh_config.parent / "void.ini" - empty = swh_config.parent / "empty.ini" + noexist = swh_config.parent / "void.yml" + empty = swh_config.parent / "empty.yml" empty.touch() # when @@ -193,12 +207,12 @@ def test_swh_config_paths(): - res = config.swh_config_paths("foo/bar.ini") + res = config.swh_config_paths("foo/bar.yml") assert res == [ - "~/.config/swh/foo/bar.ini", - "~/.swh/foo/bar.ini", - "/etc/softwareheritage/foo/bar.ini", + "~/.config/swh/foo/bar.yml", + "~/.swh/foo/bar.yml", + "/etc/softwareheritage/foo/bar.yml", ]