diff --git a/site-modules/profile/files/prometheus/update-prometheus-config b/site-modules/profile/files/prometheus/update-prometheus-config index a806ab8e..87fe42da 100755 --- a/site-modules/profile/files/prometheus/update-prometheus-config +++ b/site-modules/profile/files/prometheus/update-prometheus-config @@ -1,118 +1,133 @@ #!/usr/bin/env python3 # # This generates a static configuration for Prometheus # # Copyright © 2020 The Software Heritage Developers. # This file is released under the Apache-2.0 License. # from collections import defaultdict import copy from dataclasses import asdict, dataclass, fields import datetime import os import stat import sys -from typing import Any, Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional, Tuple import yaml @dataclass(frozen=True) class JobGroup: """Job parameters from which to group prometheus jobs""" job_name: str scrape_interval: Optional[int] scrape_timeout: Optional[int] metrics_path: Optional[str] scheme: Optional[str] + params: Optional[Tuple] @classmethod def from_dict(cls, dict): init_vars = {field.name: dict.get(field.name) for field in fields(cls)} if init_vars.get('metrics_path') == '/metrics': init_vars['metrics_path'] = None if init_vars.get('scheme') == 'http': init_vars['scheme'] = None return cls(**init_vars) def load_yaml_from_dir(dirname: str) -> Iterable[Dict[str, Any]]: """Load all yaml files from a given directory""" for filename in os.listdir(dirname): if not filename.endswith((".yml", ".yaml")): continue path = os.path.join(dirname, filename) with open(path, "r") as f: yield from yaml.safe_load(f) +def dict_factory(data): + d = dict(data) + + if d.get("params") is not None: + d["params"] = {k:list(v) for k,v in d["params"]} + + return d + def generate_scrape_configs(configs: Dict[JobGroup, List[Dict[str, Any]]]): """Generate a scrape_configs entry from a dict""" for params, targets in configs.items(): yield { **{ param: value - for param, value in asdict(params).items() + for param, value in asdict(params, dict_factory=dict_factory).items() if value is not None }, "static_configs": targets, } def merge_prometheus_config( base_config: Dict[str, Any], scrape_configs: Iterable[Dict[str, Any]] ) -> Dict[str, Any]: """Merge the main prometheus config with scrape configs""" config = copy.deepcopy(base_config) config.setdefault("scrape_configs", []).extend(scrape_configs) return config def replace_file(old_file, new_file): """Replace old_file with new_file, ensuring permissions are the same""" try: info = os.stat(old_file) os.chown(new_file, info.st_uid, info.st_gid) os.chmod(new_file, stat.S_IMODE(info.st_mode)) except FileNotFoundError: pass os.rename(new_file, old_file) if __name__ == "__main__": base_conffile = sys.argv[1] exported_dir = sys.argv[2] output = sys.argv[3] config_groups: Dict[JobGroup, List[Dict[str, Any]]] = defaultdict(list) for conf in load_yaml_from_dir(exported_dir): if 'job' in conf: conf['job_name'] = conf.pop('job') + if 'params' in conf: + params = conf.pop('params') + if params is not None: + # Hack to allow the dict serialization (used in the config_groups dict key later) + conf['params'] = tuple((k,tuple(v)) for k,v in params.items()) + group = JobGroup.from_dict(conf) for key in asdict(group): conf.pop(key, None) config_groups[group].append(conf) with open(base_conffile, "r") as f: base_config = yaml.safe_load(f) full_config = merge_prometheus_config( base_config, generate_scrape_configs(config_groups), ) now = datetime.datetime.now(tz=datetime.timezone.utc).isoformat() with open(output + ".tmp", "w") as f: print(f"# This file was generated by {sys.argv[0]} on {now}.", file=f) print(f"# Changes will be lost", file=f) print(f"", file=f) yaml.dump(full_config, f, default_flow_style=False) replace_file(output, output + ".tmp") diff --git a/site-modules/profile/manifests/prometheus/export_scrape_config.pp b/site-modules/profile/manifests/prometheus/export_scrape_config.pp index ac2fdaa8..51a3ca26 100644 --- a/site-modules/profile/manifests/prometheus/export_scrape_config.pp +++ b/site-modules/profile/manifests/prometheus/export_scrape_config.pp @@ -1,21 +1,23 @@ # Export a scrape config to the configured prometheus server define profile::prometheus::export_scrape_config ( String $target, String $job = $name, Optional[String] $prometheus_server = undef, Hash[String, String] $labels = {}, Optional[Enum['http', 'https']] $scheme = undef, Optional[String] $metrics_path = undef, + Optional[Hash[String, Array[String]]] $params = undef, ) { $static_labels = lookup('prometheus::static_labels', Hash) @@profile::prometheus::scrape_config {"${facts['swh_hostname']['short']}_${name}": prometheus_server => pick($prometheus_server, lookup('prometheus::server::certname')), target => $target, job => $job, labels => $static_labels + $labels, scheme => $scheme, metrics_path => $metrics_path, + params => $params, } } diff --git a/site-modules/profile/manifests/prometheus/scrape_config.pp b/site-modules/profile/manifests/prometheus/scrape_config.pp index 4978357d..f1a5cac0 100644 --- a/site-modules/profile/manifests/prometheus/scrape_config.pp +++ b/site-modules/profile/manifests/prometheus/scrape_config.pp @@ -1,29 +1,31 @@ # Scrape configuration for a prometheus exporter define profile::prometheus::scrape_config ( String $prometheus_server, String $target, String $job, Hash[String, String] $labels = {}, Optional[Enum['http', 'https']] $scheme = undef, Optional[String] $metrics_path = undef, + Optional[Hash[String, Array[String]]] $params = undef, ){ $directory = $profile::prometheus::server::scrape_configs_dir file {"${directory}/${name}.yaml": ensure => 'present', owner => 'root', group => 'root', mode => '0644', content => inline_yaml( [ { job_name => $job, targets => [$target], labels => $labels, scheme => $scheme, - metrics_path => $metrics_path + metrics_path => $metrics_path, + params => $params, }, ] ), notify => Exec['update-prometheus-config'], } }