diff --git a/gitlab/manage_projects.py b/gitlab/manage_projects.py new file mode 100644 --- /dev/null +++ b/gitlab/manage_projects.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +import json +import logging +from pathlib import Path +from typing import Any, Dict, Iterable, Tuple + +import click +import yaml + +import gitlab + +logger = logging.getLogger(__name__) + +Config = Dict[str, Any] + + +def get_gitlab(gitlab_instance): + gl = gitlab.Gitlab.from_config(gitlab_instance) + gl.auth() + + return gl + + +def update_project( + project, + global_settings: Dict[str, Any], + namespace_settings: Dict[str, Any], + project_settings: Dict[str, Any], +) -> Tuple: + """Given a project and settings configuration dicts, update the project. + + Returns: + Tuple (updated, updated_project). If updated is a dict, then updated_project is + the project with its attributes updated according to the configuration dicts. + Otherwise, updated_project is the same instance as 'project' input parameter. + + """ + # override from generic to specific in that order: global -> namespace -> project + config: Dict[str, Any] = { + **global_settings, + **namespace_settings, + **project_settings, + } + logger.debug( + "Project <%s>: merged configuration: %s", project.path_with_namespace, config + ) + + updated = {} + # Iterate over the new settings to apply + for attribute, value in config.items(): + existing_value = getattr(project, attribute) + # If any changes is detected + if existing_value != value: + # New settings is applied + setattr(project, attribute, value) + new_value = getattr(project, attribute) + logger.debug( + "Update attribute <%s> with value '%s' to value '%s'", + attribute, + existing_value, + new_value, + ) + updated[attribute] = dict(old=existing_value, new=new_value) + + return updated, project + + +def load_projects_configuration(projects_file: str) -> Dict[str, Any]: + """Load the configuration yaml file as Dict.""" + with open(projects_file, "r") as f: + return yaml.safe_load(f) + + +def namespaces_from_path(path_with_namespace: str) -> Iterable[str]: + """Given a path, computes the hierarchic namespaces from generic to specific.""" + namespaces = [] + # FIXME: make that a reduce call!? + for part in Path(path_with_namespace).parts[:-1]: + if namespaces: + last_part = namespaces[-1] + ns = f"{last_part}/{part}" + else: + ns = part + namespaces.append(ns) + + return namespaces + + +@click.command() +@click.option( + "--gitlab", + "-g", + "gitlab_instance", + help="Which GitLab instance to use, as configured in the python-gitlab config", +) +@click.option( + "--do-it", "do_it", is_flag=True, help="Actually perform the operations", +) +@click.argument("projects_file") +def cli(gitlab_instance: str, do_it: bool, projects_file: str,) -> None: + gl = get_gitlab(gitlab_instance) + + configuration = load_projects_configuration(projects_file) + + # Global configuration for all projects + global_settings: Dict[str, Any] = configuration["global_settings"] + + # Namespace project configuration (with possible override on the global config) + namespace_settings: Dict[str, Any] = configuration["namespace_settings"] + + # List of projects that the script should act upon (other projects are skipped) + managed_project_namespaces = configuration["managed_project_namespaces"] + + # Local specific project configuration (with possible override on the global config) + project_settings: Dict[str, Any] = configuration["project_settings"] + + # TODO: Determine whether we want to iterate over all gitlab projects or over the + # configured projects in the configuration files. For now, this iterates over the + # gitlab projects and skips non-configured ones. That way, we could discover + # projects we forgot to configure (by processing the logs afterwards). + + projects: Dict = {} + project_updated_count = 0 + + # Configure the project according to global settings (with potential specific + # project override) + for project in gl.projects.list(iterator=True): + path_with_namespace = project.path_with_namespace + # For the last print statement to explain how many got updated + projects[path_with_namespace] = project + + project_namespaces = namespaces_from_path(path_with_namespace) + if project_namespaces[0] not in managed_project_namespaces: + logger.debug("Skipped non-managed project <%s>", path_with_namespace) + continue + + namespace_config = {} + # Merge configuration from generic namespace to specific + for ns in project_namespaces: + namespace_config.update(namespace_settings.get(ns, {})) + + project_config = project_settings.get(path_with_namespace, {}) + + logger.debug("Project <%s> %s", path_with_namespace, project.id) + updates, project = update_project( + project, global_settings, namespace_config, project_config + ) + + if updates and do_it: + project.save() + project_updated_count += 1 + + if updates: + print(json.dumps({path_with_namespace: updates})) + + dry_run = not do_it + prefix_msg = "(**DRY RUN**) " if dry_run else "" + summary = { + "nb_projects": len(projects), + "nb_updated_projects": project_updated_count, + } + if dry_run: + summary["dry_run"] = dry_run + + logger.debug( + "%sNumber of projects updated: %s / %s", + prefix_msg, + project_updated_count, + len(projects), + ) + + print(json.dumps(summary)) + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(name)s:%(levelname)s %(message)s" + ) + cli(auto_envvar_prefix="SWH") diff --git a/gitlab/projects.yml b/gitlab/projects.yml new file mode 100644 --- /dev/null +++ b/gitlab/projects.yml @@ -0,0 +1,201 @@ +# [1] https://docs.gitlab.com/ee/api/projects.html#edit-project + +# List of namespaces managed by the script (It should be the first level of the +# project namespace). Other unmanaged projects will be skipped during the script +# execution. +managed_project_namespaces: + - infra + +namespace_settings: + # Dict[str, Any] of key the project namespace, and value a dict of any of the global + # settings keys above and value, whatever we want to set. + infra/puppet/3rdparty: + issues_access_level: disabled + infra/ci-cd/debs: + issues_access_level: disabled + +# Dict[str, Any] of key the project with its path namespace, and value a dict of any of +# the global settings keys above (e.g. visibility) and value, whatever we want to set +# (e.g. assuming the key 'visibility', some credentials repositories, we want to set it +# 'private'). +project_settings: + infra/ci-cd/k8s-swh-private-data: + visibility: private + infra/puppet/puppet-swh-private-data: + visibility: private + # staging extra repositories + infra/websites/www.softwareheritage.org-gandi: + visibility: private + infra/credentials: + visibility: private + infra/k8s-swh-private-data: + visibility: private + infra/annex/annex-private: + visibility: private + infra/iFWCFG: + visibility: private + +# Dict[str, Any] of key the setting to change (e.g. 'merge_method') and as value +# whatever we want to set (e.g. assuming the 'merge_method', 'ff' as in fast-forward). +# The keys were determined out of the gitlab api documentation [1]. Some were not +# supported. Those are located after the 'unsupported' comment. +global_settings: + # string (optional): See project visibility level. + visibility: public + # string (optional): Set the merge method used. + merge_method: ff + # boolean (optional): Enable Delete source branch option by default for all new merge + # requests. + remove_source_branch_after_merge: true + # string (optional): One of disabled, private, or enabled. + # releases_access_level: enabled + # boolean: Set whether or not merge requests can be merged with skipped jobs. + # allow_merge_on_skipped_pipeline: false + # string: One of disabled, private or enabled + # analytics_access_level: enabled + # boolean: Enable Auto DevOps for this project. + # auto_devops_enabled: false + # boolean: Set whether auto-closing referenced issues on default branch. + # autoclose_referenced_issues: true + # string (optional): The Git strategy. Defaults to fetch. + # build_git_strategy: fetch + # integer (optional): The maximum amount of time, in seconds, that a job can run. + # build_timeout: 3600 + # string (optional): One of disabled, private, or enabled. + # builds_access_level: enabled + # string (optional): The path to CI configuration file. + # ci_config_path: "" + # integer (optional): Default number of revisions for shallow cloning. + # ci_default_git_depth: 20 + # boolean (optional): Enable or disable prevent outdated deployment jobs. + # ci_forward_deployment_enabled: true + # boolean (optional): Enable or disable running pipelines in the parent project for + # merge requests from forks. (Introduced in GitLab 15.3.) + # ci_allow_fork_pipelines_to_run_in_parent_project: true + # boolean (optional): Set whether or not caches should be separated by branch + # protection status. + # ci_separated_caches: true + # string (optional): Set visibility of container registry, for this project, to one of + # disabled, private or enabled. + # container_registry_access_level: disabled + # string (optional): The default branch name. + # default_branch: master + # string (optional): Short project description. + # description: "" + # boolean (optional): Disable email notifications. + # emails_disabled: true + # boolean (optional): Enforce auth checks on uploads. + # enforce_auth_checks_on_uploads: true + # string (optional): One of disabled, private, or enabled. + # forking_access_level: enabled + # string (optional): One of disabled, private, or enabled. + # issues_access_level: enabled + # boolean (optional): Disable or enable the ability to keep the latest artifact for + # this project. + # keep_latest_artifact: True + # boolean (optional): Enable LFS. + # lfs_enabled: True + # string (optional): Template used to create merge commit message in merge requests. + # (Introduced in GitLab 14.5.) + # merge_commit_template: "" + # string (optional): One of disabled, private, or enabled. + # merge_requests_access_level: enabled + # boolean (optional): Set whether merge requests can only be merged when all the + # discussions are resolved. + # only_allow_merge_if_all_discussions_are_resolved: False + # boolean (optional): Set whether merge requests can only be merged with successful + # jobs. + # only_allow_merge_if_pipeline_succeeds: False + # string (optional): One of disabled, private, or enabled. + # operations_access_level: disabled + # boolean (optional): Enable or disable packages repository feature. + # packages_enabled: False + # string (optional): One of disabled, private, enabled, or public. + # pages_access_level: disabled + # boolean (optional): Show link to create/view merge request when pushing from the + # command line. + # printing_merge_request_link_enabled: true + # string (optional): One of disabled, private, or enabled. + # repository_access_level: enabled + # string (optional): Which storage shard the repository is on. (administrators only) + # repository_storage: default + # boolean (optional): Allow users to request member access. + # request_access_enabled: true + # boolean (optional): Automatically resolve merge request diffs discussions on lines + # changed with a push. + # resolve_outdated_diff_discussions: false + # boolean (optional): Allow only users with the Maintainer role to pass user-defined + # variables when triggering a pipeline. For example when the pipeline is triggered in + # the UI, with the API, or by a trigger token. + # restrict_user_defined_variables: false + # string (optional): (GitLab 14.9 and later) Security and compliance access level. One + # of disabled, private, or enabled. + # security_and_compliance_access_level: private + # boolean (optional): Enable or disable Service Desk feature. + # service_desk_enabled: false + # boolean (optional): Enable shared runners for this project. + # shared_runners_enabled: true + # string (optional): One of disabled, private, or enabled. + # snippets_access_level: enabled + # string (optional): Template used to create squash commit message in merge requests. + # (Introduced in GitLab 14.6.) + # squash_commit_template: "" + # string (optional): One of never, always, default_on, or default_off. + # squash_option: default_on + # string (optional): The commit message used to apply merge request suggestions. + # suggestion_commit_message: "" + # string (optional): One of disabled, private, or enabled. + # wiki_access_level: enabled + # ------------------------------------------------------- + # Following keys are documented but somehow not supported + # ------------------------------------------------------- + # boolean: Indicates that merges of merge requests should be blocked unless all status + # checks have passed. Defaults to false. Introduced in GitLab 15.5 with feature flag + # only_allow_merge_if_all_status_checks_passed disabled by default. + # only_allow_merge_if_all_status_checks_passed: ? + # integer: How many approvers should approve merge request by default. To configure + # approval rules, see Merge request approvals API. + # approvals_before_merge: ? + # string: Auto-cancel pending pipelines. This isn’t a boolean, but enabled/disabled. + # auto_cancel_pending_auto: ? + # string: Auto Deploy strategy (continuous, manual, or timed_incremental). + # pipelines_devops_deploy_strategy: ? + # mixed (optional): Image file for avatar of the project. + # avatar: ? + # hash (optional): Update the image cleanup policy for this project. Accepts: cadence + # (string), keep_n (integer), older_than (string), name_regex (string), + # name_regex_delete (string), name_regex_keep (string), enabled (boolean). + # container_expiration_policy_attributes: ? + # string (optional): The classification label for the project. + # external_authorization_classification_label: ? + # string (optional): Default description for Issues. Description is parsed with GitLab + # Flavored Markdown. See Templates for issues and merge requests. + # issues_template: ? + # boolean (optional): Enable or disable merge pipelines. + # merge_pipelines_enabled: ? + # string (optional): Default description for merge requests. Description is parsed + # with GitLab Flavored Markdown. See Templates for issues and merge requests. + # merge_requests_template: ? + # boolean (optional): Enable or disable merge trains. + # merge_trains_enabled: ? + # boolean (optional): Pull mirror overwrites diverged branches. + # mirror_overwrites_diverged_branches: ? + # boolean (optional): Pull mirroring triggers builds. + # mirror_trigger_builds: ? + # integer (optional): User responsible for all the activity surrounding a pull mirror + # event. (administrators only) + # mirror_user_id: ? + # boolean (optional): Enables pull mirroring in a project. + # mirror: ? + # boolean (optional): For forked projects, target merge requests to this project. If + # false, the target will be the upstream project. + # mr_default_target_self: ? + # boolean (optional): Only mirror protected branches. + # only_mirror_protected_branches: ? + # boolean (optional): If true, jobs can be viewed by non-project members. + # public_builds: ? + # string (optional): One of disabled, private, enabled or public + # requirements_access_level: ? + # string (optional): Template used to suggest names for branches created from issues. + # (Introduced in GitLab 15.6.) + # issue_branch_template: ?