Page MenuHomeSoftware Heritage
Files F7163847

D8825.id31833.diff
No OneTemporary
Actions

D8825.id31833.diff
View Options

diff --git a/gitlab/manage_projects.py b/gitlab/manage_projects.py
new file mode 100644
--- /dev/null
+++ b/gitlab/manage_projects.py
@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, Iterable, Tuple
+
+import click
+import yaml
+
+import gitlab
+
+logger = logging.getLogger(__name__)
+
+Config = Dict[str, Any]
+
+
+def get_gitlab(gitlab_instance):
+ gl = gitlab.Gitlab.from_config(gitlab_instance)
+ gl.auth()
+
+ return gl
+
+
+def update_project(
+ project,
+ global_settings: Dict[str, Any],
+ namespace_settings: Dict[str, Any],
+ project_settings: Dict[str, Any],
+) -> Tuple:
+ """Given a project and settings configuration dicts, update the project.
+
+ Returns:
+ Tuple (updated, updated_project). If updated is a dict, then updated_project is
+ the project with its attributes updated according to the configuration dicts.
+ Otherwise, updated_project is the same instance as 'project' input parameter.
+
+ """
+ # override from generic to specific in that order: global -> namespace -> project
+ config: Dict[str, Any] = {
+ **global_settings,
+ **namespace_settings,
+ **project_settings,
+ }
+ logger.debug(
+ "Project <%s>: merged configuration: %s", project.path_with_namespace, config
+ )
+
+ updated = {}
+ # Iterate over the new settings to apply
+ for attribute, value in config.items():
+ existing_value = getattr(project, attribute)
+ # If any changes is detected
+ if existing_value != value:
+ # New settings is applied
+ setattr(project, attribute, value)
+ new_value = getattr(project, attribute)
+ logger.debug(
+ "Update attribute <%s> with value '%s' to value '%s'",
+ attribute,
+ existing_value,
+ new_value,
+ )
+ updated[attribute] = dict(old=existing_value, new=new_value)
+
+ return updated, project
+
+
+def load_projects_configuration(projects_file: str) -> Dict[str, Any]:
+ """Load the configuration yaml file as Dict."""
+ with open(projects_file, "r") as f:
+ return yaml.safe_load(f)
+
+
+def namespaces_from_path(path_with_namespace: str) -> Iterable[str]:
+ """Given a path, computes the hierarchic namespaces from generic to specific."""
+ namespaces = []
+ # FIXME: make that a reduce call!?
+ for part in Path(path_with_namespace).parts[:-1]:
+ if namespaces:
+ last_part = namespaces[-1]
+ ns = f"{last_part}/{part}"
+ else:
+ ns = part
+ namespaces.append(ns)
+
+ return namespaces
+
+
+@click.command()
+@click.option(
+ "--gitlab",
+ "-g",
+ "gitlab_instance",
+ help="Which GitLab instance to use, as configured in the python-gitlab config",
+)
+@click.option(
+ "--do-it", "do_it", is_flag=True, help="Actually perform the operations",
+)
+@click.argument("projects_file")
+def cli(gitlab_instance: str, do_it: bool, projects_file: str,) -> None:
+ gl = get_gitlab(gitlab_instance)
+
+ configuration = load_projects_configuration(projects_file)
+
+ # Global configuration for all projects
+ global_settings: Dict[str, Any] = configuration["global_settings"]
+
+ # Namespace project configuration (with possible override on the global config)
+ namespace_settings: Dict[str, Any] = configuration["namespace_settings"]
+
+ # List of projects that the script should act upon (other projects are skipped)
+ managed_project_namespaces = configuration["managed_project_namespaces"]
+
+ # Local specific project configuration (with possible override on the global config)
+ project_settings: Dict[str, Any] = configuration["project_settings"]
+
+ # TODO: Determine whether we want to iterate over all gitlab projects or over the
+ # configured projects in the configuration files. For now, this iterates over the
+ # gitlab projects and skips non-configured ones. That way, we could discover
+ # projects we forgot to configure (by processing the logs afterwards).
+
+ projects: Dict = {}
+ project_updated_count = 0
+
+ # Configure the project according to global settings (with potential specific
+ # project override)
+ for project in gl.projects.list(iterator=True):
+ path_with_namespace = project.path_with_namespace
+ # For the last print statement to explain how many got updated
+ projects[path_with_namespace] = project
+
+ project_namespaces = namespaces_from_path(path_with_namespace)
+ if project_namespaces[0] not in managed_project_namespaces:
+ logger.debug("Skipped non-managed project <%s>", path_with_namespace)
+ continue
+
+ namespace_config = {}
+ # Merge configuration from generic namespace to specific
+ for ns in project_namespaces:
+ namespace_config.update(namespace_settings.get(ns, {}))
+
+ project_config = project_settings.get(path_with_namespace, {})
+
+ logger.debug("Project <%s> %s", path_with_namespace, project.id)
+ updates, project = update_project(
+ project, global_settings, namespace_config, project_config
+ )
+
+ if updates and do_it:
+ project.save()
+ project_updated_count += 1
+
+ if updates:
+ print(json.dumps({path_with_namespace: updates}))
+
+ dry_run = not do_it
+ prefix_msg = "(**DRY RUN**) " if dry_run else ""
+ summary = {
+ "nb_projects": len(projects),
+ "nb_updated_projects": project_updated_count,
+ }
+ if dry_run:
+ summary["dry_run"] = dry_run
+
+ logger.debug(
+ "%sNumber of projects updated: %s / %s",
+ prefix_msg,
+ project_updated_count,
+ len(projects),
+ )
+
+ print(json.dumps(summary))
+
+
+if __name__ == "__main__":
+ logging.basicConfig(
+ level=logging.INFO, format="%(asctime)s %(name)s:%(levelname)s %(message)s"
+ )
+ cli(auto_envvar_prefix="SWH")
diff --git a/gitlab/projects.yml b/gitlab/projects.yml
new file mode 100644
--- /dev/null
+++ b/gitlab/projects.yml
@@ -0,0 +1,201 @@
+# [1] https://docs.gitlab.com/ee/api/projects.html#edit-project
+
+# List of namespaces managed by the script (It should be the first level of the
+# project namespace). Other unmanaged projects will be skipped during the script
+# execution.
+managed_project_namespaces:
+ - infra
+
+namespace_settings:
+ # Dict[str, Any] of key the project namespace, and value a dict of any of the global
+ # settings keys above and value, whatever we want to set.
+ infra/puppet/3rdparty:
+ issues_access_level: disabled
+ infra/ci-cd/debs:
+ issues_access_level: disabled
+
+# Dict[str, Any] of key the project with its path namespace, and value a dict of any of
+# the global settings keys above (e.g. visibility) and value, whatever we want to set
+# (e.g. assuming the key 'visibility', some credentials repositories, we want to set it
+# 'private').
+project_settings:
+ infra/ci-cd/k8s-swh-private-data:
+ visibility: private
+ infra/puppet/puppet-swh-private-data:
+ visibility: private
+ # staging extra repositories
+ infra/websites/www.softwareheritage.org-gandi:
+ visibility: private
+ infra/credentials:
+ visibility: private
+ infra/k8s-swh-private-data:
+ visibility: private
+ infra/annex/annex-private:
+ visibility: private
+ infra/iFWCFG:
+ visibility: private
+
+# Dict[str, Any] of key the setting to change (e.g. 'merge_method') and as value
+# whatever we want to set (e.g. assuming the 'merge_method', 'ff' as in fast-forward).
+# The keys were determined out of the gitlab api documentation [1]. Some were not
+# supported. Those are located after the 'unsupported' comment.
+global_settings:
+ # string (optional): See project visibility level.
+ visibility: public
+ # string (optional): Set the merge method used.
+ merge_method: ff
+ # boolean (optional): Enable Delete source branch option by default for all new merge
+ # requests.
+ remove_source_branch_after_merge: true
+ # string (optional): One of disabled, private, or enabled.
+ # releases_access_level: enabled
+ # boolean: Set whether or not merge requests can be merged with skipped jobs.
+ # allow_merge_on_skipped_pipeline: false
+ # string: One of disabled, private or enabled
+ # analytics_access_level: enabled
+ # boolean: Enable Auto DevOps for this project.
+ # auto_devops_enabled: false
+ # boolean: Set whether auto-closing referenced issues on default branch.
+ # autoclose_referenced_issues: true
+ # string (optional): The Git strategy. Defaults to fetch.
+ # build_git_strategy: fetch
+ # integer (optional): The maximum amount of time, in seconds, that a job can run.
+ # build_timeout: 3600
+ # string (optional): One of disabled, private, or enabled.
+ # builds_access_level: enabled
+ # string (optional): The path to CI configuration file.
+ # ci_config_path: ""
+ # integer (optional): Default number of revisions for shallow cloning.
+ # ci_default_git_depth: 20
+ # boolean (optional): Enable or disable prevent outdated deployment jobs.
+ # ci_forward_deployment_enabled: true
+ # boolean (optional): Enable or disable running pipelines in the parent project for
+ # merge requests from forks. (Introduced in GitLab 15.3.)
+ # ci_allow_fork_pipelines_to_run_in_parent_project: true
+ # boolean (optional): Set whether or not caches should be separated by branch
+ # protection status.
+ # ci_separated_caches: true
+ # string (optional): Set visibility of container registry, for this project, to one of
+ # disabled, private or enabled.
+ # container_registry_access_level: disabled
+ # string (optional): The default branch name.
+ # default_branch: master
+ # string (optional): Short project description.
+ # description: ""
+ # boolean (optional): Disable email notifications.
+ # emails_disabled: true
+ # boolean (optional): Enforce auth checks on uploads.
+ # enforce_auth_checks_on_uploads: true
+ # string (optional): One of disabled, private, or enabled.
+ # forking_access_level: enabled
+ # string (optional): One of disabled, private, or enabled.
+ # issues_access_level: enabled
+ # boolean (optional): Disable or enable the ability to keep the latest artifact for
+ # this project.
+ # keep_latest_artifact: True
+ # boolean (optional): Enable LFS.
+ # lfs_enabled: True
+ # string (optional): Template used to create merge commit message in merge requests.
+ # (Introduced in GitLab 14.5.)
+ # merge_commit_template: ""
+ # string (optional): One of disabled, private, or enabled.
+ # merge_requests_access_level: enabled
+ # boolean (optional): Set whether merge requests can only be merged when all the
+ # discussions are resolved.
+ # only_allow_merge_if_all_discussions_are_resolved: False
+ # boolean (optional): Set whether merge requests can only be merged with successful
+ # jobs.
+ # only_allow_merge_if_pipeline_succeeds: False
+ # string (optional): One of disabled, private, or enabled.
+ # operations_access_level: disabled
+ # boolean (optional): Enable or disable packages repository feature.
+ # packages_enabled: False
+ # string (optional): One of disabled, private, enabled, or public.
+ # pages_access_level: disabled
+ # boolean (optional): Show link to create/view merge request when pushing from the
+ # command line.
+ # printing_merge_request_link_enabled: true
+ # string (optional): One of disabled, private, or enabled.
+ # repository_access_level: enabled
+ # string (optional): Which storage shard the repository is on. (administrators only)
+ # repository_storage: default
+ # boolean (optional): Allow users to request member access.
+ # request_access_enabled: true
+ # boolean (optional): Automatically resolve merge request diffs discussions on lines
+ # changed with a push.
+ # resolve_outdated_diff_discussions: false
+ # boolean (optional): Allow only users with the Maintainer role to pass user-defined
+ # variables when triggering a pipeline. For example when the pipeline is triggered in
+ # the UI, with the API, or by a trigger token.
+ # restrict_user_defined_variables: false
+ # string (optional): (GitLab 14.9 and later) Security and compliance access level. One
+ # of disabled, private, or enabled.
+ # security_and_compliance_access_level: private
+ # boolean (optional): Enable or disable Service Desk feature.
+ # service_desk_enabled: false
+ # boolean (optional): Enable shared runners for this project.
+ # shared_runners_enabled: true
+ # string (optional): One of disabled, private, or enabled.
+ # snippets_access_level: enabled
+ # string (optional): Template used to create squash commit message in merge requests.
+ # (Introduced in GitLab 14.6.)
+ # squash_commit_template: ""
+ # string (optional): One of never, always, default_on, or default_off.
+ # squash_option: default_on
+ # string (optional): The commit message used to apply merge request suggestions.
+ # suggestion_commit_message: ""
+ # string (optional): One of disabled, private, or enabled.
+ # wiki_access_level: enabled
+ # -------------------------------------------------------
+ # Following keys are documented but somehow not supported
+ # -------------------------------------------------------
+ # boolean: Indicates that merges of merge requests should be blocked unless all status
+ # checks have passed. Defaults to false. Introduced in GitLab 15.5 with feature flag
+ # only_allow_merge_if_all_status_checks_passed disabled by default.
+ # only_allow_merge_if_all_status_checks_passed: ?
+ # integer: How many approvers should approve merge request by default. To configure
+ # approval rules, see Merge request approvals API.
+ # approvals_before_merge: ?
+ # string: Auto-cancel pending pipelines. This isn’t a boolean, but enabled/disabled.
+ # auto_cancel_pending_auto: ?
+ # string: Auto Deploy strategy (continuous, manual, or timed_incremental).
+ # pipelines_devops_deploy_strategy: ?
+ # mixed (optional): Image file for avatar of the project.
+ # avatar: ?
+ # hash (optional): Update the image cleanup policy for this project. Accepts: cadence
+ # (string), keep_n (integer), older_than (string), name_regex (string),
+ # name_regex_delete (string), name_regex_keep (string), enabled (boolean).
+ # container_expiration_policy_attributes: ?
+ # string (optional): The classification label for the project.
+ # external_authorization_classification_label: ?
+ # string (optional): Default description for Issues. Description is parsed with GitLab
+ # Flavored Markdown. See Templates for issues and merge requests.
+ # issues_template: ?
+ # boolean (optional): Enable or disable merge pipelines.
+ # merge_pipelines_enabled: ?
+ # string (optional): Default description for merge requests. Description is parsed
+ # with GitLab Flavored Markdown. See Templates for issues and merge requests.
+ # merge_requests_template: ?
+ # boolean (optional): Enable or disable merge trains.
+ # merge_trains_enabled: ?
+ # boolean (optional): Pull mirror overwrites diverged branches.
+ # mirror_overwrites_diverged_branches: ?
+ # boolean (optional): Pull mirroring triggers builds.
+ # mirror_trigger_builds: ?
+ # integer (optional): User responsible for all the activity surrounding a pull mirror
+ # event. (administrators only)
+ # mirror_user_id: ?
+ # boolean (optional): Enables pull mirroring in a project.
+ # mirror: ?
+ # boolean (optional): For forked projects, target merge requests to this project. If
+ # false, the target will be the upstream project.
+ # mr_default_target_self: ?
+ # boolean (optional): Only mirror protected branches.
+ # only_mirror_protected_branches: ?
+ # boolean (optional): If true, jobs can be viewed by non-project members.
+ # public_builds: ?
+ # string (optional): One of disabled, private, enabled or public
+ # requirements_access_level: ?
+ # string (optional): Template used to suggest names for branches created from issues.
+ # (Introduced in GitLab 15.6.)
+ # issue_branch_template: ?

File Metadata

Mime Type
text/plain
Expires
Thu, Jan 30, 4:42 PM (5 h, 39 m ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217342

Event Timeline

About · Developer information · Legal