Details

Reviewers

Group Reviewers

Commits

rDSNIPecab8f25d50c: Configure gitlab projects declaratively

Summary

This allows to define the configuration for managed repositories in order from
global (.e.g. merge request merge policy) to namespace (e.g. deactivate issue tracker)
to specific project (e.g. private visibility, etc...).

This only deals with the declared namespaced projects in the configuration file. This
skips the remaining existing gitlab projects (user projects, ...).

Use:

$ python manage_projects.py --gitlab swh projects.yml | jq .
{
  "infra/ci-cd/swh-charts": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/swh-apps": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-zfs_core": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-elastic-elastic_stack": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-translate": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-redis": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-java_ks": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-treydock-keycloak": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-cassandra": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/getsentry-onpremise": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-docker": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-rabbitmq": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/swh-ansible": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-letsencrypt": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-covermymeds-pgbouncer": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/swh-grafana-dashboards": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/puppet-swh-private-data-censored": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-richardc-datacat": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-elastic-elasticsearch": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-deric-zookeeper": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-java": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-archive": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-kafka": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-grafana": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-prometheus": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-php": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-camptocamp-systemd": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-openstack-ceph": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-claranet-varnish": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-ssm-hitch": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-vcsrepo": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-icinga-icingaweb2": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-inifile": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-puppetdb": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-nginx": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-unattended_upgrades": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-saz-memcached": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-stm-debconf": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-saz-timezone": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-gunicorn": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-postgresql": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-icinga-icinga2": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-saz-resolv_conf": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-arioch-redis": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppet-extlib": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-wikimedia-kafka": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-wikimedia-zookeeper": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-mediawiki": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-mosen-cups": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-uwsgi": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/puppet-swh-systemd": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-apt": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-saz-sudo": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-stdlib": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-saz-ssh": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-role": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/swh-sysadmin-provisioning": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-theforeman-puppet": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-profile": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/puppet-swh-private-data": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-thias-php": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-postfix": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-ntp": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-mysql": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-ssm-munin": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-ripienaar-module_data": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-saz-locales": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-rlenglet-debconf_package": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-trepasi-debnet": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-swh-dar": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-concat": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-inkblot-bind": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/3rdparty/puppet-puppetlabs-apache": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    },
    "issues_access_level": {
      "old": "enabled",
      "new": "disabled"
    }
  }
}
{
  "infra/puppet/puppet-environment": {
    "merge_method": {
      "old": "merge",
      "new": "ff"
    }
  }
}
{
  "nb_projects": 86,
  "nb_updated_projects": 0,
  "dry_run": true
}

gitlab api: https://docs.gitlab.com/ee/api/projects.html#edit-project

Test Plan

Run in staging ok [1]

[1] P1526

Diff Detail

Repository

rDSNIP Code snippets

Branch

master

Lint

No Linters Available

Unit

No Unit Test Coverage

Build Status

Buildable 32770
Build 51342: arc lint + arc unit

Event Timeline

ardumont created this revision.Nov 9 2022, 4:49 PM

Herald added a reviewer: Reviewers. · View Herald TranscriptNov 9 2022, 4:49 PM

ardumont requested review of this revision.Nov 9 2022, 4:49 PM

Harbormaster completed remote builds in B32757: Diff 31818.Nov 9 2022, 4:49 PM

ardumont added inline comments.Nov 9 2022, 10:51 PM

gitlab/manage_projects.py
51–53	dict comprehension!
gitlab/projects.yml
107	Those are all configuration that will work if overriden in the configuration with the snippet code (i've double checked from the api documentation and the toplevel experiments).

Looks nice!

I guess we may want to merge three levels of settings:

global (default) settings
per-namespace settings (for instance, to have all projects under infra/puppet/3rdparty have the same config overrides, e.g. disabling the issue tracker, changing the destination of webhooks to #swh-sysadm, etc.)
and per-repository settings (in the case of visibility, it's quite obvious)

I wonder whether we should merge this config file and that of the other script, so that we get all settings in one place.

What do you think?

olasd added inline comments.Nov 10 2022, 11:03 AM

gitlab/manage_projects.py
61–63	Am I missing something, or would this skip over repos configured with an empty dict? Overall I think we should probably just have a list of namespaces in which all projects are configured unconditionally, rather than having to inline the list of all projects and keeping it up to date?

In D8825#229537, @olasd wrote:

Looks nice!

I guess we may want to merge three levels of settings:

global (default) settings

per-namespace settings (for instance, to have all projects under infra/puppet/3rdparty have the same config overrides, e.g. disabling the issue tracker, changing the destination of webhooks to #swh-sysadm, etc.)

and per-repository settings (in the case of visibility, it's quite obvious)

That'd make sense.

I wonder whether we should merge this config file and that of the other script, so that we get all settings in one place.

What do you think?

Yes, i think so too. But which script, to be sure, manage_user_groups right?

I just started it that way to simplify the reading.
I think that i'll adapt according to what you proposed here then update the diff then we land.
And finally, i'll have a stab to refactor the both of them. If that makes sense.

gitlab/manage_projects.py
61–63	Am I missing something, or would this skip over repos configured with an empty dict? I changed that locally to check against None instead indeed. Overall I think we should probably just have a list of namespaces in which all projects are configured unconditionally, rather than having to inline the list of all projects and keeping it up to date? If I apply what you mentioned earlier with a 3rd level of configuration, that should work.

ardumont added inline comments.Nov 10 2022, 11:19 AM

gitlab/manage_projects.py
61–63	Am I missing something, or would this skip over repos configured with an empty dict? I changed that locally to check against None instead indeed. Actually, I did not and somehow it was not an issue ¯\_(ツ)_/¯. Nvm this anyway as i'm changing that logic.

Adapt according to discussion and more

Harbormaster completed remote builds in B32767: Diff 31827.Nov 10 2022, 11:54 AM

ardumont retitled this revision from gitlab projects: Unify project configuration to Allow gitlab projects configuration declaratively.Nov 10 2022, 11:55 AM

ardumont edited the summary of this revision. (Show Details)

Looks great, thanks.

In your dry run I see that the script is trying to change the config of projects in user namespaces, which is not okay. Could you add (in the config, I guess) an explicit list of namespace prefixes that the script will act on, instead of letting it run on all projects?

Looks great, thanks.

\o/

In your dry run I see that the script is trying to change the config of projects in user namespaces, which is not okay. Could you add (in the config, I guess) an explicit list of namespace prefixes that the script will act on, instead of letting it run on all projects?

I'll double check that part.

ardumont edited the summary of this revision. (Show Details)Nov 10 2022, 2:47 PM

Manage only a list of namespaces and skip the other projects
Make the script outputs json (for easier parsing)

Harbormaster completed remote builds in B32768: Diff 31828.Nov 10 2022, 2:48 PM

I'm not too sure about having a single "big" object as json output, as dumping that full json structure will only work out if the process didn't crash in the middle (and when that does, you won't really know what had happened until then). Maybe output one json object per project mutated instead?

This revision is now accepted and ready to land.Nov 10 2022, 3:03 PM

ardumont edited the test plan for this revision. (Show Details)Nov 10 2022, 3:19 PM

Adapt according to last remark, dump partial json updated project.

Tested against staging again with success.

Harbormaster completed remote builds in B32770: Diff 31831.Nov 10 2022, 3:23 PM

ardumont retitled this revision from Allow gitlab projects configuration declaratively to Configure gitlab projects declaratively.Nov 10 2022, 3:26 PM

ardumont edited the summary of this revision. (Show Details)

ardumont marked an inline comment as done.

Closed by commit rDSNIPecab8f25d50c: Configure gitlab projects declaratively (authored by ardumont). · Explain WhyNov 10 2022, 3:45 PM

This revision was automatically updated to reflect the committed changes.

ardumont added a commit: rDSNIPecab8f25d50c: Configure gitlab projects declaratively.

I wonder whether we should merge this config file and that of the other script, so that we get all settings in one place.

What do you think?

Yes, i think so too. But which script, to be sure, manage_user_groups right?

I just started it that way to simplify the reading.
I think that i'll adapt according to what you proposed here then update the diff then we land.

done

And finally, i'll have a stab to refactor the both of them. If that makes sense.

Done in D8831

Configure gitlab projects declaratively
ClosedPublic
Actions

Details

Diff Detail

Event Timeline

Revision Contents
Changeset List

Diff 31831

gitlab/manage_projects.py

gitlab/projects.yml

Configure gitlab projects declarativelyClosedPublicActions

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 31831

gitlab/manage_projects.py

gitlab/projects.yml

Configure gitlab projects declaratively
ClosedPublic
Actions

Revision Contents
Changeset List