diff --git a/bin/import-puppet-module b/bin/import-puppet-module index 455816c..ad8cf59 100755 --- a/bin/import-puppet-module +++ b/bin/import-puppet-module @@ -1,201 +1,223 @@ #!/usr/bin/python3 # Import a Puppet module from the Puppet Forge to the Software Heritage # Phabricator instance +import click import copy import os import subprocess import sys import tempfile import time from phabricator import Phabricator import requests PUPPETFORGE_API = 'https://forgeapi.puppetlabs.com/v3/' PUPPETFORGE_URL = 'https://forge.puppet.com/' PHABRICATOR_EDITORS = 'PHID-PROJ-w6dmg2vssgiimpjpduga' # System administrators PHABRICATOR_TAGS = [ 'PHID-PROJ-ailxer42p4nlkaikyhlo', # Language-puppet 'PHID-PROJ-ximqdentnggqkdbc2ycg', # Sync to GitHub 'PHID-PROJ-w6dmg2vssgiimpjpduga', # System administrators ] PHABRICATOR_DATA = [ {'type': 'space', 'value': 'PHID-SPCE-yrejny25ty3jfio326zt'}, {'type': 'vcs', 'value': 'git'}, {'type': 'status', 'value': 'active'}, {'type': 'view', 'value': 'public'}, {'type': 'edit', 'value': PHABRICATOR_EDITORS}, {'type': 'policy.push', 'value': PHABRICATOR_EDITORS}, {'type': 'projects.set', 'value': PHABRICATOR_TAGS}, ] PHABRICATOR_REPO_URL = 'https://forge.softwareheritage.org/source/' MRCONFIG_MARKER = '## END SWH REPOS ##' MRCONFIG_TEMPLATE = """\ [{module_name}] checkout = git clone {phabricator_repo} {module_name} \\ \t && cd {module_name} \\ \t && git remote add upstream {upstream_repo} \\ \t && git fetch upstream """ def get_puppetforge_module_info(module_name): """Get module information from the Puppet forge""" r = requests.get(PUPPETFORGE_API + 'modules/' + module_name) if not r.ok: raise ValueError('Error when fetching %s from the Puppet forge: %s' % (module_name, ', '.join(r.json()['errors']))) return r.json() def create_phab_repo(phabricator, module_name, release_metadata): """Create a repository on the Software Hertiage forge with proper metadata""" upstream_repo = release_metadata['source'] original_description = release_metadata['summary'] puppetforge_url = PUPPETFORGE_URL + module_name.replace('-', '/') description = '''\ {description} {puppetforge_url} {upstream_repo} '''.format( description=original_description, puppetforge_url=puppetforge_url, upstream_repo=upstream_repo, ) phabricator_data = copy.deepcopy(PHABRICATOR_DATA) phabricator_name = 'puppet-%s' % module_name phabricator_data.append({'type': 'name', 'value': phabricator_name}) phabricator_data.append({'type': 'shortName', 'value': phabricator_name}) phabricator_data.append({'type': 'description', 'value': description.strip()}) return phabricator.diffusion.repository.edit(transactions=phabricator_data) -def wait_phab_repo(phabricator, id): +def wait_phab_repo(phabricator, repo_id): """Wait for the repository with the given id to be ready""" backoff = 0.5 while True: - ret = phabricator.diffusion.repository.search( - constraints={'ids': [id]} - ) + ret = phabricator.diffusion.repository.search(constraints={'ids': [repo_id]}) repo = ret.data[0] if not repo['fields']['isImporting']: break - print("Repository %s still importing, sleeping for %s seconds..." % - (id, backoff)) + print( + f"Repository {repo_id} still importing, sleeping for {backoff} seconds..." + ) time.sleep(backoff) backoff = min(backoff * 2, 15.0) -def clone_and_push_repo(module_name, phabricator_repo, upstream_repo): +def clone_and_push_repo(module_name, phabricator_repo, upstream_repo, default_branch="master", skip_clone=False): """Clone the SWH repository, add the upstream remote and push to SWH""" - subprocess.check_call(['git', 'clone', phabricator_repo, module_name]) + if not skip_clone: + subprocess.check_call(['git', 'clone', phabricator_repo, module_name]) os.chdir(module_name) - subprocess.check_call(['git', 'remote', 'add', 'upstream', upstream_repo]) + try: + subprocess.check_call(['git', 'remote', 'add', 'upstream', upstream_repo]) + except subprocess.CalledProcessError: + # already configured, skipping + pass subprocess.check_call(['git', 'fetch', 'upstream', '--tags']) - subprocess.check_call(['git', 'merge', 'upstream/master']) + subprocess.check_call(['git', 'checkout', default_branch]) + subprocess.check_call(['git', 'merge', f'upstream/{default_branch}']) subprocess.check_call(['git', 'push', '--tags', '--set-upstream', - 'origin', 'master:master']) + 'origin', f'{default_branch}:{default_branch}']) os.chdir('..') def read_mrconfig(): """Parse the .mrconfig file""" mrconfig = [] with open('.mrconfig') as f: for block in f.read().strip().split('\n\n'): mrconfig.append(block.strip()) swh_repos_index = mrconfig.index(MRCONFIG_MARKER) swh_repos = {} for i, block in enumerate(mrconfig[:swh_repos_index]): if not block.startswith('[') or ']' not in block: raise ValueError( 'Could not parse .mrconfig: missing [] in block %d' % (i + swh_repos_index + 1) ) module_name = block[1:block.index(']')] swh_repos[module_name] = block extra_repos = {} for i, block in enumerate(mrconfig[swh_repos_index + 1:]): if not block.startswith('[') or ']' not in block: raise ValueError( 'Could not parse .mrconfig: missing [] in block %d' % (i + swh_repos_index + 1) ) module_name = block[1:block.index(']')] extra_repos[module_name] = block return swh_repos, extra_repos def write_mrconfig(swh_repos, extra_repos): """Write the mrconfig from the output of read_mrconfig()""" out = [] for repo, block in sorted(swh_repos.items()): out.append(block.strip()) out.append(MRCONFIG_MARKER) for repo, block in sorted(extra_repos.items()): out.append(block.strip()) with tempfile.NamedTemporaryFile(mode='w+', dir='.', delete=False) as f: f.write('\n\n'.join(out)) f.write('\n') os.rename(f.name, '.mrconfig') def update_mrconfig(module_name, phabricator_repo, upstream_repo): """Add the new module to .mrconfig""" swh_repos, extra_repos = read_mrconfig() extra_repos[module_name] = MRCONFIG_TEMPLATE.format( module_name=module_name, phabricator_repo=phabricator_repo, upstream_repo=upstream_repo, ) write_mrconfig(swh_repos, extra_repos) ret = copy.deepcopy(swh_repos) ret.update(extra_repos) return ret def update_gitignore(modules): with tempfile.NamedTemporaryFile(mode='w+', dir='.', delete=False) as f: for module in sorted(modules): if module == 'DEFAULT': continue print('/%s/' % module, file=f) os.rename(f.name, '.gitignore') -if __name__ == '__main__': - if len(sys.argv) != 2: - print('Usage: %s publisher-module_name' % sys.argv[0], file=sys.stderr) - sys.exit(2) +@click.command() +@click.option("--default-branch", default="master", + help="Repository's default branch name: e.g. main)") +@click.option("-i", "--repo-id", type=click.INT, + help="When a repository already exists, skip creation and use it.") +@click.option("--clone/--no-clone", "clone", type=click.BOOL, + help="By default clone a repository locally, we can inhibit that step.") +@click.argument("module_name") +def main(module_name, repo_id, default_branch, clone): + """Import necessary puppet module within the swh forge - module_name = sys.argv[1] + MODULE_NAME "Pupppetlabs' module name, e.g. elastic-elasticsearch, puppetlabs-zfs_core + """ puppetforge_info = get_puppetforge_module_info(module_name) release_meta = puppetforge_info['current_release']['metadata'] upstream_repo = release_meta['source'] phabricator_repo = PHABRICATOR_REPO_URL + 'puppet-%s.git' % module_name phabricator = Phabricator() phabricator.connect() phabricator.update_interfaces() - repo = create_phab_repo(phabricator, module_name, release_meta) - repo_id = repo['object']['id'] + + if not repo_id: + repo = create_phab_repo(phabricator, module_name, release_meta) + repo_id = repo['object']['id'] + wait_phab_repo(phabricator, repo_id) - clone_and_push_repo(module_name, phabricator_repo, upstream_repo) + clone_and_push_repo( + module_name, phabricator_repo, upstream_repo, + default_branch=default_branch, skip_clone=not clone) repos = update_mrconfig(module_name, phabricator_repo, upstream_repo) update_gitignore(repos) + + +if __name__ == '__main__': + main()