diff --git a/README.md b/README.md index dd53282..db796ed 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,27 @@ swh-mirror-forge ================ Mirror swh's forge to github # Configuration In $SWH_CONFIG_PATH/mirror-forge/config.yml (SWH_CONFIG_PATH being one of /etc/softwareheritage/,~/.config/swh, or ~/.swh/), add the following information: ```yaml github: forge: ``` Docs: - github: https://github.com/settings/tokens - swh forge: https://forge.softwareheritage.org/settings/user//page/apitokens/ # Use For now, on a per repository basis: ```sh -python3 -m swh.mirror.forge.sync --repo-callsign DMOD \ - --repo-name swh-model \ - --repo-url https://forge.softwareheritage.org/source/swh-model/ \ - --repo-description "" \ - --credential-key-id 3 +python3 -m swh.mirror.forge.sync --repo-callsign DMOD --credential-key-id 3 ``` diff --git a/swh/mirror/forge/sync.py b/swh/mirror/forge/sync.py index 7623fdb..017d827 100755 --- a/swh/mirror/forge/sync.py +++ b/swh/mirror/forge/sync.py @@ -1,152 +1,205 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import json import sys import requests +from os.path import basename + from swh.core.config import SWHConfig from .api import RepositorySearch, PassphraseSearch, DiffusionUriEdit FORGE_API_URL = 'https://forge.softwareheritage.org' class SWHMirrorForge(SWHConfig): CONFIG_BASE_FILENAME = 'mirror-forge/config' DEFAULT_CONFIG = { 'github': ('str', None), 'forge': ('str', None), } def __init__(self): super().__init__() self.config = self.parse_config_file() self.token_github = self.config['github'] self.token_forge = self.config['forge'] def prepare_token(): """Prepare the needed token from the disk. Returns: tuple (token-forge, token-github) """ swh_mirror_forge = SWHMirrorForge() token_forge = swh_mirror_forge.token_forge if not token_forge: print("""Install the phabricator forge's token in $SWH_CONFIG_PATH/mirror-forge/config.yml (https://forge.softwareheritage.org/settings/user//page/apitokens/). Once the installation is done, you can trigger this script again. """) sys.exit(1) token_github = swh_mirror_forge.token_github if not token_github: print("""Install one personal github token in $SWH_CONFIG_PATH/mirror-forge/config.yml with scope public_repo (https://github.com/settings/tokens). You must be associated to https://github.com/softwareheritage organization. Once the installation is done, you can trigger this script again. """) sys.exit(1) return token_forge, token_github +def mirror_exists(data): + """Check the existence of the mirror. + + Args: + data: full information on the repository + + Returns + True if mirror already exists. False otherwise. + + """ + uris = data['attachments']['uris']['uris'] + for uri in uris: + effective_url = uri['fields']['uri']['effective'] + if 'github' in effective_url: + return True + + return False + + +def retrieve_repo_information(data): + """Given information on repository, extract the needed information for mirroring. + + Args: + data: full information on the repository + + Returns: + dict with keys phid, description, url, name. + + """ + uris = data['attachments']['uris']['uris'] + for uri in uris: + effective_url = uri['fields']['uri']['effective'] + if 'https' in effective_url and '.git' in effective_url: + elected_url = effective_url + + return { + 'phid': data['phid'], + 'description': data['fields']['name'], + 'url': elected_url, + 'name': basename(elected_url).split('.')[0], + } + + @click.command() @click.option('--repo-callsign', help="Repository's callsign") -@click.option('--repo-name', - help="Repository name (used in github)") -@click.option('--repo-url', - help="Repository's forge url (used in github)") -@click.option('--repo-description', - help="Repository's description (used in github)") @click.option('--credential-key-id', help="credential to use for access from phabricator's forge to github") @click.option('--github/--nogithub', default=True) -def run(repo_callsign, repo_name, repo_url, repo_description, - credential_key_id, github): +def run(repo_callsign, credential_key_id, github): """This will instantiate a mirror from a repository forge to github. + Args: + repo_callsign: repository's identifier callsign. This will be + used to fetch information on the repository to + mirror. + + credential_key_id: the key the forge will use to push to + modifications to github + + github: to inhibit the mirror creation in github. By default, + it creates it. Note that, in any case, a check is done + to stop if a mirror uri is already referenced in the + forge about github. + """ ### Retrieve credential access to github and phabricator's forge token_forge, token_github = prepare_token() ### Retrieve repository information query = RepositorySearch(FORGE_API_URL, token_forge) data = query.request(constraints={ "callsigns": [repo_callsign], }, attachments={ "uris": True }) - repo_phid = data[0]['phid'] - ### Check existence of mirror already set - for uri in data[0]['attachments']['uris']['uris']: - for url in uri['fields']['uri'].values(): - if 'github' in url: - print('Mirror already installed at %s, stopping.' % url) - sys.exit(0) + # Also determine an uri to provide as original url in the github mirror + + repository_information = data[0] + + if mirror_exists(repository_information): + print('Mirror already configured for %s, stopping.' % repo_callsign) + sys.exit(0) + + repo = retrieve_repo_information(repository_information) ### Create repository in github - if github or mirror: + if github: r = requests.post( 'https://api.github.com/orgs/SoftwareHeritage/repos', headers={'Authorization': 'token %s' % token_github}, data=json.dumps({ - "name": repo_name, - "description": repo_description, - "homepage": repo_url, + "name": repo['name'], + "description": repo['description'], + "homepage": repo['url'], "private": False, "has_issues": False, "has_wiki": False, "has_downloads": True })) if not r.ok: print("""Failure to create the repository in github. Status: %s""" % r.status_code) sys.exit(1) ### Retrieve credential information query = PassphraseSearch(FORGE_API_URL, token_forge) data = query.request(ids=[credential_key_id]) # Retrieve the phid for that passphrase key_phid = list(data.values())[0]['phid'] - repo_url_github = 'git@github.com:SoftwareHeritage/%s.git' % repo_name + repo['url_github'] = 'git@github.com:SoftwareHeritage/%s.git' % repo['name'] ### Install the github mirror in the forge query = DiffusionUriEdit(FORGE_API_URL, token_forge) data = query.request(transactions=[ - {"type": "repository", "value": repo_phid}, - {"type": "uri", "value": repo_url_github}, + {"type": "repository", "value": repo['phid']}, + {"type": "uri", "value": repo['url_github']}, {"type": "io", "value": "mirror"}, {"type": "display", "value": "never"}, {"type": "disable", "value": False}, {"type": "credential", "value": key_phid}, ]) - print("Repository %s mirrored at %s." % (repo_url, repo_url_github)) + print("Repository %s mirrored at %s." % (repo['url'], repo['url_github'])) sys.exit(0) if __name__ == '__main__': run()