diff --git a/swh/lister/gitlab/lister.py b/swh/lister/gitlab/lister.py --- a/swh/lister/gitlab/lister.py +++ b/swh/lister/gitlab/lister.py @@ -2,6 +2,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import random import re import time @@ -21,6 +22,63 @@ # (method disable_deleted_repo_tasks) MODEL = GitLabModel + @property + def CONFIG_BASE_FILENAME(self): + """One gitlab lister for all instances. We discriminate between the + origin on a per instance basis in the table. + + """ + return 'lister-gitlab' + + @property + def ADDITIONAL_CONFIG(self): + """Override additional config as the 'credentials' structure change + between the ancestor classes and the subclass. + + cf. request_params method below + + """ + return { + 'lister_db_url': + ('str', 'postgresql:///lister-gitlab'), + 'credentials': # credentials is a dict + ('dict', {}), + 'cache_responses': + ('bool', False), + 'cache_dir': + ('str', '~/.cache/swh/lister/%s' % self.lister_name), + } + + def request_params(self, identifier): + """Get the full parameters passed to requests given the + transport_request identifier. + + For the gitlab lister, the 'credentials' entries is configured + per instance. For example: + + - credentials: + - gitlab.com: + - username: user0 + password: + - username: user1 + password: + - ... + - other-gitlab-instance: + ... + + """ + params = { + 'headers': self.request_headers() or {} + } + # Retrieve the credentials per instance + creds = self.config['credentials'] + if creds: + creds_lister = creds[self.lister_name] + auth = random.choice(creds_lister) if creds else None + if auth: + params['auth'] = (auth['username'], auth['password']) + return params + def filter_before_inject(self, models_list): """We cannot filter so returns the models_list as is. @@ -29,6 +87,7 @@ def get_model_from_repo(self, repo): return { + 'instance': self.lister_name, 'uid': repo['id'], 'indexable': repo['id'], 'name': repo['name'], diff --git a/swh/lister/gitlab/models.py b/swh/lister/gitlab/models.py --- a/swh/lister/gitlab/models.py +++ b/swh/lister/gitlab/models.py @@ -2,19 +2,31 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, Boolean, Integer +from sqlalchemy import Column, Boolean, Integer, String from ..core.models import ModelBase class GitLabModel(ModelBase): - """a Gitlab repository""" - __tablename__ = 'main_gitlab_repos' + """a Gitlab repository from a gitlab instance - uid = Column(Integer, primary_key=True) + """ + __tablename__ = 'gitlab_repo' + + id = Column(Integer, primary_key=True) + uid = Column(Integer, index=True) + instance = Column(String, index=True) indexable = Column(Integer, index=True) fork = Column(Boolean) - def __init__(self, *args, **kwargs): + def __init__(self, uid=None, indexable=None, name=None, + full_name=None, html_url=None, origin_url=None, + origin_type=None, description=None, task_id=None, + origin_id=None, instance=None, **kwargs): + super().__init__(uid=uid, indexable=indexable, name=name, + full_name=full_name, html_url=html_url, + origin_url=origin_url, origin_type=origin_type, + description=description, task_id=task_id, + origin_id=origin_id) self.fork = kwargs.pop('fork', False) - super().__init__(*args, **kwargs) + self.instance = instance