Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gnu/lister.py
# Copyright (C) 2019 the Software Heritage developers | # Copyright (C) 2019 the Software Heritage developers | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import logging | import logging | ||||
from swh.scheduler import utils | from swh.scheduler import utils | ||||
from swh.lister.core.simple_lister import SimpleLister | from swh.lister.core.simple_lister import SimpleLister | ||||
from swh.lister.gnu.models import GNUModel | from swh.lister.gnu.models import GNUModel | ||||
from swh.lister.gnu.tree import GNUTree | from swh.lister.gnu.tree import GNUTree | ||||
from typing import Any, Dict, List | |||||
from requests import Response | |||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
class GNULister(SimpleLister): | class GNULister(SimpleLister): | ||||
MODEL = GNUModel | MODEL = GNUModel | ||||
LISTER_NAME = 'gnu' | LISTER_NAME = 'gnu' | ||||
instance = 'gnu' | instance = 'gnu' | ||||
Show All 32 Lines | def task_dict(self, origin_type, origin_url, **kwargs): | ||||
return utils.create_task_dict( | return utils.create_task_dict( | ||||
'load-archive-files', | 'load-archive-files', | ||||
kwargs.get('policy', 'oneshot'), | kwargs.get('policy', 'oneshot'), | ||||
url=origin_url, | url=origin_url, | ||||
artifacts=artifacts, | artifacts=artifacts, | ||||
retries_left=3, | retries_left=3, | ||||
) | ) | ||||
def safely_issue_request(self, identifier): | def safely_issue_request(self, identifier: int) -> None: | ||||
"""Bypass the implementation. It's now the GNUTree which deals with | """Bypass the implementation. It's now the GNUTree which deals with | ||||
querying the gnu mirror. | querying the gnu mirror. | ||||
As an implementation detail, we cannot change simply the base | As an implementation detail, we cannot change simply the base | ||||
SimpleLister as other implementation still uses it. This shall be part | SimpleLister as other implementation still uses it. This shall be part | ||||
of another refactoring pass. | of another refactoring pass. | ||||
""" | """ | ||||
return None | return None | ||||
def list_packages(self, response): | def list_packages(self, response: Response) -> List[Dict[str, Any]]: | ||||
"""List the actual gnu origins (package name) with their name, url and | """List the actual gnu origins (package name) with their name, url and | ||||
associated tarballs. | associated tarballs. | ||||
Args: | Args: | ||||
response: Unused | response: Unused | ||||
Returns: | Returns: | ||||
List of packages name, url, last modification time:: | List of packages name, url, last modification time:: | ||||
Show All 10 Lines | def list_packages(self, response: Response) -> List[Dict[str, Any]]: | ||||
'time_modified': '2016-12-06T02:37:10+00:00' | 'time_modified': '2016-12-06T02:37:10+00:00' | ||||
}, | }, | ||||
... | ... | ||||
] | ] | ||||
""" | """ | ||||
return list(self.gnu_tree.projects.values()) | return list(self.gnu_tree.projects.values()) | ||||
def get_model_from_repo(self, repo): | def get_model_from_repo(self, repo: Dict[str, Any]) -> Dict[str, Any]: | ||||
"""Transform from repository representation to model | """Transform from repository representation to model | ||||
""" | """ | ||||
return { | return { | ||||
'uid': repo['url'], | 'uid': repo['url'], | ||||
'name': repo['name'], | 'name': repo['name'], | ||||
'full_name': repo['name'], | 'full_name': repo['name'], | ||||
'html_url': repo['url'], | 'html_url': repo['url'], | ||||
'origin_url': repo['url'], | 'origin_url': repo['url'], | ||||
'time_last_updated': repo['time_modified'], | 'time_last_updated': repo['time_modified'], | ||||
'origin_type': 'tar', | 'origin_type': 'tar', | ||||
} | } |