Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/packagist/lister.py
| # Copyright (C) 2019 The Software Heritage developers | # Copyright (C) 2019 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import random | import random | ||||
| from typing import Any, List, Mapping | from typing import Any, Dict, List, Mapping | ||||
| from swh.scheduler import utils | from swh.scheduler import utils | ||||
| from swh.lister.core.simple_lister import SimpleLister | from swh.lister.core.simple_lister import SimpleLister | ||||
| from swh.lister.core.lister_transports import ListerOnePageApiTransport | from swh.lister.core.lister_transports import ListerOnePageApiTransport | ||||
| from .models import PackagistModel | from .models import PackagistModel | ||||
| Show All 36 Lines | class PackagistLister(ListerOnePageApiTransport, SimpleLister): | ||||
| PAGE = 'https://packagist.org/packages/list.json' | PAGE = 'https://packagist.org/packages/list.json' | ||||
| instance = 'packagist' | instance = 'packagist' | ||||
| def __init__(self, override_config=None): | def __init__(self, override_config=None): | ||||
| ListerOnePageApiTransport .__init__(self) | ListerOnePageApiTransport .__init__(self) | ||||
| SimpleLister.__init__(self, override_config=override_config) | SimpleLister.__init__(self, override_config=override_config) | ||||
| def task_dict(self, origin_type: str, origin_url: str, | def task_dict(self, origin_type: str, origin_url: str, | ||||
| **kwargs: Mapping[str, str]) -> Mapping[str, str]: | **kwargs: Mapping[str, str]) -> Dict[str, Any]: | ||||
| """Return task format dict | """Return task format dict | ||||
| This is overridden from the lister_base as more information is | This is overridden from the lister_base as more information is | ||||
| needed for the ingestion task creation. | needed for the ingestion task creation. | ||||
| """ | """ | ||||
| return utils.create_task_dict( | return utils.create_task_dict( | ||||
| 'load-%s' % origin_type, | 'load-%s' % origin_type, | ||||
| Show All 27 Lines | |||||