Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/pypi/lister.py
| # Copyright (C) 2018 the Software Heritage developers | # Copyright (C) 2018-2019 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import random | import random | ||||
| import xmltodict | import xmltodict | ||||
| from .models import PyPIModel | from .models import PyPIModel | ||||
| from swh.scheduler import utils | from swh.scheduler import utils | ||||
| from swh.lister.core.simple_lister import SimpleLister | from swh.lister.core.simple_lister import SimpleLister | ||||
| from swh.lister.core.lister_transports import ListerOnePageApiTransport | from swh.lister.core.lister_transports import ListerOnePageApiTransport | ||||
| class PyPILister(ListerOnePageApiTransport, SimpleLister): | class PyPILister(ListerOnePageApiTransport, SimpleLister): | ||||
| MODEL = PyPIModel | MODEL = PyPIModel | ||||
| LISTER_NAME = 'pypi' | LISTER_NAME = 'pypi' | ||||
| PAGE = 'https://pypi.org/simple/' | PAGE = 'https://pypi.org/simple/' | ||||
| instance = 'pypi' # As of today only the main pypi.org is used | |||||
| def __init__(self, override_config=None): | def __init__(self, override_config=None): | ||||
| ListerOnePageApiTransport .__init__(self) | ListerOnePageApiTransport .__init__(self) | ||||
| SimpleLister.__init__(self, override_config=override_config) | SimpleLister.__init__(self, override_config=override_config) | ||||
| def task_dict(self, origin_type, origin_url, **kwargs): | def task_dict(self, origin_type, origin_url, **kwargs): | ||||
| """(Override) Return task format dict | """(Override) Return task format dict | ||||
| This is overridden from the lister_base as more information is | This is overridden from the lister_base as more information is | ||||
| needed for the ingestion task creation. | needed for the ingestion task creation. | ||||
| """ | """ | ||||
| _type = 'origin-update-%s' % origin_type | _type = 'load-%s' % origin_type | ||||
| _policy = 'recurring' | _policy = 'recurring' | ||||
| project_name = kwargs.get('name') | project_name = kwargs.get('name') | ||||
| project_metadata_url = kwargs.get('html_url') | project_metadata_url = kwargs.get('html_url') | ||||
| return utils.create_task_dict( | return utils.create_task_dict( | ||||
| _type, _policy, project_name, origin_url, | _type, _policy, project_name, origin_url, | ||||
| project_metadata_url=project_metadata_url) | project_metadata_url=project_metadata_url) | ||||
| def list_packages(self, response): | def list_packages(self, response): | ||||
| Show All 37 Lines | |||||