Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/cran/lister.py
| # Copyright (C) 2019 the Software Heritage developers | # Copyright (C) 2019 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import subprocess | import subprocess | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import pkg_resources | import pkg_resources | ||||
| from collections import defaultdict | |||||
| from swh.lister.cran.models import CRANModel | from swh.lister.cran.models import CRANModel | ||||
| from swh.scheduler.utils import create_task_dict | from swh.scheduler.utils import create_task_dict | ||||
| from swh.core import utils | from swh.core import utils | ||||
| from swh.lister.core.simple_lister import SimpleLister | from swh.lister.core.simple_lister import SimpleLister | ||||
| class CRANLister(SimpleLister): | class CRANLister(SimpleLister): | ||||
| MODEL = CRANModel | MODEL = CRANModel | ||||
| LISTER_NAME = 'cran' | LISTER_NAME = 'cran' | ||||
| instance = 'cran' | instance = 'cran' | ||||
| descriptions = defaultdict(dict) | |||||
| def task_dict(self, origin_type, origin_url, **kwargs): | def task_dict(self, origin_type, origin_url, **kwargs): | ||||
| """Return task format dict | """Return task format dict | ||||
| This is overridden from the lister_base as more information is | This is overridden from the lister_base as more information is | ||||
| needed for the ingestion task creation. | needed for the ingestion task creation. | ||||
| """ | """ | ||||
| return create_task_dict( | return create_task_dict( | ||||
| 'load-%s' % origin_type, 'recurring', | 'load-%s' % origin_type, 'recurring', | ||||
| kwargs.get('name'), origin_url, kwargs.get('version')) | kwargs.get('name'), origin_url, kwargs.get('version'), | ||||
| project_metadata=self.descriptions[kwargs.get('name')]) | |||||
| def r_script_request(self): | def r_script_request(self): | ||||
| """Runs R script which uses inbuilt API to return a json | """Runs R script which uses inbuilt API to return a json | ||||
| response containing data about all the R packages | response containing data about all the R packages | ||||
| Returns: | Returns: | ||||
| List of dictionaries | List of dictionaries | ||||
| example | example | ||||
| Show All 22 Lines | def r_script_request(self): | ||||
| response = subprocess.run(file_path, stdout=subprocess.PIPE, | response = subprocess.run(file_path, stdout=subprocess.PIPE, | ||||
| shell=False) | shell=False) | ||||
| return json.loads(response.stdout) | return json.loads(response.stdout) | ||||
| def get_model_from_repo(self, repo): | def get_model_from_repo(self, repo): | ||||
| """Transform from repository representation to model | """Transform from repository representation to model | ||||
| """ | """ | ||||
| self.descriptions[repo["Package"]] = repo['Description'] | |||||
| project_url = 'https://cran.r-project.org/src/contrib' \ | project_url = 'https://cran.r-project.org/src/contrib' \ | ||||
| '/%(Package)s_%(Version)s.tar.gz' % repo | '/%(Package)s_%(Version)s.tar.gz' % repo | ||||
| return { | return { | ||||
| 'uid': repo["Package"], | 'uid': repo["Package"], | ||||
| 'name': repo["Package"], | 'name': repo["Package"], | ||||
| 'full_name': repo["Title"], | 'full_name': repo["Title"], | ||||
| 'version': repo["Version"], | 'version': repo["Version"], | ||||
| 'html_url': project_url, | 'html_url': project_url, | ||||
| ▲ Show 20 Lines • Show All 42 Lines • Show Last 20 Lines | |||||