diff --git a/debian/control b/debian/control index 97833e1..a3b75c8 100644 --- a/debian/control +++ b/debian/control @@ -1,23 +1,24 @@ Source: swh-lister-github Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-dateutil, python3-nose, python3-setuptools, python3-swh.core, python3-swh.scheduler (>= 0.0.10~), - python3-swh.storage, + python3-swh.storage (>= 0.0.69~), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLSGH/ Package: python3-swh.lister.github Architecture: all Depends: python3-swh.scheduler (>= 0.0.10~), + python3-swh.storage (>= 0.0.69~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage GitHub lister diff --git a/requirements.txt b/requirements.txt index a5f57c9..020ec04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ requests SQLAlchemy swh.core -swh.storage +swh.storage (>= 0.0.69) swh.scheduler (>= 0.0.10) diff --git a/swh/lister/github/base.py b/swh/lister/github/base.py index 90a85f7..a8f974c 100644 --- a/swh/lister/github/base.py +++ b/swh/lister/github/base.py @@ -1,63 +1,62 @@ # Copyright (C) 2016 The Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core import config from swh.storage import get_storage from swh.scheduler.backend import SchedulerBackend class SWHLister(config.SWHConfig): CONFIG_BASE_FILENAME = None DEFAULT_CONFIG = { 'storage_class': ('str', 'remote_storage'), 'storage_args': ('list[str]', ['http://localhost:5000/']), 'scheduling_db': ('str', 'dbname=swh-scheduler'), } ADDITIONAL_CONFIG = {} def __init__(self): self.config = self.parse_config_file( additional_configs=[self.ADDITIONAL_CONFIG]) self.storage = get_storage(self.config['storage_class'], self.config['storage_args']) self.scheduler = SchedulerBackend( scheduling_db=self.config['scheduling_db'], ) def create_origins(self, origins): """Create the origins listed, and return their ids. Args: origins: a list of origins Returns: a list of origin ids """ - - raise NotImplementedError + return self.storage.origin_add(origins) def create_tasks(self, tasks): """Create the tasks specified, and return their ids. Args: tasks (list of dict): a list of task specifications: type (str): the task type arguments (dict): the arguments for the task runner args (list of str): arguments kwargs (dict str -> str): keyword arguments next_run (datetime.datetime): when to schedule the next run Returns: a list of task ids """ returned_tasks = self.scheduler.create_tasks(tasks) id_index = self.scheduler.task_keys.index('id') return [returned_task[id_index] for returned_task in returned_tasks] def disable_tasks(self, task_ids): """Disable the tasks identified by the given ids""" self.scheduler.disable_tasks(task_ids)