diff --git a/PKG-INFO b/PKG-INFO index e4b3dfd..2b942a7 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,126 +1,126 @@ Metadata-Version: 2.1 Name: swh.lister -Version: 4.3.0 +Version: 4.3.1 Summary: Software Heritage lister Home-page: https://forge.softwareheritage.org/diffusion/DLSGH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-lister Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-lister/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing License-File: LICENSE swh-lister ========== This component from the Software Heritage stack aims to produce listings of software origins and their urls hosted on various public developer platforms or package managers. As these operations are quite similar, it provides a set of Python modules abstracting common software origins listing behaviors. It also provides several lister implementations, contained in the following Python modules: - `swh.lister.bitbucket` - `swh.lister.cgit` - `swh.lister.cran` - `swh.lister.debian` - `swh.lister.gitea` - `swh.lister.github` - `swh.lister.gitlab` - `swh.lister.gnu` - `swh.lister.golang` - `swh.lister.launchpad` - `swh.lister.maven` - `swh.lister.npm` - `swh.lister.packagist` - `swh.lister.phabricator` - `swh.lister.pypi` - `swh.lister.tuleap` - `swh.lister.gogs` - `swh.liser.fedora` Dependencies ------------ All required dependencies can be found in the `requirements*.txt` files located at the root of the repository. Local deployment ---------------- ## lister configuration Each lister implemented so far by Software Heritage (`bitbucket`, `cgit`, `cran`, `debian`, `gitea`, `github`, `gitlab`, `gnu`, `golang`, `launchpad`, `npm`, `packagist`, `phabricator`, `pypi`, `tuleap`, `maven`) must be configured by following the instructions below (please note that you have to replace `` by one of the lister name introduced above). ### Preparation steps 1. `mkdir ~/.config/swh/` 2. create configuration file `~/.config/swh/listers.yml` ### Configuration file sample Minimalistic configuration shared by all listers to add in file `~/.config/swh/listers.yml`: ```lang=yml scheduler: cls: 'remote' args: url: 'http://localhost:5008/' credentials: {} ``` Note: This expects scheduler (5008) service to run locally ## Executing a lister Once configured, a lister can be executed by using the `swh` CLI tool with the following options and commands: ``` $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister [lister_parameters] ``` Examples: ``` $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister bitbucket $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister cran $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitea url=https://codeberg.org/api/v1/ $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitlab url=https://salsa.debian.org/api/v4/ $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister npm $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister pypi ``` Licensing --------- This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. See top-level LICENSE file for the full text of the GNU General Public License along with this program. diff --git a/swh.lister.egg-info/PKG-INFO b/swh.lister.egg-info/PKG-INFO index e4b3dfd..2b942a7 100644 --- a/swh.lister.egg-info/PKG-INFO +++ b/swh.lister.egg-info/PKG-INFO @@ -1,126 +1,126 @@ Metadata-Version: 2.1 Name: swh.lister -Version: 4.3.0 +Version: 4.3.1 Summary: Software Heritage lister Home-page: https://forge.softwareheritage.org/diffusion/DLSGH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-lister Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-lister/ Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing License-File: LICENSE swh-lister ========== This component from the Software Heritage stack aims to produce listings of software origins and their urls hosted on various public developer platforms or package managers. As these operations are quite similar, it provides a set of Python modules abstracting common software origins listing behaviors. It also provides several lister implementations, contained in the following Python modules: - `swh.lister.bitbucket` - `swh.lister.cgit` - `swh.lister.cran` - `swh.lister.debian` - `swh.lister.gitea` - `swh.lister.github` - `swh.lister.gitlab` - `swh.lister.gnu` - `swh.lister.golang` - `swh.lister.launchpad` - `swh.lister.maven` - `swh.lister.npm` - `swh.lister.packagist` - `swh.lister.phabricator` - `swh.lister.pypi` - `swh.lister.tuleap` - `swh.lister.gogs` - `swh.liser.fedora` Dependencies ------------ All required dependencies can be found in the `requirements*.txt` files located at the root of the repository. Local deployment ---------------- ## lister configuration Each lister implemented so far by Software Heritage (`bitbucket`, `cgit`, `cran`, `debian`, `gitea`, `github`, `gitlab`, `gnu`, `golang`, `launchpad`, `npm`, `packagist`, `phabricator`, `pypi`, `tuleap`, `maven`) must be configured by following the instructions below (please note that you have to replace `` by one of the lister name introduced above). ### Preparation steps 1. `mkdir ~/.config/swh/` 2. create configuration file `~/.config/swh/listers.yml` ### Configuration file sample Minimalistic configuration shared by all listers to add in file `~/.config/swh/listers.yml`: ```lang=yml scheduler: cls: 'remote' args: url: 'http://localhost:5008/' credentials: {} ``` Note: This expects scheduler (5008) service to run locally ## Executing a lister Once configured, a lister can be executed by using the `swh` CLI tool with the following options and commands: ``` $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister [lister_parameters] ``` Examples: ``` $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister bitbucket $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister cran $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitea url=https://codeberg.org/api/v1/ $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister gitlab url=https://salsa.debian.org/api/v4/ $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister npm $ swh --log-level DEBUG lister -C ~/.config/swh/listers.yml run --lister pypi ``` Licensing --------- This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. See top-level LICENSE file for the full text of the GNU General Public License along with this program. diff --git a/swh/lister/gitea/tasks.py b/swh/lister/gitea/tasks.py index f6db2ed..4c70833 100644 --- a/swh/lister/gitea/tasks.py +++ b/swh/lister/gitea/tasks.py @@ -1,28 +1,21 @@ -# Copyright (C) 2020 the Software Heritage developers +# Copyright (C) 2020-2022 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Dict, Optional +from typing import Dict from celery import shared_task from .lister import GiteaLister @shared_task(name=__name__ + ".FullGiteaRelister") -def list_gitea_full( - url: str, - instance: Optional[str] = None, - api_token: Optional[str] = None, - page_size: Optional[int] = None, -) -> Dict[str, int]: +def list_gitea_full(**lister_args) -> Dict[str, int]: """Full update of a Gitea instance""" - lister = GiteaLister.from_configfile( - url=url, instance=instance, api_token=api_token, page_size=page_size - ) + lister = GiteaLister.from_configfile(**lister_args) return lister.run().dict() @shared_task(name=__name__ + ".ping") def _ping() -> str: return "OK" diff --git a/swh/lister/gitea/tests/test_tasks.py b/swh/lister/gitea/tests/test_tasks.py index a204cb1..f418e41 100644 --- a/swh/lister/gitea/tests/test_tasks.py +++ b/swh/lister/gitea/tests/test_tasks.py @@ -1,61 +1,59 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from swh.lister.pattern import ListerStats def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): res = swh_scheduler_celery_app.send_task("swh.lister.gitea.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.gitea.tasks.GiteaLister") def test_full_listing(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): lister.from_configfile.return_value = lister lister.run.return_value = ListerStats(pages=10, origins=500) kwargs = dict(url="https://try.gitea.io/api/v1") res = swh_scheduler_celery_app.send_task( "swh.lister.gitea.tasks.FullGiteaRelister", kwargs=kwargs, ) assert res res.wait() assert res.successful() - actual_kwargs = dict(**kwargs, instance=None, api_token=None, page_size=None) - - lister.from_configfile.assert_called_once_with(**actual_kwargs) + lister.from_configfile.assert_called_once_with(**kwargs) lister.run.assert_called_once_with() @patch("swh.lister.gitea.tasks.GiteaLister") def test_full_listing_params( lister, swh_scheduler_celery_app, swh_scheduler_celery_worker ): lister.from_configfile.return_value = lister lister.run.return_value = ListerStats(pages=10, origins=500) kwargs = dict( url="https://0xacab.org/api/v4", instance="0xacab", api_token="test", page_size=50, ) res = swh_scheduler_celery_app.send_task( "swh.lister.gitea.tasks.FullGiteaRelister", kwargs=kwargs, ) assert res res.wait() assert res.successful() lister.from_configfile.assert_called_once_with(**kwargs) lister.run.assert_called_once_with() diff --git a/swh/lister/gogs/tasks.py b/swh/lister/gogs/tasks.py index 81287fa..a3ad438 100644 --- a/swh/lister/gogs/tasks.py +++ b/swh/lister/gogs/tasks.py @@ -1,28 +1,21 @@ # Copyright (C) 2022 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Dict, Optional +from typing import Dict from celery import shared_task from .lister import GogsLister @shared_task(name=__name__ + ".FullGogsRelister") -def list_gogs_full( - url: str, - instance: Optional[str] = None, - api_token: Optional[str] = None, - page_size: Optional[int] = None, -) -> Dict[str, int]: +def list_gogs_full(**lister_args) -> Dict[str, int]: """Full update of a Gogs instance""" - lister = GogsLister.from_configfile( - url=url, instance=instance, api_token=api_token, page_size=page_size - ) + lister = GogsLister.from_configfile(**lister_args) return lister.run().dict() @shared_task(name=__name__ + ".ping") def _ping() -> str: return "OK" diff --git a/swh/lister/gogs/tests/test_tasks.py b/swh/lister/gogs/tests/test_tasks.py index 2f38341..9698fc3 100644 --- a/swh/lister/gogs/tests/test_tasks.py +++ b/swh/lister/gogs/tests/test_tasks.py @@ -1,61 +1,59 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from swh.lister.pattern import ListerStats def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): res = swh_scheduler_celery_app.send_task("swh.lister.gogs.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.gogs.tasks.GogsLister") def test_full_listing(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): lister.from_configfile.return_value = lister lister.run.return_value = ListerStats(pages=10, origins=500) kwargs = dict(url="https://try.gogs.io/api/v1/") res = swh_scheduler_celery_app.send_task( "swh.lister.gogs.tasks.FullGogsRelister", kwargs=kwargs, ) assert res res.wait() assert res.successful() - actual_kwargs = dict(**kwargs, instance=None, api_token=None, page_size=None) - - lister.from_configfile.assert_called_once_with(**actual_kwargs) + lister.from_configfile.assert_called_once_with(**kwargs) lister.run.assert_called_once_with() @patch("swh.lister.gogs.tasks.GogsLister") def test_full_listing_params( lister, swh_scheduler_celery_app, swh_scheduler_celery_worker ): lister.from_configfile.return_value = lister lister.run.return_value = ListerStats(pages=10, origins=500) kwargs = dict( url="https://gogs-host.com/api/v1/", instance="foo", api_token="test", page_size=50, ) res = swh_scheduler_celery_app.send_task( "swh.lister.gogs.tasks.FullGogsRelister", kwargs=kwargs, ) assert res res.wait() assert res.successful() lister.from_configfile.assert_called_once_with(**kwargs) lister.run.assert_called_once_with()