diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ iso8601 beautifulsoup4 pytz +launchpadlib \ No newline at end of file diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ lister.packagist=swh.lister.packagist:register lister.phabricator=swh.lister.phabricator:register lister.pypi=swh.lister.pypi:register + lister.launchpad=swh.lister.launchpad:register ''', classifiers=[ "Programming Language :: Python :: 3", diff --git a/swh/lister/core/tests/conftest.py b/swh/lister/core/tests/conftest.py --- a/swh/lister/core/tests/conftest.py +++ b/swh/lister/core/tests/conftest.py @@ -13,28 +13,31 @@ from swh.lister import get_lister, SUPPORTED_LISTERS from swh.lister.core.models import initialize - logger = logging.getLogger(__name__) @pytest.fixture -def swh_listers(request, postgresql_proc, postgresql, swh_scheduler): +def lister_db_url(postgresql_proc, postgresql): db_url = 'postgresql://{user}@{host}:{port}/{dbname}'.format( - host=postgresql_proc.host, - port=postgresql_proc.port, - user='postgres', - dbname='tests') - + host=postgresql_proc.host, + port=postgresql_proc.port, + user='postgres', + dbname='tests') logger.debug('lister db_url: %s', db_url) + return db_url + + +@pytest.fixture +def swh_listers(request, lister_db_url, swh_scheduler): listers = {} # Prepare schema for all listers for lister_name in SUPPORTED_LISTERS: - lister = get_lister(lister_name, db_url=db_url) + lister = get_lister(lister_name, db_url=lister_db_url) lister.scheduler = swh_scheduler # inject scheduler fixture listers[lister_name] = lister - initialize(create_engine(db_url), drop_tables=True) + initialize(create_engine(lister_db_url), drop_tables=True) # Add the load-archive-files expected by some listers (gnu, cran, ...) swh_scheduler.create_task_type({ diff --git a/swh/lister/launchpad/__init__.py b/swh/lister/launchpad/__init__.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/__init__.py @@ -0,0 +1,13 @@ +# Copyright (C) 2020 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def register(): + from .models import LaunchpadModel + from .lister import LaunchpadLister + + return {'models': [LaunchpadModel], + 'lister': LaunchpadLister, + 'task_modules': ['%s.tasks' % __name__], + } diff --git a/swh/lister/launchpad/lister.py b/swh/lister/launchpad/lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/lister.py @@ -0,0 +1,130 @@ +# Copyright (C) 2017-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import Optional, Dict, List, Union, Tuple + +from swh.lister.core.lister_base import ListerBase +from .models import LaunchpadModel + +from itertools import count +from launchpadlib.launchpad import Launchpad # type: ignore +from lazr.restfulclient.resource import Collection, Entry # type: ignore +from datetime import datetime, timedelta +from sqlalchemy import func + + +class LaunchpadLister(ListerBase): + MODEL = LaunchpadModel + LISTER_NAME = 'launchpad' + instance = 'launchpad' + launchpad: Launchpad + flush_packet_db = 20 + + def __init__(self, override_config=None): + super().__init__(override_config=override_config) + self.launchpad = Launchpad.login_anonymously( + 'softwareheritage', 'production', version='devel') + + def get_model_from_repo(self, repo: Entry + ) -> Dict[str, Union[str, datetime]]: + return { + 'uid': repo.unique_name, + 'name': repo.name, + 'full_name': repo.name, + 'origin_url': repo.git_https_url, + 'html_url': repo.web_link, + 'origin_type': 'git', + 'date_last_modified': repo.date_last_modified + } + + def lib_response_simplified(self, response: Collection + ) -> List[Dict[str, Union[str, datetime]]]: + return [self.get_model_from_repo(repo) + for repo in response[:len(response.entries)]] + + def get_git_repos(self, threshold: Optional[datetime]) -> Collection: + get_repos = self.launchpad.git_repositories.getRepositories + + return get_repos(order_by='most neglected first', + modified_since_date=threshold) + + def db_last_threshold(self) -> Optional[datetime]: + t = self.db_session.query( + func.max(self.MODEL.date_last_modified)).first() + if t: + return t[0] + else: + return None + + def ingest_data_lp(self, identifier: Optional[datetime], checks=False + ) -> Tuple[Collection, dict]: + """The core data fetch sequence. Request launchpadlib endpoint. Simplify and + filter response list of repositories. Inject repo information into + local db. Queue loader tasks for linked repositories. + + Args: + identifier: Resource identifier. + checks (bool): Additional checks required + """ + response = self.get_git_repos(identifier) + models_list = self.lib_response_simplified(response) + models_list = self.filter_before_inject(models_list) + if checks: + models_list = self.do_additional_checks(models_list) + if not models_list: + return response, {} + # inject into local db + injected = self.inject_repo_data_into_db(models_list) + # queue workers + self.schedule_missing_tasks(models_list, injected) + return response, injected + + def run(self, max_bound: Optional[datetime] = None): + """Main entry function. Sequentially fetches repository data + from the service according to the basic outline in the class + docstring, continually fetching sublists until either there + is no next index reference given or the given next index is greater + than the desired max_bound. + + Args: + max_bound : optional date to start at + Returns: + nothing + """ + status = 'uneventful' + + def ingest_git_repos(): + threshold = max_bound + for i in count(1): + response, injected_repos = self.ingest_data_lp(threshold) + if not response and not injected_repos: + return + + # batch is empty + if len(response.entries) == 0: + return + + first: datetime = response[0].date_last_modified + last: datetime = response[len( + response.entries)-1].date_last_modified + + next_date = last - timedelta(seconds=15) + + if next_date <= first: + delta = last - first + next_date = last - delta/2 + + threshold = next_date + yield i + + for i in ingest_git_repos(): + if (i % self.flush_packet_db) == 0: + self.db_session.commit() + self.db_session = self.mk_session() + status = 'eventful' + + self.db_session.commit() + self.db_session = self.mk_session() + return {'status': status} diff --git a/swh/lister/launchpad/models.py b/swh/lister/launchpad/models.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/models.py @@ -0,0 +1,15 @@ +# Copyright (C) 2017-2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from sqlalchemy import Column, String, Date + +from swh.lister.core.models import ModelBase + + +class LaunchpadModel(ModelBase): + """a Launchpad repository""" + __tablename__ = 'launchpad_repo' + + uid = Column(String, primary_key=True) + date_last_modified = Column(Date, index=True) diff --git a/swh/lister/launchpad/tasks.py b/swh/lister/launchpad/tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tasks.py @@ -0,0 +1,35 @@ +# Copyright (C) 2017-2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +# import random + +from celery import shared_task + +from .lister import LaunchpadLister + + +@shared_task(name=__name__ + '.IncrementalLaunchpadLister') +def launchpad_lister_incremental(threshold, **lister_args): + """Incremental update + """ + lister = LaunchpadLister(**lister_args) + return lister.run(max_bound=threshold) + + +@shared_task(name=__name__ + '.FullLaunchpadLister', bind=True) +def list_launchpad_full(self, **lister_args): + """Full update of Launchpad + """ + self.log.debug('%s OK, spawned full task' % (self.name)) + return launchpad_lister_incremental(threshold=None, **lister_args) + + +@shared_task(name=__name__ + '.NewLaunchpadLister', bind=True) +def list_launchpad_new(self, **lister_args): + """Update new entries of Launchpad + """ + lister = LaunchpadLister(**lister_args) + threshold = lister.db_last_threshold() + self.log.debug('%s OK, spawned new task' % (self.name)) + return launchpad_lister_incremental(threshold=threshold, **lister_args) diff --git a/swh/lister/launchpad/tests/__init__.py b/swh/lister/launchpad/tests/__init__.py new file mode 100644 diff --git a/swh/lister/launchpad/tests/conftest.py b/swh/lister/launchpad/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tests/conftest.py @@ -0,0 +1,43 @@ +from swh.lister.core.tests.conftest import * # noqa +from datetime import datetime +import json +import os +from unittest.mock import MagicMock, patch +import pytest +from swh.lister import get_lister +from swh.lister.core.models import initialize +from sqlalchemy.engine import create_engine + + +@pytest.fixture +@patch('launchpadlib.launchpad.Launchpad.login_anonymously') +def lister_launchpad(swh_listers, datadir, lister_db_url, swh_scheduler): + class Collection: + entries = [] + + def __init__(self, file): + self.entries = [Repo(r) for r in file] + + def __getitem__(self, key): + return self.entries[key] + + class Repo: + def __init__(self, d: dict): + for key in d.keys(): + if key == "date_last_modified": + setattr(self, key, datetime.fromisoformat(d[key])) + else: + setattr(self, key, d[key]) + + def mock_lp_response(page) -> Collection: + response_filepath = os.path.join(datadir, f'response{page}.json') + with open(response_filepath, 'r', encoding='utf-8') as f: + return Collection(json.load(f)) + + lister = get_lister('launchpad', db_url=lister_db_url) + lister.scheduler = swh_scheduler # inject scheduler fixture + initialize(create_engine(lister_db_url), drop_tables=True) + lister.launchpad.git_repositories.getRepositories = MagicMock( + side_effect=[mock_lp_response(i) for i in range(3)]) + + return lister diff --git a/swh/lister/launchpad/tests/data/response0.json b/swh/lister/launchpad/tests/data/response0.json new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tests/data/response0.json @@ -0,0 +1,527 @@ +[ + { + "unique_name":"~kernel-ppa/ubuntu/+source/linux/+git/unstable", + "name":"unstable", + "git_https_url":"https://git.launchpad.net/~kernel-ppa/ubuntu/+source/linux/+git/unstable", + "web_link":"https://code.launchpad.net/~kernel-ppa/ubuntu/+source/linux/+git/unstable", + "date_last_modified":"2015-05-01T12:49:18.413089+00:00" + }, + { + "unique_name":"~rvb/+git/test", + "name":"test", + "git_https_url":"https://git.launchpad.net/~rvb/+git/test", + "web_link":"https://code.launchpad.net/~rvb/+git/test", + "date_last_modified":"2015-05-01T21:22:52.263951+00:00" + }, + { + "unique_name":"~roadmr/dpkg-offline/+git/dpkg-offline", + "name":"dpkg-offline", + "git_https_url":"https://git.launchpad.net/dpkg-offline", + "web_link":"https://code.launchpad.net/~roadmr/dpkg-offline/+git/dpkg-offline", + "date_last_modified":"2015-05-01T21:26:03.608895+00:00" + }, + { + "unique_name":"~ubuntuone-hackers/conn-check/+git/gittest", + "name":"gittest", + "git_https_url":"https://git.launchpad.net/~ubuntuone-hackers/conn-check/+git/gittest", + "web_link":"https://code.launchpad.net/~ubuntuone-hackers/conn-check/+git/gittest", + "date_last_modified":"2015-05-01T22:26:18.829141+00:00" + }, + { + "unique_name":"~danrabbit/egtk/+git/egtk", + "name":"egtk", + "git_https_url":"https://git.launchpad.net/egtk", + "web_link":"https://code.launchpad.net/~danrabbit/egtk/+git/egtk", + "date_last_modified":"2015-05-01T22:51:32.664675+00:00" + }, + { + "unique_name":"~renatofilho/address-book-app/+git/address-book-app", + "name":"address-book-app", + "git_https_url":"https://git.launchpad.net/~renatofilho/address-book-app", + "web_link":"https://code.launchpad.net/~renatofilho/address-book-app/+git/address-book-app", + "date_last_modified":"2015-05-02T04:00:50.035111+00:00" + }, + { + "unique_name":"~lfaraone/+git/test", + "name":"test", + "git_https_url":"https://git.launchpad.net/~lfaraone/+git/test", + "web_link":"https://code.launchpad.net/~lfaraone/+git/test", + "date_last_modified":"2015-05-03T00:04:10.630026+00:00" + }, + { + "unique_name":"~lfaraone/+git/test2", + "name":"test2", + "git_https_url":"https://git.launchpad.net/~lfaraone/+git/test2", + "web_link":"https://code.launchpad.net/~lfaraone/+git/test2", + "date_last_modified":"2015-05-03T00:08:02.371748+00:00" + }, + { + "unique_name":"~ev/dotfiles/+git/dotfiles", + "name":"dotfiles", + "git_https_url":"https://git.launchpad.net/~ev/dotfiles", + "web_link":"https://code.launchpad.net/~ev/dotfiles/+git/dotfiles", + "date_last_modified":"2015-05-03T21:15:03.400658+00:00" + }, + { + "unique_name":"~chipaca/+git/testing", + "name":"testing", + "git_https_url":"https://git.launchpad.net/~chipaca/+git/testing", + "web_link":"https://code.launchpad.net/~chipaca/+git/testing", + "date_last_modified":"2015-05-04T01:18:21.276726+00:00" + }, + { + "unique_name":"~bafu/+git/lp-git", + "name":"lp-git", + "git_https_url":"https://git.launchpad.net/~bafu/+git/lp-git", + "web_link":"https://code.launchpad.net/~bafu/+git/lp-git", + "date_last_modified":"2015-05-04T03:16:53.783224+00:00" + }, + { + "unique_name":"~ivoks/test/+git/test", + "name":"test", + "git_https_url":"https://git.launchpad.net/~ivoks/test", + "web_link":"https://code.launchpad.net/~ivoks/test/+git/test", + "date_last_modified":"2015-05-04T11:32:33.373969+00:00" + }, + { + "unique_name":"~jjo/juju-viz/+git/juju-viz", + "name":"juju-viz", + "git_https_url":"https://git.launchpad.net/juju-viz", + "web_link":"https://code.launchpad.net/~jjo/juju-viz/+git/juju-viz", + "date_last_modified":"2015-05-04T12:18:13.011918+00:00" + }, + { + "unique_name":"~shawn111/guacamole/+git/guacamole", + "name":"guacamole", + "git_https_url":"https://git.launchpad.net/~shawn111/guacamole/+git/guacamole", + "web_link":"https://code.launchpad.net/~shawn111/guacamole/+git/guacamole", + "date_last_modified":"2015-05-04T12:18:42.970478+00:00" + }, + { + "unique_name":"~shawn111/guacamole/+git/guacamole-1", + "name":"guacamole-1", + "git_https_url":"https://git.launchpad.net/~shawn111/guacamole", + "web_link":"https://code.launchpad.net/~shawn111/guacamole/+git/guacamole-1", + "date_last_modified":"2015-05-04T12:31:33.081372+00:00" + }, + { + "unique_name":"~vthompson/music-app/+git/music-app", + "name":"music-app", + "git_https_url":"https://git.launchpad.net/~vthompson/music-app", + "web_link":"https://code.launchpad.net/~vthompson/music-app/+git/music-app", + "date_last_modified":"2015-05-04T12:48:58.598427+00:00" + }, + { + "unique_name":"~rcj/+git/ubuntu-repository-cache", + "name":"ubuntu-repository-cache", + "git_https_url":"https://git.launchpad.net/~rcj/+git/ubuntu-repository-cache", + "web_link":"https://code.launchpad.net/~rcj/+git/ubuntu-repository-cache", + "date_last_modified":"2015-05-04T15:52:54.585978+00:00" + }, + { + "unique_name":"~barry/+git/siunpacker", + "name":"siunpacker", + "git_https_url":"https://git.launchpad.net/~barry/+git/siunpacker", + "web_link":"https://code.launchpad.net/~barry/+git/siunpacker", + "date_last_modified":"2015-05-04T17:44:48.422090+00:00" + }, + { + "unique_name":"~xnox/cross-toolchain-base/+git/cross-toolchain-base", + "name":"cross-toolchain-base", + "git_https_url":"https://git.launchpad.net/~xnox/cross-toolchain-base", + "web_link":"https://code.launchpad.net/~xnox/cross-toolchain-base/+git/cross-toolchain-base", + "date_last_modified":"2015-05-04T18:21:46.441208+00:00" + }, + { + "unique_name":"~baco/+git/winsysinfo", + "name":"winsysinfo", + "git_https_url":"https://git.launchpad.net/~baco/+git/winsysinfo", + "web_link":"https://code.launchpad.net/~baco/+git/winsysinfo", + "date_last_modified":"2015-05-05T00:05:16.527676+00:00" + }, + { + "unique_name":"~xubuntu-dev/+git/libmpris2client", + "name":"libmpris2client", + "git_https_url":"https://git.launchpad.net/~xubuntu-dev/+git/libmpris2client", + "web_link":"https://code.launchpad.net/~xubuntu-dev/+git/libmpris2client", + "date_last_modified":"2015-05-05T00:35:31.663818+00:00" + }, + { + "unique_name":"~xubuntu-dev/+git/xfce4-soundmenu-plugin", + "name":"xfce4-soundmenu-plugin", + "git_https_url":"https://git.launchpad.net/~xubuntu-dev/+git/xfce4-soundmenu-plugin", + "web_link":"https://code.launchpad.net/~xubuntu-dev/+git/xfce4-soundmenu-plugin", + "date_last_modified":"2015-05-05T00:36:01.237141+00:00" + }, + { + "unique_name":"~cosmos-door/+git/fcitx-kkc", + "name":"fcitx-kkc", + "git_https_url":"https://git.launchpad.net/~cosmos-door/+git/fcitx-kkc", + "web_link":"https://code.launchpad.net/~cosmos-door/+git/fcitx-kkc", + "date_last_modified":"2015-05-05T08:07:35.588205+00:00" + }, + { + "unique_name":"~mvo/+git/snappy", + "name":"snappy", + "git_https_url":"https://git.launchpad.net/~mvo/+git/snappy", + "web_link":"https://code.launchpad.net/~mvo/+git/snappy", + "date_last_modified":"2015-05-06T06:49:00.411903+00:00" + }, + { + "unique_name":"~snappy-dev/snappy/+git/snappy", + "name":"snappy", + "git_https_url":"https://git.launchpad.net/~snappy-dev/snappy", + "web_link":"https://code.launchpad.net/~snappy-dev/snappy/+git/snappy", + "date_last_modified":"2015-05-06T07:19:47.742077+00:00" + }, + { + "unique_name":"~mvo/snappy/+git/snappy", + "name":"snappy", + "git_https_url":"https://git.launchpad.net/~mvo/snappy", + "web_link":"https://code.launchpad.net/~mvo/snappy/+git/snappy", + "date_last_modified":"2015-05-06T07:21:15.925115+00:00" + }, + { + "unique_name":"~shawn111/checkbox/+git/checkbox", + "name":"checkbox", + "git_https_url":"https://git.launchpad.net/~shawn111/checkbox/+git/checkbox", + "web_link":"https://code.launchpad.net/~shawn111/checkbox/+git/checkbox", + "date_last_modified":"2015-05-07T09:36:21.580503+00:00" + }, + { + "unique_name":"~excaliburzero/jekyll-helper/+git/jekyll-helper", + "name":"jekyll-helper", + "git_https_url":"https://git.launchpad.net/jekyll-helper", + "web_link":"https://code.launchpad.net/~excaliburzero/jekyll-helper/+git/jekyll-helper", + "date_last_modified":"2015-05-08T16:22:11.131626+00:00" + }, + { + "unique_name":"~mhall119/+git/phoenicia", + "name":"phoenicia", + "git_https_url":"https://git.launchpad.net/~mhall119/+git/phoenicia", + "web_link":"https://code.launchpad.net/~mhall119/+git/phoenicia", + "date_last_modified":"2015-05-09T03:28:59.072956+00:00" + }, + { + "unique_name":"~cmiller/+git/minecraft-packaging", + "name":"minecraft-packaging", + "git_https_url":"https://git.launchpad.net/~cmiller/+git/minecraft-packaging", + "web_link":"https://code.launchpad.net/~cmiller/+git/minecraft-packaging", + "date_last_modified":"2015-05-09T19:14:44.407377+00:00" + }, + { + "unique_name":"~cmiller/minecraft/+git/packaging", + "name":"packaging", + "git_https_url":"https://git.launchpad.net/~cmiller/minecraft/+git/packaging", + "web_link":"https://code.launchpad.net/~cmiller/minecraft/+git/packaging", + "date_last_modified":"2015-05-09T19:33:58.768691+00:00" + }, + { + "unique_name":"~cmiller/minecraft/+git/minecraft", + "name":"minecraft", + "git_https_url":"https://git.launchpad.net/~cmiller/minecraft", + "web_link":"https://code.launchpad.net/~cmiller/minecraft/+git/minecraft", + "date_last_modified":"2015-05-09T20:03:44.294262+00:00" + }, + { + "unique_name":"~canonical-launchpad-branches/launchpad/+git/launchpad-old", + "name":"launchpad-old", + "git_https_url":"https://git.launchpad.net/~canonical-launchpad-branches/launchpad/+git/launchpad-old", + "web_link":"https://code.launchpad.net/~canonical-launchpad-branches/launchpad/+git/launchpad-old", + "date_last_modified":"2015-05-12T00:58:25.229758+00:00" + }, + { + "unique_name":"~shawn111/checkbox/+git/checkbox-f", + "name":"checkbox-f", + "git_https_url":"https://git.launchpad.net/~shawn111/checkbox/+git/checkbox-f", + "web_link":"https://code.launchpad.net/~shawn111/checkbox/+git/checkbox-f", + "date_last_modified":"2015-05-12T13:39:20.906398+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux/+git/saucy", + "name":"saucy", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/saucy", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/saucy", + "date_last_modified":"2015-05-12T14:30:08.778866+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux-signed/+git/saucy", + "name":"saucy", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-signed/+git/saucy", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-signed/+git/saucy", + "date_last_modified":"2015-05-12T14:37:03.426462+00:00" + }, + { + "unique_name":"~profzoom/+git/wmfire", + "name":"wmfire", + "git_https_url":"https://git.launchpad.net/~profzoom/+git/wmfire", + "web_link":"https://code.launchpad.net/~profzoom/+git/wmfire", + "date_last_modified":"2015-05-13T04:23:37.980976+00:00" + }, + { + "unique_name":"~corey.bryant/horizon/+git/horizon", + "name":"horizon", + "git_https_url":"https://git.launchpad.net/~corey.bryant/horizon", + "web_link":"https://code.launchpad.net/~corey.bryant/horizon/+git/horizon", + "date_last_modified":"2015-05-14T14:14:13.537589+00:00" + }, + { + "unique_name":"~corey.bryant/cinder/+git/cinder", + "name":"cinder", + "git_https_url":"https://git.launchpad.net/~corey.bryant/cinder", + "web_link":"https://code.launchpad.net/~corey.bryant/cinder/+git/cinder", + "date_last_modified":"2015-05-14T14:56:48.777248+00:00" + }, + { + "unique_name":"~corey.bryant/glance/+git/glance", + "name":"glance", + "git_https_url":"https://git.launchpad.net/~corey.bryant/glance", + "web_link":"https://code.launchpad.net/~corey.bryant/glance/+git/glance", + "date_last_modified":"2015-05-14T14:57:37.088951+00:00" + }, + { + "unique_name":"~corey.bryant/keystone/+git/keystone", + "name":"keystone", + "git_https_url":"https://git.launchpad.net/~corey.bryant/keystone", + "web_link":"https://code.launchpad.net/~corey.bryant/keystone/+git/keystone", + "date_last_modified":"2015-05-14T14:58:02.676405+00:00" + }, + { + "unique_name":"~corey.bryant/nova/+git/nova", + "name":"nova", + "git_https_url":"https://git.launchpad.net/~corey.bryant/nova", + "web_link":"https://code.launchpad.net/~corey.bryant/nova/+git/nova", + "date_last_modified":"2015-05-14T14:58:39.716732+00:00" + }, + { + "unique_name":"~corey.bryant/neutron/+git/neutron", + "name":"neutron", + "git_https_url":"https://git.launchpad.net/~corey.bryant/neutron", + "web_link":"https://code.launchpad.net/~corey.bryant/neutron/+git/neutron", + "date_last_modified":"2015-05-14T15:03:44.952189+00:00" + }, + { + "unique_name":"~corey.bryant/+git/requirements", + "name":"requirements", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/requirements", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/requirements", + "date_last_modified":"2015-05-14T15:35:01.452035+00:00" + }, + { + "unique_name":"~corey.bryant/+git/cinder", + "name":"cinder", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/cinder", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/cinder", + "date_last_modified":"2015-05-14T15:35:40.510516+00:00" + }, + { + "unique_name":"~corey.bryant/+git/glance", + "name":"glance", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/glance", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/glance", + "date_last_modified":"2015-05-14T15:36:29.211892+00:00" + }, + { + "unique_name":"~corey.bryant/+git/horizon", + "name":"horizon", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/horizon", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/horizon", + "date_last_modified":"2015-05-14T15:36:54.621129+00:00" + }, + { + "unique_name":"~corey.bryant/+git/keystone", + "name":"keystone", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/keystone", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/keystone", + "date_last_modified":"2015-05-14T15:39:42.631008+00:00" + }, + { + "unique_name":"~corey.bryant/+git/neutron", + "name":"neutron", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/neutron", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/neutron", + "date_last_modified":"2015-05-14T15:40:20.019979+00:00" + }, + { + "unique_name":"~corey.bryant/+git/neutron-lbaas", + "name":"neutron-lbaas", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/neutron-lbaas", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/neutron-lbaas", + "date_last_modified":"2015-05-14T15:41:40.506852+00:00" + }, + { + "unique_name":"~corey.bryant/+git/neutron-fwaas", + "name":"neutron-fwaas", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/neutron-fwaas", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/neutron-fwaas", + "date_last_modified":"2015-05-14T15:41:44.897161+00:00" + }, + { + "unique_name":"~corey.bryant/+git/neutron-vpnaas", + "name":"neutron-vpnaas", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/neutron-vpnaas", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/neutron-vpnaas", + "date_last_modified":"2015-05-14T15:41:48.379689+00:00" + }, + { + "unique_name":"~corey.bryant/+git/nova", + "name":"nova", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/nova", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/nova", + "date_last_modified":"2015-05-14T15:41:53.411305+00:00" + }, + { + "unique_name":"~zaspire/wikimobile/+git/wikimobile", + "name":"wikimobile", + "git_https_url":"https://git.launchpad.net/~zaspire/wikimobile", + "web_link":"https://code.launchpad.net/~zaspire/wikimobile/+git/wikimobile", + "date_last_modified":"2015-05-15T08:46:05.409132+00:00" + }, + { + "unique_name":"~pmaynard/linux-application-firewall/+git/linux-application-firewall", + "name":"linux-application-firewall", + "git_https_url":"https://git.launchpad.net/linux-application-firewall", + "web_link":"https://code.launchpad.net/~pmaynard/linux-application-firewall/+git/linux-application-firewall", + "date_last_modified":"2015-05-15T12:21:46.113116+00:00" + }, + { + "unique_name":"~gerald-yang-tw/+git/realtek-rtk-btusb-dkms", + "name":"realtek-rtk-btusb-dkms", + "git_https_url":"https://git.launchpad.net/~gerald-yang-tw/+git/realtek-rtk-btusb-dkms", + "web_link":"https://code.launchpad.net/~gerald-yang-tw/+git/realtek-rtk-btusb-dkms", + "date_last_modified":"2015-05-18T08:26:46.161907+00:00" + }, + { + "unique_name":"~gerald-yang-tw/test/+git/test", + "name":"test", + "git_https_url":"https://git.launchpad.net/~gerald-yang-tw/test", + "web_link":"https://code.launchpad.net/~gerald-yang-tw/test/+git/test", + "date_last_modified":"2015-05-18T08:38:47.040365+00:00" + }, + { + "unique_name":"~gemfony/geneva/+git/geneva", + "name":"geneva", + "git_https_url":"https://git.launchpad.net/~gemfony/geneva", + "web_link":"https://code.launchpad.net/~gemfony/geneva/+git/geneva", + "date_last_modified":"2015-05-18T14:42:03.749370+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux/+git/raring", + "name":"raring", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/raring", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/raring", + "date_last_modified":"2015-05-18T15:49:44.895582+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux-meta/+git/saucy", + "name":"saucy", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-meta/+git/saucy", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-meta/+git/saucy", + "date_last_modified":"2015-05-18T15:59:07.623485+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux-meta/+git/raring", + "name":"raring", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-meta/+git/raring", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-meta/+git/raring", + "date_last_modified":"2015-05-18T16:05:23.706734+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux-signed/+git/raring", + "name":"raring", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-signed/+git/raring", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-signed/+git/raring", + "date_last_modified":"2015-05-18T16:05:25.200936+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux/+git/quantal", + "name":"quantal", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/quantal", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/quantal", + "date_last_modified":"2015-05-18T16:58:59.809000+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux-meta/+git/quantal", + "name":"quantal", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-meta/+git/quantal", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-meta/+git/quantal", + "date_last_modified":"2015-05-18T17:04:31.267631+00:00" + }, + { + "unique_name":"~ubuntu-kernel/ubuntu/+source/linux-signed/+git/quantal", + "name":"quantal", + "git_https_url":"https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-signed/+git/quantal", + "web_link":"https://code.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux-signed/+git/quantal", + "date_last_modified":"2015-05-18T17:04:32.689598+00:00" + }, + { + "unique_name":"~ki7mt/jtsdk/+git/jtsdk", + "name":"jtsdk", + "git_https_url":"https://git.launchpad.net/jtsdk", + "web_link":"https://code.launchpad.net/~ki7mt/jtsdk/+git/jtsdk", + "date_last_modified":"2015-05-18T18:45:38.235509+00:00" + }, + { + "unique_name":"~ki7mt/flsdk/+git/flsdk", + "name":"flsdk", + "git_https_url":"https://git.launchpad.net/flsdk", + "web_link":"https://code.launchpad.net/~ki7mt/flsdk/+git/flsdk", + "date_last_modified":"2015-05-18T20:37:26.560139+00:00" + }, + { + "unique_name":"~martinatkins/alamatic/+git/alamatic", + "name":"alamatic", + "git_https_url":"https://git.launchpad.net/alamatic", + "web_link":"https://code.launchpad.net/~martinatkins/alamatic/+git/alamatic", + "date_last_modified":"2015-05-19T05:07:03.315855+00:00" + }, + { + "unique_name":"~registry/zope-cmfcore/+git/zope-cmfcore", + "name":"zope-cmfcore", + "git_https_url":"https://git.launchpad.net/zope-cmfcore", + "web_link":"https://code.launchpad.net/~registry/zope-cmfcore/+git/zope-cmfcore", + "date_last_modified":"2015-05-22T21:00:16.480059+00:00" + }, + { + "unique_name":"~registry/zope-cmfdefault/+git/zope-cmfdefault", + "name":"zope-cmfdefault", + "git_https_url":"https://git.launchpad.net/zope-cmfdefault", + "web_link":"https://code.launchpad.net/~registry/zope-cmfdefault/+git/zope-cmfdefault", + "date_last_modified":"2015-05-22T22:42:01.691765+00:00" + }, + { + "unique_name":"~registry/zope-cmftopic/+git/zope-cmftopic", + "name":"zope-cmftopic", + "git_https_url":"https://git.launchpad.net/zope-cmftopic", + "web_link":"https://code.launchpad.net/~registry/zope-cmftopic/+git/zope-cmftopic", + "date_last_modified":"2015-05-22T22:59:46.944212+00:00" + }, + { + "unique_name":"~registry/zope-cmfcalendar/+git/zope-cmfcalendar", + "name":"zope-cmfcalendar", + "git_https_url":"https://git.launchpad.net/zope-cmfcalendar", + "web_link":"https://code.launchpad.net/~registry/zope-cmfcalendar/+git/zope-cmfcalendar", + "date_last_modified":"2015-05-22T23:17:21.644526+00:00" + }, + { + "unique_name":"~registry/zope-cmfuid/+git/zope-cmfuid", + "name":"zope-cmfuid", + "git_https_url":"https://git.launchpad.net/zope-cmfuid", + "web_link":"https://code.launchpad.net/~registry/zope-cmfuid/+git/zope-cmfuid", + "date_last_modified":"2015-05-22T23:33:56.253122+00:00" + }, + { + "unique_name":"~registry/zope-cmf-buildout/+git/zope-cmf-buildout", + "name":"zope-cmf-buildout", + "git_https_url":"https://git.launchpad.net/zope-cmf-buildout", + "web_link":"https://code.launchpad.net/~registry/zope-cmf-buildout/+git/zope-cmf-buildout", + "date_last_modified":"2015-05-26T01:01:50.249139+00:00" + }, + { + "unique_name":"~pspmteam/libertine/+git/libertine", + "name":"libertine", + "git_https_url":"https://git.launchpad.net/libertine", + "web_link":"https://code.launchpad.net/~pspmteam/libertine/+git/libertine", + "date_last_modified":"2015-05-27T13:45:40.133292+00:00" + } + ] \ No newline at end of file diff --git a/swh/lister/launchpad/tests/data/response1.json b/swh/lister/launchpad/tests/data/response1.json new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tests/data/response1.json @@ -0,0 +1,107 @@ +[ + { + "unique_name":"~bafu/+git/hwestack-helper", + "name":"hwestack-helper", + "git_https_url":"https://git.launchpad.net/~bafu/+git/hwestack-helper", + "web_link":"https://code.launchpad.net/~bafu/+git/hwestack-helper", + "date_last_modified":"2015-05-28T10:11:39.102130+00:00" + }, + { + "unique_name":"~cryptsetup-tpm-team/cryptsetup-tpm/+git/cryptsetup-tpm", + "name":"cryptsetup-tpm", + "git_https_url":"https://git.launchpad.net/cryptsetup-tpm", + "web_link":"https://code.launchpad.net/~cryptsetup-tpm-team/cryptsetup-tpm/+git/cryptsetup-tpm", + "date_last_modified":"2015-05-29T08:04:46.949494+00:00" + }, + { + "unique_name":"~libertine-team/libertine/+git/libertine", + "name":"libertine", + "git_https_url":"https://git.launchpad.net/~libertine-team/libertine/+git/libertine", + "web_link":"https://code.launchpad.net/~libertine-team/libertine/+git/libertine", + "date_last_modified":"2015-06-02T13:51:52.869335+00:00" + }, + { + "unique_name":"~ursinha/rosetta2wiki/+git/rosetta2wiki", + "name":"rosetta2wiki", + "git_https_url":"https://git.launchpad.net/rosetta2wiki", + "web_link":"https://code.launchpad.net/~ursinha/rosetta2wiki/+git/rosetta2wiki", + "date_last_modified":"2015-06-03T14:26:56.098899+00:00" + }, + { + "unique_name":"~rvb/+git/test-git-repo2", + "name":"test-git-repo2", + "git_https_url":"https://git.launchpad.net/~rvb/+git/test-git-repo2", + "web_link":"https://code.launchpad.net/~rvb/+git/test-git-repo2", + "date_last_modified":"2015-06-04T09:45:10.316365+00:00" + }, + { + "unique_name":"~maas-maintainers/+git/test-git-repo", + "name":"test-git-repo", + "git_https_url":"https://git.launchpad.net/~maas-maintainers/+git/test-git-repo", + "web_link":"https://code.launchpad.net/~maas-maintainers/+git/test-git-repo", + "date_last_modified":"2015-06-04T09:52:15.858834+00:00" + }, + { + "unique_name":"~zyga/+git/hwcert-tools", + "name":"hwcert-tools", + "git_https_url":"https://git.launchpad.net/~zyga/+git/hwcert-tools", + "web_link":"https://code.launchpad.net/~zyga/+git/hwcert-tools", + "date_last_modified":"2015-06-05T09:19:34.219427+00:00" + }, + { + "unique_name":"~kaxing/+git/2fa", + "name":"2fa", + "git_https_url":"https://git.launchpad.net/~kaxing/+git/2fa", + "web_link":"https://code.launchpad.net/~kaxing/+git/2fa", + "date_last_modified":"2015-06-05T10:42:56.025561+00:00" + }, + { + "unique_name":"~profzoom/+git/frobby", + "name":"frobby", + "git_https_url":"https://git.launchpad.net/~profzoom/+git/frobby", + "web_link":"https://code.launchpad.net/~profzoom/+git/frobby", + "date_last_modified":"2015-06-06T03:55:15.411463+00:00" + }, + { + "unique_name":"~dpniel/+git/test-git", + "name":"test-git", + "git_https_url":"https://git.launchpad.net/~dpniel/+git/test-git", + "web_link":"https://code.launchpad.net/~dpniel/+git/test-git", + "date_last_modified":"2015-06-08T06:29:11.354988+00:00" + }, + { + "unique_name":"~corey.bryant/+git/python-cinderclient", + "name":"python-cinderclient", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/python-cinderclient", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/python-cinderclient", + "date_last_modified":"2015-06-08T17:58:17.282686+00:00" + }, + { + "unique_name":"~corey.bryant/+git/python-glanceclient", + "name":"python-glanceclient", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/python-glanceclient", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/python-glanceclient", + "date_last_modified":"2015-06-09T15:46:32.870529+00:00" + }, + { + "unique_name":"~zyga/hwcert-tools/+git/hwcert-tools", + "name":"hwcert-tools", + "git_https_url":"https://git.launchpad.net/~zyga/hwcert-tools/+git/hwcert-tools", + "web_link":"https://code.launchpad.net/~zyga/hwcert-tools/+git/hwcert-tools", + "date_last_modified":"2015-06-09T17:15:25.299211+00:00" + }, + { + "unique_name":"~ubuntu-system-image/ubuntu-system-image/+git/documentation", + "name":"documentation", + "git_https_url":"https://git.launchpad.net/~ubuntu-system-image/ubuntu-system-image/+git/documentation", + "web_link":"https://code.launchpad.net/~ubuntu-system-image/ubuntu-system-image/+git/documentation", + "date_last_modified":"2015-06-09T17:33:52.254130+00:00" + }, + { + "unique_name":"~corey.bryant/+git/python-novaclient", + "name":"python-novaclient", + "git_https_url":"https://git.launchpad.net/~corey.bryant/+git/python-novaclient", + "web_link":"https://code.launchpad.net/~corey.bryant/+git/python-novaclient", + "date_last_modified":"2015-06-09T19:50:55.238308+00:00" + } + ] \ No newline at end of file diff --git a/swh/lister/launchpad/tests/data/response2.json b/swh/lister/launchpad/tests/data/response2.json new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tests/data/response2.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tests/test_lister.py @@ -0,0 +1,26 @@ +def test_launchpad_lister(lister_launchpad, datadir): + + lister_launchpad.run() + + assert len(lister_launchpad.launchpad + .git_repositories.getRepositories.mock_calls) == 3 + + r = lister_launchpad.scheduler.search_tasks(task_type='load-git') + assert len(r) == 90 + + for row in r: + assert row['type'] == 'load-git' + # arguments check + args = row['arguments']['args'] + assert len(args) == 0 + + # kwargs + kwargs = row['arguments']['kwargs'] + assert set(kwargs.keys()) == {'url'} + + url = kwargs['url'] + assert url.startswith('https://git.launchpad.net') + + assert row['policy'] == 'recurring' + assert row['priority'] is None + assert row['retries_left'] == 0 diff --git a/swh/lister/launchpad/tests/test_tasks.py b/swh/lister/launchpad/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/launchpad/tests/test_tasks.py @@ -0,0 +1,37 @@ +from unittest.mock import patch + + +@patch('swh.lister.launchpad.tasks.LaunchpadLister') +def test_new(lister, swh_app, celery_session_worker): + # setup the mocked LaunchpadLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.launchpad.tasks.NewLaunchpadLister' + ) + assert res + res.wait() + assert res.successful() + + assert lister.call_count == 2 + lister.db_last_threshold.assert_called_once() + lister.run.assert_called_once() + + +@patch('swh.lister.launchpad.tasks.LaunchpadLister') +def test_full(lister, swh_app, celery_session_worker): + # setup the mocked LaunchpadLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.launchpad.tasks.FullLaunchpadLister' + ) + assert res + res.wait() + assert res.successful() + + lister.assert_called_once() + lister.db_last_threshold.assert_not_called() + lister.run.assert_called_once_with(max_bound=None)