diff --git a/swh/lister/sourceforge/lister.py b/swh/lister/sourceforge/lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/lister.py @@ -0,0 +1,255 @@ +# Copyright (C) 2017 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +# Lister for projects hosted on SourceForge. +# As the SourceForge REST API does not enable to list projects, +# we will use the rsync mirror of files hosted on SourceForge +# to retrieve the projects names (we will surely miss some but +# most important ones should be retrieved) + +from swh.lister.sourceforge.models import SourceForgeModel +from swh.lister.core.indexing_lister import SWHIndexingHttpLister + +import bisect +import re +import requests +import subprocess +import string + +# url of rsync mirror for sourceforge projects +_sf_mirror_rsync_baseurl = \ + 'rsync://rsync.mirrorservice.org/downloads.sourceforge.net' +# rsync://netix.dl.sourceforge.net/sfmir + +# sample output when using rsync to list sourceforge projects +# $ rsync --list-only rsync://netix.dl.sourceforge.net/sfmir/a/ +# drwxr-xr-x 4,096 2016/08/29 20:02:18 . +# drwxr-xr-x 126 2017/11/01 01:24:09 a- +# drwxr-xr-x 10 2017/01/17 22:53:53 a0 +# drwxr-xr-x 47 2017/08/05 03:34:16 a1 +# drwxr-xr-x 4,096 2017/11/02 01:28:17 a2 +# drwxr-xr-x 105 2017/10/07 02:27:09 a3 +# drwxr-xr-x 29 2016/12/30 05:15:03 a4 +# drwxr-xr-x 10 2017/09/12 23:08:25 a5 +# drwxr-xr-x 29 2017/07/14 02:29:20 a6 +# drwxr-xr-x 10 2017/01/17 22:53:58 a7 +# drwxr-xr-x 40 2016/08/27 14:41:11 a8 +# drwxr-xr-x 10 2017/01/17 22:53:58 a9 +# drwxr-xr-x 4,096 2017/10/27 23:18:03 aa +# drwxr-xr-x 4,096 2017/10/12 10:54:39 ab +# drwxr-xr-x 8,192 2017/10/31 02:23:46 ac +# drwxr-xr-x 12,288 2017/11/01 17:05:02 ad +# drwxr-xr-x 4,096 2017/10/31 01:28:09 ae +# drwxr-xr-x 4,096 2017/11/01 01:24:09 af +# drwxr-xr-x 4,096 2017/10/29 23:25:01 ag +# drwxr-xr-x 4,096 2017/09/11 20:25:36 ah +# drwxr-xr-x 8,192 2017/10/20 17:46:55 ai +# drwxr-xr-x 4,096 2017/09/24 20:00:01 aj +# drwxr-xr-x 4,096 2017/09/25 16:30:01 ak +# drwxr-xr-x 12,288 2017/11/02 05:56:13 al +# drwxr-xr-x 8,192 2017/10/23 23:52:57 am +# drwxr-xr-x 20,480 2017/11/01 14:59:02 an +# drwxr-xr-x 4,096 2017/10/02 07:00:01 ao +# drwxr-xr-x 12,288 2017/10/31 14:15:01 ap +# drwxr-xr-x 4,096 2017/10/02 18:13:44 aq +# drwxr-xr-x 16,384 2017/10/29 14:50:01 ar +# drwxr-xr-x 16,384 2017/10/31 21:15:01 as +# drwxr-xr-x 8,192 2017/10/26 20:29:25 at +# drwxr-xr-x 16,384 2017/10/30 16:25:01 au +# drwxr-xr-x 4,096 2017/10/24 00:24:00 av +# drwxr-xr-x 4,096 2017/10/05 01:00:01 aw +# drwxr-xr-x 4,096 2017/10/27 02:26:58 ax +# drwxr-xr-x 4,096 2017/10/25 22:35:02 ay +# drwxr-xr-x 4,096 2017/10/15 17:55:01 az + +# set of characters for the first letter of a folder containing sf projects +_sf_subdir_first_char_set = string.ascii_lowercase +# set of characters for the second letter of a folder containing sf projects +_sf_subdir_second_char_set = '-' + string.digits + string.ascii_lowercase +# cache for rsync listing ouput +_projects_list_cache = {} + + +def _list_sf_projects_in_subdir(sf_mirror_rsync_baseurl, subdir): + """ + Utility function to list sourceforge projects with rsync + located in the folder xy from url (sf_mirror_rsync_baseurl)/(x)/(x)(y)/ + """ + if subdir not in _projects_list_cache: + projects = [] + try: + # call rsync to list the desired folder + output = subprocess.check_output( + ["rsync", "--list-only", "%s/%s/%s/" % + (sf_mirror_rsync_baseurl, subdir[0], subdir)], + stderr=subprocess.STDOUT) + # iterate over response lines + lines = output.decode('utf-8').split('\n') + for line in lines: + # the line corresponds to a folder + if line.startswith('drwxr-xr-x'): + columns = line.split() + # only consider folders whose first letter is the same as + # the one from the listed folder + if columns[4].startswith(subdir[0]): + projects.append(columns[4]) + except: + pass + # put retrieved projects list in cache + _projects_list_cache[subdir] = sorted(projects) + return _projects_list_cache[subdir] + + +def _next_sf_project_first_chars(current_first_chars=''): + """ + Utility function to get the next projects folder name to list with rsync. + For instance, aa -> ab, fg -> fh, gz -> h-, h7 -> h8, ... + """ + if len(current_first_chars) > 0: + first_char = current_first_chars[0] + else: + first_char = 'a' + if len(current_first_chars) > 1: + second_char = current_first_chars[1] + else: + second_char = '-' + if first_char == 'z' and second_char == 'z': + return None + elif second_char == 'z': + first_char_idx = _sf_subdir_first_char_set.index(first_char) + return _sf_subdir_first_char_set[first_char_idx+1] + \ + _sf_subdir_second_char_set[0] + else: + second_char_idx = _sf_subdir_second_char_set.index(second_char) + return first_char + _sf_subdir_second_char_set[second_char_idx+1] + + +def _next_sf_project(sf_mirror_rsync_baseurl, current_project=''): + """ + Utility function to get the next sourceforge project name. + """ + first_chars = None + if current_project: + first_chars = current_project[:2] + if len(current_project) < 2: + first_chars += '-' + else: + first_chars = 'a-' + projects = _list_sf_projects_in_subdir(sf_mirror_rsync_baseurl, + first_chars) + if not current_project: + return projects[0] + else: + idx = bisect.bisect_left(projects, current_project) + if idx < len(projects) - 1: + return projects[idx+1] + else: + next_f_chars = _next_sf_project_first_chars(first_chars) + if not next_f_chars: + return None + next_first_chars_ok = False + while not next_first_chars_ok: + projects = _list_sf_projects_in_subdir( + sf_mirror_rsync_baseurl, next_f_chars) + if len(projects) > 0: + next_first_chars_ok = True + else: + next_f_chars = _next_sf_project_first_chars(next_f_chars) + return projects[0] + + +class SourceForgeLister(SWHIndexingHttpLister): + PATH_TEMPLATE = '/rest/p/%s/' + MODEL = SourceForgeModel + + @property + def ADDITIONAL_CONFIG(self): # noqa: N802 + config = super().ADDITIONAL_CONFIG + # base url of sourceforge rsync mirror + config['sf_rsync_mirror_url'] = ('str', _sf_mirror_rsync_baseurl) + # list of sf projects to skip (those whose a call to sf rest api fails) + config['sf_projects_to_skip'] = ('list', ['cygwin-ports']) + return config + + def get_model_from_sf_project_metadata(self, repo): + model = [] + # ensure input metadata are valid + if 'tools' not in repo: + return model + # iterate over project services + for tool in repo['tools']: + # a version control system is present in the project + # and is not a link to external service like GitHub + if tool['mount_point'] == 'code' and tool['name'] != 'link': + # we need to check that the code repository is not empty first + resp = requests.get('%s%s' % (self.api_baseurl, + tool['url'])) + # code repository is not empty, now retrieve the origin url + # based on the used vcs tool + if resp.status_code == 200 and \ + b'No (more) commits' not in resp.content: + # bazaar repo special case + if tool['name'] == 'bzr': + bzr_url_template = \ + 'bzr://%s.bzr.sourceforge.net/bzrroot/%s' + origin_url = bzr_url_template %\ + (repo['shortname'], repo['shortname']) + # cvs repo special case + elif tool['name'] == 'cvs': + cvs_url_template = \ + '%s.cvs.sourceforge.net:/cvsroot/%s' + origin_url = cvs_url_template %\ + (repo['shortname'], repo['shortname']) + # for hg, git and svn + else: + origin_url = 'https://%s.code.sf.net%s' %\ + (tool['name'], tool['url']) + # append to model for each code repository found + model.append({'uid': repo['shortname'], + 'indexable': repo['shortname'], + 'name': repo['shortname'], + 'full_name': repo['name'], + 'html_url': repo['url'], + 'origin_url': origin_url, + 'origin_type': tool['name'], + 'description': repo['short_description'] + }) + return model + + def get_next_target_from_response(self, response): + # special case when the provided min_index does + # not correspond to a sf project name + if response.status_code == 404: + return _next_sf_project( + self.config['sf_rsync_mirror_url'], + re.search(r'^.*/(.*)/$', response.url).group(1)) + body = response.json() + # ensure current response is valid + if 'shortname' not in body: + return None + # get next sourceforge project name + next_project = _next_sf_project(self.config['sf_rsync_mirror_url'], + body['shortname']) + while next_project in self.config['sf_projects_to_skip']: + next_project = _next_sf_project(self.config['sf_rsync_mirror_url'], + next_project) + return next_project + + def transport_response_simplified(self, response): + if response.status_code != 200: + return [] + else: + repo = response.json() + return self.get_model_from_sf_project_metadata(repo) + + def is_within_bounds(self, inner, lower=None, upper=None): + if lower is None and upper is None: + return True + elif lower is None: + ret = inner <= upper + elif upper is None: + ret = inner >= lower + else: + ret = lower <= inner <= upper + return ret diff --git a/swh/lister/sourceforge/models.py b/swh/lister/sourceforge/models.py new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/models.py @@ -0,0 +1,15 @@ +# Copyright (C) 2017 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from sqlalchemy import Column, String + +from swh.lister.core.models import ModelBase + + +class SourceForgeModel(ModelBase): + """a SourceForge repository""" + __tablename__ = 'sourceforge_repos' + + uid = Column(String, primary_key=True) + indexable = Column(String, index=True) diff --git a/swh/lister/sourceforge/tasks.py b/swh/lister/sourceforge/tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/tasks.py @@ -0,0 +1,28 @@ +# Copyright (C) 2017 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.lister.core.tasks import (IndexingDiscoveryListerTask, + IndexingRangeListerTask, + IndexingRefreshListerTask, ListerTaskBase) + +from .lister import SourceForgeLister + + +class SourceForgeListerTask(ListerTaskBase): + def new_lister(self): + return SourceForgeLister(lister_name='sourceforge.net', + api_baseurl='https://sourceforge.net') + + +class IncrementalSourceForgeLister(SourceForgeListerTask, + IndexingDiscoveryListerTask): + task_queue = 'swh_lister_sourceforge_discover' + + +class RangeSourceForgeLister(SourceForgeListerTask, IndexingRangeListerTask): + task_queue = 'swh_lister_sourceforge_refresh' + + +class FullSourceForgeRelister(SourceForgeListerTask, IndexingRefreshListerTask): # noqa + task_queue = 'swh_lister_sourceforge_refresh' diff --git a/swh/lister/sourceforge/tests/__init__.py b/swh/lister/sourceforge/tests/__init__.py new file mode 100644 diff --git a/swh/lister/sourceforge/tests/api_empty_response.json b/swh/lister/sourceforge/tests/api_empty_response.json new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/tests/api_empty_response.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/swh/lister/sourceforge/tests/api_response.json b/swh/lister/sourceforge/tests/api_response.json new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/tests/api_response.json @@ -0,0 +1,373 @@ + +{ + "status": "active", + "preferred_support_tool": "_url", + "preferred_support_url": "http://sourceforge.net/project/memberlist.php?group_id=48010", + "labels": [], + "private": false, + "creation_date": "2002-03-02", + "socialnetworks": [], + "tools": [ + { + "mount_point": "feature-requests", + "name": "tickets", + "icons": { + "24": "images/tickets_24.png", + "32": "images/tickets_32.png", + "48": "images/tickets_48.png" + }, + "url": "/p/amiga/feature-requests/", + "tool_label": "Tickets", + "installable": true, + "mount_label": "Feature Requests" + }, + { + "mount_point": "patches", + "name": "tickets", + "icons": { + "24": "images/tickets_24.png", + "32": "images/tickets_32.png", + "48": "images/tickets_48.png" + }, + "url": "/p/amiga/patches/", + "tool_label": "Tickets", + "installable": true, + "mount_label": "Patches" + }, + { + "mount_point": "news", + "name": "blog", + "icons": { + "24": "images/blog_24.png", + "32": "images/blog_32.png", + "48": "images/blog_48.png" + }, + "url": "/p/amiga/news/", + "tool_label": "Blog", + "installable": true, + "mount_label": "News" + }, + { + "mount_point": "discussion", + "name": "discussion", + "icons": { + "24": "images/forums_24.png", + "32": "images/forums_32.png", + "48": "images/forums_48.png" + }, + "url": "/p/amiga/discussion/", + "tool_label": "Discussion", + "installable": true, + "mount_label": "Discussion" + }, + { + "mount_point": "donate", + "name": "link", + "icons": { + "24": "images/ext_24.png", + "32": "images/ext_32.png", + "48": "images/ext_48.png" + }, + "url": "/p/amiga/donate/", + "tool_label": "External Link", + "installable": true, + "mount_label": "Donate" + }, + { + "mount_point": "code", + "name": "svn", + "icons": { + "24": "images/code_24.png", + "32": "images/code_32.png", + "48": "images/code_48.png" + }, + "url": "/p/amiga/code/", + "tool_label": "SVN", + "installable": true, + "mount_label": "Code" + }, + { + "mount_point": "cvs", + "name": "cvs", + "icons": { + "24": "images/code_24.png", + "32": "images/code_32.png", + "48": "images/code_48.png" + }, + "url": "/p/amiga/cvs/", + "tool_label": "CVS", + "installable": false, + "mount_label": "Cvs" + }, + { + "sourceforge_group_id": 48010, + "mount_point": "summary", + "name": "summary", + "icons": { + "24": "images/sftheme/24x24/blog_24.png", + "32": "images/sftheme/32x32/blog_32.png", + "48": "images/sftheme/48x48/blog_48.png" + }, + "url": "/p/amiga/summary/", + "tool_label": "Summary", + "installable": false, + "mount_label": "Summary" + }, + { + "mount_point": "support", + "name": "support", + "icons": { + "24": "images/sftheme/24x24/blog_24.png", + "32": "images/sftheme/32x32/blog_32.png", + "48": "images/sftheme/48x48/blog_48.png" + }, + "url": "/p/amiga/support/", + "tool_label": "Support", + "installable": false, + "mount_label": "Support" + }, + { + "mount_point": "reviews", + "name": "reviews", + "icons": { + "24": "images/sftheme/24x24/blog_24.png", + "32": "images/sftheme/32x32/blog_32.png", + "48": "images/sftheme/48x48/blog_48.png" + }, + "url": "/p/amiga/reviews/", + "tool_label": "Reviews", + "installable": false, + "mount_label": "Reviews" + }, + { + "mount_point": "files", + "name": "files", + "icons": { + "24": "images/downloads_24.png", + "32": "images/downloads_32.png", + "48": "images/downloads_48.png" + }, + "url": "/p/amiga/files/", + "tool_label": "Files", + "installable": false, + "mount_label": "Files" + }, + { + "mount_point": "gui-requests", + "name": "tickets", + "icons": { + "24": "images/tickets_24.png", + "32": "images/tickets_32.png", + "48": "images/tickets_48.png" + }, + "url": "/p/amiga/gui-requests/", + "tool_label": "Tickets", + "installable": true, + "mount_label": "GUI Requests" + }, + { + "mount_point": "wiki", + "name": "wiki", + "icons": { + "24": "images/wiki_24.png", + "32": "images/wiki_32.png", + "48": "images/wiki_48.png" + }, + "url": "/p/amiga/wiki/", + "tool_label": "Wiki", + "installable": true, + "mount_label": "Wiki" + }, + { + "mount_point": "bugs", + "name": "tickets", + "icons": { + "24": "images/tickets_24.png", + "32": "images/tickets_32.png", + "48": "images/tickets_48.png" + }, + "url": "/p/amiga/bugs/", + "tool_label": "Tickets", + "installable": true, + "mount_label": "Bugs" + }, + { + "mount_point": "activity", + "name": "activity", + "icons": { + "24": "images/admin_24.png", + "32": "images/admin_32.png", + "48": "images/admin_48.png" + }, + "url": "/p/amiga/activity/", + "tool_label": "Tool", + "installable": false, + "mount_label": "Activity" + }, + { + "mount_point": "mailman", + "name": "mailman", + "icons": { + "24": "images/forums_24.png", + "32": "images/forums_32.png", + "48": "images/forums_48.png" + }, + "url": "/p/amiga/mailman/", + "tool_label": "Mailing Lists", + "installable": false, + "mount_label": "Mailing Lists" + } + ], + "categories": { + "developmentstatus": [ + { + "fullpath": "Development Status :: 5 - Production/Stable", + "fullname": "5 - Production/Stable", + "shortname": "production", + "id": 11 + } + ], + "environment": [ + { + "fullpath": "User Interface :: Textual :: Command-line", + "fullname": "Command-line", + "shortname": "ui_commandline", + "id": 459 + } + ], + "language": [ + { + "fullpath": "Programming Language :: Rexx", + "fullname": "Rexx", + "shortname": "rexx", + "id": 179 + }, + { + "fullpath": "Programming Language :: C", + "fullname": "C", + "shortname": "c", + "id": 164 + } + ], + "license": [ + { + "fullpath": "License :: OSI-Approved Open Source :: BSD License", + "fullname": "BSD License", + "shortname": "bsd", + "id": 187 + }, + { + "fullpath": "License :: Public Domain", + "fullname": "Public Domain", + "shortname": "publicdomain", + "id": 197 + }, + { + "fullpath": "License :: OSI-Approved Open Source :: GNU Library or Lesser General Public License version 2.0 (LGPLv2)", + "fullname": "GNU Library or Lesser General Public License version 2.0 (LGPLv2)", + "shortname": "lgpl", + "id": 16 + }, + { + "fullpath": "License :: OSI-Approved Open Source :: GNU General Public License version 2.0 (GPLv2)", + "fullname": "GNU General Public License version 2.0 (GPLv2)", + "shortname": "gpl", + "id": 15 + }, + { + "fullpath": "License :: OSI-Approved Open Source :: MIT License", + "fullname": "MIT License", + "shortname": "mit", + "id": 188 + } + ], + "database": [], + "topic": [ + { + "fullpath": "Topic :: Other/Nonlisted Topic", + "fullname": "Other/Nonlisted Topic", + "shortname": "other", + "id": 234 + } + ], + "audience": [ + { + "fullpath": "Intended Audience :: by End-User Class :: End Users/Desktop", + "fullname": "End Users/Desktop", + "shortname": "endusers", + "id": 2 + } + ], + "translation": [ + { + "fullpath": "Translations :: English", + "fullname": "English", + "shortname": "english", + "id": 275 + }, + { + "fullpath": "Translations :: Spanish", + "fullname": "Spanish", + "shortname": "spanish", + "id": 277 + } + ], + "os": [ + { + "fullpath": "Operating System :: Other Operating Systems :: AmigaOS", + "fullname": "AmigaOS", + "shortname": "amigaos", + "id": 434 + } + ] + }, + "_id": "516efddc2718467b8b82e1b2", + "name": "Amiga", + "url": "https://sourceforge.net/p/amiga/", + "icon_url": null, + "video_url": "", + "screenshots": [ + { + "url": "https://sourceforge.net/p/amiga/screenshot/234039.jpg", + "caption": "Amiga Computer's Original Logo", + "thumbnail_url": "https://sourceforge.net/p/amiga/screenshot/234039.jpg/thumb" + } + ], + "summary": "", + "short_description": "Here you'll find several Open Source Software ported/compiled for AmigaOS (and compatible systems). This project was created with the aim to give support to the ppl who don't have the time/compiler to do it for itself since most stuff compiles OOTB", + "moved_to_url": "", + "shortname": "amiga", + "developers": [ + { + "url": "https://sourceforge.net/u/synco/", + "username": "synco", + "name": "synco" + }, + { + "url": "https://sourceforge.net/u/diegocr/", + "username": "diegocr", + "name": "Diego Casorran" + }, + { + "url": "https://sourceforge.net/u/mutoid/", + "username": "mutoid", + "name": "Jean Sibart" + }, + { + "url": "https://sourceforge.net/u/yabba/", + "username": "yabba", + "name": "Stefan Burstrom" + }, + { + "url": "https://sourceforge.net/u/userid-1666728/", + "username": "bernd_afa", + "name": "Bernd Roesch" + }, + { + "url": "https://sourceforge.net/u/userid-811665/", + "username": "sonic_amiga", + "name": "Pavel Fedin" + } + ], + "external_homepage": "http://amiga.sourceforge.net" +} diff --git a/swh/lister/sourceforge/tests/test_sf_lister.py b/swh/lister/sourceforge/tests/test_sf_lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/tests/test_sf_lister.py @@ -0,0 +1,119 @@ +# Copyright (C) 2017 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import re +import requests_mock +import unittest + +from nose.tools import istest +from unittest.mock import patch + +from swh.lister.core.tests.test_lister import ( + IndexingHttpListerTesterBase, noop +) +from swh.lister.sourceforge.lister import SourceForgeLister + +_sf_baseurl = 'https://sourceforge.net' + + +@requests_mock.Mocker() +class SourceForgeListerTester(IndexingHttpListerTesterBase, unittest.TestCase): + Lister = SourceForgeLister + test_re = re.compile(r'/rest/p/([^?&]+)/') + lister_subdir = 'sourceforge' + good_api_response_file = 'api_response.json' + bad_api_response_file = 'api_empty_response.json' + first_index = 'amiga' + last_index = 'amiga' + entries_per_page = 1 + + def get_fl(self, override_config=None): + if override_config or self.fl is None: + with patch( + 'swh.scheduler.backend.SchedulerBackend.reconnect', noop + ): + self.fl = SourceForgeLister( + lister_name='sourceforge', + api_baseurl=_sf_baseurl, + override_config=override_config) + self.fl.INITIAL_BACKOFF = 1 + + self.fl.reset_backoff() + return self.fl + + def mock_response(self, request, context): + self.fl.reset_backoff() + context.status_code = 200 + if self.request_index(request) == str(self.first_index): + with open('swh/lister/%s/tests/%s' % (self.lister_subdir, + self.good_api_response_file), + 'r', encoding='utf-8') as r: + return r.read() + else: + with open('swh/lister/%s/tests/%s' % (self.lister_subdir, + self.bad_api_response_file), + 'r', encoding='utf-8') as r: + return r.read() + + def mock_sf_requests(self, http_mocker): + http_mocker.get('%s/p/amiga/code/' % _sf_baseurl, text='OK') + http_mocker.get(self.test_re, + text=self.mock_response) + + def setUp(self): + self.list_sf_projects_in_subdir_patch = patch( + 'swh.lister.sourceforge.lister._list_sf_projects_in_subdir') + self.list_sf_projects_in_subdir = \ + self.list_sf_projects_in_subdir_patch.start() + self.list_sf_projects_in_subdir.return_value = ['amiga', + 'amigaone-linux'] + + def tearDown(self): + self.list_sf_projects_in_subdir_patch.stop() + + @istest + def test_api_request(self, http_mocker): + pass + + @istest + def test_fetch_multiple_pages_nodb(self, http_mocker): + self.mock_sf_requests(http_mocker) + super(SourceForgeListerTester, self).test_fetch_multiple_pages_nodb() + + @istest + def test_fetch_multiple_pages_yesdb(self, http_mocker): + self.mock_sf_requests(http_mocker) + super(SourceForgeListerTester, self).test_fetch_multiple_pages_yesdb() + + @istest + def test_repos_list(self, http_mocker): + self.mock_sf_requests(http_mocker) + super(SourceForgeListerTester, self).test_repos_list() + + @istest + def test_fetch_none_nodb(self, http_mocker): + self.mock_sf_requests(http_mocker) + super(SourceForgeListerTester, self).test_fetch_none_nodb() + + @istest + def test_fetch_one_nodb(self, http_mocker): + self.mock_sf_requests(http_mocker) + super(SourceForgeListerTester, self).test_fetch_one_nodb() + + @istest + def test_is_within_bounds(self, http_mocker): + fl = self.get_fl() + self.assertTrue(fl.is_within_bounds("b", "a", "c")) + self.assertFalse(fl.is_within_bounds("a", "b", "c")) + self.assertTrue(fl.is_within_bounds("a", None, "c")) + self.assertTrue(fl.is_within_bounds("a", None, None)) + self.assertTrue(fl.is_within_bounds("b", "a", None)) + self.assertFalse(fl.is_within_bounds("a", "b", None)) + self.assertTrue(fl.is_within_bounds("ac", "ab", "ad")) + self.assertFalse(fl.is_within_bounds("bu", "bw", "bz")) + + @istest + def test_model_map(self, http_mocker): + self.mock_sf_requests(http_mocker) + super(SourceForgeListerTester, self).test_model_map()