diff --git a/swh/lister/sourceforge/lister.py b/swh/lister/sourceforge/lister.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/sourceforge/lister.py
@@ -0,0 +1,255 @@
+# Copyright (C) 2017 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+# Lister for projects hosted on SourceForge.
+# As the SourceForge REST API does not enable to list projects,
+# we will use the rsync mirror of files hosted on SourceForge
+# to retrieve the projects names (we will surely miss some but
+# most important ones should be retrieved)
+
+from swh.lister.sourceforge.models import SourceForgeModel
+from swh.lister.core.indexing_lister import SWHIndexingHttpLister
+
+import bisect
+import re
+import requests
+import subprocess
+import string
+
+# url of rsync mirror for sourceforge projects
+_sf_mirror_rsync_baseurl = \
+ 'rsync://rsync.mirrorservice.org/downloads.sourceforge.net'
+# rsync://netix.dl.sourceforge.net/sfmir
+
+# sample output when using rsync to list sourceforge projects
+# $ rsync --list-only rsync://netix.dl.sourceforge.net/sfmir/a/
+# drwxr-xr-x 4,096 2016/08/29 20:02:18 .
+# drwxr-xr-x 126 2017/11/01 01:24:09 a-
+# drwxr-xr-x 10 2017/01/17 22:53:53 a0
+# drwxr-xr-x 47 2017/08/05 03:34:16 a1
+# drwxr-xr-x 4,096 2017/11/02 01:28:17 a2
+# drwxr-xr-x 105 2017/10/07 02:27:09 a3
+# drwxr-xr-x 29 2016/12/30 05:15:03 a4
+# drwxr-xr-x 10 2017/09/12 23:08:25 a5
+# drwxr-xr-x 29 2017/07/14 02:29:20 a6
+# drwxr-xr-x 10 2017/01/17 22:53:58 a7
+# drwxr-xr-x 40 2016/08/27 14:41:11 a8
+# drwxr-xr-x 10 2017/01/17 22:53:58 a9
+# drwxr-xr-x 4,096 2017/10/27 23:18:03 aa
+# drwxr-xr-x 4,096 2017/10/12 10:54:39 ab
+# drwxr-xr-x 8,192 2017/10/31 02:23:46 ac
+# drwxr-xr-x 12,288 2017/11/01 17:05:02 ad
+# drwxr-xr-x 4,096 2017/10/31 01:28:09 ae
+# drwxr-xr-x 4,096 2017/11/01 01:24:09 af
+# drwxr-xr-x 4,096 2017/10/29 23:25:01 ag
+# drwxr-xr-x 4,096 2017/09/11 20:25:36 ah
+# drwxr-xr-x 8,192 2017/10/20 17:46:55 ai
+# drwxr-xr-x 4,096 2017/09/24 20:00:01 aj
+# drwxr-xr-x 4,096 2017/09/25 16:30:01 ak
+# drwxr-xr-x 12,288 2017/11/02 05:56:13 al
+# drwxr-xr-x 8,192 2017/10/23 23:52:57 am
+# drwxr-xr-x 20,480 2017/11/01 14:59:02 an
+# drwxr-xr-x 4,096 2017/10/02 07:00:01 ao
+# drwxr-xr-x 12,288 2017/10/31 14:15:01 ap
+# drwxr-xr-x 4,096 2017/10/02 18:13:44 aq
+# drwxr-xr-x 16,384 2017/10/29 14:50:01 ar
+# drwxr-xr-x 16,384 2017/10/31 21:15:01 as
+# drwxr-xr-x 8,192 2017/10/26 20:29:25 at
+# drwxr-xr-x 16,384 2017/10/30 16:25:01 au
+# drwxr-xr-x 4,096 2017/10/24 00:24:00 av
+# drwxr-xr-x 4,096 2017/10/05 01:00:01 aw
+# drwxr-xr-x 4,096 2017/10/27 02:26:58 ax
+# drwxr-xr-x 4,096 2017/10/25 22:35:02 ay
+# drwxr-xr-x 4,096 2017/10/15 17:55:01 az
+
+# set of characters for the first letter of a folder containing sf projects
+_sf_subdir_first_char_set = string.ascii_lowercase
+# set of characters for the second letter of a folder containing sf projects
+_sf_subdir_second_char_set = '-' + string.digits + string.ascii_lowercase
+# cache for rsync listing ouput
+_projects_list_cache = {}
+
+
+def _list_sf_projects_in_subdir(sf_mirror_rsync_baseurl, subdir):
+ """
+ Utility function to list sourceforge projects with rsync
+ located in the folder xy from url (sf_mirror_rsync_baseurl)/(x)/(x)(y)/
+ """
+ if subdir not in _projects_list_cache:
+ projects = []
+ try:
+ # call rsync to list the desired folder
+ output = subprocess.check_output(
+ ["rsync", "--list-only", "%s/%s/%s/" %
+ (sf_mirror_rsync_baseurl, subdir[0], subdir)],
+ stderr=subprocess.STDOUT)
+ # iterate over response lines
+ lines = output.decode('utf-8').split('\n')
+ for line in lines:
+ # the line corresponds to a folder
+ if line.startswith('drwxr-xr-x'):
+ columns = line.split()
+ # only consider folders whose first letter is the same as
+ # the one from the listed folder
+ if columns[4].startswith(subdir[0]):
+ projects.append(columns[4])
+ except:
+ pass
+ # put retrieved projects list in cache
+ _projects_list_cache[subdir] = sorted(projects)
+ return _projects_list_cache[subdir]
+
+
+def _next_sf_project_first_chars(current_first_chars=''):
+ """
+ Utility function to get the next projects folder name to list with rsync.
+ For instance, aa -> ab, fg -> fh, gz -> h-, h7 -> h8, ...
+ """
+ if len(current_first_chars) > 0:
+ first_char = current_first_chars[0]
+ else:
+ first_char = 'a'
+ if len(current_first_chars) > 1:
+ second_char = current_first_chars[1]
+ else:
+ second_char = '-'
+ if first_char == 'z' and second_char == 'z':
+ return None
+ elif second_char == 'z':
+ first_char_idx = _sf_subdir_first_char_set.index(first_char)
+ return _sf_subdir_first_char_set[first_char_idx+1] + \
+ _sf_subdir_second_char_set[0]
+ else:
+ second_char_idx = _sf_subdir_second_char_set.index(second_char)
+ return first_char + _sf_subdir_second_char_set[second_char_idx+1]
+
+
+def _next_sf_project(sf_mirror_rsync_baseurl, current_project=''):
+ """
+ Utility function to get the next sourceforge project name.
+ """
+ first_chars = None
+ if current_project:
+ first_chars = current_project[:2]
+ if len(current_project) < 2:
+ first_chars += '-'
+ else:
+ first_chars = 'a-'
+ projects = _list_sf_projects_in_subdir(sf_mirror_rsync_baseurl,
+ first_chars)
+ if not current_project:
+ return projects[0]
+ else:
+ idx = bisect.bisect_left(projects, current_project)
+ if idx < len(projects) - 1:
+ return projects[idx+1]
+ else:
+ next_f_chars = _next_sf_project_first_chars(first_chars)
+ if not next_f_chars:
+ return None
+ next_first_chars_ok = False
+ while not next_first_chars_ok:
+ projects = _list_sf_projects_in_subdir(
+ sf_mirror_rsync_baseurl, next_f_chars)
+ if len(projects) > 0:
+ next_first_chars_ok = True
+ else:
+ next_f_chars = _next_sf_project_first_chars(next_f_chars)
+ return projects[0]
+
+
+class SourceForgeLister(SWHIndexingHttpLister):
+ PATH_TEMPLATE = '/rest/p/%s/'
+ MODEL = SourceForgeModel
+
+ @property
+ def ADDITIONAL_CONFIG(self): # noqa: N802
+ config = super().ADDITIONAL_CONFIG
+ # base url of sourceforge rsync mirror
+ config['sf_rsync_mirror_url'] = ('str', _sf_mirror_rsync_baseurl)
+ # list of sf projects to skip (those whose a call to sf rest api fails)
+ config['sf_projects_to_skip'] = ('list', ['cygwin-ports'])
+ return config
+
+ def get_model_from_sf_project_metadata(self, repo):
+ model = []
+ # ensure input metadata are valid
+ if 'tools' not in repo:
+ return model
+ # iterate over project services
+ for tool in repo['tools']:
+ # a version control system is present in the project
+ # and is not a link to external service like GitHub
+ if tool['mount_point'] == 'code' and tool['name'] != 'link':
+ # we need to check that the code repository is not empty first
+ resp = requests.get('%s%s' % (self.api_baseurl,
+ tool['url']))
+ # code repository is not empty, now retrieve the origin url
+ # based on the used vcs tool
+ if resp.status_code == 200 and \
+ b'No (more) commits' not in resp.content:
+ # bazaar repo special case
+ if tool['name'] == 'bzr':
+ bzr_url_template = \
+ 'bzr://%s.bzr.sourceforge.net/bzrroot/%s'
+ origin_url = bzr_url_template %\
+ (repo['shortname'], repo['shortname'])
+ # cvs repo special case
+ elif tool['name'] == 'cvs':
+ cvs_url_template = \
+ '%s.cvs.sourceforge.net:/cvsroot/%s'
+ origin_url = cvs_url_template %\
+ (repo['shortname'], repo['shortname'])
+ # for hg, git and svn
+ else:
+ origin_url = 'https://%s.code.sf.net%s' %\
+ (tool['name'], tool['url'])
+ # append to model for each code repository found
+ model.append({'uid': repo['shortname'],
+ 'indexable': repo['shortname'],
+ 'name': repo['shortname'],
+ 'full_name': repo['name'],
+ 'html_url': repo['url'],
+ 'origin_url': origin_url,
+ 'origin_type': tool['name'],
+ 'description': repo['short_description']
+ })
+ return model
+
+ def get_next_target_from_response(self, response):
+ # special case when the provided min_index does
+ # not correspond to a sf project name
+ if response.status_code == 404:
+ return _next_sf_project(
+ self.config['sf_rsync_mirror_url'],
+ re.search(r'^.*/(.*)/$', response.url).group(1))
+ body = response.json()
+ # ensure current response is valid
+ if 'shortname' not in body:
+ return None
+ # get next sourceforge project name
+ next_project = _next_sf_project(self.config['sf_rsync_mirror_url'],
+ body['shortname'])
+ while next_project in self.config['sf_projects_to_skip']:
+ next_project = _next_sf_project(self.config['sf_rsync_mirror_url'],
+ next_project)
+ return next_project
+
+ def transport_response_simplified(self, response):
+ if response.status_code != 200:
+ return []
+ else:
+ repo = response.json()
+ return self.get_model_from_sf_project_metadata(repo)
+
+ def is_within_bounds(self, inner, lower=None, upper=None):
+ if lower is None and upper is None:
+ return True
+ elif lower is None:
+ ret = inner <= upper
+ elif upper is None:
+ ret = inner >= lower
+ else:
+ ret = lower <= inner <= upper
+ return ret
diff --git a/swh/lister/sourceforge/models.py b/swh/lister/sourceforge/models.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/sourceforge/models.py
@@ -0,0 +1,15 @@
+# Copyright (C) 2017 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from sqlalchemy import Column, String
+
+from swh.lister.core.models import ModelBase
+
+
+class SourceForgeModel(ModelBase):
+ """a SourceForge repository"""
+ __tablename__ = 'sourceforge_repos'
+
+ uid = Column(String, primary_key=True)
+ indexable = Column(String, index=True)
diff --git a/swh/lister/sourceforge/tasks.py b/swh/lister/sourceforge/tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/sourceforge/tasks.py
@@ -0,0 +1,28 @@
+# Copyright (C) 2017 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.lister.core.tasks import (IndexingDiscoveryListerTask,
+ IndexingRangeListerTask,
+ IndexingRefreshListerTask, ListerTaskBase)
+
+from .lister import SourceForgeLister
+
+
+class SourceForgeListerTask(ListerTaskBase):
+ def new_lister(self):
+ return SourceForgeLister(lister_name='sourceforge.net',
+ api_baseurl='https://sourceforge.net')
+
+
+class IncrementalSourceForgeLister(SourceForgeListerTask,
+ IndexingDiscoveryListerTask):
+ task_queue = 'swh_lister_sourceforge_discover'
+
+
+class RangeSourceForgeLister(SourceForgeListerTask, IndexingRangeListerTask):
+ task_queue = 'swh_lister_sourceforge_refresh'
+
+
+class FullSourceForgeRelister(SourceForgeListerTask, IndexingRefreshListerTask): # noqa
+ task_queue = 'swh_lister_sourceforge_refresh'
diff --git a/swh/lister/sourceforge/tests/__init__.py b/swh/lister/sourceforge/tests/__init__.py
new file mode 100644
diff --git a/swh/lister/sourceforge/tests/api_empty_response.json b/swh/lister/sourceforge/tests/api_empty_response.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/sourceforge/tests/api_empty_response.json
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/swh/lister/sourceforge/tests/api_response.json b/swh/lister/sourceforge/tests/api_response.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/sourceforge/tests/api_response.json
@@ -0,0 +1,373 @@
+
+{
+ "status": "active",
+ "preferred_support_tool": "_url",
+ "preferred_support_url": "http://sourceforge.net/project/memberlist.php?group_id=48010",
+ "labels": [],
+ "private": false,
+ "creation_date": "2002-03-02",
+ "socialnetworks": [],
+ "tools": [
+ {
+ "mount_point": "feature-requests",
+ "name": "tickets",
+ "icons": {
+ "24": "images/tickets_24.png",
+ "32": "images/tickets_32.png",
+ "48": "images/tickets_48.png"
+ },
+ "url": "/p/amiga/feature-requests/",
+ "tool_label": "Tickets",
+ "installable": true,
+ "mount_label": "Feature Requests"
+ },
+ {
+ "mount_point": "patches",
+ "name": "tickets",
+ "icons": {
+ "24": "images/tickets_24.png",
+ "32": "images/tickets_32.png",
+ "48": "images/tickets_48.png"
+ },
+ "url": "/p/amiga/patches/",
+ "tool_label": "Tickets",
+ "installable": true,
+ "mount_label": "Patches"
+ },
+ {
+ "mount_point": "news",
+ "name": "blog",
+ "icons": {
+ "24": "images/blog_24.png",
+ "32": "images/blog_32.png",
+ "48": "images/blog_48.png"
+ },
+ "url": "/p/amiga/news/",
+ "tool_label": "Blog",
+ "installable": true,
+ "mount_label": "News"
+ },
+ {
+ "mount_point": "discussion",
+ "name": "discussion",
+ "icons": {
+ "24": "images/forums_24.png",
+ "32": "images/forums_32.png",
+ "48": "images/forums_48.png"
+ },
+ "url": "/p/amiga/discussion/",
+ "tool_label": "Discussion",
+ "installable": true,
+ "mount_label": "Discussion"
+ },
+ {
+ "mount_point": "donate",
+ "name": "link",
+ "icons": {
+ "24": "images/ext_24.png",
+ "32": "images/ext_32.png",
+ "48": "images/ext_48.png"
+ },
+ "url": "/p/amiga/donate/",
+ "tool_label": "External Link",
+ "installable": true,
+ "mount_label": "Donate"
+ },
+ {
+ "mount_point": "code",
+ "name": "svn",
+ "icons": {
+ "24": "images/code_24.png",
+ "32": "images/code_32.png",
+ "48": "images/code_48.png"
+ },
+ "url": "/p/amiga/code/",
+ "tool_label": "SVN",
+ "installable": true,
+ "mount_label": "Code"
+ },
+ {
+ "mount_point": "cvs",
+ "name": "cvs",
+ "icons": {
+ "24": "images/code_24.png",
+ "32": "images/code_32.png",
+ "48": "images/code_48.png"
+ },
+ "url": "/p/amiga/cvs/",
+ "tool_label": "CVS",
+ "installable": false,
+ "mount_label": "Cvs"
+ },
+ {
+ "sourceforge_group_id": 48010,
+ "mount_point": "summary",
+ "name": "summary",
+ "icons": {
+ "24": "images/sftheme/24x24/blog_24.png",
+ "32": "images/sftheme/32x32/blog_32.png",
+ "48": "images/sftheme/48x48/blog_48.png"
+ },
+ "url": "/p/amiga/summary/",
+ "tool_label": "Summary",
+ "installable": false,
+ "mount_label": "Summary"
+ },
+ {
+ "mount_point": "support",
+ "name": "support",
+ "icons": {
+ "24": "images/sftheme/24x24/blog_24.png",
+ "32": "images/sftheme/32x32/blog_32.png",
+ "48": "images/sftheme/48x48/blog_48.png"
+ },
+ "url": "/p/amiga/support/",
+ "tool_label": "Support",
+ "installable": false,
+ "mount_label": "Support"
+ },
+ {
+ "mount_point": "reviews",
+ "name": "reviews",
+ "icons": {
+ "24": "images/sftheme/24x24/blog_24.png",
+ "32": "images/sftheme/32x32/blog_32.png",
+ "48": "images/sftheme/48x48/blog_48.png"
+ },
+ "url": "/p/amiga/reviews/",
+ "tool_label": "Reviews",
+ "installable": false,
+ "mount_label": "Reviews"
+ },
+ {
+ "mount_point": "files",
+ "name": "files",
+ "icons": {
+ "24": "images/downloads_24.png",
+ "32": "images/downloads_32.png",
+ "48": "images/downloads_48.png"
+ },
+ "url": "/p/amiga/files/",
+ "tool_label": "Files",
+ "installable": false,
+ "mount_label": "Files"
+ },
+ {
+ "mount_point": "gui-requests",
+ "name": "tickets",
+ "icons": {
+ "24": "images/tickets_24.png",
+ "32": "images/tickets_32.png",
+ "48": "images/tickets_48.png"
+ },
+ "url": "/p/amiga/gui-requests/",
+ "tool_label": "Tickets",
+ "installable": true,
+ "mount_label": "GUI Requests"
+ },
+ {
+ "mount_point": "wiki",
+ "name": "wiki",
+ "icons": {
+ "24": "images/wiki_24.png",
+ "32": "images/wiki_32.png",
+ "48": "images/wiki_48.png"
+ },
+ "url": "/p/amiga/wiki/",
+ "tool_label": "Wiki",
+ "installable": true,
+ "mount_label": "Wiki"
+ },
+ {
+ "mount_point": "bugs",
+ "name": "tickets",
+ "icons": {
+ "24": "images/tickets_24.png",
+ "32": "images/tickets_32.png",
+ "48": "images/tickets_48.png"
+ },
+ "url": "/p/amiga/bugs/",
+ "tool_label": "Tickets",
+ "installable": true,
+ "mount_label": "Bugs"
+ },
+ {
+ "mount_point": "activity",
+ "name": "activity",
+ "icons": {
+ "24": "images/admin_24.png",
+ "32": "images/admin_32.png",
+ "48": "images/admin_48.png"
+ },
+ "url": "/p/amiga/activity/",
+ "tool_label": "Tool",
+ "installable": false,
+ "mount_label": "Activity"
+ },
+ {
+ "mount_point": "mailman",
+ "name": "mailman",
+ "icons": {
+ "24": "images/forums_24.png",
+ "32": "images/forums_32.png",
+ "48": "images/forums_48.png"
+ },
+ "url": "/p/amiga/mailman/",
+ "tool_label": "Mailing Lists",
+ "installable": false,
+ "mount_label": "Mailing Lists"
+ }
+ ],
+ "categories": {
+ "developmentstatus": [
+ {
+ "fullpath": "Development Status :: 5 - Production/Stable",
+ "fullname": "5 - Production/Stable",
+ "shortname": "production",
+ "id": 11
+ }
+ ],
+ "environment": [
+ {
+ "fullpath": "User Interface :: Textual :: Command-line",
+ "fullname": "Command-line",
+ "shortname": "ui_commandline",
+ "id": 459
+ }
+ ],
+ "language": [
+ {
+ "fullpath": "Programming Language :: Rexx",
+ "fullname": "Rexx",
+ "shortname": "rexx",
+ "id": 179
+ },
+ {
+ "fullpath": "Programming Language :: C",
+ "fullname": "C",
+ "shortname": "c",
+ "id": 164
+ }
+ ],
+ "license": [
+ {
+ "fullpath": "License :: OSI-Approved Open Source :: BSD License",
+ "fullname": "BSD License",
+ "shortname": "bsd",
+ "id": 187
+ },
+ {
+ "fullpath": "License :: Public Domain",
+ "fullname": "Public Domain",
+ "shortname": "publicdomain",
+ "id": 197
+ },
+ {
+ "fullpath": "License :: OSI-Approved Open Source :: GNU Library or Lesser General Public License version 2.0 (LGPLv2)",
+ "fullname": "GNU Library or Lesser General Public License version 2.0 (LGPLv2)",
+ "shortname": "lgpl",
+ "id": 16
+ },
+ {
+ "fullpath": "License :: OSI-Approved Open Source :: GNU General Public License version 2.0 (GPLv2)",
+ "fullname": "GNU General Public License version 2.0 (GPLv2)",
+ "shortname": "gpl",
+ "id": 15
+ },
+ {
+ "fullpath": "License :: OSI-Approved Open Source :: MIT License",
+ "fullname": "MIT License",
+ "shortname": "mit",
+ "id": 188
+ }
+ ],
+ "database": [],
+ "topic": [
+ {
+ "fullpath": "Topic :: Other/Nonlisted Topic",
+ "fullname": "Other/Nonlisted Topic",
+ "shortname": "other",
+ "id": 234
+ }
+ ],
+ "audience": [
+ {
+ "fullpath": "Intended Audience :: by End-User Class :: End Users/Desktop",
+ "fullname": "End Users/Desktop",
+ "shortname": "endusers",
+ "id": 2
+ }
+ ],
+ "translation": [
+ {
+ "fullpath": "Translations :: English",
+ "fullname": "English",
+ "shortname": "english",
+ "id": 275
+ },
+ {
+ "fullpath": "Translations :: Spanish",
+ "fullname": "Spanish",
+ "shortname": "spanish",
+ "id": 277
+ }
+ ],
+ "os": [
+ {
+ "fullpath": "Operating System :: Other Operating Systems :: AmigaOS",
+ "fullname": "AmigaOS",
+ "shortname": "amigaos",
+ "id": 434
+ }
+ ]
+ },
+ "_id": "516efddc2718467b8b82e1b2",
+ "name": "Amiga",
+ "url": "https://sourceforge.net/p/amiga/",
+ "icon_url": null,
+ "video_url": "",
+ "screenshots": [
+ {
+ "url": "https://sourceforge.net/p/amiga/screenshot/234039.jpg",
+ "caption": "Amiga Computer's Original Logo",
+ "thumbnail_url": "https://sourceforge.net/p/amiga/screenshot/234039.jpg/thumb"
+ }
+ ],
+ "summary": "",
+ "short_description": "Here you'll find several Open Source Software ported/compiled for AmigaOS (and compatible systems). This project was created with the aim to give support to the ppl who don't have the time/compiler to do it for itself since most stuff compiles OOTB",
+ "moved_to_url": "",
+ "shortname": "amiga",
+ "developers": [
+ {
+ "url": "https://sourceforge.net/u/synco/",
+ "username": "synco",
+ "name": "synco"
+ },
+ {
+ "url": "https://sourceforge.net/u/diegocr/",
+ "username": "diegocr",
+ "name": "Diego Casorran"
+ },
+ {
+ "url": "https://sourceforge.net/u/mutoid/",
+ "username": "mutoid",
+ "name": "Jean Sibart"
+ },
+ {
+ "url": "https://sourceforge.net/u/yabba/",
+ "username": "yabba",
+ "name": "Stefan Burstrom"
+ },
+ {
+ "url": "https://sourceforge.net/u/userid-1666728/",
+ "username": "bernd_afa",
+ "name": "Bernd Roesch"
+ },
+ {
+ "url": "https://sourceforge.net/u/userid-811665/",
+ "username": "sonic_amiga",
+ "name": "Pavel Fedin"
+ }
+ ],
+ "external_homepage": "http://amiga.sourceforge.net"
+}
diff --git a/swh/lister/sourceforge/tests/test_sf_lister.py b/swh/lister/sourceforge/tests/test_sf_lister.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/sourceforge/tests/test_sf_lister.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2017 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import re
+import requests_mock
+import unittest
+
+from nose.tools import istest
+from unittest.mock import patch
+
+from swh.lister.core.tests.test_lister import (
+ IndexingHttpListerTesterBase, noop
+)
+from swh.lister.sourceforge.lister import SourceForgeLister
+
+_sf_baseurl = 'https://sourceforge.net'
+
+
+@requests_mock.Mocker()
+class SourceForgeListerTester(IndexingHttpListerTesterBase, unittest.TestCase):
+ Lister = SourceForgeLister
+ test_re = re.compile(r'/rest/p/([^?&]+)/')
+ lister_subdir = 'sourceforge'
+ good_api_response_file = 'api_response.json'
+ bad_api_response_file = 'api_empty_response.json'
+ first_index = 'amiga'
+ last_index = 'amiga'
+ entries_per_page = 1
+
+ def get_fl(self, override_config=None):
+ if override_config or self.fl is None:
+ with patch(
+ 'swh.scheduler.backend.SchedulerBackend.reconnect', noop
+ ):
+ self.fl = SourceForgeLister(
+ lister_name='sourceforge',
+ api_baseurl=_sf_baseurl,
+ override_config=override_config)
+ self.fl.INITIAL_BACKOFF = 1
+
+ self.fl.reset_backoff()
+ return self.fl
+
+ def mock_response(self, request, context):
+ self.fl.reset_backoff()
+ context.status_code = 200
+ if self.request_index(request) == str(self.first_index):
+ with open('swh/lister/%s/tests/%s' % (self.lister_subdir,
+ self.good_api_response_file),
+ 'r', encoding='utf-8') as r:
+ return r.read()
+ else:
+ with open('swh/lister/%s/tests/%s' % (self.lister_subdir,
+ self.bad_api_response_file),
+ 'r', encoding='utf-8') as r:
+ return r.read()
+
+ def mock_sf_requests(self, http_mocker):
+ http_mocker.get('%s/p/amiga/code/' % _sf_baseurl, text='OK')
+ http_mocker.get(self.test_re,
+ text=self.mock_response)
+
+ def setUp(self):
+ self.list_sf_projects_in_subdir_patch = patch(
+ 'swh.lister.sourceforge.lister._list_sf_projects_in_subdir')
+ self.list_sf_projects_in_subdir = \
+ self.list_sf_projects_in_subdir_patch.start()
+ self.list_sf_projects_in_subdir.return_value = ['amiga',
+ 'amigaone-linux']
+
+ def tearDown(self):
+ self.list_sf_projects_in_subdir_patch.stop()
+
+ @istest
+ def test_api_request(self, http_mocker):
+ pass
+
+ @istest
+ def test_fetch_multiple_pages_nodb(self, http_mocker):
+ self.mock_sf_requests(http_mocker)
+ super(SourceForgeListerTester, self).test_fetch_multiple_pages_nodb()
+
+ @istest
+ def test_fetch_multiple_pages_yesdb(self, http_mocker):
+ self.mock_sf_requests(http_mocker)
+ super(SourceForgeListerTester, self).test_fetch_multiple_pages_yesdb()
+
+ @istest
+ def test_repos_list(self, http_mocker):
+ self.mock_sf_requests(http_mocker)
+ super(SourceForgeListerTester, self).test_repos_list()
+
+ @istest
+ def test_fetch_none_nodb(self, http_mocker):
+ self.mock_sf_requests(http_mocker)
+ super(SourceForgeListerTester, self).test_fetch_none_nodb()
+
+ @istest
+ def test_fetch_one_nodb(self, http_mocker):
+ self.mock_sf_requests(http_mocker)
+ super(SourceForgeListerTester, self).test_fetch_one_nodb()
+
+ @istest
+ def test_is_within_bounds(self, http_mocker):
+ fl = self.get_fl()
+ self.assertTrue(fl.is_within_bounds("b", "a", "c"))
+ self.assertFalse(fl.is_within_bounds("a", "b", "c"))
+ self.assertTrue(fl.is_within_bounds("a", None, "c"))
+ self.assertTrue(fl.is_within_bounds("a", None, None))
+ self.assertTrue(fl.is_within_bounds("b", "a", None))
+ self.assertFalse(fl.is_within_bounds("a", "b", None))
+ self.assertTrue(fl.is_within_bounds("ac", "ab", "ad"))
+ self.assertFalse(fl.is_within_bounds("bu", "bw", "bz"))
+
+ @istest
+ def test_model_map(self, http_mocker):
+ self.mock_sf_requests(http_mocker)
+ super(SourceForgeListerTester, self).test_model_map()