diff --git a/swh/lister/sourceforge/lister.py b/swh/lister/sourceforge/lister.py --- a/swh/lister/sourceforge/lister.py +++ b/swh/lister/sourceforge/lister.py @@ -351,6 +351,10 @@ project=project, mount_point=tool["mount_point"], ) + if tool_name == VcsNames.MERCURIAL.value: + # SourceForge does not yet support anonymous HTTPS cloning for Mercurial + # See https://sourceforge.net/p/forge/feature-requests/727/ + url = url.replace("https://", "http://") entry = SourceForgeListerEntry( vcs=VcsNames(tool_name), url=url, last_modified=last_modified ) diff --git a/swh/lister/sourceforge/tests/data/random-mercurial.json b/swh/lister/sourceforge/tests/data/random-mercurial.json new file mode 100644 --- /dev/null +++ b/swh/lister/sourceforge/tests/data/random-mercurial.json @@ -0,0 +1,53 @@ +{ + "shortname": "random-mercurial", + "name": "Mercurial", + "_id": "4bf3fc291be1ce2f10000051", + "url": "https://sourceforge.net/p/random-mercurial/", + "private": false, + "short_description": "This is an example mercurial project", + "creation_date": "2010-11-10", + "summary": "", + "external_homepage": "", + "video_url": "", + "socialnetworks": [], + "status": "active", + "moved_to_url": "", + "preferred_support_tool": "", + "preferred_support_url": "", + "developers": [ + { + "username": "Alphare", + "name": "Raphaël Gomès", + "url": "https://sourceforge.net/u/alphare/" + } + ], + "tools": [ + { + "name": "hg", + "mount_point": "hg", + "url": "/p/random-mercurial/hg/", + "icons": { + "24": "images/code_24.png", + "32": "images/code_32.png", + "48": "images/code_48.png" + }, + "installable": true, + "tool_label": "Mercurial", + "mount_label": "Mercurial" + } + ], + "labels": [], + "categories": { + "audience": [], + "developmentstatus": [], + "environment": [], + "language": [], + "license": [], + "translation": [], + "os": [], + "database": [], + "topic": [] + }, + "icon_url": null, + "screenshots": [] +} diff --git a/swh/lister/sourceforge/tests/data/subsitemap-1.xml b/swh/lister/sourceforge/tests/data/subsitemap-1.xml --- a/swh/lister/sourceforge/tests/data/subsitemap-1.xml +++ b/swh/lister/sourceforge/tests/data/subsitemap-1.xml @@ -35,4 +35,9 @@ 2017-12-31 daily + + https://sourceforge.net/p/random-mercurial/ + 2019-05-02 + daily + diff --git a/swh/lister/sourceforge/tests/test_lister.py b/swh/lister/sourceforge/tests/test_lister.py --- a/swh/lister/sourceforge/tests/test_lister.py +++ b/swh/lister/sourceforge/tests/test_lister.py @@ -32,6 +32,7 @@ "mojunk": "p", "mramm": "p", "os3dmodels": "p", + "random-mercurial": "p", } URLS_MATCHER = { @@ -77,6 +78,7 @@ "https://git.code.sf.net/p/mojunk/git": ("git", "2017-12-31"), "https://git.code.sf.net/p/mojunk/git2": ("git", "2017-12-31"), "https://svn.code.sf.net/p/mojunk/svn": ("svn", "2017-12-31"), + "http://hg.code.sf.net/p/random-mercurial/hg": ("hg", "2019-05-02"), } @@ -115,10 +117,11 @@ # - os3dmodels (2 repos), # - mramm (3 repos), # - mojunk (3 repos), - # - backapps/website (1 repo). + # - backapps/website (1 repo), + # - random-mercurial (1 repo). # adobe and backapps itself have no repos. - assert stats.pages == 4 - assert stats.origins == 9 + assert stats.pages == 5 + assert stats.origins == 10 expected_state = { "subsitemap_last_modified": { "https://sourceforge.net/allura_sitemap/sitemap-0.xml": "2021-03-18", @@ -230,6 +233,12 @@ url="https://svn.code.sf.net/p/backapps/website/code", last_update=iso8601.parse_date("2021-02-11"), ), + ListedOrigin( + lister_id=lister.lister_obj.id, + visit_type="hg", + url="http://hg.code.sf.net/p/random-mercurial/hg", + last_update=iso8601.parse_date("2019-05-02"), + ), ] swh_scheduler.record_listed_origins(faked_listed_origins) @@ -308,10 +317,11 @@ # - os3dmodels (2 repos), # - mramm (3 repos), # - mojunk (3 repos), - # - backapps/website (1 repo). + # - backapps/website (1 repo), + # - random-mercurial (1 repo). # adobe and backapps itself have no repos. - assert stats.pages == 4 - assert stats.origins == 9 + assert stats.pages == 5 + assert stats.origins == 10 scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results assert {o.url: o.visit_type for o in scheduler_origins} == { @@ -324,6 +334,7 @@ "https://git.code.sf.net/p/mojunk/git": "git", "https://git.code.sf.net/p/mojunk/git2": "git", "https://svn.code.sf.net/p/mojunk/svn": "svn", + "http://hg.code.sf.net/p/random-mercurial/hg": "hg", } # Test `time.sleep` is called with exponential retries @@ -389,11 +400,12 @@ stats = lister.run() # - os3dmodels (2 repos), # - mojunk (3 repos), - # - backapps/website (1 repo). + # - backapps/website (1 repo), + # - random-mercurial (1 repo). # adobe and backapps itself have no repos. # Did *not* list mramm - assert stats.pages == 3 - assert stats.origins == 6 + assert stats.pages == 4 + assert stats.origins == 7 scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results res = {o.url: (o.visit_type, str(o.last_update.date())) for o in scheduler_origins} @@ -405,4 +417,5 @@ "https://git.code.sf.net/p/mojunk/git": ("git", "2017-12-31"), "https://git.code.sf.net/p/mojunk/git2": ("git", "2017-12-31"), "https://svn.code.sf.net/p/mojunk/svn": ("svn", "2017-12-31"), + "http://hg.code.sf.net/p/random-mercurial/hg": ("hg", "2019-05-02"), }