Page MenuHomeSoftware Heritage
Files F9123750

No OneTemporary
Actions

View Options

diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py
index 0253a32..646efa5 100644
--- a/swh/lister/cgit/lister.py
+++ b/swh/lister/cgit/lister.py
@@ -1,187 +1,216 @@
# Copyright (C) 2019-2021 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime, timezone
import logging
import re
from typing import Any, Dict, Iterator, List, Optional
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
import requests
from requests.exceptions import HTTPError
from swh.lister import USER_AGENT
from swh.lister.pattern import CredentialsType, StatelessLister
from swh.scheduler.interface import SchedulerInterface
from swh.scheduler.model import ListedOrigin
logger = logging.getLogger(__name__)
Repositories = List[Dict[str, Any]]
class CGitLister(StatelessLister[Repositories]):
"""Lister class for CGit repositories.
This lister will retrieve the list of published git repositories by
parsing the HTML page(s) of the index retrieved at `url`.
- For each found git repository, a query is made at the given url found
- in this index to gather published "Clone" URLs to be used as origin
- URL for that git repo.
+ The lister currently defines 2 listing behaviors:
+
+ - If the `base_git_url` is provided, the listed origin urls are computed out of the
+ base git url link and the one listed in the main listed page (resulting in less
+ HTTP queries than the 2nd behavior below). This is expected to be the main
+ deployed behavior.
+
+ - Otherwise (with no `base_git_url`), for each found git repository listed, one
+ extra HTTP query is made at the given url found in the main listing page to gather
+ published "Clone" URLs to be used as origin URL for that git repo. If several
+ "Clone" urls are provided, prefer the http/https one, if any, otherwise fallback
+ to the first one.
- If several "Clone" urls are provided, prefer the http/https one, if
- any, otherwise fallback to the first one.
"""
LISTER_NAME = "cgit"
def __init__(
self,
scheduler: SchedulerInterface,
url: str,
instance: Optional[str] = None,
credentials: Optional[CredentialsType] = None,
+ base_git_url: Optional[str] = None,
):
"""Lister class for CGit repositories.
Args:
- url (str): main URL of the CGit instance, i.e. url of the index
+ url: main URL of the CGit instance, i.e. url of the index
of published git repositories on this instance.
- instance (str): Name of cgit instance. Defaults to url's hostname
+ instance: Name of cgit instance. Defaults to url's hostname
if unset.
+ base_git_url: Optional base git url which allows the origin url
+ computations.
"""
if not instance:
instance = urlparse(url).hostname
assert instance is not None # Make mypy happy
super().__init__(
scheduler=scheduler, url=url, instance=instance, credentials=credentials,
)
self.session = requests.Session()
self.session.headers.update(
{"Accept": "application/html", "User-Agent": USER_AGENT}
)
+ self.base_git_url = base_git_url
def _get_and_parse(self, url: str) -> BeautifulSoup:
"""Get the given url and parse the retrieved HTML using BeautifulSoup"""
response = self.session.get(url)
response.raise_for_status()
return BeautifulSoup(response.text, features="html.parser")
def get_pages(self) -> Iterator[Repositories]:
"""Generate git 'project' URLs found on the current CGit server
The last_update date is retrieved on the list of repo page to avoid
to compute it on the repository details which only give a date per branch
"""
next_page: Optional[str] = self.url
while next_page:
bs_idx = self._get_and_parse(next_page)
page_results = []
for tr in bs_idx.find("div", {"class": "content"}).find_all(
"tr", {"class": ""}
):
- url = urljoin(self.url, tr.find("a")["href"])
+ repository_link = tr.find("a")["href"]
+ repo_url = None
+ git_url = None
+
+ base_url = urljoin(self.url, repository_link)
+ if self.base_git_url: # mapping provided
+ # computing git url
+ git_url = base_url.replace(self.url, self.base_git_url)
+ else:
+ # we compute the git detailed page url from which we will retrieve
+ # the git url (cf. self.get_origins_from_page)
+ repo_url = base_url
+
span = tr.find("span", {"class": re.compile("age-")})
if span:
last_updated_date = span["title"]
else:
last_updated_date = None
page_results.append(
- {"url": url, "last_updated_date": last_updated_date}
+ {
+ "url": repo_url,
+ "git_url": git_url,
+ "last_updated_date": last_updated_date,
+ }
)
yield page_results
try:
pager = bs_idx.find("ul", {"class": "pager"})
current_page = pager.find("a", {"class": "current"})
if current_page:
next_page = current_page.parent.next_sibling.a["href"]
next_page = urljoin(self.url, next_page)
except (AttributeError, KeyError):
# no pager, or no next page
next_page = None
def get_origins_from_page(
self, repositories: Repositories
) -> Iterator[ListedOrigin]:
"""Convert a page of cgit repositories into a list of ListedOrigins."""
assert self.lister_obj.id is not None
- for repository in repositories:
- origin_url = self._get_origin_from_repository_url(repository["url"])
+ for repo in repositories:
+ origin_url = repo["git_url"] or self._get_origin_from_repository_url(
+ repo["url"]
+ )
if origin_url is None:
continue
yield ListedOrigin(
lister_id=self.lister_obj.id,
url=origin_url,
visit_type="git",
- last_update=_parse_last_updated_date(repository),
+ last_update=_parse_last_updated_date(repo),
)
def _get_origin_from_repository_url(self, repository_url: str) -> Optional[str]:
"""Extract the git url from the repository page"""
try:
bs = self._get_and_parse(repository_url)
except HTTPError as e:
logger.warning(
"Unexpected HTTP status code %s on %s",
e.response.status_code,
e.response.url,
)
return None
# origin urls are listed on the repository page
# TODO check if forcing https is better or not ?
# <link rel='vcs-git' href='git://...' title='...'/>
# <link rel='vcs-git' href='http://...' title='...'/>
# <link rel='vcs-git' href='https://...' title='...'/>
urls = [x["href"] for x in bs.find_all("a", {"rel": "vcs-git"})]
if not urls:
return None
# look for the http/https url, if any, and use it as origin_url
for url in urls:
if urlparse(url).scheme in ("http", "https"):
origin_url = url
break
else:
# otherwise, choose the first one
origin_url = urls[0]
return origin_url
def _parse_last_updated_date(repository: Dict[str, Any]) -> Optional[datetime]:
"""Parse the last updated date"""
date = repository.get("last_updated_date")
if not date:
return None
parsed_date = None
for date_format in ("%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S (%Z)"):
try:
parsed_date = datetime.strptime(date, date_format)
# force UTC to avoid naive datetime
if not parsed_date.tzinfo:
parsed_date = parsed_date.replace(tzinfo=timezone.utc)
break
except Exception:
pass
if not parsed_date:
logger.warning(
"Could not parse %s last_updated date: %s", repository["url"], date,
)
return parsed_date
diff --git a/swh/lister/cgit/tasks.py b/swh/lister/cgit/tasks.py
index e6ef159..9a95648 100644
--- a/swh/lister/cgit/tasks.py
+++ b/swh/lister/cgit/tasks.py
@@ -1,21 +1,25 @@
# Copyright (C) 2019-2021 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from typing import Dict, Optional
from celery import shared_task
from .lister import CGitLister
@shared_task(name=__name__ + ".CGitListerTask")
-def list_cgit(url: str, instance: Optional[str] = None,) -> Dict[str, str]:
+def list_cgit(
+ url: str, instance: Optional[str] = None, base_git_url: Optional[str] = None
+) -> Dict[str, str]:
"""Lister task for CGit instances"""
- lister = CGitLister.from_configfile(url=url, instance=instance)
+ lister = CGitLister.from_configfile(
+ url=url, instance=instance, base_git_url=base_git_url
+ )
return lister.run().dict()
@shared_task(name=__name__ + ".ping")
def _ping():
return "OK"
diff --git a/swh/lister/cgit/tests/data/https_git.baserock.org/cgit b/swh/lister/cgit/tests/data/https_git.baserock.org/cgit
new file mode 100644
index 0000000..46fde7e
--- /dev/null
+++ b/swh/lister/cgit/tests/data/https_git.baserock.org/cgit
@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang='en'>
+ <head>
+ <title>Lorry Depot</title>
+ <meta name='generator' content='cgit v1.2.1'/>
+ <meta name='robots' content='index, nofollow'/>
+ <link rel='stylesheet' type='text/css' href='/cgit-css/cgit.css'/>
+ <link rel='shortcut icon' href='/favicon.ico'/>
+ </head>
+ <body>
+ <div id='cgit'><table id='header'>
+ <tr>
+ <td class='logo' rowspan='2'><a href='/cgit/'><img src='/cgit-css/cgit.png' alt='cgit logo'/></a></td>
+ <td class='main'>Lorry Depot</td></tr>
+ <tr><td class='sub'>Mirror Repositories</td></tr></table>
+ <table class='tabs'><tr><td>
+ <a class='active' href='/cgit/'>index</a></td><td class='form'><form method='get' action='/cgit/'>
+ <input type='search' name='q' size='10' value=''/>
+ <input type='submit' value='search'/>
+ </form></td></tr></table>
+ <div class='content'>
+ <table summary='repository list' class='list nowrap'><tr class='nohover'><th class='left'><a href='/cgit/?s=name'>Name</a></th><th class='left'><a href='/cgit/?s=desc'>Description</a></th><th class='left'><a href='/cgit/?s=idle'>Idle</a></th><th class='left'>Links</th></tr>
+ <tr><td class='toplevel-repo'><a title='baserock/baserock/baserock-chroot.git' href='/cgit/baserock/baserock/baserock-chroot.git/'>baserock/baserock/baserock-chroot.git</a></td><td><a href='/cgit/baserock/baserock/baserock-chroot.git/'>Tools for working with Baserock chroots
+ </a></td><td><span class='age-years' title='2015-05-20 09:59:01 +0000'>6 years</span></td><td><a class='button' href='/cgit/baserock/baserock/baserock-chroot.git/'>summary</a><a class='button' href='/cgit/baserock/baserock/baserock-chroot.git/log/'>log</a><a class='button' href='/cgit/baserock/baserock/baserock-chroot.git/tree/'>tree</a></td></tr>
+ <tr><td class='toplevel-repo'><a title='baserock/baserock/bsp-support.git' href='/cgit/baserock/baserock/bsp-support.git/'>baserock/baserock/bsp-support.git</a></td><td><a href='/cgit/baserock/baserock/bsp-support.git/'>Programs and configuration needed to support specific devices
+ </a></td><td><span class='age-years' title='2015-10-25 19:53:10 +0000'>5 years</span></td><td><a class='button' href='/cgit/baserock/baserock/bsp-support.git/'>summary</a><a class='button' href='/cgit/baserock/baserock/bsp-support.git/log/'>log</a><a class='button' href='/cgit/baserock/baserock/bsp-support.git/tree/'>tree</a></td></tr>
+ <tr><td class='toplevel-repo'><a title='baserock/baserock/definitions.git' href='/cgit/baserock/baserock/definitions.git/'>baserock/baserock/definitions.git</a></td><td><a href='/cgit/baserock/baserock/definitions.git/'>gitlab.com: baserock/definitions.git
+ </a></td><td><span class='age-months' title='2019-05-15 17:58:47 +0000'>21 months</span></td><td><a class='button' href='/cgit/baserock/baserock/definitions.git/'>summary</a><a class='button' href='/cgit/baserock/baserock/definitions.git/log/'>log</a><a class='button' href='/cgit/baserock/baserock/definitions.git/tree/'>tree</a></td></tr>
+ </table>
+ </div> <!-- class=content -->
+ </div> <!-- id=cgit -->
+ </body>
+</html>
diff --git a/swh/lister/cgit/tests/data/https_git.eclipse.org/c b/swh/lister/cgit/tests/data/https_git.eclipse.org/c
new file mode 100644
index 0000000..8cd1d4b
--- /dev/null
+++ b/swh/lister/cgit/tests/data/https_git.eclipse.org/c
@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang='en'>
+ <head>
+ <title>Eclipse Git repositories</title>
+ <meta name='generator' content='cgit v1.2.1'/>
+ <meta name='robots' content='index, nofollow'/>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="keywords" content="eclipse.org, Eclipse Foundation, Eclipse, Git, Source code, Source" />
+ </head>
+ <body>
+ <!-- /#breadcrumb -->
+ <main>
+ <div class="container-fluid legacy-page" id="novaContent">
+ <div class="col-md-24"><div id='cgit'><table id='header'>
+ <tr>
+ <td class='logo' rowspan='2'><a href='/c/'><img src='/git.png' alt='cgit logo'/></a></td>
+ <td class='main'>Eclipse Git repositories</td></tr>
+ <tr><td class='sub'>To use Git in Eclipse, check out the EGit project.</td></tr></table>
+ <table class='tabs'><tr><td>
+ <a class='active' href='/c/'>index</a><a href='/c/?p=about'>about</a></td><td class='form'><form method='get' action='/c/'>
+ <input type='search' name='q' size='10' value=''/>
+ <input type='submit' value='search'/>
+ </form></td></tr></table>
+ <div class='content'><table summary='repository list' class='list nowrap'><tr class='nohover'><th class='left'><a href='/c/?s=name'>Name</a></th><th class='left'><a href='/c/?s=desc'>Description</a></th><th class='left'><a href='/c/?s=idle'>Idle</a></th></tr>
+ <tr class='nohover-highlight'><td colspan='3' class='reposection'>3p</td></tr>
+ <tr><td class='sublevel-repo'><a title='3p.git' href='/c/3p/3p.git/'>3p.git</a></td><td><a href='/c/3p/3p.git/'>3P - PolarSys Packaging</a></td><td><span class='age-months' title='2020-01-17 23:52:00 +0000'>12 months</span></td></tr>
+ <tr><td class='sublevel-repo'><a title='qvtd.git' href='/c/www.eclipse.org/qvtd.git/'>qvtd.git</a></td><td><a href='/c/www.eclipse.org/qvtd.git/'>www.eclipse.org project repository</a></td><td><span class='age-years' title='2018-07-20 23:51:54 +0000'>3 years</span></td></tr>
+ <tr><td class='sublevel-repo'><a title='r4e.git' href='/c/www.eclipse.org/r4e.git/'>r4e.git</a></td><td><a href='/c/www.eclipse.org/r4e.git/'>www.eclipse.org project repository</a></td><td><span class='age-years' title='2018-11-17 00:57:51 +0000'>2 years</span></td></tr>
+ <tr><td class='sublevel-repo'><a title='rap.git' href='/c/www.eclipse.org/rap.git/'>rap.git</a></td><td><a href='/c/www.eclipse.org/rap.git/'>RAP project website</a></td><td><span class='age-weeks' title='2020-12-20 01:16:37 +0000'>6 weeks</span></td></tr>
+ <tr class='nohover-highlight'><td colspan='3' class='reposection'>zenoh</td></tr>
+ <tr><td class='sublevel-repo'><a title='zenoh.git' href='/c/zenoh/zenoh.git/'>zenoh.git</a></td><td><a href='/c/zenoh/zenoh.git/'>Unnamed repository; edit this file 'description' to name the repository.</a></td><td><span class='age-months' title='2020-01-25 05:07:25 +0000'>12 months</span></td></tr>
+ </table></div> <!-- class=content -->
+ </div>
+ </div>
+ </main>
+ <p id="back-to-top">
+ <a class="visible-xs" href="#top">Back to the top</a>
+ </p>
+ </body>
+</html>
diff --git a/swh/lister/cgit/tests/data/https_jff.email/cgit b/swh/lister/cgit/tests/data/https_jff.email/cgit
new file mode 100644
index 0000000..ebc0289
--- /dev/null
+++ b/swh/lister/cgit/tests/data/https_jff.email/cgit
@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang='en'>
+ <head>
+ <title>Jörgs Debian Repository</title>
+ <meta name='generator' content='cgit v1.1'/>
+ <meta name='robots' content='index, nofollow'/>
+ <link rel='stylesheet' type='text/css' href='/cgit-css/cgit.css'/>
+ <link rel='shortcut icon' href='/favicon.ico'/>
+ </head>
+ <body>
+ <div id='cgit'><table id='header'>
+ <tr>
+ <td class='logo' rowspan='2'><a href='/cgit/'><img src='/cgit-css/cgit.png' alt='cgit logo'/></a></td>
+ <td class='main'>Jörgs Debian Repository</td></tr>
+ <tr><td class='sub'>ask debian-cgit@jff.email</td></tr></table>
+ <table class='tabs'><tr><td>
+ <a class='active' href='/cgit/'>index</a></td><td class='form'><form method='get' action='/cgit/'>
+ <input type='text' name='q' size='10' value=''/>
+ <input type='submit' value='search'/>
+ </form></td></tr></table>
+ <div class='content'><table summary='repository list' class='list nowrap'>
+ <tr class='nohover'><th class='left'><a href='/cgit/?s=name'>Name</a></th><th class='left'><a href='/cgit/?s=desc'>Description</a></th><th class='left'><a href='/cgit/?s=owner'>Owner</a></th><th class='left'><a href='/cgit/?s=idle'>Idle</a></th></tr>
+ <tr><td class='toplevel-repo'><a title='gnome-pie.git' href='/cgit/gnome-pie.git/'>gnome-pie.git</a></td><td><a href='/cgit/gnome-pie.git/'>Debian repo for gnome-pie</a></td><td><a href='/cgit/?q=debian@jff.email'>debian@jff.email</a></td><td><span class='age-months' title='2020-06-01 15:22:58 +0000'>8 months</span></td></tr>
+ <tr><td class='toplevel-repo'><a title='gs5.git' href='/cgit/gs5.git/'>gs5.git</a></td><td><a href='/cgit/gs5.git/'>Clone of gs5</a></td><td><a href='/cgit/?q=jff@jff.email'>jff@jff.email</a></td><td><span class='age-years' title='2018-10-01 17:24:55 +0000'>2 years</span></td></tr>
+ <tr><td class='toplevel-repo'><a title='libunistring.git' href='/cgit/libunistring.git/'>libunistring.git</a></td><td><a href='/cgit/libunistring.git/'>Debian repo for libunistring</a></td><td><a href='/cgit/?q=debian@jff.email'>debian@jff.email</a></td><td><span class='age-months' title='2020-06-02 06:00:00 +0000'>8 months</span></td></tr>
+ <tr><td class='toplevel-repo'><a title='mailgraph.git' href='/cgit/mailgraph.git/'>mailgraph.git</a></td><td><a href='/cgit/mailgraph.git/'>Debian repo for mailgraph</a></td><td><a href='/cgit/?q=debian@jff.email'>debian@jff.email</a></td><td><span class='age-years' title='2018-09-09 19:18:23 +0000'>2 years</span></td></tr>
+ <tr><td class='toplevel-repo'><a title='mwc.git' href='/cgit/mwc.git/'>mwc.git</a></td><td><a href='/cgit/mwc.git/'>Debian repo for mwc</a></td><td><a href='/cgit/?q=debian@jff.email'>debian@jff.email</a></td><td><span class='age-months' title='2019-07-14 14:58:18 +0000'>19 months</span></td></tr>
+ <tr><td class='toplevel-repo'><a title='xtrkcad.git' href='/cgit/xtrkcad.git/'>xtrkcad.git</a></td><td><a href='/cgit/xtrkcad.git/'>Debian repo for xtrkcad</a></td><td><a href='/cgit/?q=debian@jff.email'>debian@jff.email</a></td><td><span class='age-months' title='2020-08-24 19:27:40 +0000'>5 months</span></td></tr>
+ </table></div> <!-- class=content -->
+ <div class='footer'>generated by <a href='https://git.zx2c4.com/cgit/about/'>cgit v1.1</a> at 2021-01-29 14:18:26 +0000</div>
+ </div> <!-- id=cgit -->
+ </body>
+</html>
diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py
index cf4c9fc..f963a4e 100644
--- a/swh/lister/cgit/tests/test_lister.py
+++ b/swh/lister/cgit/tests/test_lister.py
@@ -1,198 +1,233 @@
# Copyright (C) 2019-2021 The Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime, timedelta, timezone
from typing import List
import pytest
from swh.core.pytest_plugin import requests_mock_datadir_factory
from swh.lister import __version__
from swh.lister.cgit.lister import CGitLister, _parse_last_updated_date
from swh.lister.pattern import ListerStats
def test_lister_cgit_get_pages_one_page(requests_mock_datadir, swh_scheduler):
url = "https://git.savannah.gnu.org/cgit/"
lister_cgit = CGitLister(swh_scheduler, url=url)
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
assert len(flattened_repos) == 977
assert (
flattened_repos[0]["url"] == "https://git.savannah.gnu.org/cgit/elisp-es.git/"
)
# note the url below is NOT a subpath of /cgit/
assert (
flattened_repos[-1]["url"] == "https://git.savannah.gnu.org/path/to/yetris.git/"
) # noqa
# note the url below is NOT on the same server
assert flattened_repos[-2]["url"] == "http://example.org/cgit/xstarcastle.git/"
def test_lister_cgit_get_pages_with_pages(requests_mock_datadir, swh_scheduler):
url = "https://git.tizen/cgit/"
lister_cgit = CGitLister(swh_scheduler, url=url)
repos: List[List[str]] = list(lister_cgit.get_pages())
flattened_repos = sum(repos, [])
# we should have 16 repos (listed on 3 pages)
assert len(repos) == 3
assert len(flattened_repos) == 16
def test_lister_cgit_run_with_page(requests_mock_datadir, swh_scheduler):
"""cgit lister supports pagination"""
url = "https://git.tizen/cgit/"
lister_cgit = CGitLister(swh_scheduler, url=url)
stats = lister_cgit.run()
expected_nb_origins = 16
assert stats == ListerStats(pages=3, origins=expected_nb_origins)
# test page parsing
scheduler_origins = swh_scheduler.get_listed_origins(
lister_cgit.lister_obj.id
).results
assert len(scheduler_origins) == expected_nb_origins
# test listed repositories
for listed_origin in scheduler_origins:
assert listed_origin.visit_type == "git"
assert listed_origin.url.startswith("https://git.tizen")
# test user agent content
assert len(requests_mock_datadir.request_history) != 0
for request in requests_mock_datadir.request_history:
assert "User-Agent" in request.headers
user_agent = request.headers["User-Agent"]
assert "Software Heritage Lister" in user_agent
assert __version__ in user_agent
def test_lister_cgit_run_populates_last_update(requests_mock_datadir, swh_scheduler):
"""cgit lister returns last updated date"""
url = "https://git.tizen/cgit"
urls_without_date = [
f"https://git.tizen.org/cgit/{suffix_url}"
for suffix_url in ["All-Projects", "All-Users", "Lock-Projects",]
]
lister_cgit = CGitLister(swh_scheduler, url=url)
stats = lister_cgit.run()
expected_nb_origins = 16
assert stats == ListerStats(pages=3, origins=expected_nb_origins)
# test page parsing
scheduler_origins = swh_scheduler.get_listed_origins(
lister_cgit.lister_obj.id
).results
assert len(scheduler_origins) == expected_nb_origins
# test listed repositories
for listed_origin in scheduler_origins:
if listed_origin.url in urls_without_date:
assert listed_origin.last_update is None
else:
assert listed_origin.last_update is not None
@pytest.mark.parametrize(
"date_str,expected_date",
[
({}, None),
("unexpected date", None),
("2020-0140-10 10:10:10 (GMT)", None),
(
"2020-01-10 10:10:10 (GMT)",
datetime(
year=2020,
month=1,
day=10,
hour=10,
minute=10,
second=10,
tzinfo=timezone.utc,
),
),
(
"2019-08-04 05:10:41 +0100",
datetime(
year=2019,
month=8,
day=4,
hour=5,
minute=10,
second=41,
tzinfo=timezone(timedelta(hours=1)),
),
),
],
)
def test_lister_cgit_date_parsing(date_str, expected_date):
"""test cgit lister date parsing"""
repository = {"url": "url", "last_updated_date": date_str}
assert _parse_last_updated_date(repository) == expected_date
requests_mock_datadir_missing_url = requests_mock_datadir_factory(
ignore_urls=["https://git.tizen/cgit/adaptation/ap_samsung/audio-hal-e4x12/",]
)
def test_lister_cgit_get_origin_from_repo_failing(
requests_mock_datadir_missing_url, swh_scheduler
):
url = "https://git.tizen/cgit/"
lister_cgit = CGitLister(swh_scheduler, url=url)
stats = lister_cgit.run()
expected_nb_origins = 15
assert stats == ListerStats(pages=3, origins=expected_nb_origins)
@pytest.mark.parametrize(
"credentials, expected_credentials",
[
(None, []),
({"key": "value"}, []),
(
{"cgit": {"tizen": [{"username": "user", "password": "pass"}]}},
[{"username": "user", "password": "pass"}],
),
],
)
def test_lister_cgit_instantiation_with_credentials(
credentials, expected_credentials, swh_scheduler
):
url = "https://git.tizen/cgit/"
lister = CGitLister(
swh_scheduler, url=url, instance="tizen", credentials=credentials
)
# Credentials are allowed in constructor
assert lister.credentials == expected_credentials
def test_lister_cgit_from_configfile(swh_scheduler_config, mocker):
load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar")
load_from_envvar.return_value = {
"scheduler": {"cls": "local", **swh_scheduler_config},
"url": "https://git.tizen/cgit/",
"instance": "tizen",
"credentials": {},
}
lister = CGitLister.from_configfile()
assert lister.scheduler is not None
assert lister.credentials is not None
+
+
+@pytest.mark.parametrize(
+ "url,base_git_url,expected_nb_origins",
+ [
+ ("https://git.eclipse.org/c", "https://eclipse.org/r", 5),
+ ("https://git.baserock.org/cgit/", "https://git.baserock.org/git/", 3),
+ ("https://jff.email/cgit/", "git://jff.email/opt/git/", 6),
+ ],
+)
+def test_lister_cgit_with_base_git_url(
+ url, base_git_url, expected_nb_origins, requests_mock_datadir, swh_scheduler
+):
+ """With base git url provided, listed urls should be the computed origin urls
+
+ """
+ lister_cgit = CGitLister(swh_scheduler, url=url, base_git_url=base_git_url,)
+
+ stats = lister_cgit.run()
+
+ assert stats == ListerStats(pages=1, origins=expected_nb_origins)
+
+ # test page parsing
+ scheduler_origins = swh_scheduler.get_listed_origins(
+ lister_cgit.lister_obj.id
+ ).results
+ assert len(scheduler_origins) == expected_nb_origins
+
+ # test listed repositories
+ for listed_origin in scheduler_origins:
+ assert listed_origin.visit_type == "git"
+ assert listed_origin.url.startswith(base_git_url)
+ assert (
+ listed_origin.url.startswith(url) is False
+ ), f"url should be mapped to {base_git_url}"
diff --git a/swh/lister/cgit/tests/test_tasks.py b/swh/lister/cgit/tests/test_tasks.py
index f348221..b9a00cd 100644
--- a/swh/lister/cgit/tests/test_tasks.py
+++ b/swh/lister/cgit/tests/test_tasks.py
@@ -1,35 +1,35 @@
# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.lister.pattern import ListerStats
def test_cgit_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
res = swh_scheduler_celery_app.send_task("swh.lister.cgit.tasks.ping")
assert res
res.wait()
assert res.successful()
assert res.result == "OK"
def test_cgit_lister_task(
swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker
):
# setup the mocked CGitLister
lister = mocker.patch("swh.lister.cgit.tasks.CGitLister")
lister.from_configfile.return_value = lister
lister.run.return_value = ListerStats(pages=10, origins=500)
- kwargs = dict(url="https://git.kernel.org/", instance="kernel")
+ kwargs = dict(url="https://git.kernel.org/", instance="kernel", base_git_url=None)
res = swh_scheduler_celery_app.send_task(
"swh.lister.cgit.tasks.CGitListerTask", kwargs=kwargs,
)
assert res
res.wait()
assert res.successful()
lister.from_configfile.assert_called_once_with(**kwargs)
lister.run.assert_called_once_with()

File Metadata

Mime Type
text/x-diff
Expires
Sat, Jun 21, 6:03 PM (1 w, 6 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3277016

Event Timeline

About · Developer information · Legal