diff --git a/swh/lister/cgit/lister.py b/swh/lister/cgit/lister.py --- a/swh/lister/cgit/lister.py +++ b/swh/lister/cgit/lister.py @@ -28,12 +28,18 @@ This lister will retrieve the list of published git repositories by parsing the HTML page(s) of the index retrieved at `url`. - For each found git repository, a query is made at the given url found - in this index to gather published "Clone" URLs to be used as origin - URL for that git repo. + The lister currently defines 2 listing behaviors: + + - If the `base_git_url` is provided, the listed origin urls are computed out of the + base git url link and the one listed in the main listed page (resulting in less + queries than the 2nd behavior below). This is expected to be the main behavior + used. + + - Otherwise, with no `base_git_url`, for each found git repository, a query is made + at the given url found in this index to gather published "Clone" URLs to be used + as origin URL for that git repo. If several "Clone" urls are provided, prefer the + http/https one, if any, otherwise fallback to the first one. - If several "Clone" urls are provided, prefer the http/https one, if - any, otherwise fallback to the first one. """ LISTER_NAME = "cgit" @@ -44,6 +50,7 @@ url: str, instance: Optional[str] = None, credentials: Optional[CredentialsType] = None, + base_git_url: Optional[str] = None, ): """Lister class for CGit repositories. @@ -66,6 +73,7 @@ self.session.headers.update( {"Accept": "application/html", "User-Agent": USER_AGENT} ) + self.base_git_url = base_git_url def _get_and_parse(self, url: str) -> BeautifulSoup: """Get the given url and parse the retrieved HTML using BeautifulSoup""" @@ -87,7 +95,14 @@ for tr in bs_idx.find("div", {"class": "content"}).find_all( "tr", {"class": ""} ): - url = urljoin(self.url, tr.find("a")["href"]) + repository_link = tr.find("a")["href"] + repo_url = None + git_url = None + + if self.base_git_url: # mapping provided, we compute the git url + git_url = urljoin(self.base_git_url, repository_link) + else: # mapping not provided, we compute the url as before + repo_url = urljoin(self.url, repository_link) span = tr.find("span", {"class": re.compile("age-")}) if span: last_updated_date = span["title"] @@ -95,7 +110,11 @@ last_updated_date = None page_results.append( - {"url": url, "last_updated_date": last_updated_date} + { + "url": repo_url, + "git_url": git_url, + "last_updated_date": last_updated_date, + } ) yield page_results @@ -114,11 +133,17 @@ def get_origins_from_page( self, repositories: Repositories ) -> Iterator[ListedOrigin]: - """Convert a page of cgit repositories into a list of ListedOrigins.""" + """Convert a page of cgit repositories into a list of ListedOrigins. + + If the base_git_url is not set, this will try and parse the extra + + """ assert self.lister_obj.id is not None - for repository in repositories: - origin_url = self._get_origin_from_repository_url(repository["url"]) + for repo in repositories: + origin_url = repo["git_url"] or self._get_origin_from_repository_url( + repo["url"] + ) if origin_url is None: continue @@ -126,7 +151,7 @@ lister_id=self.lister_obj.id, url=origin_url, visit_type="git", - last_update=_parse_last_updated_date(repository), + last_update=_parse_last_updated_date(repo), ) def _get_origin_from_repository_url(self, repository_url: str) -> Optional[str]: diff --git a/swh/lister/cgit/tests/data/https_git.eclipse.org/c b/swh/lister/cgit/tests/data/https_git.eclipse.org/c new file mode 100644 --- /dev/null +++ b/swh/lister/cgit/tests/data/https_git.eclipse.org/c @@ -0,0 +1,42 @@ + + + + Eclipse Git repositories + + + + + + + + + +
+
+
+
+
+

+ Back to the top +

+ + diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -196,3 +196,28 @@ lister = CGitLister.from_configfile() assert lister.scheduler is not None assert lister.credentials is not None + + +def test_lister_cgit_with_base_git_url(requests_mock_datadir, swh_scheduler): + """With base git url provided, listed urls should be the computed origin urls + + """ + url = "https://git.eclipse.org/c" + base_git_url = "https://eclipse.org/c" + lister_cgit = CGitLister(swh_scheduler, url=url, base_git_url=base_git_url,) + + stats = lister_cgit.run() + + expected_nb_origins = 5 # 5 "sublevel-section" class in html + assert stats == ListerStats(pages=1, origins=expected_nb_origins) + + # test page parsing + scheduler_origins = swh_scheduler.get_listed_origins( + lister_cgit.lister_obj.id + ).results + assert len(scheduler_origins) == expected_nb_origins + + # test listed repositories + for listed_origin in scheduler_origins: + assert listed_origin.visit_type == "git" + assert listed_origin.url.startswith(base_git_url)