Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/sourceforge/tests/test_lister.py
# Copyright (C) 2021-2022 The Software Heritage developers | # Copyright (C) 2021-2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import datetime | import datetime | ||||
import functools | import functools | ||||
import json | import json | ||||
from pathlib import Path | from pathlib import Path | ||||
import re | import re | ||||
from iso8601 import iso8601 | from iso8601 import iso8601 | ||||
import pytest | import pytest | ||||
from requests.exceptions import HTTPError | from requests.exceptions import HTTPError | ||||
from swh.lister import USER_AGENT | from swh.lister import USER_AGENT_TEMPLATE | ||||
from swh.lister.sourceforge.lister import ( | from swh.lister.sourceforge.lister import ( | ||||
MAIN_SITEMAP_URL, | MAIN_SITEMAP_URL, | ||||
PROJECT_API_URL_FORMAT, | PROJECT_API_URL_FORMAT, | ||||
SourceForgeLister, | SourceForgeLister, | ||||
SourceForgeListerState, | SourceForgeListerState, | ||||
) | ) | ||||
from swh.lister.tests.test_utils import assert_sleep_calls | from swh.lister.tests.test_utils import assert_sleep_calls | ||||
from swh.lister.utils import WAIT_EXP_BASE | from swh.lister.utils import WAIT_EXP_BASE | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | def get_cvs_info_page(datadir): | ||||
return Path(datadir, "aaron.html").read_text() | return Path(datadir, "aaron.html").read_text() | ||||
def get_bzr_repo_page(datadir, repo_name): | def get_bzr_repo_page(datadir, repo_name): | ||||
return Path(datadir, f"{repo_name}.html").read_text() | return Path(datadir, f"{repo_name}.html").read_text() | ||||
def _check_request_headers(request): | def _check_request_headers(request): | ||||
return request.headers.get("User-Agent") == USER_AGENT | return ( | ||||
request.headers.get("User-Agent") | |||||
== USER_AGENT_TEMPLATE % SourceForgeLister.LISTER_NAME | |||||
) | |||||
def _check_listed_origins(lister, swh_scheduler): | def _check_listed_origins(lister, swh_scheduler): | ||||
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results | scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results | ||||
res = {o.url: (o.visit_type, str(o.last_update.date())) for o in scheduler_origins} | res = {o.url: (o.visit_type, str(o.last_update.date())) for o in scheduler_origins} | ||||
assert res == { | assert res == { | ||||
"https://svn.code.sf.net/p/backapps/website/code": ("svn", "2021-02-11"), | "https://svn.code.sf.net/p/backapps/website/code": ("svn", "2021-02-11"), | ||||
"https://git.code.sf.net/p/os3dmodels/git": ("git", "2017-03-31"), | "https://git.code.sf.net/p/os3dmodels/git": ("git", "2017-03-31"), | ||||
▲ Show 20 Lines • Show All 451 Lines • Show Last 20 Lines |