Differential D5653 Diff 20199 swh/web/tests/common/test_origin_save.py

Changeset View

Standalone View

swh/web/tests/common/test_origin_save.py

# Copyright (C) 2019-2021 The Software Heritage developers		# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from datetime import datetime, timedelta, timezone		from datetime import datetime, timedelta, timezone
from functools import partial		from functools import partial
import re		import re
from typing import Optional		from typing import Optional

import pytest		import pytest
import requests		import requests

from swh.core.pytest_plugin import get_response_cb		from swh.core.pytest_plugin import get_response_cb
		from swh.web.common.exc import BadInputExc
from swh.web.common.models import (		from swh.web.common.models import (
SAVE_REQUEST_ACCEPTED,		SAVE_REQUEST_ACCEPTED,
SAVE_TASK_FAILED,		SAVE_TASK_FAILED,
SAVE_TASK_RUNNING,		SAVE_TASK_RUNNING,
SAVE_TASK_SCHEDULED,		SAVE_TASK_SCHEDULED,
SAVE_TASK_SUCCEEDED,		SAVE_TASK_SUCCEEDED,
VISIT_STATUS_FULL,		VISIT_STATUS_FULL,
SaveOriginRequest,		SaveOriginRequest,
)		)
from swh.web.common.origin_save import (		from swh.web.common.origin_save import (
		_check_origin_exists,
get_save_origin_requests,		get_save_origin_requests,
get_save_origin_task_info,		get_save_origin_task_info,
		origin_exists,
refresh_save_origin_request_statuses,		refresh_save_origin_request_statuses,
)		)
from swh.web.common.typing import OriginVisitInfo, SaveOriginRequestInfo		from swh.web.common.typing import (
		OriginExistenceCheckInfo,
		OriginVisitInfo,
		SaveOriginRequestInfo,
		)
from swh.web.config import get_config		from swh.web.config import get_config

_es_url = "http://esnode1.internal.softwareheritage.org:9200"		_es_url = "http://esnode1.internal.softwareheritage.org:9200"
_es_workers_index_url = "%s/swh_workers-*" % _es_url		_es_workers_index_url = "%s/swh_workers-*" % _es_url

_origin_url = "https://gitlab.com/inkscape/inkscape"		_origin_url = "https://gitlab.com/inkscape/inkscape"
_visit_type = "git"		_visit_type = "git"
_task_id = 203525448		_task_id = 203525448
▲ Show 20 Lines • Show All 269 Lines • ▼ Show 20 Lines	assert sor.to_dict() == SaveOriginRequestInfo(
save_request_status=sor.status,		save_request_status=sor.status,
save_task_status=sor.loading_task_status,		save_task_status=sor.loading_task_status,
visit_status=sor.visit_status,		visit_status=sor.visit_status,
visit_date=_visit_date.isoformat() if _visit_date else None,		visit_date=_visit_date.isoformat() if _visit_date else None,
loading_task_id=sor.loading_task_id,		loading_task_id=sor.loading_task_id,
)		)


		def test_origin_exists_404(requests_mock):
		"""Origin which does not exist should be reported as inexistent"""
		url_ko = "https://example.org/some-inexistant-url"
		requests_mock.head(url_ko, status_code=404)

		actual_result = origin_exists(url_ko)
		assert actual_result == OriginExistenceCheckInfo(
		origin_url=url_ko, exists=False, last_modified=None, content_length=None,
		)

		with pytest.raises(BadInputExc, match="not exist"):
		_check_origin_exists(url_ko)


		def test_origin_exists_200_no_data(requests_mock):
		anlambertUnsubmitted Not Done Inline Actions Comment is not correct here. anlambert: Comment is not correct here.
		"""Existing origin should be reported as such (no extra information)"""
		url = "http://example.org/real-url"
		requests_mock.head(
		url, status_code=200,
		)

		actual_result = origin_exists(url)
		assert actual_result == OriginExistenceCheckInfo(
		origin_url=url, exists=True, last_modified=None, content_length=None,
		)

		# passes the check
		_check_origin_exists(url)


		def test_origin_exists_200_with_data(requests_mock):
		anlambertUnsubmitted Not Done Inline Actions same here anlambert: same here
		"""Existing origin should be reported as such (+ extra information)"""
		url = "http://example.org/real-url"
		requests_mock.head(
		url,
		status_code=200,
		headers={
		"content-length": "10",
		"last-modified": "Sun, 21 Aug 2011 16:26:32 GMT",
		},
		)

		actual_result = origin_exists(url)
		assert actual_result == OriginExistenceCheckInfo(
		origin_url=url,
		exists=True,
		content_length=10,
		last_modified="Sun, 21 Aug 2011 16:26:32 GMT",
		)

		# passes the check
		_check_origin_exists(url)


@pytest.mark.django_db		@pytest.mark.django_db
@pytest.mark.parametrize("visit_status", ["created", "ongoing",])		@pytest.mark.parametrize("visit_status", ["created", "ongoing",])
def test_get_save_origin_requests_no_visit_date_found(mocker, visit_status):		def test_get_save_origin_requests_no_visit_date_found(mocker, visit_status):
"""Uneventful visits with failed visit status are marked as failed		"""Uneventful visits with failed visit status are marked as failed

"""		"""
sors = _get_save_origin_requests(		sors = _get_save_origin_requests(
mocker, load_status="scheduled", visit_status=visit_status,		mocker, load_status="scheduled", visit_status=visit_status,
▲ Show 20 Lines • Show All 102 Lines • Show Last 20 Lines