Differential D5896 Diff 21275 swh/web/tests/common/test_origin_save.py

Changeset View

Standalone View

swh/web/tests/common/test_origin_save.py

Show All 14 Lines
from swh.core.pytest_plugin import get_response_cb		from swh.core.pytest_plugin import get_response_cb
from swh.web.common.exc import BadInputExc		from swh.web.common.exc import BadInputExc
from swh.web.common.models import (		from swh.web.common.models import (
SAVE_REQUEST_ACCEPTED,		SAVE_REQUEST_ACCEPTED,
SAVE_TASK_FAILED,		SAVE_TASK_FAILED,
SAVE_TASK_RUNNING,		SAVE_TASK_RUNNING,
SAVE_TASK_SCHEDULED,		SAVE_TASK_SCHEDULED,
SAVE_TASK_SUCCEEDED,		SAVE_TASK_SUCCEEDED,
		VISIT_STATUS_CREATED,
VISIT_STATUS_FULL,		VISIT_STATUS_FULL,
SaveOriginRequest,		SaveOriginRequest,
)		)
from swh.web.common.origin_save import (		from swh.web.common.origin_save import (
_check_origin_exists,		_check_origin_exists,
_check_visit_type_savable,		_check_visit_type_savable,
_visit_type_task,		_visit_type_task,
_visit_type_task_privileged,		_visit_type_task_privileged,
▲ Show 20 Lines • Show All 444 Lines • ▼ Show 20 Lines	def test_get_save_origin_requests_no_visit_date_found(mocker, visit_status):

"""		"""
sors = _get_save_origin_requests(		sors = _get_save_origin_requests(
mocker, load_status="scheduled", visit_status=visit_status,		mocker, load_status="scheduled", visit_status=visit_status,
)		)
# check no visit date has been found		# check no visit date has been found
assert len(sors) == 1		assert len(sors) == 1
assert sors[0]["save_task_status"] == SAVE_TASK_RUNNING		assert sors[0]["save_task_status"] == SAVE_TASK_RUNNING
assert sors[0]["visit_date"] is None		assert sors[0]["visit_date"] is not None
assert sors[0]["visit_status"] == visit_status		assert sors[0]["visit_status"] == visit_status


@pytest.mark.django_db		@pytest.mark.django_db
@pytest.mark.parametrize("visit_status", ["not_found", "failed",])		@pytest.mark.parametrize("visit_status", ["not_found", "failed",])
def test_get_save_origin_requests_no_failed_status_override(mocker, visit_status):		def test_get_save_origin_requests_no_failed_status_override(mocker, visit_status):
"""Uneventful visits with failed statuses (failed, not found) are marked as failed		"""Uneventful visits with failed statuses (failed, not found) are marked as failed

"""		"""
sors = _get_save_origin_requests(		sors = _get_save_origin_requests(
mocker, load_status="uneventful", visit_status=visit_status		mocker, load_status="uneventful", visit_status=visit_status
)		)
assert len(sors) == 1		assert len(sors) == 1

assert sors[0]["save_task_status"] == SAVE_TASK_FAILED		assert sors[0]["save_task_status"] == SAVE_TASK_FAILED
visit_date = sors[0]["visit_date"]		visit_date = sors[0]["visit_date"]
if visit_status == "failed":
assert visit_date is None
else:
assert visit_date is not None		assert visit_date is not None

sors = get_save_origin_requests(_visit_type, _origin_url)		sors = get_save_origin_requests(_visit_type, _origin_url)
assert len(sors) == 1		assert len(sors) == 1
assert sors[0]["save_task_status"] == SAVE_TASK_FAILED		assert sors[0]["save_task_status"] == SAVE_TASK_FAILED
assert sors[0]["visit_status"] == visit_status		assert sors[0]["visit_status"] == visit_status


@pytest.mark.django_db		@pytest.mark.django_db
Show All 25 Lines	def test_get_visit_info_incomplete_visit_still_successful(mocker, load_status):
"""		"""
sors = _get_save_origin_requests(		sors = _get_save_origin_requests(
mocker, load_status=load_status, visit_status=None,		mocker, load_status=load_status, visit_status=None,
)		)
assert len(sors) == 1		assert len(sors) == 1

assert sors[0]["save_task_status"] == SAVE_TASK_SUCCEEDED		assert sors[0]["save_task_status"] == SAVE_TASK_SUCCEEDED
# As the entry is missing the following information though		# As the entry is missing the following information though
assert sors[0]["visit_date"] is None		assert sors[0]["visit_date"] is not None
assert sors[0]["visit_status"] is None		assert sors[0]["visit_status"] is None

# It's still detected as to be updated by the refresh routine		# It's still detected as to be updated by the refresh routine
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()
assert len(sors) == 1		assert len(sors) == 1

assert sors[0]["save_task_status"] == SAVE_TASK_SUCCEEDED		assert sors[0]["save_task_status"] == SAVE_TASK_SUCCEEDED
assert sors[0]["visit_date"] is None		assert sors[0]["visit_date"] is not None
assert sors[0]["visit_status"] is None		assert sors[0]["visit_status"] is None


@pytest.mark.django_db		@pytest.mark.django_db
def test_refresh_in_progress_save_request_statuses(mocker, api_client, archive_data):		def test_refresh_in_progress_save_request_statuses(mocker, api_client, archive_data):
"""Refresh a pending save origins requests and update if the status changes		"""Refresh a pending save origins requests and update if the status changes
"""		"""
date_now = datetime.now(tz=timezone.utc)		date_now = datetime.now(tz=timezone.utc)
date_pivot = date_now - timedelta(days=30)		date_pivot = date_now - timedelta(days=30)
visit_started_date = date_now - timedelta(minutes=1)		visit_started_date = date_now - timedelta(minutes=1)

# returned visit status		# returned visit status
sors = _get_save_origin_requests(		SaveOriginRequest.objects.create(
mocker, load_status=SAVE_TASK_SCHEDULED, visit_status="created",		request_date=datetime.now(tz=timezone.utc),
		visit_type=_visit_type,
		visit_status=VISIT_STATUS_CREATED,
		origin_url=_origin_url,
		status=SAVE_REQUEST_ACCEPTED,
		visit_date=None,
		loading_task_id=_task_id,
)		)
assert len(sors) == 1
		# mock scheduler and archives
		_mock_scheduler(
		mocker, task_status="next_run_scheduled", task_run_status=SAVE_TASK_SCHEDULED
		)
		mock_archive = mocker.patch("swh.web.common.origin_save.archive")
		mock_archive.lookup_origin.return_value = {"url": _origin_url}
		mock_get_origin_visits = mocker.patch(
		"swh.web.common.origin_save.get_origin_visits"
		)
		# create a visit for the save request with status created
		visit_date = datetime.now(tz=timezone.utc).isoformat()
		visit_info = OriginVisitInfo(
		date=visit_date,
		formatted_date="",
		metadata={},
		origin=_origin_url,
		snapshot="", # make mypy happy
		status=VISIT_STATUS_CREATED,
		type=_visit_type,
		url="",
		visit=34,
		)
		mock_get_origin_visits.return_value = [visit_info]

# make the scheduler return a running event		# make the scheduler return a running event
_mock_scheduler(		_mock_scheduler(
mocker,		mocker,
task_status="next_run_scheduled",		task_status="next_run_scheduled",
task_run_status="started",		task_run_status="started",
visit_started_date=visit_started_date,		visit_started_date=visit_started_date,
)		)

# The visit is detected but still running		# The visit is detected but still running
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()

		assert mock_get_origin_visits.called and mock_get_origin_visits.call_count == 1
assert len(sors) == 1		assert len(sors) == 1

for sor in sors:		for sor in sors:
assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot		assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot
# The status is updated		# The status is updated
assert sor["save_task_status"] == SAVE_TASK_RUNNING		assert sor["save_task_status"] == SAVE_TASK_RUNNING
# but the following entries are missing so it's not updated		# but the following entries are missing so it's not updated
assert sor["visit_date"] is None		assert sor["visit_date"] is not None
assert sor["visit_status"] == "created"		assert sor["visit_status"] == VISIT_STATUS_CREATED

# make the visit status completed		# make the visit status completed
# make the scheduler return a running event		# make the scheduler return a running event
_mock_scheduler(		_mock_scheduler(
mocker,		mocker,
task_status="completed",		task_status="completed",
task_run_status="eventful",		task_run_status="eventful",
visit_started_date=visit_started_date,		visit_started_date=visit_started_date,
)		)

# This time around, the origin returned will have all information updated		# This time around, the origin returned will have all required information updated
mock_get_origin_visits = mocker.patch(		# (visit date and visit status in final state)
"swh.web.common.origin_save.get_origin_visits"
)
# create a visit for the save request with status created
visit_date = datetime.now(tz=timezone.utc).isoformat()		visit_date = datetime.now(tz=timezone.utc).isoformat()
		anlambertUnsubmitted Done Inline Actions To remove, no ? anlambert: To remove, no ?
		ardumontAuthorUnsubmitted Done Inline Actions quite ;) ardumont: quite ;)
visit_info = OriginVisitInfo(		visit_info.update({"date": visit_date, "status": VISIT_STATUS_FULL})
date=visit_date,
formatted_date="",
metadata={},
origin=_origin_url,
snapshot="", # make mypy happy
status="full",
type=_visit_type,
url="",
visit=34,
)
mock_get_origin_visits.return_value = [visit_info]		mock_get_origin_visits.return_value = [visit_info]

# Detected entry, this time it should be updated		# Detected entry, this time it should be updated
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()
assert len(sors) == 1		assert len(sors) == 1

		assert mock_get_origin_visits.called and mock_get_origin_visits.call_count == 1 + 1

for sor in sors:		for sor in sors:
assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot		assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot
# as it turns out, in this test, this won't update anything as no new status got		# as it turns out, in this test, this won't update anything as no new status got
# returned by the scheduler		# returned by the scheduler
assert sor["save_task_status"] == SAVE_TASK_SUCCEEDED		assert sor["save_task_status"] == SAVE_TASK_SUCCEEDED
assert sor["visit_date"] == visit_date		assert sor["visit_date"] == visit_date
assert sor["visit_status"] == "full"		assert sor["visit_status"] == VISIT_STATUS_FULL

# Once in final state, a sor should not be updated anymore		# Once in final state, a sor should not be updated anymore
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()
assert len(sors) == 0		assert len(sors) == 0


@pytest.mark.django_db		@pytest.mark.django_db
def test_refresh_save_request_statuses(mocker, api_client, archive_data):		def test_refresh_save_request_statuses(mocker, api_client, archive_data):
"""Refresh filters save origins requests and update if changes		"""Refresh filters save origins requests and update if changes

"""		"""
date_now = datetime.now(tz=timezone.utc)		date_now = datetime.now(tz=timezone.utc)
date_pivot = date_now - timedelta(days=30)		date_pivot = date_now - timedelta(days=30)
# returned visit status		# returned visit status
sors = _get_save_origin_requests(		SaveOriginRequest.objects.create(
mocker, load_status=SAVE_TASK_SCHEDULED, visit_status=None,		request_date=datetime.now(tz=timezone.utc),
		visit_type=_visit_type,
		visit_status=None,
		origin_url=_origin_url,
		status=SAVE_REQUEST_ACCEPTED,
		visit_date=None,
		loading_task_id=_task_id,
)		)
assert len(sors) == 1
		# mock scheduler and archives
		_mock_scheduler(
		mocker, task_status="next_run_scheduled", task_run_status=SAVE_TASK_SCHEDULED
		)
		mock_archive = mocker.patch("swh.web.common.origin_save.archive")
		mock_archive.lookup_origin.return_value = {"url": _origin_url}
		mock_get_origin_visits = mocker.patch(
		"swh.web.common.origin_save.get_origin_visits"
		)
		# create a visit for the save request with status created
		visit_date = datetime.now(tz=timezone.utc).isoformat()
		visit_info = OriginVisitInfo(
		date=visit_date,
		formatted_date="",
		metadata={},
		origin=_origin_url,
		snapshot="", # make mypy happy
		status=VISIT_STATUS_CREATED,
		type=_visit_type,
		url="",
		visit=34,
		)
		mock_get_origin_visits.return_value = [visit_info]

# no changes so refresh does detect the entry but does nothing		# no changes so refresh does detect the entry but does nothing
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()
assert len(sors) == 1		assert len(sors) == 1

for sor in sors:		for sor in sors:
assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot		assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot
# as it turns out, in this test, this won't update anything as no new status got		# as it turns out, in this test, this won't update anything as no new status got
# returned by the scheduler		# returned by the scheduler
assert sor["save_task_status"] == SAVE_TASK_SCHEDULED		assert sor["save_task_status"] == SAVE_TASK_RUNNING
# Information is empty		# Information is empty
assert sor["visit_date"] is None		assert sor["visit_date"] == visit_date
assert sor["visit_status"] is None		assert sor["visit_status"] == VISIT_STATUS_CREATED

# make the scheduler return eventful event for that origin		# A save code now entry is detected for update, but as nothing changes, the entry
_mock_scheduler(mocker)		# remains in the same state
		anlambertUnsubmitted Not Done Inline Actions s/And/An/ anlambert: s/And/An/
# updates will be detected, entry should be updated but we are still missing info
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()
assert len(sors) == 1		assert len(sors) == 1

for sor in sors:		for sor in sors:
assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot		assert iso8601.parse_date(sor["save_request_date"]) >= date_pivot
# The status is updated		# Status is not updated as no new information is available on the visit status
assert sor["save_task_status"] == SAVE_TASK_SUCCEEDED		# and the task status has not moved
# but the following entries are missing so it's not updated		assert sor["save_task_status"] == SAVE_TASK_RUNNING
assert sor["visit_date"] is None		# Information is empty
assert sor["visit_status"] is None		assert sor["visit_date"] == visit_date
		assert sor["visit_status"] == VISIT_STATUS_CREATED

# This time around, the origin returned will have all information updated		# This time around, the origin returned will have all information updated
mock_get_origin_visits = mocker.patch(
"swh.web.common.origin_save.get_origin_visits"
)
# create a visit for the save request with status created		# create a visit for the save request with status created
visit_date = datetime.now(tz=timezone.utc).isoformat()		visit_date = datetime.now(tz=timezone.utc).isoformat()
visit_info = OriginVisitInfo(		visit_info = OriginVisitInfo(
date=visit_date,		date=visit_date,
formatted_date="",		formatted_date="",
metadata={},		metadata={},
origin=_origin_url,		origin=_origin_url,
snapshot="", # make mypy happy		snapshot="", # make mypy happy
status="full",		status=VISIT_STATUS_FULL,
type=_visit_type,		type=_visit_type,
url="",		url="",
visit=34,		visit=34,
)		)
mock_get_origin_visits.return_value = [visit_info]		mock_get_origin_visits.return_value = [visit_info]

# Detected entry, this time it should be updated		# Detected entry, this time it should be updated
sors = refresh_save_origin_request_statuses()		sors = refresh_save_origin_request_statuses()
Show All 13 Lines