Differential D4005 Diff 14138 swh/web/tests/common/test_origin_save.py

Changeset View

Standalone View

swh/web/tests/common/test_origin_save.py

# Copyright (C) 2019-2020 The Software Heritage developers		# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from datetime import datetime, timedelta, timezone		from datetime import datetime, timedelta, timezone
from functools import partial		from functools import partial
import re		import re

import pytest		import pytest
import requests		import requests

from swh.core.pytest_plugin import get_response_cb		from swh.core.pytest_plugin import get_response_cb
from swh.web.common.models import SaveOriginRequest		from swh.web.common.models import (
		SAVE_REQUEST_ACCEPTED,
		SAVE_TASK_FAILED,
		SAVE_TASK_RUNNING,
		SAVE_TASK_SUCCEEDED,
		SaveOriginRequest,
		)
from swh.web.common.origin_save import (		from swh.web.common.origin_save import (
get_save_origin_requests,		get_save_origin_requests,
get_save_origin_task_info,		get_save_origin_task_info,
)		)
from swh.web.common.typing import OriginVisitInfo		from swh.web.common.typing import OriginVisitInfo
from swh.web.config import get_config		from swh.web.config import get_config

_es_url = "http://esnode1.internal.softwareheritage.org:9200"		_es_url = "http://esnode1.internal.softwareheritage.org:9200"
Show All 27 Lines	def test_get_save_origin_task_info_with_es(mocker):
_get_save_origin_task_info_test(mocker, es_available=True, full_info=False)		_get_save_origin_task_info_test(mocker, es_available=True, full_info=False)


@pytest.mark.django_db		@pytest.mark.django_db
def test_get_save_origin_task_info_without_es(mocker):		def test_get_save_origin_task_info_without_es(mocker):
_get_save_origin_task_info_test(mocker, es_available=False)		_get_save_origin_task_info_test(mocker, es_available=False)


def _mock_scheduler(mocker, task_status="succeeded", task_archived=False):		def _mock_scheduler(mocker, task_status="completed", task_archived=False):
mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler")		mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler")
task = (		task = (
{		{
"arguments": {"args": [], "kwargs": {"repo_url": _origin_url},},		"arguments": {"args": [], "kwargs": {"repo_url": _origin_url},},
"current_interval": timedelta(days=64),		"current_interval": timedelta(days=64),
"id": _task_id,		"id": _task_id,
"next_run": datetime.now(tz=timezone.utc) + timedelta(days=64),		"next_run": datetime.now(tz=timezone.utc) + timedelta(days=64),
"policy": "oneshot",		"policy": "oneshot",
"priority": "high",		"priority": "high",
"retries_left": 0,		"retries_left": 0,
"status": "disabled",		"status": task_status,
"type": "load-git",		"type": "load-git",
}		}
if not task_archived		if not task_archived
else None		else None
)		)
mock_scheduler.get_tasks.return_value = [dict(task) if task else None]		mock_scheduler.get_tasks.return_value = [dict(task) if task else None]

task_run = {		task_run = {
Show All 19 Lines	if es_available:
swh_web_config.update({"es_workers_index_url": _es_workers_index_url})		swh_web_config.update({"es_workers_index_url": _es_workers_index_url})
else:		else:
swh_web_config.update({"es_workers_index_url": ""})		swh_web_config.update({"es_workers_index_url": ""})

sor = SaveOriginRequest.objects.create(		sor = SaveOriginRequest.objects.create(
request_date=datetime.now(tz=timezone.utc),		request_date=datetime.now(tz=timezone.utc),
visit_type=_visit_type,		visit_type=_visit_type,
origin_url="https://gitlab.com/inkscape/inkscape",		origin_url="https://gitlab.com/inkscape/inkscape",
status="accepted",		status=SAVE_REQUEST_ACCEPTED,
visit_date=datetime.now(tz=timezone.utc) + timedelta(hours=1),		visit_date=datetime.now(tz=timezone.utc) + timedelta(hours=1),
loading_task_id=_task_id,		loading_task_id=_task_id,
)		)

task, task_run = _mock_scheduler(mocker, task_archived=task_archived)		task, task_run = _mock_scheduler(mocker, task_archived=task_archived)

es_response = requests.post("%s/_search" % _es_workers_index_url).json()		es_response = requests.post("%s/_search" % _es_workers_index_url).json()

▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines

@pytest.mark.django_db		@pytest.mark.django_db
def test_get_save_origin_requests_find_visit_date(mocker):		def test_get_save_origin_requests_find_visit_date(mocker):
# create a save request		# create a save request
SaveOriginRequest.objects.create(		SaveOriginRequest.objects.create(
request_date=datetime.now(tz=timezone.utc),		request_date=datetime.now(tz=timezone.utc),
visit_type=_visit_type,		visit_type=_visit_type,
origin_url=_origin_url,		origin_url=_origin_url,
status="accepted",		status=SAVE_REQUEST_ACCEPTED,
visit_date=None,		visit_date=None,
loading_task_id=_task_id,		loading_task_id=_task_id,
)		)

# mock scheduler and services		# mock scheduler and services
_mock_scheduler(mocker)		_mock_scheduler(mocker)
mock_service = mocker.patch("swh.web.common.origin_save.service")		mock_service = mocker.patch("swh.web.common.origin_save.service")
mock_service.lookup_origin.return_value = {"url": _origin_url}		mock_service.lookup_origin.return_value = {"url": _origin_url}
Show All 13 Lines	visit_info = OriginVisitInfo(
url="",		url="",
visit=34,		visit=34,
)		)
mock_get_origin_visits.return_value = [visit_info]		mock_get_origin_visits.return_value = [visit_info]

# check visit date has been correctly found		# check visit date has been correctly found
sors = get_save_origin_requests(_visit_type, _origin_url)		sors = get_save_origin_requests(_visit_type, _origin_url)
assert len(sors) == 1		assert len(sors) == 1
		assert sors[0]["save_task_status"] == SAVE_TASK_SUCCEEDED
assert sors[0]["visit_date"] == visit_date		assert sors[0]["visit_date"] == visit_date
mock_get_origin_visits.assert_called_once()		mock_get_origin_visits.assert_called_once()

# check visit is not searched again when it has been found		# check visit is not searched again when it has been found
get_save_origin_requests(_visit_type, _origin_url)		get_save_origin_requests(_visit_type, _origin_url)
mock_get_origin_visits.assert_called_once()		mock_get_origin_visits.assert_called_once()

# check visit date are not searched for save requests older than		# check visit date are not searched for save requests older than
# one month		# one month
sor = SaveOriginRequest.objects.create(		sor = SaveOriginRequest.objects.create(
visit_type=_visit_type,		visit_type=_visit_type,
origin_url=_origin_url,		origin_url=_origin_url,
status="accepted",		status=SAVE_REQUEST_ACCEPTED,
loading_task_id=_task_id,		loading_task_id=_task_id,
visit_date=None,		visit_date=None,
)		)
sor.request_date = datetime.now(tz=timezone.utc) - timedelta(days=31)		sor.request_date = datetime.now(tz=timezone.utc) - timedelta(days=31)
sor.save()		sor.save()

_mock_scheduler(mocker, task_status="failed")		_mock_scheduler(mocker, task_status="disabled")

sors = get_save_origin_requests(_visit_type, _origin_url)		sors = get_save_origin_requests(_visit_type, _origin_url)

assert len(sors) == 2		assert len(sors) == 2
		assert sors[0]["save_task_status"] == SAVE_TASK_FAILED
assert sors[0]["visit_date"] is None		assert sors[0]["visit_date"] is None
mock_get_origin_visits.assert_called_once()		mock_get_origin_visits.assert_called_once()


@pytest.mark.django_db		@pytest.mark.django_db
def test_get_save_origin_requests_no_visit_date_found(mocker):		def test_get_save_origin_requests_no_visit_date_found(mocker):
# create a save request		# create a save request
SaveOriginRequest.objects.create(		SaveOriginRequest.objects.create(
request_date=datetime.now(tz=timezone.utc),		request_date=datetime.now(tz=timezone.utc),
visit_type=_visit_type,		visit_type=_visit_type,
origin_url=_origin_url,		origin_url=_origin_url,
status="accepted",		status=SAVE_REQUEST_ACCEPTED,
visit_date=None,		visit_date=None,
loading_task_id=_task_id,		loading_task_id=_task_id,
)		)

# mock scheduler and services		# mock scheduler and services
_mock_scheduler(mocker)		_mock_scheduler(mocker, task_status="next_run_scheduled")
mock_service = mocker.patch("swh.web.common.origin_save.service")		mock_service = mocker.patch("swh.web.common.origin_save.service")
mock_service.lookup_origin.return_value = {"url": _origin_url}		mock_service.lookup_origin.return_value = {"url": _origin_url}
mock_get_origin_visits = mocker.patch(		mock_get_origin_visits = mocker.patch(
"swh.web.common.origin_save.get_origin_visits"		"swh.web.common.origin_save.get_origin_visits"
)		)
# create a visit for the save request with status created		# create a visit for the save request with status created
visit_date = datetime.now(tz=timezone.utc).isoformat()		visit_date = datetime.now(tz=timezone.utc).isoformat()
visit_info = OriginVisitInfo(		visit_info = OriginVisitInfo(
date=visit_date,		date=visit_date,
formatted_date="",		formatted_date="",
metadata={},		metadata={},
origin=_origin_url,		origin=_origin_url,
snapshot=None,		snapshot=None,
status="created",		status="created",
type=_visit_type,		type=_visit_type,
url="",		url="",
visit=34,		visit=34,
)		)
mock_get_origin_visits.return_value = [visit_info]		mock_get_origin_visits.return_value = [visit_info]

# check no visit date has been found		# check no visit date has been found
sors = get_save_origin_requests(_visit_type, _origin_url)		sors = get_save_origin_requests(_visit_type, _origin_url)
assert len(sors) == 1		assert len(sors) == 1
		assert sors[0]["save_task_status"] == SAVE_TASK_RUNNING
assert sors[0]["visit_date"] is None		assert sors[0]["visit_date"] is None
mock_get_origin_visits.assert_called_once()		mock_get_origin_visits.assert_called_once()