Changeset View
Changeset View
Standalone View
Standalone View
swh/web/tests/api/views/test_origin_save.py
# Copyright (C) 2018-2019 The Software Heritage developers | # Copyright (C) 2018-2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import pytest | |||||
from datetime import datetime, timedelta | from datetime import datetime, timedelta | ||||
from django.utils import timezone | from django.utils import timezone | ||||
from rest_framework.test import APITestCase | |||||
from unittest.mock import patch | |||||
from swh.web.common.utils import reverse | from swh.web.common.utils import reverse | ||||
from swh.web.common.models import ( | from swh.web.common.models import ( | ||||
SaveUnauthorizedOrigin, SaveOriginRequest, | SaveUnauthorizedOrigin, SaveOriginRequest, | ||||
SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, | SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, | ||||
SAVE_REQUEST_PENDING | SAVE_REQUEST_PENDING | ||||
) | ) | ||||
from swh.web.common.models import ( | from swh.web.common.models import ( | ||||
SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, | SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, | ||||
SAVE_TASK_SCHEDULED, SAVE_TASK_FAILED, SAVE_TASK_SUCCEED | SAVE_TASK_SCHEDULED, SAVE_TASK_FAILED, SAVE_TASK_SUCCEED | ||||
) | ) | ||||
from swh.web.tests.testcase import WebTestCase | |||||
pytestmark = pytest.mark.django_db | |||||
class SaveApiTestCase(WebTestCase, APITestCase): | |||||
@classmethod | @pytest.fixture(autouse=True) | ||||
def setUpTestData(cls): # noqa: N802 | def populate_db(): | ||||
SaveUnauthorizedOrigin.objects.create( | SaveUnauthorizedOrigin.objects.create( | ||||
url='https://github.com/user/illegal_repo') | url='https://github.com/user/illegal_repo') | ||||
SaveUnauthorizedOrigin.objects.create( | SaveUnauthorizedOrigin.objects.create( | ||||
url='https://gitlab.com/user_to_exclude') | url='https://gitlab.com/user_to_exclude') | ||||
def test_invalid_visit_type(self): | |||||
def test_invalid_visit_type(api_client): | |||||
url = reverse('api-1-save-origin', | url = reverse('api-1-save-origin', | ||||
url_args={'visit_type': 'foo', | url_args={'visit_type': 'foo', | ||||
'origin_url': 'https://github.com/torvalds/linux'}) # noqa | 'origin_url': 'https://github.com/torvalds/linux'}) | ||||
response = api_client.post(url) | |||||
assert response.status_code == 400 | |||||
response = self.client.post(url) | |||||
self.assertEqual(response.status_code, 400) | |||||
def test_invalid_origin_url(self): | def test_invalid_origin_url(api_client): | ||||
url = reverse('api-1-save-origin', | url = reverse('api-1-save-origin', | ||||
url_args={'visit_type': 'git', | url_args={'visit_type': 'git', | ||||
'origin_url': 'bar'}) | 'origin_url': 'bar'}) | ||||
response = self.client.post(url) | response = api_client.post(url) | ||||
self.assertEqual(response.status_code, 400) | assert response.status_code == 400 | ||||
def check_created_save_request_status(self, mock_scheduler, origin_url, | |||||
def check_created_save_request_status(api_client, mocker, origin_url, | |||||
scheduler_task_status, | scheduler_task_status, | ||||
expected_request_status, | expected_request_status, | ||||
expected_task_status=None, | expected_task_status=None, | ||||
visit_date=None): | visit_date=None): | ||||
mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') | |||||
if not scheduler_task_status: | if not scheduler_task_status: | ||||
mock_scheduler.get_tasks.return_value = [] | mock_scheduler.get_tasks.return_value = [] | ||||
else: | else: | ||||
mock_scheduler.get_tasks.return_value = \ | mock_scheduler.get_tasks.return_value = [{ | ||||
[{ | |||||
'priority': 'high', | 'priority': 'high', | ||||
'policy': 'oneshot', | 'policy': 'oneshot', | ||||
'type': 'load-git', | 'type': 'load-git', | ||||
'arguments': { | 'arguments': { | ||||
'kwargs': { | 'kwargs': { | ||||
'repo_url': origin_url | 'repo_url': origin_url | ||||
}, | }, | ||||
'args': [] | 'args': [] | ||||
}, | }, | ||||
'status': scheduler_task_status, | 'status': scheduler_task_status, | ||||
'id': 1, | 'id': 1, | ||||
}] | }] | ||||
mock_scheduler.create_tasks.return_value = \ | mock_scheduler.create_tasks.return_value = [{ | ||||
[{ | |||||
'priority': 'high', | 'priority': 'high', | ||||
'policy': 'oneshot', | 'policy': 'oneshot', | ||||
'type': 'load-git', | 'type': 'load-git', | ||||
'arguments': { | 'arguments': { | ||||
'kwargs': { | 'kwargs': { | ||||
'repo_url': origin_url | 'repo_url': origin_url | ||||
}, | }, | ||||
'args': [] | 'args': [] | ||||
}, | }, | ||||
'status': 'next_run_not_scheduled', | 'status': 'next_run_not_scheduled', | ||||
'id': 1, | 'id': 1, | ||||
}] | }] | ||||
url = reverse('api-1-save-origin', | url = reverse('api-1-save-origin', | ||||
url_args={'visit_type': 'git', | url_args={'visit_type': 'git', | ||||
'origin_url': origin_url}) | 'origin_url': origin_url}) | ||||
with patch('swh.web.common.origin_save._get_visit_info_for_save_request') as mock_visit_date: # noqa | mock_visit_date = mocker.patch(('swh.web.common.origin_save.' | ||||
'_get_visit_info_for_save_request')) | |||||
mock_visit_date.return_value = (visit_date, None) | mock_visit_date.return_value = (visit_date, None) | ||||
response = self.client.post(url) | response = api_client.post(url) | ||||
if expected_request_status != SAVE_REQUEST_REJECTED: | if expected_request_status != SAVE_REQUEST_REJECTED: | ||||
self.assertEqual(response.status_code, 200) | assert response.status_code == 200, response.data | ||||
self.assertEqual(response.data['save_request_status'], | assert (response.data['save_request_status'] == | ||||
expected_request_status) | expected_request_status) | ||||
self.assertEqual(response.data['save_task_status'], | assert response.data['save_task_status'] == expected_task_status | ||||
expected_task_status) | |||||
else: | else: | ||||
self.assertEqual(response.status_code, 403) | assert response.status_code == 403, response.data | ||||
def check_save_request_status(self, mock_scheduler, origin_url, | def check_save_request_status(api_client, mocker, origin_url, | ||||
expected_request_status, | expected_request_status, | ||||
expected_task_status, | expected_task_status, | ||||
scheduler_task_status='next_run_not_scheduled', # noqa | scheduler_task_status='next_run_not_scheduled', | ||||
visit_date=None): | visit_date=None): | ||||
mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') | |||||
mock_scheduler.get_tasks.return_value = \ | mock_scheduler.get_tasks.return_value = [{ | ||||
[{ | |||||
'priority': 'high', | 'priority': 'high', | ||||
'policy': 'oneshot', | 'policy': 'oneshot', | ||||
'type': 'load-git', | 'type': 'load-git', | ||||
'arguments': { | 'arguments': { | ||||
'kwargs': { | 'kwargs': { | ||||
'repo_url': origin_url | 'repo_url': origin_url | ||||
}, | }, | ||||
'args': [] | 'args': [] | ||||
}, | }, | ||||
'status': scheduler_task_status, | 'status': scheduler_task_status, | ||||
'id': 1, | 'id': 1, | ||||
}] | }] | ||||
url = reverse('api-1-save-origin', | url = reverse('api-1-save-origin', | ||||
url_args={'visit_type': 'git', | url_args={'visit_type': 'git', | ||||
'origin_url': origin_url}) | 'origin_url': origin_url}) | ||||
with patch('swh.web.common.origin_save._get_visit_info_for_save_request') as mock_visit_date: # noqa | mock_visit_date = mocker.patch(('swh.web.common.origin_save.' | ||||
'_get_visit_info_for_save_request')) | |||||
mock_visit_date.return_value = (visit_date, None) | mock_visit_date.return_value = (visit_date, None) | ||||
response = self.client.get(url) | response = api_client.get(url) | ||||
self.assertEqual(response.status_code, 200) | assert response.status_code == 200, response.data | ||||
save_request_data = response.data[0] | save_request_data = response.data[0] | ||||
self.assertEqual(save_request_data['save_request_status'], | assert (save_request_data['save_request_status'] == | ||||
expected_request_status) | expected_request_status) | ||||
self.assertEqual(save_request_data['save_task_status'], | assert save_request_data['save_task_status'] == expected_task_status | ||||
expected_task_status) | |||||
# Check that save task status is still available when | # Check that save task status is still available when | ||||
# the scheduler task has been archived | # the scheduler task has been archived | ||||
mock_scheduler.get_tasks.return_value = [] | mock_scheduler.get_tasks.return_value = [] | ||||
response = self.client.get(url) | response = api_client.get(url) | ||||
self.assertEqual(response.status_code, 200) | assert response.status_code == 200 | ||||
save_request_data = response.data[0] | save_request_data = response.data[0] | ||||
self.assertEqual(save_request_data['save_task_status'], | assert save_request_data['save_task_status'] == expected_task_status | ||||
expected_task_status) | |||||
@patch('swh.web.common.origin_save.scheduler') | def test_save_request_rejected(api_client, mocker): | ||||
def test_save_request_rejected(self, mock_scheduler): | |||||
origin_url = 'https://github.com/user/illegal_repo' | origin_url = 'https://github.com/user/illegal_repo' | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
None, SAVE_REQUEST_REJECTED) | None, SAVE_REQUEST_REJECTED) | ||||
self.check_save_request_status(mock_scheduler, origin_url, | check_save_request_status(api_client, mocker, origin_url, | ||||
SAVE_REQUEST_REJECTED, | SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_CREATED) | ||||
SAVE_TASK_NOT_CREATED) | |||||
@patch('swh.web.common.origin_save.scheduler') | |||||
def test_save_request_pending(self, mock_scheduler): | def test_save_request_pending(api_client, mocker): | ||||
origin_url = 'https://unkwownforge.com/user/repo' | origin_url = 'https://unkwownforge.com/user/repo' | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, | ||||
None, SAVE_REQUEST_PENDING, | origin_url, None, SAVE_REQUEST_PENDING, | ||||
SAVE_TASK_NOT_CREATED) | |||||
self.check_save_request_status(mock_scheduler, origin_url, | |||||
SAVE_REQUEST_PENDING, | |||||
SAVE_TASK_NOT_CREATED) | SAVE_TASK_NOT_CREATED) | ||||
check_save_request_status(api_client, mocker, origin_url, | |||||
SAVE_REQUEST_PENDING, SAVE_TASK_NOT_CREATED) | |||||
@patch('swh.web.common.origin_save.scheduler') | def test_save_request_succeed(api_client, mocker): | ||||
def test_save_request_succeed(self, mock_scheduler): | |||||
origin_url = 'https://github.com/Kitware/CMake' | origin_url = 'https://github.com/Kitware/CMake' | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
None, SAVE_REQUEST_ACCEPTED, | None, SAVE_REQUEST_ACCEPTED, | ||||
SAVE_TASK_NOT_YET_SCHEDULED) | SAVE_TASK_NOT_YET_SCHEDULED) | ||||
self.check_save_request_status(mock_scheduler, origin_url, | check_save_request_status(api_client, mocker, origin_url, | ||||
SAVE_REQUEST_ACCEPTED, | SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED, | ||||
SAVE_TASK_SCHEDULED, | scheduler_task_status='next_run_scheduled') | ||||
scheduler_task_status='next_run_scheduled') # noqa | check_save_request_status(api_client, mocker, origin_url, | ||||
self.check_save_request_status(mock_scheduler, origin_url, | SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEED, | ||||
SAVE_REQUEST_ACCEPTED, | |||||
SAVE_TASK_SUCCEED, | |||||
scheduler_task_status='completed', | scheduler_task_status='completed', | ||||
visit_date=None) # noqa | visit_date=None) | ||||
visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) | visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) | ||||
self.check_save_request_status(mock_scheduler, origin_url, | check_save_request_status(api_client, mocker, origin_url, | ||||
SAVE_REQUEST_ACCEPTED, | SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEED, | ||||
SAVE_TASK_SUCCEED, | |||||
scheduler_task_status='completed', | scheduler_task_status='completed', | ||||
visit_date=visit_date) # noqa | visit_date=visit_date) | ||||
@patch('swh.web.common.origin_save.scheduler') | def test_save_request_failed(api_client, mocker): | ||||
def test_save_request_failed(self, mock_scheduler): | |||||
origin_url = 'https://gitlab.com/inkscape/inkscape' | origin_url = 'https://gitlab.com/inkscape/inkscape' | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
None, SAVE_REQUEST_ACCEPTED, | None, SAVE_REQUEST_ACCEPTED, | ||||
SAVE_TASK_NOT_YET_SCHEDULED) | SAVE_TASK_NOT_YET_SCHEDULED) | ||||
self.check_save_request_status(mock_scheduler, origin_url, | check_save_request_status(api_client, mocker, origin_url, | ||||
SAVE_REQUEST_ACCEPTED, | SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED, | ||||
SAVE_TASK_SCHEDULED, | scheduler_task_status='next_run_scheduled') | ||||
scheduler_task_status='next_run_scheduled') # noqa | check_save_request_status(api_client, mocker, origin_url, | ||||
self.check_save_request_status(mock_scheduler, origin_url, | SAVE_REQUEST_ACCEPTED, SAVE_TASK_FAILED, | ||||
SAVE_REQUEST_ACCEPTED, | scheduler_task_status='disabled') | ||||
SAVE_TASK_FAILED, | |||||
scheduler_task_status='disabled') # noqa | |||||
@patch('swh.web.common.origin_save.scheduler') | def test_create_save_request_only_when_needed(api_client, mocker): | ||||
def test_create_save_request_only_when_needed(self, mock_scheduler): | |||||
origin_url = 'https://github.com/webpack/webpack' | origin_url = 'https://github.com/webpack/webpack' | ||||
SaveOriginRequest.objects.create(visit_type='git', | SaveOriginRequest.objects.create(visit_type='git', origin_url=origin_url, | ||||
origin_url=origin_url, | status=SAVE_REQUEST_ACCEPTED, | ||||
status=SAVE_REQUEST_ACCEPTED, # noqa | |||||
loading_task_id=56) | loading_task_id=56) | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
'next_run_not_scheduled', | 'next_run_not_scheduled', | ||||
SAVE_REQUEST_ACCEPTED, | SAVE_REQUEST_ACCEPTED, | ||||
SAVE_TASK_NOT_YET_SCHEDULED) | SAVE_TASK_NOT_YET_SCHEDULED) | ||||
sors = list(SaveOriginRequest.objects.filter(visit_type='git', | sors = list(SaveOriginRequest.objects.filter(visit_type='git', | ||||
origin_url=origin_url)) | origin_url=origin_url)) | ||||
self.assertEqual(len(sors), 1) | assert len(sors) == 1 | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
'next_run_scheduled', | 'next_run_scheduled', | ||||
SAVE_REQUEST_ACCEPTED, | SAVE_REQUEST_ACCEPTED, | ||||
SAVE_TASK_SCHEDULED) | SAVE_TASK_SCHEDULED) | ||||
sors = list(SaveOriginRequest.objects.filter(visit_type='git', | sors = list(SaveOriginRequest.objects.filter(visit_type='git', | ||||
origin_url=origin_url)) | origin_url=origin_url)) | ||||
self.assertEqual(len(sors), 1) | assert len(sors) == 1 | ||||
visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) | visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
'completed', | 'completed', SAVE_REQUEST_ACCEPTED, | ||||
SAVE_REQUEST_ACCEPTED, | |||||
SAVE_TASK_NOT_YET_SCHEDULED, | SAVE_TASK_NOT_YET_SCHEDULED, | ||||
visit_date=visit_date) | visit_date=visit_date) | ||||
sors = list(SaveOriginRequest.objects.filter(visit_type='git', | sors = list(SaveOriginRequest.objects.filter(visit_type='git', | ||||
origin_url=origin_url)) | origin_url=origin_url)) | ||||
self.assertEqual(len(sors), 2) | assert len(sors) == 2 | ||||
self.check_created_save_request_status(mock_scheduler, origin_url, | check_created_save_request_status(api_client, mocker, origin_url, | ||||
'disabled', | 'disabled', SAVE_REQUEST_ACCEPTED, | ||||
SAVE_REQUEST_ACCEPTED, | |||||
SAVE_TASK_NOT_YET_SCHEDULED) | SAVE_TASK_NOT_YET_SCHEDULED) | ||||
sors = list(SaveOriginRequest.objects.filter(visit_type='git', | sors = list(SaveOriginRequest.objects.filter(visit_type='git', | ||||
origin_url=origin_url)) | origin_url=origin_url)) | ||||
self.assertEqual(len(sors), 3) | assert len(sors) == 3 | ||||
def test_get_save_requests_unknown_origin(self): | def test_get_save_requests_unknown_origin(api_client): | ||||
unknown_origin_url = 'https://gitlab.com/foo/bar' | unknown_origin_url = 'https://gitlab.com/foo/bar' | ||||
url = reverse('api-1-save-origin', | url = reverse('api-1-save-origin', | ||||
url_args={'visit_type': 'git', | url_args={'visit_type': 'git', | ||||
'origin_url': unknown_origin_url}) | 'origin_url': unknown_origin_url}) | ||||
response = self.client.get(url) | response = api_client.get(url) | ||||
self.assertEqual(response.status_code, 404) | assert response.status_code == 404 | ||||
self.assertEqual(response.data, { | assert response.data == { | ||||
'exception': 'NotFoundExc', | 'exception': 'NotFoundExc', | ||||
'reason': ('No save requests found for visit of type ' | 'reason': ('No save requests found for visit of type ' | ||||
'git on origin with url %s.') % unknown_origin_url | 'git on origin with url %s.') % unknown_origin_url | ||||
}) | } |