Differential D6113 Diff 22160 swh/web/tests/api/views/test_vault.py

Changeset View

Standalone View

swh/web/tests/api/views/test_vault.py

# Copyright (C) 2017-2020 The Software Heritage developers		# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

		import re

from hypothesis import given		from hypothesis import given
		import pytest

from swh.model import hashutil		from swh.model.identifiers import CoreSWHID
from swh.vault.exc import NotFoundExc		from swh.vault.exc import NotFoundExc
from swh.web.common.utils import reverse		from swh.web.common.utils import reverse
from swh.web.tests.strategies import (		from swh.web.tests.strategies import (
directory,		directory,
revision,		revision,
unknown_directory,		unknown_directory,
unknown_revision,		unknown_revision,
)		)
from swh.web.tests.utils import (		from swh.web.tests.utils import (
check_api_get_responses,		check_api_get_responses,
check_api_post_responses,		check_api_post_responses,
check_http_get_response,		check_http_get_response,
check_http_post_response,		check_http_post_response,
)		)

		#####################
		# Current API:


@given(directory(), revision())		@given(directory(), revision())
def test_api_vault_cook(api_client, mocker, directory, revision):		def test_api_vault_cook(api_client, mocker, directory, revision):
mock_archive = mocker.patch("swh.web.api.views.vault.archive")		mock_archive = mocker.patch("swh.web.api.views.vault.archive")

for obj_type, obj_id in (		for bundle_type, swhid in (
("directory", directory),		("flat", f"swh:1:dir:{directory}"),
("revision_gitfast", revision),		("gitfast", f"swh:1:rev:{revision}"),
):		):

fetch_url = reverse(		fetch_url = reverse(
f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},		f"api-1-vault-fetch-{bundle_type}", url_args={"swhid": swhid},
		)
		stub_cook = {
		"type": bundle_type,
		"progress_msg": None,
		"task_id": 1,
		"task_status": "done",
		"swhid": swhid,
		}
		stub_fetch = b"content"

		mock_archive.vault_cook.return_value = stub_cook
		mock_archive.vault_fetch.return_value = stub_fetch

		email = "test@test.mail"
		url = reverse(
		f"api-1-vault-cook-{bundle_type}",
		url_args={"swhid": swhid},
		query_params={"email": email},
		)

		rv = check_api_post_responses(api_client, url, data=None, status_code=200)
		assert rv.data == {
		"fetch_url": rv.wsgi_request.build_absolute_uri(fetch_url),
		"progress_message": None,
		"id": 1,
		"status": "done",
		"swhid": swhid,
		}
		mock_archive.vault_cook.assert_called_with(
		bundle_type, CoreSWHID.from_string(swhid), email
		)

		rv = check_http_get_response(api_client, fetch_url, status_code=200)
		assert rv["Content-Type"] == "application/gzip"
		assert rv.content == stub_fetch
		mock_archive.vault_fetch.assert_called_with(
		bundle_type, CoreSWHID.from_string(swhid)
		)


		@given(directory(), revision(), unknown_directory(), unknown_revision())
		def test_api_vault_cook_notfound(
		api_client, mocker, directory, revision, unknown_directory, unknown_revision
		):
		mock_vault = mocker.patch("swh.web.common.archive.vault")
		mock_vault.cook.side_effect = NotFoundExc("object not found")
		mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found")
		mock_vault.progress.side_effect = NotFoundExc("cooking request not found")

		for bundle_type, swhid in (
		("flat", f"swh:1:dir:{directory}"),
		("gitfast", f"swh:1:rev:{revision}"),
		):

		url = reverse(f"api-1-vault-cook-{bundle_type}", url_args={"swhid": swhid})

		rv = check_api_get_responses(api_client, url, status_code=404)

		assert rv.data["exception"] == "NotFoundExc"
		assert rv.data["reason"] == f"Cooking of {swhid} was never requested."
		mock_vault.progress.assert_called_with(
		bundle_type, CoreSWHID.from_string(swhid)
		)

		for bundle_type, swhid in (
		("flat", f"swh:1:dir:{unknown_directory}"),
		("gitfast", f"swh:1:rev:{unknown_revision}"),
		):
		url = reverse(f"api-1-vault-cook-{bundle_type}", url_args={"swhid": swhid})
		rv = check_api_post_responses(api_client, url, data=None, status_code=404)

		assert rv.data["exception"] == "NotFoundExc"
		assert rv.data["reason"] == f"{swhid} not found."
		mock_vault.cook.assert_called_with(
		bundle_type, CoreSWHID.from_string(swhid), email=None
		)

		fetch_url = reverse(
		f"api-1-vault-fetch-{bundle_type}", url_args={"swhid": swhid},
		)

		rv = check_api_get_responses(api_client, fetch_url, status_code=404)
		assert rv.data["exception"] == "NotFoundExc"
		assert rv.data["reason"] == f"Cooked archive for {swhid} not found."
		mock_vault.fetch.assert_called_with(bundle_type, CoreSWHID.from_string(swhid))


		@pytest.mark.parametrize("bundle_type", ["flat", "gitfast"])
		def test_api_vault_cook_error_content(api_client, mocker, bundle_type):
		swhid = "swh:1:cnt:" + "0" * 40

		email = "test@test.mail"
		url = reverse(
		f"api-1-vault-cook-{bundle_type}",
		url_args={"swhid": swhid},
		query_params={"email": email},
		)

		rv = check_api_post_responses(api_client, url, data=None, status_code=400)
		assert rv.data == {
		"exception": "BadInputExc",
		"reason": (
		"Content objects do not need to be cooked, "
		"use `/api/1/content/raw/` instead."
		),
		}


		@pytest.mark.parametrize(
		"bundle_type,swhid_type,hint",
		[
		("flat", "rev", True),
		("flat", "rel", False),
		("flat", "snp", False),
		("gitfast", "dir", True),
		("gitfast", "rel", False),
		("gitfast", "snp", False),
		],
		)
		def test_api_vault_cook_error(api_client, mocker, bundle_type, swhid_type, hint):
		swhid = f"swh:1:{swhid_type}:" + "0" * 40

		email = "test@test.mail"
		url = reverse(
		f"api-1-vault-cook-{bundle_type}",
		url_args={"swhid": swhid},
		query_params={"email": email},
		)

		rv = check_api_post_responses(api_client, url, data=None, status_code=400)
		assert rv.data["exception"] == "BadInputExc"
		if hint:
		assert re.match(
		r"Only .* can be cooked as .* bundles\. Use .*", rv.data["reason"]
		)
		else:
		assert re.match(r"Only .* can be cooked as .* bundles\.", rv.data["reason"])


		#####################
		# Legacy API:


		@given(directory(), revision())
		def test_api_vault_cook_legacy(api_client, mocker, directory, revision):
		mock_archive = mocker.patch("swh.web.api.views.vault.archive")

		for obj_type, bundle_type, obj_id in (
		("directory", "flat", directory),
		("revision_gitfast", "gitfast", revision),
		):
		swhid = f"swh:1:{obj_type[:3]}:{obj_id}"

		fetch_url = reverse(
		f"api-1-vault-fetch-{bundle_type}", url_args={"swhid": swhid},
)		)
stub_cook = {		stub_cook = {
"type": obj_type,		"type": obj_type,
"progress_msg": None,		"progress_msg": None,
"task_id": 1,		"task_id": 1,
"task_status": "done",		"task_status": "done",
"object_id": obj_id,		"swhid": swhid,
}		}
stub_fetch = b"content"		stub_fetch = b"content"

mock_archive.vault_cook.return_value = stub_cook		mock_archive.vault_cook.return_value = stub_cook
mock_archive.vault_fetch.return_value = stub_fetch		mock_archive.vault_fetch.return_value = stub_fetch

email = "test@test.mail"		email = "test@test.mail"
url = reverse(		url = reverse(
f"api-1-vault-cook-{obj_type}",		f"api-1-vault-cook-{obj_type}",
url_args={f"{obj_type[:3]}_id": obj_id},		url_args={f"{obj_type[:3]}_id": obj_id},
query_params={"email": email},		query_params={"email": email},
)		)

rv = check_api_post_responses(api_client, url, data=None, status_code=200)		rv = check_api_post_responses(api_client, url, data=None, status_code=200)
assert rv.data == {		assert rv.data == {
"fetch_url": rv.wsgi_request.build_absolute_uri(fetch_url),		"fetch_url": rv.wsgi_request.build_absolute_uri(fetch_url),
"obj_type": obj_type,
"progress_message": None,		"progress_message": None,
"id": 1,		"id": 1,
"status": "done",		"status": "done",
"obj_id": obj_id,		"swhid": swhid,
}		}
mock_archive.vault_cook.assert_called_with(		mock_archive.vault_cook.assert_called_with(
obj_type, hashutil.hash_to_bytes(obj_id), email		bundle_type, CoreSWHID.from_string(swhid), email
)		)

rv = check_http_get_response(api_client, fetch_url, status_code=200)		rv = check_http_get_response(api_client, fetch_url, status_code=200)
assert rv["Content-Type"] == "application/gzip"		assert rv["Content-Type"] == "application/gzip"
assert rv.content == stub_fetch		assert rv.content == stub_fetch
mock_archive.vault_fetch.assert_called_with(		mock_archive.vault_fetch.assert_called_with(
obj_type, hashutil.hash_to_bytes(obj_id)		bundle_type, CoreSWHID.from_string(swhid)
)		)


@given(directory(), revision())		@given(directory(), revision())
def test_api_vault_cook_uppercase_hash(api_client, directory, revision):		def test_api_vault_cook_uppercase_hash_legacy(api_client, directory, revision):

for obj_type, obj_id in (		for obj_type, obj_id in (
("directory", directory),		("directory", directory),
("revision_gitfast", revision),		("revision_gitfast", revision),
):		):

url = reverse(		url = reverse(
f"api-1-vault-cook-{obj_type}-uppercase-checksum",		f"api-1-vault-cook-{obj_type}-uppercase-checksum",
Show All 19 Lines	):
redirect_url = reverse(		redirect_url = reverse(
f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},		f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},
)		)

assert rv["location"] == redirect_url		assert rv["location"] == redirect_url


@given(directory(), revision(), unknown_directory(), unknown_revision())		@given(directory(), revision(), unknown_directory(), unknown_revision())
def test_api_vault_cook_notfound(		def test_api_vault_cook_notfound_legacy(
api_client, mocker, directory, revision, unknown_directory, unknown_revision		api_client, mocker, directory, revision, unknown_directory, unknown_revision
):		):
mock_vault = mocker.patch("swh.web.common.archive.vault")		mock_vault = mocker.patch("swh.web.common.archive.vault")
mock_vault.cook.side_effect = NotFoundExc("object not found")		mock_vault.cook.side_effect = NotFoundExc("object not found")
mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found")		mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found")
mock_vault.progress.side_effect = NotFoundExc("cooking request not found")		mock_vault.progress.side_effect = NotFoundExc("cooking request not found")

for obj_type, obj_id in (		for obj_type, bundle_type, obj_id in (
("directory", directory),		("directory", "flat", directory),
("revision_gitfast", revision),		("revision_gitfast", "gitfast", revision),
):		):

obj_name = obj_type.split("_")[0]

url = reverse(		url = reverse(
f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},		f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},
)		)

		swhid = f"swh:1:{obj_type[:3]}:{obj_id}"

rv = check_api_get_responses(api_client, url, status_code=404)		rv = check_api_get_responses(api_client, url, status_code=404)

assert rv.data["exception"] == "NotFoundExc"		assert rv.data["exception"] == "NotFoundExc"
assert (		assert rv.data["reason"] == f"Cooking of {swhid} was never requested."
rv.data["reason"]		mock_vault.progress.assert_called_with(
== f"Cooking of {obj_name} '{obj_id}' was never requested."		bundle_type, CoreSWHID.from_string(swhid)
)		)
mock_vault.progress.assert_called_with(obj_type, hashutil.hash_to_bytes(obj_id))

for obj_type, obj_id in (		for obj_type, bundle_type, obj_id in (
("directory", unknown_directory),		("directory", "flat", unknown_directory),
("revision_gitfast", unknown_revision),		("revision_gitfast", "gitfast", unknown_revision),
):		):
obj_name = obj_type.split("_")[0]		swhid = f"swh:1:{obj_type[:3]}:{obj_id}"

url = reverse(		url = reverse(
f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id}		f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id}
)		)
rv = check_api_post_responses(api_client, url, data=None, status_code=404)		rv = check_api_post_responses(api_client, url, data=None, status_code=404)

assert rv.data["exception"] == "NotFoundExc"		assert rv.data["exception"] == "NotFoundExc"
assert rv.data["reason"] == f"{obj_name.title()} '{obj_id}' not found."		assert rv.data["reason"] == f"{swhid} not found."
mock_vault.cook.assert_called_with(		mock_vault.cook.assert_called_with(
obj_type, hashutil.hash_to_bytes(obj_id), email=None		bundle_type, CoreSWHID.from_string(swhid), email=None
)		)

fetch_url = reverse(		fetch_url = reverse(
f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},		f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id},
)		)

rv = check_api_get_responses(api_client, fetch_url, status_code=404)		# Redirected to the current 'fetch' url
assert rv.data["exception"] == "NotFoundExc"		rv = check_http_get_response(api_client, fetch_url, status_code=302)
assert (		redirect_url = reverse(
rv.data["reason"] == f"Cooked archive for {obj_name} '{obj_id}' not found."		f"api-1-vault-fetch-{bundle_type}", url_args={"swhid": swhid},
)		)
mock_vault.fetch.assert_called_with(obj_type, hashutil.hash_to_bytes(obj_id))		assert rv["location"] == redirect_url

		rv = check_api_get_responses(api_client, redirect_url, status_code=404)
		assert rv.data["exception"] == "NotFoundExc"
		assert rv.data["reason"] == f"Cooked archive for {swhid} not found."
		mock_vault.fetch.assert_called_with(bundle_type, CoreSWHID.from_string(swhid))