diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -p no:flask -p no:pytest_swh_storage --ignore=swh/web/tests/random_fixtures_test.py +addopts = -p no:flask -p no:pytest_swh_storage --ignore=swh/web/tests/random_fixtures_test.py -k test_api_raw_ norecursedirs = build docs node_modules .tox DJANGO_SETTINGS_MODULE = swh.web.settings.tests filterwarnings = diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -2,7 +2,7 @@ swh.core >= 0.0.95 swh.counters >= 0.5.1 swh.indexer >= 0.4.1 -swh.model >= 2.6.3 +swh.model >= 6.2.0 swh.scheduler >= 0.7.0 swh.search >= 0.2.0 swh.storage >= 0.31.0 diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py --- a/swh/web/api/urls.py +++ b/swh/web/api/urls.py @@ -13,6 +13,7 @@ import swh.web.api.views.origin # noqa import swh.web.api.views.origin_save # noqa import swh.web.api.views.ping # noqa +import swh.web.api.views.raw # noqa import swh.web.api.views.release # noqa import swh.web.api.views.revision # noqa import swh.web.api.views.snapshot # noqa diff --git a/swh/web/api/views/metadata.py b/swh/web/api/views/metadata.py --- a/swh/web/api/views/metadata.py +++ b/swh/web/api/views/metadata.py @@ -16,9 +16,7 @@ from swh.web.api.apiurls import api_route from swh.web.common import archive, converters from swh.web.common.exc import BadInputExc, NotFoundExc -from swh.web.common.utils import reverse - -SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" +from swh.web.common.utils import SWHID_RE, reverse @api_route( diff --git a/swh/web/api/views/raw.py b/swh/web/api/views/raw.py new file mode 100644 --- /dev/null +++ b/swh/web/api/views/raw.py @@ -0,0 +1,115 @@ +# Copyright (C) 2018-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.http import HttpResponse + +from swh.model.git_objects import ( + content_git_object, + directory_git_object, + revision_git_object, + release_git_object, + snapshot_git_object, +) +import swh.model.model as model +from swh.model.swhids import CoreSWHID, ObjectType +from swh.web.api.apidoc import api_doc, format_docstring +from swh.web.api.apiurls import api_route +from swh.web.common.archive import storage +from swh.web.common.exc import NotFoundExc +from swh.web.common.utils import SWHID_RE + + +@api_route( + f"/raw/(?P{SWHID_RE})/", + "api-1-raw-object", +) +@api_doc("/raw/") +@format_docstring() +def api_raw_object(request, swhid_raw): + """ + .. http:get:: /api/1/raw// + + Get the object corresponding to the SWHID in raw form. + + This endpoint exposes the internal representation (see + :func:`swh.model.git_objects.*_git_object` in our data + model module for details), and so can be used to fetch a binary + blob which hashes to the same identifier. + + :param string swhid: the object's SWHID + + :resheader Content-Type: application/octet-stream + + :statuscode 200: no error + :statuscode 400: an invalid SWHID has been provided + :statuscode 404: the requested object can not be found in the archive + + **Example:** + + .. parsed-literal:: + + :swh_web_api:`raw/swh:1:snp:6a3a2cf0b2b90ce7ae1cf0a221ed68035b686f5a` + """ + + swhid = CoreSWHID.from_string(swhid_raw) + object_id = swhid.object_id + object_type = swhid.object_type + + def not_found(): + return NotFoundExc(f"Object with id {swhid_raw} not found.") + + if object_type == ObjectType.CONTENT: + results = storage.content_find({"sha1_git": object_id}) + if len(results) == 0: + raise not_found() + result = content_git_object(results[0]) + + elif object_type == ObjectType.DIRECTORY: + entries = [] + page_token = None + while True: + batch = storage.directory_get_entries( + directory_id=object_id, + page_token=page_token, + ) + if batch is None: + raise not_found() + entries += batch.results + if batch.next_page_token is None: + break + page_token = batch.next_page_token + result = directory_git_object( + model.Directory( + id=object_id, + entries=entries, + ) + ) + + elif object_type == ObjectType.REVISION: + result = storage.revision_get([object_id])[0] + if result is None: + raise not_found() + result = revision_git_object(result) + + elif object_type == ObjectType.RELEASE: + result = storage.release_get([object_id])[0] + if result is None: + raise not_found() + result = release_git_object(result) + + elif object_type == ObjectType.SNAPSHOT: + result = storage.snapshot_get(object_id) + if result is None: + raise not_found() + result = snapshot_git_object(result) + + else: + raise ValueError(f"Unexpected object type variant: {object_type}") + + response = HttpResponse(result, content_type="application/octet-stream") + filename = swhid.replace(":", "_") + "_raw" + response["Content-disposition"] = f"attachment; filename={filename}" + + return response diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py --- a/swh/web/api/views/vault.py +++ b/swh/web/api/views/vault.py @@ -15,13 +15,12 @@ from swh.web.api.views.utils import api_lookup from swh.web.common import archive, query from swh.web.common.exc import BadInputExc -from swh.web.common.utils import reverse +from swh.web.common.utils import SWHID_RE, reverse + ###################################################### # Common -SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" - # XXX: a bit spaghetti. Would be better with class-based views. def _dispatch_cook_progress(request, bundle_type: str, swhid: CoreSWHID): diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -39,6 +39,8 @@ SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) +SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" + swh_object_icons = { "alias": "mdi mdi-star", "branch": "mdi mdi-source-branch", diff --git a/swh/web/tests/api/views/test_raw.py b/swh/web/tests/api/views/test_raw.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/api/views/test_raw.py @@ -0,0 +1,62 @@ +# Copyright (C) 2015-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import hashlib + +from swh.model.git_objects import content_git_object +from swh.web.common.utils import reverse +# from swh.web.tests.data import random_content +from swh.web.tests.utils import ( + # check_api_get_responses, + check_http_get_response, +) + + +# def test_api_raw_content_ko_not_found(api_client): +# unknown_content_ = random_content() +# +# url = reverse( +# "api-1-raw-object", url_args={"q": "sha1_git:%s" % unknown_content_["sha1_git"]} +# ) +# rv = check_api_get_responses(api_client, url, status_code=404) +# assert rv.data == { +# "exception": "NotFoundExc", +# "reason": "Content with sha1 checksum equals to %s not found!" +# % unknown_content_["sha1_git"], +# } + + +def test_api_raw_content(api_client, archive_data, content): + object_id = content["sha1_git"] + object_ty = "cnt" + url = reverse( + "api-1-raw-object", url_args={"swhid_raw": f"swh:1:{object_ty}:{object_id}"} + ) + + rv = check_http_get_response(api_client, url, status_code=200) + assert rv["Content-Type"] == "application/octet-stream" + assert ( + rv["Content-disposition"] + == f"attachment; filename=swh_1_{object_ty}_{object_id}_raw" + ) + expected_data = archive_data.content_get_data(object_id) + assert rv.content == content_git_object(expected_data) + + +def test_api_raw_directory(api_client, archive_data, directory): + object_id = directory + object_ty = "dir" + url = reverse( + "api-1-raw-object", url_args={"swhid_raw": f"swh:1:{object_ty}:{object_id}"} + ) + + rv = check_http_get_response(api_client, url, status_code=200) + assert rv["Content-Type"] == "application/octet-stream" + assert ( + rv["Content-disposition"] + == f"attachment; filename=swh_1_{object_ty}_{object_id}_raw" + ) + sha1_git = hashlib.new("sha1", rv.content).digest() + assert sha1_git == object_id