diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -p no:flask -p no:pytest_swh_storage --ignore=swh/web/tests/random_fixtures_test.py +addopts = -p no:flask -p no:pytest_swh_storage --ignore=swh/web/tests/random_fixtures_test.py -k test_api_raw_ norecursedirs = build docs node_modules .tox DJANGO_SETTINGS_MODULE = swh.web.settings.tests filterwarnings = diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -2,7 +2,7 @@ swh.core >= 0.0.95 swh.counters >= 0.5.1 swh.indexer >= 0.4.1 -swh.model >= 2.6.3 +swh.model >= 6.2.0 swh.scheduler >= 0.7.0 swh.search >= 0.2.0 swh.storage >= 0.31.0 diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py --- a/swh/web/api/urls.py +++ b/swh/web/api/urls.py @@ -13,6 +13,7 @@ import swh.web.api.views.origin # noqa import swh.web.api.views.origin_save # noqa import swh.web.api.views.ping # noqa +import swh.web.api.views.raw # noqa import swh.web.api.views.release # noqa import swh.web.api.views.revision # noqa import swh.web.api.views.snapshot # noqa diff --git a/swh/web/api/views/metadata.py b/swh/web/api/views/metadata.py --- a/swh/web/api/views/metadata.py +++ b/swh/web/api/views/metadata.py @@ -16,9 +16,7 @@ from swh.web.api.apiurls import api_route from swh.web.common import archive, converters from swh.web.common.exc import BadInputExc, NotFoundExc -from swh.web.common.utils import reverse - -SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" +from swh.web.common.utils import SWHID_RE, reverse @api_route( diff --git a/swh/web/api/views/raw.py b/swh/web/api/views/raw.py new file mode 100644 --- /dev/null +++ b/swh/web/api/views/raw.py @@ -0,0 +1,115 @@ +# Copyright (C) 2018-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.http import HttpResponse + +from swh.model.git_objects import ( + content_git_object, + directory_git_object, + revision_git_object, + release_git_object, + snapshot_git_object, +) +from swh.model import model +from swh.model.swhids import CoreSWHID, ObjectType +from swh.web.api.apidoc import api_doc, format_docstring +from swh.web.api.apiurls import api_route +from swh.web.common import archive +from swh.web.common.exc import NotFoundExc +from swh.web.common.utils import SWHID_RE + + +@api_route( + f"/raw/(?P{SWHID_RE})/", + "api-1-raw-object", +) +@api_doc("/raw/") +@format_docstring() +def api_raw_object(request, swhid): + """ + .. http:get:: /api/1/raw// + + Get the object corresponding to the SWHID in raw form. + + This endpoint exposes the internal representation (see + :func:`swh.model.git_objects.*_git_object` in our data + model module for details), and so can be used to fetch a binary + blob which hashes to the same identifier. + + :param string swhid: the object's SWHID + + :resheader Content-Type: application/octet-stream + + :statuscode 200: no error + :statuscode 400: an invalid SWHID has been provided + :statuscode 404: the requested object can not be found in the archive + + **Example:** + + .. parsed-literal:: + + :swh_web_api:`raw/swh:1:snp:6a3a2cf0b2b90ce7ae1cf0a221ed68035b686f5a` + """ + + swhid = CoreSWHID.from_string(swhid) + object_id = swhid.object_id + object_type = swhid.object_type + + def not_found(): + return NotFoundExc(f"Object with id {swhid} not found.") + + if object_type == ObjectType.CONTENT: + results = archive.storage.content_find({"sha1_git": object_id}) + if len(results) == 0: + raise not_found() + result = content_git_object(results[0]) + + elif object_type == ObjectType.DIRECTORY: + entries = [] + page_token = None + while True: + batch = archive.storage.directory_get_entries( + directory_id=object_id, + page_token=page_token, + ) + if batch is None: + raise not_found() + entries += batch.results + if batch.next_page_token is None: + break + page_token = batch.next_page_token + result = directory_git_object( + model.Directory( + id=object_id, + entries=entries, + ) + ) + + elif object_type == ObjectType.REVISION: + result = archive.storage.revision_get([object_id])[0] + if result is None: + raise not_found() + result = revision_git_object(result) + + elif object_type == ObjectType.RELEASE: + result = archive.storage.release_get([object_id])[0] + if result is None: + raise not_found() + result = release_git_object(result) + + elif object_type == ObjectType.SNAPSHOT: + result = archive.storage.snapshot_get(object_id) + if result is None: + raise not_found() + result = snapshot_git_object(result) + + else: + raise ValueError(f"Unexpected object type variant: {object_type}") + + response = HttpResponse(result, content_type="application/octet-stream") + filename = swhid.replace(":", "_") + "_raw" + response["Content-disposition"] = f"attachment; filename={filename}" + + return response diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py --- a/swh/web/api/views/vault.py +++ b/swh/web/api/views/vault.py @@ -15,13 +15,12 @@ from swh.web.api.views.utils import api_lookup from swh.web.common import archive, query from swh.web.common.exc import BadInputExc -from swh.web.common.utils import reverse +from swh.web.common.utils import SWHID_RE, reverse + ###################################################### # Common -SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" - # XXX: a bit spaghetti. Would be better with class-based views. def _dispatch_cook_progress(request, bundle_type: str, swhid: CoreSWHID): diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -39,6 +39,8 @@ SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) +SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" + swh_object_icons = { "alias": "mdi mdi-star", "branch": "mdi mdi-source-branch", diff --git a/swh/web/tests/api/views/test_raw.py b/swh/web/tests/api/views/test_raw.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/api/views/test_raw.py @@ -0,0 +1,77 @@ +# Copyright (C) 2015-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import hashlib + +from swh.model.git_objects import content_git_object +from swh.web.common.utils import reverse +from swh.web.tests.data import random_swhid +from swh.web.tests.utils import ( + check_api_get_responses, + check_http_get_response, +) + + +def test_api_raw_not_found(api_client): + unknown_swhid = random_swhid() + + url = reverse("api-1-raw-object", url_args={"swhid": str(unknown_swhid)}) + rv = check_api_get_responses(api_client, url, status_code=404) + assert rv.data == { + "exception": "NotFoundExc", + "reason": f"Object with id {unknown_swhid} not found.", + } + + +def test_api_raw_content(api_client, archive_data, content): + object_id = content["sha1_git"] + object_ty = "cnt" + url = reverse( + "api-1-raw-object", + url_args={"swhid": f"swh:1:{object_ty}:{object_id}"}, + ) + + rv = check_http_get_response(api_client, url, status_code=200) + assert rv["Content-Type"] == "application/octet-stream" + assert ( + rv["Content-disposition"] + == f"attachment; filename=swh_1_{object_ty}_{object_id}_raw" + ) + expected_data = archive_data.content_get_data(object_id) + assert rv.content == content_git_object(expected_data) + sha1_git = hashlib.new("sha1", rv.content).digest() + assert sha1_git == object_id + + +def _test_api_raw_hash(api_client, archive_data, object_id, object_ty): + url = reverse( + "api-1-raw-object", + url_args={"swhid": f"swh:1:{object_ty}:{object_id}"}, + ) + + rv = check_http_get_response(api_client, url, status_code=200) + assert rv["Content-Type"] == "application/octet-stream" + assert ( + rv["Content-disposition"] + == f"attachment; filename=swh_1_{object_ty}_{object_id}_raw" + ) + sha1_git = hashlib.new("sha1", rv.content).digest() + assert sha1_git == object_id + + +def test_api_raw_directory(api_client, archive_data, directory): + _test_api_raw_hash(api_client, archive_data, directory, "dir") + + +def test_api_raw_revision(api_client, archive_data, revision): + _test_api_raw_hash(api_client, archive_data, revision["id"], "rev") + + +def test_api_raw_release(api_client, archive_data, release): + _test_api_raw_hash(api_client, archive_data, release["id"], "rel") + + +def test_api_raw_snapshot(api_client, archive_data, snapshot): + _test_api_raw_hash(api_client, archive_data, snapshot["id"], "snp") diff --git a/swh/web/tests/common/test_archive.py b/swh/web/tests/common/test_archive.py --- a/swh/web/tests/common/test_archive.py +++ b/swh/web/tests/common/test_archive.py @@ -789,39 +789,50 @@ root_rev_log = archive_data.revision_log(root_rev) rev = root_rev_log[-1]["id"] - assert archive.lookup_revision_through( - { - "origin_url": origin["url"], - "branch_name": branch_name, - "ts": None, - "sha1_git": rev, - } - ) == archive.lookup_revision_with_context_by(origin["url"], branch_name, None, rev) + assert ( + archive.lookup_revision_through( + { + "origin_url": origin["url"], + "branch_name": branch_name, + "ts": None, + "sha1_git": rev, + } + ) + == archive.lookup_revision_with_context_by( + origin["url"], branch_name, None, rev + ) + ) def test_lookup_revision_through_with_revision_by(archive_data, origin): branches = _get_origin_branches(archive_data, origin) branch_name = random.choice(list(branches.keys())) - assert archive.lookup_revision_through( - { - "origin_url": origin["url"], - "branch_name": branch_name, - "ts": None, - } - ) == archive.lookup_revision_by(origin["url"], branch_name, None) + assert ( + archive.lookup_revision_through( + { + "origin_url": origin["url"], + "branch_name": branch_name, + "ts": None, + } + ) + == archive.lookup_revision_by(origin["url"], branch_name, None) + ) def test_lookup_revision_through_with_context(ancestor_revisions): sha1_git = ancestor_revisions["sha1_git"] sha1_git_root = ancestor_revisions["sha1_git_root"] - assert archive.lookup_revision_through( - { - "sha1_git_root": sha1_git_root, - "sha1_git": sha1_git, - } - ) == archive.lookup_revision_with_context(sha1_git_root, sha1_git) + assert ( + archive.lookup_revision_through( + { + "sha1_git_root": sha1_git_root, + "sha1_git": sha1_git, + } + ) + == archive.lookup_revision_with_context(sha1_git_root, sha1_git) + ) def test_lookup_revision_through_with_revision(revision): diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -71,6 +71,25 @@ return hash_to_hex(bytes(random.randint(0, 255) for _ in range(32))) +def random_object_type(): + return random.choice( + [ + ObjectType.CONTENT, + ObjectType.DIRECTORY, + ObjectType.REVISION, + ObjectType.RELEASE, + ObjectType.SNAPSHOT, + ] + ) + + +def random_swhid(): + return CoreSWHID( + object_type=random_object_type(), + object_id=random_sha1(), + ) + + def random_content(): return { "sha1": random_sha1(),