diff --git a/docs/uri-scheme-api-origin.rst b/docs/uri-scheme-api-origin.rst --- a/docs/uri-scheme-api-origin.rst +++ b/docs/uri-scheme-api-origin.rst @@ -8,3 +8,17 @@ .. autosimple:: swh.web.api.views.origin.api_origin_visits .. autosimple:: swh.web.api.views.origin.api_origin_visit + +.. autosimple:: swh.web.api.views.origin.api_origin_visit + +.. autosimple:: swh.web.save_code_now.api_views.api_save_origin + +.. autosimple:: swh.web.save_origin_webhooks.bitbucket.api_origin_save_webhook_bitbucket + +.. autosimple:: swh.web.save_origin_webhooks.gitea.api_origin_save_webhook_gitea + +.. autosimple:: swh.web.save_origin_webhooks.github.api_origin_save_webhook_github + +.. autosimple:: swh.web.save_origin_webhooks.gitlab.api_origin_save_webhook_gitlab + +.. autosimple:: swh.web.save_origin_webhooks.sourceforge.api_origin_save_webhook_sourceforge diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -231,6 +231,7 @@ text = re.sub(r"([^:])//", r"\1/", text) # transform references to api endpoints doc into valid rst links text = re.sub(":http:get:`([^,`]*)`", r"`\1 <\1doc/>`_", text) + text = re.sub(":http:post:`([^,`]*)`", r"`\1 <\1doc/>`_", text) # transform references to some elements into bold text text = re.sub(":http:header:`(.*)`", r"**\1**", text) text = re.sub(":func:`(.*)`", r"**\1**", text) diff --git a/swh/web/save_code_now/api_views.py b/swh/web/save_code_now/api_views.py --- a/swh/web/save_code_now/api_views.py +++ b/swh/web/save_code_now/api_views.py @@ -6,6 +6,7 @@ import os from typing import Optional, cast +from django.conf import settings from rest_framework.request import Request from swh.web.api.apidoc import api_doc, format_docstring @@ -33,6 +34,27 @@ return docstring +def _webhook_info_doc() -> str: + docstring = "" + if "swh.web.save_origin_webhooks" in settings.SWH_DJANGO_APPS: + docstring = """ + :>json boolean from_webhook: indicates if the save request was created + from a popular forge webhook receiver + (see :http:post:`/api/1/origin/save/webhook/github/` for instance) + :>json string webhook_origin: indicates which forge type sent the webhook, + currently the supported types are:""" + + # instantiate webhook receivers + from swh.web.save_origin_webhooks import urls # noqa + from swh.web.save_origin_webhooks.generic_receiver import SUPPORTED_FORGE_TYPES + + webhook_forge_types = sorted(list(SUPPORTED_FORGE_TYPES)) + for visit_type in webhook_forge_types[:-1]: + docstring += f"**{visit_type}**, " + docstring += f"and **{webhook_forge_types[-1]}**" + return docstring + + save_code_now_api_urls = APIUrls() @@ -45,7 +67,9 @@ api_urls=save_code_now_api_urls, ) @api_doc("/origin/save/", category="Request archival") -@format_docstring(visit_types=_savable_visit_types()) +@format_docstring( + visit_types=_savable_visit_types(), webhook_info_doc=_webhook_info_doc() +) def api_save_origin(request: Request, visit_type: str, origin_url: str): """ .. http:get:: /api/1/origin/save/(visit_type)/url/(origin_url)/ @@ -102,6 +126,7 @@ otherwise. :>json string note: optional note giving details about the save request, for instance why it has been rejected + {webhook_info_doc} :statuscode 200: no error :statuscode 400: an invalid visit type or origin url has been provided @@ -110,6 +135,13 @@ """ + def _cleanup_sor_data(sor): + del sor["id"] + if "swh.web.save_origin_webhooks" not in settings.SWH_DJANGO_APPS: + del sor["from_webhook"] + del sor["webhook_origin"] + return sor + data = request.data or {} if request.method == "POST": sor = create_save_origin_request( @@ -122,10 +154,8 @@ user_id=cast(Optional[int], request.user.id), **data, ) - del sor["id"] - return sor + return _cleanup_sor_data(sor) + else: sors = get_save_origin_requests(visit_type, origin_url) - for sor in sors: - del sor["id"] - return sors + return [_cleanup_sor_data(sor) for sor in sors] diff --git a/swh/web/save_code_now/migrations/0013_saveoriginrequest_webhook_info.py b/swh/web/save_code_now/migrations/0013_saveoriginrequest_webhook_info.py new file mode 100644 --- /dev/null +++ b/swh/web/save_code_now/migrations/0013_saveoriginrequest_webhook_info.py @@ -0,0 +1,26 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("swh_web_save_code_now", "0012_saveoriginrequest_note"), + ] + + operations = [ + migrations.AddField( + model_name="saveoriginrequest", + name="from_webhook", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="saveoriginrequest", + name="webhook_origin", + field=models.CharField(max_length=200, null=True), + ), + ] diff --git a/swh/web/save_code_now/models.py b/swh/web/save_code_now/models.py --- a/swh/web/save_code_now/models.py +++ b/swh/web/save_code_now/models.py @@ -103,6 +103,8 @@ # store ids of users that submitted the request as string list user_ids = models.TextField(null=True) note = models.TextField(null=True) + from_webhook = models.BooleanField(default=False) + webhook_origin = models.CharField(max_length=200, null=True) class Meta: app_label = "swh_web_save_code_now" @@ -129,6 +131,8 @@ visit_date=visit_date.isoformat() if visit_date else None, loading_task_id=self.loading_task_id, note=self.note, + from_webhook=self.from_webhook, + webhook_origin=self.webhook_origin, ) def __str__(self) -> str: diff --git a/swh/web/save_code_now/origin_save.py b/swh/web/save_code_now/origin_save.py --- a/swh/web/save_code_now/origin_save.py +++ b/swh/web/save_code_now/origin_save.py @@ -404,6 +404,8 @@ origin_url: str, privileged_user: bool = False, user_id: Optional[int] = None, + from_webhook: bool = False, + webhook_origin: Optional[str] = None, **kwargs, ) -> SaveOriginRequestInfo: """Create a loading task to save a software origin into the archive. @@ -426,6 +428,8 @@ privileged: Whether the user has some more privilege than other (bypass review, access to privileged other visit types) user_id: User identifier (provided when authenticated) + from_webhook: Indicates if the save request is created from a webhook receiver + webhook_origin: Indicates which forge type sent the webhook kwargs: Optional parameters (e.g. artifact_url, artifact_filename, artifact_version) @@ -545,6 +549,8 @@ status=save_request_status, loading_task_id=task["id"], user_ids=f'"{user_id}"' if user_id else None, + from_webhook=from_webhook, + webhook_origin=webhook_origin, ) # save request must be manually reviewed for acceptation @@ -568,6 +574,8 @@ origin_url=origin_url, status=save_request_status, user_ids=f'"{user_id}"' if user_id else None, + from_webhook=from_webhook, + webhook_origin=webhook_origin, ) # origin can not be saved as its url is blacklisted, # log the request to the database anyway @@ -577,6 +585,8 @@ origin_url=origin_url, status=save_request_status, user_ids=f'"{user_id}"' if user_id else None, + from_webhook=from_webhook, + webhook_origin=webhook_origin, ) if save_request_status == SAVE_REQUEST_REJECTED: diff --git a/swh/web/save_code_now/tests/test_migrations.py b/swh/web/save_code_now/tests/test_migrations.py --- a/swh/web/save_code_now/tests/test_migrations.py +++ b/swh/web/save_code_now/tests/test_migrations.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -10,6 +10,7 @@ MIGRATION_0010 = "0010_saveoriginrequest_user_id" MIGRATION_0011 = "0011_saveoriginrequest_user_ids" MIGRATION_0012 = "0012_saveoriginrequest_note" +MIGRATION_0013 = "0013_saveoriginrequest_webhook_info" def test_migrations_09_add_visit_status_to_sor_model(migrator): @@ -58,3 +59,21 @@ new_model = new_state.apps.get_model(APP_NAME, "SaveOriginRequest") assert hasattr(new_model, "note") is True + + +def test_migrations_13_add_webhook_info_to_sor_model(migrator): + """Ensures the migration adds the from_webhook field to SaveOriginRequest table""" + + old_state = migrator.apply_initial_migration( + (APP_NAME, MIGRATION_0012), + ) + old_model = old_state.apps.get_model(APP_NAME, "SaveOriginRequest") + + assert hasattr(old_model, "from_webhook") is False + assert hasattr(old_model, "webhook_origin") is False + + new_state = migrator.apply_tested_migration((APP_NAME, MIGRATION_0013)) + new_model = new_state.apps.get_model(APP_NAME, "SaveOriginRequest") + + assert hasattr(new_model, "from_webhook") is True + assert hasattr(new_model, "webhook_origin") is True diff --git a/swh/web/save_code_now/tests/test_origin_save.py b/swh/web/save_code_now/tests/test_origin_save.py --- a/swh/web/save_code_now/tests/test_origin_save.py +++ b/swh/web/save_code_now/tests/test_origin_save.py @@ -338,6 +338,8 @@ visit_date=_visit_date.isoformat() if _visit_date else None, loading_task_id=sor.loading_task_id, note=note, + from_webhook=False, + webhook_origin=None, ) diff --git a/swh/web/save_code_now/tests/test_origin_save_api.py b/swh/web/save_code_now/tests/test_origin_save_api.py --- a/swh/web/save_code_now/tests/test_origin_save_api.py +++ b/swh/web/save_code_now/tests/test_origin_save_api.py @@ -95,6 +95,8 @@ response = check_api_post_responses(api_client, url, data=None, status_code=200) assert response.data["save_request_status"] == expected_request_status assert response.data["save_task_status"] == expected_task_status + assert response.data["from_webhook"] is False + assert response.data["webhook_origin"] is None else: check_api_post_responses(api_client, url, data=None, status_code=403) @@ -139,6 +141,8 @@ assert save_request_data["save_request_status"] == expected_request_status assert save_request_data["save_task_status"] == expected_task_status assert save_request_data["visit_status"] == visit_status + assert save_request_data["from_webhook"] is False + assert save_request_data["webhook_origin"] is None if scheduler_task_run_status is not None: # Check that save task status is still available when diff --git a/swh/web/save_origin_webhooks/generic_receiver.py b/swh/web/save_origin_webhooks/generic_receiver.py --- a/swh/web/save_origin_webhooks/generic_receiver.py +++ b/swh/web/save_origin_webhooks/generic_receiver.py @@ -16,6 +16,9 @@ webhooks_api_urls = APIUrls() +SUPPORTED_FORGE_TYPES = set() + + class OriginSaveWebhookReceiver(abc.ABC): FORGE_TYPE: str WEBHOOK_GUIDE_URL: str @@ -64,6 +67,7 @@ request or missing data in webhook payload """ self.__name__ = "api_origin_save_webhook_{self.FORGE_TYPE.lower()}" + SUPPORTED_FORGE_TYPES.add(self.FORGE_TYPE.lower()) api_doc( f"/origin/save/webhook/{self.FORGE_TYPE.lower()}/", category="Request archival", @@ -118,7 +122,10 @@ ) save_request = create_save_origin_request( - visit_type=visit_type, origin_url=repo_url + visit_type=visit_type, + origin_url=repo_url, + from_webhook=True, + webhook_origin=self.FORGE_TYPE.lower(), ) return { diff --git a/swh/web/save_origin_webhooks/tests/utils.py b/swh/web/save_origin_webhooks/tests/utils.py --- a/swh/web/save_origin_webhooks/tests/utils.py +++ b/swh/web/save_origin_webhooks/tests/utils.py @@ -5,6 +5,7 @@ from typing import Any, Dict +from swh.web.save_code_now.models import SaveOriginRequest from swh.web.tests.helpers import check_api_post_responses from swh.web.utils import reverse @@ -40,6 +41,11 @@ task = dict(tasks[0].items()) assert task["arguments"]["kwargs"]["url"] == expected_origin_url + request = SaveOriginRequest.objects.get( + origin_url=expected_origin_url, visit_type=expected_visit_type + ) + assert request.from_webhook + def origin_save_webhook_receiver_invalid_request_test( forge_type: str, diff --git a/swh/web/utils/typing.py b/swh/web/utils/typing.py --- a/swh/web/utils/typing.py +++ b/swh/web/utils/typing.py @@ -248,6 +248,10 @@ """Status of the scheduled task""" note: Optional[str] """Optional note associated to the request, for instance rejection reason""" + from_webhook: bool + """Indicates if request was created from a webhook receiver""" + webhook_origin: Optional[str] + """Indicates from which forge type a webhook was received""" class OriginExistenceCheckInfo(TypedDict):