Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9697201
D8889.id32101.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
13 KB
Subscribers
None
D8889.id32101.diff
View Options
diff --git a/docs/uri-scheme-api-origin.rst b/docs/uri-scheme-api-origin.rst
--- a/docs/uri-scheme-api-origin.rst
+++ b/docs/uri-scheme-api-origin.rst
@@ -8,3 +8,17 @@
.. autosimple:: swh.web.api.views.origin.api_origin_visits
.. autosimple:: swh.web.api.views.origin.api_origin_visit
+
+.. autosimple:: swh.web.api.views.origin.api_origin_visit
+
+.. autosimple:: swh.web.save_code_now.api_views.api_save_origin
+
+.. autosimple:: swh.web.save_origin_webhooks.bitbucket.api_origin_save_webhook_bitbucket
+
+.. autosimple:: swh.web.save_origin_webhooks.gitea.api_origin_save_webhook_gitea
+
+.. autosimple:: swh.web.save_origin_webhooks.github.api_origin_save_webhook_github
+
+.. autosimple:: swh.web.save_origin_webhooks.gitlab.api_origin_save_webhook_gitlab
+
+.. autosimple:: swh.web.save_origin_webhooks.sourceforge.api_origin_save_webhook_sourceforge
diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py
--- a/swh/web/api/apidoc.py
+++ b/swh/web/api/apidoc.py
@@ -231,6 +231,7 @@
text = re.sub(r"([^:])//", r"\1/", text)
# transform references to api endpoints doc into valid rst links
text = re.sub(":http:get:`([^,`]*)`", r"`\1 <\1doc/>`_", text)
+ text = re.sub(":http:post:`([^,`]*)`", r"`\1 <\1doc/>`_", text)
# transform references to some elements into bold text
text = re.sub(":http:header:`(.*)`", r"**\1**", text)
text = re.sub(":func:`(.*)`", r"**\1**", text)
diff --git a/swh/web/save_code_now/api_views.py b/swh/web/save_code_now/api_views.py
--- a/swh/web/save_code_now/api_views.py
+++ b/swh/web/save_code_now/api_views.py
@@ -6,6 +6,7 @@
import os
from typing import Optional, cast
+from django.conf import settings
from rest_framework.request import Request
from swh.web.api.apidoc import api_doc, format_docstring
@@ -33,6 +34,27 @@
return docstring
+def _webhook_info_doc() -> str:
+ docstring = ""
+ if "swh.web.save_origin_webhooks" in settings.SWH_DJANGO_APPS:
+ docstring = """
+ :>json boolean from_webhook: indicates if the save request was created
+ from a popular forge webhook receiver
+ (see :http:post:`/api/1/origin/save/webhook/github/` for instance)
+ :>json string webhook_origin: indicates which forge type sent the webhook,
+ currently the supported types are:"""
+
+ # instantiate webhook receivers
+ from swh.web.save_origin_webhooks import urls # noqa
+ from swh.web.save_origin_webhooks.generic_receiver import SUPPORTED_FORGE_TYPES
+
+ webhook_forge_types = sorted(list(SUPPORTED_FORGE_TYPES))
+ for visit_type in webhook_forge_types[:-1]:
+ docstring += f"**{visit_type}**, "
+ docstring += f"and **{webhook_forge_types[-1]}**"
+ return docstring
+
+
save_code_now_api_urls = APIUrls()
@@ -45,7 +67,9 @@
api_urls=save_code_now_api_urls,
)
@api_doc("/origin/save/", category="Request archival")
-@format_docstring(visit_types=_savable_visit_types())
+@format_docstring(
+ visit_types=_savable_visit_types(), webhook_info_doc=_webhook_info_doc()
+)
def api_save_origin(request: Request, visit_type: str, origin_url: str):
"""
.. http:get:: /api/1/origin/save/(visit_type)/url/(origin_url)/
@@ -102,6 +126,7 @@
otherwise.
:>json string note: optional note giving details about the save request,
for instance why it has been rejected
+ {webhook_info_doc}
:statuscode 200: no error
:statuscode 400: an invalid visit type or origin url has been provided
@@ -110,6 +135,13 @@
"""
+ def _cleanup_sor_data(sor):
+ del sor["id"]
+ if "swh.web.save_origin_webhooks" not in settings.SWH_DJANGO_APPS:
+ del sor["from_webhook"]
+ del sor["webhook_origin"]
+ return sor
+
data = request.data or {}
if request.method == "POST":
sor = create_save_origin_request(
@@ -122,10 +154,8 @@
user_id=cast(Optional[int], request.user.id),
**data,
)
- del sor["id"]
- return sor
+ return _cleanup_sor_data(sor)
+
else:
sors = get_save_origin_requests(visit_type, origin_url)
- for sor in sors:
- del sor["id"]
- return sors
+ return [_cleanup_sor_data(sor) for sor in sors]
diff --git a/swh/web/save_code_now/migrations/0013_saveoriginrequest_webhook_info.py b/swh/web/save_code_now/migrations/0013_saveoriginrequest_webhook_info.py
new file mode 100644
--- /dev/null
+++ b/swh/web/save_code_now/migrations/0013_saveoriginrequest_webhook_info.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("swh_web_save_code_now", "0012_saveoriginrequest_note"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="saveoriginrequest",
+ name="from_webhook",
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name="saveoriginrequest",
+ name="webhook_origin",
+ field=models.CharField(max_length=200, null=True),
+ ),
+ ]
diff --git a/swh/web/save_code_now/models.py b/swh/web/save_code_now/models.py
--- a/swh/web/save_code_now/models.py
+++ b/swh/web/save_code_now/models.py
@@ -103,6 +103,8 @@
# store ids of users that submitted the request as string list
user_ids = models.TextField(null=True)
note = models.TextField(null=True)
+ from_webhook = models.BooleanField(default=False)
+ webhook_origin = models.CharField(max_length=200, null=True)
class Meta:
app_label = "swh_web_save_code_now"
@@ -129,6 +131,8 @@
visit_date=visit_date.isoformat() if visit_date else None,
loading_task_id=self.loading_task_id,
note=self.note,
+ from_webhook=self.from_webhook,
+ webhook_origin=self.webhook_origin,
)
def __str__(self) -> str:
diff --git a/swh/web/save_code_now/origin_save.py b/swh/web/save_code_now/origin_save.py
--- a/swh/web/save_code_now/origin_save.py
+++ b/swh/web/save_code_now/origin_save.py
@@ -404,6 +404,8 @@
origin_url: str,
privileged_user: bool = False,
user_id: Optional[int] = None,
+ from_webhook: bool = False,
+ webhook_origin: Optional[str] = None,
**kwargs,
) -> SaveOriginRequestInfo:
"""Create a loading task to save a software origin into the archive.
@@ -426,6 +428,8 @@
privileged: Whether the user has some more privilege than other (bypass
review, access to privileged other visit types)
user_id: User identifier (provided when authenticated)
+ from_webhook: Indicates if the save request is created from a webhook receiver
+ webhook_origin: Indicates which forge type sent the webhook
kwargs: Optional parameters (e.g. artifact_url, artifact_filename,
artifact_version)
@@ -545,6 +549,8 @@
status=save_request_status,
loading_task_id=task["id"],
user_ids=f'"{user_id}"' if user_id else None,
+ from_webhook=from_webhook,
+ webhook_origin=webhook_origin,
)
# save request must be manually reviewed for acceptation
@@ -568,6 +574,8 @@
origin_url=origin_url,
status=save_request_status,
user_ids=f'"{user_id}"' if user_id else None,
+ from_webhook=from_webhook,
+ webhook_origin=webhook_origin,
)
# origin can not be saved as its url is blacklisted,
# log the request to the database anyway
@@ -577,6 +585,8 @@
origin_url=origin_url,
status=save_request_status,
user_ids=f'"{user_id}"' if user_id else None,
+ from_webhook=from_webhook,
+ webhook_origin=webhook_origin,
)
if save_request_status == SAVE_REQUEST_REJECTED:
diff --git a/swh/web/save_code_now/tests/test_migrations.py b/swh/web/save_code_now/tests/test_migrations.py
--- a/swh/web/save_code_now/tests/test_migrations.py
+++ b/swh/web/save_code_now/tests/test_migrations.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 The Software Heritage developers
+# Copyright (C) 2021-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -10,6 +10,7 @@
MIGRATION_0010 = "0010_saveoriginrequest_user_id"
MIGRATION_0011 = "0011_saveoriginrequest_user_ids"
MIGRATION_0012 = "0012_saveoriginrequest_note"
+MIGRATION_0013 = "0013_saveoriginrequest_webhook_info"
def test_migrations_09_add_visit_status_to_sor_model(migrator):
@@ -58,3 +59,21 @@
new_model = new_state.apps.get_model(APP_NAME, "SaveOriginRequest")
assert hasattr(new_model, "note") is True
+
+
+def test_migrations_13_add_webhook_info_to_sor_model(migrator):
+ """Ensures the migration adds the from_webhook field to SaveOriginRequest table"""
+
+ old_state = migrator.apply_initial_migration(
+ (APP_NAME, MIGRATION_0012),
+ )
+ old_model = old_state.apps.get_model(APP_NAME, "SaveOriginRequest")
+
+ assert hasattr(old_model, "from_webhook") is False
+ assert hasattr(old_model, "webhook_origin") is False
+
+ new_state = migrator.apply_tested_migration((APP_NAME, MIGRATION_0013))
+ new_model = new_state.apps.get_model(APP_NAME, "SaveOriginRequest")
+
+ assert hasattr(new_model, "from_webhook") is True
+ assert hasattr(new_model, "webhook_origin") is True
diff --git a/swh/web/save_code_now/tests/test_origin_save.py b/swh/web/save_code_now/tests/test_origin_save.py
--- a/swh/web/save_code_now/tests/test_origin_save.py
+++ b/swh/web/save_code_now/tests/test_origin_save.py
@@ -338,6 +338,8 @@
visit_date=_visit_date.isoformat() if _visit_date else None,
loading_task_id=sor.loading_task_id,
note=note,
+ from_webhook=False,
+ webhook_origin=None,
)
diff --git a/swh/web/save_code_now/tests/test_origin_save_api.py b/swh/web/save_code_now/tests/test_origin_save_api.py
--- a/swh/web/save_code_now/tests/test_origin_save_api.py
+++ b/swh/web/save_code_now/tests/test_origin_save_api.py
@@ -95,6 +95,8 @@
response = check_api_post_responses(api_client, url, data=None, status_code=200)
assert response.data["save_request_status"] == expected_request_status
assert response.data["save_task_status"] == expected_task_status
+ assert response.data["from_webhook"] is False
+ assert response.data["webhook_origin"] is None
else:
check_api_post_responses(api_client, url, data=None, status_code=403)
@@ -139,6 +141,8 @@
assert save_request_data["save_request_status"] == expected_request_status
assert save_request_data["save_task_status"] == expected_task_status
assert save_request_data["visit_status"] == visit_status
+ assert save_request_data["from_webhook"] is False
+ assert save_request_data["webhook_origin"] is None
if scheduler_task_run_status is not None:
# Check that save task status is still available when
diff --git a/swh/web/save_origin_webhooks/generic_receiver.py b/swh/web/save_origin_webhooks/generic_receiver.py
--- a/swh/web/save_origin_webhooks/generic_receiver.py
+++ b/swh/web/save_origin_webhooks/generic_receiver.py
@@ -16,6 +16,9 @@
webhooks_api_urls = APIUrls()
+SUPPORTED_FORGE_TYPES = set()
+
+
class OriginSaveWebhookReceiver(abc.ABC):
FORGE_TYPE: str
WEBHOOK_GUIDE_URL: str
@@ -64,6 +67,7 @@
request or missing data in webhook payload
"""
self.__name__ = "api_origin_save_webhook_{self.FORGE_TYPE.lower()}"
+ SUPPORTED_FORGE_TYPES.add(self.FORGE_TYPE.lower())
api_doc(
f"/origin/save/webhook/{self.FORGE_TYPE.lower()}/",
category="Request archival",
@@ -118,7 +122,10 @@
)
save_request = create_save_origin_request(
- visit_type=visit_type, origin_url=repo_url
+ visit_type=visit_type,
+ origin_url=repo_url,
+ from_webhook=True,
+ webhook_origin=self.FORGE_TYPE.lower(),
)
return {
diff --git a/swh/web/save_origin_webhooks/tests/utils.py b/swh/web/save_origin_webhooks/tests/utils.py
--- a/swh/web/save_origin_webhooks/tests/utils.py
+++ b/swh/web/save_origin_webhooks/tests/utils.py
@@ -5,6 +5,7 @@
from typing import Any, Dict
+from swh.web.save_code_now.models import SaveOriginRequest
from swh.web.tests.helpers import check_api_post_responses
from swh.web.utils import reverse
@@ -40,6 +41,11 @@
task = dict(tasks[0].items())
assert task["arguments"]["kwargs"]["url"] == expected_origin_url
+ request = SaveOriginRequest.objects.get(
+ origin_url=expected_origin_url, visit_type=expected_visit_type
+ )
+ assert request.from_webhook
+
def origin_save_webhook_receiver_invalid_request_test(
forge_type: str,
diff --git a/swh/web/utils/typing.py b/swh/web/utils/typing.py
--- a/swh/web/utils/typing.py
+++ b/swh/web/utils/typing.py
@@ -248,6 +248,10 @@
"""Status of the scheduled task"""
note: Optional[str]
"""Optional note associated to the request, for instance rejection reason"""
+ from_webhook: bool
+ """Indicates if request was created from a webhook receiver"""
+ webhook_origin: Optional[str]
+ """Indicates from which forge type a webhook was received"""
class OriginExistenceCheckInfo(TypedDict):
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, Aug 17, 11:06 PM (6 d, 23 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217361
Attached To
D8889: save_code_now: Add from_webhook field to SaveOriginRequest model
Event Timeline
Log In to Comment