diff --git a/swh/web/save_origin_webhooks/bitbucket.py b/swh/web/save_origin_webhooks/bitbucket.py --- a/swh/web/save_origin_webhooks/bitbucket.py +++ b/swh/web/save_origin_webhooks/bitbucket.py @@ -28,7 +28,7 @@ def is_push_event(self, request: Request) -> bool: return request.headers["X-Event-Key"] == "repo:push" - def extract_repo_url_and_visit_type(self, request: Request) -> Tuple[str, str]: + def extract_repo_info(self, request: Request) -> Tuple[str, str, bool]: repo_url = ( request.data.get("repository", {}) .get("links", {}) @@ -38,7 +38,9 @@ if repo_url: repo_url += ".git" - return repo_url, "git" + private = request.data.get("repository", {}).get("is_private", False) + + return repo_url, "git", private api_origin_save_webhook_bitbucket = BitbucketOriginSaveWebhookReceiver() diff --git a/swh/web/save_origin_webhooks/generic_receiver.py b/swh/web/save_origin_webhooks/generic_receiver.py --- a/swh/web/save_origin_webhooks/generic_receiver.py +++ b/swh/web/save_origin_webhooks/generic_receiver.py @@ -33,7 +33,9 @@ ... @abc.abstractmethod - def extract_repo_url_and_visit_type(self, request: Request) -> Tuple[str, str]: + def extract_repo_info(self, request: Request) -> Tuple[str, str, bool]: + """Extract and return a tuple (repository_url, visit_type, private) from + the forge webhook payload.""" ... def __init__(self): @@ -100,7 +102,7 @@ f"{self.FORGE_TYPE} webhook, it should be 'application/json'." ) - repo_url, visit_type = self.extract_repo_url_and_visit_type(request) + repo_url, visit_type, private = self.extract_repo_info(request) if not repo_url: raise BadInputExc( f"Repository URL could not be extracted from {self.FORGE_TYPE} webhook " @@ -110,6 +112,10 @@ raise BadInputExc( f"Visit type could not be determined for repository {repo_url}." ) + if private: + raise BadInputExc( + f"Repository {repo_url} is private and cannot be cloned without authentication." + ) save_request = create_save_origin_request( visit_type=visit_type, origin_url=repo_url diff --git a/swh/web/save_origin_webhooks/gitea.py b/swh/web/save_origin_webhooks/gitea.py --- a/swh/web/save_origin_webhooks/gitea.py +++ b/swh/web/save_origin_webhooks/gitea.py @@ -21,8 +21,10 @@ def is_push_event(self, request: Request) -> bool: return request.headers[f"X-{self.FORGE_TYPE}-Event"] == "push" - def extract_repo_url_and_visit_type(self, request: Request) -> Tuple[str, str]: - return request.data.get("repository", {}).get("clone_url", ""), "git" + def extract_repo_info(self, request: Request) -> Tuple[str, str, bool]: + repo_url = request.data.get("repository", {}).get("clone_url", "") + private = request.data.get("repository", {}).get("private", False) + return repo_url, "git", private api_origin_save_webhook_gitea = GiteaOriginSaveWebhookReceiver() diff --git a/swh/web/save_origin_webhooks/github.py b/swh/web/save_origin_webhooks/github.py --- a/swh/web/save_origin_webhooks/github.py +++ b/swh/web/save_origin_webhooks/github.py @@ -32,8 +32,10 @@ def is_push_event(self, request: Request) -> bool: return request.headers[f"X-{self.FORGE_TYPE}-Event"] == "push" - def extract_repo_url_and_visit_type(self, request: Request) -> Tuple[str, str]: - return request.data.get("repository", {}).get("html_url", ""), "git" + def extract_repo_info(self, request: Request) -> Tuple[str, str, bool]: + repo_url = request.data.get("repository", {}).get("html_url", "") + private = request.data.get("repository", {}).get("private", False) + return repo_url, "git", private api_origin_save_webhook_github = GitHubOriginSaveWebhookReceiver() diff --git a/swh/web/save_origin_webhooks/gitlab.py b/swh/web/save_origin_webhooks/gitlab.py --- a/swh/web/save_origin_webhooks/gitlab.py +++ b/swh/web/save_origin_webhooks/gitlab.py @@ -27,8 +27,13 @@ def is_push_event(self, request: Request) -> bool: return request.headers["X-Gitlab-Event"] == "Push Hook" - def extract_repo_url_and_visit_type(self, request: Request) -> Tuple[str, str]: - return request.data.get("repository", {}).get("git_http_url", ""), "git" + def extract_repo_info(self, request: Request) -> Tuple[str, str, bool]: + repo_url = request.data.get("repository", {}).get("git_http_url", "") + # visibility_level values: 0 = private, 10 = internal, 20 = public + visibility_level = request.data.get("repository", {}).get( + "visibility_level", 20 + ) + return repo_url, "git", visibility_level != 20 api_origin_save_webhook_gitlab = GitlabOriginSaveWebhookReceiver() diff --git a/swh/web/save_origin_webhooks/sourceforge.py b/swh/web/save_origin_webhooks/sourceforge.py --- a/swh/web/save_origin_webhooks/sourceforge.py +++ b/swh/web/save_origin_webhooks/sourceforge.py @@ -34,9 +34,10 @@ # SourceForge only support webhooks for push events return True - def extract_repo_url_and_visit_type(self, request: Request) -> Tuple[str, str]: + def extract_repo_info(self, request: Request) -> Tuple[str, str, bool]: repo_url = "" visit_type = "" + private = False project_full_name = request.data.get("repository", {}).get("full_name") if project_full_name: project_name = project_full_name.split("/")[2] @@ -46,6 +47,7 @@ response = requests.get(project_api_url) if response.ok: project_data = response.json() + private = project_data.get("private", False) for tool in project_data.get("tools", []): if tool.get("mount_point") == "code" and tool.get( "url", "" @@ -55,7 +57,7 @@ ) visit_type = tool.get("name", "") - return repo_url, visit_type + return repo_url, visit_type, private api_origin_save_webhook_sourceforge = SourceforgeOriginSaveWebhookReceiver() diff --git a/swh/web/save_origin_webhooks/tests/test_bitbucket.py b/swh/web/save_origin_webhooks/tests/test_bitbucket.py --- a/swh/web/save_origin_webhooks/tests/test_bitbucket.py +++ b/swh/web/save_origin_webhooks/tests/test_bitbucket.py @@ -13,6 +13,7 @@ origin_save_webhook_receiver_invalid_event_test, origin_save_webhook_receiver_invalid_request_test, origin_save_webhook_receiver_no_repo_url_test, + origin_save_webhook_receiver_private_repo_test, origin_save_webhook_receiver_test, ) @@ -86,3 +87,19 @@ payload=payload, api_client=api_client, ) + + +def test_origin_save_bitbucket_webhook_receiver_private_repo(api_client, datadir): + with open(os.path.join(datadir, "bitbucket_webhook_payload.json"), "rb") as payload: + payload = json.load(payload) + payload["repository"]["is_private"] = True + origin_save_webhook_receiver_private_repo_test( + forge_type="Bitbucket", + http_headers={ + "User-Agent": "Bitbucket-Webhooks/2.0", + "X-Event-Key": "repo:push", + }, + payload=payload, + expected_origin_url="https://bitbucket.org/johndoe/webhook-test.git", + api_client=api_client, + ) diff --git a/swh/web/save_origin_webhooks/tests/test_gitea.py b/swh/web/save_origin_webhooks/tests/test_gitea.py --- a/swh/web/save_origin_webhooks/tests/test_gitea.py +++ b/swh/web/save_origin_webhooks/tests/test_gitea.py @@ -15,6 +15,7 @@ origin_save_webhook_receiver_invalid_event_test, origin_save_webhook_receiver_invalid_request_test, origin_save_webhook_receiver_no_repo_url_test, + origin_save_webhook_receiver_private_repo_test, origin_save_webhook_receiver_test, ) @@ -85,3 +86,18 @@ payload=payload, api_client=api_client, ) + + +def test_origin_save_gitea_webhook_receiver_private_repo(api_client, datadir): + with open(os.path.join(datadir, "gitea_webhook_payload.json"), "rb") as payload: + payload = json.load(payload) + payload["repository"]["private"] = True + origin_save_webhook_receiver_private_repo_test( + forge_type="Gitea", + http_headers={ + "X-Gitea-Event": "push", + }, + payload=payload, + api_client=api_client, + expected_origin_url="https://try.gitea.io/johndoe/webhook-test.git", + ) diff --git a/swh/web/save_origin_webhooks/tests/test_github.py b/swh/web/save_origin_webhooks/tests/test_github.py --- a/swh/web/save_origin_webhooks/tests/test_github.py +++ b/swh/web/save_origin_webhooks/tests/test_github.py @@ -17,6 +17,7 @@ origin_save_webhook_receiver_invalid_event_test, origin_save_webhook_receiver_invalid_request_test, origin_save_webhook_receiver_no_repo_url_test, + origin_save_webhook_receiver_private_repo_test, origin_save_webhook_receiver_test, ) @@ -108,3 +109,19 @@ ) assert resp.data == {"message": "pong"} + + +def test_origin_save_github_webhook_receiver_private_repo(api_client, datadir): + with open(os.path.join(datadir, "github_webhook_payload.json"), "rb") as payload: + payload = json.load(payload) + payload["repository"]["private"] = True + origin_save_webhook_receiver_private_repo_test( + forge_type="GitHub", + http_headers={ + "User-Agent": "GitHub-Hookshot/ede37db", + "X-GitHub-Event": "push", + }, + payload=payload, + api_client=api_client, + expected_origin_url="https://github.com/johndoe/webhook-test", + ) diff --git a/swh/web/save_origin_webhooks/tests/test_gitlab.py b/swh/web/save_origin_webhooks/tests/test_gitlab.py --- a/swh/web/save_origin_webhooks/tests/test_gitlab.py +++ b/swh/web/save_origin_webhooks/tests/test_gitlab.py @@ -13,6 +13,7 @@ origin_save_webhook_receiver_invalid_event_test, origin_save_webhook_receiver_invalid_request_test, origin_save_webhook_receiver_no_repo_url_test, + origin_save_webhook_receiver_private_repo_test, origin_save_webhook_receiver_test, ) @@ -86,3 +87,19 @@ payload=payload, api_client=api_client, ) + + +def test_origin_save_gitlab_webhook_receiver_private_repo(api_client, datadir): + with open(os.path.join(datadir, "gitlab_webhook_payload.json"), "rb") as payload: + payload = json.load(payload) + payload["repository"]["visibility_level"] = 0 + origin_save_webhook_receiver_private_repo_test( + forge_type="GitLab", + http_headers={ + "User-Agent": "GitLab/15.6.0-pre", + "X-Gitlab-Event": "Push Hook", + }, + payload=payload, + api_client=api_client, + expected_origin_url="https://gitlab.com/johndoe/test.git", + ) diff --git a/swh/web/save_origin_webhooks/tests/test_sourceforge.py b/swh/web/save_origin_webhooks/tests/test_sourceforge.py --- a/swh/web/save_origin_webhooks/tests/test_sourceforge.py +++ b/swh/web/save_origin_webhooks/tests/test_sourceforge.py @@ -7,11 +7,13 @@ import os import pytest +import requests from .utils import ( origin_save_webhook_receiver_invalid_content_type_test, origin_save_webhook_receiver_invalid_request_test, origin_save_webhook_receiver_no_repo_url_test, + origin_save_webhook_receiver_private_repo_test, origin_save_webhook_receiver_test, ) @@ -98,3 +100,50 @@ payload=payload, api_client=api_client, ) + + +@pytest.mark.parametrize( + "payload_file,origin_url,visit_type", + [ + ( + "sourceforge_webhook_payload_hg.json", + "http://hg.code.sf.net/p/webhook-test-hg/code", + "hg", + ), + ( + "sourceforge_webhook_payload_git.json", + "https://git.code.sf.net/p/webhook-test-git/code", + "git", + ), + ( + "sourceforge_webhook_payload_svn.json", + "https://svn.code.sf.net/p/webhook-test-svn/code/", + "svn", + ), + ], +) +def test_origin_save_sourceforge_webhook_receiver_private_repo( + api_client, + datadir, + requests_mock_datadir, + requests_mock, + payload_file, + origin_url, + visit_type, +): + # override sourceforge REST API response + repo_data_url = f"https://sourceforge.net/rest/p/webhook-test-{visit_type}" + repo_data = requests.get(repo_data_url).json() + repo_data["private"] = True + requests_mock.get(repo_data_url, json=repo_data) + + with open(os.path.join(datadir, payload_file), "rb") as payload: + origin_save_webhook_receiver_private_repo_test( + forge_type="SourceForge", + http_headers={ + "User-Agent": "Allura Webhook (https://allura.apache.org/)", + }, + payload=json.load(payload), + expected_origin_url=origin_url, + api_client=api_client, + ) diff --git a/swh/web/save_origin_webhooks/tests/utils.py b/swh/web/save_origin_webhooks/tests/utils.py --- a/swh/web/save_origin_webhooks/tests/utils.py +++ b/swh/web/save_origin_webhooks/tests/utils.py @@ -141,3 +141,29 @@ f"Repository URL could not be extracted from {forge_type} webhook payload." ), } + + +def origin_save_webhook_receiver_private_repo_test( + forge_type: str, + http_headers: Dict[str, Any], + payload: Dict[str, Any], + api_client, + expected_origin_url: str, +): + url = reverse(f"api-1-origin-save-webhook-{forge_type.lower()}") + + resp = check_api_post_responses( + api_client, + url, + status_code=400, + data=payload, + **django_http_headers(http_headers), + ) + + assert resp.data == { + "exception": "BadInputExc", + "reason": ( + f"Repository {expected_origin_url} is private and cannot be cloned " + "without authentication." + ), + }