diff --git a/swh/web/client/cli.py b/swh/web/client/cli.py --- a/swh/web/client/cli.py +++ b/swh/web/client/cli.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -51,6 +51,7 @@ if not conf: raise ValueError(f"Cannot parse configuration file: {config_file}") + # TODO: Determine what the following conditional is for if config_file == DEFAULT_CONFIG_PATH: try: conf = conf["swh"]["web"]["client"] @@ -126,6 +127,57 @@ sys.stderr.close() +@web.group(name="save", context_settings=CONTEXT_SETTINGS) +@click.pass_context +def savecodenow(ctx: Context,): + """Subcommand to interact from the cli with the save code now feature + + """ + pass + + +@savecodenow.command("submit-request") +@click.option("--delimiter", "-d", default=",") +@click.pass_context +def submit_request(ctx, delimiter: str) -> None: + """Submit new save code now request through cli pipe. The expected format of the request + if one csv row ``,``. + + Example: + + cat list-origins | swh web save submit-request + + echo svn;https://svn-url\ngit;https://git-url | swh web save \ + submit-request --delimiter ';' + + Prints: + The output of save code now requests as json output. + + """ + import json + import logging + import sys + + logging.basicConfig(level=logging.INFO, stream=sys.stderr) + + client = ctx.obj["client"] + + processed_origins = [] + for origin in sys.stdin: + visit_type, origin = origin.rstrip().split(delimiter) + + try: + saved_origin = client.origin_save(visit_type, origin) + logging.info("Submitted origin (%s, %s)", visit_type, origin) + processed_origins.append(saved_origin) + except Exception as e: + logging.warning( + "Issue for origin (%s, %s)\n%s", origin, visit_type, e, + ) + logging.debug("Origin saved: %s", len(processed_origins)) + print(json.dumps(processed_origins)) + + @web.group(name="auth", context_settings=CONTEXT_SETTINGS) @click.option( "--oidc-server-url", diff --git a/swh/web/client/client.py b/swh/web/client/client.py --- a/swh/web/client/client.py +++ b/swh/web/client/client.py @@ -608,3 +608,21 @@ q = r.links["next"]["url"] else: done = True + + def origin_save(self, visit_type: str, origin: str) -> Dict: + """Save code now query for the origin with visit_type. + + Args: + visit_type: Type of the visit + origin: the origin to save + + Returns: + The resulting dict of the visit saved + + Raises: + requests.HTTPError: if HTTP request fails + + """ + q = f"origin/save/{visit_type}/url/{origin}/" + r = self._call(q, http_method="post") + return r.json() diff --git a/swh/web/client/tests/api_data_static.py b/swh/web/client/tests/api_data_static.py new file mode 100644 --- /dev/null +++ b/swh/web/client/tests/api_data_static.py @@ -0,0 +1,38 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +API_DATA_STATIC = { + "origin/save/git/url/https://gitlab.org/gazelle/itest/": r""" + { + "visit_type": "git", + "origin_url": "https://gitlab.org/gazelle/itest", + "save_request_date": "2021-04-20T11:34:38.752929+00:00", + "save_request_status": "accepted", + "save_task_status": "not yet scheduled", + "visit_date": null + } + """, + "origin/save/git/url/https://git.renater.fr/anonscm/git/6po/6po.git/": r""" + { + "visit_type": "git", + "origin_url": "https://git.renater.fr/anonscm/git/6po/6po.git", + "save_request_date": "2021-04-20T11:34:40.115226+00:00", + "save_request_status": "accepted", + "save_task_status": "not yet scheduled", + "visit_date": null + } + """, + "origin/save/git/url/https://github.com/colobot/colobot/": r""" + { + "visit_type": "git", + "origin_url": "https://github.com/colobot/colobot", + "save_request_date": "2021-04-20T11:40:47.667492+00:00", + "save_request_status": "accepted", + "save_task_status": "not yet scheduled", + "visit_date": null + } + """, +} diff --git a/swh/web/client/tests/conftest.py b/swh/web/client/tests/conftest.py --- a/swh/web/client/tests/conftest.py +++ b/swh/web/client/tests/conftest.py @@ -1,13 +1,17 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os + import pytest +import yaml from swh.web.client.client import WebAPIClient from .api_data import API_DATA, API_URL +from .api_data_static import API_DATA_STATIC @pytest.fixture @@ -42,6 +46,10 @@ requests_mock.register_uri("POST", f"{API_URL}/known/", json=known_callback) + # Add some other post urls to mock + for api_call, data in API_DATA_STATIC.items(): + requests_mock.post(f"{API_URL}/{api_call}", text=data) + return requests_mock @@ -49,3 +57,27 @@ def web_api_client(): # use the fake base API URL that matches API data return WebAPIClient(api_url=API_URL) + + +@pytest.fixture +def cli_global_config_dict(): + """Define a basic configuration yaml for the cli. + + """ + return { + "api_url": API_URL, + "bearer_token": None, + } + + +@pytest.fixture +def cli_config_path(tmp_path, cli_global_config_dict, monkeypatch): + """Write a global.yml file and writes it in the environment + + """ + config_path = os.path.join(tmp_path, "global.yml") + with open(config_path, "w") as f: + f.write(yaml.dump(cli_global_config_dict)) + monkeypatch.setenv("SWH_CONFIG_FILE", config_path) + + return config_path diff --git a/swh/web/client/tests/test_cli.py b/swh/web/client/tests/test_cli.py --- a/swh/web/client/tests/test_cli.py +++ b/swh/web/client/tests/test_cli.py @@ -1,11 +1,14 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import json +import os + from click.testing import CliRunner -from swh.web.client.cli import auth +from swh.web.client.cli import auth, web runner = CliRunner() @@ -52,3 +55,61 @@ mock_logout.side_effect = Exception("Auth error") result = runner.invoke(auth, [command, oidc_profile["refresh_token"]]) assert result.exit_code == 1 + + +def test_save_code_now_through_cli(mocker, web_api_mock, tmp_path, cli_config_path): + """Trigger save code now from the cli creates new save code now requests""" + origins = [ + ("git", "https://gitlab.org/gazelle/itest"), + ("git", "https://git.renater.fr/anonscm/git/6po/6po.git"), + ("git", "https://github.com/colobot/colobot"), + # this will be rejected + ("tig", "invalid-and-refusing-to-save-this"), + ] + origins_csv = "\n".join(map(lambda t: ",".join(t), origins)) + origins_csv = f"{origins_csv}\n" + + temp_file = os.path.join(tmp_path, "tmp.csv") + with open(temp_file, "w") as f: + f.write(origins_csv) + + with open(temp_file, "r") as f: + result = runner.invoke( + web, + ["--config-file", cli_config_path, "save", "submit-request"], + input=f, + catch_exceptions=False, + ) + + assert result.exit_code == 0, f"Unexpected output: {result.output}" + actual_save_requests = json.loads(result.output.strip()) + assert len(actual_save_requests) == 3 + + expected_save_requests = [ + { + "origin_url": "https://gitlab.org/gazelle/itest", + "save_request_date": "2021-04-20T11:34:38.752929+00:00", + "save_request_status": "accepted", + "save_task_status": "not yet scheduled", + "visit_date": None, + "visit_type": "git", + }, + { + "origin_url": "https://git.renater.fr/anonscm/git/6po/6po.git", + "save_request_date": "2021-04-20T11:34:40.115226+00:00", + "save_request_status": "accepted", + "save_task_status": "not yet scheduled", + "visit_date": None, + "visit_type": "git", + }, + { + "origin_url": "https://github.com/colobot/colobot", + "save_request_date": "2021-04-20T11:40:47.667492+00:00", + "save_request_status": "accepted", + "save_task_status": "not yet scheduled", + "visit_date": None, + "visit_type": "git", + }, + ] + for actual_save_request in actual_save_requests: + assert actual_save_request in expected_save_requests diff --git a/swh/web/client/tests/test_web_api_client.py b/swh/web/client/tests/test_web_api_client.py --- a/swh/web/client/tests/test_web_api_client.py +++ b/swh/web/client/tests/test_web_api_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020 The Software Heritage developers +# Copyright (C) 2020-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -6,6 +6,7 @@ import json from dateutil.parser import parse as parse_date +import pytest from swh.model.identifiers import REVISION, CoreSWHID from swh.web.client.client import typify_json @@ -177,6 +178,23 @@ assert visit in actual_visits +@pytest.mark.parametrize( + "visit_type,origin", + [ + ("git", "https://gitlab.org/gazelle/itest"), + ("git", "https://git.renater.fr/anonscm/git/6po/6po.git"), + ("git", "https://github.com/colobot/colobot"), + ], +) +def test_origin_save(visit_type, origin, web_api_client, web_api_mock): + """Post save code now is allowed from the client.""" + save_request = web_api_client.origin_save(visit_type, origin) + + assert save_request is not None + assert save_request["save_request_status"] == "accepted" + assert save_request["visit_date"] is None + + def test_known(web_api_client, web_api_mock): # full list of SWHIDs for which we mock a {known: True} answer known_swhids = [