diff --git a/swh/deposit/cli/admin.py b/swh/deposit/cli/admin.py index 2ffcfb0b..e009574c 100644 --- a/swh/deposit/cli/admin.py +++ b/swh/deposit/cli/admin.py @@ -1,284 +1,285 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information # WARNING: do not import unnecessary things here to keep cli startup time under # control from __future__ import annotations from typing import TYPE_CHECKING import click from swh.deposit.cli import deposit if TYPE_CHECKING: from swh.deposit.models import DepositCollection @deposit.group("admin") @click.option( "--config-file", "-C", default=None, type=click.Path(exists=True, dir_okay=False,), help="Optional extra configuration file.", ) @click.option( "--platform", default="development", type=click.Choice(["development", "production"]), help="development or production platform", ) @click.pass_context def admin(ctx, config_file: str, platform: str): """Server administration tasks (manipulate user or collections)""" from swh.deposit.config import setup_django_for # configuration happens here setup_django_for(platform, config_file=config_file) @admin.group("user") @click.pass_context def user(ctx): """Manipulate user.""" # configuration happens here pass def _create_collection(name: str) -> DepositCollection: """Create the collection with name if it does not exist. Args: name: collection name Returns: collection: the existing collection object """ # to avoid loading too early django namespaces from swh.deposit.models import DepositCollection try: collection = DepositCollection.objects.get(name=name) click.echo(f"Collection '{name}' exists, skipping.") except DepositCollection.DoesNotExist: click.echo(f"Create collection '{name}'.") collection = DepositCollection.objects.create(name=name) click.echo(f"Collection '{name}' created.") return collection @user.command("create") @click.option("--username", required=True, help="User's name") @click.option("--password", required=True, help="Desired user's password (plain).") @click.option("--firstname", default="", help="User's first name") @click.option("--lastname", default="", help="User's last name") @click.option("--email", default="", help="User's email") @click.option("--collection", help="User's collection") @click.option("--provider-url", default="", help="Provider URL") @click.option("--domain", default="", help="The domain") @click.pass_context def user_create( ctx, username: str, password: str, firstname: str, lastname: str, email: str, collection: str, provider_url: str, domain: str, ): """Create a user with some needed information (password, collection) If the collection does not exist, the collection is then created alongside. The password is stored encrypted using django's utilities. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositClient # If collection is not provided, fallback to username if not collection: collection = username # create the collection if it does not exist collection_ = _create_collection(collection) # user create/update try: user = DepositClient.objects.get(username=username) # type: ignore click.echo(f"Update user '{username}'.") user.set_password(password) action_done = "updated" except DepositClient.DoesNotExist: click.echo(f"Create user '{username}'.") user = DepositClient.objects.create_user( # type: ignore username=username, password=password ) action_done = "created" user.collections = [collection_.id] user.first_name = firstname user.last_name = lastname user.email = email user.is_active = True user.provider_url = provider_url user.domain = domain user.save() click.echo(f"User '{username}' {action_done}.") @user.command("list") @click.pass_context def user_list(ctx): """List existing users. This entrypoint is not paginated yet as there is not a lot of entry. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositClient users = DepositClient.objects.all() if not users: output = "Empty user list" else: output = "\n".join((user.username for user in users)) click.echo(output) @user.command("exists") @click.argument("username", required=True) @click.pass_context def user_exists(ctx, username: str): """Check if user exists. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositClient try: DepositClient.objects.get(username=username) # type: ignore click.echo(f"User {username} exists.") ctx.exit(0) except DepositClient.DoesNotExist: click.echo(f"User {username} does not exist.") ctx.exit(1) @admin.group("collection") @click.pass_context def collection(ctx): """Manipulate collections.""" pass @collection.command("create") @click.option("--name", required=True, help="Collection's name") @click.pass_context def collection_create(ctx, name): _create_collection(name) @collection.command("list") @click.pass_context def collection_list(ctx): """List existing collections. This entrypoint is not paginated yet as there is not a lot of entry. """ # to avoid loading too early django namespaces from swh.deposit.models import DepositCollection collections = DepositCollection.objects.all() if not collections: output = "Empty collection list" else: output = "\n".join((col.name for col in collections)) click.echo(output) @admin.group("deposit") @click.pass_context def adm_deposit(ctx): """Manipulate deposit.""" pass @adm_deposit.command("reschedule") @click.option("--deposit-id", required=True, help="Deposit identifier") @click.pass_context def adm_deposit_reschedule(ctx, deposit_id): """Reschedule the deposit loading This will: - check the deposit's status to something reasonable (failed or done). That means that the checks have passed alright but something went wrong during the loading (failed: loading failed, done: loading ok, still for some reasons as in bugs, we need to reschedule it) - reset the deposit's status to 'verified' (prior to any loading but after the checks which are fine) and removes the different archives' identifiers (swh-id, ...) - trigger back the loading task through the scheduler """ # to avoid loading too early django namespaces from datetime import datetime from swh.deposit.config import ( DEPOSIT_STATUS_LOAD_FAILURE, DEPOSIT_STATUS_LOAD_SUCCESS, DEPOSIT_STATUS_VERIFIED, APIConfig, ) from swh.deposit.models import Deposit try: deposit = Deposit.objects.get(pk=deposit_id) except Deposit.DoesNotExist: - click.echo("Deposit %s does not exist." % deposit_id) + click.echo(f"Deposit {deposit_id} does not exist.") ctx.exit(1) # Check the deposit is in a reasonable state accepted_statuses = [DEPOSIT_STATUS_LOAD_SUCCESS, DEPOSIT_STATUS_LOAD_FAILURE] if deposit.status == DEPOSIT_STATUS_VERIFIED: - click.echo("Deposit %s's status already set for rescheduling." % (deposit_id)) + click.echo(f"Deposit {deposit_id} already set for rescheduling.") ctx.exit(0) if deposit.status not in accepted_statuses: click.echo( - "Deposit %s's status be one of %s." - % (deposit_id, ", ".join(accepted_statuses)) + f"Deposit {deposit_id} cannot be rescheduled (status: {deposit.status}).\n" + "Rescheduling deposit is only accepted for deposit with status: " + f"{', '.join(accepted_statuses)}." ) ctx.exit(1) task_id = deposit.load_task_id if not task_id: click.echo( - "Deposit %s cannot be rescheduled. It misses the " - "associated task." % deposit_id + f"Deposit {deposit_id} cannot be rescheduled. It misses the " + "associated scheduler task id (field load_task_id)." ) ctx.exit(1) # Reset the deposit's state deposit.swhid = None deposit.swhid_context = None deposit.status = DEPOSIT_STATUS_VERIFIED deposit.save() - # Trigger back the deposit + # Schedule back the deposit loading task scheduler = APIConfig().scheduler scheduler.set_status_tasks( [task_id], status="next_run_not_scheduled", next_run=datetime.now() ) diff --git a/swh/deposit/tests/cli/test_admin.py b/swh/deposit/tests/cli/test_admin.py index 73262aab..c05659b3 100644 --- a/swh/deposit/tests/cli/test_admin.py +++ b/swh/deposit/tests/cli/test_admin.py @@ -1,189 +1,317 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.deposit.cli.admin import admin as cli +from swh.deposit.config import ( + DEPOSIT_STATUS_DEPOSITED, + DEPOSIT_STATUS_PARTIAL, + DEPOSIT_STATUS_VERIFIED, +) from swh.deposit.models import DepositClient, DepositCollection +from swh.scheduler.utils import create_oneshot_task_dict @pytest.fixture(autouse=True) def enable_db_access_for_all_tests(db): pass def test_cli_admin_user_list_nothing(cli_runner): result = cli_runner.invoke(cli, ["user", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == "Empty user list\n" def test_cli_admin_user_list_with_users(cli_runner, deposit_user): result = cli_runner.invoke(cli, ["user", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == f"{deposit_user.username}\n" # only 1 user def test_cli_admin_collection_list_nothing(cli_runner): result = cli_runner.invoke(cli, ["collection", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == "Empty collection list\n" def test_cli_admin_collection_list_with_collections(cli_runner, deposit_collection): from swh.deposit.tests.conftest import create_deposit_collection new_collection = create_deposit_collection("something") result = cli_runner.invoke(cli, ["collection", "list",]) assert result.exit_code == 0, f"Unexpected output: {result.output}" collections = "\n".join([deposit_collection.name, new_collection.name]) assert result.output == f"{collections}\n" def test_cli_admin_user_exists_unknown(cli_runner): result = cli_runner.invoke(cli, ["user", "exists", "unknown"]) assert result.exit_code == 1, f"Unexpected output: {result.output}" assert result.output == "User unknown does not exist.\n" def test_cli_admin_user_exists(cli_runner, deposit_user): result = cli_runner.invoke(cli, ["user", "exists", deposit_user.username]) assert result.exit_code == 0, f"Unexpected output: {result.output}" assert result.output == f"User {deposit_user.username} exists.\n" def test_cli_admin_create_collection(cli_runner): collection_name = "something" try: DepositCollection.objects.get(name=collection_name) except DepositCollection.DoesNotExist: pass result = cli_runner.invoke( cli, ["collection", "create", "--name", collection_name,] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" collection = DepositCollection.objects.get(name=collection_name) assert collection is not None assert ( result.output == f"""Create collection '{collection_name}'. Collection '{collection_name}' created. """ ) result2 = cli_runner.invoke( cli, ["collection", "create", "--name", collection_name,] ) assert result2.exit_code == 0, f"Unexpected output: {result.output}" assert ( result2.output == f"""Collection '{collection_name}' exists, skipping. """ ) def test_cli_admin_user_create(cli_runner): user_name = "user" collection_name = user_name try: DepositClient.objects.get(username=user_name) except DepositClient.DoesNotExist: pass try: DepositCollection.objects.get(name=collection_name) except DepositCollection.DoesNotExist: pass result = cli_runner.invoke( cli, ["user", "create", "--username", user_name, "--password", "password",] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" user = DepositClient.objects.get(username=user_name) assert user is not None collection = DepositCollection.objects.get(name=collection_name) assert collection is not None assert ( result.output == f"""Create collection '{user_name}'. Collection '{collection_name}' created. Create user '{user_name}'. User '{user_name}' created. """ ) assert collection.name == collection_name assert user.username == user_name first_password = user.password assert first_password is not None assert user.collections == [collection.id] assert user.is_active is True assert user.domain == "" assert user.provider_url == "" assert user.email == "" assert user.first_name == "" assert user.last_name == "" # create a user that already exists result2 = cli_runner.invoke( cli, [ "user", "create", "--username", "user", "--password", "another-password", # changing password "--collection", collection_name, # specifying the collection this time "--firstname", "User", "--lastname", "no one", "--email", "user@org.org", "--provider-url", "http://some-provider.org", "--domain", "domain", ], ) assert result2.exit_code == 0, f"Unexpected output: {result2.output}" user = DepositClient.objects.get(username=user_name) assert user is not None assert user.username == user_name assert user.collections == [collection.id] assert user.is_active is True second_password = user.password assert second_password is not None assert second_password != first_password, "Password should have changed" assert user.domain == "domain" assert user.provider_url == "http://some-provider.org" assert user.email == "user@org.org" assert user.first_name == "User" assert user.last_name == "no one" assert ( result2.output == f"""Collection '{collection_name}' exists, skipping. Update user '{user_name}'. User '{user_name}' updated. """ ) + + +def test_cli_admin_reschedule_unknown_deposit(cli_runner): + """Rescheduling unknown deposit should report failure + + """ + unknown_deposit_id = 666 + + from swh.deposit.models import Deposit + + try: + Deposit.objects.get(id=unknown_deposit_id) + except Deposit.DoesNotExist: + pass + + result = cli_runner.invoke( + cli, ["deposit", "reschedule", "--deposit-id", unknown_deposit_id] + ) + + assert result.output == f"Deposit {unknown_deposit_id} does not exist.\n" + assert result.exit_code == 1 + + +def test_cli_admin_reschedule_verified_deposit(cli_runner, complete_deposit): + """Rescheduling verified deposit should do nothing but report + + """ + deposit = complete_deposit + deposit.status = "verified" + deposit.save() + + result = cli_runner.invoke( + cli, ["deposit", "reschedule", "--deposit-id", deposit.id] + ) + + assert result.output == f"Deposit {deposit.id} already set for rescheduling.\n" + assert result.exit_code == 0 + + +@pytest.mark.parametrize( + "status_to_check", [DEPOSIT_STATUS_PARTIAL, DEPOSIT_STATUS_DEPOSITED] +) +def test_cli_admin_reschedule_unaccepted_deposit_status( + status_to_check, cli_runner, complete_deposit +): + """Rescheduling verified deposit should do nothing but report + + """ + deposit = complete_deposit + deposit.status = status_to_check # not accepted status will fail the check + deposit.save() + + result = cli_runner.invoke( + cli, ["deposit", "reschedule", "--deposit-id", deposit.id] + ) + + assert result.output == ( + f"Deposit {deposit.id} cannot be rescheduled (status: {deposit.status}).\n" + "Rescheduling deposit is only accepted for deposit with status: done, failed.\n" + ) + assert result.exit_code == 1 + + +def test_cli_admin_reschedule_missing_task_id(cli_runner, complete_deposit): + """Rescheduling deposit with no load_task_id cannot work. + + """ + deposit = complete_deposit + deposit.load_task_id = "" # drop the load-task-id so it fails the check + deposit.save() + + result = cli_runner.invoke( + cli, ["deposit", "reschedule", "--deposit-id", deposit.id] + ) + + assert result.output == ( + f"Deposit {deposit.id} cannot be rescheduled. It misses the " + "associated scheduler task id (field load_task_id).\n" + ) + assert result.exit_code == 1 + + +def test_cli_admin_reschedule_nominal(cli_runner, complete_deposit, swh_scheduler): + """Rescheduling deposit with no load_task_id cannot work. + + """ + deposit = complete_deposit + + from swh.deposit.models import Deposit + + # create a task to keep a reference on it + task = create_oneshot_task_dict( + "load-deposit", url=deposit.origin_url, deposit_id=deposit.id, retries_left=3 + ) + scheduled_task = swh_scheduler.create_tasks([task])[0] + # disable it + swh_scheduler.set_status_tasks([scheduled_task["id"]], status="disabled") + + # Now update the deposit state with some swhid and relevant load_task_id + deposit = complete_deposit + deposit.load_task_id = scheduled_task["id"] + deposit.swhid = "swh:1:dir:02ed6084fb0e8384ac58980e07548a547431cf74" + deposit.swhid_context = f"{deposit.swhid};origin=https://url/external-id" + deposit.save() + + # Reschedule it + result = cli_runner.invoke( + cli, ["deposit", "reschedule", "--deposit-id", deposit.id] + ) + assert result.exit_code == 0 + + # Now, ensure the deposit and the associated task are in the right shape + deposit = Deposit.objects.get(id=deposit.id) + + # got reset to a state which allows rescheduling + assert deposit.id + assert deposit.swhid is None + assert deposit.swhid_context is None + assert deposit.status == DEPOSIT_STATUS_VERIFIED + + task = swh_scheduler.search_tasks(task_id=deposit.load_task_id)[0] + assert task["status"] == "next_run_not_scheduled"