diff --git a/swh/deposit/api/common.py b/swh/deposit/api/common.py --- a/swh/deposit/api/common.py +++ b/swh/deposit/api/common.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2021 The Software Heritage developers +# Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -27,7 +27,7 @@ from swh.deposit.api.checks import check_metadata from swh.deposit.api.converters import convert_status_detail from swh.deposit.auth import HasDepositPermission, KeycloakBasicAuthentication -from swh.deposit.models import Deposit +from swh.deposit.models import DEPOSIT_METADATA_ONLY, Deposit from swh.deposit.utils import NAMESPACES, compute_metadata_context from swh.model import hashutil from swh.model.model import ( @@ -885,6 +885,7 @@ if isinstance(swhid_ref, QualifiedSWHID): deposit.swhid = str(extended_swhid_from_qualified(swhid_ref)) deposit.swhid_context = str(swhid_ref) + deposit.type = DEPOSIT_METADATA_ONLY deposit.complete_date = depo_request.date deposit.reception_date = depo_request.date deposit.save() diff --git a/swh/deposit/migrations/0021_deposit_origin_url_20201124_1438.py b/swh/deposit/migrations/0021_deposit_origin_url_20201124_1438.py --- a/swh/deposit/migrations/0021_deposit_origin_url_20201124_1438.py +++ b/swh/deposit/migrations/0021_deposit_origin_url_20201124_1438.py @@ -21,7 +21,7 @@ migrations.AddField( model_name="deposit", name="origin_url", field=models.TextField(null=True), ), - migrations.RunPython(fill_origin_url), + # migrations.RunPython(fill_origin_url), migrations.AlterField( model_name="deposit", name="external_id", field=models.TextField(null=True), ), diff --git a/swh/deposit/migrations/0022_auto_20220223_1542.py b/swh/deposit/migrations/0022_auto_20220223_1542.py new file mode 100644 --- /dev/null +++ b/swh/deposit/migrations/0022_auto_20220223_1542.py @@ -0,0 +1,59 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.db import migrations, models + +from swh.deposit.config import DEPOSIT_STATUS_LOAD_SUCCESS +from swh.deposit.models import ( + DEPOSIT_CODE, + DEPOSIT_METADATA_ONLY, + DEPOSIT_TYPES, + Deposit, +) + + +def fill_deposit_type(apps, schema_editor): + """Fill the new field metadata_only on existing data. This will mark metadata only + deposits all deposits whose status is done, their complete date is exactly the + reception date, and they have their swhid filled in. + + """ + + for deposit in Deposit.objects.all(): + deposit.type = ( + DEPOSIT_METADATA_ONLY + if ( + deposit.status == DEPOSIT_STATUS_LOAD_SUCCESS + and deposit.type is None + and deposit.complete_date == deposit.reception_date + and deposit.complete_date is not None + and deposit.swhid is not None + and deposit.swhid_context is not None + ) + else DEPOSIT_CODE + ) + deposit.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("deposit", "0021_deposit_origin_url_20201124_1438"), + ] + + operations = [ + migrations.AddField( + model_name="deposit", + name="type", + field=models.CharField( + choices=DEPOSIT_TYPES, default=DEPOSIT_CODE, max_length=4, + ), + preserve_default=False, + ), + # Migrate and make the operations possibly reversible + migrations.RunPython( + fill_deposit_type, reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/swh/deposit/models.py b/swh/deposit/models.py --- a/swh/deposit/models.py +++ b/swh/deposit/models.py @@ -114,6 +114,15 @@ ) +DEPOSIT_METADATA_ONLY = "meta" +DEPOSIT_CODE = "code" + +DEPOSIT_TYPES = [ + (DEPOSIT_METADATA_ONLY, DEPOSIT_METADATA_ONLY), + (DEPOSIT_CODE, DEPOSIT_CODE), +] + + class Deposit(models.Model): """Deposit reception table @@ -147,6 +156,8 @@ load_task_id = models.TextField( blank=True, null=True, verbose_name="Scheduler's associated loading task id" ) + type = models.CharField(max_length=4, choices=DEPOSIT_TYPES, default=DEPOSIT_CODE) + raw_metadata: Optional[str] = None class Meta: diff --git a/swh/deposit/tests/conftest.py b/swh/deposit/tests/conftest.py --- a/swh/deposit/tests/conftest.py +++ b/swh/deposit/tests/conftest.py @@ -200,6 +200,7 @@ ("PORT", postgresql_proc.port), # noqa } ) + with django_db_blocker.unblock(): setup_databases( verbosity=request.config.option.verbose, interactive=False, keepdb=False diff --git a/swh/deposit/tests_migration/test_migrations.py b/swh/deposit/tests_migration/test_migrations.py --- a/swh/deposit/tests_migration/test_migrations.py +++ b/swh/deposit/tests_migration/test_migrations.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 The Software Heritage developers +# Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -45,3 +45,23 @@ new_deposit = new_state.apps.get_model("deposit", "Deposit") assert hasattr(new_deposit, "origin_url") is True + + +def test_migrations_22_add_deposit_type_column(migrator): + """22 migration should add the new type column""" + from swh.deposit.models import DEPOSIT_CODE, Deposit + + old_state = migrator.apply_initial_migration( + ("deposit", "0021_deposit_origin_url_20201124_1438") + ) + old_deposit = old_state.apps.get_model("deposit", "Deposit") + + assert hasattr(old_deposit, "type") is False + + # Migrate to the latest schema + new_state = migrator.apply_tested_migration(("deposit", "0022_auto_20220223_1542")) + new_deposit = new_state.apps.get_model("deposit", "Deposit") + + assert hasattr(new_deposit, "type") is True + + assert Deposit().type == DEPOSIT_CODE