diff --git a/swh/storage/algos/diff.py b/swh/storage/algos/diff.py --- a/swh/storage/algos/diff.py +++ b/swh/storage/algos/diff.py @@ -15,14 +15,13 @@ import collections from typing import Any, Dict -from swh.model.hashutil import hash_to_bytes -from swh.model.identifiers import directory_identifier +from swh.model.model import Directory from swh.storage.interface import StorageInterface from .dir_iterators import DirectoryIterator, DoubleDirectoryIterator, Remaining # get the hash identifier for an empty directory -_empty_dir_hash = hash_to_bytes(directory_identifier({"entries": []})) +_empty_dir_hash = Directory(entries=()).id def _get_rev(storage: StorageInterface, rev_id: bytes) -> Dict[str, Any]: diff --git a/swh/storage/algos/dir_iterators.py b/swh/storage/algos/dir_iterators.py --- a/swh/storage/algos/dir_iterators.py +++ b/swh/storage/algos/dir_iterators.py @@ -13,11 +13,10 @@ from enum import Enum -from swh.model.hashutil import hash_to_bytes -from swh.model.identifiers import directory_identifier +from swh.model.model import Directory # get the hash identifier for an empty directory -_empty_dir_hash = hash_to_bytes(directory_identifier({"entries": []})) +_empty_dir_hash = Directory(entries=()).id def _get_dir(storage, dir_id): diff --git a/swh/storage/algos/revisions_walker.py b/swh/storage/algos/revisions_walker.py --- a/swh/storage/algos/revisions_walker.py +++ b/swh/storage/algos/revisions_walker.py @@ -382,7 +382,7 @@ """ Return the path checksum identifier in the source tree of the provided revision. If the path corresponds to a directory, the - value computed by :func:`swh.model.identifiers.directory_identifier` + value computed by :meth:`swh.model.Directory.compute_hash` will be returned. If the path corresponds to a file, its sha1 checksum will be returned. diff --git a/swh/storage/api/serializers.py b/swh/storage/api/serializers.py --- a/swh/storage/api/serializers.py +++ b/swh/storage/api/serializers.py @@ -7,9 +7,7 @@ from typing import Callable, Dict, List, Tuple -import swh.model.identifiers as identifiers -from swh.model.identifiers import CoreSWHID, ExtendedSWHID, ObjectType, QualifiedSWHID -import swh.model.model as model +from swh.model import model, swhids from swh.storage import interface @@ -30,8 +28,8 @@ return getattr(model, d.pop("__type__"))(d["value"]) -def _decode_identifiers_enum(d): - return getattr(identifiers, d.pop("__type__"))(d["value"]) +def _decode_swhids_enum(d): + return getattr(swhids, d.pop("__type__"))(d["value"]) def _decode_storage_enum(d): @@ -40,21 +38,21 @@ ENCODERS: List[Tuple[type, str, Callable]] = [ (model.BaseModel, "model", _encode_model_object), - (CoreSWHID, "core_swhid", str), - (ExtendedSWHID, "extended_swhid", str), - (QualifiedSWHID, "qualified_swhid", str), - (ObjectType, "identifiers_enum", _encode_enum), + (swhids.CoreSWHID, "core_swhid", str), + (swhids.ExtendedSWHID, "extended_swhid", str), + (swhids.QualifiedSWHID, "qualified_swhid", str), + (swhids.ObjectType, "identifiers_enum", _encode_enum), (model.MetadataAuthorityType, "model_enum", _encode_enum), (interface.ListOrder, "storage_enum", _encode_enum), ] DECODERS: Dict[str, Callable] = { - "core_swhid": CoreSWHID.from_string, - "extended_swhid": ExtendedSWHID.from_string, - "qualified_swhid": QualifiedSWHID.from_string, + "core_swhid": swhids.CoreSWHID.from_string, + "extended_swhid": swhids.ExtendedSWHID.from_string, + "qualified_swhid": swhids.QualifiedSWHID.from_string, "model": lambda d: getattr(model, d.pop("__type__")).from_dict(d), - "identifiers_enum": _decode_identifiers_enum, + "identifiers_enum": _decode_swhids_enum, "model_enum": _decode_model_enum, "storage_enum": _decode_storage_enum, } diff --git a/swh/storage/backfill.py b/swh/storage/backfill.py --- a/swh/storage/backfill.py +++ b/swh/storage/backfill.py @@ -19,7 +19,6 @@ from typing import Any, Callable, Dict, Optional from swh.core.db import BaseDb -from swh.model.identifiers import ExtendedObjectType from swh.model.model import ( BaseModel, Directory, @@ -32,6 +31,7 @@ SnapshotBranch, TargetType, ) +from swh.model.swhids import ExtendedObjectType from swh.storage.postgresql.converters import ( db_to_extid, db_to_raw_extrinsic_metadata, diff --git a/swh/storage/cassandra/cql.py b/swh/storage/cassandra/cql.py --- a/swh/storage/cassandra/cql.py +++ b/swh/storage/cassandra/cql.py @@ -39,7 +39,6 @@ ) from swh.core.utils import grouper -from swh.model.identifiers import CoreSWHID from swh.model.model import ( Content, Person, @@ -48,6 +47,7 @@ Timestamp, TimestampWithTimezone, ) +from swh.model.swhids import CoreSWHID from swh.storage.interface import ListOrder from ..utils import remove_keys diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -28,8 +28,6 @@ from swh.core.api.classes import stream_results from swh.core.api.serializers import msgpack_dumps, msgpack_loads from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_hex -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID -from swh.model.identifiers import ObjectType as SwhidObjectType from swh.model.model import ( Content, Directory, @@ -50,6 +48,8 @@ SnapshotBranch, TargetType, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID +from swh.model.swhids import ObjectType as SwhidObjectType from swh.storage.interface import ( VISIT_STATUSES, ListOrder, diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -22,8 +22,8 @@ Union, ) -from swh.model.identifiers import ExtendedSWHID from swh.model.model import Content, Sha1Git, SkippedContent +from swh.model.swhids import ExtendedSWHID from swh.storage.cassandra import CassandraStorage from swh.storage.cassandra.model import ( BaseRow, diff --git a/swh/storage/interface.py b/swh/storage/interface.py --- a/swh/storage/interface.py +++ b/swh/storage/interface.py @@ -11,7 +11,6 @@ from swh.core.api import remote_api_endpoint from swh.core.api.classes import PagedResult as CorePagedResult -from swh.model.identifiers import ExtendedSWHID, ObjectType from swh.model.model import ( Content, Directory, @@ -32,6 +31,7 @@ Snapshot, SnapshotBranch, ) +from swh.model.swhids import ExtendedSWHID, ObjectType class ListOrder(Enum): diff --git a/swh/storage/migrate_extrinsic_metadata.py b/swh/storage/migrate_extrinsic_metadata.py --- a/swh/storage/migrate_extrinsic_metadata.py +++ b/swh/storage/migrate_extrinsic_metadata.py @@ -39,13 +39,6 @@ from swh.core.db import BaseDb from swh.model.hashutil import hash_to_hex -from swh.model.identifiers import ( - CoreSWHID, - ExtendedObjectType, - ExtendedSWHID, - ObjectType, - QualifiedSWHID, -) from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -53,6 +46,13 @@ RawExtrinsicMetadata, Sha1Git, ) +from swh.model.swhids import ( + CoreSWHID, + ExtendedObjectType, + ExtendedSWHID, + ObjectType, + QualifiedSWHID, +) from swh.storage import get_storage from swh.storage.algos.origin import iter_origin_visit_statuses, iter_origin_visits from swh.storage.algos.snapshot import snapshot_get_all_branches diff --git a/swh/storage/postgresql/converters.py b/swh/storage/postgresql/converters.py --- a/swh/storage/postgresql/converters.py +++ b/swh/storage/postgresql/converters.py @@ -8,15 +8,13 @@ import warnings from swh.core.utils import encode_with_unescape -from swh.model.identifiers import CoreSWHID, ExtendedSWHID -from swh.model.identifiers import ObjectType as SwhidObjectType -from swh.model.identifiers import origin_identifier from swh.model.model import ( ExtID, MetadataAuthority, MetadataAuthorityType, MetadataFetcher, ObjectType, + Origin, Person, RawExtrinsicMetadata, Release, @@ -25,6 +23,8 @@ Timestamp, TimestampWithTimezone, ) +from swh.model.swhids import CoreSWHID, ExtendedSWHID +from swh.model.swhids import ObjectType as SwhidObjectType from ..utils import map_optional @@ -303,7 +303,7 @@ warnings.warn( "Fetching raw_extrinsic_metadata row with URL target", DeprecationWarning ) - target = "swh:1:ori:" + origin_identifier({"url": target}) + target = Origin(url=target).swhid() return RawExtrinsicMetadata( target=ExtendedSWHID.from_string(target), diff --git a/swh/storage/postgresql/db.py b/swh/storage/postgresql/db.py --- a/swh/storage/postgresql/db.py +++ b/swh/storage/postgresql/db.py @@ -14,8 +14,8 @@ from swh.core.db.db_utils import jsonize as _jsonize from swh.core.db.db_utils import stored_procedure from swh.model.hashutil import DEFAULT_ALGORITHMS -from swh.model.identifiers import ObjectType from swh.model.model import SHA1_SIZE, OriginVisit, OriginVisitStatus, Sha1Git +from swh.model.swhids import ObjectType from swh.storage.interface import ListOrder logger = logging.getLogger(__name__) diff --git a/swh/storage/postgresql/storage.py b/swh/storage/postgresql/storage.py --- a/swh/storage/postgresql/storage.py +++ b/swh/storage/postgresql/storage.py @@ -20,7 +20,6 @@ from swh.core.api.serializers import msgpack_dumps, msgpack_loads from swh.core.db.common import db_transaction, db_transaction_generator from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex -from swh.model.identifiers import ExtendedObjectType, ExtendedSWHID, ObjectType from swh.model.model import ( SHA1_SIZE, Content, @@ -43,6 +42,7 @@ SnapshotBranch, TargetType, ) +from swh.model.swhids import ExtendedObjectType, ExtendedSWHID, ObjectType from swh.storage.exc import HashCollision, StorageArgumentException, StorageDBError from swh.storage.interface import ( VISIT_STATUSES, diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_cran.py b/swh/storage/tests/migrate_extrinsic_metadata/test_cran.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_cran.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_cran.py @@ -11,7 +11,6 @@ import json from unittest.mock import Mock, call -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -19,6 +18,7 @@ Origin, RawExtrinsicMetadata, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage.migrate_extrinsic_metadata import cran_package_from_url, handle_row FETCHER = MetadataFetcher( diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py b/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_debian.py @@ -15,7 +15,6 @@ import attr import pytest -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -33,6 +32,7 @@ Timestamp, TimestampWithTimezone, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage import get_storage from swh.storage.interface import ListOrder, PagedResult from swh.storage.migrate_extrinsic_metadata import debian_origins_from_row, handle_row diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_deposit.py b/swh/storage/tests/migrate_extrinsic_metadata/test_deposit.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_deposit.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_deposit.py @@ -11,7 +11,6 @@ import json from unittest.mock import MagicMock, Mock, call -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -19,6 +18,7 @@ Origin, RawExtrinsicMetadata, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage.migrate_extrinsic_metadata import ( DEPOSIT_COLS, cran_package_from_url, diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_gnu.py b/swh/storage/tests/migrate_extrinsic_metadata/test_gnu.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_gnu.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_gnu.py @@ -11,7 +11,6 @@ import json from unittest.mock import Mock, call -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -19,6 +18,7 @@ Origin, RawExtrinsicMetadata, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage.migrate_extrinsic_metadata import cran_package_from_url, handle_row FETCHER = MetadataFetcher( diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_nixguix.py b/swh/storage/tests/migrate_extrinsic_metadata/test_nixguix.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_nixguix.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_nixguix.py @@ -11,7 +11,6 @@ import json from unittest.mock import Mock, call -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -19,6 +18,7 @@ Origin, RawExtrinsicMetadata, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage.migrate_extrinsic_metadata import cran_package_from_url, handle_row FETCHER = MetadataFetcher( diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_npm.py b/swh/storage/tests/migrate_extrinsic_metadata/test_npm.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_npm.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_npm.py @@ -11,7 +11,6 @@ import json from unittest.mock import Mock, call -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -19,6 +18,7 @@ Origin, RawExtrinsicMetadata, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage.migrate_extrinsic_metadata import ( handle_row, npm_package_from_source_url, diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py b/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py @@ -13,7 +13,6 @@ import attr -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.model.model import ( MetadataAuthority, MetadataAuthorityType, @@ -26,6 +25,7 @@ SnapshotBranch, TargetType, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID from swh.storage import get_storage from swh.storage.interface import PagedResult from swh.storage.migrate_extrinsic_metadata import ( diff --git a/swh/storage/tests/storage_data.py b/swh/storage/tests/storage_data.py --- a/swh/storage/tests/storage_data.py +++ b/swh/storage/tests/storage_data.py @@ -10,8 +10,6 @@ from swh.model import from_disk from swh.model.hashutil import hash_to_bytes -from swh.model.identifiers import CoreSWHID, ExtendedObjectType, ExtendedSWHID -from swh.model.identifiers import ObjectType as SwhidObjectType from swh.model.model import ( Content, Directory, @@ -35,6 +33,8 @@ Timestamp, TimestampWithTimezone, ) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID +from swh.model.swhids import ObjectType as SwhidObjectType class StorageData: diff --git a/swh/storage/tests/storage_tests.py b/swh/storage/tests/storage_tests.py --- a/swh/storage/tests/storage_tests.py +++ b/swh/storage/tests/storage_tests.py @@ -23,7 +23,6 @@ from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes from swh.model.hypothesis_strategies import objects from swh.model.hypothesis_strategies import snapshots as unknown_snapshot -from swh.model.identifiers import CoreSWHID, ObjectType from swh.model.model import ( Content, Directory, @@ -39,6 +38,7 @@ SnapshotBranch, TargetType, ) +from swh.model.swhids import CoreSWHID, ObjectType from swh.storage import get_storage from swh.storage.cassandra.storage import CassandraStorage from swh.storage.common import origin_url_to_sha1 as sha1 diff --git a/swh/storage/tests/test_postgresql_converters.py b/swh/storage/tests/test_postgresql_converters.py --- a/swh/storage/tests/test_postgresql_converters.py +++ b/swh/storage/tests/test_postgresql_converters.py @@ -7,7 +7,6 @@ import pytest -from swh.model.identifiers import ExtendedSWHID from swh.model.model import ( ObjectType, Person, @@ -17,6 +16,7 @@ Timestamp, TimestampWithTimezone, ) +from swh.model.swhids import ExtendedSWHID from swh.storage.postgresql import converters