diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core[db,http] >= 0.14.4 -swh.model >= 0.3 +swh.model >= 3.0.0 swh.objstorage >= 0.0.17 swh.scheduler >= 0.7.0 swh.storage >= 0.29.0 diff --git a/swh/vault/api/serializers.py b/swh/vault/api/serializers.py --- a/swh/vault/api/serializers.py +++ b/swh/vault/api/serializers.py @@ -5,7 +5,7 @@ from typing import Callable, Dict, List, Tuple -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID ENCODERS: List[Tuple[type, str, Callable]] = [ (CoreSWHID, "core_swhid", str), diff --git a/swh/vault/backend.py b/swh/vault/backend.py --- a/swh/vault/backend.py +++ b/swh/vault/backend.py @@ -13,7 +13,7 @@ from swh.core.db import BaseDb from swh.core.db.common import db_transaction -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID from swh.scheduler import get_scheduler from swh.scheduler.utils import create_oneshot_task_dict from swh.storage import get_storage diff --git a/swh/vault/cache.py b/swh/vault/cache.py --- a/swh/vault/cache.py +++ b/swh/vault/cache.py @@ -4,7 +4,7 @@ # See top-level LICENSE file for more information from swh.model import hashutil -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID from swh.objstorage.factory import get_objstorage from swh.objstorage.objstorage import compute_hash diff --git a/swh/vault/cli.py b/swh/vault/cli.py --- a/swh/vault/cli.py +++ b/swh/vault/cli.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: import io - from swh.model.identifiers import CoreSWHID + from swh.model.swhids import CoreSWHID class SwhidParamType(click.ParamType): @@ -26,7 +26,7 @@ def convert(self, value, param, ctx): from swh.model.exceptions import ValidationError - from swh.model.identifiers import CoreSWHID + from swh.model.swhids import CoreSWHID try: return CoreSWHID.from_string(value) @@ -70,7 +70,7 @@ and outputs it to the given file. """ from swh.core import config - from swh.model.identifiers import ObjectType + from swh.model.swhids import ObjectType from swh.objstorage.exc import ObjNotFoundError from swh.objstorage.factory import get_objstorage from swh.storage import get_storage diff --git a/swh/vault/cookers/__init__.py b/swh/vault/cookers/__init__.py --- a/swh/vault/cookers/__init__.py +++ b/swh/vault/cookers/__init__.py @@ -10,7 +10,7 @@ from swh.core.config import load_named_config from swh.core.config import read as read_config -from swh.model.identifiers import CoreSWHID, ObjectType +from swh.model.swhids import CoreSWHID, ObjectType from swh.storage import get_storage from swh.vault import get_vault from swh.vault.cookers.base import DEFAULT_CONFIG, DEFAULT_CONFIG_PATH, BaseVaultCooker diff --git a/swh/vault/cookers/base.py b/swh/vault/cookers/base.py --- a/swh/vault/cookers/base.py +++ b/swh/vault/cookers/base.py @@ -11,7 +11,7 @@ from psycopg2.extensions import QueryCanceledError -from swh.model.identifiers import CoreSWHID, ObjectType +from swh.model.swhids import CoreSWHID, ObjectType from swh.storage.interface import StorageInterface MAX_BUNDLE_SIZE = 2 ** 29 # 512 MiB diff --git a/swh/vault/cookers/directory.py b/swh/vault/cookers/directory.py --- a/swh/vault/cookers/directory.py +++ b/swh/vault/cookers/directory.py @@ -6,7 +6,7 @@ import tarfile import tempfile -from swh.model.identifiers import ObjectType +from swh.model.swhids import ObjectType from swh.vault.cookers.base import BaseVaultCooker from swh.vault.to_disk import DirectoryBuilder diff --git a/swh/vault/cookers/git_bare.py b/swh/vault/cookers/git_bare.py --- a/swh/vault/cookers/git_bare.py +++ b/swh/vault/cookers/git_bare.py @@ -30,12 +30,9 @@ import zlib from swh.core.api.classes import stream_results_optional -from swh.model import identifiers +from swh.model import git_objects from swh.model.hashutil import hash_to_bytehex, hash_to_hex from swh.model.model import ( - Content, - DirectoryEntry, - ObjectType, Person, Release, Revision, @@ -46,6 +43,9 @@ TargetType, TimestampWithTimezone, ) +from swh.model.model import Content, DirectoryEntry +from swh.model.model import ObjectType as ModelObjectType +from swh.model.swhids import CoreSWHID, ObjectType from swh.storage.algos.revisions_walker import DFSRevisionsWalker from swh.storage.algos.snapshot import snapshot_get_all_branches from swh.vault.cookers.base import BaseVaultCooker @@ -78,9 +78,7 @@ class GitBareCooker(BaseVaultCooker): BUNDLE_TYPE = "git_bare" - SUPPORTED_OBJECT_TYPES = { - identifiers.ObjectType[obj_type.name] for obj_type in RootObjectType - } + SUPPORTED_OBJECT_TYPES = {ObjectType[obj_type.name] for obj_type in RootObjectType} use_fsck = True @@ -317,14 +315,12 @@ # First, try to cook using swh-graph, as it is more efficient than # swh-storage for querying the history - obj_swhid = identifiers.CoreSWHID( - object_type=identifiers.ObjectType.REVISION, object_id=obj_id, - ) + obj_swhid = CoreSWHID(object_type=ObjectType.REVISION, object_id=obj_id,) try: revision_ids = ( swhid.object_id for swhid in map( - identifiers.CoreSWHID.from_string, + CoreSWHID.from_string, self.graph.visit_nodes(str(obj_swhid), edges="rev:rev"), ) ) @@ -366,24 +362,22 @@ # First, try to cook using swh-graph, as it is more efficient than # swh-storage for querying the history - obj_swhid = identifiers.CoreSWHID( - object_type=identifiers.ObjectType.SNAPSHOT, object_id=obj_id, - ) + obj_swhid = CoreSWHID(object_type=ObjectType.SNAPSHOT, object_id=obj_id,) try: - swhids: Iterable[identifiers.CoreSWHID] = map( - identifiers.CoreSWHID.from_string, + swhids: Iterable[CoreSWHID] = map( + CoreSWHID.from_string, self.graph.visit_nodes(str(obj_swhid), edges="snp:*,rel:*,rev:rev"), ) for swhid in swhids: - if swhid.object_type is identifiers.ObjectType.REVISION: + if swhid.object_type is ObjectType.REVISION: revision_ids.append(swhid.object_id) - elif swhid.object_type is identifiers.ObjectType.RELEASE: + elif swhid.object_type is ObjectType.RELEASE: release_ids.append(swhid.object_id) - elif swhid.object_type is identifiers.ObjectType.DIRECTORY: + elif swhid.object_type is ObjectType.DIRECTORY: directory_ids.append(swhid.object_id) - elif swhid.object_type is identifiers.ObjectType.CONTENT: + elif swhid.object_type is ObjectType.CONTENT: content_ids.append(swhid.object_id) - elif swhid.object_type is identifiers.ObjectType.SNAPSHOT: + elif swhid.object_type is ObjectType.SNAPSHOT: assert ( swhid.object_id == obj_id ), f"Snapshot {obj_id.hex()} references a different snapshot" @@ -454,7 +448,7 @@ def write_revision_node(self, revision: Dict[str, Any]) -> bool: """Writes a revision object to disk""" - git_object = identifiers.revision_git_object(revision) + git_object = git_objects.revision_git_object(revision) return self.write_object(revision["id"], git_object) def load_releases(self, obj_ids: List[Sha1Git]) -> List[Release]: @@ -475,15 +469,15 @@ target to the list of objects to visit""" for release in self.load_releases(obj_ids): assert release.target, "{release.swhid(}) has no target" - if release.target_type is ObjectType.REVISION: + if release.target_type is ModelObjectType.REVISION: self.push_revision_subgraph(release.target) - elif release.target_type is ObjectType.DIRECTORY: + elif release.target_type is ModelObjectType.DIRECTORY: self._push(self._dir_stack, [release.target]) - elif release.target_type is ObjectType.CONTENT: + elif release.target_type is ModelObjectType.CONTENT: self._push(self._cnt_stack, [release.target]) - elif release.target_type is ObjectType.RELEASE: + elif release.target_type is ModelObjectType.RELEASE: self.push_releases_subgraphs([release.target]) - elif release.target_type is ObjectType.SNAPSHOT: + elif release.target_type is ModelObjectType.SNAPSHOT: raise NotImplementedError( f"{release.swhid()} targets a snapshot: {release.target!r}" ) @@ -495,7 +489,7 @@ def write_release_node(self, release: Dict[str, Any]) -> bool: """Writes a release object to disk""" - git_object = identifiers.release_git_object(release) + git_object = git_objects.release_git_object(release) return self.write_object(release["id"], git_object) def load_directories(self, obj_ids: List[Sha1Git]) -> None: @@ -514,7 +508,7 @@ entries = [entry.to_dict() for entry in entries_it] directory = {"id": obj_id, "entries": entries} - git_object = identifiers.directory_git_object(directory) + git_object = git_objects.directory_git_object(directory) self.write_object(obj_id, git_object) # Add children to the stack @@ -573,7 +567,7 @@ self.write_content(content.sha1_git, datum) def write_content(self, obj_id: Sha1Git, content: bytes) -> None: - header = identifiers.git_object_header("blob", len(content)) + header = git_objects.git_object_header("blob", len(content)) self.write_object(obj_id, header + content) def _expect_mismatched_object_error(self, obj_id): diff --git a/swh/vault/cookers/revision_flat.py b/swh/vault/cookers/revision_flat.py --- a/swh/vault/cookers/revision_flat.py +++ b/swh/vault/cookers/revision_flat.py @@ -8,7 +8,7 @@ import tempfile from swh.model import hashutil -from swh.model.identifiers import ObjectType +from swh.model.swhids import ObjectType from swh.vault.cookers.base import BaseVaultCooker from swh.vault.cookers.utils import revision_log from swh.vault.to_disk import DirectoryBuilder diff --git a/swh/vault/cookers/revision_gitfast.py b/swh/vault/cookers/revision_gitfast.py --- a/swh/vault/cookers/revision_gitfast.py +++ b/swh/vault/cookers/revision_gitfast.py @@ -18,7 +18,7 @@ from swh.model import hashutil from swh.model.from_disk import DentryPerms, mode_to_perms -from swh.model.identifiers import ObjectType +from swh.model.swhids import ObjectType from swh.model.toposort import toposort from swh.vault.cookers.base import BaseVaultCooker from swh.vault.cookers.utils import revision_log diff --git a/swh/vault/cooking_tasks.py b/swh/vault/cooking_tasks.py --- a/swh/vault/cooking_tasks.py +++ b/swh/vault/cooking_tasks.py @@ -5,7 +5,7 @@ from celery import current_app as app -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID from swh.vault.cookers import get_cooker diff --git a/swh/vault/in_memory_backend.py b/swh/vault/in_memory_backend.py --- a/swh/vault/in_memory_backend.py +++ b/swh/vault/in_memory_backend.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID from .cache import VaultCache diff --git a/swh/vault/interface.py b/swh/vault/interface.py --- a/swh/vault/interface.py +++ b/swh/vault/interface.py @@ -8,7 +8,7 @@ from typing_extensions import Protocol, runtime_checkable from swh.core.api import remote_api_endpoint -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID @runtime_checkable diff --git a/swh/vault/tests/test_backend.py b/swh/vault/tests/test_backend.py --- a/swh/vault/tests/test_backend.py +++ b/swh/vault/tests/test_backend.py @@ -11,8 +11,8 @@ import psycopg2 import pytest -from swh.model.identifiers import CoreSWHID from swh.model.model import Content +from swh.model.swhids import CoreSWHID from swh.vault.exc import NotFoundExc from swh.vault.tests.vault_testing import hash_content diff --git a/swh/vault/tests/test_cache.py b/swh/vault/tests/test_cache.py --- a/swh/vault/tests/test_cache.py +++ b/swh/vault/tests/test_cache.py @@ -5,7 +5,7 @@ from swh.model import hashutil -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID TEST_TYPE_1 = "revision_gitfast" TEST_TYPE_2 = "directory" diff --git a/swh/vault/tests/test_cli.py b/swh/vault/tests/test_cli.py --- a/swh/vault/tests/test_cli.py +++ b/swh/vault/tests/test_cli.py @@ -10,7 +10,7 @@ import click.testing import pytest -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID from swh.vault.cli import vault as vault_cli_group from swh.vault.cookers.base import BaseVaultCooker from swh.vault.in_memory_backend import InMemoryVaultBackend diff --git a/swh/vault/tests/test_cookers.py b/swh/vault/tests/test_cookers.py --- a/swh/vault/tests/test_cookers.py +++ b/swh/vault/tests/test_cookers.py @@ -26,7 +26,6 @@ from swh.loader.git.from_disk import GitLoaderFromDisk from swh.model import from_disk, hashutil -from swh.model.identifiers import CoreSWHID, ObjectType from swh.model.model import ( Directory, DirectoryEntry, @@ -35,6 +34,7 @@ RevisionType, TimestampWithTimezone, ) +from swh.model.swhids import CoreSWHID, ObjectType from swh.vault.cookers import DirectoryCooker, GitBareCooker, RevisionGitfastCooker from swh.vault.tests.vault_testing import hash_content from swh.vault.to_disk import HIDDEN_MESSAGE, SKIPPED_MESSAGE diff --git a/swh/vault/tests/test_cookers_base.py b/swh/vault/tests/test_cookers_base.py --- a/swh/vault/tests/test_cookers_base.py +++ b/swh/vault/tests/test_cookers_base.py @@ -6,7 +6,7 @@ import textwrap from unittest.mock import MagicMock -from swh.model.identifiers import CoreSWHID +from swh.model.swhids import CoreSWHID from swh.vault.cookers.base import BaseVaultCooker TEST_BUNDLE_CHUNKS = [b"test content 1\n", b"test content 2\n", b"test content 3\n"]