Differential D6112 Diff 22228 swh/vault/cookers/__init__.py

Changeset View

Standalone View

swh/vault/cookers/init.py

# Copyright (C) 2017-2021 The Software Heritage developers		# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from __future__ import annotations		from __future__ import annotations

import os		import os
from typing import Any, Dict		from typing import Any, Dict, List, Type

from swh.core.config import load_named_config		from swh.core.config import load_named_config
from swh.core.config import read as read_config		from swh.core.config import read as read_config
		from swh.model.identifiers import CoreSWHID, ObjectType
from swh.storage import get_storage		from swh.storage import get_storage
from swh.vault import get_vault		from swh.vault import get_vault
from swh.vault.cookers.base import DEFAULT_CONFIG, DEFAULT_CONFIG_PATH		from swh.vault.cookers.base import DEFAULT_CONFIG, DEFAULT_CONFIG_PATH, BaseVaultCooker
from swh.vault.cookers.directory import DirectoryCooker		from swh.vault.cookers.directory import DirectoryCooker
from swh.vault.cookers.git_bare import GitBareCooker		from swh.vault.cookers.git_bare import GitBareCooker
from swh.vault.cookers.revision_flat import RevisionFlatCooker		from swh.vault.cookers.revision_flat import RevisionFlatCooker
from swh.vault.cookers.revision_gitfast import RevisionGitfastCooker		from swh.vault.cookers.revision_gitfast import RevisionGitfastCooker

COOKER_TYPES = {		_COOKER_CLS: List[Type[BaseVaultCooker]] = [
"directory": DirectoryCooker,		DirectoryCooker,
"revision_flat": RevisionFlatCooker,		RevisionFlatCooker,
"revision_gitfast": RevisionGitfastCooker,		RevisionGitfastCooker,
"snapshot_git_bare": GitBareCooker,		GitBareCooker,
"revision_git_bare": GitBareCooker,		]
"directory_git_bare": GitBareCooker,		COOKER_TYPES: Dict[str, List[Type[BaseVaultCooker]]] = {}
}


def get_cooker_cls(bundle_type):		for _cooker_cls in _COOKER_CLS:
return COOKER_TYPES[bundle_type]		COOKER_TYPES.setdefault(_cooker_cls.BUNDLE_TYPE, []).append(_cooker_cls)


		def get_cooker_cls(bundle_type: str, object_type: ObjectType):
		cookers = COOKER_TYPES.get(bundle_type)

		if not cookers:
		raise ValueError(f"{bundle_type} is not a valid bundle type.")

		for cooker in cookers:
		try:
		cooker.check_object_type(object_type)
		except ValueError:
		pass
		else:
		return cooker

		raise ValueError(
		f"{object_type.name.lower()} objects do not have a {bundle_type} cooker"
		)


def check_config(cfg: Dict[str, Any]) -> Dict[str, Any]:		def check_config(cfg: Dict[str, Any]) -> Dict[str, Any]:
"""Ensure the configuration is ok to run a vault worker, and propagate defaults		"""Ensure the configuration is ok to run a vault worker, and propagate defaults

Raises:		Raises:
EnvironmentError if the configuration is not for remote instance		EnvironmentError if the configuration is not for remote instance
ValueError if one of the following keys is missing: vault, storage		ValueError if one of the following keys is missing: vault, storage
Show All 21 Lines	if "storage" not in vcfg:
vcfg["storage"] = cfg.get("storage")		vcfg["storage"] = cfg.get("storage")

if not vcfg.get("storage"):		if not vcfg.get("storage"):
raise ValueError("invalid configuration: missing 'storage' config entry.")		raise ValueError("invalid configuration: missing 'storage' config entry.")

return cfg		return cfg


def get_cooker(bundle_type: str, obj_id: str):		def get_cooker(bundle_type: str, swhid: CoreSWHID):
"""Instantiate a cooker class of type bundle_type.		"""Instantiate a cooker class of type bundle_type.

Returns:		Returns:
Cooker class in charge of cooking the bundle_type with id obj_id.		Cooker class in charge of cooking the bundle_type with id swhid.

Raises:		Raises:
ValueError in case of a missing top-level vault key configuration or a storage		ValueError in case of a missing top-level vault key configuration or a storage
key.		key.
EnvironmentError in case the vault configuration reference a non remote class.		EnvironmentError in case the vault configuration reference a non remote class.

"""		"""
if "SWH_CONFIG_FILENAME" in os.environ:		if "SWH_CONFIG_FILENAME" in os.environ:
cfg = read_config(os.environ["SWH_CONFIG_FILENAME"], DEFAULT_CONFIG)		cfg = read_config(os.environ["SWH_CONFIG_FILENAME"], DEFAULT_CONFIG)
else:		else:
cfg = load_named_config(DEFAULT_CONFIG_PATH, DEFAULT_CONFIG)		cfg = load_named_config(DEFAULT_CONFIG_PATH, DEFAULT_CONFIG)
cooker_cls = get_cooker_cls(bundle_type)		cooker_cls = get_cooker_cls(bundle_type, swhid.object_type)

cfg = check_config(cfg)		cfg = check_config(cfg)
vcfg = cfg["vault"]		vcfg = cfg["vault"]

storage = get_storage(**vcfg.pop("storage"))		storage = get_storage(**vcfg.pop("storage"))
backend = get_vault(**vcfg)		backend = get_vault(**vcfg)

try:		try:
from swh.graph.client import RemoteGraphClient # optional dependency		from swh.graph.client import RemoteGraphClient # optional dependency

graph = RemoteGraphClient(**vcfg["graph"]) if vcfg.get("graph") else None		graph = RemoteGraphClient(**vcfg["graph"]) if vcfg.get("graph") else None
except ModuleNotFoundError:		except ModuleNotFoundError:
if vcfg.get("graph"):		if vcfg.get("graph"):
raise EnvironmentError(		raise EnvironmentError(
"Graph configuration required but module is not installed."		"Graph configuration required but module is not installed."
)		)
else:		else:
graph = None		graph = None

return cooker_cls(		return cooker_cls(
bundle_type,		swhid,
obj_id,
backend=backend,		backend=backend,
storage=storage,		storage=storage,
graph=graph,		graph=graph,
max_bundle_size=cfg["max_bundle_size"],		max_bundle_size=cfg["max_bundle_size"],
)		)