Differential D7868 Diff 28448 swh/loader/git/loader.py

Changeset View

Standalone View

swh/loader/git/loader.py

	Show All 13 Lines

	import dulwich.client			import dulwich.client
	from dulwich.errors import GitProtocolError, NotGitRepository			from dulwich.errors import GitProtocolError, NotGitRepository
	from dulwich.object_store import ObjectStoreGraphWalker			from dulwich.object_store import ObjectStoreGraphWalker
	from dulwich.objects import ShaFile			from dulwich.objects import ShaFile
	from dulwich.pack import PackData, PackInflater			from dulwich.pack import PackData, PackInflater

	from swh.core.statsd import Statsd			from swh.core.statsd import Statsd
	from swh.loader.core.loader import DVCSLoader
	from swh.loader.exception import NotFound			from swh.loader.exception import NotFound
	from swh.model import hashutil			from swh.model import hashutil
	from swh.model.model import (			from swh.model.model import (
	BaseContent,			BaseContent,
	Directory,			Directory,
	Release,			Release,
	Revision,			Revision,
	Snapshot,			Snapshot,
	SnapshotBranch,			SnapshotBranch,
	TargetType,			TargetType,
	)			)
	from swh.storage.algos.snapshot import snapshot_get_latest			from swh.storage.algos.snapshot import snapshot_get_latest
	from swh.storage.interface import StorageInterface			from swh.storage.interface import StorageInterface

	from . import converters, dumb, utils			from . import converters, dumb, utils
				from .base import BaseGitLoader
	from .utils import HexBytes			from .utils import HexBytes

	logger = logging.getLogger(__name__)			logger = logging.getLogger(__name__)


	class RepoRepresentation:			class RepoRepresentation:
	"""Repository representation for a Software Heritage origin."""			"""Repository representation for a Software Heritage origin."""

	▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
	@dataclass			@dataclass
	class FetchPackReturn:			class FetchPackReturn:
	remote_refs: Dict[bytes, HexBytes]			remote_refs: Dict[bytes, HexBytes]
	symbolic_refs: Dict[bytes, HexBytes]			symbolic_refs: Dict[bytes, HexBytes]
	pack_buffer: SpooledTemporaryFile			pack_buffer: SpooledTemporaryFile
	pack_size: int			pack_size: int


	class GitLoader(DVCSLoader):			class GitLoader(BaseGitLoader):
	"""A bulk loader for a git repository			"""A bulk loader for a git repository

	Emits the following statsd stats:			Emits the following statsd stats:

	* increments ``swh_loader_git``			* increments ``swh_loader_git``
	* histogram ``swh_loader_git_ignored_refs_percent`` is the ratio of refs ignored			* histogram ``swh_loader_git_ignored_refs_percent`` is the ratio of refs ignored
	over all refs of the remote repository			over all refs of the remote repository
	* histogram ``swh_loader_git_known_refs_percent`` is the ratio of (non-ignored)			* histogram ``swh_loader_git_known_refs_percent`` is the ratio of (non-ignored)
	▲ Show 20 Lines • Show All 450 Lines • Show Last 20 Lines