Differential D3435 Diff 16093 swh/loader/mercurial/from_bundle.py

Changeset View

Standalone View

swh/loader/mercurial/from_bundle.py

This file was copied from swh/loader/mercurial/loader.py.

	# Copyright (C) 2017-2020 The Software Heritage developers			# Copyright (C) 2017-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	"""This document contains a SWH loader for ingesting repository data			"""This document contains a SWH loader for ingesting repository data
	from Mercurial version 2 bundle files.			from Mercurial version 2 bundle files.

	"""			"""

	# NOTE: The code here does expensive work twice in places because of the			# NOTE: The code here does expensive work twice in places because of the
	# intermediate need to check for what is missing before sending to the database			# intermediate need to check for what is missing before sending to the database
	# and the desire to not juggle very large amounts of data.			# and the desire to not juggle very large amounts of data.

	# TODO: Decide whether to also serialize to disk and read back more quickly			# TODO: Decide whether to also serialize to disk and read back more quickly
	# from there. Maybe only for very large repos and fast drives.			# from there. Maybe only for very large repos and fast drives.
	# - Avi			# - Avi


	import datetime			import datetime
	import os			import os
	from queue import Empty
	import random			import random
	import re			import re
				import time
				from queue import Empty
	from shutil import rmtree			from shutil import rmtree
	from tempfile import mkdtemp			from tempfile import mkdtemp
	import time
	from typing import Any, Dict, Iterable, List, Optional			from typing import Any, Dict, Iterable, List, Optional

	import billiard			import billiard
	from dateutil import parser
	import hglib			import hglib
				from dateutil import parser

	from swh.core.config import merge_configs			from swh.core.config import merge_configs
	from swh.loader.core.loader import DVCSLoader			from swh.loader.core.loader import DVCSLoader
	from swh.loader.core.utils import clean_dangling_folders			from swh.loader.core.utils import clean_dangling_folders
	from swh.model import identifiers			from swh.model import identifiers
	from swh.model.hashutil import (			from swh.model.hashutil import (
	DEFAULT_ALGORITHMS,			DEFAULT_ALGORITHMS,
	MultiHash,			MultiHash,
				marmouteUnsubmitted Done Inline Actions This seems like unrelated change. marmoute: This seems like unrelated change.
				acezarAuthorUnsubmitted Done Inline Actions Same answer: isort in precommit acezar: Same answer: isort in precommit
	hash_to_bytehex,			hash_to_bytehex,
	hash_to_bytes,			hash_to_bytes,
	hash_to_hex,			hash_to_hex,
	)			)
	from swh.model.model import (			from swh.model.model import (
	BaseContent,			BaseContent,
	Content,			Content,
	Directory,			Directory,
	▲ Show 20 Lines • Show All 593 Lines • Show Last 20 Lines