Differential D6622 Diff 24063 swh/loader/svn/utils.py

Changeset View

Standalone View

swh/loader/svn/utils.py

# Copyright (C) 2016-2020 The Software Heritage developers		# Copyright (C) 2016-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import errno		import errno
		import logging
import os		import os
import shutil		import shutil
from subprocess import PIPE, Popen, call		from subprocess import PIPE, Popen, call
import tempfile		import tempfile
		from typing import Tuple

from dateutil import parser		from dateutil import parser

from swh.model.model import Optional, Timestamp		from swh.model.model import Optional, Timestamp

		logger = logging.getLogger(__name__)


def strdate_to_timestamp(strdate: Optional[str]) -> Timestamp:		def strdate_to_timestamp(strdate: Optional[str]) -> Timestamp:
"""Convert a string date to an int timestamp.		"""Convert a string date to an int timestamp.

Args:		Args:
strdate: A string representing a date with format like		strdate: A string representing a date with format like
'YYYY-mm-DDTHH:MM:SS.800722Z'		'YYYY-mm-DDTHH:MM:SS.800722Z'

▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines	def read_lines(self):
else:		else:
self._buffer = ""		self._buffer = ""
if len(lines) == 1 and not lines[0]:		if len(lines) == 1 and not lines[0]:
lines = []		lines = []
return (lines, False)		return (lines, False)


def init_svn_repo_from_dump(		def init_svn_repo_from_dump(
dump_path, prefix=None, suffix=None, root_dir="/tmp", gzip=False		dump_path: str,
):		prefix: Optional[str] = None,
"""Given a path to a svn dump.		suffix: Optional[str] = None,
Initialize an svn repository with the content of said dump.		root_dir: str = "/tmp",
		gzip: bool = False,
		cleanup_dump: bool = True,
		) -> Tuple[str, str]:
		"""Given a path to a svn dump, initialize an svn repository with the content of said
		dump.

Returns:		Args:
A tuple:		dump_path: The dump to the path
- temporary folder (str): containing the mounted repository		prefix: optional prefix file name for the working directory
- repo_path (str): path to the mounted repository inside the		suffix: optional suffix file name for the working directory
temporary folder		root_dir: the root directory where the working directory is created
		gzip: Boolean to determine whether we treat the dump as compressed or not.
		cleanup_dump: Whether we want this function call to clean up the dump at the end
		of the repository initialization.

Raises:		Raises:
ValueError in case of failure to run the command to uncompress		ValueError in case of failure to run the command to uncompress and load the
and load the dump.		dump.

		Returns:
		A tuple:
		- temporary folder: containing the mounted repository
		- repo_path: path to the mounted repository inside the temporary folder

"""		"""
project_name = os.path.basename(os.path.dirname(dump_path))		project_name = os.path.basename(os.path.dirname(dump_path))
temp_dir = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_dir)		temp_dir = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_dir)

try:		try:
repo_path = os.path.join(temp_dir, project_name)		repo_path = os.path.join(temp_dir, project_name)

Show All 18 Lines	try:
if r != 0:		if r != 0:
raise ValueError(		raise ValueError(
"Failed to mount the svn dump for project %s" % project_name		"Failed to mount the svn dump for project %s" % project_name
)		)
return temp_dir, repo_path		return temp_dir, repo_path
except Exception as e:		except Exception as e:
shutil.rmtree(temp_dir)		shutil.rmtree(temp_dir)
raise e		raise e
		finally:
		if cleanup_dump:
		try:
		# At this time, the temporary svn repository is mounted from the dump or
		# the svn repository failed to mount. Either way, we can drop the dump.
		vlorentzUnsubmitted Done Inline Actions Can `os.remove` fail without an error? vlorentz: Can `os.remove` fail without an error?
		ardumontAuthorUnsubmitted Done Inline Actions i'll check ardumont: i'll check
		ardumontAuthorUnsubmitted Done Inline Actions I guess it could happen say in the scenario where the exception already got raised. I'll wrap this in a try except. Diff on its way. ardumont: I guess it could happen say in the scenario where the exception already got raised. I'll wrap…
		ardumontAuthorUnsubmitted Done Inline Actions diff "update" on its way. ardumont: diff "update" on its way.
		os.remove(dump_path)
		assert not os.path.exists(dump_path)
		except OSError as e:
		logger.warn("Failure to remove the dump %s: %s", dump_path, e)


def init_svn_repo_from_archive_dump(		def init_svn_repo_from_archive_dump(
archive_path, prefix=None, suffix=None, root_dir="/tmp"		archive_path: str,
):		prefix: Optional[str] = None,
"""Given a path to an archive containing an svn dump.		suffix: Optional[str] = None,
Initialize an svn repository with the content of said dump.		root_dir: str = "/tmp",
		cleanup_dump: bool = True,
Returns:		) -> Tuple[str, str]:
A tuple:		"""Given a path to an archive containing an svn dump, initializes an svn repository
- temporary folder (str): containing the mounted repository		with the content of the uncompressed dump.
- repo_path (str): path to the mounted repository inside the
temporary folder

		Args:
		archive_path: The archive svn dump path
		prefix: optional prefix file name for the working directory
		suffix: optional suffix file name for the working directory
		root_dir: the root directory where the working directory is created
		gzip: Boolean to determine whether we treat the dump as compressed or not.
		cleanup_dump: Whether we want this function call to clean up the dump at the end
		of the repository initialization.
Raises:		Raises:
ValueError in case of failure to run the command to uncompress		ValueError in case of failure to run the command to uncompress
and load the dump.		and load the dump.

		Returns:
		A tuple:
		- temporary folder: containing the mounted repository
		- repo_path: path to the mounted repository inside the
		temporary folder

"""		"""
return init_svn_repo_from_dump(		return init_svn_repo_from_dump(
archive_path, prefix=prefix, suffix=suffix, root_dir=root_dir, gzip=True		archive_path,
		prefix=prefix,
		suffix=suffix,
		root_dir=root_dir,
		gzip=True,
		cleanup_dump=cleanup_dump,
)		)