diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py
--- a/swh/loader/svn/loader.py
+++ b/swh/loader/svn/loader.py
@@ -234,7 +234,7 @@
hash_data_per_revs = svnrepo.swh_hash_data_at_revision(revision_start)
rev = revision_start
- rev, _, commit, _, root_dir = list(hash_data_per_revs)[0]
+ commit, root_dir = list(hash_data_per_revs)[0]
dir_id = root_dir.hash
swh_revision = self.build_swh_revision(rev, commit, dir_id, parents)
@@ -402,9 +402,9 @@
self.latest_snapshot, self.latest_revision = latest_snapshot_revision
local_dirname = tempfile.mkdtemp(
- suffix="-%s" % os.getpid(),
- prefix=TEMPORARY_DIR_PREFIX_PATTERN,
dir=self.temp_directory,
+ prefix=TEMPORARY_DIR_PREFIX_PATTERN,
+ suffix="-%s" % os.getpid(),
)
try:
diff --git a/swh/loader/svn/svn.py b/swh/loader/svn/svn.py
--- a/swh/loader/svn/svn.py
+++ b/swh/loader/svn/svn.py
@@ -13,11 +13,19 @@
import os
import shutil
import tempfile
+from typing import Dict, Iterator, List, Optional, Tuple, Union
from subvertpy import client, properties
from subvertpy.ra import Auth, RemoteAccess, get_username_provider
-from swh.model.from_disk import Directory
+from swh.model.from_disk import Directory as DirectoryFromDisk
+from swh.model.model import (
+ Content,
+ Directory,
+ Person,
+ SkippedContent,
+ TimestampWithTimezone,
+)
from . import converters, ra
@@ -74,23 +82,23 @@
}
)
- def head_revision(self):
+ def head_revision(self) -> int:
"""Retrieve current head revision.
"""
return self.conn.get_latest_revnum()
- def initial_revision(self):
+ def initial_revision(self) -> int:
"""Retrieve the initial revision from which the remote url appeared.
"""
return 1
- def convert_commit_message(self, msg):
+ def convert_commit_message(self, msg: Union[str, bytes]) -> bytes:
"""Simply encode the commit message.
Args:
- msg (str): the commit message to convert.
+ msg: the commit message to convert.
Returns:
The transformed message as bytes.
@@ -100,12 +108,12 @@
return msg
return msg.encode("utf-8")
- def convert_commit_date(self, date):
+ def convert_commit_date(self, date: str) -> TimestampWithTimezone:
"""Convert the message commit date into a timestamp in swh format.
The precision is kept.
Args:
- date (str): the commit date to convert.
+ date: the commit date to convert.
Returns:
The transformed date.
@@ -113,19 +121,19 @@
"""
return converters.svn_date_to_swh_date(date)
- def convert_commit_author(self, author):
+ def convert_commit_author(self, author: Optional[bytes]) -> Person:
"""Convert the commit author into an swh person.
Args:
- author (str): the commit author to convert.
+ author: the commit author to convert.
Returns:
- Person: a model object
+ Person as model object
"""
return converters.svn_author_to_swh_person(author)
- def __to_entry(self, log_entry):
+ def __to_entry(self, log_entry: Tuple) -> Dict:
changed_paths, rev, revprops, has_children = log_entry
author_date = self.convert_commit_date(
@@ -147,7 +155,7 @@
"message": message,
}
- def logs(self, revision_start, revision_end):
+ def logs(self, revision_start: int, revision_end: int) -> Iterator[Dict]:
"""Stream svn logs between revision_start and revision_end by chunks of
block_size logs.
@@ -178,7 +186,7 @@
):
yield self.__to_entry(log_entry)
- def export(self, revision):
+ def export(self, revision: int) -> None:
"""Export the repository to a given version.
"""
@@ -189,10 +197,10 @@
ignore_keywords=True,
)
- def export_temporary(self, revision):
- """Export the repository to a given revision in a temporary location.
- This is up to the caller of this function to clean up the
- temporary location when done (cf. self.clean_fs method)
+ def export_temporary(self, revision: int) -> Tuple[str, bytes]:
+ """Export the repository to a given revision in a temporary location. This is up
+ to the caller of this function to clean up the temporary location when done (cf.
+ self.clean_fs method)
Args:
revision: Revision to export at
@@ -203,7 +211,7 @@
"""
local_dirname = tempfile.mkdtemp(
- prefix="check-revision-%s." % revision, dir=self.local_dirname
+ dir=self.local_dirname, prefix=f"check-revision-{revision}."
)
local_name = os.path.basename(self.remote_url)
local_url = os.path.join(local_dirname, local_name)
@@ -212,7 +220,18 @@
)
return local_dirname, os.fsencode(local_url)
- def swh_hash_data_per_revision(self, start_revision, end_revision):
+ def swh_hash_data_per_revision(
+ self, start_revision: int, end_revision: int
+ ) -> Iterator[
+ Tuple[
+ int,
+ Optional[int],
+ Dict,
+ Tuple[List[Content], List[SkippedContent], List[Directory]],
+ DirectoryFromDisk,
+ ],
+ ]:
+
"""Compute swh hash data per each revision between start_revision and
end_revision.
@@ -221,11 +240,14 @@
end_revision: ending revision
Yields:
- tuple (rev, nextrev, commit, objects_per_path)
+ Tuple (rev, nextrev, commit, objects_per_path):
+
- rev: current revision
- - nextrev: next revision
+ - nextrev: next revision or None if we reached end_revision.
- commit: commit data (author, date, message) for such revision
- - objects_per_path: dictionary of path, swh hash data with type
+ - objects_per_path: Tuple of list of objects between start_revision and
+ end_revision
+ - complete Directory representation
"""
for commit in self.logs(start_revision, end_revision):
@@ -239,16 +261,20 @@
yield rev, nextrev, commit, objects, self.swhreplay.directory
- def swh_hash_data_at_revision(self, revision):
- """Compute the hash data at revision.
+ def swh_hash_data_at_revision(
+ self, revision: int
+ ) -> Iterator[Tuple[Dict, DirectoryFromDisk]]:
+ """Compute the information at a given svn revision. This is expected to be used
+ for update only.
- Expected to be used for update only.
+ Yields:
+ The tuple (commit dictionary, targeted directory object).
"""
- # Update the disk at revision
+ # Update disk representation of the repository at revision id
self.export(revision)
# Compute the current hashes on disk
- directory = Directory.from_disk(
+ directory = DirectoryFromDisk.from_disk(
path=os.fsencode(self.local_url), max_content_length=self.max_content_length
)
@@ -260,18 +286,17 @@
# Retrieve the commit information for revision
commit = list(self.logs(revision, revision))[0]
- yield revision, revision + 1, commit, {}, directory
+ yield commit, directory
- def clean_fs(self, local_dirname=None):
+ def clean_fs(self, local_dirname: Optional[str] = None) -> None:
"""Clean up the local working copy.
Args:
- local_dirname (str): Path to remove recursively if
- provided. Otherwise, remove the temporary upper root tree
- used for svn repository loading.
+ local_dirname: Path to remove recursively if provided. Otherwise, remove the
+ temporary upper root tree used for svn repository loading.
"""
- dirname = local_dirname if local_dirname else self.local_dirname
+ dirname = local_dirname or self.local_dirname
if os.path.exists(dirname):
logger.debug("cleanup %s", dirname)
shutil.rmtree(dirname)