Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/ra.py
# Copyright (C) 2016-2021 The Software Heritage developers | # Copyright (C) 2016-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
"""Remote Access client to svn server. | """Remote Access client to svn server. | ||||
""" | """ | ||||
import codecs | import codecs | ||||
import os | import os | ||||
import shutil | import shutil | ||||
import tempfile | import tempfile | ||||
from typing import Dict, List, Tuple | from typing import List, Optional, Tuple | ||||
import click | import click | ||||
from subvertpy import delta, properties | from subvertpy import delta, properties | ||||
from subvertpy.ra import Auth, RemoteAccess, get_username_provider | from subvertpy.ra import Auth, RemoteAccess, get_username_provider | ||||
from swh.model import from_disk, hashutil | from swh.model import from_disk, hashutil | ||||
from swh.model.model import Content, Directory, SkippedContent | from swh.model.model import Content, Directory, SkippedContent | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | |||||
DEFAULT_FLAG = 0 | DEFAULT_FLAG = 0 | ||||
EXEC_FLAG = 1 | EXEC_FLAG = 1 | ||||
NOEXEC_FLAG = 2 | NOEXEC_FLAG = 2 | ||||
SVN_PROPERTY_EOL = "svn:eol-style" | SVN_PROPERTY_EOL = "svn:eol-style" | ||||
# EOL state check mess | |||||
EOL_STYLE = {} | |||||
anlambert: Could you add a small docstring to explain the purpose of that class ? | |||||
Done Inline ActionsAny suggestion? I'm not sure I understand it myself... vlorentz: Any suggestion? I'm not sure I understand it myself... | |||||
Not Done Inline ActionsCould be something like: Class used to persist some file states (end of lines style for instance) across revisions replay when associated subversion properties are modified. anlambert: Could be something like: `Class used to persist some file states (end of lines style for… | |||||
class FileEditor: | class FileEditor: | ||||
"""File Editor in charge of updating file on disk and memory objects. | """File Editor in charge of updating file on disk and memory objects. | ||||
""" | """ | ||||
__slots__ = ["directory", "path", "fullpath", "executable", "link"] | __slots__ = [ | ||||
"directory", | |||||
"path", | |||||
"fullpath", | |||||
"executable", | |||||
"link", | |||||
"eol_style", | |||||
"svn_special_path_non_link_data", | |||||
] | |||||
eol_style: Optional[str] | |||||
"""EOL state check mess""" | |||||
# keep track of non link file content with svn:special property set | svn_special_path_non_link_data: bytes | ||||
svn_special_path_non_link_data: Dict[str, bytes] = {} | """keep track of non link file content with svn:special property set""" | ||||
def __init__(self, directory, rootpath, path): | def __init__(self, directory, rootpath, path): | ||||
Not Done Inline ActionsPlease add types ;) ardumont: Please add types ;) | |||||
self.directory = directory | self.directory = directory | ||||
self.path = path | self.path = path | ||||
# default value: 0, 1: set the flag, 2: remove the exec flag | # default value: 0, 1: set the flag, 2: remove the exec flag | ||||
self.executable = DEFAULT_FLAG | self.executable = DEFAULT_FLAG | ||||
self.link = None | self.link = None | ||||
self.fullpath = os.path.join(rootpath, path) | self.fullpath = os.path.join(rootpath, path) | ||||
self.eol_style = None | |||||
self.svn_special_path_non_link_data = None | |||||
def change_prop(self, key, value): | def change_prop(self, key, value): | ||||
if key == properties.PROP_EXECUTABLE: | if key == properties.PROP_EXECUTABLE: | ||||
if value is None: # bit flip off | if value is None: # bit flip off | ||||
self.executable = NOEXEC_FLAG | self.executable = NOEXEC_FLAG | ||||
else: | else: | ||||
self.executable = EXEC_FLAG | self.executable = EXEC_FLAG | ||||
elif key == properties.PROP_SPECIAL: | elif key == properties.PROP_SPECIAL: | ||||
# Possibly a symbolic link. We cannot check further at | # Possibly a symbolic link. We cannot check further at | ||||
# that moment though, patch(s) not being applied yet | # that moment though, patch(s) not being applied yet | ||||
self.link = value is not None | self.link = value is not None | ||||
elif key == SVN_PROPERTY_EOL: | elif key == SVN_PROPERTY_EOL: | ||||
# backup end of line style for file | # backup end of line style for file | ||||
EOL_STYLE[self.fullpath] = value | self.eol_style = value | ||||
def __make_symlink(self, src): | def __make_symlink(self, src): | ||||
"""Convert the svnlink to a symlink on disk. | """Convert the svnlink to a symlink on disk. | ||||
This function expects self.fullpath to be a svn link. | This function expects self.fullpath to be a svn link. | ||||
Args: | Args: | ||||
src (bytes): Path to the link's source | src (bytes): Path to the link's source | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | def close(self): | ||||
# same file as the export operation. | # same file as the export operation. | ||||
with open(self.fullpath, "rb") as f: | with open(self.fullpath, "rb") as f: | ||||
content = f.read() | content = f.read() | ||||
with open(self.fullpath, "wb") as f: | with open(self.fullpath, "wb") as f: | ||||
exported_data = content.split(b"\x00")[0] | exported_data = content.split(b"\x00")[0] | ||||
if exported_data != content: | if exported_data != content: | ||||
# keep track of original file content in order to restore | # keep track of original file content in order to restore | ||||
# it if the svn:special property gets unset in another revision | # it if the svn:special property gets unset in another revision | ||||
self.svn_special_path_non_link_data[self.fullpath] = content | self.svn_special_path_non_link_data = content | ||||
f.write(exported_data) | f.write(exported_data) | ||||
elif os.path.islink(self.fullpath): | elif os.path.islink(self.fullpath): | ||||
# path was a symbolic link in previous revision but got the property | # path was a symbolic link in previous revision but got the property | ||||
# svn:special unset in current one, revert its content to svn link format | # svn:special unset in current one, revert its content to svn link format | ||||
self.__make_svnlink() | self.__make_svnlink() | ||||
elif self.fullpath in self.svn_special_path_non_link_data: | elif self.svn_special_path_non_link_data is not None: | ||||
# path was a non link file with the svn:special property previously set | # path was a non link file with the svn:special property previously set | ||||
# and got truncated on export, restore its original content | # and got truncated on export, restore its original content | ||||
with open(self.fullpath, "wb") as f: | with open(self.fullpath, "wb") as f: | ||||
f.write(self.svn_special_path_non_link_data[self.fullpath]) | f.write(self.svn_special_path_non_link_data) | ||||
del self.svn_special_path_non_link_data[self.fullpath] | self.svn_special_path_non_link_data = None | ||||
if not is_link: # if a link, do nothing regarding flag | if not is_link: # if a link, do nothing regarding flag | ||||
if self.executable == EXEC_FLAG: | if self.executable == EXEC_FLAG: | ||||
os.chmod(self.fullpath, 0o755) | os.chmod(self.fullpath, 0o755) | ||||
elif self.executable == NOEXEC_FLAG: | elif self.executable == NOEXEC_FLAG: | ||||
os.chmod(self.fullpath, 0o644) | os.chmod(self.fullpath, 0o644) | ||||
# And now compute file's checksums | # And now compute file's checksums | ||||
eol_style = EOL_STYLE.get(self.fullpath, None) | if self.eol_style and not is_link: | ||||
if eol_style and not is_link: | |||||
# ensure to normalize line endings as defined by svn:eol-style | # ensure to normalize line endings as defined by svn:eol-style | ||||
# property to get the same file checksum as after an export | # property to get the same file checksum as after an export | ||||
# or checkout operation with subversion | # or checkout operation with subversion | ||||
with open(self.fullpath, "rb") as f: | with open(self.fullpath, "rb") as f: | ||||
data = f.read() | data = f.read() | ||||
data = _normalize_line_endings(data, eol_style) | data = _normalize_line_endings(data, self.eol_style) | ||||
mode = os.lstat(self.fullpath).st_mode | mode = os.lstat(self.fullpath).st_mode | ||||
self.directory[self.path] = from_disk.Content.from_bytes( | self.directory[self.path] = from_disk.Content.from_bytes( | ||||
mode=mode, data=data | mode=mode, data=data | ||||
) | ) | ||||
else: | else: | ||||
self.directory[self.path] = from_disk.Content.from_file(path=self.fullpath) | self.directory[self.path] = from_disk.Content.from_file(path=self.fullpath) | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | def remove_child(self, path): | ||||
entry_removed = None | entry_removed = None | ||||
else: | else: | ||||
del self.directory[path] | del self.directory[path] | ||||
fpath = os.path.join(self.rootpath, path) | fpath = os.path.join(self.rootpath, path) | ||||
if isinstance(entry_removed, from_disk.Directory): | if isinstance(entry_removed, from_disk.Directory): | ||||
shutil.rmtree(fpath) | shutil.rmtree(fpath) | ||||
else: | else: | ||||
os.remove(fpath) | os.remove(fpath) | ||||
Not Done Inline Actionsmake it a more sensible debug statement instead? ardumont: make it a more sensible debug statement instead? | |||||
Not Done Inline Actionsnot sure there is an interest to log such a debug statement here, looks like a debug print that has been accidentally committed. anlambert: not sure there is an interest to log such a debug statement here, looks like a debug print that… | |||||
Done Inline Actionsindeed vlorentz: indeed | |||||
# when deleting a directory ensure to remove any eol style setting for the | |||||
# file it contains as they can be added again later in another revision | |||||
# without the svn:eol-style property set | |||||
fullpath = os.path.join(self.rootpath, path) | |||||
for eol_path in list(EOL_STYLE): | |||||
if eol_path.startswith(fullpath): | |||||
del EOL_STYLE[eol_path] | |||||
def update_checksum(self): | def update_checksum(self): | ||||
raise NotImplementedError("This should be implemented.") | raise NotImplementedError("This should be implemented.") | ||||
def open_directory(self, *args): | def open_directory(self, *args): | ||||
raise NotImplementedError("This should be implemented.") | raise NotImplementedError("This should be implemented.") | ||||
def add_directory(self, *args): | def add_directory(self, *args): | ||||
raise NotImplementedError("This should be implemented.") | raise NotImplementedError("This should be implemented.") | ||||
▲ Show 20 Lines • Show All 216 Lines • Show Last 20 Lines |
Could you add a small docstring to explain the purpose of that class ?