Changeset View
Standalone View
swh/loader/svn/ra.py
Show All 13 Lines | |||||
from subvertpy import delta, properties | from subvertpy import delta, properties | ||||
from subvertpy.ra import RemoteAccess, Auth, get_username_provider | from subvertpy.ra import RemoteAccess, Auth, get_username_provider | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.model.from_disk import Content, Directory | from swh.model.from_disk import Content, Directory | ||||
CRLF = b'\r\n' | _eol_style = { | ||||
olasd: This should probably be hardcoded to a "native to Software Heritage" line ending style, rather… | |||||
Not Done Inline ActionsOf course, will be changed in upcoming updated difff anlambert: Of course, will be changed in upcoming updated difff | |||||
'native': b'\n', | |||||
'CRLF': b'\r\n', | |||||
'LF': b'\n', | |||||
'CR': b'\r' | |||||
} | |||||
Not Done Inline ActionsDoes this function replicate what svn does when encountering mixed line endings on the file sent by the server (e.g. a file with CRLFs with a lone LF)? Does this ever happen? olasd: Does this function replicate what svn does when encountering mixed line endings on the file… | |||||
Not Done Inline ActionsYes, the function will guarantee that all line endings will be the same after processing the buffer of bytes. anlambert: Yes, the function will guarantee that all line endings will be the same after processing the… | |||||
def _normalize_line_endings(lines, eol_style='native'): | |||||
"""Normalize line endings to unix (\n), windows (\r\n) or mac (\r). | |||||
Args: | |||||
lines (bytes): The lines to normalize | |||||
line_ending (str): The line ending format as defined for | |||||
svn:eol-style property. Acceptable values are 'native', | |||||
'CRLF', 'LF' and 'CR' | |||||
Returns | |||||
bytes: lines with endings normalized | |||||
""" | |||||
lines = lines.replace(_eol_style['CRLF'], _eol_style['LF'])\ | |||||
.replace(_eol_style['CR'], _eol_style['LF']) | |||||
if _eol_style[eol_style] != _eol_style['LF']: | |||||
lines = lines.replace(_eol_style['LF'], _eol_style[eol_style]) | |||||
return lines | |||||
def apply_txdelta_handler(sbuf, target_stream): | def apply_txdelta_handler(sbuf, target_stream): | ||||
"""Return a function that can be called repeatedly with txdelta windows. | """Return a function that can be called repeatedly with txdelta windows. | ||||
When done, closes the target_stream. | When done, closes the target_stream. | ||||
Adapted from subvertpy.delta.apply_txdelta_handler to close the | Adapted from subvertpy.delta.apply_txdelta_handler to close the | ||||
stream when done. | stream when done. | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | |||||
DEFAULT_FLAG = 0 | DEFAULT_FLAG = 0 | ||||
EXEC_FLAG = 1 | EXEC_FLAG = 1 | ||||
NOEXEC_FLAG = 2 | NOEXEC_FLAG = 2 | ||||
SVN_PROPERTY_EOL = 'svn:eol-style' | SVN_PROPERTY_EOL = 'svn:eol-style' | ||||
# EOL state check mess | # EOL state check mess | ||||
EOL_CHECK = {} | EOL_STYLE = {} | ||||
class SWHFileEditor: | class SWHFileEditor: | ||||
"""File Editor in charge of updating file on disk and memory objects. | """File Editor in charge of updating file on disk and memory objects. | ||||
""" | """ | ||||
__slots__ = ['directory', 'path', 'fullpath', 'executable', 'link'] | __slots__ = ['directory', 'path', 'fullpath', 'executable', 'link'] | ||||
Show All 10 Lines | def change_prop(self, key, value): | ||||
if value is None: # bit flip off | if value is None: # bit flip off | ||||
self.executable = NOEXEC_FLAG | self.executable = NOEXEC_FLAG | ||||
else: | else: | ||||
self.executable = EXEC_FLAG | self.executable = EXEC_FLAG | ||||
elif key == properties.PROP_SPECIAL: | elif key == properties.PROP_SPECIAL: | ||||
# Possibly a symbolic link. We cannot check further at | # Possibly a symbolic link. We cannot check further at | ||||
# that moment though, patch(s) not being applied yet | # that moment though, patch(s) not being applied yet | ||||
self.link = True | self.link = True | ||||
elif key == SVN_PROPERTY_EOL: # Detect inconsistent repositories | elif key == SVN_PROPERTY_EOL: | ||||
if value in ['LF', 'native']: | # backup end of line style for file | ||||
EOL_CHECK[self.fullpath] = value | EOL_STYLE[self.fullpath] = value | ||||
else: | |||||
if self.fullpath in EOL_CHECK: | |||||
del EOL_CHECK[self.fullpath] | |||||
def __make_symlink(self, src): | def __make_symlink(self, src): | ||||
"""Convert the svnlink to a symlink on disk. | """Convert the svnlink to a symlink on disk. | ||||
This function expects self.fullpath to be a svn link. | This function expects self.fullpath to be a svn link. | ||||
Args: | Args: | ||||
src (bytes): Path to the link's source | src (bytes): Path to the link's source | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | def close(self): | ||||
self.link = False | self.link = False | ||||
if not is_link: # if a link, do nothing regarding flag | if not is_link: # if a link, do nothing regarding flag | ||||
if self.executable == EXEC_FLAG: | if self.executable == EXEC_FLAG: | ||||
os.chmod(self.fullpath, 0o755) | os.chmod(self.fullpath, 0o755) | ||||
elif self.executable == NOEXEC_FLAG: | elif self.executable == NOEXEC_FLAG: | ||||
os.chmod(self.fullpath, 0o644) | os.chmod(self.fullpath, 0o644) | ||||
check_eol = EOL_CHECK.get(self.fullpath) | |||||
if check_eol: | |||||
raw_content = open(self.fullpath, 'rb').read() | |||||
if CRLF in raw_content: # CRLF | |||||
msg = 'Inconsistency. CRLF detected in a converted ' \ | |||||
'file %s (%s: %s)' % ( | |||||
self.fullpath, SVN_PROPERTY_EOL, check_eol) | |||||
raise ValueError(msg) | |||||
# And now compute file's checksums | # And now compute file's checksums | ||||
eol_style = EOL_STYLE.get(self.fullpath, None) | |||||
if eol_style: | |||||
# ensure to normalize line endings as defined by svn:eol-style | |||||
# property to get the same file checksum as after an export | |||||
# or checkout operation with subversion | |||||
with open(self.fullpath, 'rb') as f: | |||||
data = f.read() | |||||
data = _normalize_line_endings(data, eol_style) | |||||
mode = os.lstat(self.fullpath).st_mode | |||||
self.directory[self.path] = Content.from_bytes(mode=mode, | |||||
Not Done Inline ActionsThat's interesting, that could be used for the svn symlink use case as well. Today, the svn symlink disk representation is changed on disk (before/after patch appliance/hash computation, don't remember the right combination here but you grok the idea). Having a Content.from_svn_symlink (specific to that loader) would be preferable. Nice. ardumont: That's interesting, that could be used for the svn symlink use case as well.
Today, the svn… | |||||
data=data) | |||||
else: | |||||
self.directory[self.path] = Content.from_file(path=self.fullpath, | self.directory[self.path] = Content.from_file(path=self.fullpath, | ||||
data=True) | data=True) | ||||
class BaseDirSWHEditor: | class BaseDirSWHEditor: | ||||
"""Base class implementation of dir editor. | """Base class implementation of dir editor. | ||||
see :class:`SWHDirEditor` for an implementation that hashes every | see :class:`SWHDirEditor` for an implementation that hashes every | ||||
directory encountered. | directory encountered. | ||||
Show All 36 Lines | def remove_child(self, path): | ||||
entry_removed = None | entry_removed = None | ||||
else: | else: | ||||
del self.directory[path] | del self.directory[path] | ||||
fpath = os.path.join(self.rootpath, path) | fpath = os.path.join(self.rootpath, path) | ||||
if isinstance(entry_removed, Directory): | if isinstance(entry_removed, Directory): | ||||
shutil.rmtree(fpath) | shutil.rmtree(fpath) | ||||
else: | else: | ||||
os.remove(fpath) | os.remove(fpath) | ||||
if path in EOL_CHECK: | if path in EOL_STYLE: | ||||
del EOL_CHECK[path] | del EOL_STYLE[path] | ||||
def update_checksum(self): | def update_checksum(self): | ||||
raise NotImplementedError('This should be implemented.') | raise NotImplementedError('This should be implemented.') | ||||
def open_directory(self, *args): | def open_directory(self, *args): | ||||
raise NotImplementedError('This should be implemented.') | raise NotImplementedError('This should be implemented.') | ||||
def add_directory(self, *args): | def add_directory(self, *args): | ||||
▲ Show 20 Lines • Show All 192 Lines • Show Last 20 Lines |
This should probably be hardcoded to a "native to Software Heritage" line ending style, rather than depend on the platform on which the loader runs.
(I propose b'\n', obviously)