Differential D1611 Diff 5367 swh/objstorage/backends/pathslicing.py

Changeset View

Standalone View

swh/objstorage/backends/pathslicing.py

Show First 20 Lines • Show All 51 Lines • ▼ Show 20 Lines	def _write_obj_file(hex_obj_id, objstorage):
(tmp, tmp_path) = tempfile.mkstemp(suffix='.tmp', prefix='hex_obj_id.',		(tmp, tmp_path) = tempfile.mkstemp(suffix='.tmp', prefix='hex_obj_id.',
dir=dir)		dir=dir)

# Open the file and yield it for writing.		# Open the file and yield it for writing.
tmp_f = os.fdopen(tmp, 'wb')		tmp_f = os.fdopen(tmp, 'wb')
with gzip.GzipFile(filename=tmp_path, fileobj=tmp_f) as f:		with gzip.GzipFile(filename=tmp_path, fileobj=tmp_f) as f:
yield f		yield f

		# Make sure the contents of the temporary file are written to disk
		tmp_f.flush()
		if objstorage.use_fdatasync:
		os.fdatasync(tmp)
		else:
		os.fsync(tmp)

# Then close the temporary file and move it to the right directory.		# Then close the temporary file and move it to the right directory.
tmp_f.close()		tmp_f.close()
os.chmod(tmp_path, FILE_MODE)		os.chmod(tmp_path, FILE_MODE)
os.rename(tmp_path, path)		os.rename(tmp_path, path)


@contextmanager		@contextmanager
def _read_obj_file(hex_obj_id, objstorage):		def _read_obj_file(hex_obj_id, objstorage):
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines	def __init__(self, root, slicing, **kwargs):
self.root = root		self.root = root
# Make a list of tuples where each tuple contains the beginning		# Make a list of tuples where each tuple contains the beginning
# and the end of each slicing.		# and the end of each slicing.
self.bounds = [		self.bounds = [
slice(*map(int, sbounds.split(':')))		slice(*map(int, sbounds.split(':')))
for sbounds in slicing.split('/')		for sbounds in slicing.split('/')
if sbounds		if sbounds
]		]

		zackUnsubmitted Not Done Inline Actions Is this because some python versions don't have it, or just for OS portability? I'm asking because we now have evidence (in T1817) that fdatasync is critical for us, in the sense that we can lose data if it's not available. So just silently ignoring the fact it's not available makes me shiver :-) I'm tempted to suggest failing to initialize if fdatasync is not available. Either way, we should log an error message if it's not available. zack: Is this because some python versions don't have it, or just for OS portability? I'm asking…
		olasdAuthorUnsubmitted Done Inline Actions The new version falls back to fsync (available on all platforms) if fdatasync isn't available. olasd: The new version falls back to fsync (available on all platforms) if fdatasync isn't available.
		zackUnsubmitted Not Done Inline Actions thumb up zack: thumb up
		self.use_fdatasync = hasattr(os, 'fdatasync')

self.check_config(check_write=False)		self.check_config(check_write=False)

def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check whether this object storage is properly configured"""		"""Check whether this object storage is properly configured"""

root = self.root		root = self.root

if not os.path.isdir(root):		if not os.path.isdir(root):
▲ Show 20 Lines • Show All 238 Lines • Show Last 20 Lines