diff --git a/swh/vault/backend.py b/swh/vault/backend.py --- a/swh/vault/backend.py +++ b/swh/vault/backend.py @@ -79,7 +79,7 @@ self.cache = VaultCache(self.config['cache']) self.db = None self.reconnect() - self.smtp_server = smtplib.SMTP('localhost') + self.smtp_server = smtplib.SMTP('localhost', 25) def reconnect(self): """Reconnect to the database.""" @@ -268,13 +268,25 @@ msg['From'] = NOTIF_EMAIL_FROM msg['To'] = email - self.smtp_server.send_message(msg) + self._smtp_send(msg) if n_id is not None: cursor.execute(''' DELETE FROM vault_notif_email WHERE id = %s''', (n_id,)) + def _smtp_send(self, msg): + # Reconnect if needed + try: + status = self.smtp_server.noop()[0] + except: # smtplib.SMTPServerDisconnected + status = -1 + if status != 250: + self.smtp_server.connect() + + # Send the message + self.smtp_server.send_message(msg) + @autocommit def _cache_expire(self, cond, *args, cursor=None): """Low-level expiration method, used by cache_expire_* methods""" diff --git a/swh/vault/cookers/base.py b/swh/vault/cookers/base.py --- a/swh/vault/cookers/base.py +++ b/swh/vault/cookers/base.py @@ -194,21 +194,17 @@ content = get_filtered_file_content(self.storage, file_data) self._create_file(path, content, file_data['perms']) - def _create_file(self, path, content, perms=0o100644): + def _create_file(self, path, content, mode=0o100644): """Create the given file and fill it with content. """ - if perms not in (0o100644, 0o100755, 0o120000): - logging.warning('File {} has invalid permission {}, ' - 'defaulting to 644.'.format(path, perms)) - perms = 0o100644 - - if perms == 0o120000: # Symbolic link + perms = mode_to_perms(mode) + if perms == DentryPerms.symlink: os.symlink(content, path) else: with open(path, 'wb') as f: f.write(content) - os.chmod(path, perms & 0o777) + os.chmod(path, perms.value) def _get_file_content(self, obj_id): """Get the content of the given file. diff --git a/swh/vault/cookers/revision_gitfast.py b/swh/vault/cookers/revision_gitfast.py --- a/swh/vault/cookers/revision_gitfast.py +++ b/swh/vault/cookers/revision_gitfast.py @@ -12,6 +12,7 @@ from .base import BaseVaultCooker, get_filtered_file_content from swh.model import hashutil +from swh.model.from_disk import mode_to_perms class RevisionGitfastCooker(BaseVaultCooker): @@ -206,7 +207,7 @@ yield from self._compute_blob_command_content(f) commands.append(fastimport.commands.FileModifyCommand( path=os.path.join(root, fname), - mode=f['perms'], + mode=mode_to_perms(f['perms']).value, dataref=(b':' + self.mark(f['sha1'])), data=None, )) diff --git a/swh/vault/tests/test_cookers.py b/swh/vault/tests/test_cookers.py --- a/swh/vault/tests/test_cookers.py +++ b/swh/vault/tests/test_cookers.py @@ -151,7 +151,7 @@ directory = Directory.from_disk(path=bytes(p)) self.assertEqual(obj_id_hex, hashutil.hash_to_hex(directory.hash)) - def test_filtered_objects(self): + def test_directory_filtered_objects(self): repo = TestRepo() with repo as rp: file_1, id_1 = hash_content(b'test1') @@ -184,6 +184,29 @@ self.assertEqual((p / 'hidden_file').read_bytes(), HIDDEN_MESSAGE) self.assertEqual((p / 'absent_file').read_bytes(), SKIPPED_MESSAGE) + def test_directory_bogus_perms(self): + # Some early git repositories have 664/775 permissions... let's check + # if all the weird modes are properly normalized in the directory + # cooker. + repo = TestRepo() + with repo as rp: + (rp / 'file').write_text(TEST_CONTENT) + (rp / 'file').chmod(0o664) + (rp / 'executable').write_bytes(TEST_EXECUTABLE) + (rp / 'executable').chmod(0o775) + (rp / 'wat').write_text(TEST_CONTENT) + (rp / 'wat').chmod(0o604) + c = repo.commit() + self.load(str(rp)) + + obj_id_hex = repo.repo[c].tree.decode() + obj_id = hashutil.hash_to_bytes(obj_id_hex) + + with self.cook_extract_directory(obj_id) as p: + self.assertEqual((p / 'file').stat().st_mode, 0o100644) + self.assertEqual((p / 'executable').stat().st_mode, 0o100755) + self.assertEqual((p / 'wat').stat().st_mode, 0o100644) + class TestRevisionGitfastCooker(BaseTestCookers, unittest.TestCase): def test_revision_simple(self): @@ -307,7 +330,7 @@ with self.cook_extract_revision_gitfast(obj_id) as (ert, p): self.assertEqual(ert.repo.refs[b'HEAD'].decode(), obj_id_hex) - def test_filtered_objects(self): + def test_revision_filtered_objects(self): repo = TestRepo() with repo as rp: file_1, id_1 = hash_content(b'test1') @@ -339,3 +362,26 @@ self.assertEqual((p / 'file').read_bytes(), b'test1') self.assertEqual((p / 'hidden_file').read_bytes(), HIDDEN_MESSAGE) self.assertEqual((p / 'absent_file').read_bytes(), SKIPPED_MESSAGE) + + def test_revision_bogus_perms(self): + # Some early git repositories have 664/775 permissions... let's check + # if all the weird modes are properly normalized in the revision + # cooker. + repo = TestRepo() + with repo as rp: + (rp / 'file').write_text(TEST_CONTENT) + (rp / 'file').chmod(0o664) + (rp / 'executable').write_bytes(TEST_EXECUTABLE) + (rp / 'executable').chmod(0o775) + (rp / 'wat').write_text(TEST_CONTENT) + (rp / 'wat').chmod(0o604) + repo.commit('initial commit') + self.load(str(rp)) + obj_id_hex = repo.repo.refs[b'HEAD'].decode() + obj_id = hashutil.hash_to_bytes(obj_id_hex) + + with self.cook_extract_revision_gitfast(obj_id) as (ert, p): + ert.checkout(b'HEAD') + self.assertEqual((p / 'file').stat().st_mode, 0o100644) + self.assertEqual((p / 'executable').stat().st_mode, 0o100755) + self.assertEqual((p / 'wat').stat().st_mode, 0o100644)