Differential D2041 Diff 6861 swh/storage/tests/test_storage.py

Changeset View

Standalone View

swh/storage/tests/test_storage.py

# Copyright (C) 2015-2019 The Software Heritage developers		# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import copy		import copy
import datetime		import datetime
import itertools		import itertools
		import queue
import random		import random
		import sys
		import threading
		import time
import unittest		import unittest
from collections import defaultdict		from collections import defaultdict
from unittest.mock import Mock, patch		from unittest.mock import Mock, patch

import psycopg2		import psycopg2
import pytest		import pytest

from hypothesis import given, strategies, settings, HealthCheck		from hypothesis import given, strategies, settings, HealthCheck
▲ Show 20 Lines • Show All 3,949 Lines • ▼ Show 20 Lines	def test_content_add_objstorage_exception(self):
self.storage.content_add([self.cont])		self.storage.content_add([self.cont])

self.assertEqual(e.exception.args, ('mocked broken objstorage',))		self.assertEqual(e.exception.args, ('mocked broken objstorage',))
missing = list(self.storage.content_missing([self.cont]))		missing = list(self.storage.content_missing([self.cont]))
self.assertEqual(missing, [self.cont['sha1']])		self.assertEqual(missing, [self.cont['sha1']])


@pytest.mark.db		@pytest.mark.db
		class TestStorageRaceConditions(TestStorageData, StorageTestDbFixture,
		unittest.TestCase):
		def test_content_add_race(self):

		results = queue.Queue()

		def thread1():
		db1 = None
		try:
		db1 = self.storage.get_db()
		with db1.transaction() as cur1:
		ret = self.storage.content_add([self.cont], db=db1,
		anlambertUnsubmitted Not Done Inline Actions For your information, the test also fails when not providing the `db` and `cur` parameter. Are they really needed ? anlambert: For your information, the test also fails when not providing the `db` and `cur` parameter. Are…
		olasdAuthorUnsubmitted Not Done Inline Actions They are not really (in the current state of this code). Technically, to be sure that the race condition triggers all the time, we need to add a synchronization point between the threads, so that we are sure that both transactions are open at the same time. This means synchronizing before the `content_add` call, but after `db.transaction()`. olasd: They are not really (in the current state of this code). Technically, to be sure that the race…
		cur=cur1)
		results.put(('thread1', 'data', ret))
		except Exception:
		results.put(('thread1', 'exc', sys.exc_info()))
		finally:
		if db1:
		self.storage.put_db(db1)

		def thread2():
		db2 = None
		try:
		db2 = self.storage.get_db()
		with db2.transaction() as cur2:
		ret = self.storage.content_add([self.cont], db=db2,
		cur=cur2)
		results.put(('thread2', 'data', ret))
		except Exception as e:
		results.put(('thread2', 'exc', e))
		finally:
		if db2:
		self.storage.put_db(db2)

		t1 = threading.Thread(target=thread1)
		t2 = threading.Thread(target=thread2)
		t1.start()
		#time.sleep(1)
		t2.start()
		t1.join()
		t2.join()

		r1 = results.get(block=False)
		r2 = results.get(block=False)

		with pytest.raises(queue.Empty):
		results.get(block=False)
		assert {r1[0], r2[0]} == {'thread1', 'thread2'}
		assert r1[1] == 'data', 'Got exception %r in %s' % (r1[2], r1[0])
		assert r2[1] == 'data', 'Got exception %r in %s' % (r2[2], r2[0])


		@pytest.mark.db
@pytest.mark.property_based		@pytest.mark.property_based
class PropTestLocalStorage(CommonPropTestStorage, StorageTestDbFixture,		class PropTestLocalStorage(CommonPropTestStorage, StorageTestDbFixture,
unittest.TestCase):		unittest.TestCase):
pass		pass


class AlteringSchemaTest(TestStorageData, StorageTestDbFixture,		class AlteringSchemaTest(TestStorageData, StorageTestDbFixture,
unittest.TestCase):		unittest.TestCase):
▲ Show 20 Lines • Show All 61 Lines • Show Last 20 Lines