Differential D101 Diff 358 swh/storage/tests/test_storage.py

Changeset View

Standalone View

swh/storage/tests/test_storage.py

# Copyright (C) 2015 The Software Heritage developers		# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import datetime		import datetime
import os		import os
import psycopg2		import psycopg2
import shutil		import shutil
▲ Show 20 Lines • Show All 281 Lines • ▼ Show 20 Lines	def setUp(self):
'type': 'git',		'type': 'git',
}		}

self.origin2 = {		self.origin2 = {
'url': 'file:///dev/zero',		'url': 'file:///dev/zero',
'type': 'git',		'type': 'git',
}		}

		self.date_visit1 = datetime.datetime(2015, 1, 1, 23, 0, 0,
		tzinfo=datetime.timezone.utc)

self.occurrence = {		self.occurrence = {
'branch': b'master',		'branch': b'master',
'target': b'67890123456789012345',		'target': b'67890123456789012345',
'target_type': 'revision',		'target_type': 'revision',
'date': datetime.datetime(2015, 1, 1, 23, 0, 0,
tzinfo=datetime.timezone.utc),
}		}

		self.date_visit2 = datetime.datetime(2015, 1, 1, 23, 0, 0,
		tzinfo=datetime.timezone.utc)

self.occurrence2 = {		self.occurrence2 = {
'branch': b'master',		'branch': b'master',
'target': self.revision2['id'],		'target': self.revision2['id'],
'target_type': 'revision',		'target_type': 'revision',
'date': datetime.datetime(2015, 1, 1, 23, 0, 0,
tzinfo=datetime.timezone.utc),
}		}

		self.date_visit3 = datetime.datetime(2015, 1, 1, 23, 0, 0,
		tzinfo=datetime.timezone.utc)

# template occurrence to be filled in test (cf. revision_log_by)		# template occurrence to be filled in test (cf. revision_log_by)
self.occurrence3 = {		self.occurrence3 = {
'branch': b'master',		'branch': b'master',
'target_type': 'revision',		'target_type': 'revision',
'date': datetime.datetime(2015, 1, 1, 23, 0, 0,
tzinfo=datetime.timezone.utc),
}		}

self.release = {		self.release = {
'id': b'87659012345678901234',		'id': b'87659012345678901234',
'name': b'v0.0.1',		'name': b'v0.0.1',
'author': {		'author': {
'name': b'olasd',		'name': b'olasd',
'email': b'nic@olasd.fr',		'email': b'nic@olasd.fr',
▲ Show 20 Lines • Show All 430 Lines • ▼ Show 20 Lines	def revision_log_by(self):
# given		# given
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)
self.storage.revision_add([self.revision3,		self.storage.revision_add([self.revision3,
self.revision4])		self.revision4])

# occurrence3 targets 'revision4'		# occurrence3 targets 'revision4'
# with branch 'master' and origin origin_id		# with branch 'master' and origin origin_id
occurrence3 = self.occurrence3.copy()		occurrence3 = self.occurrence3.copy()
		date_visit1 = self.date_visit3
		origin_visit1 = self.storage.origin_visit_add(origin_id,
		date_visit1)
occurrence3.update({		occurrence3.update({
'origin': origin_id,		'origin': origin_id,
'target': self.revision4['id'],		'target': self.revision4['id'],
		'visit': origin_visit1['visit'],
})		})

self.storage.occurrence_add([occurrence3])		self.storage.occurrence_add([occurrence3])

# self.revision4 -is-child-of-> self.revision3		# self.revision4 -is-child-of-> self.revision3
# when		# when
actual_results = list(self.storage.revision_log_by(		actual_results = list(self.storage.revision_log_by(
origin_id,		origin_id,
branch_name=occurrence3['branch'],		branch_name=occurrence3['branch'],
timestamp=occurrence3['date']))		timestamp=date_visit1))

# hack: ids generated		# hack: ids generated
for actual_result in actual_results:		for actual_result in actual_results:
del actual_result['author']['id']		del actual_result['author']['id']
del actual_result['committer']['id']		del actual_result['committer']['id']

self.assertEqual(len(actual_results), 2)		self.assertEqual(len(actual_results), 2)
self.assertEquals(actual_results[0], self.revision4)		self.assertEquals(actual_results[0], self.revision4)
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines	def revision_get_by(self):
self.storage.content_add([self.cont2])		self.storage.content_add([self.cont2])
self.storage.directory_add([self.dir2]) # point to self.cont		self.storage.directory_add([self.dir2]) # point to self.cont
self.storage.revision_add([self.revision2]) # points to self.dir		self.storage.revision_add([self.revision2]) # points to self.dir
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)

# occurrence2 points to 'revision2' with branch 'master', we		# occurrence2 points to 'revision2' with branch 'master', we
# need to point to the right origin		# need to point to the right origin
occurrence2 = self.occurrence2.copy()		occurrence2 = self.occurrence2.copy()
occurrence2.update({'origin': origin_id})		date_visit1 = self.date_visit2
		origin_visit1 = self.storage.origin_visit_add(origin_id, date_visit1)
		occurrence2.update({
		'origin': origin_id,
		'visit': origin_visit1['visit'],
		})
self.storage.occurrence_add([occurrence2])		self.storage.occurrence_add([occurrence2])

# we want only revision 2		# we want only revision 2
expected_revisions = list(self.storage.revision_get(		expected_revisions = list(self.storage.revision_get(
[self.revision2['id']]))		[self.revision2['id']]))

# when		# when
actual_results = list(self.storage.revision_get_by(		actual_results = list(self.storage.revision_get_by(
Show All 20 Lines	def revision_get_by_multiple_occurrence(self):
# given		# given
self.storage.content_add([self.cont2])		self.storage.content_add([self.cont2])
self.storage.directory_add([self.dir2])		self.storage.directory_add([self.dir2])
self.storage.revision_add([self.revision2, self.revision3])		self.storage.revision_add([self.revision2, self.revision3])
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)

# occurrence2 points to 'revision2' with branch 'master', we		# occurrence2 points to 'revision2' with branch 'master', we
# need to point to the right origin		# need to point to the right origin
		date_visit1 = self.date_visit2
		origin_visit1 = self.storage.origin_visit_add(origin_id, date_visit1)
occurrence2 = self.occurrence2.copy()		occurrence2 = self.occurrence2.copy()
occurrence2.update({'origin': origin_id,		occurrence2.update({
'date': occurrence2['date']})		'origin': origin_id,
		'visit': origin_visit1['visit']
		})

dt = datetime.timedelta(days=1)		dt = datetime.timedelta(days=1)
		date_visit2 = date_visit1 + dt
		origin_visit2 = self.storage.origin_visit_add(origin_id, date_visit2)
occurrence3 = self.occurrence2.copy()		occurrence3 = self.occurrence2.copy()
occurrence3.update({'origin': origin_id,		occurrence3.update({
'date': occurrence3['date'] + dt,		'origin': origin_id,
'target': self.revision3['id']})		'visit': origin_visit2['visit'],
		'target': self.revision3['id'],
		})
# 2 occurrences on same revision with lower validity date with 1h delta		# 2 occurrences on same revision with lower validity date with 1h delta
self.storage.occurrence_add([occurrence2])		self.storage.occurrence_add([occurrence2])
self.storage.occurrence_add([occurrence3])		self.storage.occurrence_add([occurrence3])

# when		# when
actual_results0 = list(self.storage.revision_get_by(		actual_results0 = list(self.storage.revision_get_by(
origin_id,		origin_id,
occurrence2['branch'],		occurrence2['branch'],
occurrence2['date']))		date_visit1))

# hack: ids are generated		# hack: ids are generated
del actual_results0[0]['author']['id']		del actual_results0[0]['author']['id']
del actual_results0[0]['committer']['id']		del actual_results0[0]['committer']['id']

self.assertEquals(len(actual_results0), 1)		self.assertEquals(len(actual_results0), 1)
self.assertEqual(actual_results0, [self.revision2])		self.assertEqual(actual_results0, [self.revision2])

# when		# when
actual_results1 = list(self.storage.revision_get_by(		actual_results1 = list(self.storage.revision_get_by(
origin_id,		origin_id,
occurrence2['branch'],		occurrence2['branch'],
occurrence2['date'] + dt/3)) # closer to occurrence2		date_visit1 + dt/3)) # closer to first visit

# hack: ids are generated		# hack: ids are generated
del actual_results1[0]['author']['id']		del actual_results1[0]['author']['id']
del actual_results1[0]['committer']['id']		del actual_results1[0]['committer']['id']

self.assertEquals(len(actual_results1), 1)		self.assertEquals(len(actual_results1), 1)
self.assertEqual(actual_results1, [self.revision2])		self.assertEqual(actual_results1, [self.revision2])

# when		# when
actual_results2 = list(self.storage.revision_get_by(		actual_results2 = list(self.storage.revision_get_by(
origin_id,		origin_id,
occurrence2['branch'],		occurrence2['branch'],
occurrence2['date'] + 2*dt/3)) # closer to occurrence3		date_visit1 + 2*dt/3)) # closer to second visit

del actual_results2[0]['author']['id']		del actual_results2[0]['author']['id']
del actual_results2[0]['committer']['id']		del actual_results2[0]['committer']['id']

self.assertEquals(len(actual_results2), 1)		self.assertEquals(len(actual_results2), 1)
self.assertEqual(actual_results2, [self.revision3])		self.assertEqual(actual_results2, [self.revision3])

# when		# when
actual_results3 = list(self.storage.revision_get_by(		actual_results3 = list(self.storage.revision_get_by(
origin_id,		origin_id,
occurrence3['branch'],		occurrence3['branch'],
occurrence3['date']))		date_visit2))

# hack: ids are generated		# hack: ids are generated
del actual_results3[0]['author']['id']		del actual_results3[0]['author']['id']
del actual_results3[0]['committer']['id']		del actual_results3[0]['committer']['id']

self.assertEquals(len(actual_results3), 1)		self.assertEquals(len(actual_results3), 1)
self.assertEqual(actual_results3, [self.revision3])		self.assertEqual(actual_results3, [self.revision3])

▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines
def release_get_by(self):		def release_get_by(self):
# given		# given
self.storage.revision_add([self.revision2]) # points to self.dir		self.storage.revision_add([self.revision2]) # points to self.dir
self.storage.release_add([self.release3])		self.storage.release_add([self.release3])
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)

# occurrence2 points to 'revision2' with branch 'master', we		# occurrence2 points to 'revision2' with branch 'master', we
# need to point to the right origin		# need to point to the right origin
		origin_visit = self.storage.origin_visit_add(origin_id,
		self.date_visit2)
occurrence2 = self.occurrence2.copy()		occurrence2 = self.occurrence2.copy()
occurrence2.update({'origin': origin_id})		occurrence2.update({
		'origin': origin_id,
		'visit': origin_visit['visit'],
		})

self.storage.occurrence_add([occurrence2])		self.storage.occurrence_add([occurrence2])

# we want only revision 2		# we want only revision 2
expected_releases = list(self.storage.release_get(		expected_releases = list(self.storage.release_get(
[self.release3['id']]))		[self.release3['id']]))

# when		# when
actual_results = list(self.storage.release_get_by(		actual_results = list(self.storage.release_get_by(
Show All 32 Lines	def origin_get(self):

self.assertEqual(actual_origin1, {'id': id,		self.assertEqual(actual_origin1, {'id': id,
'type': self.origin['type'],		'type': self.origin['type'],
'url': self.origin['url'],		'url': self.origin['url'],
'lister': None,		'lister': None,
'project': None})		'project': None})

@istest		@istest
def origin_visit_get(self):		def origin_visit_add(self):
# 1- given		# given
		olasdUnsubmitted Done Inline Actions It'd be clearer if the tests were split between the add and update functions. olasd: It'd be clearer if the tests were split between the add and update functions.
		ardumontAuthorUnsubmitted Done Inline Actions I'm on it. ardumont: I'm on it.
self.assertIsNone(self.storage.origin_get(self.origin2))		self.assertIsNone(self.storage.origin_get(self.origin2))

self.storage.content_add([self.cont2])
self.storage.directory_add([self.dir2])
self.storage.revision_add([self.revision2, self.revision3])
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)
		self.assertIsNotNone(origin_id)

# occurrence2 points to 'revision2' with branch 'master', we		# when
# need to point to the right origin		origin_visit1 = self.storage.origin_visit_add(
occurrence2 = self.occurrence2.copy()		origin_id,
occurrence2.update({'origin': origin_id,		ts=self.date_visit2)
'date': occurrence2['date']})

dt = datetime.timedelta(days=1)

occurrence3 = self.occurrence2.copy()
occurrence3.update({'origin': origin_id,
'date': occurrence3['date'] + dt,
'target': self.revision3['id']})

# 2 occurrences on same revision with lower validity date with 1h delta		# then
self.storage.occurrence_add([occurrence2])		self.assertEquals(origin_visit1['origin'], origin_id)
		self.assertIsNotNone(origin_visit1['visit'])
		self.assertTrue(origin_visit1['visit'] > 0)

# when
actual_origin_visits = list(self.storage.origin_visit_get(origin_id))		actual_origin_visits = list(self.storage.origin_visit_get(origin_id))
self.assertEquals(len(actual_origin_visits), 1)
self.assertEquals(actual_origin_visits,		self.assertEquals(actual_origin_visits,
[{		[{
'origin': origin_id,		'origin': origin_id,
'date': occurrence2['date'],		'date': self.date_visit2,
'visit': 1		'visit': origin_visit1['visit'],
		'status': 'ongoing',
}])		}])

# 2- given		@istest
self.storage.occurrence_add([occurrence3])		def origin_visit_update(self):
		# given
		origin_id = self.storage.origin_add_one(self.origin2)
		origin_id2 = self.storage.origin_add_one(self.origin)

		origin_visit1 = self.storage.origin_visit_add(
		origin_id,
		ts=self.date_visit2)

		origin_visit2 = self.storage.origin_visit_add(
		origin_id,
		ts=self.date_visit3)

		origin_visit3 = self.storage.origin_visit_add(
		origin_id2,
		ts=self.date_visit3)

# when		# when
		self.storage.origin_visit_update(origin_id, origin_visit1['visit'],
		status='full')
		self.storage.origin_visit_update(origin_id2, origin_visit3['visit'],
		status='partial')

		# then
actual_origin_visits = list(self.storage.origin_visit_get(origin_id))		actual_origin_visits = list(self.storage.origin_visit_get(origin_id))
self.assertEquals(len(actual_origin_visits), 2)
self.assertEquals(actual_origin_visits,		self.assertEquals(actual_origin_visits,
[{		[{
'origin': origin_id,		'origin': origin_visit2['origin'],
'date': occurrence2['date'],		'date': self.date_visit2,
'visit': 1		'visit': origin_visit1['visit'],
}, {		'status': 'full'
'origin': origin_id,		},
'date': occurrence3['date'],		{
'visit': 2		'origin': origin_visit2['origin'],
		'date': self.date_visit3,
		'visit': origin_visit2['visit'],
		'status': 'ongoing'
		}])

		actual_origin_visits2 = list(self.storage.origin_visit_get(origin_id2))
		self.assertEquals(actual_origin_visits2,
		[{
		'origin': origin_visit3['origin'],
		'date': self.date_visit3,
		'visit': origin_visit3['visit'],
		'status': 'partial'
}])		}])

@istest		@istest
def occurrence_add(self):		def occurrence_add(self):
		occur = self.occurrence.copy()

origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)
		date_visit1 = self.date_visit1
		origin_visit1 = self.storage.origin_visit_add(origin_id, date_visit1)

revision = self.revision.copy()		revision = self.revision.copy()
revision['id'] = self.occurrence['target']		revision['id'] = occur['target']
self.storage.revision_add([revision])		self.storage.revision_add([revision])

occur = self.occurrence		occur.update({
occur['origin'] = origin_id		'origin': origin_id,
self.storage.occurrence_add([occur])		'visit': origin_visit1['visit'],
		})
self.storage.occurrence_add([occur])		self.storage.occurrence_add([occur])

test_query = '''		test_query = '''
with indiv_occurrences as (		with indiv_occurrences as (
select origin, branch, target, target_type, unnest(visits) as visit		select origin, branch, target, target_type, unnest(visits) as visit
from occurrence_history		from occurrence_history
)		)
select origin, branch, target, target_type, date		select origin, branch, target, target_type, date
from indiv_occurrences		from indiv_occurrences
left join origin_visit using(origin, visit)		left join origin_visit using(origin, visit)
order by origin, date'''		order by origin, date'''

self.cursor.execute(test_query)		self.cursor.execute(test_query)
ret = self.cursor.fetchall()		ret = self.cursor.fetchall()
self.assertEqual(len(ret), 1)		self.assertEqual(len(ret), 1)
self.assertEqual(		self.assertEqual(
(ret[0][0], ret[0][1].tobytes(), ret[0][2].tobytes(),		(ret[0][0], ret[0][1].tobytes(), ret[0][2].tobytes(),
ret[0][3], ret[0][4]),		ret[0][3], ret[0][4]),
(occur['origin'], occur['branch'], occur['target'],		(occur['origin'], occur['branch'], occur['target'],
occur['target_type'], occur['date']))		occur['target_type'], self.date_visit1))

orig_date = occur['date']		date_visit2 = date_visit1 + datetime.timedelta(hours=10)
occur['date'] += datetime.timedelta(hours=10)
self.storage.occurrence_add([occur])		origin_visit2 = self.storage.origin_visit_add(origin_id, date_visit2)
		occur2 = occur.copy()
		occur2.update({
		'visit': origin_visit2['visit'],
		})
		self.storage.occurrence_add([occur2])

self.cursor.execute(test_query)		self.cursor.execute(test_query)
ret = self.cursor.fetchall()		ret = self.cursor.fetchall()
self.assertEqual(len(ret), 2)		self.assertEqual(len(ret), 2)
self.assertEqual(		self.assertEqual(
(ret[0][0], ret[0][1].tobytes(), ret[0][2].tobytes(),		(ret[0][0], ret[0][1].tobytes(), ret[0][2].tobytes(),
ret[0][3], ret[0][4]),		ret[0][3], ret[0][4]),
(occur['origin'], occur['branch'], occur['target'],		(occur['origin'], occur['branch'], occur['target'],
occur['target_type'], orig_date))		occur['target_type'], date_visit1))
self.assertEqual(		self.assertEqual(
(ret[1][0], ret[1][1].tobytes(), ret[1][2].tobytes(),		(ret[1][0], ret[1][1].tobytes(), ret[1][2].tobytes(),
ret[1][3], ret[1][4]),		ret[1][3], ret[1][4]),
(occur['origin'], occur['branch'], occur['target'],		(occur2['origin'], occur2['branch'], occur2['target'],
occur['target_type'], occur['date']))		occur2['target_type'], date_visit2))

@istest		@istest
def occurrence_get(self):		def occurrence_get(self):
# given		# given
		occur = self.occurrence.copy()
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)
		origin_visit1 = self.storage.origin_visit_add(origin_id,
		self.date_visit1)

revision = self.revision.copy()		revision = self.revision.copy()
revision['id'] = self.occurrence['target']		revision['id'] = occur['target']
self.storage.revision_add([revision])		self.storage.revision_add([revision])

occur = self.occurrence		occur.update({
occur['origin'] = origin_id		'origin': origin_id,
		'visit': origin_visit1['visit'],
		})
self.storage.occurrence_add([occur])		self.storage.occurrence_add([occur])
self.storage.occurrence_add([occur])		self.storage.occurrence_add([occur])

# when		# when
actual_occurrence = list(self.storage.occurrence_get(origin_id))		actual_occurrence = list(self.storage.occurrence_get(origin_id))

# then		# then
expected_occur = occur.copy()		expected_occurrence = self.occurrence.copy()
del expected_occur['date']		expected_occurrence.update({
		'origin': origin_id
		})
self.assertEquals(len(actual_occurrence), 1)		self.assertEquals(len(actual_occurrence), 1)
self.assertEquals(actual_occurrence[0], expected_occur)		self.assertEquals(actual_occurrence[0], expected_occurrence)

@istest		@istest
def content_find_occurrence_with_present_content(self):		def content_find_occurrence_with_present_content(self):
# 1. with something to find		# 1. with something to find
# given		# given
self.storage.content_add([self.cont2])		self.storage.content_add([self.cont2])
self.storage.directory_add([self.dir2]) # point to self.cont		self.storage.directory_add([self.dir2]) # point to self.cont
self.storage.revision_add([self.revision2]) # points to self.dir		self.storage.revision_add([self.revision2]) # points to self.dir
origin_id = self.storage.origin_add_one(self.origin2)		origin_id = self.storage.origin_add_one(self.origin2)
occurrence = self.occurrence2
occurrence.update({'origin': origin_id})		occurrence = self.occurrence2.copy()
		origin_visit1 = self.storage.origin_visit_add(origin_id,
		self.date_visit2)
		occurrence.update({
		'origin': origin_id,
		'visit': origin_visit1['visit'],
		})

self.storage.occurrence_add([occurrence])		self.storage.occurrence_add([occurrence])

# when		# when
occ = self.storage.content_find_occurrence(		occ = self.storage.content_find_occurrence(
{'sha1': self.cont2['sha1']})		{'sha1': self.cont2['sha1']})

# then		# then
self.assertEquals(occ['origin_type'], self.origin2['type'])		self.assertEquals(occ['origin_type'], self.origin2['type'])
▲ Show 20 Lines • Show All 358 Lines • Show Last 20 Lines