Differential D1958 Diff 6591 swh/indexer/tests/utils.py

Changeset View

Standalone View

swh/indexer/tests/utils.py

Show All 30 Lines	INDEXER_CFG_KEY: {
'cls': 'memory',		'cls': 'memory',
'args': {		'args': {
},		},
},		},
}		}

ORIGINS = [		ORIGINS = [
{		{
'lister': None,
'project': None,
'type': 'git',		'type': 'git',
'url': 'https://github.com/SoftwareHeritage/swh-storage'},		'url': 'https://github.com/SoftwareHeritage/swh-storage'},
{		{
'lister': None,
'project': None,
'type': 'ftp',		'type': 'ftp',
'url': 'rsync://ftp.gnu.org/gnu/3dldf'},		'url': 'rsync://ftp.gnu.org/gnu/3dldf'},
{		{
'lister': None,
'project': None,
'type': 'deposit',		'type': 'deposit',
'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'},		'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'},
{		{
'lister': None,
'project': None,
'type': 'pypi',		'type': 'pypi',
'url': 'https://pypi.org/project/limnoria/'},		'url': 'https://pypi.org/project/limnoria/'},
{		{
'lister': None,
'project': None,
'type': 'svn',		'type': 'svn',
'url': 'http://0-512-md.googlecode.com/svn/'},		'url': 'http://0-512-md.googlecode.com/svn/'},
{		{
'lister': None,
'project': None,
'type': 'git',		'type': 'git',
'url': 'https://github.com/librariesio/yarn-parser'},		'url': 'https://github.com/librariesio/yarn-parser'},
{		{
'lister': None,
'project': None,
'type': 'git',		'type': 'git',
'url': 'https://github.com/librariesio/yarn-parser.git'},		'url': 'https://github.com/librariesio/yarn-parser.git'},
]		]

SNAPSHOTS = [		SNAPSHOTS = [
{		{
'origin': 'https://github.com/SoftwareHeritage/swh-storage',		'origin': 'https://github.com/SoftwareHeritage/swh-storage',
'branches': {		'branches': {
Show All 27 Lines	SNAPSHOTS = [
'target_type': 'revision'},		'target_type': 'revision'},
b'3DLDF-2.0.3.tar.gz': {		b'3DLDF-2.0.3.tar.gz': {
'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee'		'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee'
b'\xcc\x1a\xb4`\x8c\x8by',		b'\xcc\x1a\xb4`\x8c\x8by',
'target_type': 'revision'},		'target_type': 'revision'},
b'3DLDF-2.0.tar.gz': {		b'3DLDF-2.0.tar.gz': {
'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G'		'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G'
b'\xd3\xd1m',		b'\xd3\xd1m',
b'target_type': 'revision'}		'target_type': 'revision'}
}},		}},
{		{
'origin': 'https://forge.softwareheritage.org/source/jesuisgpl/',		'origin': 'https://forge.softwareheritage.org/source/jesuisgpl/',
'branches': {		'branches': {
b'master': {		b'master': {
'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{'		'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{'
b'\xa6\xe9\x99\xb1\x9e]q\xeb',		b'\xa6\xe9\x99\xb1\x9e]q\xeb',
'target_type': 'revision'}		'target_type': 'revision'}
Show All 39 Lines	SNAPSHOTS = [
'target': hash_to_bytes(		'target': hash_to_bytes(
'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),		'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
'target_type': 'revision'}}},		'target_type': 'revision'}}},
]		]


REVISIONS = [{		REVISIONS = [{
'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),		'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
		'message': 'Improve search functionality',
'author': {		'author': {
'id': 26,
'name': b'Andrew Nesbitt',		'name': b'Andrew Nesbitt',
'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>',		'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>',
'email': b'andrewnez@gmail.com'		'email': b'andrewnez@gmail.com'
},		},
'committer': {		'committer': {
'id': 26,
'name': b'Andrew Nesbitt',		'name': b'Andrew Nesbitt',
'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>',		'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>',
'email': b'andrewnez@gmail.com'		'email': b'andrewnez@gmail.com'
},		},
		'committer_date': {
		'negative_utc': None,
		'offset': 120,
		'timestamp': {'microseconds': 0, 'seconds': 1380883849}
		},
		vlorentzUnsubmitted Done Inline Actions nitpick: use the same formatting as the author date vlorentz: nitpick: use the same formatting as the author date
		'type': 'git',
'synthetic': False,		'synthetic': False,
'date': {		'date': {
'negative_utc': False,		'negative_utc': False,
'timestamp': {		'timestamp': {
'seconds': 1487596456,		'seconds': 1487596456,
'microseconds': 0		'microseconds': 0
},		},
'offset': 0		'offset': 0
},		},
'directory': b'10'		'directory': b'10'
}]		}]

DIRECTORY_ID = b'10'		DIRECTORY_ID = b'10'

DIRECTORY = [{		DIRECTORY = [{
		vlorentzUnsubmitted Done Inline Actions the var name should be `DIRECTORY_ENTRIES`, btw vlorentz: the var name should be `DIRECTORY_ENTRIES`, btw
'sha1_git': b'abc',
'name': b'index.js',		'name': b'index.js',
'target': b'abc',
'length': 897,
'status': 'visible',
'type': 'file',		'type': 'file',
		'target': b'abc',
'perms': 33188,		'perms': 33188,
'sha1': b'bcd'
},		},
{		{
'sha1_git': b'aab',
'name': b'package.json',		'name': b'package.json',
'target': b'aab',
'length': 712,
'status': 'visible',
'type': 'file',		'type': 'file',
		'target': b'aab',
'perms': 33188,		'perms': 33188,
'sha1': b'cde'
},		},
vlorentzUnsubmitted Not Done Inline Actions because of this, directory_ls used to return `{'target': b'aab', .., 'sha1': b'cde'}` (`b'aab'` being a a sha1_git). vlorentz: because of this, directory_ls used to return `{'target': b'aab', .., 'sha1': b'cde'}` (`b'aab'`…
{		{
'target': b'11',
'type': 'dir',
'length': None,
'name': b'.github',		'name': b'.github',
'sha1': None,		'type': 'dir',
		'target': b'11',
'perms': 16384,		'perms': 16384,
'sha1_git': None,
'status': None,
'sha256': None
}		}
]		]

SHA1_TO_LICENSES = {		SHA1_TO_LICENSES = {
'01c9379dfc33803963d07c1ccc748d3fe4c96bb5': ['GPL'],		'01c9379dfc33803963d07c1ccc748d3fe4c96bb5': ['GPL'],
'02fb2c89e14f7fab46701478c83779c7beb7b069': ['Apache2.0'],		'02fb2c89e14f7fab46701478c83779c7beb7b069': ['Apache2.0'],
'103bc087db1d26afc3a0283f38663d081e9b01e6': ['MIT'],		'103bc087db1d26afc3a0283f38663d081e9b01e6': ['MIT'],
'688a5ef812c53907562fe379d4b3851e69c7cb15': ['AGPL'],		'688a5ef812c53907562fe379d4b3851e69c7cb15': ['AGPL'],
▲ Show 20 Lines • Show All 336 Lines • ▼ Show 20 Lines	storage.directory_add([{
'entries': DIRECTORY,		'entries': DIRECTORY,
}])		}])
for (obj_id, content) in OBJ_STORAGE_DATA.items():		for (obj_id, content) in OBJ_STORAGE_DATA.items():
content_hashes = hashutil.MultiHash.from_data(content).digest()		content_hashes = hashutil.MultiHash.from_data(content).digest()
storage.content_add([{		storage.content_add([{
'data': content,		'data': content,
'length': len(content),		'length': len(content),
'status': 'visible',		'status': 'visible',
'sha1': hash_to_bytes(obj_id),		'sha1': hash_to_bytes(obj_id),
'sha1_git': hash_to_bytes(obj_id),		'sha1_git': hash_to_bytes(obj_id),
		vlorentzUnsubmitted Not Done Inline Actions Still not happy about this, but let's fix it later. vlorentz: Still not happy about this, but let's fix it later.
		ardumontAuthorUnsubmitted Done Inline Actions neither am i but yeah, not right now ;) ardumont: neither am i but yeah, not right now ;)
'sha256': content_hashes['sha256'],		'sha256': content_hashes['sha256'],
'blake2s256': content_hashes['blake2s256']		'blake2s256': content_hashes['blake2s256']
}])		}])


class CommonContentIndexerTest(metaclass=abc.ABCMeta):		class CommonContentIndexerTest(metaclass=abc.ABCMeta):
legacy_get_format = False		legacy_get_format = False
"""True if and only if the tested indexer uses the legacy format.		"""True if and only if the tested indexer uses the legacy format.
see: https://forge.softwareheritage.org/T1433		see: https://forge.softwareheritage.org/T1433

"""		"""
def get_indexer_results(self, ids):		def get_indexer_results(self, ids):
"""Override this for indexers that don't have a mock storage."""		"""Override this for indexers that don't have a mock storage."""
return self.indexer.idx_storage.state		return self.indexer.idx_storage.state

def assert_legacy_results_ok(self, sha1s, expected_results=None):		def assert_legacy_results_ok(self, sha1s, expected_results=None):
# XXX old format, remove this when all endpoints are		# XXX old format, remove this when all endpoints are
# updated to the new one		# updated to the new one
# see: https://forge.softwareheritage.org/T1433		# see: https://forge.softwareheritage.org/T1433
		vlorentzUnsubmitted Not Done Inline Actions But, this new code adds `{'target': b'aab', .., 'sha1': b'aab'}`; so the sha1 changed. You should not populate `storage.content_add` from `DIRECTORY_ENTRIES`. Add a new constant (with the same data that was deleted) instead. vlorentz: But, this new code adds `{'target': b'aab', .., 'sha1': b'aab'}`; so the sha1 changed. You…
		ardumontAuthorUnsubmitted Done Inline Actions But, this new code adds {'target': b'aab', .., 'sha1': b'aab'}; so the sha1 changed. yes, i hear you. But, I remember i tried to add a constants in the objstorage (b'aab', b'abc') first but it failed the same. The lookup of the sha1 stopped being b'cde' (within the indexer run) when removing the extra fields in the DIRECTORY_ENTRIES (necessary to pass the validation). Looks like as you said some misbehavior in the in-memory storage somehow... ardumont: > But, this new code adds {'target': b'aab', .., 'sha1': b'aab'}; so the sha1 changed. yes, i…
		ardumontAuthorUnsubmitted Done Inline Actions The actual fix is to target b'cde' for that particular entry... (then i don't have to fix any other failing tests) ardumont: The actual fix is to target b'cde' for that particular entry... (then i don't have to fix any…
		vlorentzUnsubmitted Not Done Inline Actions No. The actual fix is to use `storage.content_add([{'target': b'aab', .., 'sha1': b'cde'}])` vlorentz: No. The actual fix is to use `storage.content_add([{'target': b'aab', .., 'sha1': b'cde'}])`
		ardumontAuthorUnsubmitted Done Inline Actions Well, i mean the real fix would be to untangle this, this is not maintainable as is. It's not clear where what is coming from where. As a first approximation though, just changing the less cogs possible (only the requisite from the validation introduction) to have the tests green as before is the way. Then improving. And No. The actual fix is to use storage.content_add([{'target': b'aab', .., 'sha1': b'cde'}]) Even though correct, that does not add up to clarity... ardumont: Well, i mean the real fix would be to untangle this, this is not maintainable as is. It's not…
sha1s = [sha1 if isinstance(sha1, bytes) else hash_to_bytes(sha1)		sha1s = [sha1 if isinstance(sha1, bytes) else hash_to_bytes(sha1)
for sha1 in sha1s]		for sha1 in sha1s]
actual_results = list(self.get_indexer_results(sha1s))		actual_results = list(self.get_indexer_results(sha1s))

if expected_results is None:		if expected_results is None:
expected_results = self.expected_results		expected_results = self.expected_results

self.assertEqual(len(expected_results), len(actual_results),		self.assertEqual(len(expected_results), len(actual_results),
▲ Show 20 Lines • Show All 156 Lines • Show Last 20 Lines