Page MenuHomeSoftware Heritage
Differential D1564 Diff 5322 swh/indexer/tests/utils.py

Changeset View

Standalone View

View Options

swh/indexer/tests/utils.py

Show All 30 Lines INDEXER_CFG_KEY: {
'cls': 'memory', 'cls': 'memory',
'args': { 'args': {
}, },
}, },
} }
ORIGINS = [ ORIGINS = [
{ {
'id': 52189575,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'git', 'type': 'git',
'url': 'https://github.com/SoftwareHeritage/swh-storage'}, 'url': 'https://github.com/SoftwareHeritage/swh-storage'},
{ {
'id': 4423668,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'ftp', 'type': 'ftp',
'url': 'rsync://ftp.gnu.org/gnu/3dldf'}, 'url': 'rsync://ftp.gnu.org/gnu/3dldf'},
{ {
'id': 77775770,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'deposit', 'type': 'deposit',
'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'}, 'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'},
{ {
'id': 85072327,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'pypi', 'type': 'pypi',
'url': 'https://pypi.org/project/limnoria/'}, 'url': 'https://pypi.org/project/limnoria/'},
{ {
'id': 49908349,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'svn', 'type': 'svn',
'url': 'http://0-512-md.googlecode.com/svn/'}, 'url': 'http://0-512-md.googlecode.com/svn/'},
{ {
'id': 54974445,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'git', 'type': 'git',
'url': 'https://github.com/librariesio/yarn-parser'}, 'url': 'https://github.com/librariesio/yarn-parser'},
{ {
'id': 54974446,
'lister': None, 'lister': None,
'project': None, 'project': None,
'type': 'git', 'type': 'git',
'url': 'https://github.com/librariesio/yarn-parser.git'}, 'url': 'https://github.com/librariesio/yarn-parser.git'},
] ]
SNAPSHOTS = { SNAPSHOTS = [
52189575: { {
'origin': 'https://github.com/SoftwareHeritage/swh-storage',
'branches': { 'branches': {
b'refs/heads/add-revision-origin-cache': { b'refs/heads/add-revision-origin-cache': {
'target': b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0' 'target': b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0'
b's\xe7/\xe9l\x1e', b's\xe7/\xe9l\x1e',
'target_type': 'revision'}, 'target_type': 'revision'},
b'HEAD': { b'HEAD': {
'target': b'8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}' 'target': b'8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}'
b'\xac\xefrm', b'\xac\xefrm',
'target_type': 'revision'}, 'target_type': 'revision'},
b'refs/tags/v0.0.103': { b'refs/tags/v0.0.103': {
'target': b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+' 'target': b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+'
b'\x0f\xdd', b'\x0f\xdd',
'target_type': 'release'}, 'target_type': 'release'},
}}, }},
4423668: { {
'origin': 'rsync://ftp.gnu.org/gnu/3dldf',
'branches': { 'branches': {
b'3DLDF-1.1.4.tar.gz': { b'3DLDF-1.1.4.tar.gz': {
'target': b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90|\xd3\xfc' 'target': b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90|\xd3\xfc'
b'"G\x99\x11', b'"G\x99\x11',
'target_type': 'revision'}, 'target_type': 'revision'},
b'3DLDF-2.0.2.tar.gz': { b'3DLDF-2.0.2.tar.gz': {
'target': b'\xb6\x0e\xe7\x9e9\xac\xaa\x19\x9e=' 'target': b'\xb6\x0e\xe7\x9e9\xac\xaa\x19\x9e='
b'\xd1\xc5\x00\\\xc6\xfc\xe0\xa6\xb4V', b'\xd1\xc5\x00\\\xc6\xfc\xe0\xa6\xb4V',
'target_type': 'revision'}, 'target_type': 'revision'},
b'3DLDF-2.0.3-examples.tar.gz': { b'3DLDF-2.0.3-examples.tar.gz': {
'target': b'!H\x19\xc0\xee\x82-\x12F1\xbd\x97' 'target': b'!H\x19\xc0\xee\x82-\x12F1\xbd\x97'
b'\xfe\xadZ\x80\x80\xc1\x83\xff', b'\xfe\xadZ\x80\x80\xc1\x83\xff',
'target_type': 'revision'}, 'target_type': 'revision'},
b'3DLDF-2.0.3.tar.gz': { b'3DLDF-2.0.3.tar.gz': {
'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee' 'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee'
b'\xcc\x1a\xb4`\x8c\x8by', b'\xcc\x1a\xb4`\x8c\x8by',
'target_type': 'revision'}, 'target_type': 'revision'},
b'3DLDF-2.0.tar.gz': { b'3DLDF-2.0.tar.gz': {
'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G' 'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G'
b'\xd3\xd1m', b'\xd3\xd1m',
b'target_type': 'revision'} b'target_type': 'revision'}
}}, }},
77775770: { {
'origin': 'https://forge.softwareheritage.org/source/jesuisgpl/',
'branches': { 'branches': {
b'master': { b'master': {
'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{' 'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{'
b'\xa6\xe9\x99\xb1\x9e]q\xeb', b'\xa6\xe9\x99\xb1\x9e]q\xeb',
'target_type': 'revision'} 'target_type': 'revision'}
}, },
'id': b"h\xc0\xd2a\x04\xd4~'\x8d\xd6\xbe\x07\xeda\xfa\xfbV" 'id': b"h\xc0\xd2a\x04\xd4~'\x8d\xd6\xbe\x07\xeda\xfa\xfbV"
b"\x1d\r "}, b"\x1d\r "},
85072327: { {
'origin': 'https://pypi.org/project/limnoria/',
'branches': { 'branches': {
b'HEAD': { b'HEAD': {
'target': b'releases/2018.09.09', 'target': b'releases/2018.09.09',
'target_type': 'alias'}, 'target_type': 'alias'},
b'releases/2018.09.01': { b'releases/2018.09.01': {
'target': b'<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d' 'target': b'<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d'
b'\xbb\xdfF\xfdw\xcf', b'\xbb\xdfF\xfdw\xcf',
'target_type': 'revision'}, 'target_type': 'revision'},
b'releases/2018.09.09': { b'releases/2018.09.09': {
'target': b'\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k' 'target': b'\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k'
b'A\x10\x9d\xc5\xfa2\xf8t', b'A\x10\x9d\xc5\xfa2\xf8t',
'target_type': 'revision'}}, 'target_type': 'revision'}},
'id': b'{\xda\x8e\x84\x7fX\xff\x92\x80^\x93V\x18\xa3\xfay' 'id': b'{\xda\x8e\x84\x7fX\xff\x92\x80^\x93V\x18\xa3\xfay'
b'\x12\x9e\xd6\xb3'}, b'\x12\x9e\xd6\xb3'},
49908349: { {
'origin': 'http://0-512-md.googlecode.com/svn/',
'branches': { 'branches': {
b'master': { b'master': {
'target': b'\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8' 'target': b'\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8'
b'\xc9\xad#.\x1bw=\x18', b'\xc9\xad#.\x1bw=\x18',
'target_type': 'revision'}}, 'target_type': 'revision'}},
'id': b'\xa1\xa2\x8c\n\xb3\x87\xa8\xf9\xe0a\x8c\xb7' 'id': b'\xa1\xa2\x8c\n\xb3\x87\xa8\xf9\xe0a\x8c\xb7'
b'\x05\xea\xb8\x1f\xc4H\xf4s'}, b'\x05\xea\xb8\x1f\xc4H\xf4s'},
54974445: { {
'origin': 'https://github.com/librariesio/yarn-parser',
'branches': { 'branches': {
b'HEAD': { b'HEAD': {
'target': hash_to_bytes( 'target': hash_to_bytes(
'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
'target_type': 'revision'}}}, 'target_type': 'revision'}}},
54974446: { {
'origin': 'https://github.com/librariesio/yarn-parser.git',
'branches': { 'branches': {
b'HEAD': { b'HEAD': {
'target': hash_to_bytes( 'target': hash_to_bytes(
'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
'target_type': 'revision'}}}, 'target_type': 'revision'}}},
} ]
REVISIONS = [{ REVISIONS = [{
'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), 'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'),
'author': { 'author': {
'id': 26, 'id': 26,
'name': b'Andrew Nesbitt', 'name': b'Andrew Nesbitt',
'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>', 'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>',
▲ Show 20 Lines • Show All 372 Lines • ▼ Show 20 Lines
def fill_obj_storage(obj_storage): def fill_obj_storage(obj_storage):
"""Add some content in an object storage.""" """Add some content in an object storage."""
for (obj_id, content) in OBJ_STORAGE_DATA.items(): for (obj_id, content) in OBJ_STORAGE_DATA.items():
obj_storage.add(content, obj_id=hash_to_bytes(obj_id)) obj_storage.add(content, obj_id=hash_to_bytes(obj_id))
def fill_storage(storage): def fill_storage(storage):
for origin in ORIGINS: for origin in ORIGINS:
origin = origin.copy()
del origin['id']
storage.origin_add_one(origin) storage.origin_add_one(origin)
for (orig_pseudo_id, snap) in SNAPSHOTS.items(): for snap in SNAPSHOTS:
for orig in ORIGINS: origin_url = snap['origin']
if orig_pseudo_id == orig['id']: visit = storage.origin_visit_add(origin_url, datetime.datetime.now())
origin_id = storage.origin_get(
{'type': orig['type'], 'url': orig['url']})['id']
break
else:
assert False
visit = storage.origin_visit_add(origin_id, datetime.datetime.now())
snap_id = snap.get('id') or \ snap_id = snap.get('id') or \
bytes([random.randint(0, 255) for _ in range(32)]) bytes([random.randint(0, 255) for _ in range(32)])
storage.snapshot_add(origin_id, visit['visit'], { storage.snapshot_add([{
'id': snap_id, 'id': snap_id,
'branches': snap['branches'] 'branches': snap['branches']
}) }])
storage.origin_visit_update(
origin_url, visit['visit'], status='full', snapshot=snap_id)
storage.revision_add(REVISIONS) storage.revision_add(REVISIONS)
storage.directory_add([{ storage.directory_add([{
'id': DIRECTORY_ID, 'id': DIRECTORY_ID,
'entries': DIRECTORY, 'entries': DIRECTORY,
}]) }])
for (obj_id, content) in OBJ_STORAGE_DATA.items(): for (obj_id, content) in OBJ_STORAGE_DATA.items():
content_hashes = hashutil.MultiHash.from_data(content).digest() content_hashes = hashutil.MultiHash.from_data(content).digest()
storage.content_add([{ storage.content_add([{
▲ Show 20 Lines • Show All 188 Lines • Show Last 20 Lines
About · Developer information · Legal