Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/utils.py
Show All 30 Lines | INDEXER_CFG_KEY: { | ||||
'cls': 'memory', | 'cls': 'memory', | ||||
'args': { | 'args': { | ||||
}, | }, | ||||
}, | }, | ||||
} | } | ||||
ORIGINS = [ | ORIGINS = [ | ||||
{ | { | ||||
'id': 52189575, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'git', | 'type': 'git', | ||||
'url': 'https://github.com/SoftwareHeritage/swh-storage'}, | 'url': 'https://github.com/SoftwareHeritage/swh-storage'}, | ||||
{ | { | ||||
'id': 4423668, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'ftp', | 'type': 'ftp', | ||||
'url': 'rsync://ftp.gnu.org/gnu/3dldf'}, | 'url': 'rsync://ftp.gnu.org/gnu/3dldf'}, | ||||
{ | { | ||||
'id': 77775770, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'deposit', | 'type': 'deposit', | ||||
'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'}, | 'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'}, | ||||
{ | { | ||||
'id': 85072327, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'pypi', | 'type': 'pypi', | ||||
'url': 'https://pypi.org/project/limnoria/'}, | 'url': 'https://pypi.org/project/limnoria/'}, | ||||
{ | { | ||||
'id': 49908349, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'svn', | 'type': 'svn', | ||||
'url': 'http://0-512-md.googlecode.com/svn/'}, | 'url': 'http://0-512-md.googlecode.com/svn/'}, | ||||
{ | { | ||||
'id': 54974445, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'git', | 'type': 'git', | ||||
'url': 'https://github.com/librariesio/yarn-parser'}, | 'url': 'https://github.com/librariesio/yarn-parser'}, | ||||
{ | { | ||||
'id': 54974446, | |||||
'lister': None, | 'lister': None, | ||||
'project': None, | 'project': None, | ||||
'type': 'git', | 'type': 'git', | ||||
'url': 'https://github.com/librariesio/yarn-parser.git'}, | 'url': 'https://github.com/librariesio/yarn-parser.git'}, | ||||
] | ] | ||||
SNAPSHOTS = { | SNAPSHOTS = [ | ||||
52189575: { | { | ||||
'origin': 'https://github.com/SoftwareHeritage/swh-storage', | |||||
'branches': { | 'branches': { | ||||
b'refs/heads/add-revision-origin-cache': { | b'refs/heads/add-revision-origin-cache': { | ||||
'target': b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0' | 'target': b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0' | ||||
b's\xe7/\xe9l\x1e', | b's\xe7/\xe9l\x1e', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'HEAD': { | b'HEAD': { | ||||
'target': b'8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}' | 'target': b'8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}' | ||||
b'\xac\xefrm', | b'\xac\xefrm', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'refs/tags/v0.0.103': { | b'refs/tags/v0.0.103': { | ||||
'target': b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+' | 'target': b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+' | ||||
b'\x0f\xdd', | b'\x0f\xdd', | ||||
'target_type': 'release'}, | 'target_type': 'release'}, | ||||
}}, | }}, | ||||
4423668: { | { | ||||
'origin': 'rsync://ftp.gnu.org/gnu/3dldf', | |||||
'branches': { | 'branches': { | ||||
b'3DLDF-1.1.4.tar.gz': { | b'3DLDF-1.1.4.tar.gz': { | ||||
'target': b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90|\xd3\xfc' | 'target': b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90|\xd3\xfc' | ||||
b'"G\x99\x11', | b'"G\x99\x11', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'3DLDF-2.0.2.tar.gz': { | b'3DLDF-2.0.2.tar.gz': { | ||||
'target': b'\xb6\x0e\xe7\x9e9\xac\xaa\x19\x9e=' | 'target': b'\xb6\x0e\xe7\x9e9\xac\xaa\x19\x9e=' | ||||
b'\xd1\xc5\x00\\\xc6\xfc\xe0\xa6\xb4V', | b'\xd1\xc5\x00\\\xc6\xfc\xe0\xa6\xb4V', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'3DLDF-2.0.3-examples.tar.gz': { | b'3DLDF-2.0.3-examples.tar.gz': { | ||||
'target': b'!H\x19\xc0\xee\x82-\x12F1\xbd\x97' | 'target': b'!H\x19\xc0\xee\x82-\x12F1\xbd\x97' | ||||
b'\xfe\xadZ\x80\x80\xc1\x83\xff', | b'\xfe\xadZ\x80\x80\xc1\x83\xff', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'3DLDF-2.0.3.tar.gz': { | b'3DLDF-2.0.3.tar.gz': { | ||||
'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee' | 'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee' | ||||
b'\xcc\x1a\xb4`\x8c\x8by', | b'\xcc\x1a\xb4`\x8c\x8by', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'3DLDF-2.0.tar.gz': { | b'3DLDF-2.0.tar.gz': { | ||||
'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G' | 'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G' | ||||
b'\xd3\xd1m', | b'\xd3\xd1m', | ||||
b'target_type': 'revision'} | b'target_type': 'revision'} | ||||
}}, | }}, | ||||
77775770: { | { | ||||
'origin': 'https://forge.softwareheritage.org/source/jesuisgpl/', | |||||
'branches': { | 'branches': { | ||||
b'master': { | b'master': { | ||||
'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{' | 'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{' | ||||
b'\xa6\xe9\x99\xb1\x9e]q\xeb', | b'\xa6\xe9\x99\xb1\x9e]q\xeb', | ||||
'target_type': 'revision'} | 'target_type': 'revision'} | ||||
}, | }, | ||||
'id': b"h\xc0\xd2a\x04\xd4~'\x8d\xd6\xbe\x07\xeda\xfa\xfbV" | 'id': b"h\xc0\xd2a\x04\xd4~'\x8d\xd6\xbe\x07\xeda\xfa\xfbV" | ||||
b"\x1d\r "}, | b"\x1d\r "}, | ||||
85072327: { | { | ||||
'origin': 'https://pypi.org/project/limnoria/', | |||||
'branches': { | 'branches': { | ||||
b'HEAD': { | b'HEAD': { | ||||
'target': b'releases/2018.09.09', | 'target': b'releases/2018.09.09', | ||||
'target_type': 'alias'}, | 'target_type': 'alias'}, | ||||
b'releases/2018.09.01': { | b'releases/2018.09.01': { | ||||
'target': b'<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d' | 'target': b'<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d' | ||||
b'\xbb\xdfF\xfdw\xcf', | b'\xbb\xdfF\xfdw\xcf', | ||||
'target_type': 'revision'}, | 'target_type': 'revision'}, | ||||
b'releases/2018.09.09': { | b'releases/2018.09.09': { | ||||
'target': b'\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k' | 'target': b'\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k' | ||||
b'A\x10\x9d\xc5\xfa2\xf8t', | b'A\x10\x9d\xc5\xfa2\xf8t', | ||||
'target_type': 'revision'}}, | 'target_type': 'revision'}}, | ||||
'id': b'{\xda\x8e\x84\x7fX\xff\x92\x80^\x93V\x18\xa3\xfay' | 'id': b'{\xda\x8e\x84\x7fX\xff\x92\x80^\x93V\x18\xa3\xfay' | ||||
b'\x12\x9e\xd6\xb3'}, | b'\x12\x9e\xd6\xb3'}, | ||||
49908349: { | { | ||||
'origin': 'http://0-512-md.googlecode.com/svn/', | |||||
'branches': { | 'branches': { | ||||
b'master': { | b'master': { | ||||
'target': b'\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8' | 'target': b'\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8' | ||||
b'\xc9\xad#.\x1bw=\x18', | b'\xc9\xad#.\x1bw=\x18', | ||||
'target_type': 'revision'}}, | 'target_type': 'revision'}}, | ||||
'id': b'\xa1\xa2\x8c\n\xb3\x87\xa8\xf9\xe0a\x8c\xb7' | 'id': b'\xa1\xa2\x8c\n\xb3\x87\xa8\xf9\xe0a\x8c\xb7' | ||||
b'\x05\xea\xb8\x1f\xc4H\xf4s'}, | b'\x05\xea\xb8\x1f\xc4H\xf4s'}, | ||||
54974445: { | { | ||||
'origin': 'https://github.com/librariesio/yarn-parser', | |||||
'branches': { | 'branches': { | ||||
b'HEAD': { | b'HEAD': { | ||||
'target': hash_to_bytes( | 'target': hash_to_bytes( | ||||
'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), | '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), | ||||
'target_type': 'revision'}}}, | 'target_type': 'revision'}}}, | ||||
54974446: { | { | ||||
'origin': 'https://github.com/librariesio/yarn-parser.git', | |||||
'branches': { | 'branches': { | ||||
b'HEAD': { | b'HEAD': { | ||||
'target': hash_to_bytes( | 'target': hash_to_bytes( | ||||
'8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), | '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), | ||||
'target_type': 'revision'}}}, | 'target_type': 'revision'}}}, | ||||
} | ] | ||||
REVISIONS = [{ | REVISIONS = [{ | ||||
'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), | 'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), | ||||
'author': { | 'author': { | ||||
'id': 26, | 'id': 26, | ||||
'name': b'Andrew Nesbitt', | 'name': b'Andrew Nesbitt', | ||||
'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>', | 'fullname': b'Andrew Nesbitt <andrewnez@gmail.com>', | ||||
▲ Show 20 Lines • Show All 372 Lines • ▼ Show 20 Lines | |||||
def fill_obj_storage(obj_storage): | def fill_obj_storage(obj_storage): | ||||
"""Add some content in an object storage.""" | """Add some content in an object storage.""" | ||||
for (obj_id, content) in OBJ_STORAGE_DATA.items(): | for (obj_id, content) in OBJ_STORAGE_DATA.items(): | ||||
obj_storage.add(content, obj_id=hash_to_bytes(obj_id)) | obj_storage.add(content, obj_id=hash_to_bytes(obj_id)) | ||||
def fill_storage(storage): | def fill_storage(storage): | ||||
for origin in ORIGINS: | for origin in ORIGINS: | ||||
origin = origin.copy() | |||||
del origin['id'] | |||||
storage.origin_add_one(origin) | storage.origin_add_one(origin) | ||||
for (orig_pseudo_id, snap) in SNAPSHOTS.items(): | for snap in SNAPSHOTS: | ||||
for orig in ORIGINS: | origin_url = snap['origin'] | ||||
if orig_pseudo_id == orig['id']: | visit = storage.origin_visit_add(origin_url, datetime.datetime.now()) | ||||
origin_id = storage.origin_get( | |||||
{'type': orig['type'], 'url': orig['url']})['id'] | |||||
break | |||||
else: | |||||
assert False | |||||
visit = storage.origin_visit_add(origin_id, datetime.datetime.now()) | |||||
snap_id = snap.get('id') or \ | snap_id = snap.get('id') or \ | ||||
bytes([random.randint(0, 255) for _ in range(32)]) | bytes([random.randint(0, 255) for _ in range(32)]) | ||||
storage.snapshot_add(origin_id, visit['visit'], { | storage.snapshot_add([{ | ||||
'id': snap_id, | 'id': snap_id, | ||||
'branches': snap['branches'] | 'branches': snap['branches'] | ||||
}) | }]) | ||||
storage.origin_visit_update( | |||||
origin_url, visit['visit'], status='full', snapshot=snap_id) | |||||
storage.revision_add(REVISIONS) | storage.revision_add(REVISIONS) | ||||
storage.directory_add([{ | storage.directory_add([{ | ||||
'id': DIRECTORY_ID, | 'id': DIRECTORY_ID, | ||||
'entries': DIRECTORY, | 'entries': DIRECTORY, | ||||
}]) | }]) | ||||
for (obj_id, content) in OBJ_STORAGE_DATA.items(): | for (obj_id, content) in OBJ_STORAGE_DATA.items(): | ||||
content_hashes = hashutil.MultiHash.from_data(content).digest() | content_hashes = hashutil.MultiHash.from_data(content).digest() | ||||
storage.content_add([{ | storage.content_add([{ | ||||
▲ Show 20 Lines • Show All 188 Lines • Show Last 20 Lines |