Differential D7941 Diff 28669 swh/indexer/tests/utils.py

Changeset View

Standalone View

swh/indexer/tests/utils.py

Show All 13 Lines
from swh.core.api.classes import stream_results		from swh.core.api.classes import stream_results
from swh.indexer.storage import INDEXER_CFG_KEY		from swh.indexer.storage import INDEXER_CFG_KEY
from swh.model import hashutil		from swh.model import hashutil
from swh.model.hashutil import hash_to_bytes		from swh.model.hashutil import hash_to_bytes
from swh.model.model import (		from swh.model.model import (
Content,		Content,
Directory,		Directory,
DirectoryEntry,		DirectoryEntry,
		ObjectType,
Origin,		Origin,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
Person,		Person,
		Release,
Revision,		Revision,
RevisionType,		RevisionType,
Snapshot,		Snapshot,
SnapshotBranch,		SnapshotBranch,
TargetType,		TargetType,
TimestampWithTimezone,		TimestampWithTimezone,
)		)
from swh.storage.utils import now		from swh.storage.utils import now

BASE_TEST_CONFIG: Dict[str, Dict[str, Any]] = {		BASE_TEST_CONFIG: Dict[str, Dict[str, Any]] = {
"storage": {"cls": "memory"},		"storage": {"cls": "memory"},
"objstorage": {"cls": "memory"},		"objstorage": {"cls": "memory"},
INDEXER_CFG_KEY: {"cls": "memory"},		INDEXER_CFG_KEY: {"cls": "memory"},
}		}


ORIGIN_VISITS = [		ORIGIN_VISITS = [
{"type": "git", "origin": "https://github.com/SoftwareHeritage/swh-storage"},		{"type": "git", "origin": "https://github.com/SoftwareHeritage/swh-storage"},
{"type": "ftp", "origin": "rsync://ftp.gnu.org/gnu/3dldf"},		{"type": "ftp", "origin": "rsync://ftp.gnu.org/gnu/3dldf"},
{		{
"type": "deposit",		"type": "deposit",
"origin": "https://forge.softwareheritage.org/source/jesuisgpl/",		"origin": "https://forge.softwareheritage.org/source/jesuisgpl/",
},		},
{"type": "pypi", "origin": "https://pypi.org/project/limnoria/"},		{
		"type": "pypi",
		"origin": "https://old-pypi.example.org/project/limnoria/",
		}, # with rev head
		{"type": "pypi", "origin": "https://pypi.org/project/limnoria/"}, # with rel head
{"type": "svn", "origin": "http://0-512-md.googlecode.com/svn/"},		{"type": "svn", "origin": "http://0-512-md.googlecode.com/svn/"},
{"type": "git", "origin": "https://github.com/librariesio/yarn-parser"},		{"type": "git", "origin": "https://github.com/librariesio/yarn-parser"},
{"type": "git", "origin": "https://github.com/librariesio/yarn-parser.git"},		{"type": "git", "origin": "https://github.com/librariesio/yarn-parser.git"},
		{"type": "git", "origin": "https://npm.example.org/yarn-parser"},
]		]

ORIGINS = [Origin(url=visit["origin"]) for visit in ORIGIN_VISITS]		ORIGINS = [Origin(url=visit["origin"]) for visit in ORIGIN_VISITS]


DIRECTORY = Directory(		DIRECTORY = Directory(
id=hash_to_bytes("34f335a750111ca0a8b64d8034faec9eedc396be"),		id=hash_to_bytes("34f335a750111ca0a8b64d8034faec9eedc396be"),
entries=(		entries=(
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines	date=TimestampWithTimezone.from_datetime(
datetime.datetime(2017, 2, 20, 16, 14, 16, tzinfo=_utc_plus_2)		datetime.datetime(2017, 2, 20, 16, 14, 16, tzinfo=_utc_plus_2)
),		),
directory=DIRECTORY2.id,		directory=DIRECTORY2.id,
parents=(),		parents=(),
)		)

REVISIONS = [REVISION]		REVISIONS = [REVISION]

		RELEASE = Release(
		name=b"v0.0.0",
		message=None,
		author=Person(
		name=b"Andrew Nesbitt",
		fullname=b"Andrew Nesbitt <andrewnez@gmail.com>",
		email=b"andrewnez@gmail.com",
		),
		synthetic=False,
		date=TimestampWithTimezone.from_datetime(
		datetime.datetime(2017, 2, 20, 16, 14, 16, tzinfo=_utc_plus_2)
		),
		target_type=ObjectType.DIRECTORY,
		target=DIRECTORY2.id,
		)

		RELEASES = [RELEASE]

SNAPSHOTS = [		SNAPSHOTS = [
		# https://github.com/SoftwareHeritage/swh-storage
Snapshot(		Snapshot(
id=hash_to_bytes("a50fde72265343b7d28cecf6db20d98a81d21965"),		id=hash_to_bytes("a50fde72265343b7d28cecf6db20d98a81d21965"),
branches={		branches={
b"refs/heads/add-revision-origin-cache": SnapshotBranch(		b"refs/heads/add-revision-origin-cache": SnapshotBranch(
target=b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0s\xe7/\xe9l\x1e',		target=b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0s\xe7/\xe9l\x1e',
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
b"refs/head/master": SnapshotBranch(		b"refs/head/master": SnapshotBranch(
target=b"8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}\xac\xefrm",		target=b"8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}\xac\xefrm",
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
b"HEAD": SnapshotBranch(		b"HEAD": SnapshotBranch(
target=b"refs/head/master", target_type=TargetType.ALIAS		target=b"refs/head/master", target_type=TargetType.ALIAS
),		),
b"refs/tags/v0.0.103": SnapshotBranch(		b"refs/tags/v0.0.103": SnapshotBranch(
target=b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+\x0f\xdd',		target=b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+\x0f\xdd',
target_type=TargetType.RELEASE,		target_type=TargetType.RELEASE,
),		),
},		},
),		),
		# rsync://ftp.gnu.org/gnu/3dldf
Snapshot(		Snapshot(
id=hash_to_bytes("2c67f69a416bca4e1f3fcd848c588fab88ad0642"),		id=hash_to_bytes("2c67f69a416bca4e1f3fcd848c588fab88ad0642"),
branches={		branches={
b"3DLDF-1.1.4.tar.gz": SnapshotBranch(		b"3DLDF-1.1.4.tar.gz": SnapshotBranch(
target=b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90\|\xd3\xfc"G\x99\x11',		target=b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90\|\xd3\xfc"G\x99\x11',
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
b"3DLDF-2.0.2.tar.gz": SnapshotBranch(		b"3DLDF-2.0.2.tar.gz": SnapshotBranch(
Show All 9 Lines	Snapshot(
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
b"3DLDF-2.0.tar.gz": SnapshotBranch(		b"3DLDF-2.0.tar.gz": SnapshotBranch(
target=b"F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G\xd3\xd1m",		target=b"F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G\xd3\xd1m",
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
},		},
),		),
		# https://forge.softwareheritage.org/source/jesuisgpl/",
Snapshot(		Snapshot(
id=hash_to_bytes("68c0d26104d47e278dd6be07ed61fafb561d0d20"),		id=hash_to_bytes("68c0d26104d47e278dd6be07ed61fafb561d0d20"),
branches={		branches={
b"master": SnapshotBranch(		b"master": SnapshotBranch(
target=b"\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{\xa6\xe9\x99\xb1\x9e]q\xeb", # noqa		target=b"\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{\xa6\xe9\x99\xb1\x9e]q\xeb", # noqa
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
)		)
},		},
),		),
		# https://old-pypi.example.org/project/limnoria/
Snapshot(		Snapshot(
id=hash_to_bytes("f255245269e15fc99d284affd79f766668de0b67"),		id=hash_to_bytes("f255245269e15fc99d284affd79f766668de0b67"),
branches={		branches={
b"HEAD": SnapshotBranch(		b"HEAD": SnapshotBranch(
target=b"releases/2018.09.09", target_type=TargetType.ALIAS		target=b"releases/2018.09.09", target_type=TargetType.ALIAS
),		),
b"releases/2018.09.01": SnapshotBranch(		b"releases/2018.09.01": SnapshotBranch(
target=b"<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d\xbb\xdfF\xfdw\xcf",		target=b"<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d\xbb\xdfF\xfdw\xcf",
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
b"releases/2018.09.09": SnapshotBranch(		b"releases/2018.09.09": SnapshotBranch(
target=b"\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8kA\x10\x9d\xc5\xfa2\xf8t", # noqa		target=b"\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8kA\x10\x9d\xc5\xfa2\xf8t", # noqa
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
),		),
},		},
),		),
		# https://pypi.org/project/limnoria/
		Snapshot(
		branches={
		b"HEAD": SnapshotBranch(
		target=b"releases/2018.09.09", target_type=TargetType.ALIAS
		),
		b"releases/2018.09.01": SnapshotBranch(
		target=b"<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d\xbb\xdfF\xfdw\xcf",
		target_type=TargetType.RELEASE,
		),
		b"releases/2018.09.09": SnapshotBranch(
		target=b"\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8kA\x10\x9d\xc5\xfa2\xf8t", # noqa
		target_type=TargetType.RELEASE,
		),
		},
		),
		# http://0-512-md.googlecode.com/svn/
Snapshot(		Snapshot(
id=hash_to_bytes("a1a28c0ab387a8f9e0618cb705eab81fc448f473"),		id=hash_to_bytes("a1a28c0ab387a8f9e0618cb705eab81fc448f473"),
branches={		branches={
b"master": SnapshotBranch(		b"master": SnapshotBranch(
target=b"\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8\xc9\xad#.\x1bw=\x18",		target=b"\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8\xc9\xad#.\x1bw=\x18",
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
)		)
},		},
),		),
		# https://github.com/librariesio/yarn-parser
Snapshot(		Snapshot(
id=hash_to_bytes("bb4fd3a836930ce629d912864319637040ff3040"),		id=hash_to_bytes("bb4fd3a836930ce629d912864319637040ff3040"),
branches={		branches={
b"HEAD": SnapshotBranch(		b"HEAD": SnapshotBranch(
target=REVISION.id,		target=REVISION.id,
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
)		)
},		},
),		),
		# https://github.com/librariesio/yarn-parser.git
Snapshot(		Snapshot(
id=hash_to_bytes("bb4fd3a836930ce629d912864319637040ff3040"),		id=hash_to_bytes("bb4fd3a836930ce629d912864319637040ff3040"),
branches={		branches={
b"HEAD": SnapshotBranch(		b"HEAD": SnapshotBranch(
target=REVISION.id,		target=REVISION.id,
target_type=TargetType.REVISION,		target_type=TargetType.REVISION,
)		)
},		},
),		),
		# https://npm.example.org/yarn-parser
		Snapshot(
		branches={
		b"HEAD": SnapshotBranch(
		target=RELEASE.id,
		target_type=TargetType.RELEASE,
		)
		},
		),
]		]

		assert len(SNAPSHOTS) == len(ORIGIN_VISITS)


SHA1_TO_LICENSES = {		SHA1_TO_LICENSES = {
"01c9379dfc33803963d07c1ccc748d3fe4c96bb5": ["GPL"],		"01c9379dfc33803963d07c1ccc748d3fe4c96bb5": ["GPL"],
"02fb2c89e14f7fab46701478c83779c7beb7b069": ["Apache2.0"],		"02fb2c89e14f7fab46701478c83779c7beb7b069": ["Apache2.0"],
"103bc087db1d26afc3a0283f38663d081e9b01e6": ["MIT"],		"103bc087db1d26afc3a0283f38663d081e9b01e6": ["MIT"],
"688a5ef812c53907562fe379d4b3851e69c7cb15": ["AGPL"],		"688a5ef812c53907562fe379d4b3851e69c7cb15": ["AGPL"],
"da39a3ee5e6b4b0d3255bfef95601890afd80709": [],		"da39a3ee5e6b4b0d3255bfef95601890afd80709": [],
}		}
▲ Show 20 Lines • Show All 326 Lines • ▼ Show 20 Lines	def fill_obj_storage(obj_storage):
for (obj_id, content) in OBJ_STORAGE_DATA.items():		for (obj_id, content) in OBJ_STORAGE_DATA.items():
obj_storage.add(content, obj_id=hash_to_bytes(obj_id))		obj_storage.add(content, obj_id=hash_to_bytes(obj_id))


def fill_storage(storage):		def fill_storage(storage):
storage.origin_add(ORIGINS)		storage.origin_add(ORIGINS)
storage.directory_add([DIRECTORY, DIRECTORY2])		storage.directory_add([DIRECTORY, DIRECTORY2])
storage.revision_add(REVISIONS)		storage.revision_add(REVISIONS)
		storage.release_add(RELEASES)
storage.snapshot_add(SNAPSHOTS)		storage.snapshot_add(SNAPSHOTS)

for visit, snapshot in zip(ORIGIN_VISITS, SNAPSHOTS):		for visit, snapshot in zip(ORIGIN_VISITS, SNAPSHOTS):
assert snapshot.id is not None		assert snapshot.id is not None

visit = storage.origin_visit_add(		visit = storage.origin_visit_add(
[OriginVisit(origin=visit["origin"], date=now(), type=visit["type"])]		[OriginVisit(origin=visit["origin"], date=now(), type=visit["type"])]
)[0]		)[0]
▲ Show 20 Lines • Show All 151 Lines • Show Last 20 Lines