Differential D8610 Diff 31104 swh/lister/nixguix/lister.py

Changeset View

Standalone View

swh/lister/nixguix/lister.py

Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines
class ArtifactNatureMistyped(ValueError):		class ArtifactNatureMistyped(ValueError):
"""Raised when a remote artifact's neither a tarball nor a file. It's probably a		"""Raised when a remote artifact's neither a tarball nor a file. It's probably a
misconfiguration in the manifest that badly typed a vcs repository."""		misconfiguration in the manifest that badly typed a vcs repository."""

pass		pass


@dataclass		@dataclass
class OriginUpstream:
"""Upstream origin (e.g. NixOS/nixpkgs, Guix/Guix)."""

origin: str
"""Canonical url of the repository"""
version: int
"""Version of the repository (dismissed?)"""
revision: str
"""Revision of the repository (dismissed?)"""


@dataclass
class Artifact:		class Artifact:
"""Metadata information on Remote Artifact with url (tarball or file)."""		"""Metadata information on Remote Artifact with url (tarball or file)."""

origin: str		origin: str
"""Canonical url retrieve the tarball artifact."""		"""Canonical url retrieve the tarball artifact."""
visit_type: str		visit_type: str
"""Either 'tar' or 'file' """		"""Either 'tar' or 'file' """
fallback_urls: List[str]		fallback_urls: List[str]
Show All 13 Lines	class VCS:
ref: Optional[str] = None		ref: Optional[str] = None
"""Reference either a svn commit id, a git commit, ..."""		"""Reference either a svn commit id, a git commit, ..."""


class ArtifactType(Enum):		class ArtifactType(Enum):
"""The possible artifact types listed out of the manifest."""		"""The possible artifact types listed out of the manifest."""

ARTIFACT = "artifact"		ARTIFACT = "artifact"
ORIGIN = "origin"
VCS = "vcs"		VCS = "vcs"


PageResult = Tuple[ArtifactType, Union[Artifact, VCS, OriginUpstream]]		PageResult = Tuple[ArtifactType, Union[Artifact, VCS]]


VCS_SUPPORTED = ("git", "svn", "hg")		VCS_SUPPORTED = ("git", "svn", "hg")

# Rough approximation of what we can find of mimetypes for tarballs "out there"		# Rough approximation of what we can find of mimetypes for tarballs "out there"
POSSIBLE_TARBALL_MIMETYPES = set(MIMETYPE_TO_ARCHIVE_FORMAT.keys())		POSSIBLE_TARBALL_MIMETYPES = set(MIMETYPE_TO_ARCHIVE_FORMAT.keys())


▲ Show 20 Lines • Show All 153 Lines • ▼ Show 20 Lines	class NixGuixLister(StatelessLister[PageResult]):

def get_pages(self) -> Iterator[PageResult]:		def get_pages(self) -> Iterator[PageResult]:
"""Yield one page per "typed" origin referenced in manifest."""		"""Yield one page per "typed" origin referenced in manifest."""
# fetch and parse the manifest...		# fetch and parse the manifest...
response = self.http_request(self.url)		response = self.http_request(self.url)

# ... if any		# ... if any
raw_data = response.json()		raw_data = response.json()
version = raw_data["version"]		yield ArtifactType.VCS, VCS(origin=self.origin_upstream, type="git")
revision = raw_data["revision"]
yield ArtifactType.ORIGIN, OriginUpstream(
self.origin_upstream,
version,
revision,
)

# grep '"type"' guix-sources.json \| sort \| uniq		# grep '"type"' guix-sources.json \| sort \| uniq
# "type": false <<<<<<<<< noise		# "type": false <<<<<<<<< noise
# "type": "git",		# "type": "git",
# "type": "hg",		# "type": "hg",
# "type": "no-origin", <<<<<<<<< noise		# "type": "no-origin", <<<<<<<<< noise
# "type": "svn",		# "type": "svn",
# "type": "url",		# "type": "url",
Show All 24 Lines	def get_pages(self) -> Iterator[PageResult]:
urls = artifact.get("urls")		urls = artifact.get("urls")
if not urls:		if not urls:
# Nothing to fetch		# Nothing to fetch
logger.warning("Skipping url <%s>: empty artifact", artifact)		logger.warning("Skipping url <%s>: empty artifact", artifact)
continue		continue

assert urls is not None		assert urls is not None

# Deal with misplaced origins

# FIXME: T3294: Fix missing scheme in urls		# FIXME: T3294: Fix missing scheme in urls
origin, *fallback_urls = urls		origin, *fallback_urls = urls

integrity = artifact.get("integrity")		integrity = artifact.get("integrity")
if integrity is None:		if integrity is None:
logger.warning("Skipping url <%s>: missing integrity field", origin)		logger.warning("Skipping url <%s>: missing integrity field", origin)
continue		continue

▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines	def vcs_to_listed_origin(self, artifact: VCS) -> Iterator[ListedOrigin]:
assert self.lister_obj.id is not None		assert self.lister_obj.id is not None
# FIXME: What to do with the "ref" (e.g. git/hg/svn commit, ...)		# FIXME: What to do with the "ref" (e.g. git/hg/svn commit, ...)
yield ListedOrigin(		yield ListedOrigin(
lister_id=self.lister_obj.id,		lister_id=self.lister_obj.id,
url=artifact.origin,		url=artifact.origin,
visit_type=artifact.type,		visit_type=artifact.type,
)		)

def origin_to_listed_origin(
self, origin_upstream: OriginUpstream
) -> Iterator[ListedOrigin]:
"""Given an upstream origin, yield a ListedOrigin."""
assert self.lister_obj.id is not None
yield ListedOrigin(
lister_id=self.lister_obj.id,
url=origin_upstream.origin,
visit_type="git", # both nixpkgs and guix are git origins so far
)

def artifact_to_listed_origin(self, artifact: Artifact) -> Iterator[ListedOrigin]:		def artifact_to_listed_origin(self, artifact: Artifact) -> Iterator[ListedOrigin]:
"""Given an artifact (tarball, file), yield one ListedOrigin."""		"""Given an artifact (tarball, file), yield one ListedOrigin."""
assert self.lister_obj.id is not None		assert self.lister_obj.id is not None
yield ListedOrigin(		yield ListedOrigin(
lister_id=self.lister_obj.id,		lister_id=self.lister_obj.id,
url=artifact.origin,		url=artifact.origin,
visit_type=artifact.visit_type,		visit_type=artifact.visit_type,
extra_loader_arguments={		extra_loader_arguments={
Show All 12 Lines