Page MenuHomeSoftware Heritage
Differential D8189

Use original attr module instead of attrs
ClosedPublic
Actions

Authored by douardda on Aug 4 2022, 4:24 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Commits
rDVAUca71a59eda6f: Use original attr module instead of attrs
Summary

because this later is not yet available on debian, and we do not use new
API features anyway in test_cookers.py

Should fix deb packaging.

Diff Detail

Repository
rDVAU Software Heritage Vault
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

douardda created this revision.Aug 4 2022, 4:24 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptAug 4 2022, 4:24 PM
swh-public-ci added a comment.Aug 4 2022, 4:27 PM

Build is green

Patch application report for D8189 (id=29554)

Rebasing onto 186295793d...

Current branch diff-target is up to date.
Changes applied before test
commit ca71a59eda6f516765010111574e803630b492ec
Author: David Douard <david.douard@sdfa3.org>
Date:   Thu Aug 4 16:22:26 2022 +0200

    Use original attr module instead of attrs
    
    because this later is not yet available on debian, and we do not use new
    API features anyway in test_cookers.py
    
    Should fix deb packaging.

See https://jenkins.softwareheritage.org/job/DVAU/job/tests-on-diff/249/ for more details.

Harbormaster completed remote builds in B30620: Diff 29554.Aug 4 2022, 4:27 PM
douardda requested review of this revision.Aug 4 2022, 4:27 PM
ardumont accepted this revision.Aug 4 2022, 4:28 PM
This revision is now accepted and ready to land.Aug 4 2022, 4:28 PM
Closed by commit rDVAUca71a59eda6f: Use original attr module instead of attrs (authored by douardda). · Explain WhyAug 4 2022, 4:33 PM
This revision was automatically updated to reflect the committed changes.
douardda added a commit: rDVAUca71a59eda6f: Use original attr module instead of attrs.

Revision Contents
Changeset List

PathSize
5 lines
swh/
vault/
tests/
6 lines

Diff 29556

View Options

requirements-test.txt

Show First 20 Lines • Show All 281 Lines • ▼ Show 20 Linesclass PackageLoader(BaseLoader, Generic[TPackageInfo]):
def extra_branches(self) -> Dict[bytes, Mapping[str, Any]]: def extra_branches(self) -> Dict[bytes, Mapping[str, Any]]:
"""Return an extra dict of branches that are used to update the set of """Return an extra dict of branches that are used to update the set of
branches. branches.
""" """
return {} return {}
def finalize_visit(
self,
snapshot: Optional[Snapshot],
visit: OriginVisit,
status_visit: str,
status_load: str,
failed_branches: List[str],
) -> Dict[str, Any]:
"""Finalize the visit:
- flush eventual unflushed data to storage
- update origin visit's status
- return the task's status
"""
self.storage.flush()
snapshot_id: Optional[bytes] = None
if snapshot and snapshot.id: # to prevent the snapshot.id to b""
snapshot_id = snapshot.id
assert visit.visit
visit_status = OriginVisitStatus(
origin=self.url,
visit=visit.visit,
type=self.visit_type,
date=now(),
status=status_visit,
snapshot=snapshot_id,
)
self.storage.origin_visit_status_add([visit_status])
result: Dict[str, Any] = {
"status": status_load,
}
if snapshot_id:
result["snapshot_id"] = hash_to_hex(snapshot_id)
if failed_branches:
logger.warning("%d failed branches", len(failed_branches))
for i, urls in enumerate(islice(failed_branches, 50)):
prefix_url = "Failed branches: " if i == 0 else ""
logger.warning("%s%s", prefix_url, urls)
return result
def load(self) -> Dict: def load(self) -> Dict:
"""Load for a specific origin the associated contents. """Load for a specific origin the associated contents.
for each package version of the origin for each package version of the origin
1. Fetch the files for one package version By default, this can be 1. Fetch the files for one package version By default, this can be
implemented as a simple HTTP request. Loaders with more specific implemented as a simple HTTP request. Loaders with more specific
requirements can override this, e.g.: the PyPI loader checks the requirements can override this, e.g.: the PyPI loader checks the
Show All 32 Lines def load(self) -> Dict:
""" """
status_load = "uneventful" # either: eventful, uneventful, failed status_load = "uneventful" # either: eventful, uneventful, failed
status_visit = "full" # see swh.model.model.OriginVisitStatus status_visit = "full" # see swh.model.model.OriginVisitStatus
tmp_revisions = {} # type: Dict[str, List] tmp_revisions = {} # type: Dict[str, List]
snapshot = None snapshot = None
failed_branches: List[str] = [] failed_branches: List[str] = []
def finalize_visit() -> Dict[str, Any]:
"""Finalize the visit:
- flush eventual unflushed data to storage
- update origin visit's status
- return the task's status
"""
self.storage.flush()
snapshot_id: Optional[bytes] = None
if snapshot and snapshot.id: # to prevent the snapshot.id to b""
snapshot_id = snapshot.id
assert visit.visit
visit_status = OriginVisitStatus(
origin=self.url,
visit=visit.visit,
type=self.visit_type,
date=now(),
status=status_visit,
snapshot=snapshot_id,
)
self.storage.origin_visit_status_add([visit_status])
result: Dict[str, Any] = {
"status": status_load,
}
if snapshot_id:
result["snapshot_id"] = hash_to_hex(snapshot_id)
if failed_branches:
logger.warning("%d failed branches", len(failed_branches))
for i, urls in enumerate(islice(failed_branches, 50)):
prefix_url = "Failed branches: " if i == 0 else ""
logger.warning("%s%s", prefix_url, urls)
return result
# Prepare origin and origin_visit # Prepare origin and origin_visit
origin = Origin(url=self.url) origin = Origin(url=self.url)
try: try:
self.storage.origin_add([origin]) self.storage.origin_add([origin])
visit = list( visit = list(
self.storage.origin_visit_add( self.storage.origin_visit_add(
[ [
OriginVisit( OriginVisit(
Show All 10 Lines def load(self) -> Dict:
try: try:
last_snapshot = self.last_snapshot() last_snapshot = self.last_snapshot()
logger.debug("last snapshot: %s", last_snapshot) logger.debug("last snapshot: %s", last_snapshot)
known_artifacts = self.known_artifacts(last_snapshot) known_artifacts = self.known_artifacts(last_snapshot)
logger.debug("known artifacts: %s", known_artifacts) logger.debug("known artifacts: %s", known_artifacts)
except Exception as e: except Exception as e:
logger.exception("Failed to get previous state for %s", self.url) logger.exception("Failed to get previous state for %s", self.url)
sentry_sdk.capture_exception(e) sentry_sdk.capture_exception(e)
status_visit = "failed" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="failed",
status_load="failed",
)
load_exceptions: List[Exception] = [] load_exceptions: List[Exception] = []
try: try:
versions = self.get_versions() versions = self.get_versions()
except NotFound: except NotFound:
status_visit = "not_found" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="not_found",
status_load="failed",
)
except Exception: except Exception:
status_visit = "failed" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="failed",
status_load="failed",
)
for version in versions: for version in versions:
logger.debug("version: %s", version) logger.debug("version: %s", version)
tmp_revisions[version] = [] tmp_revisions[version] = []
# `p_` stands for `package_` # `p_` stands for `package_`
for branch_name, p_info in self.get_package_info(version): for branch_name, p_info in self.get_package_info(version):
logger.debug("package_info: %s", p_info) logger.debug("package_info: %s", p_info)
revision_id = self.resolve_revision_from(known_artifacts, p_info) revision_id = self.resolve_revision_from(known_artifacts, p_info)
Show All 24 Lines def load(self) -> Dict:
tmp_revisions[version].append((branch_name, revision_id)) tmp_revisions[version].append((branch_name, revision_id))
if load_exceptions: if load_exceptions:
status_visit = "partial" status_visit = "partial"
if not tmp_revisions: if not tmp_revisions:
# We could not load any revisions; fail completely # We could not load any revisions; fail completely
status_visit = "failed" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="failed",
status_load="failed",
)
try: try:
# Retrieve the default release version (the "latest" one) # Retrieve the default release version (the "latest" one)
default_version = self.get_default_version() default_version = self.get_default_version()
logger.debug("default version: %s", default_version) logger.debug("default version: %s", default_version)
# Retrieve extra branches # Retrieve extra branches
extra_branches = self.extra_branches() extra_branches = self.extra_branches()
logger.debug("extra branches: %s", extra_branches) logger.debug("extra branches: %s", extra_branches)
Show All 26 Lines def load(self) -> Dict:
except Exception as e: except Exception as e:
logger.exception( logger.exception(
"Failed to load extrinsic origin metadata for %s", self.url "Failed to load extrinsic origin metadata for %s", self.url
) )
sentry_sdk.capture_exception(e) sentry_sdk.capture_exception(e)
status_visit = "partial" status_visit = "partial"
status_load = "failed" status_load = "failed"
return finalize_visit() return self.finalize_visit(
snapshot=snapshot,
visit=visit,
failed_branches=failed_branches,
status_visit=status_visit,
status_load=status_load,
)
def _load_directory( def _load_directory(
self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], tmpdir: str self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], tmpdir: str
) -> Tuple[str, from_disk.Directory]: ) -> Tuple[str, from_disk.Directory]:
uncompressed_path = self.uncompress(dl_artifacts, dest=tmpdir) uncompressed_path = self.uncompress(dl_artifacts, dest=tmpdir)
logger.debug("uncompressed_path: %s", uncompressed_path) logger.debug("uncompressed_path: %s", uncompressed_path)
directory = from_disk.Directory.from_disk( directory = from_disk.Directory.from_disk(
▲ Show 20 Lines • Show All 295 Lines • Show Last 20 Lines
View Options

swh/vault/tests/test_cookers.py

Show First 20 Lines • Show All 281 Lines • ▼ Show 20 Linesclass PackageLoader(BaseLoader, Generic[TPackageInfo]):
def extra_branches(self) -> Dict[bytes, Mapping[str, Any]]: def extra_branches(self) -> Dict[bytes, Mapping[str, Any]]:
"""Return an extra dict of branches that are used to update the set of """Return an extra dict of branches that are used to update the set of
branches. branches.
""" """
return {} return {}
def finalize_visit(
self,
snapshot: Optional[Snapshot],
visit: OriginVisit,
status_visit: str,
status_load: str,
failed_branches: List[str],
) -> Dict[str, Any]:
"""Finalize the visit:
- flush eventual unflushed data to storage
- update origin visit's status
- return the task's status
"""
self.storage.flush()
snapshot_id: Optional[bytes] = None
if snapshot and snapshot.id: # to prevent the snapshot.id to b""
snapshot_id = snapshot.id
assert visit.visit
visit_status = OriginVisitStatus(
origin=self.url,
visit=visit.visit,
type=self.visit_type,
date=now(),
status=status_visit,
snapshot=snapshot_id,
)
self.storage.origin_visit_status_add([visit_status])
result: Dict[str, Any] = {
"status": status_load,
}
if snapshot_id:
result["snapshot_id"] = hash_to_hex(snapshot_id)
if failed_branches:
logger.warning("%d failed branches", len(failed_branches))
for i, urls in enumerate(islice(failed_branches, 50)):
prefix_url = "Failed branches: " if i == 0 else ""
logger.warning("%s%s", prefix_url, urls)
return result
def load(self) -> Dict: def load(self) -> Dict:
"""Load for a specific origin the associated contents. """Load for a specific origin the associated contents.
for each package version of the origin for each package version of the origin
1. Fetch the files for one package version By default, this can be 1. Fetch the files for one package version By default, this can be
implemented as a simple HTTP request. Loaders with more specific implemented as a simple HTTP request. Loaders with more specific
requirements can override this, e.g.: the PyPI loader checks the requirements can override this, e.g.: the PyPI loader checks the
Show All 32 Lines def load(self) -> Dict:
""" """
status_load = "uneventful" # either: eventful, uneventful, failed status_load = "uneventful" # either: eventful, uneventful, failed
status_visit = "full" # see swh.model.model.OriginVisitStatus status_visit = "full" # see swh.model.model.OriginVisitStatus
tmp_revisions = {} # type: Dict[str, List] tmp_revisions = {} # type: Dict[str, List]
snapshot = None snapshot = None
failed_branches: List[str] = [] failed_branches: List[str] = []
def finalize_visit() -> Dict[str, Any]:
"""Finalize the visit:
- flush eventual unflushed data to storage
- update origin visit's status
- return the task's status
"""
self.storage.flush()
snapshot_id: Optional[bytes] = None
if snapshot and snapshot.id: # to prevent the snapshot.id to b""
snapshot_id = snapshot.id
assert visit.visit
visit_status = OriginVisitStatus(
origin=self.url,
visit=visit.visit,
type=self.visit_type,
date=now(),
status=status_visit,
snapshot=snapshot_id,
)
self.storage.origin_visit_status_add([visit_status])
result: Dict[str, Any] = {
"status": status_load,
}
if snapshot_id:
result["snapshot_id"] = hash_to_hex(snapshot_id)
if failed_branches:
logger.warning("%d failed branches", len(failed_branches))
for i, urls in enumerate(islice(failed_branches, 50)):
prefix_url = "Failed branches: " if i == 0 else ""
logger.warning("%s%s", prefix_url, urls)
return result
# Prepare origin and origin_visit # Prepare origin and origin_visit
origin = Origin(url=self.url) origin = Origin(url=self.url)
try: try:
self.storage.origin_add([origin]) self.storage.origin_add([origin])
visit = list( visit = list(
self.storage.origin_visit_add( self.storage.origin_visit_add(
[ [
OriginVisit( OriginVisit(
Show All 10 Lines def load(self) -> Dict:
try: try:
last_snapshot = self.last_snapshot() last_snapshot = self.last_snapshot()
logger.debug("last snapshot: %s", last_snapshot) logger.debug("last snapshot: %s", last_snapshot)
known_artifacts = self.known_artifacts(last_snapshot) known_artifacts = self.known_artifacts(last_snapshot)
logger.debug("known artifacts: %s", known_artifacts) logger.debug("known artifacts: %s", known_artifacts)
except Exception as e: except Exception as e:
logger.exception("Failed to get previous state for %s", self.url) logger.exception("Failed to get previous state for %s", self.url)
sentry_sdk.capture_exception(e) sentry_sdk.capture_exception(e)
status_visit = "failed" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="failed",
status_load="failed",
)
load_exceptions: List[Exception] = [] load_exceptions: List[Exception] = []
try: try:
versions = self.get_versions() versions = self.get_versions()
except NotFound: except NotFound:
status_visit = "not_found" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="not_found",
status_load="failed",
)
except Exception: except Exception:
status_visit = "failed" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="failed",
status_load="failed",
)
for version in versions: for version in versions:
logger.debug("version: %s", version) logger.debug("version: %s", version)
tmp_revisions[version] = [] tmp_revisions[version] = []
# `p_` stands for `package_` # `p_` stands for `package_`
for branch_name, p_info in self.get_package_info(version): for branch_name, p_info in self.get_package_info(version):
logger.debug("package_info: %s", p_info) logger.debug("package_info: %s", p_info)
revision_id = self.resolve_revision_from(known_artifacts, p_info) revision_id = self.resolve_revision_from(known_artifacts, p_info)
Show All 24 Lines def load(self) -> Dict:
tmp_revisions[version].append((branch_name, revision_id)) tmp_revisions[version].append((branch_name, revision_id))
if load_exceptions: if load_exceptions:
status_visit = "partial" status_visit = "partial"
if not tmp_revisions: if not tmp_revisions:
# We could not load any revisions; fail completely # We could not load any revisions; fail completely
status_visit = "failed" return self.finalize_visit(
status_load = "failed" snapshot=snapshot,
return finalize_visit() visit=visit,
failed_branches=failed_branches,
status_visit="failed",
status_load="failed",
)
try: try:
# Retrieve the default release version (the "latest" one) # Retrieve the default release version (the "latest" one)
default_version = self.get_default_version() default_version = self.get_default_version()
logger.debug("default version: %s", default_version) logger.debug("default version: %s", default_version)
# Retrieve extra branches # Retrieve extra branches
extra_branches = self.extra_branches() extra_branches = self.extra_branches()
logger.debug("extra branches: %s", extra_branches) logger.debug("extra branches: %s", extra_branches)
Show All 26 Lines def load(self) -> Dict:
except Exception as e: except Exception as e:
logger.exception( logger.exception(
"Failed to load extrinsic origin metadata for %s", self.url "Failed to load extrinsic origin metadata for %s", self.url
) )
sentry_sdk.capture_exception(e) sentry_sdk.capture_exception(e)
status_visit = "partial" status_visit = "partial"
status_load = "failed" status_load = "failed"
return finalize_visit() return self.finalize_visit(
snapshot=snapshot,
visit=visit,
failed_branches=failed_branches,
status_visit=status_visit,
status_load=status_load,
)
def _load_directory( def _load_directory(
self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], tmpdir: str self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], tmpdir: str
) -> Tuple[str, from_disk.Directory]: ) -> Tuple[str, from_disk.Directory]:
uncompressed_path = self.uncompress(dl_artifacts, dest=tmpdir) uncompressed_path = self.uncompress(dl_artifacts, dest=tmpdir)
logger.debug("uncompressed_path: %s", uncompressed_path) logger.debug("uncompressed_path: %s", uncompressed_path)
directory = from_disk.Directory.from_disk( directory = from_disk.Directory.from_disk(
▲ Show 20 Lines • Show All 295 Lines • Show Last 20 Lines
About · Developer information · Legal