Page MenuHomeSoftware Heritage
Differential D7149

conftest: Fix scheduler loading task names
ClosedPublic
Actions

Authored by anlambert on Feb 10 2022, 4:05 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Commits
rDWAPPS786d081bd312: conftest: Fix scheduler loading task names
Summary

Numerous tests are broken since rDSCHc46ffadf7adf24c7eb3ffce062e8ade3818c79cc
has landed in swh-scheduler.

Ensure to restore the old loading task names for tests to succeed.

Diff Detail

Repository
rDWAPPS Web applications
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 26756
Build 41836: Phabricator diff pipeline on jenkinsJenkins console · Jenkins
Build 41835: arc lint + arc unit

Event Timeline

anlambert created this revision.Feb 10 2022, 4:05 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptFeb 10 2022, 4:05 PM
swh-public-ci added a comment.Feb 10 2022, 4:19 PM

Build is green

Patch application report for D7149 (id=25911)

Rebasing onto 13f1591bb2...

Current branch diff-target is up to date.
Changes applied before test
commit 786d081bd31270f0ae50447caea9536f978b32dc
Author: Antoine Lambert <anlambert@softwareheritage.org>
Date:   Thu Feb 10 16:04:40 2022 +0100

    conftest: Fix scheduler loading task names
    
    Numerous tests are broken since rDSCHc46ffadf7adf24c7eb3ffce062e8ade3818c79cc
    has landed in swh-scheduler.
    
    Ensure to restore the old loading task names for tests to succeed.

See https://jenkins.softwareheritage.org/job/DWAPPS/job/tests-on-diff/1372/ for more details.

Harbormaster completed remote builds in B26756: Diff 25911.Feb 10 2022, 4:19 PM
anlambert requested review of this revision.Feb 10 2022, 4:19 PM
ardumont accepted this revision.Feb 10 2022, 4:41 PM
This revision is now accepted and ready to land.Feb 10 2022, 4:41 PM
Closed by commit rDWAPPS786d081bd312: conftest: Fix scheduler loading task names (authored by anlambert). · Explain WhyFeb 10 2022, 4:47 PM
This revision was automatically updated to reflect the committed changes.
anlambert added a commit: rDWAPPS786d081bd312: conftest: Fix scheduler loading task names.

Revision Contents
Changeset List

PathSizeCoverage (All)Coverage (Touched)
swh/
web/
tests/
2 lines94%
Loading...

Diff 25911

View Options

swh/web/tests/conftest.py

# Copyright (C) 2015-2021 The Software Heritage developers # Copyright (C) 2015-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from __future__ import annotations from __future__ import annotations
import binascii import binascii
import datetime import datetime
import enum import enum
from functools import lru_cache from functools import lru_cache
import hashlib import hashlib
import re import re
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from typing import (
Any,
Dict,
Generic,
Iterable,
List,
Optional,
Tuple,
Type,
TypeVar,
Union,
)
import warnings import warnings
import attr import attr
from attrs_strict import type_validator from attrs_strict import type_validator
from .collections import ImmutableDict from .collections import ImmutableDict
from .exceptions import ValidationError from .exceptions import ValidationError
from .fields.hashes import validate_sha1 from .fields.hashes import validate_sha1
▲ Show 20 Lines • Show All 707 Lines • ▼ Show 20 Lines_swhid_type_map = {
"rel": RELEASE, "rel": RELEASE,
"rev": REVISION, "rev": REVISION,
"dir": DIRECTORY, "dir": DIRECTORY,
"cnt": CONTENT, "cnt": CONTENT,
"emd": RAW_EXTRINSIC_METADATA, "emd": RAW_EXTRINSIC_METADATA,
} }
@attr.s(frozen=True, kw_only=True) # type of the "object_type" attribute of the SWHID class; either
class CoreSWHID: # ObjectType or ExtendedObjectType
""" _TObjectType = TypeVar("_TObjectType", ObjectType, ExtendedObjectType)
Dataclass holding the relevant info associated to a SoftWare Heritage
persistent IDentifier (SWHID). # the SWHID class itself (this is used so that X.from_string() can return X
# for all X subclass of _BaseSWHID)
Unlike `QualifiedSWHID`, it is restricted to core SWHIDs, ie. SWHIDs _TSWHID = TypeVar("_TSWHID", bound="_BaseSWHID")
with no qualifiers.
Raises:
swh.model.exceptions.ValidationError: In case of invalid object type or id
To get the raw SWHID string from an instance of this class,
use the :func:`str` function:
>>> swhid = CoreSWHID(
... object_type=ObjectType.CONTENT,
... object_id=bytes.fromhex('8ff44f081d43176474b267de5451f2c2e88089d0'),
... )
>>> str(swhid)
'swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0'
And vice-versa with :meth:`CoreSWHID.from_string`: @attr.s(frozen=True, kw_only=True)
class _BaseSWHID(Generic[_TObjectType]):
"""Common base class for CoreSWHID, QualifiedSWHID, and ExtendedSWHID.
>>> swhid == CoreSWHID.from_string( This is an "abstract" class and should not be instantiated directly;
... "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0" it only exists to deduplicate code between these three SWHID classes."""
... )
True
"""
namespace = attr.ib(type=str, default=SWHID_NAMESPACE) namespace = attr.ib(type=str, default=SWHID_NAMESPACE)
"""the namespace of the identifier, defaults to ``swh``""" """the namespace of the identifier, defaults to ``swh``"""
scheme_version = attr.ib(type=int, default=SWHID_VERSION) scheme_version = attr.ib(type=int, default=SWHID_VERSION)
"""the scheme version of the identifier, defaults to 1""" """the scheme version of the identifier, defaults to 1"""
object_type = attr.ib( # overridden by subclasses
type=ObjectType, validator=type_validator(), converter=ObjectType object_type: _TObjectType
)
"""the type of object the identifier points to""" """the type of object the identifier points to"""
object_id = attr.ib(type=bytes, validator=type_validator()) object_id = attr.ib(type=bytes, validator=type_validator())
"""object's identifier""" """object's identifier"""
@namespace.validator @namespace.validator
def check_namespace(self, attribute, value): def check_namespace(self, attribute, value):
if value != SWHID_NAMESPACE: if value != SWHID_NAMESPACE:
Show All 23 Lines def __str__(self) -> str:
self.namespace, self.namespace,
str(self.scheme_version), str(self.scheme_version),
self.object_type.value, self.object_type.value,
hash_to_hex(self.object_id), hash_to_hex(self.object_id),
] ]
) )
@classmethod @classmethod
def from_string(cls, s: str) -> CoreSWHID: def from_string(cls: Type[_TSWHID], s: str) -> _TSWHID:
parts = _parse_swhid(s) parts = _parse_swhid(s)
if parts.pop("qualifiers"): if parts.pop("qualifiers"):
olasdUnsubmitted
Not Done
Inline Actions
parts = _parse_swhid(s)
- if parts.pop("qualifiers"):
+ if "qualifiers" in parts:
raise ValidationError(f"{cls.__name__} does not support qualifiers.")
olasd:
vlorentzAuthorUnsubmitted
Done
Inline Actions

Nope, parts["qualifiers"] is an empty dict

vlorentz: Nope, `parts["qualifiers"]` is an empty dict
raise ValidationError("CoreSWHID does not support qualifiers.") raise ValidationError(f"{cls.__name__} does not support qualifiers.")
try: try:
return CoreSWHID(**parts) return cls(**parts)
except ValueError as e: except ValueError as e:
raise ValidationError(*e.args) from None raise ValidationError(*e.args) from None
@attr.s(frozen=True, kw_only=True)
class CoreSWHID(_BaseSWHID[ObjectType]):
"""
Dataclass holding the relevant info associated to a SoftWare Heritage
persistent IDentifier (SWHID).
Unlike `QualifiedSWHID`, it is restricted to core SWHIDs, ie. SWHIDs
with no qualifiers.
Raises:
swh.model.exceptions.ValidationError: In case of invalid object type or id
To get the raw SWHID string from an instance of this class,
use the :func:`str` function:
>>> swhid = CoreSWHID(
... object_type=ObjectType.CONTENT,
... object_id=bytes.fromhex('8ff44f081d43176474b267de5451f2c2e88089d0'),
... )
>>> str(swhid)
'swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0'
And vice-versa with :meth:`CoreSWHID.from_string`:
>>> swhid == CoreSWHID.from_string(
... "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0"
... )
True
"""
object_type = attr.ib(
type=ObjectType, validator=type_validator(), converter=ObjectType
)
"""the type of object the identifier points to"""
def _parse_core_swhid(swhid: Union[str, CoreSWHID, None]) -> Optional[CoreSWHID]: def _parse_core_swhid(swhid: Union[str, CoreSWHID, None]) -> Optional[CoreSWHID]:
"""Alias of CoreSWHID.from_string to make mypy happy...... """Alias of CoreSWHID.from_string to make mypy happy......
https://github.com/python/mypy/issues/6172""" https://github.com/python/mypy/issues/6172"""
if swhid is None or isinstance(swhid, CoreSWHID): if swhid is None or isinstance(swhid, CoreSWHID):
return swhid return swhid
else: else:
return CoreSWHID.from_string(swhid) return CoreSWHID.from_string(swhid)
def _parse_lines_qualifier( def _parse_lines_qualifier(
lines: Union[str, Tuple[int, Optional[int]], None] lines: Union[str, Tuple[int, Optional[int]], None]
) -> Optional[Tuple[int, Optional[int]]]: ) -> Optional[Tuple[int, Optional[int]]]:
if lines is None or isinstance(lines, tuple): if lines is None or isinstance(lines, tuple):
return lines return lines
elif "-" in lines: elif "-" in lines:
return tuple(map(int, lines.split("-", 2))) # type: ignore return tuple(map(int, lines.split("-", 2))) # type: ignore
else: else:
return (int(lines), None) return (int(lines), None)
@attr.s(frozen=True, kw_only=True) @attr.s(frozen=True, kw_only=True)
class QualifiedSWHID: class QualifiedSWHID(_BaseSWHID[ObjectType]):
""" """
Dataclass holding the relevant info associated to a SoftWare Heritage Dataclass holding the relevant info associated to a SoftWare Heritage
persistent IDentifier (SWHID) persistent IDentifier (SWHID)
Raises: Raises:
swh.model.exceptions.ValidationError: In case of invalid object type or id swh.model.exceptions.ValidationError: In case of invalid object type or id
To get the raw SWHID string from an instance of this class, To get the raw SWHID string from an instance of this class,
Show All 10 Linesclass QualifiedSWHID(_BaseSWHID[ObjectType]):
And vice-versa with :meth:`QualifiedSWHID.from_string`: And vice-versa with :meth:`QualifiedSWHID.from_string`:
>>> swhid == QualifiedSWHID.from_string( >>> swhid == QualifiedSWHID.from_string(
... "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0;lines=5-10" ... "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0;lines=5-10"
... ) ... )
True True
""" """
namespace = attr.ib(type=str, default=SWHID_NAMESPACE)
"""the namespace of the identifier, defaults to ``swh``"""
scheme_version = attr.ib(type=int, default=SWHID_VERSION)
"""the scheme version of the identifier, defaults to 1"""
object_type = attr.ib( object_type = attr.ib(
type=ObjectType, validator=type_validator(), converter=ObjectType type=ObjectType, validator=type_validator(), converter=ObjectType
) )
"""the type of object the identifier points to""" """the type of object the identifier points to"""
object_id = attr.ib(type=bytes, validator=type_validator())
"""object's identifier"""
# qualifiers: # qualifiers:
origin = attr.ib(type=Optional[str], default=None, validator=type_validator()) origin = attr.ib(type=Optional[str], default=None, validator=type_validator())
"""the software origin where an object has been found or observed in the wild, """the software origin where an object has been found or observed in the wild,
as an URI""" as an URI"""
visit = attr.ib(type=Optional[CoreSWHID], default=None, converter=_parse_core_swhid) visit = attr.ib(type=Optional[CoreSWHID], default=None, converter=_parse_core_swhid)
"""the core identifier of a snapshot corresponding to a specific visit """the core identifier of a snapshot corresponding to a specific visit
Show All 19 Linesclass QualifiedSWHID(_BaseSWHID[ObjectType]):
lines = attr.ib( lines = attr.ib(
type=Optional[Tuple[int, Optional[int]]], type=Optional[Tuple[int, Optional[int]]],
default=None, default=None,
validator=type_validator(), validator=type_validator(),
converter=_parse_lines_qualifier, converter=_parse_lines_qualifier,
) )
"""lines: line number(s) of interest, usually within a content object""" """lines: line number(s) of interest, usually within a content object"""
@namespace.validator
def check_namespace(self, attribute, value):
if value != SWHID_NAMESPACE:
raise ValidationError(
"Invalid SWHID: invalid namespace: %(namespace)s",
params={"namespace": value},
)
@scheme_version.validator
def check_scheme_version(self, attribute, value):
if value != SWHID_VERSION:
raise ValidationError(
"Invalid SWHID: invalid version: %(version)s", params={"version": value}
)
@object_id.validator
def check_object_id(self, attribute, value):
if len(value) != 20:
raise ValidationError(
"Invalid SWHID: invalid checksum: %(object_id)s",
params={"object_id": hash_to_hex(value)},
)
@visit.validator @visit.validator
def check_visit(self, attribute, value): def check_visit(self, attribute, value):
if value and value.object_type != ObjectType.SNAPSHOT: if value and value.object_type != ObjectType.SNAPSHOT:
raise ValidationError( raise ValidationError(
f"The 'visit' qualifier must be a 'snp' SWHID, " f"The 'visit' qualifier must be a 'snp' SWHID, "
f"not '{value.object_type.value}'" f"not '{value.object_type.value}'"
) )
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines def from_string(cls, s: str) -> QualifiedSWHID:
) )
try: try:
return QualifiedSWHID(**parts, **qualifiers) return QualifiedSWHID(**parts, **qualifiers)
except ValueError as e: except ValueError as e:
raise ValidationError(*e.args) from None raise ValidationError(*e.args) from None
@attr.s(frozen=True, kw_only=True) @attr.s(frozen=True, kw_only=True)
class ExtendedSWHID: class ExtendedSWHID(_BaseSWHID[ExtendedObjectType]):
""" """
Dataclass holding the relevant info associated to a SoftWare Heritage Dataclass holding the relevant info associated to a SoftWare Heritage
persistent IDentifier (SWHID). persistent IDentifier (SWHID).
It extends `CoreSWHID`, by allowing non-standard object types; and should It extends `CoreSWHID`, by allowing non-standard object types; and should
only be used internally to Software Heritage. only be used internally to Software Heritage.
Raises: Raises:
Show All 12 Linesclass ExtendedSWHID(_BaseSWHID[ExtendedObjectType]):
And vice-versa with :meth:`CoreSWHID.from_string`: And vice-versa with :meth:`CoreSWHID.from_string`:
>>> swhid == ExtendedSWHID.from_string( >>> swhid == ExtendedSWHID.from_string(
... "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0" ... "swh:1:cnt:8ff44f081d43176474b267de5451f2c2e88089d0"
... ) ... )
True True
""" """
namespace = attr.ib(type=str, default=SWHID_NAMESPACE)
"""the namespace of the identifier, defaults to ``swh``"""
scheme_version = attr.ib(type=int, default=SWHID_VERSION)
"""the scheme version of the identifier, defaults to 1"""
object_type = attr.ib( object_type = attr.ib(
type=ExtendedObjectType, type=ExtendedObjectType,
validator=type_validator(), validator=type_validator(),
converter=ExtendedObjectType, converter=ExtendedObjectType,
) )
"""the type of object the identifier points to""" """the type of object the identifier points to"""
object_id = attr.ib(type=bytes, validator=type_validator())
"""object's identifier"""
@namespace.validator
def check_namespace(self, attribute, value):
if value != SWHID_NAMESPACE:
raise ValidationError(
"Invalid SWHID: invalid namespace: %(namespace)s",
params={"namespace": value},
)
@scheme_version.validator
def check_scheme_version(self, attribute, value):
if value != SWHID_VERSION:
raise ValidationError(
"Invalid SWHID: invalid version: %(version)s", params={"version": value}
)
@object_id.validator
def check_object_id(self, attribute, value):
if len(value) != 20:
raise ValidationError(
"Invalid SWHID: invalid checksum: %(object_id)s",
params={"object_id": hash_to_hex(value)},
)
def __str__(self) -> str:
return SWHID_SEP.join(
[
self.namespace,
str(self.scheme_version),
self.object_type.value,
hash_to_hex(self.object_id),
]
)
@classmethod
def from_string(cls, s: str) -> ExtendedSWHID:
parts = _parse_swhid(s)
if parts.pop("qualifiers"):
raise ValidationError("ExtendedSWHID does not support qualifiers.")
return ExtendedSWHID(**parts)
@attr.s(frozen=True) @attr.s(frozen=True)
class SWHID: class SWHID:
""" """
Deprecated alternative to QualifiedSWHID. Deprecated alternative to QualifiedSWHID.
Args: Args:
namespace (str): the namespace of the identifier, defaults to ``swh`` namespace (str): the namespace of the identifier, defaults to ``swh``
▲ Show 20 Lines • Show All 196 Lines • Show Last 20 Lines
About · Developer information · Legal