Page MenuHomeSoftware Heritage
Differential D2495

package.loader: Fail loading early when failing to create an origin/visit
ClosedPublic
Actions

Authored by ardumont on Jan 6 2020, 2:14 PM.

Details

Reviewers
douardda
Group Reviewers
Reviewers
Commits
rDLDBASE85c54dfbe1c8: package.loader: Stop loading when failing to create an origin/visit
Summary

Fix [1] unbound value error when failure to create either origin or origin-visit.

[1] https://sentry.softwareheritage.org/share/issue/6c4750b56a114817bb088ec6e4965faf/

Test Plan

tox

Diff Detail

Repository
rDLDBASE Generic VCS/Package Loader
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

ardumont created this revision.Jan 6 2020, 2:14 PM
swh-public-ci added a comment.Jan 6 2020, 2:16 PM

Build is green
See https://jenkins.softwareheritage.org/job/DLDBASE/job/tox/298/ for more details.

Harbormaster completed remote builds in B9957: Diff 8861.Jan 6 2020, 2:16 PM
ardumont retitled this revision from package.loader: Stop immediately loading when failing to create an origin/visit to package.loader: Fail loading early when failing to create an origin/visit.Jan 6 2020, 2:31 PM
ardumont updated this revision to Diff 8865.Jan 6 2020, 3:03 PM

Plug to fixture branch

swh-public-ci added a comment.Jan 6 2020, 3:05 PM

Build is green
See https://jenkins.softwareheritage.org/job/DLDBASE/job/tox/303/ for more details.

Harbormaster completed remote builds in B9961: Diff 8865.Jan 6 2020, 3:05 PM
ardumont edited the summary of this revision. (Show Details)Jan 6 2020, 3:06 PM
ardumont added a project: Core Loader.
ardumont updated this revision to Diff 8870.Jan 6 2020, 3:11 PM

Rebase on latest master

swh-public-ci added a comment.Jan 6 2020, 3:13 PM

Build is green
See https://jenkins.softwareheritage.org/job/DLDBASE/job/tox/304/ for more details.

Harbormaster completed remote builds in B9966: Diff 8870.Jan 6 2020, 3:13 PM
douardda accepted this revision.Jan 6 2020, 3:34 PM
douardda added inline comments.
swh/loader/package/loader.py
271

s/Failure/Failed/ ?

This revision is now accepted and ready to land.Jan 6 2020, 3:34 PM
ardumont updated this revision to Diff 8871.Jan 6 2020, 3:37 PM
  • Fix error message
  • plug to master branch
swh/loader/package/loader.py
271

yes

swh-public-ci added a comment.Jan 6 2020, 3:39 PM

Build is green
See https://jenkins.softwareheritage.org/job/DLDBASE/job/tox/305/ for more details.

Harbormaster completed remote builds in B9967: Diff 8871.Jan 6 2020, 3:39 PM
Closed by commit rDLDBASE85c54dfbe1c8: package.loader: Stop loading when failing to create an origin/visit (authored by ardumont). · Explain WhyJan 6 2020, 4:37 PM
This revision was automatically updated to reflect the committed changes.
ardumont added a commit: rDLDBASE85c54dfbe1c8: package.loader: Stop loading when failing to create an origin/visit.

Revision Contents
Changeset List

PathSize
swh/
loader/
package/
10 lines
tests/
35 lines

Diff 8877

View Options

swh/loader/package/loader.py

# Copyright (C) 2017-2019 The Software Heritage developers # Copyright (C) 2017-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import email.utils import email.utils
import logging import logging
from os import path from os import path
import re import re
import subprocess import subprocess
from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple
import attr
from dateutil.parser import parse as parse_date from dateutil.parser import parse as parse_date
from debian.changelog import Changelog from debian.changelog import Changelog
from debian.deb822 import Dsc from debian.deb822 import Dsc
from typing import Any, Generator, List, Mapping, Optional, Sequence, Tuple
from swh.loader.package.loader import PackageLoader from swh.loader.package.loader import BasePackageInfo, PackageLoader
from swh.loader.package.utils import download, release_name from swh.loader.package.utils import download, release_name
from swh.model.model import ( from swh.model.model import (
Sha1Git, Sha1Git,
Person, Person,
Revision, Revision,
RevisionType, RevisionType,
TimestampWithTimezone, TimestampWithTimezone,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
UPLOADERS_SPLIT = re.compile(r"(?<=\>)\s*,\s*") UPLOADERS_SPLIT = re.compile(r"(?<=\>)\s*,\s*")
class DebianLoader(PackageLoader): class DebianPackageInfo(BasePackageInfo):
raw = attr.ib(type=Dict[str, Any])
class DebianLoader(PackageLoader[DebianPackageInfo]):
"""Load debian origins into swh archive. """Load debian origins into swh archive.
""" """
visit_type = "deb" visit_type = "deb"
def __init__(self, url: str, date: str, packages: Mapping[str, Any]): def __init__(self, url: str, date: str, packages: Mapping[str, Any]):
"""Debian Loader implementation. """Debian Loader implementation.
Show All 40 Linesclass DebianLoader(PackageLoader[DebianPackageInfo]):
def get_versions(self) -> Sequence[str]: def get_versions(self) -> Sequence[str]:
"""Returns the keys of the packages input (e.g. """Returns the keys of the packages input (e.g.
stretch/contrib/0.7.2-3, etc...) stretch/contrib/0.7.2-3, etc...)
""" """
return list(self.packages.keys()) return list(self.packages.keys())
def get_package_info( def get_package_info(self, version: str) -> Iterator[Tuple[str, DebianPackageInfo]]:
ardumontUnsubmitted
Done
Inline Actions

same, Iterator...

ardumont: same, Iterator...
self, version: str
) -> Generator[Tuple[str, Mapping[str, Any]], None, None]:
meta = self.packages[version] meta = self.packages[version]
p_info = meta.copy() p_info = DebianPackageInfo(url=self.url, filename=None, raw=meta,)
p_info["raw"] = meta
yield release_name(version), p_info yield release_name(version), p_info
def resolve_revision_from( def resolve_revision_from(
self, known_package_artifacts: Mapping, artifact_metadata: Mapping self, known_package_artifacts: Mapping, artifact_metadata: Mapping
) -> Optional[bytes]: ) -> Optional[bytes]:
return resolve_revision_from(known_package_artifacts, artifact_metadata) return resolve_revision_from(known_package_artifacts, artifact_metadata)
def download_package( def download_package(
self, p_info: Mapping[str, Any], tmpdir: str self, p_info: DebianPackageInfo, tmpdir: str
) -> List[Tuple[str, Mapping]]: ) -> List[Tuple[str, Mapping]]:
"""Contrary to other package loaders (1 package, 1 artifact), """Contrary to other package loaders (1 package, 1 artifact),
`a_metadata` represents the package's datafiles set to fetch: `p_info.files` represents the package's datafiles set to fetch:
ardumontUnsubmitted
Done
Inline Actions

p_info.raw now

ardumont: p_info.raw now
- <package-version>.orig.tar.gz - <package-version>.orig.tar.gz
- <package-version>.dsc - <package-version>.dsc
- <package-version>.diff.gz - <package-version>.diff.gz
This is delegated to the `download_package` function. This is delegated to the `download_package` function.
""" """
all_hashes = download_package(p_info, tmpdir) all_hashes = download_package(p_info.raw, tmpdir)
logger.debug("all_hashes: %s", all_hashes) logger.debug("all_hashes: %s", all_hashes)
res = [] res = []
for hashes in all_hashes.values(): for hashes in all_hashes.values():
res.append((tmpdir, hashes)) res.append((tmpdir, hashes))
logger.debug("res: %s", res) logger.debug("res: %s", res)
return res return res
def uncompress( def uncompress(
▲ Show 20 Lines • Show All 296 Lines • Show Last 20 Lines
View Options

swh/loader/package/tests/test_loader.py

# Copyright (C) 2017-2019 The Software Heritage developers # Copyright (C) 2017-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import email.utils import email.utils
import logging import logging
from os import path from os import path
import re import re
import subprocess import subprocess
from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence, Tuple
import attr
from dateutil.parser import parse as parse_date from dateutil.parser import parse as parse_date
from debian.changelog import Changelog from debian.changelog import Changelog
from debian.deb822 import Dsc from debian.deb822 import Dsc
from typing import Any, Generator, List, Mapping, Optional, Sequence, Tuple
from swh.loader.package.loader import PackageLoader from swh.loader.package.loader import BasePackageInfo, PackageLoader
from swh.loader.package.utils import download, release_name from swh.loader.package.utils import download, release_name
from swh.model.model import ( from swh.model.model import (
Sha1Git, Sha1Git,
Person, Person,
Revision, Revision,
RevisionType, RevisionType,
TimestampWithTimezone, TimestampWithTimezone,
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
UPLOADERS_SPLIT = re.compile(r"(?<=\>)\s*,\s*") UPLOADERS_SPLIT = re.compile(r"(?<=\>)\s*,\s*")
class DebianLoader(PackageLoader): class DebianPackageInfo(BasePackageInfo):
raw = attr.ib(type=Dict[str, Any])
class DebianLoader(PackageLoader[DebianPackageInfo]):
"""Load debian origins into swh archive. """Load debian origins into swh archive.
""" """
visit_type = "deb" visit_type = "deb"
def __init__(self, url: str, date: str, packages: Mapping[str, Any]): def __init__(self, url: str, date: str, packages: Mapping[str, Any]):
"""Debian Loader implementation. """Debian Loader implementation.
Show All 40 Linesclass DebianLoader(PackageLoader[DebianPackageInfo]):
def get_versions(self) -> Sequence[str]: def get_versions(self) -> Sequence[str]:
"""Returns the keys of the packages input (e.g. """Returns the keys of the packages input (e.g.
stretch/contrib/0.7.2-3, etc...) stretch/contrib/0.7.2-3, etc...)
""" """
return list(self.packages.keys()) return list(self.packages.keys())
def get_package_info( def get_package_info(self, version: str) -> Iterator[Tuple[str, DebianPackageInfo]]:
ardumontUnsubmitted
Done
Inline Actions

same, Iterator...

ardumont: same, Iterator...
self, version: str
) -> Generator[Tuple[str, Mapping[str, Any]], None, None]:
meta = self.packages[version] meta = self.packages[version]
p_info = meta.copy() p_info = DebianPackageInfo(url=self.url, filename=None, raw=meta,)
p_info["raw"] = meta
yield release_name(version), p_info yield release_name(version), p_info
def resolve_revision_from( def resolve_revision_from(
self, known_package_artifacts: Mapping, artifact_metadata: Mapping self, known_package_artifacts: Mapping, artifact_metadata: Mapping
) -> Optional[bytes]: ) -> Optional[bytes]:
return resolve_revision_from(known_package_artifacts, artifact_metadata) return resolve_revision_from(known_package_artifacts, artifact_metadata)
def download_package( def download_package(
self, p_info: Mapping[str, Any], tmpdir: str self, p_info: DebianPackageInfo, tmpdir: str
) -> List[Tuple[str, Mapping]]: ) -> List[Tuple[str, Mapping]]:
"""Contrary to other package loaders (1 package, 1 artifact), """Contrary to other package loaders (1 package, 1 artifact),
`a_metadata` represents the package's datafiles set to fetch: `p_info.files` represents the package's datafiles set to fetch:
ardumontUnsubmitted
Done
Inline Actions

p_info.raw now

ardumont: p_info.raw now
- <package-version>.orig.tar.gz - <package-version>.orig.tar.gz
- <package-version>.dsc - <package-version>.dsc
- <package-version>.diff.gz - <package-version>.diff.gz
This is delegated to the `download_package` function. This is delegated to the `download_package` function.
""" """
all_hashes = download_package(p_info, tmpdir) all_hashes = download_package(p_info.raw, tmpdir)
logger.debug("all_hashes: %s", all_hashes) logger.debug("all_hashes: %s", all_hashes)
res = [] res = []
for hashes in all_hashes.values(): for hashes in all_hashes.values():
res.append((tmpdir, hashes)) res.append((tmpdir, hashes))
logger.debug("res: %s", res) logger.debug("res: %s", res)
return res return res
def uncompress( def uncompress(
▲ Show 20 Lines • Show All 296 Lines • Show Last 20 Lines
About · Developer information · Legal