Page MenuHomeSoftware Heritage
Differential D8611

nixguix: Deal with connection error with server
ClosedPublic
Actions

Authored by ardumont on Oct 4 2022, 2:57 PM.

Details

Reviewers
anlambert
Group Reviewers
Reviewers
Maniphest Tasks
T3781: Replace the Nixguix loader with a lister
Commits
rDLS0f8f293f968e: nixguix: Deal with connection error with server
Summary

When that arises, we skip the origins.

Related to T3781

Diff Detail

Repository
rDLS Listers
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 32075
Build 50219: Phabricator diff pipeline on jenkinsJenkins console · Jenkins
Build 50218: arc lint + arc unit

Event Timeline

ardumont created this revision.Oct 4 2022, 2:57 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptOct 4 2022, 2:57 PM
swh-public-ci added a comment.Oct 4 2022, 3:02 PM

Build is green

Patch application report for D8611 (id=31105)

Rebasing onto d92474bbda...

Current branch diff-target is up to date.
Changes applied before test
commit 0f8f293f968ee1efc8b269f4073bbac99a4f3615
Author: Antoine R. Dumont (@ardumont) <ardumont@softwareheritage.org>
Date:   Tue Oct 4 14:57:01 2022 +0200

    nixguix: Deal with connection error with server
    
    When that arises, we skip the origins.
    
    Related to T3781

See https://jenkins.softwareheritage.org/job/DLS/job/tests-on-diff/744/ for more details.

Harbormaster completed remote builds in B32075: Diff 31105.Oct 4 2022, 3:02 PM
ardumont requested review of this revision.Oct 4 2022, 3:02 PM
anlambert accepted this revision.Oct 4 2022, 3:48 PM
This revision is now accepted and ready to land.Oct 4 2022, 3:48 PM
ardumont added a child revision: D8612: nixguix: Add support for pseudo url with missing schema.Oct 4 2022, 4:01 PM
Closed by commit rDLS0f8f293f968e: nixguix: Deal with connection error with server (authored by ardumont). · Explain WhyOct 4 2022, 4:02 PM
This revision was automatically updated to reflect the committed changes.
ardumont added a commit: rDLS0f8f293f968e: nixguix: Deal with connection error with server.
ardumont mentioned this in T3781: Replace the Nixguix loader with a lister.Oct 4 2022, 4:05 PM

Revision Contents
Changeset List

PathSizeCoverage (All)Coverage (Touched)
swh/
lister/
nixguix/
4 lines99%
Loading...
tests/
data/
7 lines--
14 lines100%
Loading...

Diff 31105

View Options

swh/lister/nixguix/lister.py

Show First 20 Lines • Show All 1,155 Lines • ▼ Show 20 Lines def test_release_add(self, swh_storage):
# already present so nothing added # already present so nothing added
actual_result = swh_storage.release_add([data.release, data.release2]) actual_result = swh_storage.release_add([data.release, data.release2])
assert actual_result == {"release:add": 0} assert actual_result == {"release:add": 0}
swh_storage.refresh_stat_counters() swh_storage.refresh_stat_counters()
assert swh_storage.stat_counters()["release"] == 2 assert swh_storage.stat_counters()["release"] == 2
def test_release_add_same_twice(self, swh_storage):
result = swh_storage.release_add([data.release, data.release])
assert list(result) == ["release:add"]
assert 1 <= result["release:add"] <= 2
def test_release_add_from_generator(self, swh_storage): def test_release_add_from_generator(self, swh_storage):
def _rel_gen(): def _rel_gen():
yield data.release yield data.release
yield data.release2 yield data.release2
actual_result = swh_storage.release_add(_rel_gen()) actual_result = swh_storage.release_add(_rel_gen())
assert actual_result == {"release:add": 2} assert actual_result == {"release:add": 2}
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines
def test_release_add_twice(self, swh_storage): def test_release_add_twice(self, swh_storage):
actual_result = swh_storage.release_add([data.release]) actual_result = swh_storage.release_add([data.release])
assert actual_result == {"release:add": 1} assert actual_result == {"release:add": 1}
assert list(swh_storage.journal_writer.journal.objects) == [ assert list(swh_storage.journal_writer.journal.objects) == [
("release", Release.from_dict(data.release)) ("release", Release.from_dict(data.release))
] ]
actual_result = swh_storage.release_add([data.release, data.release2]) actual_result = swh_storage.release_add(
[data.release, data.release2, data.release, data.release2]
)
assert actual_result == {"release:add": 1} assert actual_result == {"release:add": 1}
assert list(swh_storage.journal_writer.journal.objects) == [ assert set(swh_storage.journal_writer.journal.objects) == set(
[
("release", Release.from_dict(data.release)), ("release", Release.from_dict(data.release)),
("release", Release.from_dict(data.release2)), ("release", Release.from_dict(data.release2)),
] ]
)
def test_release_add_name_clash(self, swh_storage): def test_release_add_name_clash(self, swh_storage):
release1 = data.release.copy() release1 = data.release.copy()
release2 = data.release2.copy() release2 = data.release2.copy()
release1["author"] = { release1["author"] = {
"fullname": b"John Doe <john.doe@example.com>", "fullname": b"John Doe <john.doe@example.com>",
"name": b"John Doe", "name": b"John Doe",
▲ Show 20 Lines • Show All 2,722 Lines • Show Last 20 Lines
View Options

swh/lister/nixguix/tests/data/guix-swh_sources.json

Show First 20 Lines • Show All 1,155 Lines • ▼ Show 20 Lines def test_release_add(self, swh_storage):
# already present so nothing added # already present so nothing added
actual_result = swh_storage.release_add([data.release, data.release2]) actual_result = swh_storage.release_add([data.release, data.release2])
assert actual_result == {"release:add": 0} assert actual_result == {"release:add": 0}
swh_storage.refresh_stat_counters() swh_storage.refresh_stat_counters()
assert swh_storage.stat_counters()["release"] == 2 assert swh_storage.stat_counters()["release"] == 2
def test_release_add_same_twice(self, swh_storage):
result = swh_storage.release_add([data.release, data.release])
assert list(result) == ["release:add"]
assert 1 <= result["release:add"] <= 2
def test_release_add_from_generator(self, swh_storage): def test_release_add_from_generator(self, swh_storage):
def _rel_gen(): def _rel_gen():
yield data.release yield data.release
yield data.release2 yield data.release2
actual_result = swh_storage.release_add(_rel_gen()) actual_result = swh_storage.release_add(_rel_gen())
assert actual_result == {"release:add": 2} assert actual_result == {"release:add": 2}
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines
def test_release_add_twice(self, swh_storage): def test_release_add_twice(self, swh_storage):
actual_result = swh_storage.release_add([data.release]) actual_result = swh_storage.release_add([data.release])
assert actual_result == {"release:add": 1} assert actual_result == {"release:add": 1}
assert list(swh_storage.journal_writer.journal.objects) == [ assert list(swh_storage.journal_writer.journal.objects) == [
("release", Release.from_dict(data.release)) ("release", Release.from_dict(data.release))
] ]
actual_result = swh_storage.release_add([data.release, data.release2]) actual_result = swh_storage.release_add(
[data.release, data.release2, data.release, data.release2]
)
assert actual_result == {"release:add": 1} assert actual_result == {"release:add": 1}
assert list(swh_storage.journal_writer.journal.objects) == [ assert set(swh_storage.journal_writer.journal.objects) == set(
[
("release", Release.from_dict(data.release)), ("release", Release.from_dict(data.release)),
("release", Release.from_dict(data.release2)), ("release", Release.from_dict(data.release2)),
] ]
)
def test_release_add_name_clash(self, swh_storage): def test_release_add_name_clash(self, swh_storage):
release1 = data.release.copy() release1 = data.release.copy()
release2 = data.release2.copy() release2 = data.release2.copy()
release1["author"] = { release1["author"] = {
"fullname": b"John Doe <john.doe@example.com>", "fullname": b"John Doe <john.doe@example.com>",
"name": b"John Doe", "name": b"John Doe",
▲ Show 20 Lines • Show All 2,722 Lines • Show Last 20 Lines
View Options

swh/lister/nixguix/tests/test_lister.py

Show First 20 Lines • Show All 1,155 Lines • ▼ Show 20 Lines def test_release_add(self, swh_storage):
# already present so nothing added # already present so nothing added
actual_result = swh_storage.release_add([data.release, data.release2]) actual_result = swh_storage.release_add([data.release, data.release2])
assert actual_result == {"release:add": 0} assert actual_result == {"release:add": 0}
swh_storage.refresh_stat_counters() swh_storage.refresh_stat_counters()
assert swh_storage.stat_counters()["release"] == 2 assert swh_storage.stat_counters()["release"] == 2
def test_release_add_same_twice(self, swh_storage):
result = swh_storage.release_add([data.release, data.release])
assert list(result) == ["release:add"]
assert 1 <= result["release:add"] <= 2
def test_release_add_from_generator(self, swh_storage): def test_release_add_from_generator(self, swh_storage):
def _rel_gen(): def _rel_gen():
yield data.release yield data.release
yield data.release2 yield data.release2
actual_result = swh_storage.release_add(_rel_gen()) actual_result = swh_storage.release_add(_rel_gen())
assert actual_result == {"release:add": 2} assert actual_result == {"release:add": 2}
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines
def test_release_add_twice(self, swh_storage): def test_release_add_twice(self, swh_storage):
actual_result = swh_storage.release_add([data.release]) actual_result = swh_storage.release_add([data.release])
assert actual_result == {"release:add": 1} assert actual_result == {"release:add": 1}
assert list(swh_storage.journal_writer.journal.objects) == [ assert list(swh_storage.journal_writer.journal.objects) == [
("release", Release.from_dict(data.release)) ("release", Release.from_dict(data.release))
] ]
actual_result = swh_storage.release_add([data.release, data.release2]) actual_result = swh_storage.release_add(
[data.release, data.release2, data.release, data.release2]
)
assert actual_result == {"release:add": 1} assert actual_result == {"release:add": 1}
assert list(swh_storage.journal_writer.journal.objects) == [ assert set(swh_storage.journal_writer.journal.objects) == set(
[
("release", Release.from_dict(data.release)), ("release", Release.from_dict(data.release)),
("release", Release.from_dict(data.release2)), ("release", Release.from_dict(data.release2)),
] ]
)
def test_release_add_name_clash(self, swh_storage): def test_release_add_name_clash(self, swh_storage):
release1 = data.release.copy() release1 = data.release.copy()
release2 = data.release2.copy() release2 = data.release2.copy()
release1["author"] = { release1["author"] = {
"fullname": b"John Doe <john.doe@example.com>", "fullname": b"John Doe <john.doe@example.com>",
"name": b"John Doe", "name": b"John Doe",
▲ Show 20 Lines • Show All 2,722 Lines • Show Last 20 Lines
About · Developer information · Legal