Page MenuHomeSoftware Heritage
Differential D1372

swh.journal.publisher: Remove publisher
ClosedPublic
Actions

Authored by ardumont on Apr 9 2019, 12:13 PM.

Details

Reviewers
olasd
douardda
vlorentz
Group Reviewers
Reviewers
Commits
rDJNL989d96fa30f4: swh.journal.publisher: Remove publisher
Summary

Unrelated to this diff, tests will fail.

There is an ongoing fix for the hypothesis strategy in D1367 to fix
the failed tests (related to writer/replayer).

Test Plan

tox

Diff Detail

Repository
rDJNL Journal infrastructure
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

ardumont created this revision.Apr 9 2019, 12:13 PM
swh-public-ci added a comment.Apr 9 2019, 12:16 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DJNL/job/tox/127/
See console output for more information: https://jenkins.softwareheritage.org/job/DJNL/job/tox/127/console

Harbormaster failed remote builds in B5213: Diff 4407!Apr 9 2019, 12:16 PM
ardumont updated this revision to Diff 4410.Apr 9 2019, 12:21 PM

Rebase on master

swh-public-ci added a comment.Apr 9 2019, 12:24 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DJNL/job/tox/129/
See console output for more information: https://jenkins.softwareheritage.org/job/DJNL/job/tox/129/console

Harbormaster failed remote builds in B5217: Diff 4410!Apr 9 2019, 12:24 PM
ardumont mentioned this in D1374: docker-compose: Remove storage listener and journal publisher.Apr 9 2019, 1:14 PM
ardumont mentioned this in D1358: Add extra configuration to kafka consumer from journal.client.Apr 9 2019, 1:57 PM
olasd accepted this revision.Apr 9 2019, 3:06 PM

presses F

The few references left to swh.journal.publisher should disappear when the backfiller lands.

This revision is now accepted and ready to land.Apr 9 2019, 3:06 PM
ardumont updated this revision to Diff 4428.Apr 9 2019, 4:13 PM

Merge on master

swh-public-ci added a comment.Apr 9 2019, 4:17 PM

Build has FAILED

Link to build: https://jenkins.softwareheritage.org/job/DJNL/job/tox/132/
See console output for more information: https://jenkins.softwareheritage.org/job/DJNL/job/tox/132/console

Harbormaster failed remote builds in B5235: Diff 4428!Apr 9 2019, 4:17 PM
This revision was landed with ongoing or failed builds.Apr 9 2019, 4:18 PM
Closed by commit rDJNL989d96fa30f4: swh.journal.publisher: Remove publisher (authored by ardumont). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 4429

View Options

swh/journal/cli.py

Show First 20 Lines • Show All 680 Lines • ▼ Show 20 Lines def test_record_listed_origins(self, swh_scheduler, listed_origins):
ret = swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.record_listed_origins(listed_origins)
assert set(returned.url for returned in ret) == set( assert set(returned.url for returned in ret) == set(
origin.url for origin in listed_origins origin.url for origin in listed_origins
) )
assert all(origin.first_seen == origin.last_seen for origin in ret) assert all(origin.first_seen == origin.last_seen for origin in ret)
def test_record_listed_origins_with_duplicate(self, swh_scheduler, listed_origins):
# the duplicates must be in the same page to raise the "on conflict error"
listed_origins.insert(0, listed_origins[0])
ret = swh_scheduler.record_listed_origins(listed_origins)
# without the duplicate
assert len(ret) == len(listed_origins) - 1
def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins): def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins):
# First, insert `cutoff` origins # First, insert `cutoff` origins
cutoff = 100 cutoff = 100
assert cutoff < len(listed_origins) assert cutoff < len(listed_origins)
ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff]) ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff])
assert len(ret) == cutoff assert len(ret) == cutoff
▲ Show 20 Lines • Show All 801 Lines • Show Last 20 Lines
View Options

swh/journal/publisher.py

Show First 20 Lines • Show All 680 Lines • ▼ Show 20 Lines def test_record_listed_origins(self, swh_scheduler, listed_origins):
ret = swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.record_listed_origins(listed_origins)
assert set(returned.url for returned in ret) == set( assert set(returned.url for returned in ret) == set(
origin.url for origin in listed_origins origin.url for origin in listed_origins
) )
assert all(origin.first_seen == origin.last_seen for origin in ret) assert all(origin.first_seen == origin.last_seen for origin in ret)
def test_record_listed_origins_with_duplicate(self, swh_scheduler, listed_origins):
# the duplicates must be in the same page to raise the "on conflict error"
listed_origins.insert(0, listed_origins[0])
ret = swh_scheduler.record_listed_origins(listed_origins)
# without the duplicate
assert len(ret) == len(listed_origins) - 1
def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins): def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins):
# First, insert `cutoff` origins # First, insert `cutoff` origins
cutoff = 100 cutoff = 100
assert cutoff < len(listed_origins) assert cutoff < len(listed_origins)
ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff]) ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff])
assert len(ret) == cutoff assert len(ret) == cutoff
▲ Show 20 Lines • Show All 801 Lines • Show Last 20 Lines
View Options

swh/journal/tests/conftest.py

Show First 20 Lines • Show All 680 Lines • ▼ Show 20 Lines def test_record_listed_origins(self, swh_scheduler, listed_origins):
ret = swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.record_listed_origins(listed_origins)
assert set(returned.url for returned in ret) == set( assert set(returned.url for returned in ret) == set(
origin.url for origin in listed_origins origin.url for origin in listed_origins
) )
assert all(origin.first_seen == origin.last_seen for origin in ret) assert all(origin.first_seen == origin.last_seen for origin in ret)
def test_record_listed_origins_with_duplicate(self, swh_scheduler, listed_origins):
# the duplicates must be in the same page to raise the "on conflict error"
listed_origins.insert(0, listed_origins[0])
ret = swh_scheduler.record_listed_origins(listed_origins)
# without the duplicate
assert len(ret) == len(listed_origins) - 1
def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins): def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins):
# First, insert `cutoff` origins # First, insert `cutoff` origins
cutoff = 100 cutoff = 100
assert cutoff < len(listed_origins) assert cutoff < len(listed_origins)
ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff]) ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff])
assert len(ret) == cutoff assert len(ret) == cutoff
▲ Show 20 Lines • Show All 801 Lines • Show Last 20 Lines
View Options

swh/journal/tests/test_direct_writer.py

Show First 20 Lines • Show All 680 Lines • ▼ Show 20 Lines def test_record_listed_origins(self, swh_scheduler, listed_origins):
ret = swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.record_listed_origins(listed_origins)
assert set(returned.url for returned in ret) == set( assert set(returned.url for returned in ret) == set(
origin.url for origin in listed_origins origin.url for origin in listed_origins
) )
assert all(origin.first_seen == origin.last_seen for origin in ret) assert all(origin.first_seen == origin.last_seen for origin in ret)
def test_record_listed_origins_with_duplicate(self, swh_scheduler, listed_origins):
# the duplicates must be in the same page to raise the "on conflict error"
listed_origins.insert(0, listed_origins[0])
ret = swh_scheduler.record_listed_origins(listed_origins)
# without the duplicate
assert len(ret) == len(listed_origins) - 1
def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins): def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins):
# First, insert `cutoff` origins # First, insert `cutoff` origins
cutoff = 100 cutoff = 100
assert cutoff < len(listed_origins) assert cutoff < len(listed_origins)
ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff]) ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff])
assert len(ret) == cutoff assert len(ret) == cutoff
▲ Show 20 Lines • Show All 801 Lines • Show Last 20 Lines
View Options

swh/journal/tests/test_publisher_kafka.py

Show First 20 Lines • Show All 680 Lines • ▼ Show 20 Lines def test_record_listed_origins(self, swh_scheduler, listed_origins):
ret = swh_scheduler.record_listed_origins(listed_origins) ret = swh_scheduler.record_listed_origins(listed_origins)
assert set(returned.url for returned in ret) == set( assert set(returned.url for returned in ret) == set(
origin.url for origin in listed_origins origin.url for origin in listed_origins
) )
assert all(origin.first_seen == origin.last_seen for origin in ret) assert all(origin.first_seen == origin.last_seen for origin in ret)
def test_record_listed_origins_with_duplicate(self, swh_scheduler, listed_origins):
# the duplicates must be in the same page to raise the "on conflict error"
listed_origins.insert(0, listed_origins[0])
ret = swh_scheduler.record_listed_origins(listed_origins)
# without the duplicate
assert len(ret) == len(listed_origins) - 1
def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins): def test_record_listed_origins_upsert(self, swh_scheduler, listed_origins):
# First, insert `cutoff` origins # First, insert `cutoff` origins
cutoff = 100 cutoff = 100
assert cutoff < len(listed_origins) assert cutoff < len(listed_origins)
ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff]) ret = swh_scheduler.record_listed_origins(listed_origins[:cutoff])
assert len(ret) == cutoff assert len(ret) == cutoff
▲ Show 20 Lines • Show All 801 Lines • Show Last 20 Lines
View Options

swh/journal/tests/test_publisher_no_kafka.py

Loading...
About · Developer information · Legal