Page MenuHomeSoftware Heritage
Differential D7023

Add missing maven lister task subscription to lister config
ClosedPublic
Actions

Authored by ardumont on Jan 24 2022, 11:58 AM.

Details

Reviewers
vsellier
Group Reviewers
Reviewers
Maniphest Tasks
T3746: staging: Deploy maven indexer/lister/loader
Commits
rDENV1f5c1393573b: Add missing maven lister task subscription to lister config
Summary

This actually triggers the listing now [1]

[1]

swh-lister_1                        | [2022-01-24 10:53:48,054: INFO/ForkPoolWorker-1] Downloading text index from http://swh-lister-maven-nginx/export.fld.
swh-lister_1                        | [2022-01-24 10:53:48,068: INFO/ForkPoolWorker-1] Found 2 poms.
swh-lister_1                        | [2022-01-24 10:53:48,068: INFO/ForkPoolWorker-1] Fetching poms..
swh-lister_1                        | [2022-01-24 10:53:48,068: INFO/ForkPoolWorker-1] Fetching URL https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.0/sprova4j-0.1.0.pom with params {}
swh-lister_1                        | [2022-01-24 10:53:48,206: INFO/ForkPoolWorker-1] Fetching URL https://repo1.maven.org/maven2/al/aldi/sprova4j/0.1.1/sprova4j-0.1.1.pom with params {}
swh-lister_1                        | [2022-01-24 10:53:48,310: INFO/ForkPoolWorker-1] Task swh.lister.maven.tasks.FullMavenLister[f88b25c8-7237-4887-948b-11371958b0cc] succeeded in 0.2693615449825302s: {'pages': 4, 'origins': 4}

Related to T3746

Test Plan
$ doco up
... (wait for it)
$ swh-doco exec swh-scheduler swh scheduler task add list-maven-full url=https://repo1.maven.org/maven2/ index_url=http://swh-lister-maven-nginx/export.fld

Diff Detail

Repository
rDENV Development environment
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 26289
Build 41100: arc lint + arc unit

Revision Contents
Changeset List

PathSize
docker/
conf/
1 line

Diff 25451

View Options

docker/conf/lister.yml

Show First 20 Lines • Show All 565 Lines • ▼ Show 20 Lines def test_content_add_collision(self):
cont1b = cont1.copy() cont1b = cont1.copy()
sha256_array = bytearray(cont1b['sha256']) sha256_array = bytearray(cont1b['sha256'])
sha256_array[0] += 1 sha256_array[0] += 1
cont1b['sha256'] = bytes(sha256_array) cont1b['sha256'] = bytes(sha256_array)
with self.assertRaises(HashCollision) as cm: with self.assertRaises(HashCollision) as cm:
self.storage.content_add([cont1, cont1b]) self.storage.content_add([cont1, cont1b])
self.assertEqual(cm.exception.args[0], 'sha1') self.assertIn(cm.exception.args[0], ['sha1', 'sha1_git', 'blake2s256'])
def test_skipped_content_add(self): def test_skipped_content_add(self):
cont = self.skipped_cont.copy() cont = self.skipped_cont.copy()
cont2 = self.skipped_cont2.copy() cont2 = self.skipped_cont2.copy()
cont2['blake2s256'] = None cont2['blake2s256'] = None
self.storage.content_add([cont, cont, cont2]) self.storage.content_add([cont, cont, cont2])
▲ Show 20 Lines • Show All 194 Lines • ▼ Show 20 Lines def test_revision_log(self):
self.storage.revision_add([self.revision3, self.storage.revision_add([self.revision3,
self.revision4]) self.revision4])
# when # when
actual_results = list(self.storage.revision_log( actual_results = list(self.storage.revision_log(
[self.revision4['id']])) [self.revision4['id']]))
# hack: ids generated # hack: ids generated
for actual_result in actual_results: for actual_result in actual_results:
if 'id' in actual_result['author']:
del actual_result['author']['id'] del actual_result['author']['id']
if 'id' in actual_result['committer']:
del actual_result['committer']['id'] del actual_result['committer']['id']
douarddaUnsubmitted
Not Done
Inline Actions

These hunks (in test_storage.py) should not be in this Diff. Having to do so in order to make your test_in_memory pass (I guess it's the reason for these modifications) prove your implementation does not respect the "API" of the original storage class.

It's probably no big deal here, but it's a bad practice...

douardda: These hunks (in test_storage.py) should not be in this Diff. Having to do so in order to make…
vlorentzAuthorUnsubmitted
Done
Inline Actions

comments like # hack: ids generated show that these IDs are quirks from the SQL backend, not features.

vlorentz: comments like `# hack: ids generated` show that these IDs are quirks from the SQL backend, not…
vlorentzAuthorUnsubmitted
Done
Inline Actions

Actually, according to Storage's docstrings, actual_result['author'] and actual_result['committer'] should not even exist.

vlorentz: Actually, according to Storage's docstrings, `actual_result['author']` and `actual_result…
self.assertEqual(len(actual_results), 2) # rev4 -child-> rev3 self.assertEqual(len(actual_results), 2) # rev4 -child-> rev3
self.assertEqual(actual_results[0], self.assertEqual(actual_results[0],
self.normalize_entity(self.revision4)) self.normalize_entity(self.revision4))
self.assertEqual(actual_results[1], self.assertEqual(actual_results[1],
self.normalize_entity(self.revision3)) self.normalize_entity(self.revision3))
def test_revision_log_with_limit(self): def test_revision_log_with_limit(self):
# given # given
# self.revision4 -is-child-of-> self.revision3 # self.revision4 -is-child-of-> self.revision3
self.storage.revision_add([self.revision3, self.storage.revision_add([self.revision3,
self.revision4]) self.revision4])
actual_results = list(self.storage.revision_log( actual_results = list(self.storage.revision_log(
[self.revision4['id']], 1)) [self.revision4['id']], 1))
# hack: ids generated # hack: ids generated
for actual_result in actual_results: for actual_result in actual_results:
if 'id' in actual_result['author']:
del actual_result['author']['id'] del actual_result['author']['id']
if 'id' in actual_result['committer']:
del actual_result['committer']['id'] del actual_result['committer']['id']
self.assertEqual(len(actual_results), 1) self.assertEqual(len(actual_results), 1)
self.assertEqual(actual_results[0], self.revision4) self.assertEqual(actual_results[0], self.revision4)
@staticmethod @staticmethod
def _short_revision(revision): def _short_revision(revision):
return [revision['id'], revision['parents']] return [revision['id'], revision['parents']]
Show All 27 Linesclass CommonTestStorage(TestStorageData):
def test_revision_get(self): def test_revision_get(self):
self.storage.revision_add([self.revision]) self.storage.revision_add([self.revision])
actual_revisions = list(self.storage.revision_get( actual_revisions = list(self.storage.revision_get(
[self.revision['id'], self.revision2['id']])) [self.revision['id'], self.revision2['id']]))
# when # when
if 'id' in actual_revisions[0]['author']:
del actual_revisions[0]['author']['id'] # hack: ids are generated del actual_revisions[0]['author']['id'] # hack: ids are generated
if 'id' in actual_revisions[0]['committer']:
del actual_revisions[0]['committer']['id'] del actual_revisions[0]['committer']['id']
self.assertEqual(len(actual_revisions), 2) self.assertEqual(len(actual_revisions), 2)
self.assertEqual(actual_revisions[0], self.assertEqual(actual_revisions[0],
self.normalize_entity(self.revision)) self.normalize_entity(self.revision))
self.assertIsNone(actual_revisions[1]) self.assertIsNone(actual_revisions[1])
def test_revision_get_no_parents(self): def test_revision_get_no_parents(self):
self.storage.revision_add([self.revision3]) self.storage.revision_add([self.revision3])
▲ Show 20 Lines • Show All 1,116 Lines • Show Last 20 Lines
About · Developer information · Legal