Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/test_indexer.py
| Show First 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | class TrivialContentPartitionIndexer(ContentPartitionIndexer[str]): | ||||
| ) -> Iterable[Sha1]: | ) -> Iterable[Sha1]: | ||||
| return iter([b"excluded hash", b"other excluded hash"]) | return iter([b"excluded hash", b"other excluded hash"]) | ||||
| def persist_index_computations(self, results: List[str]) -> Dict[str, int]: | def persist_index_computations(self, results: List[str]) -> Dict[str, int]: | ||||
| self._results.append(results) # type: ignore | self._results.append(results) # type: ignore | ||||
| return {"nb_added": len(results)} | return {"nb_added": len(results)} | ||||
| def check_sentry(sentry_events): | def check_sentry(sentry_events, tags): | ||||
| assert len(sentry_events) == 1 | assert len(sentry_events) == 1 | ||||
| sentry_event = sentry_events.pop() | sentry_event = sentry_events.pop() | ||||
| assert sentry_event.get("tags") == tags | |||||
| assert "'_TestException'" in str(sentry_event) | assert "'_TestException'" in str(sentry_event) | ||||
| def test_content_indexer_catch_exceptions(sentry_events): | def test_content_indexer_catch_exceptions(sentry_events): | ||||
| indexer = CrashingContentIndexer(config=BASE_TEST_CONFIG) | indexer = CrashingContentIndexer(config=BASE_TEST_CONFIG) | ||||
| indexer.objstorage = Mock() | indexer.objstorage = Mock() | ||||
| indexer.objstorage.get.return_value = b"content" | indexer.objstorage.get.return_value = b"content" | ||||
| indexer.objstorage.get_batch.return_value = [b"content"] | indexer.objstorage.get_batch.return_value = [b"content"] | ||||
| sha1 = b"\x12" * 20 | sha1 = b"\x12" * 20 | ||||
| # As task, catching exceptions | # As task, catching exceptions | ||||
| assert indexer.run([sha1]) == {"status": "failed"} | assert indexer.run([sha1]) == {"status": "failed"} | ||||
| check_sentry(sentry_events) | check_sentry(sentry_events, {"swh-indexer-content-sha1": sha1.hex()}) | ||||
| # As journal client, catching exceptions | # As journal client, catching exceptions | ||||
| assert indexer.process_journal_objects({"content": [{"sha1": sha1}]}) == { | assert indexer.process_journal_objects({"content": [{"sha1": sha1}]}) == { | ||||
| "status": "failed" | "status": "failed" | ||||
| } | } | ||||
| check_sentry(sentry_events) | check_sentry(sentry_events, {"swh-indexer-content-sha1": sha1.hex()}) | ||||
| indexer.catch_exceptions = False | indexer.catch_exceptions = False | ||||
| # As task, not catching exceptions | # As task, not catching exceptions | ||||
| with pytest.raises(_TestException): | with pytest.raises(_TestException): | ||||
| indexer.run([sha1]) | indexer.run([sha1]) | ||||
| assert sentry_events == [] | assert sentry_events == [] | ||||
| # As journal client, not catching exceptions | # As journal client, not catching exceptions | ||||
| with pytest.raises(_TestException): | with pytest.raises(_TestException): | ||||
| assert indexer.process_journal_objects({"content": [{"sha1": sha1}]}) == { | assert indexer.process_journal_objects({"content": [{"sha1": sha1}]}) == { | ||||
| "status": "failed" | "status": "failed" | ||||
| } | } | ||||
| assert sentry_events == [] | assert sentry_events == [] | ||||
| def test_directory_indexer_catch_exceptions(sentry_events): | def test_directory_indexer_catch_exceptions(sentry_events): | ||||
| indexer = CrashingDirectoryIndexer(config=BASE_TEST_CONFIG) | indexer = CrashingDirectoryIndexer(config=BASE_TEST_CONFIG) | ||||
| indexer.storage = Mock() | indexer.storage = Mock() | ||||
| indexer.storage.directory_get.return_value = [DIRECTORY2] | indexer.storage.directory_get.return_value = [DIRECTORY2] | ||||
| sha1 = DIRECTORY2.id | sha1 = DIRECTORY2.id | ||||
| swhid = str(DIRECTORY2.swhid()) | |||||
| # As task, catching exceptions | # As task, catching exceptions | ||||
| assert indexer.run([sha1]) == {"status": "failed"} | assert indexer.run([sha1]) == {"status": "failed"} | ||||
| check_sentry(sentry_events) | check_sentry(sentry_events, {"swh-indexer-directory-swhid": swhid}) | ||||
| # As journal client, catching exceptions | # As journal client, catching exceptions | ||||
| assert indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]}) == { | assert indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]}) == { | ||||
| "status": "failed" | "status": "failed" | ||||
| } | } | ||||
| check_sentry(sentry_events) | check_sentry(sentry_events, {"swh-indexer-directory-swhid": swhid}) | ||||
| indexer.catch_exceptions = False | indexer.catch_exceptions = False | ||||
| # As task, not catching exceptions | # As task, not catching exceptions | ||||
| with pytest.raises(_TestException): | with pytest.raises(_TestException): | ||||
| indexer.run([b"foo"]) | indexer.run([b"foo"]) | ||||
| assert sentry_events == [] | assert sentry_events == [] | ||||
| # As journal client, not catching exceptions | # As journal client, not catching exceptions | ||||
| with pytest.raises(_TestException): | with pytest.raises(_TestException): | ||||
| indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]}) | indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]}) | ||||
| assert sentry_events == [] | assert sentry_events == [] | ||||
| def test_origin_indexer_catch_exceptions(sentry_events): | def test_origin_indexer_catch_exceptions(sentry_events): | ||||
| indexer = CrashingOriginIndexer(config=BASE_TEST_CONFIG) | indexer = CrashingOriginIndexer(config=BASE_TEST_CONFIG) | ||||
| origin_url = "http://example.org" | origin_url = "http://example.org" | ||||
| # As task, catching exceptions | # As task, catching exceptions | ||||
| assert indexer.run([origin_url]) == {"status": "failed"} | assert indexer.run([origin_url]) == {"status": "failed"} | ||||
| check_sentry(sentry_events) | check_sentry(sentry_events, {"swh-indexer-origin-url": origin_url}) | ||||
| # As journal client, catching exceptions | # As journal client, catching exceptions | ||||
| assert indexer.process_journal_objects({"origin": [{"url": origin_url}]}) == { | assert indexer.process_journal_objects({"origin": [{"url": origin_url}]}) == { | ||||
| "status": "failed" | "status": "failed" | ||||
| } | } | ||||
| check_sentry(sentry_events) | check_sentry(sentry_events, {"swh-indexer-origin-url": origin_url}) | ||||
| indexer.catch_exceptions = False | indexer.catch_exceptions = False | ||||
| # As task, not catching exceptions | # As task, not catching exceptions | ||||
| with pytest.raises(_TestException): | with pytest.raises(_TestException): | ||||
| indexer.run([origin_url]) | indexer.run([origin_url]) | ||||
| assert sentry_events == [] | assert sentry_events == [] | ||||
| ▲ Show 20 Lines • Show All 49 Lines • Show Last 20 Lines | |||||