Differential D8216 Diff 29632 swh/indexer/tests/test_indexer.py

Changeset View

Standalone View

swh/indexer/tests/test_indexer.py

# Copyright (C) 2020 The Software Heritage developers		# Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from typing import Any, Dict, Iterable, List, Optional		from typing import Any, Dict, Iterable, List, Optional
from unittest.mock import Mock		from unittest.mock import Mock

import pytest		import pytest
		import sentry_sdk

from swh.indexer.indexer import (		from swh.indexer.indexer import (
ContentIndexer,		ContentIndexer,
ContentPartitionIndexer,		ContentPartitionIndexer,
DirectoryIndexer,		DirectoryIndexer,
OriginIndexer,		OriginIndexer,
)		)
from swh.indexer.storage import PagedResult, Sha1		from swh.indexer.storage import PagedResult, Sha1
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines	def test_content_indexer_catch_exceptions(sentry_events):

# As task, not catching exceptions		# As task, not catching exceptions
with pytest.raises(_TestException):		with pytest.raises(_TestException):
indexer.run([sha1])		indexer.run([sha1])
assert sentry_events == []		assert sentry_events == []

# As journal client, not catching exceptions		# As journal client, not catching exceptions
with pytest.raises(_TestException):		with pytest.raises(_TestException):
assert indexer.process_journal_objects({"content": [{"sha1": sha1}]}) == {		indexer.process_journal_objects({"content": [{"sha1": sha1}]})
"status": "failed"
}
assert sentry_events == []		assert sentry_events == []

		# As journal client, check the frontend will be able to get the tag when reporting
		try:
		indexer.process_journal_objects({"content": [{"sha1": sha1}]})
		except Exception:
		sentry_sdk.capture_exception()
		else:
		assert False
		check_sentry(sentry_events, {"swh-indexer-content-sha1": sha1.hex()})


def test_directory_indexer_catch_exceptions(sentry_events):		def test_directory_indexer_catch_exceptions(sentry_events):
indexer = CrashingDirectoryIndexer(config=BASE_TEST_CONFIG)		indexer = CrashingDirectoryIndexer(config=BASE_TEST_CONFIG)
indexer.storage = Mock()		indexer.storage = Mock()
indexer.storage.directory_get.return_value = [DIRECTORY2]		indexer.storage.directory_get.return_value = [DIRECTORY2]

sha1 = DIRECTORY2.id		sha1 = DIRECTORY2.id
swhid = str(DIRECTORY2.swhid())		swhid = str(DIRECTORY2.swhid())
Show All 15 Lines	with pytest.raises(_TestException):
indexer.run([b"foo"])		indexer.run([b"foo"])
assert sentry_events == []		assert sentry_events == []

# As journal client, not catching exceptions		# As journal client, not catching exceptions
with pytest.raises(_TestException):		with pytest.raises(_TestException):
indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]})		indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]})
assert sentry_events == []		assert sentry_events == []

		# As journal client, check the frontend will be able to get the tag when reporting
		try:
		indexer.process_journal_objects({"directory": [DIRECTORY2.to_dict()]})
		except Exception:
		sentry_sdk.capture_exception()
		else:
		assert False
		check_sentry(sentry_events, {"swh-indexer-directory-swhid": swhid})


def test_origin_indexer_catch_exceptions(sentry_events):		def test_origin_indexer_catch_exceptions(sentry_events):
indexer = CrashingOriginIndexer(config=BASE_TEST_CONFIG)		indexer = CrashingOriginIndexer(config=BASE_TEST_CONFIG)

origin_url = "http://example.org"		origin_url = "http://example.org"

# As task, catching exceptions		# As task, catching exceptions
assert indexer.run([origin_url]) == {"status": "failed"}		assert indexer.run([origin_url]) == {"status": "failed"}
Show All 12 Lines	with pytest.raises(_TestException):
indexer.run([origin_url])		indexer.run([origin_url])
assert sentry_events == []		assert sentry_events == []

# As journal client, not catching exceptions		# As journal client, not catching exceptions
with pytest.raises(_TestException):		with pytest.raises(_TestException):
indexer.process_journal_objects({"origin": [{"url": origin_url}]})		indexer.process_journal_objects({"origin": [{"url": origin_url}]})
assert sentry_events == []		assert sentry_events == []

		# As journal client, check the frontend will be able to get the tag when reporting
		try:
		indexer.process_journal_objects({"origin": [{"url": origin_url}]})
		except Exception:
		sentry_sdk.capture_exception()
		else:
		assert False
		check_sentry(sentry_events, {"swh-indexer-origin-url": origin_url})


def test_content_partition_indexer_catch_exceptions():		def test_content_partition_indexer_catch_exceptions():
indexer = CrashingContentPartitionIndexer(		indexer = CrashingContentPartitionIndexer(
config={**BASE_TEST_CONFIG, "write_batch_size": 42}		config={**BASE_TEST_CONFIG, "write_batch_size": 42}
)		)

assert indexer.run(0, 42) == {"status": "failed"}		assert indexer.run(0, 42) == {"status": "failed"}

Show All 36 Lines