Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/test_cli.py
| Show All 36 Lines | tools = [ | ||||
| 'tool_configuration': {}, | 'tool_configuration': {}, | ||||
| } | } | ||||
| for i in range(2) | for i in range(2) | ||||
| ] | ] | ||||
| tools = idx_storage.indexer_configuration_add(tools) | tools = idx_storage.indexer_configuration_add(tools) | ||||
| origin_metadata = [ | origin_metadata = [ | ||||
| { | { | ||||
| 'id': origin_id, | 'id': 'file://dev/%04d' % origin_id, | ||||
| 'origin_url': 'file:///dev/zero', | |||||
| 'from_revision': hash_to_bytes('abcd{:0>4}'.format(origin_id)), | 'from_revision': hash_to_bytes('abcd{:0>4}'.format(origin_id)), | ||||
| 'indexer_configuration_id': tools[origin_id % 2]['id'], | 'indexer_configuration_id': tools[origin_id % 2]['id'], | ||||
| 'metadata': {'name': 'origin %d' % origin_id}, | 'metadata': {'name': 'origin %d' % origin_id}, | ||||
| 'mappings': ['mapping%d' % (origin_id % 10)] | 'mappings': ['mapping%d' % (origin_id % 10)] | ||||
| } | } | ||||
| for origin_id in range(nb_rows) | for origin_id in range(nb_rows) | ||||
| ] | ] | ||||
| revision_metadata = [ | revision_metadata = [ | ||||
| Show All 23 Lines | |||||
| def _assert_tasks_for_origins(tasks, origins): | def _assert_tasks_for_origins(tasks, origins): | ||||
| expected_kwargs = {"policy_update": "update-dups"} | expected_kwargs = {"policy_update": "update-dups"} | ||||
| assert {task['type'] for task in tasks} == {'index-origin-metadata'} | assert {task['type'] for task in tasks} == {'index-origin-metadata'} | ||||
| assert all(len(task['arguments']['args']) == 1 for task in tasks) | assert all(len(task['arguments']['args']) == 1 for task in tasks) | ||||
| for task in tasks: | for task in tasks: | ||||
| assert task['arguments']['kwargs'] == expected_kwargs, task | assert task['arguments']['kwargs'] == expected_kwargs, task | ||||
| assert _origins_in_task_args(tasks) == set(origins) | assert _origins_in_task_args(tasks) == set([ | ||||
| 'file://dev/%04d' % i for i in origins]) | |||||
| def invoke(scheduler, catch_exceptions, args): | def invoke(scheduler, catch_exceptions, args): | ||||
| runner = CliRunner() | runner = CliRunner() | ||||
| with patch('swh.indexer.cli.get_scheduler') as get_scheduler_mock, \ | with patch('swh.indexer.cli.get_scheduler') as get_scheduler_mock, \ | ||||
| tempfile.NamedTemporaryFile('a', suffix='.yml') as config_fd: | tempfile.NamedTemporaryFile('a', suffix='.yml') as config_fd: | ||||
| config_fd.write(CLI_CONFIG) | config_fd.write(CLI_CONFIG) | ||||
| config_fd.seek(0) | config_fd.seek(0) | ||||
| ▲ Show 20 Lines • Show All 225 Lines • ▼ Show 20 Lines | def test_origin_metadata_reindex_filter_one_tool( | ||||
| tasks = indexer_scheduler.search_tasks() | tasks = indexer_scheduler.search_tasks() | ||||
| assert len(tasks) == 6 | assert len(tasks) == 6 | ||||
| _assert_tasks_for_origins( | _assert_tasks_for_origins( | ||||
| tasks, | tasks, | ||||
| [x*2 for x in range(55)]) | [x*2 for x in range(55)]) | ||||
| def test_journal_client(storage, indexer_scheduler): | def test_journal_client(storage, indexer_scheduler): | ||||
| """Tests the re-indexing when origin_batch_size*task_batch_size is a | """Test the 'swh indexer journal-client' cli tool.""" | ||||
| divisor of nb_origins.""" | |||||
| message = FakeKafkaMessage('swh.journal.objects.origin_visit', 'bogus', { | message = FakeKafkaMessage('swh.journal.objects.origin_visit', 'bogus', { | ||||
| 'status': 'full', | 'status': 'full', | ||||
| 'origin': { | 'origin': { | ||||
| 'url': 'file:///dev/zero', | 'url': 'file://dev/0000', | ||||
| } | } | ||||
| }) | }) | ||||
| consumer = MockedKafkaConsumer([message]) | consumer = MockedKafkaConsumer([message]) | ||||
| with patch('swh.journal.client.Consumer', | with patch('swh.journal.client.Consumer', | ||||
| return_value=consumer): | return_value=consumer): | ||||
| result = invoke(indexer_scheduler, False, [ | result = invoke(indexer_scheduler, False, [ | ||||
| Show All 12 Lines | def test_journal_client(storage, indexer_scheduler): | ||||
| assert result.exit_code == 0, result.output | assert result.exit_code == 0, result.output | ||||
| assert result.output == expected_output | assert result.output == expected_output | ||||
| # Check scheduled tasks | # Check scheduled tasks | ||||
| tasks = indexer_scheduler.search_tasks() | tasks = indexer_scheduler.search_tasks() | ||||
| assert len(tasks) == 1 | assert len(tasks) == 1 | ||||
| _assert_tasks_for_origins( | _assert_tasks_for_origins( | ||||
| tasks, | tasks, | ||||
| ['file:///dev/zero']) | [0]) | ||||