Changeset View
Changeset View
Standalone View
Standalone View
swh/journal/tests/test_backfill.py
- This file was added.
# Copyright (C) 2019 The Software Heritage developers | |||||
# See the AUTHORS file at the top-level directory of this distribution | |||||
# License: GNU General Public License version 3, or any later version | |||||
# See top-level LICENSE file for more information | |||||
import pytest | |||||
from swh.journal.backfill import ( | |||||
JournalBackfiller, compute_query, PARTITION_KEY | |||||
) | |||||
TEST_CONFIG = { | |||||
'brokers': ['localhost'], | |||||
'final_prefix': 'swh.tmp_journal.new', | |||||
'client_id': 'swh.journal.client.test', | |||||
'storage_dbconn': 'service=swh-dev', | |||||
} | |||||
def test_config_ko_missing_mandatory_key(): | |||||
"""Missing configuration key will make the initialization fail | |||||
""" | |||||
for key in TEST_CONFIG.keys(): | |||||
config = TEST_CONFIG.copy() | |||||
config.pop(key) | |||||
with pytest.raises(ValueError) as e: | |||||
JournalBackfiller(config) | |||||
error = ('Configuration error: The following keys must be' | |||||
' provided: %s' % (','.join([key]), )) | |||||
assert e.value.args[0] == error | |||||
def test_config_ko_unknown_object_type(): | |||||
"""Parse arguments will fail if the object type is unknown | |||||
""" | |||||
backfiller = JournalBackfiller(TEST_CONFIG) | |||||
with pytest.raises(ValueError) as e: | |||||
backfiller.parse_arguments('unknown-object-type', 1, 2) | |||||
error = ('Object type unknown-object-type is not supported. ' | |||||
'The only possible values are %s' % ( | |||||
', '.join(PARTITION_KEY))) | |||||
assert e.value.args[0] == error | |||||
def test_compute_query_content(): | |||||
query, where_args, column_aliases = compute_query( | |||||
'content', '\x000000', '\x000001') | |||||
assert where_args == ['\x000000', '\x000001'] | |||||
assert column_aliases == [ | |||||
'sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'status', | |||||
'ctime' | |||||
] | |||||
assert query == ''' | |||||
select sha1,sha1_git,sha256,blake2s256,length,status,ctime | |||||
from content | |||||
where (sha1) >= %s and (sha1) < %s | |||||
''' | |||||
def test_compute_query_skipped_content(): | |||||
query, where_args, column_aliases = compute_query( | |||||
'skipped_content', None, None) | |||||
assert where_args == [] | |||||
assert column_aliases == [ | |||||
'sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'ctime', | |||||
'status', 'reason', | |||||
] | |||||
assert query == ''' | |||||
select sha1,sha1_git,sha256,blake2s256,length,ctime,status,reason | |||||
from skipped_content | |||||
''' | |||||
def test_compute_query_origin_visit(): | |||||
query, where_args, column_aliases = compute_query( | |||||
'origin_visit', 1, 10) | |||||
assert where_args == [1, 10] | |||||
assert column_aliases == [ | |||||
'type', 'url', 'date', 'snapshot', 'status' | |||||
] | |||||
assert query == ''' | |||||
select type,url,date,snapshot,status | |||||
from origin_visit | |||||
left join origin on origin_visit.origin=origin.id | |||||
where (origin_visit.origin) >= %s and (origin_visit.origin) < %s | |||||
''' | |||||
def test_compute_query_release(): | |||||
query, where_args, column_aliases = compute_query( | |||||
'release', '\x000002', '\x000003') | |||||
assert where_args == ['\x000002', '\x000003'] | |||||
assert column_aliases == [ | |||||
'id', 'date', 'date_offset', 'comment', 'name', 'synthetic', | |||||
'date_neg_utc_offset', 'target', 'target_type', 'author_id', | |||||
'author_name', 'author_email', 'author_fullname'] | |||||
assert query == ''' | |||||
select release.id as id,date,date_offset,comment,release.name as name,synthetic,date_neg_utc_offset,target,target_type,a.id as author_id,a.name as author_name,a.email as author_email,a.fullname as author_fullname | |||||
from release | |||||
left join person a on release.author=a.id | |||||
where (release.id) >= %s and (release.id) < %s | |||||
''' # noqa |