diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -7,6 +7,8 @@ import pytest from requests.utils import parse_header_links +from swh.model.model import Origin + from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api.utils import enrich_origin_visit, enrich_origin @@ -107,23 +109,24 @@ @given(new_origin(), visit_dates(3), new_snapshots(3)) def test_api_lookup_origin_visits(api_client, archive_data, new_origin, visit_dates, new_snapshots): + archive_data.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin['url'], visit_date, type='git') + new_origin.url, visit_date, type='git') archive_data.snapshot_add([new_snapshots[i]]) archive_data.origin_visit_update( - new_origin['url'], origin_visit['visit'], - snapshot=new_snapshots[i]['id']) + new_origin.url, origin_visit['visit'], + snapshot=new_snapshots[i].id) - all_visits = list(reversed(get_origin_visits(new_origin))) + all_visits = list(reversed(get_origin_visits(new_origin.to_dict()))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:])): url = reverse('api-1-origin-visits', - url_args={'origin_url': new_origin['url']}, + url_args={'origin_url': new_origin.url}, query_params={'per_page': 2, 'last_visit': last_visit}) @@ -146,20 +149,20 @@ archive_data.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin['url'], visit_date, type='git') + new_origin.url, visit_date, type='git') archive_data.snapshot_add([new_snapshots[i]]) archive_data.origin_visit_update( - new_origin['url'], origin_visit['visit'], - snapshot=new_snapshots[i]['id']) + new_origin.url, origin_visit['visit'], + snapshot=new_snapshots[i].id) - all_visits = list(reversed(get_origin_visits(new_origin))) + all_visits = list(reversed(get_origin_visits(new_origin.to_dict()))) for last_visit, expected_visits in ( (None, all_visits[:2]), (all_visits[1]['visit'], all_visits[2:4])): url = reverse('api-1-origin-visits', - url_args={'origin_url': new_origin['url']}, + url_args={'origin_url': new_origin.url}, query_params={'per_page': 2, 'last_visit': last_visit}) @@ -182,14 +185,14 @@ archive_data.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin['url'], visit_date, type='git') + new_origin.url, visit_date, type='git') visit_id = origin_visit['visit'] archive_data.snapshot_add([new_snapshots[i]]) archive_data.origin_visit_update( - new_origin['url'], origin_visit['visit'], - snapshot=new_snapshots[i]['id']) + new_origin.url, origin_visit['visit'], + snapshot=new_snapshots[i].id) url = reverse('api-1-origin-visit', - url_args={'origin_url': new_origin['url'], + url_args={'origin_url': new_origin.url, 'visit_id': visit_id}) rv = api_client.get(url) @@ -197,7 +200,7 @@ assert rv['Content-Type'] == 'application/json' expected_visit = archive_data.origin_visit_get_by( - new_origin['url'], visit_id) + new_origin.url, visit_id) expected_visit = enrich_origin_visit( expected_visit, with_origin_link=True, @@ -212,13 +215,13 @@ archive_data.origin_add_one(new_origin) url = reverse('api-1-origin-visit-latest', - url_args={'origin_url': new_origin['url']}) + url_args={'origin_url': new_origin.url}) rv = api_client.get(url) assert rv.status_code == 404, rv.data assert rv.data == { 'exception': 'NotFoundExc', - 'reason': 'No visit for origin %s found' % new_origin['url'] + 'reason': 'No visit for origin %s found' % new_origin.url } @@ -230,16 +233,16 @@ visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin['url'], visit_date, type='git') + new_origin.url, visit_date, type='git') visit_ids.append(origin_visit['visit']) archive_data.snapshot_add([new_snapshots[0]]) archive_data.origin_visit_update( - new_origin['url'], visit_ids[0], - snapshot=new_snapshots[0]['id']) + new_origin.url, visit_ids[0], + snapshot=new_snapshots[0].id) url = reverse('api-1-origin-visit-latest', - url_args={'origin_url': new_origin['url']}) + url_args={'origin_url': new_origin.url}) rv = api_client.get(url) @@ -247,7 +250,7 @@ assert rv['Content-Type'] == 'application/json' expected_visit = archive_data.origin_visit_get_by( - new_origin['url'], visit_ids[1]) + new_origin.url, visit_ids[1]) expected_visit = enrich_origin_visit( expected_visit, with_origin_link=True, @@ -265,16 +268,16 @@ visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin['url'], visit_date, type='git') + new_origin.url, visit_date, type='git') visit_ids.append(origin_visit['visit']) archive_data.snapshot_add([new_snapshots[0]]) archive_data.origin_visit_update( - new_origin['url'], visit_ids[0], - snapshot=new_snapshots[0]['id']) + new_origin.url, visit_ids[0], + snapshot=new_snapshots[0].id) url = reverse('api-1-origin-visit-latest', - url_args={'origin_url': new_origin['url']}, + url_args={'origin_url': new_origin.url}, query_params={'require_snapshot': True}) rv = api_client.get(url) @@ -283,7 +286,7 @@ assert rv['Content-Type'] == 'application/json' expected_visit = archive_data.origin_visit_get_by( - new_origin['url'], visit_ids[0]) + new_origin.url, visit_ids[0]) expected_visit = enrich_origin_visit( expected_visit, with_origin_link=True, @@ -379,14 +382,14 @@ def test_api_origin_not_found(api_client, new_origin): url = reverse('api-1-origin', - url_args={'origin_url': new_origin['url']}) + url_args={'origin_url': new_origin.url}) rv = api_client.get(url) assert rv.status_code == 404, rv.data assert rv['Content-Type'] == 'application/json' assert rv.data == { 'exception': 'NotFoundExc', - 'reason': 'Origin with url %s not found!' % new_origin['url'] + 'reason': 'Origin with url %s not found!' % new_origin.url } @@ -514,7 +517,7 @@ mocker.patch('swh.web.common.service.search', None) archive_data.origin_add([ - {'url': 'http://foobar/{}'.format(i)} + Origin(url='http://foobar/{}'.format(i)) for i in range(2000) ]) diff --git a/swh/web/tests/api/views/test_release.py b/swh/web/tests/api/views/test_release.py --- a/swh/web/tests/api/views/test_release.py +++ b/swh/web/tests/api/views/test_release.py @@ -6,11 +6,12 @@ from datetime import datetime from hypothesis import given -from swh.model.hashutil import hash_to_bytes +from swh.model.hashutil import hash_to_bytes, hash_to_hex +from swh.model.model import Person, Release, TimestampWithTimezone from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 from swh.web.tests.strategies import ( - release, sha1, content, directory + release, content, directory ) @@ -32,45 +33,43 @@ assert rv.data == expected_release -@given(sha1(), sha1(), sha1(), content(), directory(), release()) +@given(content(), directory(), release()) def test_api_release_target_type_not_a_revision(api_client, archive_data, - new_rel1, new_rel2, - new_rel3, content, - directory, release): - for new_rel_id, target_type, target in ( - (new_rel1, 'content', content), - (new_rel2, 'directory', directory), - (new_rel3, 'release', release)): + content, directory, release): + for target_type, target in (('content', content), ('directory', directory), + ('release', release)): if target_type == 'content': target = target['sha1_git'] - sample_release = { - 'author': { - 'email': b'author@company.org', - 'fullname': b'author ', - 'name': b'author' - }, - 'date': { - 'timestamp': int(datetime.now().timestamp()), - 'offset': 0, - 'negative_utc': False, - }, - 'id': hash_to_bytes(new_rel_id), - 'message': b'sample release message', - 'name': b'sample release', - 'synthetic': False, - 'target': hash_to_bytes(target), - 'target_type': target_type - } + sample_release = Release( + author=Person( + email=b'author@company.org', + fullname=b'author ', + name=b'author' + ), + date=TimestampWithTimezone( + timestamp=int(datetime.now().timestamp()), + offset=0, + negative_utc=False, + ), + message=b'sample release message', + name=b'sample release', + synthetic=False, + target=hash_to_bytes(target), + target_type=target_type + ) archive_data.release_add([sample_release]) - url = reverse('api-1-release', url_args={'sha1_git': new_rel_id}) + new_release_id = hash_to_hex(sample_release.id) + + url = reverse('api-1-release', + url_args={'sha1_git': new_release_id}) rv = api_client.get(url) - expected_release = archive_data.release_get(new_rel_id) + expected_release = archive_data.release_get(new_release_id) if target_type == 'content': url_args = {'q': 'sha1_git:%s' % target} diff --git a/swh/web/tests/api/views/test_snapshot.py b/swh/web/tests/api/views/test_snapshot.py --- a/swh/web/tests/api/views/test_snapshot.py +++ b/swh/web/tests/api/views/test_snapshot.py @@ -8,6 +8,7 @@ from hypothesis import given from swh.model.hashutil import hash_to_hex +from swh.model.model import Snapshot from swh.web.api.utils import enrich_snapshot from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 @@ -155,7 +156,7 @@ for branch in snp_dict['branches'].keys(): snp_dict['branches'][branch] = None break - archive_data.snapshot_add([snp_dict]) + archive_data.snapshot_add([Snapshot.from_dict(snp_dict)]) url = reverse('api-1-snapshot', url_args={'snapshot_id': snp_id}) rv = api_client.get(url) diff --git a/swh/web/tests/browse/views/test_origin.py b/swh/web/tests/browse/views/test_origin.py --- a/swh/web/tests/browse/views/test_origin.py +++ b/swh/web/tests/browse/views/test_origin.py @@ -13,6 +13,7 @@ from hypothesis import given from swh.model.hashutil import hash_to_bytes +from swh.model.model import Snapshot from swh.web.browse.utils import process_snapshot_branches from swh.web.common.exc import NotFoundExc from swh.web.common.utils import ( @@ -268,7 +269,7 @@ 'target': hash_to_bytes(revisions[i-1]), } - archive_data.snapshot_add([snp_dict]) + archive_data.snapshot_add([Snapshot.from_dict(snp_dict)]) visit = archive_data.origin_visit_add( new_origin['url'], visit_dates[0], type='git') archive_data.origin_visit_update(new_origin['url'], visit['visit'], @@ -293,7 +294,7 @@ 'target': hash_to_bytes(revisions[i]), } - archive_data.snapshot_add([snp_dict]) + archive_data.snapshot_add([Snapshot.from_dict(snp_dict)]) visit = archive_data.origin_visit_add( new_origin['url'], visit_dates[0], type='git') archive_data.origin_visit_update(new_origin['url'], visit['visit'], diff --git a/swh/web/tests/browse/views/test_revision.py b/swh/web/tests/browse/views/test_revision.py --- a/swh/web/tests/browse/views/test_revision.py +++ b/swh/web/tests/browse/views/test_revision.py @@ -222,7 +222,7 @@ url = reverse('browse-revision', url_args={'sha1_git': revision}, - query_params={'origin': new_origin['url']}) + query_params={'origin': new_origin.url}) resp = client.get(url) assert resp.status_code == 404 diff --git a/swh/web/tests/common/test_service.py b/swh/web/tests/common/test_service.py --- a/swh/web/tests/common/test_service.py +++ b/swh/web/tests/common/test_service.py @@ -15,6 +15,7 @@ from swh.model.identifiers import ( CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT ) +from swh.model.model import Directory, DirectoryEntry, Origin, Revision from swh.web.common import service from swh.web.common.exc import BadInputExc, NotFoundExc @@ -187,14 +188,14 @@ archive_data.origin_add_one(new_origin) for ts in visit_dates: archive_data.origin_visit_add( - new_origin['url'], ts, type='git') + new_origin.url, ts, type='git') actual_origin_visits = list( - service.lookup_origin_visits(new_origin['url'], per_page=100)) + service.lookup_origin_visits(new_origin.url, per_page=100)) - expected_visits = archive_data.origin_visit_get(new_origin['url']) + expected_visits = archive_data.origin_visit_get(new_origin.url) for expected_visit in expected_visits: - expected_visit['origin'] = new_origin['url'] + expected_visit['origin'] = new_origin.url assert actual_origin_visits == expected_visits @@ -205,14 +206,14 @@ visits = [] for ts in visit_dates: visits.append(archive_data.origin_visit_add( - new_origin['url'], ts, type='git')) + new_origin.url, ts, type='git')) visit = random.choice(visits)['visit'] actual_origin_visit = service.lookup_origin_visit( - new_origin['url'], visit) + new_origin.url, visit) expected_visit = dict(archive_data.origin_visit_get_by( - new_origin['url'], visit)) + new_origin.url, visit)) assert actual_origin_visit == expected_visit @@ -221,9 +222,9 @@ def test_lookup_origin(archive_data, new_origin): archive_data.origin_add_one(new_origin) - actual_origin = service.lookup_origin({'url': new_origin['url']}) + actual_origin = service.lookup_origin({'url': new_origin.url}) expected_origin = archive_data.origin_get( - {'url': new_origin['url']}) + {'url': new_origin.url}) assert actual_origin == expected_origin @@ -345,59 +346,37 @@ assert e.match('Revision %s not found' % unknown_revision_) -def test_lookup_directory_with_revision_unknown_content(archive_data): +@given(new_revision()) +def test_lookup_directory_with_revision_unknown_content(archive_data, + new_revision): unknown_content_ = random_content() - unknown_revision_ = random_sha1() - unknown_directory_ = random_sha1() dir_path = 'README.md' + + # A directory that points to unknown content + dir = Directory(entries=[ + DirectoryEntry( + name=bytes(dir_path.encode('utf-8')), + type='file', + target=hash_to_bytes(unknown_content_['sha1_git']), + perms=DentryPerms.content + ) + ]) + # Create a revision that points to a directory # Which points to unknown content - revision = { - 'author': { - 'name': b'abcd', - 'email': b'abcd@company.org', - 'fullname': b'abcd abcd' - }, - 'committer': { - 'email': b'aaaa@company.org', - 'fullname': b'aaaa aaa', - 'name': b'aaa' - }, - 'committer_date': { - 'negative_utc': False, - 'offset': 0, - 'timestamp': 1437511651 - }, - 'date': { - 'negative_utc': False, - 'offset': 0, - 'timestamp': 1437511651 - }, - 'message': b'bleh', - 'metadata': [], - 'parents': [], - 'synthetic': False, - 'type': 'git', - 'id': hash_to_bytes(unknown_revision_), - 'directory': hash_to_bytes(unknown_directory_) - } - # A directory that points to unknown content - dir = { - 'id': hash_to_bytes(unknown_directory_), - 'entries': [{ - 'name': bytes(dir_path.encode('utf-8')), - 'type': 'file', - 'target': hash_to_bytes(unknown_content_['sha1_git']), - 'perms': DentryPerms.content - }] - } + new_revision = new_revision.to_dict() + new_revision['directory'] = dir.id + del new_revision['id'] + new_revision = Revision.from_dict(new_revision) + # Add the directory and revision in mem archive_data.directory_add([dir]) - archive_data.revision_add([revision]) + archive_data.revision_add([new_revision]) + new_revision_id = hash_to_hex(new_revision.id) with pytest.raises(NotFoundExc) as e: - service.lookup_directory_with_revision(unknown_revision_, dir_path) - assert e.match('Content not found for revision %s' % unknown_revision_) + service.lookup_directory_with_revision(new_revision_id, dir_path) + assert e.match('Content not found for revision %s' % new_revision_id) @given(revision()) @@ -500,8 +479,9 @@ @given(new_revision()) def test_lookup_revision_invalid_msg(archive_data, new_revision): + new_revision = new_revision.to_dict() new_revision['message'] = b'elegant fix for bug \xff' - archive_data.revision_add([new_revision]) + archive_data.revision_add([Revision.from_dict(new_revision)]) revision = service.lookup_revision(hash_to_hex(new_revision['id'])) assert revision['message'] is None @@ -513,9 +493,9 @@ archive_data.revision_add([new_revision]) revision_message = service.lookup_revision_message( - hash_to_hex(new_revision['id'])) + hash_to_hex(new_revision.id)) - assert revision_message == {'message': new_revision['message']} + assert revision_message == {'message': new_revision.message} def test_lookup_revision_msg_no_rev(): @@ -925,7 +905,7 @@ def test_lookup_origin_missing_trailing_slash(archive_data): - deb_origin = {'url': 'http://snapshot.debian.org/package/r-base/'} + deb_origin = Origin(url='http://snapshot.debian.org/package/r-base/') archive_data.origin_add_one(deb_origin) - origin_info = service.lookup_origin({'url': deb_origin['url'][:-1]}) - assert origin_info['url'] == deb_origin['url'] + origin_info = service.lookup_origin({'url': deb_origin.url[:-1]}) + assert origin_info['url'] == deb_origin.url diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -16,9 +16,9 @@ from swh.indexer.mimetype import MimetypeIndexer from swh.indexer.ctags import CtagsIndexer from swh.indexer.storage import get_indexer_storage -from swh.model.from_disk import Content, Directory -from swh.model.hashutil import hash_to_hex, hash_to_bytes, DEFAULT_ALGORITHMS -from swh.model.identifiers import directory_identifier +from swh.model import from_disk +from swh.model.hashutil import hash_to_hex, DEFAULT_ALGORITHMS +from swh.model.model import Content, Directory, Origin from swh.loader.git.from_disk import GitLoaderFromArchive from swh.search import get_search from swh.storage.algos.dir_iterators import dir_iterator @@ -35,10 +35,7 @@ # Configuration for git loader _TEST_LOADER_CONFIG = { 'storage': { - 'cls': 'validate', - 'storage': { - 'cls': 'memory' - } + 'cls': 'memory', }, 'save_data': False, 'max_content_size': 100 * 1024 * 1024, @@ -190,7 +187,8 @@ for i in range(250): url = 'https://many.origins/%d' % (i+1) - storage.origin_add([{'url': url}]) + # storage.origin_add([{'url': url}]) + storage.origin_add([Origin(url=url)]) search.origin_update([{'url': url, 'has_visits': True}]) visit = storage.origin_visit_add(url, '2019-12-03 13:55:05', 'tar') storage.origin_visit_update( @@ -260,9 +258,7 @@ idx_storage = get_indexer_storage('memory', {}) # Add the empty directory to the test archive - empty_dir_id = directory_identifier({'entries': []}) - empty_dir_id_bin = hash_to_bytes(empty_dir_id) - storage.directory_add([{'id': empty_dir_id_bin, 'entries': []}]) + storage.directory_add([Directory(entries=[])]) # Return tests data return { @@ -367,11 +363,11 @@ """ test_contents_dir = os.path.join( os.path.dirname(__file__), data_path).encode('utf-8') - directory = Directory.from_disk(path=test_contents_dir) + directory = from_disk.Directory.from_disk(path=test_contents_dir) contents = [] for name, obj in directory.items(): - if isinstance(obj, Content): + if isinstance(obj, from_disk.Content): c = obj.to_model().with_data().to_dict() c['status'] = 'visible' sha1 = hash_to_hex(c['sha1']) @@ -384,7 +380,7 @@ language = get_hljs_language_from_filename(filename) data_dict[key] = {'sha1': sha1, 'language': language} - contents.append(c) + contents.append(Content.from_dict(c)) storage = get_tests_data()['storage'] storage.content_add(contents) diff --git a/swh/web/tests/misc/test_badges.py b/swh/web/tests/misc/test_badges.py --- a/swh/web/tests/misc/test_badges.py +++ b/swh/web/tests/misc/test_badges.py @@ -60,7 +60,7 @@ for object_type, object_id in ( (CONTENT, unknown_content['sha1_git']), (DIRECTORY, unknown_directory), - (ORIGIN, new_origin['url']), + (ORIGIN, new_origin.url), (RELEASE, unknown_release), (REVISION, unknown_revision), (SNAPSHOT, unknown_snapshot) diff --git a/swh/web/tests/strategies.py b/swh/web/tests/strategies.py --- a/swh/web/tests/strategies.py +++ b/swh/web/tests/strategies.py @@ -16,6 +16,7 @@ from swh.model.hashutil import hash_to_hex, hash_to_bytes from swh.model.identifiers import directory_identifier +from swh.model.model import Person, Revision, TimestampWithTimezone from swh.storage.algos.revisions_walker import get_revisions_walker from swh.model.hypothesis_strategies import ( origins as new_origin_strategy, snapshots as new_snapshot @@ -255,9 +256,9 @@ Hypothesis strategy returning a random origin not ingested into the test archive. """ - return new_origin_strategy().map(lambda origin: origin.to_dict()).filter( + return new_origin_strategy().filter( lambda origin: get_tests_data()['storage'].origin_get( - [origin])[0] is None) + [origin.to_dict()])[0] is None) def new_origins(nb_origins=None): @@ -326,11 +327,11 @@ name = draw(text(min_size=5, max_size=30, alphabet=characters(min_codepoint=0, max_codepoint=255))) email = '%s@company.org' % name - return { - 'name': name.encode(), - 'email': email.encode(), - 'fullname': ('%s <%s>' % (name, email)).encode() - } + return Person( + name=name.encode(), + email=email.encode(), + fullname=('%s <%s>' % (name, email)).encode() + ) @composite @@ -355,20 +356,18 @@ Hypothesis strategy returning random raw swh revision data not ingested into the test archive. """ - return { - 'id': draw(unknown_revision().map(hash_to_bytes)), - 'directory': draw(sha1().map(hash_to_bytes)), - 'author': draw(new_person()), - 'committer': draw(new_person()), - 'message': draw( + return Revision( + directory=draw(sha1().map(hash_to_bytes)), + author=draw(new_person()), + committer=draw(new_person()), + message=draw( text(min_size=20, max_size=100).map(lambda t: t.encode())), - 'date': draw(new_swh_date()), - 'committer_date': draw(new_swh_date()), - 'synthetic': False, - 'type': 'git', - 'parents': [], - 'metadata': [], - } + date=TimestampWithTimezone.from_datetime(draw(new_swh_date())), + committer_date=TimestampWithTimezone.from_datetime( + draw(new_swh_date())), + synthetic=False, + type='git', + ) def revisions(min_size=2, max_size=8): @@ -398,8 +397,7 @@ def new_snapshots(nb_snapshots=None): min_size = nb_snapshots if nb_snapshots else 2 max_size = nb_snapshots if nb_snapshots else 8 - return lists(new_snapshot(min_size=2, max_size=10, only_objects=True) - .map(lambda snp: snp.to_dict()), + return lists(new_snapshot(min_size=2, max_size=10, only_objects=True), min_size=min_size, max_size=max_size)