diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ *.egg-info version.txt .vscode/ +/.tox/ diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1 @@ -nose +pytest diff --git a/swh/storage/tests/test_converters.py b/swh/storage/tests/test_converters.py --- a/swh/storage/tests/test_converters.py +++ b/swh/storage/tests/test_converters.py @@ -5,12 +5,9 @@ import unittest -from nose.plugins.attrib import attr - from swh.storage import converters -@attr('!db') class TestConverters(unittest.TestCase): def setUp(self): self.maxDiff = None @@ -21,7 +18,7 @@ 1, b'fullname', b'name', b'email') # then - self.assertEquals(actual_author, { + self.assertEqual(actual_author, { 'id': 1, 'fullname': b'fullname', 'name': b'name', @@ -55,7 +52,7 @@ }) # then - self.assertEquals(actual_revision, { + self.assertEqual(actual_revision, { 'id': 'revision-id', 'author': { 'id': 'auth-id', @@ -98,7 +95,7 @@ }) # then - self.assertEquals(actual_release, { + self.assertEqual(actual_release, { 'author': { 'id': 'auth-id', 'fullname': b'auth-fullname', @@ -122,4 +119,4 @@ db_data = converters.git_headers_to_db(raw_data) loop = converters.db_to_git_headers(db_data) - self.assertEquals(raw_data, loop) + self.assertEqual(raw_data, loop) diff --git a/swh/storage/tests/test_db.py b/swh/storage/tests/test_db.py --- a/swh/storage/tests/test_db.py +++ b/swh/storage/tests/test_db.py @@ -6,7 +6,7 @@ import os import unittest -from nose.plugins.attrib import attr +import pytest from swh.core.tests.db_testing import SingleDbTestFixture from swh.model.hashutil import hash_to_bytes @@ -14,7 +14,7 @@ from . import SQL_DIR -@attr('db') +@pytest.mark.db class TestDb(SingleDbTestFixture, unittest.TestCase): TEST_DB_NAME = 'softwareheritage-test-storage' TEST_DB_DUMP = os.path.join(SQL_DIR, '*.sql') diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -11,15 +11,14 @@ from unittest.mock import Mock, patch import psycopg2 -from nose.plugins.attrib import attr +import pytest -from swh.core.tests.db_testing import DbTestFixture from swh.model import from_disk, identifiers from swh.model.hashutil import hash_to_bytes from swh.storage.tests.storage_testing import StorageTestFixture -@attr('db') +@pytest.mark.db class BaseTestStorage(StorageTestFixture): def setUp(self): super().setUp() @@ -582,7 +581,7 @@ datums = self.cursor.fetchall() - self.assertEquals(2, len(datums)) + self.assertEqual(2, len(datums)) datum = datums[0] self.assertEqual( (datum[0].tobytes(), datum[1].tobytes(), datum[2].tobytes(), @@ -783,10 +782,10 @@ del actual_result['committer']['id'] self.assertEqual(len(actual_results), 2) # rev4 -child-> rev3 - self.assertEquals(actual_results[0], - self.normalize_entity(self.revision4)) - self.assertEquals(actual_results[1], - self.normalize_entity(self.revision3)) + self.assertEqual(actual_results[0], + self.normalize_entity(self.revision4)) + self.assertEqual(actual_results[1], + self.normalize_entity(self.revision3)) def test_revision_log_with_limit(self): # given @@ -802,7 +801,7 @@ del actual_result['committer']['id'] self.assertEqual(len(actual_results), 1) - self.assertEquals(actual_results[0], self.revision4) + self.assertEqual(actual_results[0], self.revision4) @staticmethod def _short_revision(revision): @@ -819,10 +818,10 @@ [self.revision4['id']])) self.assertEqual(len(actual_results), 2) # rev4 -child-> rev3 - self.assertEquals(list(actual_results[0]), - self._short_revision(self.revision4)) - self.assertEquals(list(actual_results[1]), - self._short_revision(self.revision3)) + self.assertEqual(list(actual_results[0]), + self._short_revision(self.revision4)) + self.assertEqual(list(actual_results[1]), + self._short_revision(self.revision3)) def test_revision_shortlog_with_limit(self): # given @@ -833,8 +832,8 @@ [self.revision4['id']], 1)) self.assertEqual(len(actual_results), 1) - self.assertEquals(list(actual_results[0]), - self._short_revision(self.revision4)) + self.assertEqual(list(actual_results[0]), + self._short_revision(self.revision4)) def test_revision_get(self): self.storage.revision_add([self.revision]) @@ -883,9 +882,9 @@ for actual_release in actual_releases: del actual_release['author']['id'] # hack: ids are generated - self.assertEquals([self.normalize_entity(self.release), - self.normalize_entity(self.release2)], - [actual_releases[0], actual_releases[1]]) + self.assertEqual([self.normalize_entity(self.release), + self.normalize_entity(self.release2)], + [actual_releases[0], actual_releases[1]]) def test_origin_add_one(self): origin0 = self.storage.origin_get(self.origin) @@ -1010,20 +1009,20 @@ ts=self.date_visit2) # then - self.assertEquals(origin_visit1['origin'], origin_id) + self.assertEqual(origin_visit1['origin'], origin_id) self.assertIsNotNone(origin_visit1['visit']) self.assertTrue(origin_visit1['visit'] > 0) actual_origin_visits = list(self.storage.origin_visit_get(origin_id)) - self.assertEquals(actual_origin_visits, - [{ - 'origin': origin_id, - 'date': self.date_visit2, - 'visit': origin_visit1['visit'], - 'status': 'ongoing', - 'metadata': None, - 'snapshot': None, - }]) + self.assertEqual(actual_origin_visits, + [{ + 'origin': origin_id, + 'date': self.date_visit2, + 'visit': origin_visit1['visit'], + 'status': 'ongoing', + 'metadata': None, + 'snapshot': None, + }]) def test_origin_visit_update(self): # given @@ -1055,7 +1054,7 @@ # then actual_origin_visits = list(self.storage.origin_visit_get(origin_id)) - self.assertEquals(actual_origin_visits, [{ + self.assertEqual(actual_origin_visits, [{ 'origin': origin_visit2['origin'], 'date': self.date_visit2, 'visit': origin_visit1['visit'], @@ -1073,38 +1072,38 @@ actual_origin_visits_bis = list(self.storage.origin_visit_get( origin_id, limit=1)) - self.assertEquals(actual_origin_visits_bis, - [{ - 'origin': origin_visit2['origin'], - 'date': self.date_visit2, - 'visit': origin_visit1['visit'], - 'status': 'full', - 'metadata': visit1_metadata, - 'snapshot': None, - }]) + self.assertEqual(actual_origin_visits_bis, + [{ + 'origin': origin_visit2['origin'], + 'date': self.date_visit2, + 'visit': origin_visit1['visit'], + 'status': 'full', + 'metadata': visit1_metadata, + 'snapshot': None, + }]) actual_origin_visits_ter = list(self.storage.origin_visit_get( origin_id, last_visit=origin_visit1['visit'])) - self.assertEquals(actual_origin_visits_ter, - [{ - 'origin': origin_visit2['origin'], - 'date': self.date_visit3, - 'visit': origin_visit2['visit'], - 'status': 'ongoing', - 'metadata': None, - 'snapshot': None, - }]) + self.assertEqual(actual_origin_visits_ter, + [{ + 'origin': origin_visit2['origin'], + 'date': self.date_visit3, + 'visit': origin_visit2['visit'], + 'status': 'ongoing', + 'metadata': None, + 'snapshot': None, + }]) actual_origin_visits2 = list(self.storage.origin_visit_get(origin_id2)) - self.assertEquals(actual_origin_visits2, - [{ - 'origin': origin_visit3['origin'], - 'date': self.date_visit3, - 'visit': origin_visit3['visit'], - 'status': 'partial', - 'metadata': None, - 'snapshot': None, - }]) + self.assertEqual(actual_origin_visits2, + [{ + 'origin': origin_visit3['origin'], + 'date': self.date_visit3, + 'visit': origin_visit3['visit'], + 'status': 'partial', + 'metadata': None, + 'snapshot': None, + }]) def test_origin_visit_get_by(self): origin_id = self.storage.origin_add_one(self.origin2) @@ -1146,7 +1145,7 @@ origin_visit1['origin'], origin_visit1['visit']) # then - self.assertEquals(actual_origin_visit1, expected_origin_visit) + self.assertEqual(actual_origin_visit1, expected_origin_visit) def test_origin_visit_get_by_no_result(self): # No result @@ -1366,8 +1365,8 @@ # Add snapshot to visit1, latest snapshot = visit 1 snapshot self.storage.snapshot_add(origin_id, visit1_id, self.complete_snapshot) - self.assertEquals(self.complete_snapshot, - self.storage.snapshot_get_latest(origin_id)) + self.assertEqual(self.complete_snapshot, + self.storage.snapshot_get_latest(origin_id)) # Status filter: both visits are status=ongoing, so no snapshot # returned @@ -1378,7 +1377,7 @@ # Mark the first visit as completed and check status filter again self.storage.origin_visit_update(origin_id, visit1_id, status='full') - self.assertEquals( + self.assertEqual( self.complete_snapshot, self.storage.snapshot_get_latest(origin_id, allowed_statuses=['full']), @@ -1386,11 +1385,11 @@ # Add snapshot to visit2 and check that the new snapshot is returned self.storage.snapshot_add(origin_id, visit2_id, self.empty_snapshot) - self.assertEquals(self.empty_snapshot, - self.storage.snapshot_get_latest(origin_id)) + self.assertEqual(self.empty_snapshot, + self.storage.snapshot_get_latest(origin_id)) # Check that the status filter is still working - self.assertEquals( + self.assertEqual( self.complete_snapshot, self.storage.snapshot_get_latest(origin_id, allowed_statuses=['full']), @@ -1561,11 +1560,11 @@ # add it actual_tools = list(self.storage.tool_add([tool])) - self.assertEquals(len(actual_tools), 1) + self.assertEqual(len(actual_tools), 1) actual_tool = actual_tools[0] self.assertIsNotNone(actual_tool) # now it exists new_id = actual_tool.pop('id') - self.assertEquals(actual_tool, tool) + self.assertEqual(actual_tool, tool) actual_tools2 = list(self.storage.tool_add([tool])) actual_tool2 = actual_tools2[0] diff --git a/tox.ini b/tox.ini new file mode 100644 --- /dev/null +++ b/tox.ini @@ -0,0 +1,17 @@ +[tox] +envlist=flake8,py3 + +[testenv:py3] +deps = + -r requirements-test.txt + pytest-cov + pifpaf +commands = + pifpaf run postgresql -- pytest --cov=swh --cov-branch {posargs} + +[testenv:flake8] +skip_install = true +deps = + flake8 +commands = + {envpython} -m flake8 diff --git a/utils/dump_revisions.py b/utils/dump_revisions.py --- a/utils/dump_revisions.py +++ b/utils/dump_revisions.py @@ -8,6 +8,7 @@ from swh.storage import converters, db from swh.model import identifiers + QUERY = ''' select r.id, @@ -44,6 +45,7 @@ if id != computed_id: dump_revision(revision) + if __name__ == '__main__': swh_db = db.Db.connect('service=swh', cursor_factory=psycopg2.extras.RealDictCursor) diff --git a/utils/fix_revisions_from_dump.py b/utils/fix_revisions_from_dump.py --- a/utils/fix_revisions_from_dump.py +++ b/utils/fix_revisions_from_dump.py @@ -16,6 +16,7 @@ def author_date_to_negutc(rev): rev['date']['negative_utc'] = True + DATE_NEGUTC_FIX = ('set author negutc', [ (None, None), (author_date_to_negutc, 'date_neg_utcoffset = true'), @@ -29,6 +30,7 @@ def committer_date_to_negutc(rev): rev['committer_date']['negative_utc'] = True + COMMITTER_DATE_NEGUTC_FIX = ('set committer negutc', [ (None, None), (committer_date_to_negutc, 'committer_date_neg_utcoffset = true'), @@ -38,6 +40,7 @@ def message_to_empty(rev): rev['message'] = b'' + MESSAGE_EMPTY_FIX = ('empty instead of null message', [ (None, None), (message_to_empty, "message = ''"), @@ -47,6 +50,7 @@ def message_to_null(rev): rev['message'] = None + MESSAGE_NULL_FIX = ('null instead of empty message', [ (None, None), (message_to_null, "message = NULL"), @@ -88,6 +92,7 @@ def author_name_doublespace(rev): rev['author']['name'] = b''.join([rev['author']['name'], b' ']) + AUTHOR_NAME_ADD_SPC_FIX = ('author double space', [ (None, None), (author_name_doublespace, 'trailing space author name') @@ -97,6 +102,7 @@ def committer_name_doublespace(rev): rev['committer']['name'] = b''.join([rev['committer']['name'], b' ']) + COMMITTER_NAME_ADD_SPC_FIX = ('committer double space', [ (None, None), (committer_name_doublespace, 'trailing space committer name') @@ -106,6 +112,7 @@ def author_name_null(rev): rev['author']['name'] = None + AUTHOR_NAME_NULL_FIX = ('author name null', [ (None, None), (author_name_null, 'None author name') @@ -115,6 +122,7 @@ def author_email_null(rev): rev['author']['email'] = None + AUTHOR_EMAIL_NULL_FIX = ('author email null', [ (None, None), (author_email_null, 'None author email') @@ -124,6 +132,7 @@ def committer_name_null(rev): rev['committer']['name'] = None + COMMITTER_NAME_NULL_FIX = ('committer name null', [ (None, None), (committer_name_null, 'None committer name') @@ -133,6 +142,7 @@ def committer_email_null(rev): rev['committer']['email'] = None + COMMITTER_EMAIL_NULL_FIX = ('committer email null', [ (None, None), (committer_email_null, 'None committer email') @@ -143,6 +153,7 @@ rev['author'] = b''.join([ identifiers.normalize_author(rev['author']), b' ']) + AUTHOR_ADD_SPC_FIX = ('add trailing space to author specification', [ (None, None), (author_add_spc, 'add trailing space to author spec') @@ -153,6 +164,7 @@ rev['committer'] = b''.join([ identifiers.normalize_author(rev['committer']), b' ']) + COMMITTER_ADD_SPC_FIX = ('add trailing space to committer specification', [ (None, None), (committer_add_spc, 'add trailing space to committer spec') @@ -224,6 +236,7 @@ else: return id, [] + if __name__ == '__main__': for hash in sys.stdin.readlines():