diff --git a/Makefile.tests b/Makefile.tests index 40efae8..86dd12d 100644 --- a/Makefile.tests +++ b/Makefile.tests @@ -1,62 +1,64 @@ test-connect-db: psql -d $(DB_TEST) test-create-db: cd $(SQL_FOLDER) && make clean initdb DBNAME=$(DB_TEST) test-drop-db: cd $(SQL_FOLDER) && make clean dropdb DBNAME=$(DB_TEST) tests: PYTHONPATH=$(_PYPATH) make test test-run-back: PYTHONPATH=$(_PYPATH) $(SWH_BACK) $(FLAG) --config ./resources/test/back.ini test-http: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_http.py test-swhrepo: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_swhrepo.py test-api: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api*.py test-api-post-per-type: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_post_*.py test-api-content: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_content.py test-api-directory: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_directory.py test-api-revision: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_revision.py test-api-release: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_release.py test-api-occurrence: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_occurrence.py test-api-home: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_home.py test-api-origin: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_origin.py test-api-person: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_person.py test-api-pickle: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_api_pickle.py test-file: $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_file.py test-remote-loader: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_remote_loader.py test-local-loader: PYTHONPATH=$(_PYPATH) $(NOSE) $(NOSEFLAGS) $(TESTDIR)/test_local_loader.py + +test-loaders: test-local-loader test-remote-loader diff --git a/swh/tests/test_local_loader.py b/swh/tests/test_local_loader.py index 078cfd6..e7fff3d 100644 --- a/swh/tests/test_local_loader.py +++ b/swh/tests/test_local_loader.py @@ -1,231 +1,243 @@ # coding: utf-8 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest import pygit2 import tempfile import shutil from nose.plugins.attrib import attr from nose.tools import istest from swh.store import db, models from swh.gitloader import loader from swh.conf import reader import test_initdb +from test_utils import list_files_from from test_git_utils import create_commit_with_content, create_tag @attr('slow') class TestRemoteLoader(unittest.TestCase): def setUp(self): """Initialize a git repository for the remaining test to manipulate. """ tmp_git_folder_path = tempfile.mkdtemp(prefix='test-sgloader.', dir='/tmp') self.tmp_git_repo = pygit2.init_repository(tmp_git_folder_path) - conf_back = reader.read('./resources/test/back.ini', + self.conf_back = reader.read('./resources/test/back.ini', {'port': ('int', 9999)}) - self.db_url = conf_back['db_url'] - self.content_storage_dir = conf_back['content_storage_dir'] + self.db_url = self.conf_back['db_url'] self.conf = { 'action': 'load', 'repo_path': self.tmp_git_repo.workdir, 'backend-type': 'local', - 'backend': './resources/test/back.ini' + 'backend': './resources/test/back.ini' } test_initdb.prepare_db(self.db_url) def tearDown(self): """Destroy the test git repository. """ shutil.rmtree(self.tmp_git_repo.workdir) - shutil.rmtree(self.content_storage_dir, ignore_errors=True) + shutil.rmtree(self.conf_back['content_storage_dir'], ignore_errors=True) @istest def should_fail_on_bad_action(self): # when try: loader.load({'action': 'unknown'}) except: pass @istest def should_fail_on_inexistant_folder(self): # when try: loader.load({'action': 'load', 'repo_path': 'something-that-definitely-does-not-exist'}) except: pass @istest def should_fail_on_inexistant_backend_type(self): # when try: loader.load({'action': 'load', 'repo_path': '.', 'backend-type': 'unknown'}) # only local or remote supported except: pass @istest def remote_loader(self): """Trigger loader and make sure everything is ok. """ # given commit0 = create_commit_with_content(self.tmp_git_repo, 'blob 0', 'commit msg 0') commit1 = create_commit_with_content(self.tmp_git_repo, 'blob 1', 'commit msg 1', [commit0.hex]) commit2 = create_commit_with_content(self.tmp_git_repo, 'blob 2', 'commit msg 2', [commit1.hex]) commit3 = create_commit_with_content(self.tmp_git_repo, None, 'commit msg 3', [commit2.hex]) commit4 = create_commit_with_content(self.tmp_git_repo, 'blob 4', 'commit msg 4', [commit3.hex]) # when loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf_back['content_storage_dir'])) + self.assertEquals(nb_files, 4+5+4, "4 blobs, 4 trees, 5 commits were created so 13 files.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 5, "Should be 5 commits") self.assertEquals( models.count_directories(db_conn), 5, "Should be 5 trees") self.assertEquals( models.count_contents(db_conn), 4, "Should be 4 blobs as we created one commit without data!") self.assertEquals( models.count_release(db_conn), 0, "No tag created so 0 release.") self.assertEquals( models.count_occurrence(db_conn), 1, "Should be 1 reference (master) so 1 occurrence.") # given commit5 = create_commit_with_content(self.tmp_git_repo, 'new blob 5', 'commit msg 5', [commit4.hex]) commit6 = create_commit_with_content(self.tmp_git_repo, 'new blob and last 6', 'commit msg 6', [commit5.hex]) commit7 = create_commit_with_content(self.tmp_git_repo, 'new blob 7', 'commit msg 7', [commit6.hex]) # when loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf_back['content_storage_dir'])) + self.assertEquals(nb_files, 13+3+3+3, "3 commits + 3 trees + 3 blobs so 9 more.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 8, "Should be 5+3 == 8 commits now") self.assertEquals( models.count_directories(db_conn), 8, "Should be 5+3 == 8 trees") self.assertEquals( models.count_contents(db_conn), 7, "Should be 4+3 == 7 blobs") self.assertEquals( models.count_release(db_conn), 0, "No tag created so 0 release.") self.assertEquals( models.count_occurrence(db_conn), 2, "Should be 1 reference which changed twice so 2 occurrences (master changed).") # given create_commit_with_content(self.tmp_git_repo, None, 'commit 8 with parent 2', [commit7.hex]) # when loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf_back['content_storage_dir'])) + self.assertEquals(nb_files, 22+1, "1 commit more without blob so no tree either.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 9, "Should be 8+1 == 9 commits now") self.assertEquals( models.count_directories(db_conn), 8, "Should be 8 trees (new commit without blob so no new tree)") self.assertEquals( models.count_contents(db_conn), 7, "Should be 7 blobs (new commit without new blob)") self.assertEquals( models.count_release(db_conn), 0, "No tag created so 0 release.") self.assertEquals( models.count_occurrence(db_conn), 3, "Should be 1 reference which changed thrice so 3 occurrences (master changed again).") self.assertEquals( models.count_person(db_conn), 2, "1 author + 1 committer") # add tag create_tag(self.tmp_git_repo, '0.0.1', commit5, 'bad ass release 0.0.1, towards infinity...') create_tag(self.tmp_git_repo, '0.0.2', commit7, 'release 0.0.2... and beyond') loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf_back['content_storage_dir'])) + self.assertEquals(nb_files, 23+2, "2 tags more.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 9, "Should be 8+1 == 9 commits now") self.assertEquals( models.count_directories(db_conn), 8, "Should be 8 trees (new commit without blob so no new tree)") self.assertEquals( models.count_contents(db_conn), 7, "Should be 7 blobs (new commit without new blob)") self.assertEquals( models.count_release(db_conn), 2, "Should be 2 annotated tags so 2 releases") self.assertEquals( models.count_occurrence(db_conn), 3, "master did not change this time so still 3 occurrences") self.assertEquals( models.count_person(db_conn), 3, "1 author + 1 committer + 1 tagger") diff --git a/swh/tests/test_remote_loader.py b/swh/tests/test_remote_loader.py index 0d71cc4..2ad6803 100644 --- a/swh/tests/test_remote_loader.py +++ b/swh/tests/test_remote_loader.py @@ -1,228 +1,242 @@ # coding: utf-8 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest import pygit2 import tempfile import shutil from nose.plugins.attrib import attr from nose.tools import istest from swh.store import db, models from swh.gitloader import loader from swh.conf import reader import test_initdb from test_git_utils import create_commit_with_content, create_tag +from test_utils import list_files_from @attr('slow') class TestRemoteLoader(unittest.TestCase): def setUp(self): """Initialize a git repository for the remaining test to manipulate. """ tmp_git_folder_path = tempfile.mkdtemp(prefix='test-sgloader.', dir='/tmp') self.tmp_git_repo = pygit2.init_repository(tmp_git_folder_path) self.conf = reader.read('./resources/test/back.ini', {'port': ('int', 9999)}) self.db_url = self.conf['db_url'] self.conf.update({ 'action': 'load', 'repo_path': self.tmp_git_repo.workdir, 'backend-type': 'remote', 'backend': 'http://localhost:%s' % self.conf['port'] }) test_initdb.prepare_db(self.db_url) def tearDown(self): """Destroy the test git repository. """ shutil.rmtree(self.tmp_git_repo.workdir) + shutil.rmtree(self.conf['content_storage_dir'], ignore_errors=True) @istest def should_fail_on_bad_action(self): # when try: loader.load({'action': 'unknown'}) except: pass @istest def should_fail_on_inexistant_folder(self): # when try: loader.load({'action': 'load', 'repo_path': 'something-that-definitely-does-not-exist'}) except: pass @istest def should_fail_on_inexistant_backend_type(self): # when try: loader.load({'action': 'load', 'repo_path': '.', 'backend-type': 'unknown'}) # only local or remote supported except: pass @istest def remote_loader(self): """Trigger loader and make sure everything is ok. """ # given commit0 = create_commit_with_content(self.tmp_git_repo, 'blob 0', 'commit msg 0') commit1 = create_commit_with_content(self.tmp_git_repo, 'blob 1', 'commit msg 1', [commit0.hex]) commit2 = create_commit_with_content(self.tmp_git_repo, 'blob 2', 'commit msg 2', [commit1.hex]) commit3 = create_commit_with_content(self.tmp_git_repo, None, 'commit msg 3', [commit2.hex]) commit4 = create_commit_with_content(self.tmp_git_repo, 'blob 4', 'commit msg 4', [commit3.hex]) # when loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf['content_storage_dir'])) + self.assertEquals(nb_files, 4+5+4, "4 blobs, 4 trees, 5 commits were created so 13 files.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 5, "Should be 5 commits") self.assertEquals( models.count_directories(db_conn), 5, "Should be 5 trees") self.assertEquals( models.count_contents(db_conn), 4, "Should be 4 blobs as we created one commit without data!") self.assertEquals( models.count_release(db_conn), 0, "No tag created so 0 release.") self.assertEquals( models.count_occurrence(db_conn), 1, "Should be 1 reference (master) so 1 occurrence.") # given commit5 = create_commit_with_content(self.tmp_git_repo, 'new blob 5', 'commit msg 5', [commit4.hex]) commit6 = create_commit_with_content(self.tmp_git_repo, 'new blob and last 6', 'commit msg 6', [commit5.hex]) commit7 = create_commit_with_content(self.tmp_git_repo, 'new blob 7', 'commit msg 7', [commit6.hex]) # when loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf['content_storage_dir'])) + self.assertEquals(nb_files, 13+3+3+3, "3 commits + 3 trees + 3 blobs so 9 more.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 8, "Should be 5+3 == 8 commits now") self.assertEquals( models.count_directories(db_conn), 8, "Should be 5+3 == 8 trees") self.assertEquals( models.count_contents(db_conn), 7, "Should be 4+3 == 7 blobs") self.assertEquals( models.count_release(db_conn), 0, "No tag created so 0 release.") self.assertEquals( models.count_occurrence(db_conn), 2, "Should be 1 reference which changed twice so 2 occurrences (master changed).") # given create_commit_with_content(self.tmp_git_repo, None, 'commit 8 with parent 2', [commit7.hex]) # when loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf['content_storage_dir'])) + self.assertEquals(nb_files, 22+1, "1 commit more without blob so no tree either.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 9, "Should be 8+1 == 9 commits now") self.assertEquals( models.count_directories(db_conn), 8, "Should be 8 trees (new commit without blob so no new tree)") self.assertEquals( models.count_contents(db_conn), 7, "Should be 7 blobs (new commit without new blob)") self.assertEquals( models.count_release(db_conn), 0, "No tag created so 0 release.") self.assertEquals( models.count_occurrence(db_conn), 3, "Should be 1 reference which changed thrice so 3 occurrences (master changed again).") self.assertEquals( models.count_person(db_conn), 2, "1 author + 1 committer") # add tag create_tag(self.tmp_git_repo, '0.0.1', commit5, 'bad ass release 0.0.1, towards infinity...') create_tag(self.tmp_git_repo, '0.0.2', commit7, 'release 0.0.2... and beyond') loader.load(self.conf) # then + nb_files = len(list_files_from(self.conf['content_storage_dir'])) + self.assertEquals(nb_files, 23+2, "2 tags more.") + with db.connect(self.db_url) as db_conn: self.assertEquals( models.count_revisions(db_conn), 9, "Should be 8+1 == 9 commits now") self.assertEquals( models.count_directories(db_conn), 8, "Should be 8 trees (new commit without blob so no new tree)") self.assertEquals( models.count_contents(db_conn), 7, "Should be 7 blobs (new commit without new blob)") self.assertEquals( models.count_release(db_conn), 2, "Should be 2 annotated tags so 2 releases") self.assertEquals( models.count_occurrence(db_conn), 3, "master did not change this time so still 3 occurrences") self.assertEquals( models.count_person(db_conn), 3, "1 author + 1 committer + 1 tagger") diff --git a/swh/tests/test_utils.py b/swh/tests/test_utils.py index e0b6eeb..a1831b8 100644 --- a/swh/tests/test_utils.py +++ b/swh/tests/test_utils.py @@ -1,31 +1,42 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import time +import os from swh.backend import api import test_initdb def now(): "Build the date as of now in the api's format." return time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) +def list_files_from(root_path): + """Compute the list of files from root_path. + + """ + f = [] + for (dirpath, dirnames, filenames) in os.walk(root_path): + f.extend(filenames) + return f + + def app_client(db_url="dbname=softwareheritage-dev-test"): """Setup the application ready for testing. """ api.app.config['conf'] = {'db_url': db_url, 'content_storage_dir': '/tmp/swh-git-loader/content-storage', 'log_dir': '/tmp/swh-git-loader/log', 'folder_depth': 2, 'storage_compression': None, 'debug': 'true'} api.app.config['TESTING'] = True app = api.app.test_client() test_initdb.prepare_db(db_url) return app, db_url