Differential D799 Diff 2551 swh/loader/git/tests/test_loader.py

Changeset View

Standalone View

swh/loader/git/tests/test_loader.py

# Copyright (C) 2018 The Software Heritage developers		# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import os.path		import os.path
import zipfile
import tempfile
import subprocess		import subprocess

from swh.loader.git.loader import GitLoader, GitLoaderFromArchive		from swh.loader.git.loader import GitLoader, GitLoaderFromArchive
from swh.loader.core.tests import BaseLoaderTest		from swh.loader.core.tests import BaseLoaderTest

from . import TEST_LOADER_CONFIG		from . import TEST_LOADER_CONFIG


class GitLoaderFromArchive(GitLoaderFromArchive):		class GitLoaderFromArchive(GitLoaderFromArchive):
def project_name_from_archive(self, archive_path):		def project_name_from_archive(self, archive_path):
# We don't want the project name to be 'resources'.		# We don't want the project name to be 'resources'.
return 'testrepo'		return 'testrepo'

		def parse_config_file(self, args, *kwargs):
		return TEST_LOADER_CONFIG


CONTENT1 = {		CONTENT1 = {
'33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d', # README v1		'33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d', # README v1
'349c4ff7d21f1ec0eda26f3d9284c293e3425417', # README v2		'349c4ff7d21f1ec0eda26f3d9284c293e3425417', # README v2
'799c11e348d39f1704022b8354502e2f81f3c037', # file1.txt		'799c11e348d39f1704022b8354502e2f81f3c037', # file1.txt
'4bdb40dfd6ec75cb730e678b5d7786e30170c5fb', # file2.txt		'4bdb40dfd6ec75cb730e678b5d7786e30170c5fb', # file2.txt
}		}

SNAPSHOT_ID = 'bdf3b06d6017e0d9ad6447a73da6ff1ae9efb8f0'		SNAPSHOT_ID = 'bdf3b06d6017e0d9ad6447a73da6ff1ae9efb8f0'

SNAPSHOT1 = {		SNAPSHOT1 = {
'id': SNAPSHOT_ID,		'id': SNAPSHOT_ID,
'branches': {		'branches': {
'HEAD': {		'HEAD': {
'target': '2f01f5ca7e391a2f08905990277faf81e709a649',		'target': '2f01f5ca7e391a2f08905990277faf81e709a649',
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines	REVISIONS1 = {
'1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b':		'1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b':
'fbf70528223d263661b5ad4b80f26caf3860eb8e',		'fbf70528223d263661b5ad4b80f26caf3860eb8e',
'79f65ac75f79dda6ff03d66e1242702ab67fb51c':		'79f65ac75f79dda6ff03d66e1242702ab67fb51c':
'5df34ec74d6f69072d9a0a6677d8efbed9b12e60',		'5df34ec74d6f69072d9a0a6677d8efbed9b12e60',
'b0a77609903f767a2fd3d769904ef9ef68468b87':		'b0a77609903f767a2fd3d769904ef9ef68468b87':
'9ca0c7d6ffa3f9f0de59fd7912e08f11308a1338',		'9ca0c7d6ffa3f9f0de59fd7912e08f11308a1338',
'bd746cd1913721b269b395a56a97baf6755151c2':		'bd746cd1913721b269b395a56a97baf6755151c2':
'e1d0d894835f91a0f887a4bc8b16f81feefdfbd5',		'e1d0d894835f91a0f887a4bc8b16f81feefdfbd5',
}		}


class BaseGitLoaderTest(BaseLoaderTest):		class BaseGitLoaderTest(BaseLoaderTest):
def setUp(self, archive_name, uncompress_archive, filename='testrepo'):		def setUp(self, archive_name, uncompress_archive, filename='testrepo'):
super().setUp(archive_name=archive_name, filename=filename,		super().setUp(archive_name=archive_name, filename=filename,
prefix_tmp_folder_name='swh.loader.git.',		prefix_tmp_folder_name='swh.loader.git.',
start_path=os.path.dirname(__file__),		start_path=os.path.dirname(__file__),
uncompress_archive=uncompress_archive)		uncompress_archive=uncompress_archive)


class GitLoaderTest(GitLoader):		class GitLoaderTest(GitLoader):
def parse_config_file(self, args, *kwargs):		def parse_config_file(self, args, *kwargs):
return TEST_LOADER_CONFIG		return TEST_LOADER_CONFIG


class BaseDirGitLoaderTest(BaseGitLoaderTest):		class BaseDirGitLoaderTest(BaseGitLoaderTest):
"""Mixin base loader test to prepare the git		"""Mixin base loader test to prepare the git
repository to uncompress, load and test the results.		repository to uncompress, load and test the results.

This sets up		This sets up

"""		"""
def setUp(self):		def setUp(self):
super().setUp('testrepo.tgz', True)		super().setUp('testrepo.tgz', uncompress_archive=True)
self.loader = GitLoaderTest()		self.loader = GitLoaderTest()
self.storage = self.loader.storage		self.storage = self.loader.storage

def load(self):		def load(self):
return self.loader.load(		return self.loader.load(
origin_url=self.repo_url,		origin_url=self.repo_url,
visit_date='2016-05-03 15:16:32+00',		visit_date='2016-05-03 15:16:32+00',
directory=self.destination_path)		directory=self.destination_path)


class GitLoaderFromArchiveTest(GitLoaderFromArchive):		class BaseGitLoaderFromArchiveTest(BaseGitLoaderTest):
		ardumontUnsubmitted Not Done Inline Actions Maybe rename to remove the zip notion? ardumont: Maybe rename to remove the zip notion?
		anlambertAuthorUnsubmitted Done Inline Actions proposal: BaseGitLoaderFromArchiveTest anlambert: proposal: BaseGitLoaderFromArchiveTest
		ardumontUnsubmitted Not Done Inline Actions sounds good! ardumont: sounds good!
def parse_config_file(self, args, *kwargs):
return TEST_LOADER_CONFIG


class BaseZipGitLoaderTest(BaseGitLoaderTest):
"""Mixin base loader test to prepare the git		"""Mixin base loader test to prepare the git
repository to uncompress, load and test the results.		repository to uncompress, load and test the results.

This sets up		This sets up

"""		"""
def setUp(self):		def setUp(self):
super().setUp('testrepo.tgz', True)		super().setUp('testrepo.tgz', uncompress_archive=False)
		ardumontUnsubmitted Not Done Inline Actions Can you explicit the boolean value with its name? I think it's to inhibit the tarball uncompression step, not completely sure though. ardumont: Can you explicit the boolean value with its name? I think it's to inhibit the tarball…
		anlambertAuthorUnsubmitted Done Inline Actions sure! anlambert: sure!
self._setup_zip()		self.loader = GitLoaderFromArchive()
self.loader = GitLoaderFromArchiveTest()
self.storage = self.loader.storage		self.storage = self.loader.storage

def _setup_zip(self):
self._zip_file = tempfile.NamedTemporaryFile('ab', suffix='.zip')
dest_dir = os.path.normpath(self.destination_path) + '/'
with zipfile.ZipFile(self._zip_file, 'a') as zip_writer:
for root, dirs, files in os.walk(dest_dir):
assert root.startswith(dest_dir)
relative_root = os.path.join(
'testrepo',
root[len(dest_dir):])
for file_ in files:
zip_writer.write(
filename=os.path.join(root, file_),
arcname=os.path.join(relative_root, file_))
self.destination_path = self._zip_file.name
self.tmp_root_path = None
self.repo_url = 'file://' + self.destination_path

def tearDown(self):
self._zip_file.close()
super().tearDown()

def load(self):		def load(self):
return self.loader.load(		return self.loader.load(
origin_url=self.repo_url,		origin_url=self.repo_url,
visit_date='2016-05-03 15:16:32+00',		visit_date='2016-05-03 15:16:32+00',
archive_path=self.destination_path)		archive_path=self.destination_path)


class GitLoaderTests:		class GitLoaderTests:
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines	def test_load_changed(self):

# TODO: how to check the snapshot id?		# TODO: how to check the snapshot id?
# self.assertSnapshotEqual(SNAPSHOT1)		# self.assertSnapshotEqual(SNAPSHOT1)

self.assertEqual(self.loader.load_status(), {'status': 'eventful'})		self.assertEqual(self.loader.load_status(), {'status': 'eventful'})
self.assertEqual(self.loader.visit_status(), 'full')		self.assertEqual(self.loader.visit_status(), 'full')


class ZipGitLoaderTest(BaseZipGitLoaderTest, GitLoaderTests):		class GitLoaderFromArchiveTest(BaseGitLoaderFromArchiveTest, GitLoaderTests):
"""Tests for GitLoaderFromArchive. Imports the common ones		"""Tests for GitLoaderFromArchive. Imports the common ones
from GitLoaderTests."""		from GitLoaderTests."""
pass		pass