Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/npm/tests/test_loader.py
# Copyright (C) 2019 The Software Heritage developers | # Copyright (C) 2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import os | import os | ||||
import unittest | import unittest | ||||
import requests_mock | import requests_mock | ||||
from swh.loader.core.tests import BaseLoaderStorageTest | from swh.loader.core.tests import BaseLoaderStorageTest | ||||
from swh.loader.npm.loader import NpmLoader | from swh.loader.npm.loader import NpmLoader | ||||
from swh.model.identifiers import snapshot_identifier | |||||
from .common import ( | from .common import ( | ||||
empty_package, empty_package_url, empty_package_metadata_url, | |||||
package, package_url, package_metadata_url, init_test_data, | package, package_url, package_metadata_url, init_test_data, | ||||
EMPTY_PACKAGE_METADATA_JSON_FILENAME, | |||||
FIRST_VISIT_PACKAGE_METADATA_JSON_FILENAME, | FIRST_VISIT_PACKAGE_METADATA_JSON_FILENAME, | ||||
SECOND_VISIT_PACKAGE_METADATA_JSON_FILENAME, | SECOND_VISIT_PACKAGE_METADATA_JSON_FILENAME, | ||||
) | ) | ||||
_LOADER_TESTS_CONFIG = { | _LOADER_TESTS_CONFIG = { | ||||
'content_packet_size': 10000, | 'content_packet_size': 10000, | ||||
'content_packet_size_bytes': 104857600, | 'content_packet_size_bytes': 104857600, | ||||
'content_size_limit': 104857600, | 'content_size_limit': 104857600, | ||||
▲ Show 20 Lines • Show All 170 Lines • ▼ Show 20 Lines | def parse_config_file(self, *args, **kwargs): | ||||
return _LOADER_TESTS_CONFIG | return _LOADER_TESTS_CONFIG | ||||
@requests_mock.Mocker() | @requests_mock.Mocker() | ||||
class TestNpmLoader(unittest.TestCase, BaseLoaderStorageTest): | class TestNpmLoader(unittest.TestCase, BaseLoaderStorageTest): | ||||
@classmethod | @classmethod | ||||
def setUpClass(cls): | def setUpClass(cls): | ||||
cls.reset_loader() | |||||
@classmethod | |||||
def reset_loader(cls): | |||||
cls.loader = NpmLoaderTest() | cls.loader = NpmLoaderTest() | ||||
cls.storage = cls.loader.storage | cls.storage = cls.loader.storage | ||||
def reset_loader_counters(self): | def reset_loader_counters(self): | ||||
counters_reset = dict.fromkeys(self.loader.counters.keys(), 0) | counters_reset = dict.fromkeys(self.loader.counters.keys(), 0) | ||||
self.loader.counters.update(counters_reset) | self.loader.counters.update(counters_reset) | ||||
def test_npm_loader_empty_package(self, m): | |||||
init_test_data(m, EMPTY_PACKAGE_METADATA_JSON_FILENAME, | |||||
empty_package_metadata_url) | |||||
self.loader.load(empty_package, empty_package_url, | |||||
empty_package_metadata_url) | |||||
self.assertCountContents(0) | |||||
self.assertCountDirectories(0) | |||||
self.assertCountRevisions(0) | |||||
self.assertCountReleases(0) | |||||
self.assertCountSnapshots(1) | |||||
expected_branches = {} | |||||
self.assertSnapshotEqual( | |||||
snapshot_identifier({'branches': expected_branches}), | |||||
expected_branches | |||||
) | |||||
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | |||||
self.assertEqual(self.loader.visit_status(), 'full') | |||||
self.assertFalse(os.path.exists(self.loader.temp_directory)) | |||||
def test_npm_loader_first_visit(self, m): | def test_npm_loader_first_visit(self, m): | ||||
init_test_data(m, FIRST_VISIT_PACKAGE_METADATA_JSON_FILENAME) | self.reset_loader() | ||||
init_test_data(m, FIRST_VISIT_PACKAGE_METADATA_JSON_FILENAME, | |||||
package_metadata_url) | |||||
self.loader.load(package, package_url, package_metadata_url) | self.loader.load(package, package_url, package_metadata_url) | ||||
self.assertCountContents(len(_expected_new_contents_first_visit)) | self.assertCountContents(len(_expected_new_contents_first_visit)) | ||||
self.assertCountDirectories(len(_expected_new_directories_first_visit)) | self.assertCountDirectories(len(_expected_new_directories_first_visit)) | ||||
self.assertCountRevisions(3, '3 releases so 3 revisions should be created') # noqa | self.assertCountRevisions(3, '3 releases so 3 revisions should be created') # noqa | ||||
self.assertCountReleases(0, 'No release is created by the npm loader') | self.assertCountReleases(0, 'No release is created by the npm loader') | ||||
self.assertCountSnapshots(1, 'Only 1 snapshot targeting all revisions') | self.assertCountSnapshots(1, 'Only 1 snapshot targeting all revisions') | ||||
Show All 14 Lines | def test_npm_loader_first_visit(self, m): | ||||
self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
self.assertFalse(os.path.exists(self.loader.temp_directory)) | self.assertFalse(os.path.exists(self.loader.temp_directory)) | ||||
def test_npm_loader_first_visit_again(self, m): | def test_npm_loader_first_visit_again(self, m): | ||||
self.reset_loader_counters() | self.reset_loader_counters() | ||||
init_test_data(m, FIRST_VISIT_PACKAGE_METADATA_JSON_FILENAME) | init_test_data(m, FIRST_VISIT_PACKAGE_METADATA_JSON_FILENAME, | ||||
package_metadata_url) | |||||
self.loader.load(package, package_url, package_metadata_url) | self.loader.load(package, package_url, package_metadata_url) | ||||
# previously loaded objects should still be here | # previously loaded objects should still be here | ||||
self.assertCountContents(len(_expected_new_contents_first_visit)) | self.assertCountContents(len(_expected_new_contents_first_visit)) | ||||
self.assertCountDirectories(len(_expected_new_directories_first_visit)) | self.assertCountDirectories(len(_expected_new_directories_first_visit)) | ||||
self.assertCountRevisions(len(_expected_new_revisions_first_visit)) | self.assertCountRevisions(len(_expected_new_revisions_first_visit)) | ||||
self.assertCountReleases(0) | self.assertCountReleases(0) | ||||
self.assertCountSnapshots(1) | self.assertCountSnapshots(1) | ||||
self.assertSnapshotEqual(_expected_new_snapshot_first_visit, | self.assertSnapshotEqual(_expected_new_snapshot_first_visit, | ||||
_expected_branches_first_visit) | _expected_branches_first_visit) | ||||
# no objects should have been loaded in that visit | # no objects should have been loaded in that visit | ||||
counters_reset = dict.fromkeys(self.loader.counters.keys(), 0) | counters_reset = dict.fromkeys(self.loader.counters.keys(), 0) | ||||
self.assertEqual(self.loader.counters, counters_reset) | self.assertEqual(self.loader.counters, counters_reset) | ||||
self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | self.assertEqual(self.loader.load_status(), {'status': 'uneventful'}) | ||||
self.assertEqual(self.loader.visit_status(), 'full') | self.assertEqual(self.loader.visit_status(), 'full') | ||||
self.assertFalse(os.path.exists(self.loader.temp_directory)) | self.assertFalse(os.path.exists(self.loader.temp_directory)) | ||||
def test_npm_loader_second_visit(self, m): | def test_npm_loader_second_visit(self, m): | ||||
self.reset_loader_counters() | self.reset_loader_counters() | ||||
init_test_data(m, SECOND_VISIT_PACKAGE_METADATA_JSON_FILENAME) | init_test_data(m, SECOND_VISIT_PACKAGE_METADATA_JSON_FILENAME, | ||||
package_metadata_url) | |||||
self.loader.load(package, package_url, package_metadata_url) | self.loader.load(package, package_url, package_metadata_url) | ||||
expected_nb_contents = sum([len(_expected_new_contents_first_visit), | expected_nb_contents = sum([len(_expected_new_contents_first_visit), | ||||
len(_expected_new_contents_second_visit)]) | len(_expected_new_contents_second_visit)]) | ||||
expected_nb_directories = sum([len(_expected_new_directories_first_visit), # noqa | expected_nb_directories = sum([len(_expected_new_directories_first_visit), # noqa | ||||
len(_expected_new_directories_second_visit)]) # noqa | len(_expected_new_directories_second_visit)]) # noqa | ||||
Show All 32 Lines |