Changeset View
Changeset View
Standalone View
Standalone View
swh/web/tests/data.py
# Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import os | import os | ||||
from swh.indexer.language import LanguageIndexer | from swh.indexer.language import LanguageIndexer | ||||
from swh.indexer.fossology_license import FossologyLicenseIndexer | from swh.indexer.fossology_license import FossologyLicenseIndexer | ||||
from swh.indexer.mimetype import MimetypeIndexer | from swh.indexer.mimetype import MimetypeIndexer | ||||
from swh.indexer.ctags import CtagsIndexer | from swh.indexer.ctags import CtagsIndexer | ||||
from swh.indexer.storage import get_indexer_storage | from swh.indexer.storage import get_indexer_storage | ||||
from swh.model.hashutil import hash_to_hex, DEFAULT_ALGORITHMS | from swh.model.hashutil import hash_to_hex, hash_to_bytes, DEFAULT_ALGORITHMS | ||||
from swh.model.identifiers import directory_identifier | |||||
from swh.loader.git.from_disk import GitLoaderFromArchive | from swh.loader.git.from_disk import GitLoaderFromArchive | ||||
from swh.storage.algos.dir_iterators import dir_iterator | from swh.storage.algos.dir_iterators import dir_iterator | ||||
# Module used to initialize data that will be provided as tests input | # Module used to initialize data that will be provided as tests input | ||||
# Configuration for git loader | # Configuration for git loader | ||||
_TEST_LOADER_CONFIG = { | _TEST_LOADER_CONFIG = { | ||||
'storage': { | 'storage': { | ||||
▲ Show 20 Lines • Show All 109 Lines • ▼ Show 20 Lines | _TEST_ORIGINS = [ | ||||
'url': 'https://github.com/wcoder/highlightjs-line-numbers.js', | 'url': 'https://github.com/wcoder/highlightjs-line-numbers.js', | ||||
'archive': 'highlightjs-line-numbers.js.zip' | 'archive': 'highlightjs-line-numbers.js.zip' | ||||
}, | }, | ||||
{ | { | ||||
'id': 2, | 'id': 2, | ||||
'type': 'git', | 'type': 'git', | ||||
'url': 'https://github.com/memononen/libtess2', | 'url': 'https://github.com/memononen/libtess2', | ||||
'archive': 'libtess2.zip' | 'archive': 'libtess2.zip' | ||||
}, | |||||
{ | |||||
'id': 3, | |||||
'type': 'git', | |||||
'url': 'repo_with_submodules', | |||||
'archive': 'repo_with_submodules.tgz' | |||||
} | } | ||||
] | ] | ||||
# Tests data initialization | # Tests data initialization | ||||
def _init_tests_data(): | def _init_tests_data(): | ||||
# Load git repositories from archives | # Load git repositories from archives | ||||
loader = GitLoaderFromArchive(config=_TEST_LOADER_CONFIG) | loader = GitLoaderFromArchive(config=_TEST_LOADER_CONFIG) | ||||
Show All 29 Lines | for origin in _TEST_ORIGINS: | ||||
revisions.add(rev_id) | revisions.add(rev_id) | ||||
for rev in storage.revision_get(revisions): | for rev in storage.revision_get(revisions): | ||||
dir_id = rev['directory'] | dir_id = rev['directory'] | ||||
directories.add(hash_to_hex(dir_id)) | directories.add(hash_to_hex(dir_id)) | ||||
for entry in dir_iterator(storage, dir_id): | for entry in dir_iterator(storage, dir_id): | ||||
if entry['type'] == 'file': | if entry['type'] == 'file': | ||||
contents.add(entry['sha1']) | contents.add(entry['sha1']) | ||||
else: | elif entry['type'] == 'dir': | ||||
directories.add(hash_to_hex(entry['target'])) | directories.add(hash_to_hex(entry['target'])) | ||||
# Get all checksums for each content | # Get all checksums for each content | ||||
contents_metadata = storage.content_get_metadata(contents) | contents_metadata = storage.content_get_metadata(contents) | ||||
contents = [] | contents = [] | ||||
for content_metadata in contents_metadata: | for content_metadata in contents_metadata: | ||||
contents.append({ | contents.append({ | ||||
algo: hash_to_hex(content_metadata[algo]) | algo: hash_to_hex(content_metadata[algo]) | ||||
Show All 12 Lines | for idx_name, idx_class in (('mimetype_indexer', _MimetypeIndexer), | ||||
('ctags_indexer', _CtagsIndexer)): | ('ctags_indexer', _CtagsIndexer)): | ||||
idx = idx_class() | idx = idx_class() | ||||
idx.storage = storage | idx.storage = storage | ||||
idx.objstorage = storage.objstorage | idx.objstorage = storage.objstorage | ||||
idx.idx_storage = idx_storage | idx.idx_storage = idx_storage | ||||
idx.register_tools(idx.config['tools']) | idx.register_tools(idx.config['tools']) | ||||
indexers[idx_name] = idx | indexers[idx_name] = idx | ||||
# Add the empty directory to the test archive | |||||
empty_dir_id = directory_identifier({'entries': []}) | |||||
empty_dir_id_bin = hash_to_bytes(empty_dir_id) | |||||
storage.directory_add([{'id': empty_dir_id_bin, 'entries': []}]) | |||||
# Return tests data | # Return tests data | ||||
return { | return { | ||||
'storage': storage, | 'storage': storage, | ||||
'idx_storage': idx_storage, | 'idx_storage': idx_storage, | ||||
**indexers, | **indexers, | ||||
'origins': _TEST_ORIGINS, | 'origins': _TEST_ORIGINS, | ||||
'contents': contents, | 'contents': contents, | ||||
'directories': list(directories), | 'directories': list(directories), | ||||
Show All 17 Lines |