Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/tests/__init__.py
# Copyright (C) 2018-2019 The Software Heritage developers | # Copyright (C) 2018-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
TEST_LOADER_CONFIG = { | import os | ||||
"storage": { | import subprocess | ||||
"cls": "pipeline", | |||||
"steps": [ | from typing import Optional | ||||
{"cls": "filter"}, | |||||
{ | |||||
"cls": "buffer", | def prepare_repository_from_archive( | ||||
"min_batch_size": { | archive_path: str, | ||||
"content": 10, | filename: Optional[str] = None, | ||||
"content_bytes": 100 * 1024 * 1024, | tmp_path: str = "/tmp", | ||||
"directory": 10, | uncompress_archive: bool = True, | ||||
"revision": 10, | ) -> str: | ||||
"release": 10, | if uncompress_archive: | ||||
}, | # uncompress folder/repositories/dump for the loader to ingest | ||||
}, | subprocess.check_output(["tar", "xf", archive_path, "-C", tmp_path]) | ||||
{"cls": "memory"}, | # build the origin url (or some derivative form) | ||||
], | _fname = filename if filename else os.path.basename(archive_path) | ||||
ardumont: I will definitely move that to the loader-core now.
Also, note that the `uncompress_archive`… | |||||
Done Inline Actionsardumont: D3431 | |||||
}, | return f"file://{tmp_path}/{_fname}" | ||||
"max_content_size": 100 * 1024 * 1024, | |||||
"pack_size_bytes": 4 * 1024 * 1024 * 1024, | |||||
"save_data": False, | |||||
} |
I will definitely move that to the loader-core now.
Also, note that the uncompress_archive option is no longer needed (i dropped its use).