Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/loader.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| """Loader in charge of injecting either new or existing svn mirrors to | """Loader in charge of injecting either new or existing svn mirrors to | ||||
| swh-storage. | swh-storage. | ||||
| """ | """ | ||||
| from mmap import ACCESS_WRITE, mmap | from mmap import ACCESS_WRITE, mmap | ||||
| import os | import os | ||||
| import pty | import pty | ||||
| import re | import re | ||||
| import shutil | import shutil | ||||
| from subprocess import Popen | from subprocess import Popen | ||||
| import tempfile | import tempfile | ||||
| from typing import Any, Dict, Iterator, List, Optional, Tuple | from typing import Any, Dict, Iterator, List, Optional, Tuple | ||||
| from subvertpy import SubversionException | |||||
| from swh.core.config import merge_configs | from swh.core.config import merge_configs | ||||
| from swh.loader.core.loader import BaseLoader | from swh.loader.core.loader import BaseLoader | ||||
| from swh.loader.core.utils import clean_dangling_folders | from swh.loader.core.utils import clean_dangling_folders | ||||
| from swh.loader.exception import NotFound | |||||
| from swh.loader.svn.svn import SvnRepo | |||||
| from swh.model import from_disk, hashutil | from swh.model import from_disk, hashutil | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| Content, | Content, | ||||
| Directory, | Directory, | ||||
| Origin, | Origin, | ||||
| Revision, | Revision, | ||||
| SkippedContent, | SkippedContent, | ||||
| Snapshot, | Snapshot, | ||||
| SnapshotBranch, | SnapshotBranch, | ||||
| TargetType, | TargetType, | ||||
| ) | ) | ||||
| from swh.storage.algos.snapshot import snapshot_get_latest | from swh.storage.algos.snapshot import snapshot_get_latest | ||||
| from . import converters, svn | from . import converters | ||||
| from .exception import SvnLoaderHistoryAltered, SvnLoaderUneventful | from .exception import SvnLoaderHistoryAltered, SvnLoaderUneventful | ||||
| from .utils import ( | from .utils import ( | ||||
| OutputStream, | OutputStream, | ||||
| init_svn_repo_from_archive_dump, | init_svn_repo_from_archive_dump, | ||||
| init_svn_repo_from_dump, | init_svn_repo_from_dump, | ||||
| ) | ) | ||||
| DEFAULT_BRANCH = b"HEAD" | DEFAULT_BRANCH = b"HEAD" | ||||
| ▲ Show 20 Lines • Show All 349 Lines • ▼ Show 20 Lines | def prepare(self, *args, **kwargs): | ||||
| local_dirname = self.destination_path | local_dirname = self.destination_path | ||||
| else: | else: | ||||
| local_dirname = tempfile.mkdtemp( | local_dirname = tempfile.mkdtemp( | ||||
| suffix="-%s" % os.getpid(), | suffix="-%s" % os.getpid(), | ||||
| prefix=TEMPORARY_DIR_PREFIX_PATTERN, | prefix=TEMPORARY_DIR_PREFIX_PATTERN, | ||||
| dir=self.temp_directory, | dir=self.temp_directory, | ||||
| ) | ) | ||||
| self.svnrepo = svn.SvnRepo( | try: | ||||
| self.svnrepo = SvnRepo( | |||||
| self.svn_url, self.origin_url, local_dirname, self.max_content_length | self.svn_url, self.origin_url, local_dirname, self.max_content_length | ||||
| ) | ) | ||||
| except SubversionException as e: | |||||
| error_msgs = [ | |||||
| "Unable to connect to a repository at URL", | |||||
| "Unknown URL type", | |||||
| ] | |||||
| for msg in error_msgs: | |||||
| if msg in e.args[0]: | |||||
| self._load_status = "uneventful" | |||||
| raise NotFound(e) | |||||
| raise | |||||
| try: | try: | ||||
| revision_start, revision_end, revision_parents = self.start_from( | revision_start, revision_end, revision_parents = self.start_from( | ||||
| self.start_from_scratch | self.start_from_scratch | ||||
| ) | ) | ||||
| self.swh_revision_gen = self.process_svn_revisions( | self.swh_revision_gen = self.process_svn_revisions( | ||||
| self.svnrepo, revision_start, revision_end, revision_parents | self.svnrepo, revision_start, revision_end, revision_parents | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 336 Lines • Show Last 20 Lines | |||||