Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/loader.py
| # Copyright (C) 2015-2021 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| """Loader in charge of injecting either new or existing svn mirrors to | """Loader in charge of injecting either new or existing svn mirrors to | ||||
| swh-storage. | swh-storage. | ||||
| """ | """ | ||||
| from datetime import datetime | |||||
| from mmap import ACCESS_WRITE, mmap | from mmap import ACCESS_WRITE, mmap | ||||
| import os | import os | ||||
| import pty | import pty | ||||
| import re | import re | ||||
| import shutil | import shutil | ||||
| from subprocess import Popen | from subprocess import Popen | ||||
| import tempfile | import tempfile | ||||
| from typing import Dict, Iterator, List, Optional, Tuple | from typing import Dict, Iterator, List, Optional, Tuple | ||||
| import iso8601 | |||||
| from subvertpy import SubversionException | from subvertpy import SubversionException | ||||
| from swh.loader.core.loader import BaseLoader | from swh.loader.core.loader import BaseLoader | ||||
| from swh.loader.core.utils import clean_dangling_folders | from swh.loader.core.utils import clean_dangling_folders | ||||
| from swh.loader.exception import NotFound | from swh.loader.exception import NotFound | ||||
| from swh.loader.svn.svn import SvnRepo | from swh.loader.svn.svn import SvnRepo | ||||
| from swh.model import from_disk, hashutil | from swh.model import from_disk, hashutil | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| Show All 33 Lines | class SvnLoader(BaseLoader): | ||||
| visit_type = "svn" | visit_type = "svn" | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| storage: StorageInterface, | storage: StorageInterface, | ||||
| url: str, | url: str, | ||||
| origin_url: Optional[str] = None, | origin_url: Optional[str] = None, | ||||
| visit_date: Optional[str] = None, | visit_date: Optional[datetime] = None, | ||||
| destination_path: Optional[str] = None, | destination_path: Optional[str] = None, | ||||
| swh_revision: Optional[str] = None, | swh_revision: Optional[str] = None, | ||||
| start_from_scratch: bool = False, | start_from_scratch: bool = False, | ||||
| temp_directory: str = "/tmp", | temp_directory: str = "/tmp", | ||||
| debug: bool = False, | debug: bool = False, | ||||
| check_revision: int = 0, | check_revision: int = 0, | ||||
| max_content_size: Optional[int] = None, | max_content_size: Optional[int] = None, | ||||
| ): | ): | ||||
| Show All 17 Lines | ): | ||||
| self._skipped_contents: List[SkippedContent] = [] | self._skipped_contents: List[SkippedContent] = [] | ||||
| self._directories: List[Directory] = [] | self._directories: List[Directory] = [] | ||||
| self._revisions: List[Revision] = [] | self._revisions: List[Revision] = [] | ||||
| self._snapshot: Optional[Snapshot] = None | self._snapshot: Optional[Snapshot] = None | ||||
| # internal state, current visit | # internal state, current visit | ||||
| self._last_revision = None | self._last_revision = None | ||||
| self._visit_status = "full" | self._visit_status = "full" | ||||
| self._load_status = "uneventful" | self._load_status = "uneventful" | ||||
| if visit_date: | self.visit_date = visit_date | ||||
| self.visit_date = iso8601.parse_date(visit_date) | |||||
| else: | |||||
| self.visit_date = None | |||||
| self.destination_path = destination_path | self.destination_path = destination_path | ||||
| self.start_from_scratch = start_from_scratch | self.start_from_scratch = start_from_scratch | ||||
| self.snapshot = None | self.snapshot = None | ||||
| # state from previous visit | # state from previous visit | ||||
| self.latest_snapshot = None | self.latest_snapshot = None | ||||
| self.latest_revision = None | self.latest_revision = None | ||||
| def pre_cleanup(self): | def pre_cleanup(self): | ||||
| ▲ Show 20 Lines • Show All 435 Lines • ▼ Show 20 Lines | def __init__( | ||||
| self, | self, | ||||
| storage: StorageInterface, | storage: StorageInterface, | ||||
| url: str, | url: str, | ||||
| archive_path: str, | archive_path: str, | ||||
| origin_url: Optional[str] = None, | origin_url: Optional[str] = None, | ||||
| destination_path: Optional[str] = None, | destination_path: Optional[str] = None, | ||||
| swh_revision: Optional[str] = None, | swh_revision: Optional[str] = None, | ||||
| start_from_scratch: bool = False, | start_from_scratch: bool = False, | ||||
| visit_date: Optional[str] = None, | visit_date: Optional[datetime] = None, | ||||
| temp_directory: str = "/tmp", | temp_directory: str = "/tmp", | ||||
| debug: bool = False, | debug: bool = False, | ||||
| check_revision: int = 0, | check_revision: int = 0, | ||||
| max_content_size: Optional[int] = None, | max_content_size: Optional[int] = None, | ||||
| ): | ): | ||||
| super().__init__( | super().__init__( | ||||
| storage=storage, | storage=storage, | ||||
| url=url, | url=url, | ||||
| ▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | class SvnLoaderFromRemoteDump(SvnLoader): | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| storage: StorageInterface, | storage: StorageInterface, | ||||
| url: str, | url: str, | ||||
| origin_url: Optional[str] = None, | origin_url: Optional[str] = None, | ||||
| destination_path: Optional[str] = None, | destination_path: Optional[str] = None, | ||||
| swh_revision: Optional[str] = None, | swh_revision: Optional[str] = None, | ||||
| start_from_scratch: bool = False, | start_from_scratch: bool = False, | ||||
| visit_date: Optional[str] = None, | visit_date: Optional[datetime] = None, | ||||
| temp_directory: str = "/tmp", | temp_directory: str = "/tmp", | ||||
| debug: bool = False, | debug: bool = False, | ||||
| check_revision: int = 0, | check_revision: int = 0, | ||||
| max_content_size: Optional[int] = None, | max_content_size: Optional[int] = None, | ||||
| ): | ): | ||||
| super().__init__( | super().__init__( | ||||
| storage=storage, | storage=storage, | ||||
| url=url, | url=url, | ||||
| ▲ Show 20 Lines • Show All 156 Lines • Show Last 20 Lines | |||||