Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from abc import ABCMeta, abstractmethod | from abc import ABCMeta, abstractmethod | ||||
| import datetime | import datetime | ||||
| import hashlib | import hashlib | ||||
| import logging | import logging | ||||
| import os | import os | ||||
| from typing import Any, Dict, Iterable, Optional, Tuple | from typing import Any, Dict, Iterable, Optional | ||||
| from swh.core.config import load_from_envvar | from swh.core.config import load_from_envvar | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| BaseContent, | BaseContent, | ||||
| Content, | Content, | ||||
| Directory, | Directory, | ||||
| Origin, | Origin, | ||||
| OriginVisit, | OriginVisit, | ||||
| OriginVisitStatus, | OriginVisitStatus, | ||||
| Release, | Release, | ||||
| Revision, | Revision, | ||||
| Sha1Git, | Sha1Git, | ||||
| SkippedContent, | SkippedContent, | ||||
| Snapshot, | Snapshot, | ||||
| ) | ) | ||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from swh.storage.utils import now | from swh.storage.utils import now | ||||
| DEFAULT_CONFIG: Dict[str, Any] = { | DEFAULT_CONFIG: Dict[str, Any] = { | ||||
| "max_content_size": 100 * 1024 * 1024, | "max_content_size": 100 * 1024 * 1024, | ||||
| "save_data": False, | "save_data": False, | ||||
| "save_data_path": "", | "save_data_path": "", | ||||
| "storage": {"cls": "memory"}, | |||||
| } | } | ||||
| class BaseLoader(metaclass=ABCMeta): | class BaseLoader(metaclass=ABCMeta): | ||||
| """Mixin base class for loader. | """Mixin base class for loader. | ||||
| To use this class, you must: | To use this class, you must: | ||||
| Show All 23 Lines | class BaseLoader(metaclass=ABCMeta): | ||||
| You can take a look at some example classes: | You can take a look at some example classes: | ||||
| - :class:`SvnLoader` | - :class:`SvnLoader` | ||||
| """ | """ | ||||
| def __init__( | def __init__( | ||||
| self, logging_class: Optional[str] = None, | self, | ||||
| logging_class: Optional[str] = None, | |||||
| config: Optional[Dict[str, Any]] = None, | |||||
| ): | ): | ||||
| if config: | |||||
| self.config = config | |||||
| else: | |||||
| self.config = load_from_envvar(DEFAULT_CONFIG) | self.config = load_from_envvar(DEFAULT_CONFIG) | ||||
| self.storage = get_storage(**self.config["storage"]) | self.storage = get_storage(**self.config["storage"]) | ||||
| if logging_class is None: | if logging_class is None: | ||||
| logging_class = "%s.%s" % ( | logging_class = "%s.%s" % ( | ||||
| self.__class__.__module__, | self.__class__.__module__, | ||||
| self.__class__.__name__, | self.__class__.__name__, | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 346 Lines • Show Last 20 Lines | |||||