Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | def __init__( | ||||
| storage: StorageInterface, | storage: StorageInterface, | ||||
| origin_url: str, | origin_url: str, | ||||
| logging_class: Optional[str] = None, | logging_class: Optional[str] = None, | ||||
| save_data_path: Optional[str] = None, | save_data_path: Optional[str] = None, | ||||
| max_content_size: Optional[int] = None, | max_content_size: Optional[int] = None, | ||||
| lister_name: Optional[str] = None, | lister_name: Optional[str] = None, | ||||
| lister_instance_name: Optional[str] = None, | lister_instance_name: Optional[str] = None, | ||||
| metadata_fetcher_credentials: CredentialsType = None, | metadata_fetcher_credentials: CredentialsType = None, | ||||
| **kwargs, | |||||
| ): | ): | ||||
| if lister_name == "": | if lister_name == "": | ||||
| raise ValueError("lister_name must not be the empty string") | raise ValueError("lister_name must not be the empty string") | ||||
| if lister_name is None and lister_instance_name is not None: | if lister_name is None and lister_instance_name is not None: | ||||
| raise ValueError( | raise ValueError( | ||||
| f"lister_name is None but lister_instance_name is {lister_instance_name!r}" | f"lister_name is None but lister_instance_name is {lister_instance_name!r}" | ||||
| ) | ) | ||||
| if lister_name is not None and lister_instance_name is None: | if lister_name is not None and lister_instance_name is None: | ||||
| Show All 13 Lines | ): | ||||
| self.__class__.__module__, | self.__class__.__module__, | ||||
| self.__class__.__name__, | self.__class__.__name__, | ||||
| ) | ) | ||||
| self.log = logging.getLogger(logging_class) | self.log = logging.getLogger(logging_class) | ||||
| _log = logging.getLogger("requests.packages.urllib3.connectionpool") | _log = logging.getLogger("requests.packages.urllib3.connectionpool") | ||||
| _log.setLevel(logging.WARN) | _log.setLevel(logging.WARN) | ||||
| self.log.setLevel( | |||||
| logging.DEBUG if kwargs.get("debug") is True else logging.INFO | |||||
| ) | |||||
| # possibly overridden in self.prepare method | # possibly overridden in self.prepare method | ||||
| self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
| self.loaded_snapshot_id = None | self.loaded_snapshot_id = None | ||||
| if save_data_path: | if save_data_path: | ||||
| path = save_data_path | path = save_data_path | ||||
| os.stat(path) | os.stat(path) | ||||
| ▲ Show 20 Lines • Show All 480 Lines • Show Last 20 Lines | |||||