Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/nixguix/loader.py
| # Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import requests | |||||
| from typing import Any, Dict, Iterator, Mapping, Optional, Tuple | from typing import Any, Dict, Iterator, Mapping, Optional, Tuple | ||||
| import attr | import attr | ||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| MetadataAuthority, | MetadataAuthority, | ||||
| MetadataAuthorityType, | MetadataAuthorityType, | ||||
| Revision, | Revision, | ||||
| RevisionType, | RevisionType, | ||||
| TargetType, | TargetType, | ||||
| Snapshot, | Snapshot, | ||||
| BaseModel, | BaseModel, | ||||
| Sha1Git, | Sha1Git, | ||||
| ) | ) | ||||
| from swh.loader.package.utils import EMPTY_AUTHOR | from swh.loader.package.utils import EMPTY_AUTHOR, api_info | ||||
| from swh.loader.package.loader import ( | from swh.loader.package.loader import ( | ||||
| BasePackageInfo, | BasePackageInfo, | ||||
| PackageLoader, | PackageLoader, | ||||
| RawExtrinsicMetadataCore, | RawExtrinsicMetadataCore, | ||||
| ) | ) | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| ▲ Show 20 Lines • Show All 168 Lines • ▼ Show 20 Lines | ) -> Optional[Revision]: | ||||
| "when": self.visit_date.isoformat(), | "when": self.visit_date.isoformat(), | ||||
| "raw": p_info.raw_info, | "raw": p_info.raw_info, | ||||
| }, | }, | ||||
| }, | }, | ||||
| ) | ) | ||||
| def retrieve_sources(url: str) -> bytes: | def retrieve_sources(url: str) -> bytes: | ||||
| response = requests.get(url, allow_redirects=True) | return api_info(url, allow_redirects=True) | ||||
| if response.status_code != 200: | |||||
| raise ValueError("Got %d HTTP code on %s", response.status_code, url) | |||||
| return response.content | |||||
| def parse_sources(raw_sources: bytes) -> Dict[str, Any]: | def parse_sources(raw_sources: bytes) -> Dict[str, Any]: | ||||
| return json.loads(raw_sources.decode("utf-8")) | return json.loads(raw_sources.decode("utf-8")) | ||||
| def clean_sources(sources: Dict[str, Any]) -> Dict[str, Any]: | def clean_sources(sources: Dict[str, Any]) -> Dict[str, Any]: | ||||
| """Validate and clean the sources structure. First, it ensures all top | """Validate and clean the sources structure. First, it ensures all top | ||||
| ▲ Show 20 Lines • Show All 55 Lines • Show Last 20 Lines | |||||