Differential D2854 Diff 10151 swh/loader/package/functional/loader.py

Changeset View

Standalone View

swh/loader/package/functional/loader.py

# Copyright (C) 2020 The Software Heritage developers		# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import json		import json
import requests		import requests
		import tempfile

from typing import Dict, Optional, Any, Mapping		from typing import Dict, Optional, Any, Mapping, Tuple

from swh.model import hashutil		from swh.model import hashutil
		from swh.model import from_disk

from swh.model.model import (		from swh.model.model import (
Sha1Git, Revision, RevisionType		Sha1Git, Revision, RevisionType, Release, ObjectType
)		)

from swh.loader.package.utils import EMPTY_AUTHOR		from swh.loader.package.utils import EMPTY_AUTHOR

from swh.loader.package.loader import PackageLoader		from swh.loader.package.loader import PackageLoader


def retrieve_sources(url: str) -> Dict[str, Any]:		def retrieve_sources(url: str) -> Dict[str, Any]:
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines	def build_revision(self, a_metadata: Dict, uncompressed_path: str,
metadata={		metadata={
'extrinsic': {		'extrinsic': {
'provider': self.provider_url,		'provider': self.provider_url,
'when': self.visit_date.isoformat(),		'when': self.visit_date.isoformat(),
'raw': a_metadata,		'raw': a_metadata,
},		},
}		}
)		)

		def _load_target(
		self, p_info, origin) \
		-> Tuple[str, Optional[Sha1Git], bool]:

		revision_id, loaded = self._load_revision(p_info, origin)

		if loaded is True:
		return 'revision', revision_id, loaded

		import logging
		logger = logging.getLogger(__name__)
		logger.error("fILE\n\n")

		with tempfile.TemporaryDirectory() as tmpdir:
		try:
		dl_artifacts = self.download_package(p_info, tmpdir)
		except Exception:
		logger.exception('Unable to retrieve %s',
		p_info)
		return ('', None, False)

		content = from_disk.Content.from_file(
		path=dl_artifacts[0][0],
		max_content_length=self.max_content_size)

		ardumontUnsubmitted Not Done Inline Actions beware `to_model` can return a `Content` or a `SkippedContent`. That will not be the same endpoint for the storage: Content: `self.storage.content_add` SkippedContent: `self.storage.skipped_content_add` ardumont: beware `to_model` can return a `Content` or a `SkippedContent`. That will not be the same…
		self.storage.content_add([content.to_model()])

		release = Release(
		name=b'',
		message=b'',
		author=EMPTY_AUTHOR,
		date=None,
		target_type=ObjectType.CONTENT,
		target=content.hash,
		synthetic=True
		)

		return 'release', release.id, True
		ardumontUnsubmitted Not Done Inline Actions Discussing this case in [1], in the end, we could simply add a SnapshotBranch which targets a TargetType Content. No need to create a Release object. [1] https://forge.softwareheritage.org/D3745?id=13193#inline-25905 ardumont: Discussing this case in [1], in the end, we could simply add a SnapshotBranch which targets a…

		return '', None, False