diff --git a/swh/loader/package/functional/loader.py b/swh/loader/package/functional/loader.py --- a/swh/loader/package/functional/loader.py +++ b/swh/loader/package/functional/loader.py @@ -5,13 +5,15 @@ import json import requests +import tempfile -from typing import Dict, Optional, Any, Mapping +from typing import Dict, Optional, Any, Mapping, Tuple from swh.model import hashutil +from swh.model import from_disk from swh.model.model import ( - Sha1Git, Revision, RevisionType + Sha1Git, Revision, RevisionType, Release, ObjectType ) from swh.loader.package.utils import EMPTY_AUTHOR @@ -113,3 +115,45 @@ }, } ) + + def _load_target( + self, p_info, origin) \ + -> Tuple[str, Optional[Sha1Git], bool]: + + revision_id, loaded = self._load_revision(p_info, origin) + + if loaded is True: + return 'revision', revision_id, loaded + + import logging + logger = logging.getLogger(__name__) + logger.error("fILE\n\n") + + with tempfile.TemporaryDirectory() as tmpdir: + try: + dl_artifacts = self.download_package(p_info, tmpdir) + except Exception: + logger.exception('Unable to retrieve %s', + p_info) + return ('', None, False) + + content = from_disk.Content.from_file( + path=dl_artifacts[0][0].encode('utf-8'), + max_content_length=self.max_content_size) + + self.storage.content_add([content.to_model()]) + + release = Release( + name=b'iop', + message=b'iop', + author=EMPTY_AUTHOR, + date=None, + target_type=ObjectType.CONTENT, + target=content.hash, + synthetic=True + ) + self.storage.release_add([release]) + + return 'release', release.id, True + + return '', None, False diff --git a/swh/loader/package/functional/tests/test_functional.py b/swh/loader/package/functional/tests/test_functional.py --- a/swh/loader/package/functional/tests/test_functional.py +++ b/swh/loader/package/functional/tests/test_functional.py @@ -40,12 +40,12 @@ stats = get_stats(loader.storage) assert { - 'content': 1, + 'content': 2, 'directory': 3, 'origin': 1, 'origin_visit': 1, 'person': 1, - 'release': 0, + 'release': 1, 'revision': 1, 'skipped_content': 0, 'snapshot': 1 @@ -54,7 +54,7 @@ origin_visit = next(loader.storage.origin_visit_get(sources_url)) # The visit is partial because urls pointing to non tarball file # are not handled yet - assert origin_visit['status'] == 'partial' + assert origin_visit['status'] == 'full' assert origin_visit['type'] == 'functional' diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -172,6 +172,13 @@ """ return None + def resolve_target_from( + self, known_artifacts: Dict, artifact_metadata: Dict) \ + -> Tuple[str, Optional[bytes]]: + return 'revision', self.resolve_revision_from( + known_artifacts, + artifact_metadata) + def download_package(self, p_info: Mapping[str, Any], tmpdir: str) -> List[Tuple[str, Mapping]]: """Download artifacts for a specific package. All downloads happen in @@ -290,19 +297,23 @@ # `p_` stands for `package_` for branch_name, p_info in self.get_package_info(version): logger.debug('package_info: %s', p_info) - revision_id = self.resolve_revision_from( + target_type, target_id = self.resolve_target_from( known_artifacts, p_info['raw']) - if revision_id is None: - (revision_id, loaded) = \ - self._load_revision(p_info, origin) + if target_id is None: + (target_type, target_id, loaded) = \ + self._load_target(p_info, origin) + if loaded: status_load = 'eventful' else: status_visit = 'partial' - if revision_id is None: + if target_id is None: continue - tmp_revisions[version].append((branch_name, revision_id)) + tmp_revisions[version].append(( + branch_name, + target_type, + target_id)) logger.debug('tmp_revisions: %s', tmp_revisions) # Build and load the snapshot @@ -320,9 +331,9 @@ 'target': branch_name.encode('utf-8'), } - for branch_name, target in branch_name_revisions: + for branch_name, target_type, target in branch_name_revisions: branches[branch_name.encode('utf-8')] = { - 'target_type': 'revision', + 'target_type': target_type, 'target': target, } @@ -436,3 +447,9 @@ self.storage.revision_add([revision]) return (revision.id, True) + + def _load_target( + self, p_info, origin) \ + -> Tuple[str, Optional[Sha1Git], bool]: + revision_id, loaded = self._load_revision(p_info, origin) + return 'revision', revision_id, loaded