Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/filter.py
Show First 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | def content_add(self, content: Iterable[Dict]) -> Dict: | ||||
return self.storage.content_add( | return self.storage.content_add( | ||||
x for x in contents if x['sha256'] in contents_to_add | x for x in contents if x['sha256'] in contents_to_add | ||||
) | ) | ||||
def skipped_content_add(self, content: Iterable[Dict]) -> Dict: | def skipped_content_add(self, content: Iterable[Dict]) -> Dict: | ||||
contents = list(content) | contents = list(content) | ||||
contents_to_add = self._filter_missing_skipped_contents(contents) | contents_to_add = self._filter_missing_skipped_contents(contents) | ||||
return self.storage.skipped_content_add( | return self.storage.skipped_content_add( | ||||
x for x in contents if x['sha1_git'] in contents_to_add | x for x in contents | ||||
if x.get('sha1_git') is None or x['sha1_git'] in contents_to_add | |||||
) | ) | ||||
def directory_add(self, directories: Iterable[Dict]) -> Dict: | def directory_add(self, directories: Iterable[Dict]) -> Dict: | ||||
directories = list(directories) | directories = list(directories) | ||||
missing_ids = self._filter_missing_ids( | missing_ids = self._filter_missing_ids( | ||||
'directory', | 'directory', | ||||
(d['id'] for d in directories) | (d['id'] for d in directories) | ||||
) | ) | ||||
Show All 40 Lines | def _filter_missing_skipped_contents( | ||||
Args: | Args: | ||||
content_hashes: List of sha1_git to check for existence in swh | content_hashes: List of sha1_git to check for existence in swh | ||||
storage | storage | ||||
""" | """ | ||||
objects_seen = self.objects_seen['skipped_content'] | objects_seen = self.objects_seen['skipped_content'] | ||||
missing_hashes = [] | missing_hashes = [] | ||||
for hashes in content_hashes: | for hashes in content_hashes: | ||||
if hashes['sha1_git'] in objects_seen: | if hashes.get('sha1_git') is None \ | ||||
or hashes['sha1_git'] in objects_seen: | |||||
continue | continue | ||||
objects_seen.add(hashes['sha1_git']) | objects_seen.add(hashes['sha1_git']) | ||||
missing_hashes.append(hashes) | missing_hashes.append(hashes) | ||||
return {c['sha1_git'] | return {c['sha1_git'] | ||||
for c in self.storage.skipped_content_missing(missing_hashes)} | for c in self.storage.skipped_content_missing(missing_hashes)} | ||||
def _filter_missing_ids( | def _filter_missing_ids( | ||||
Show All 28 Lines |