Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/db.py
Show First 20 Lines • Show All 262 Lines • ▼ Show 20 Lines | def store_tmp_bytea(self, ids, cur=None): | ||||
self.copy_to(({'id': elem} for elem in ids), 'tmp_bytea', | self.copy_to(({'id': elem} for elem in ids), 'tmp_bytea', | ||||
['id'], cur) | ['id'], cur) | ||||
def content_update_from_temp(self, keys_to_update, cur=None): | def content_update_from_temp(self, keys_to_update, cur=None): | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""select swh_content_update(ARRAY[%s] :: text[])""" % | cur.execute("""select swh_content_update(ARRAY[%s] :: text[])""" % | ||||
keys_to_update) | keys_to_update) | ||||
content_get_metadata_keys = ['sha1', 'sha1_git', 'sha256', 'length', | content_get_metadata_keys = [ | ||||
'status'] | 'sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', 'status'] | ||||
skipped_content_keys = [ | |||||
'sha1', 'sha1_git', 'sha256', 'blake2s256', | |||||
'length', 'reason', 'status', 'origin'] | |||||
def content_get_metadata_from_temp(self, cur=None): | def content_get_metadata_from_temp(self, cur=None): | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""select t.id as sha1, %s from tmp_bytea t | cur.execute("""select t.id as sha1, %s from tmp_bytea t | ||||
left join content on t.id = content.sha1 | left join content on t.id = content.sha1 | ||||
""" % ', '.join(self.content_get_metadata_keys[1:])) | """ % ', '.join(self.content_get_metadata_keys[1:])) | ||||
yield from cursor_to_bytes(cur) | yield from cursor_to_bytes(cur) | ||||
def content_missing_from_temp(self, cur=None): | def content_missing_from_temp(self, cur=None): | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""SELECT sha1, sha1_git, sha256 | cur.execute("""SELECT sha1, sha1_git, sha256, blake2s256 | ||||
FROM swh_content_missing()""") | FROM swh_content_missing()""") | ||||
yield from cursor_to_bytes(cur) | yield from cursor_to_bytes(cur) | ||||
def content_missing_per_sha1_from_temp(self, cur=None): | def content_missing_per_sha1_from_temp(self, cur=None): | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""SELECT * | cur.execute("""SELECT * | ||||
FROM swh_content_missing_per_sha1()""") | FROM swh_content_missing_per_sha1()""") | ||||
yield from cursor_to_bytes(cur) | yield from cursor_to_bytes(cur) | ||||
def skipped_content_missing_from_temp(self, cur=None): | def skipped_content_missing_from_temp(self, cur=None): | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""SELECT sha1, sha1_git, sha256 | cur.execute("""SELECT sha1, sha1_git, sha256, blake2s256 | ||||
FROM swh_skipped_content_missing()""") | FROM swh_skipped_content_missing()""") | ||||
yield from cursor_to_bytes(cur) | yield from cursor_to_bytes(cur) | ||||
def occurrence_get(self, origin_id, cur=None): | def occurrence_get(self, origin_id, cur=None): | ||||
"""Retrieve latest occurrence's information by origin_id. | """Retrieve latest occurrence's information by origin_id. | ||||
""" | """ | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""SELECT origin, branch, target, target_type, | cur.execute("""SELECT origin, branch, target, target_type, | ||||
(select max(date) from origin_visit | (select max(date) from origin_visit | ||||
where origin=%s) as date | where origin=%s) as date | ||||
FROM occurrence | FROM occurrence | ||||
WHERE origin=%s | WHERE origin=%s | ||||
""", | """, | ||||
(origin_id, origin_id)) | (origin_id, origin_id)) | ||||
yield from cursor_to_bytes(cur) | yield from cursor_to_bytes(cur) | ||||
def content_find(self, sha1=None, sha1_git=None, sha256=None, cur=None): | content_find_cols = ['sha1', 'sha1_git', 'sha256', 'blake2s256', 'length', | ||||
'ctime', 'status'] | |||||
def content_find(self, sha1=None, sha1_git=None, sha256=None, | |||||
blake2s256=None, cur=None): | |||||
"""Find the content optionally on a combination of the following | """Find the content optionally on a combination of the following | ||||
checksums sha1, sha1_git or sha256. | checksums sha1, sha1_git, sha256 or blake2s256. | ||||
Args: | Args: | ||||
sha1: sha1 content | sha1: sha1 content | ||||
git_sha1: the sha1 computed `a la git` sha1 of the content | git_sha1: the sha1 computed `a la git` sha1 of the content | ||||
sha256: sha256 content | sha256: sha256 content | ||||
blake2s256: blake2s256 content | |||||
Returns: | Returns: | ||||
The triplet (sha1, sha1_git, sha256) if found or None. | The tuple (sha1, sha1_git, sha256, blake2s256) if found or None. | ||||
""" | """ | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("""SELECT sha1, sha1_git, sha256, length, ctime, status | cur.execute("""SELECT %s | ||||
FROM swh_content_find(%s, %s, %s) | FROM swh_content_find(%%s, %%s, %%s, %%s) | ||||
LIMIT 1""", (sha1, sha1_git, sha256)) | LIMIT 1""" % ','.join(self.content_find_cols), | ||||
(sha1, sha1_git, sha256, blake2s256)) | |||||
content = line_to_bytes(cur.fetchone()) | content = line_to_bytes(cur.fetchone()) | ||||
if set(content) == {None}: | if set(content) == {None}: | ||||
return None | return None | ||||
else: | else: | ||||
return content | return content | ||||
provenance_cols = ['content', 'revision', 'origin', 'visit', 'path'] | provenance_cols = ['content', 'revision', 'origin', 'visit', 'path'] | ||||
▲ Show 20 Lines • Show All 629 Lines • Show Last 20 Lines |