Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/db.py
Show First 20 Lines • Show All 696 Lines • ▼ Show 20 Lines | def content_archive_get(self, content_id, cur=None): | ||||
ORDER BY content_id | ORDER BY content_id | ||||
""" | """ | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute(query, (content_id,)) | cur.execute(query, (content_id,)) | ||||
content_id, present, ongoing, mtimes = cur.fetchone() | content_id, present, ongoing, mtimes = cur.fetchone() | ||||
return (content_id, present, dict(zip(ongoing, mtimes))) | return (content_id, present, dict(zip(ongoing, mtimes))) | ||||
def content_archive_get_copies(self, previous_content=None, limit=1000, | def content_archive_get_copies(self, last_content=None, limit=1000, | ||||
cur=None): | cur=None): | ||||
"""Get the list of copies for `limit` contents starting after | """Get the list of copies for `limit` contents starting after | ||||
`previous_content`. | `last_content`. | ||||
Args: | Args: | ||||
previous_content: sha1 of the last content retrieved. May be None | last_content: sha1 of the last content retrieved. May be None | ||||
to start at the beginning. | to start at the beginning. | ||||
limit: number of contents to retrieve. Can be None to retrieve all | limit: number of contents to retrieve. Can be None to retrieve all | ||||
objects (will be slow). | objects (will be slow). | ||||
Yields: | Yields: | ||||
A tuple (content_id, present_copies, ongoing_copies), where | A tuple (content_id, present_copies, ongoing_copies), where | ||||
ongoing_copies is a dict mapping copy to mtime. | ongoing_copies is a dict mapping copy to mtime. | ||||
Show All 19 Lines | def content_archive_get_copies(self, last_content=None, limit=1000, | ||||
ORDER BY key | ORDER BY key | ||||
) AS ongoing_mtime | ) AS ongoing_mtime | ||||
FROM content_archive | FROM content_archive | ||||
WHERE content_id > %s | WHERE content_id > %s | ||||
ORDER BY content_id | ORDER BY content_id | ||||
LIMIT %s | LIMIT %s | ||||
""" | """ | ||||
if previous_content is None: | if last_content is None: | ||||
previous_content = b'' | last_content = b'' | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute(query, (previous_content, limit)) | cur.execute(query, (last_content, limit)) | ||||
for content_id, present, ongoing, mtimes in cursor_to_bytes(cur): | |||||
yield (content_id, present, dict(zip(ongoing, mtimes))) | |||||
def content_archive_get_unarchived_copies( | |||||
self, retention_policy, last_content=None, | |||||
limit=1000, cur=None): | |||||
""" Get the list of copies for `limit` contents starting after | |||||
`last_content`. Yields only copies with number of present | |||||
smaller than `retention policy`. | |||||
Args: | |||||
last_content: sha1 of the last content retrieved. May be None | |||||
to start at the beginning. | |||||
retention_policy: number of presentcopies required. | |||||
limit: number of contents to retrieve. Can be None to retrieve all | |||||
objects (will be slow). | |||||
Yields: | |||||
A tuple (content_id, present_copies, ongoing_copies), where | |||||
ongoing_copies is a dict mapping copy to mtime. | |||||
""" | |||||
query = """SELECT content_id, | |||||
array( | |||||
SELECT key | |||||
FROM jsonb_each(copies) | |||||
WHERE value->>'status' = 'present' | |||||
ORDER BY key | |||||
) AS present, | |||||
array( | |||||
SELECT key | |||||
FROM jsonb_each(copies) | |||||
WHERE value->>'status' = 'ongoing' | |||||
ORDER BY key | |||||
) AS ongoing, | |||||
array( | |||||
SELECT value->'mtime' | |||||
FROM jsonb_each(copies) | |||||
WHERE value->>'status' = 'ongoing' | |||||
ORDER BY key | |||||
) AS ongoing_mtime | |||||
FROM content_archive | |||||
WHERE content_id > %s AND num_present < %s | |||||
ORDER BY content_id | |||||
LIMIT %s | |||||
""" | |||||
if last_content is None: | |||||
last_content = b'' | |||||
cur = self._cursor(cur) | |||||
cur.execute(query, (last_content, retention_policy, limit)) | |||||
for content_id, present, ongoing, mtimes in cursor_to_bytes(cur): | for content_id, present, ongoing, mtimes in cursor_to_bytes(cur): | ||||
yield (content_id, present, dict(zip(ongoing, mtimes))) | yield (content_id, present, dict(zip(ongoing, mtimes))) | ||||
def content_archive_update(self, content_id, archive_id, | def content_archive_update(self, content_id, archive_id, | ||||
new_status=None, cur=None): | new_status=None, cur=None): | ||||
zack: same comments about the function prototype as above | |||||
""" Update the status of an archive content and set its mtime to | """ Update the status of an archive content and set its mtime to | ||||
Change the mtime of an archived content for the given archive and set | Change the mtime of an archived content for the given archive and set | ||||
it's mtime to the current time. | it's mtime to the current time. | ||||
Args: | Args: | ||||
content_id (str): content sha1 | content_id (str): content sha1 | ||||
archive_id (str): name of the archive | archive_id (str): name of the archive | ||||
Show All 23 Lines |
same comments about the function prototype as above