Changeset View
Changeset View
Standalone View
Standalone View
swh/archiver/db.py
# Copyright (C) 2015-2017 The Software Heritage developers | # Copyright (C) 2015-2017 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import datetime | import datetime | ||||
from swh.storage.db import BaseDb, cursor_to_bytes, stored_procedure | from swh.core.db import BaseDb | ||||
from swh.core.db.db_utils import stored_procedure | |||||
def utcnow(): | def utcnow(): | ||||
return datetime.datetime.now(tz=datetime.timezone.utc) | return datetime.datetime.now(tz=datetime.timezone.utc) | ||||
class ArchiverDb(BaseDb): | class ArchiverDb(BaseDb): | ||||
"""Proxy to the SWH's archiver DB | """Proxy to the SWH's archiver DB | ||||
""" | """ | ||||
def archive_ls(self, cur=None): | def archive_ls(self, cur=None): | ||||
""" Get all the archives registered on the server. | """ Get all the archives registered on the server. | ||||
Yields: | Yields: | ||||
a tuple (server_id, server_url) for each archive server. | a tuple (server_id, server_url) for each archive server. | ||||
""" | """ | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("SELECT * FROM archive") | cur.execute("SELECT * FROM archive") | ||||
yield from cursor_to_bytes(cur) | yield from cur | ||||
def content_archive_get(self, content_id, cur=None): | def content_archive_get(self, content_id, cur=None): | ||||
""" Get the archival status of a content in a specific server. | """ Get the archival status of a content in a specific server. | ||||
Retrieve from the database the archival status of the given content | Retrieve from the database the archival status of the given content | ||||
in the given archive server. | in the given archive server. | ||||
Args: | Args: | ||||
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | def content_archive_get_copies(self, last_content=None, limit=1000, | ||||
from content_copies | from content_copies | ||||
where status = 'present' and %s | where status = 'present' and %s | ||||
group by content_id | group by content_id | ||||
order by content_id | order by content_id | ||||
limit %%(limit)s""" % last_content_clause | limit %%(limit)s""" % last_content_clause | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute(query, vars) | cur.execute(query, vars) | ||||
for content_id, present in cursor_to_bytes(cur): | for content_id, present in cur: | ||||
yield (content_id, present, {}) | yield (content_id, present, {}) | ||||
def content_archive_get_unarchived_copies( | def content_archive_get_unarchived_copies( | ||||
self, retention_policy, last_content=None, | self, retention_policy, last_content=None, | ||||
limit=1000, cur=None): | limit=1000, cur=None): | ||||
""" Get the list of copies for `limit` contents starting after | """ Get the list of copies for `limit` contents starting after | ||||
`last_content`. Yields only copies with number of present | `last_content`. Yields only copies with number of present | ||||
smaller than `retention policy`. | smaller than `retention policy`. | ||||
Show All 32 Lines | def content_archive_get_unarchived_copies( | ||||
where status = 'present' and %s | where status = 'present' and %s | ||||
group by content_id | group by content_id | ||||
having count(archive_id) < %%(retention_policy)s | having count(archive_id) < %%(retention_policy)s | ||||
order by content_id | order by content_id | ||||
limit %%(limit)s""" % last_content_clause | limit %%(limit)s""" % last_content_clause | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute(query, vars) | cur.execute(query, vars) | ||||
for content_id, present in cursor_to_bytes(cur): | for content_id, present in cur: | ||||
yield (content_id, present, {}) | yield (content_id, present, {}) | ||||
@stored_procedure('swh_mktemp_content_archive') | @stored_procedure('swh_mktemp_content_archive') | ||||
def mktemp_content_archive(self, cur=None): | def mktemp_content_archive(self, cur=None): | ||||
"""Trigger the creation of the temporary table tmp_content_archive | """Trigger the creation of the temporary table tmp_content_archive | ||||
during the lifetime of the transaction. | during the lifetime of the transaction. | ||||
""" | """ | ||||
Show All 20 Lines | class ArchiverDb(BaseDb): | ||||
def content_archive_get_missing(self, backend_name, cur=None): | def content_archive_get_missing(self, backend_name, cur=None): | ||||
"""Retrieve the content missing from backend_name. | """Retrieve the content missing from backend_name. | ||||
""" | """ | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute("select * from swh_content_archive_missing(%s)", | cur.execute("select * from swh_content_archive_missing(%s)", | ||||
(backend_name,)) | (backend_name,)) | ||||
yield from cursor_to_bytes(cur) | yield from cur | ||||
def content_archive_get_unknown(self, cur=None): | def content_archive_get_unknown(self, cur=None): | ||||
"""Retrieve unknown sha1 from archiver db. | """Retrieve unknown sha1 from archiver db. | ||||
""" | """ | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
cur.execute('select * from swh_content_archive_unknown()') | cur.execute('select * from swh_content_archive_unknown()') | ||||
yield from cursor_to_bytes(cur) | yield from cur | ||||
def content_archive_update(self, content_id, archive_id, | def content_archive_update(self, content_id, archive_id, | ||||
new_status=None, cur=None): | new_status=None, cur=None): | ||||
""" Update the status of an archive content and set its mtime to | """ Update the status of an archive content and set its mtime to | ||||
Change the mtime of an archived content for the given archive and set | Change the mtime of an archived content for the given archive and set | ||||
it's mtime to the current time. | it's mtime to the current time. | ||||
Show All 21 Lines |