diff --git a/debian/control b/debian/control --- a/debian/control +++ b/debian/control @@ -14,7 +14,7 @@ python3-swh.model (>= 0.0.15), python3-swh.objstorage (>= 0.0.17), python3-swh.scheduler (>= 0.0.14), - python3-swh.storage (>= 0.0.88), + python3-swh.storage (>= 0.0.100), python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/source/swh-archiver/ diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -3,4 +3,4 @@ swh.model >= 0.0.15 swh.objstorage >= 0.0.17 swh.scheduler >= 0.0.14 -swh.storage >= 0.0.88 +swh.storage >= 0.0.100 diff --git a/swh/archiver/db.py b/swh/archiver/db.py --- a/swh/archiver/db.py +++ b/swh/archiver/db.py @@ -166,14 +166,13 @@ """Add new content archive entries from temporary table. Use from archiver.storage module:: - - self.db.mktemp_content_archive() + db.mktemp_content_archive(cur) # copy data over to the temp table - self.db.copy_to([{'colname': id0}, {'colname': id1}], - 'tmp_cache_content', - ['colname'], cur) + db.copy_to([{'colname': id0}, {'colname': id1}], + 'tmp_cache_content', + ['colname'], cur) # insert into the main table - self.db.add_content_archive_from_temp(cur) + db.add_content_archive_from_temp(cur) """ pass diff --git a/swh/archiver/storage.py b/swh/archiver/storage.py --- a/swh/archiver/storage.py +++ b/swh/archiver/storage.py @@ -27,23 +27,26 @@ """ try: if isinstance(dbconn, psycopg2.extensions.connection): - self.db = ArchiverDb(dbconn) + self._db = ArchiverDb(dbconn) else: - self.db = ArchiverDb.connect(dbconn) + self._db = ArchiverDb.connect(dbconn) except psycopg2.OperationalError as e: raise StorageDBError(e) + def get_db(self): + return self._db + @db_transaction_generator - def archive_ls(self, cur=None): + def archive_ls(self, db=None, cur=None): """ Get all the archives registered on the server. Yields: a tuple (server_id, server_url) for each archive server. """ - yield from self.db.archive_ls(cur) + yield from db.archive_ls(cur) @db_transaction - def content_archive_get(self, content_id, cur=None): + def content_archive_get(self, content_id, db=None, cur=None): """ Get the archival status of a content. Retrieve from the database the archival status of the given content @@ -55,11 +58,11 @@ A tuple (content_id, present_copies, ongoing_copies), where ongoing_copies is a dict mapping copy to mtime. """ - return self.db.content_archive_get(content_id, cur) + return db.content_archive_get(content_id, cur) @db_transaction_generator def content_archive_get_copies(self, last_content=None, limit=1000, - cur=None): + db=None, cur=None): """ Get the list of copies for `limit` contents starting after `last_content`. @@ -74,13 +77,12 @@ ongoing_copies is a dict mapping copy to mtime. """ - yield from self.db.content_archive_get_copies(last_content, limit, - cur) + yield from db.content_archive_get_copies(last_content, limit, cur) @db_transaction_generator def content_archive_get_unarchived_copies( self, retention_policy, last_content=None, - limit=1000, cur=None): + limit=1000, db=None, cur=None): """ Get the list of copies for `limit` contents starting after `last_content`. Yields only copies with number of present smaller than `retention policy`. @@ -97,11 +99,12 @@ ongoing_copies is a dict mapping copy to mtime. """ - yield from self.db.content_archive_get_unarchived_copies( + yield from db.content_archive_get_unarchived_copies( retention_policy, last_content, limit, cur) @db_transaction_generator - def content_archive_get_missing(self, content_ids, backend_name, cur=None): + def content_archive_get_missing(self, content_ids, backend_name, db=None, + cur=None): """Retrieve missing sha1s from source_name. Args: @@ -112,9 +115,7 @@ missing sha1s from backend_name """ - db = self.db - - db.mktemp_content_archive() + db.mktemp_content_archive(cur) db.copy_to(content_ids, 'tmp_content_archive', ['content_id'], cur) @@ -122,7 +123,7 @@ yield content_id[0] @db_transaction_generator - def content_archive_get_unknown(self, content_ids, cur=None): + def content_archive_get_unknown(self, content_ids, db=None, cur=None): """Retrieve unknown sha1s from content_archive. Args: @@ -132,9 +133,7 @@ Unknown sha1s from content_archive """ - db = self.db - - db.mktemp_content_archive() + db.mktemp_content_archive(cur) db.copy_to(content_ids, 'tmp_content_archive', ['content_id'], cur) @@ -143,7 +142,7 @@ @db_transaction def content_archive_update(self, content_id, archive_id, - new_status=None, cur=None): + new_status=None, db=None, cur=None): """ Update the status of an archive content and set its mtime to now Change the mtime of an archived content for the given archive and set @@ -157,11 +156,11 @@ the function only change the mtime of the content for the given archive. """ - self.db.content_archive_update(content_id, archive_id, new_status, cur) + db.content_archive_update(content_id, archive_id, new_status, cur) @db_transaction def content_archive_add( - self, content_ids, sources_present, cur=None): + self, content_ids, sources_present, db=None, cur=None): """Insert a new entry in db about content_id. Args: @@ -169,8 +168,6 @@ sources_present ([str]): List of source names where contents are present """ - db = self.db - # Prepare copies dictionary copies = {} for source in sources_present: @@ -182,7 +179,7 @@ copies = json.dumps(copies) num_present = len(sources_present) - db.mktemp('content_archive') + db.mktemp('content_archive', cur) db.copy_to( ({'content_id': id, 'copies': copies,