Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_archiver.py
| Show First 20 Lines • Show All 106 Lines • ▼ Show 20 Lines | def __get_missing(self): | ||||
| self.cursor.execute("""SELECT content_id | self.cursor.execute("""SELECT content_id | ||||
| FROM content_archive | FROM content_archive | ||||
| WHERE status='missing'""") | WHERE status='missing'""") | ||||
| return self.cursor.fetchall() | return self.cursor.fetchall() | ||||
| def __create_director(self, batch_size=5000, archival_max_age=3600, | def __create_director(self, batch_size=5000, archival_max_age=3600, | ||||
| retention_policy=1, asynchronous=False): | retention_policy=1, asynchronous=False): | ||||
| config = { | config = { | ||||
| 'objstorage_type': 'local_objstorage', | |||||
| 'objstorage_path': self.objroot, | 'objstorage_path': self.objroot, | ||||
| 'objstorage_slicing': '0:2/2:4/4:6', | |||||
| 'batch_max_size': batch_size, | 'batch_max_size': batch_size, | ||||
| 'archival_max_age': archival_max_age, | 'archival_max_age': archival_max_age, | ||||
| 'retention_policy': retention_policy, | 'retention_policy': retention_policy, | ||||
| 'asynchronous': asynchronous # Avoid depending on queue for tests. | 'asynchronous': asynchronous # Avoid depending on queue for tests. | ||||
| } | } | ||||
| director = ArchiverDirector(db_conn_archiver=self.conn, | director = ArchiverDirector(db_conn_archiver=self.conn, | ||||
| db_conn_storage=self.conn_storage, | |||||
| config=config) | config=config) | ||||
| return director | return director | ||||
| def __create_worker(self, batch={}, config={}): | def __create_worker(self, batch={}, config={}): | ||||
| mstorage_args = [ | mobjstorage_args = self.archiver.master_objstorage_args | ||||
| self.archiver.master_storage.db.conn, # master storage db | |||||
| # connection | |||||
| self.objroot # object storage path | |||||
| ] | |||||
| if not config: | if not config: | ||||
| config = self.archiver.config | config = self.archiver.config | ||||
| return ArchiverWorker(batch, | return ArchiverWorker(batch, | ||||
| archiver_args=self.conn, | archiver_args=self.conn, | ||||
| master_storage_args=mstorage_args, | master_objstorage_args=mobjstorage_args, | ||||
| slave_storages=[self.storage_data], | slave_objstorages=[self.storage_data], | ||||
| config=config) | config=config) | ||||
| # Integration test | # Integration test | ||||
| def archive_missing_content(self): | def archive_missing_content(self): | ||||
| """ Run archiver on a missing content should archive it. | """ Run archiver on a missing content should archive it. | ||||
| """ | """ | ||||
| content_data = b'archive_missing_content' | content_data = b'archive_missing_content' | ||||
| ▲ Show 20 Lines • Show All 132 Lines • Show Last 20 Lines | |||||