Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/backends/azure.py
| # Copyright (C) 2016-2020 The Software Heritage developers | # Copyright (C) 2016-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import datetime | import datetime | ||||
| from itertools import product | from itertools import product | ||||
| import string | import string | ||||
| from typing import Dict, Optional, Union | from typing import Dict, Optional, Union | ||||
| import warnings | import warnings | ||||
| from azure.storage.blob import ( | from azure.storage.blob import ( | ||||
| ContainerClient, | ContainerClient, | ||||
| ContainerSasPermissions, | ContainerSasPermissions, | ||||
| generate_container_sas, | generate_container_sas, | ||||
| ) | ) | ||||
| from azure.core.exceptions import ResourceNotFoundError | from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError | ||||
| from swh.objstorage.objstorage import ( | from swh.objstorage.objstorage import ( | ||||
| ObjStorage, | ObjStorage, | ||||
| compute_hash, | compute_hash, | ||||
| compressors, | compressors, | ||||
| decompressors, | decompressors, | ||||
| ) | ) | ||||
| from swh.objstorage.exc import ObjNotFoundError, Error | from swh.objstorage.exc import ObjNotFoundError, Error | ||||
| ▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | def add(self, content, obj_id=None, check_presence=True): | ||||
| hex_obj_id = self._internal_id(obj_id) | hex_obj_id = self._internal_id(obj_id) | ||||
| # Send the compressed content | # Send the compressed content | ||||
| compressor = compressors[self.compression]() | compressor = compressors[self.compression]() | ||||
| data = compressor.compress(content) | data = compressor.compress(content) | ||||
| data += compressor.flush() | data += compressor.flush() | ||||
| client = self.get_blob_client(hex_obj_id) | client = self.get_blob_client(hex_obj_id) | ||||
| try: | |||||
| client.upload_blob(data=data, length=len(data)) | client.upload_blob(data=data, length=len(data)) | ||||
| except ResourceExistsError: | |||||
| # There's a race condition between check_presence and upload_blob, | |||||
| # that we can't get rid of as the azure api doesn't allow atomic | |||||
| # replaces or renaming a blob. As the restore operation explicitly | |||||
| # removes the blob, it should be safe to just ignore the error. | |||||
| pass | |||||
| return obj_id | return obj_id | ||||
| def restore(self, content, obj_id=None): | def restore(self, content, obj_id=None): | ||||
| """Restore a content. | """Restore a content. | ||||
| """ | """ | ||||
| if obj_id is None: | if obj_id is None: | ||||
| ▲ Show 20 Lines • Show All 122 Lines • Show Last 20 Lines | |||||