Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/backends/azure.py
| Show All 32 Lines | |||||
| def get_container_url( | def get_container_url( | ||||
| account_name: str, | account_name: str, | ||||
| account_key: str, | account_key: str, | ||||
| container_name: str, | container_name: str, | ||||
| access_policy: str = "read_only", | access_policy: str = "read_only", | ||||
| expiry: datetime.timedelta = datetime.timedelta(days=365), | expiry: datetime.timedelta = datetime.timedelta(days=365), | ||||
| container_url_template: str = ( | |||||
| "https://{account_name}.blob.core.windows.net/{container_name}?{signature}" | |||||
| ), | |||||
| **kwargs, | **kwargs, | ||||
| ) -> str: | ) -> str: | ||||
| """Get the full url, for the given container on the given account, with a | """Get the full url, for the given container on the given account, with a | ||||
| Shared Access Signature granting the specified access policy. | Shared Access Signature granting the specified access policy. | ||||
| Args: | Args: | ||||
| account_name: name of the storage account for which to generate the URL | account_name: name of the storage account for which to generate the URL | ||||
| account_key: shared account key of the storage account used to generate the SAS | account_key: shared account key of the storage account used to generate the SAS | ||||
| Show All 22 Lines | signature = generate_container_sas( | ||||
| account_name, | account_name, | ||||
| container_name, | container_name, | ||||
| account_key=account_key, | account_key=account_key, | ||||
| permission=access_policies[access_policy], | permission=access_policies[access_policy], | ||||
| start=current_time + datetime.timedelta(minutes=-1), | start=current_time + datetime.timedelta(minutes=-1), | ||||
| expiry=current_time + expiry, | expiry=current_time + expiry, | ||||
| ) | ) | ||||
| return f"https://{account_name}.blob.core.windows.net/{container_name}?{signature}" | return container_url_template.format( | ||||
| account_name=account_name, | |||||
| container_name=container_name, | |||||
| signature=signature, | |||||
| ) | |||||
| class AzureCloudObjStorage(ObjStorage): | class AzureCloudObjStorage(ObjStorage): | ||||
| """ObjStorage backend for Azure blob storage accounts. | """ObjStorage backend for Azure blob storage accounts. | ||||
| Args: | Args: | ||||
| container_url: the URL of the container in which the objects are stored. | container_url: the URL of the container in which the objects are stored. | ||||
| account_name: (deprecated) the name of the storage account under which objects are | account_name: (deprecated) the name of the storage account under which objects are | ||||
| Show All 13 Lines | class AzureCloudObjStorage(ObjStorage): | ||||
| PRIMARY_HASH: Literal["sha1"] = "sha1" | PRIMARY_HASH: Literal["sha1"] = "sha1" | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| container_url: Optional[str] = None, | container_url: Optional[str] = None, | ||||
| account_name: Optional[str] = None, | account_name: Optional[str] = None, | ||||
| api_secret_key: Optional[str] = None, | api_secret_key: Optional[str] = None, | ||||
| container_name: Optional[str] = None, | container_name: Optional[str] = None, | ||||
| connection_string: Optional[str] = None, | |||||
| compression="gzip", | compression="gzip", | ||||
| **kwargs, | **kwargs, | ||||
| ): | ): | ||||
| if container_url is None: | if container_url is None and connection_string is None: | ||||
| if account_name is None or api_secret_key is None or container_name is None: | if account_name is None or api_secret_key is None or container_name is None: | ||||
| raise ValueError( | raise ValueError( | ||||
| "AzureCloudObjStorage must have a container_url or all three " | "AzureCloudObjStorage must have a container_url, a connection_string," | ||||
| "account_name, api_secret_key and container_name" | "or all three account_name, api_secret_key and container_name" | ||||
| ) | ) | ||||
| else: | else: | ||||
| warnings.warn( | warnings.warn( | ||||
| "The Azure objstorage account secret key parameters are " | "The Azure objstorage account secret key parameters are " | ||||
| "deprecated, please use container URLs instead.", | "deprecated, please use container URLs instead.", | ||||
| DeprecationWarning, | DeprecationWarning, | ||||
| ) | ) | ||||
| container_url = get_container_url( | container_url = get_container_url( | ||||
| account_name=account_name, | account_name=account_name, | ||||
| account_key=api_secret_key, | account_key=api_secret_key, | ||||
| container_name=container_name, | container_name=container_name, | ||||
| access_policy="full", | access_policy="full", | ||||
| ) | ) | ||||
| elif connection_string: | |||||
| if container_name is None: | |||||
| raise ValueError( | |||||
| "container_name is required when using connection_string." | |||||
| ) | |||||
| self.container_name = container_name | |||||
| super().__init__(**kwargs) | super().__init__(**kwargs) | ||||
| self.container_url = container_url | self.container_url = container_url | ||||
| self.connection_string = connection_string | |||||
| self.compression = compression | self.compression = compression | ||||
| def get_container_client(self, hex_obj_id): | def get_container_client(self, hex_obj_id): | ||||
| """Get the container client for the container that contains the object with | """Get the container client for the container that contains the object with | ||||
| internal id hex_obj_id | internal id hex_obj_id | ||||
| This is used to allow the PrefixedAzureCloudObjStorage to dispatch the | This is used to allow the PrefixedAzureCloudObjStorage to dispatch the | ||||
| client according to the prefix of the object id. | client according to the prefix of the object id. | ||||
| """ | """ | ||||
| if self.connection_string: | |||||
| return ContainerClient.from_connection_string( | |||||
| self.connection_string, self.container_name | |||||
| ) | |||||
| else: | |||||
| return ContainerClient.from_container_url(self.container_url) | return ContainerClient.from_container_url(self.container_url) | ||||
| @contextlib.asynccontextmanager | @contextlib.asynccontextmanager | ||||
| async def get_async_container_clients(self): | async def get_async_container_clients(self): | ||||
| """Returns a collection of container clients, to be passed to | """Returns a collection of container clients, to be passed to | ||||
| ``get_async_blob_client``. | ``get_async_blob_client``. | ||||
| Each container may not be used in more than one asyncio loop.""" | Each container may not be used in more than one asyncio loop.""" | ||||
| if self.connection_string: | |||||
| client = AsyncContainerClient.from_connection_string( | |||||
| self.connection_string, self.container_name | |||||
| ) | |||||
| else: | |||||
| client = AsyncContainerClient.from_container_url(self.container_url) | client = AsyncContainerClient.from_container_url(self.container_url) | ||||
| async with client: | async with client: | ||||
| yield {"": client} | yield {"": client} | ||||
| def get_blob_client(self, hex_obj_id): | def get_blob_client(self, hex_obj_id): | ||||
| """Get the azure blob client for the given hex obj id""" | """Get the azure blob client for the given hex obj id""" | ||||
| container_client = self.get_container_client(hex_obj_id) | container_client = self.get_container_client(hex_obj_id) | ||||
| return container_client.get_blob_client(blob=hex_obj_id) | return container_client.get_blob_client(blob=hex_obj_id) | ||||
| ▲ Show 20 Lines • Show All 258 Lines • Show Last 20 Lines | |||||