Differential D8756 Diff 31615 swh/objstorage/tests/test_objstorage_azure.py

Changeset View

Standalone View

swh/objstorage/tests/test_objstorage_azure.py

# Copyright (C) 2016-2021 The Software Heritage developers		# Copyright (C) 2016-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import asyncio		import asyncio
import base64		import base64
import collections		import collections
from dataclasses import dataclass		from dataclasses import dataclass
		import os
		import secrets
		import shutil
		import subprocess
		import tempfile
import unittest		import unittest
from unittest.mock import patch		from unittest.mock import patch
from urllib.parse import parse_qs, urlparse		from urllib.parse import parse_qs, urlparse

from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError		from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
		from azure.storage.blob import BlobServiceClient
import pytest		import pytest

from swh.model.hashutil import hash_to_hex		from swh.model.hashutil import hash_to_hex
import swh.objstorage.backends.azure		import swh.objstorage.backends.azure
from swh.objstorage.exc import Error		from swh.objstorage.exc import Error
from swh.objstorage.factory import get_objstorage		from swh.objstorage.factory import get_objstorage
from swh.objstorage.objstorage import decompressors		from swh.objstorage.objstorage import decompressors

from .objstorage_testing import ObjStorageTestFixture		from .objstorage_testing import ObjStorageTestFixture

		AZURITE_EXE = shutil.which(
		"azurite-blob", path=os.environ.get("AZURITE_PATH", os.environ.get("PATH"))
		)


		olasdUnsubmitted Done Inline Actions Ugh. Just set AZURITE_PATH to your `node_modules/.bin` and avoid this `.js` ugliness? olasd: Ugh. Just set AZURITE_PATH to your `node_modules/.bin` and avoid this `.js` ugliness?
		vlorentzAuthorUnsubmitted Done Inline Actions oooh! I was looking for `node_modules/bin`, that's why I didn't find it I'll use `azurite-blob` instead of `azurite` itself, so it doesn't needlessly spawn other the queue and table services. vlorentz: oooh! I was looking for `node_modules/bin`, that's why I didn't find it I'll use `azurite…
@dataclass		@dataclass
class MockListedObject:		class MockListedObject:
name: str		name: str


class MockAsyncDownloadClient:		class MockAsyncDownloadClient:
def __init__(self, blob_data):		def __init__(self, blob_data):
self.blob_data = blob_data		self.blob_data = blob_data
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	class MockBlobClient:

def delete_blob(self):		def delete_blob(self):
if self.blob not in self.container.blobs:		if self.blob not in self.container.blobs:
raise ResourceNotFoundError("Blob not found")		raise ResourceNotFoundError("Blob not found")

del self.container.blobs[self.blob]		del self.container.blobs[self.blob]


		@pytest.mark.skipif(not AZURITE_EXE, reason="azurite not found in AZURITE_PATH or PATH")
		class TestAzuriteCloudObjStorage(ObjStorageTestFixture, unittest.TestCase):
		compression = "none"

		@classmethod
		def setUpClass(cls):
		super().setUpClass()

		host = "127.0.0.1"

		cls._azurite_path = tempfile.mkdtemp()

		cls._azurite_proc = subprocess.Popen(
		[
		AZURITE_EXE,
		"--blobHost",
		host,
		"--blobPort",
		"0",
		],
		stdout=subprocess.PIPE,
		cwd=cls._azurite_path,
		)

		prefix = b"Azurite Blob service successfully listens on "
		for line in cls._azurite_proc.stdout:
		if line.startswith(prefix):
		base_url = line[len(prefix) :].decode().strip()
		break
		else:
		assert False, "Did not get Azurite Blob service port."

		# https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite#well-known-storage-account-and-key
		account_name = "devstoreaccount1"
		account_key = (
		"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq"
		"/K1SZFPTOtr/KBHBeksoGMGw=="
		)

		container_url = f"{base_url}/{account_name}"
		cls._connection_string = (
		f"DefaultEndpointsProtocol=https;"
		f"AccountName={account_name};"
		f"AccountKey={account_key};"
		f"BlobEndpoint={container_url};"
		)

		@classmethod
		olasdUnsubmitted Not Done Inline Actions As this includes the `container_name`, is the separate argument needed for `get_objstorage`? If azurite supports that, it may be nicer to get a shared access signature and pass it through to the existing connection_url argument (which would introduce coverage of the code that retrieves shared access signatures as well). olasd: As this includes the `container_name`, is the separate argument needed for `get_objstorage`?
		olasdUnsubmitted Not Done Inline Actions s/connection_url/container_url/, of course olasd: s/connection_url/container_url/, of course
		vlorentzAuthorUnsubmitted Done Inline Actions It's needed by `ContainerClient.from_connection_string` for some reason vlorentz: It's needed by `ContainerClient.from_connection_string` for some reason
		def tearDownClass(cls):
		super().tearDownClass()
		cls._azurite_proc.kill()
		cls._azurite_proc.wait(2)
		shutil.rmtree(cls._azurite_path)

		def setUp(self):
		super().setUp()
		self._container_name = secrets.token_hex(10)
		client = BlobServiceClient.from_connection_string(self._connection_string)
		client.create_container(self._container_name)

		self.storage = get_objstorage(
		"azure",
		connection_string=self._connection_string,
		container_name=self._container_name,
		compression=self.compression,
		)


		class TestAzuriteCloudObjStorageGzip(TestAzuriteCloudObjStorage):
		compression = "gzip"


def get_MockContainerClient():		def get_MockContainerClient():
blobs = collections.defaultdict(dict) # {container_url: {blob_id: blob}}		blobs = collections.defaultdict(dict) # {container_url: {blob_id: blob}}

class MockContainerClient:		class MockContainerClient:
def __init__(self, container_url):		def __init__(self, container_url):
self.container_url = container_url		self.container_url = container_url
self.blobs = blobs[self.container_url]		self.blobs = blobs[self.container_url]

Show All 23 Lines	class MockContainerClient:
yield from future		yield from future

def __aexit__(self, *args):		def __aexit__(self, *args):
return self		return self

return MockContainerClient		return MockContainerClient


class TestAzureCloudObjStorage(ObjStorageTestFixture, unittest.TestCase):		class TestMockedAzureCloudObjStorage(ObjStorageTestFixture, unittest.TestCase):
compression = "none"		compression = "none"

def setUp(self):		def setUp(self):
super().setUp()		super().setUp()
ContainerClient = get_MockContainerClient()		ContainerClient = get_MockContainerClient()
patcher = patch(		patcher = patch(
"swh.objstorage.backends.azure.ContainerClient", ContainerClient		"swh.objstorage.backends.azure.ContainerClient", ContainerClient
)		)
Show All 40 Lines	def test_trailing_data_on_stored_blob(self):
with self.assertRaises(Error) as e:		with self.assertRaises(Error) as e:
self.storage.check(obj_id)		self.storage.check(obj_id)
else:		else:
with self.assertRaises(Error) as e:		with self.assertRaises(Error) as e:
self.storage.get(obj_id)		self.storage.get(obj_id)
assert "trailing data" in e.exception.args[0]		assert "trailing data" in e.exception.args[0]


class TestAzureCloudObjStorageGzip(TestAzureCloudObjStorage):		class TestMockedAzureCloudObjStorageGzip(TestMockedAzureCloudObjStorage):
compression = "gzip"		compression = "gzip"


class TestAzureCloudObjStorageZlib(TestAzureCloudObjStorage):		class TestMockedAzureCloudObjStorageZlib(TestMockedAzureCloudObjStorage):
compression = "zlib"		compression = "zlib"


class TestAzureCloudObjStorageLzma(TestAzureCloudObjStorage):		class TestMockedAzureCloudObjStorageLzma(TestMockedAzureCloudObjStorage):
compression = "lzma"		compression = "lzma"


class TestAzureCloudObjStorageBz2(TestAzureCloudObjStorage):		class TestMockedAzureCloudObjStorageBz2(TestMockedAzureCloudObjStorage):
compression = "bz2"		compression = "bz2"


class TestPrefixedAzureCloudObjStorage(ObjStorageTestFixture, unittest.TestCase):		class TestPrefixedAzureCloudObjStorage(ObjStorageTestFixture, unittest.TestCase):
def setUp(self):		def setUp(self):
super().setUp()		super().setUp()
self.ContainerClient = get_MockContainerClient()		self.ContainerClient = get_MockContainerClient()
patcher = patch(		patcher = patch(
▲ Show 20 Lines • Show All 111 Lines • Show Last 20 Lines