Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/backends/libcloud.py
# Copyright (C) 2016-2017 The Software Heritage developers | # Copyright (C) 2016-2017 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import abc | import abc | ||||
from collections import OrderedDict | from collections import OrderedDict | ||||
from collections.abc import Iterator | from typing import Iterator, Optional | ||||
from typing import Optional | |||||
from urllib.parse import urlencode | from urllib.parse import urlencode | ||||
from libcloud.storage import providers | from libcloud.storage import providers | ||||
import libcloud.storage.drivers.s3 | import libcloud.storage.drivers.s3 | ||||
from libcloud.storage.types import ObjectDoesNotExistError, Provider | from libcloud.storage.types import ObjectDoesNotExistError, Provider | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.objstorage.exc import Error, ObjNotFoundError | from swh.objstorage.exc import Error, ObjNotFoundError | ||||
from swh.objstorage.interface import ObjId | from swh.objstorage.interface import CompositeObjId, ObjId | ||||
from swh.objstorage.objstorage import ( | from swh.objstorage.objstorage import ( | ||||
ObjStorage, | ObjStorage, | ||||
compressors, | compressors, | ||||
compute_hash, | compute_hash, | ||||
decompressors, | decompressors, | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | def _get_provider(self): | ||||
"%s must implement `get_provider` method" % type(self) | "%s must implement `get_provider` method" % type(self) | ||||
) | ) | ||||
def check_config(self, *, check_write): | def check_config(self, *, check_write): | ||||
"""Check the configuration for this object storage""" | """Check the configuration for this object storage""" | ||||
# FIXME: hopefully this blew up during instantiation | # FIXME: hopefully this blew up during instantiation | ||||
return True | return True | ||||
def __contains__(self, obj_id): | def __contains__(self, obj_id: ObjId) -> bool: | ||||
try: | try: | ||||
self._get_object(obj_id) | self._get_object(obj_id) | ||||
except ObjNotFoundError: | except ObjNotFoundError: | ||||
return False | return False | ||||
else: | else: | ||||
return True | return True | ||||
def __iter__(self): | def __iter__(self) -> Iterator[CompositeObjId]: | ||||
"""Iterate over the objects present in the storage | """Iterate over the objects present in the storage | ||||
Warning: Iteration over the contents of a cloud-based object storage | Warning: Iteration over the contents of a cloud-based object storage | ||||
may have bad efficiency: due to the very high amount of objects in it | may have bad efficiency: due to the very high amount of objects in it | ||||
and the fact that it is remote, get all the contents of the current | and the fact that it is remote, get all the contents of the current | ||||
object storage may result in a lot of network requests. | object storage may result in a lot of network requests. | ||||
You almost certainly don't want to use this method in production. | You almost certainly don't want to use this method in production. | ||||
▲ Show 20 Lines • Show All 116 Lines • Show Last 20 Lines |