Cesar77RR's picture
download
raw
19.9 kB
'Implementation of an image service that uses Glance as the backend'
import copy
import itertools
import random
import shutil
import sys
import time
import glanceclient.exc
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import timeutils
import six
from six.moves import range
from six.moves import urllib
from cinder import exception
from cinder.i18n import _
glance_opts = [cfg.ListOpt('allowed_direct_url_schemes', default=[], help='A list of url schemes that can be downloaded directly via the direct_url. Currently supported schemes: [file, cinder].'), cfg.StrOpt('glance_catalog_info', default='image:glance:publicURL', help='Info to match when looking for glance in the service catalog. Format is: separated values of the form: <service_type>:<service_name>:<endpoint_type> - Only used if glance_api_servers are not provided.')]
glance_core_properties_opts = [cfg.ListOpt('glance_core_properties', default=['checksum', 'container_format', 'disk_format', 'image_name', 'image_id', 'min_disk', 'min_ram', 'name', 'size'], help='Default core properties of image')]
CONF = cfg.CONF
CONF.register_opts(glance_opts)
CONF.register_opts(glance_core_properties_opts)
CONF.import_opt('glance_api_version', 'cinder.common.config')
LOG = logging.getLogger(__name__)
def _parse_image_ref(image_href):
'Parse an image href into composite parts.\n\n :param image_href: href of an image\n :returns: a tuple of the form (image_id, netloc, use_ssl)\n :raises ValueError:\n\n '
url = urllib.parse.urlparse(image_href)
netloc = url.netloc
image_id = url.path.split('/')[(-1)]
use_ssl = (url.scheme == 'https')
return (image_id, netloc, use_ssl)
def _create_glance_client(context, netloc, use_ssl, version=None):
'Instantiate a new glanceclient.Client object.'
if (version is None):
version = CONF.glance_api_version
params = {}
if use_ssl:
scheme = 'https'
params['insecure'] = CONF.glance_api_insecure
params['ssl_compression'] = CONF.glance_api_ssl_compression
params['cacert'] = CONF.glance_ca_certificates_file
else:
scheme = 'http'
if (CONF.auth_strategy == 'keystone'):
params['token'] = context.auth_token
if (CONF.glance_request_timeout is not None):
params['timeout'] = CONF.glance_request_timeout
endpoint = ('%s://%s' % (scheme, netloc))
params['global_request_id'] = context.global_id
return glanceclient.Client(str(version), endpoint, **params)
def get_api_servers(context):
'Return Iterable over shuffled api servers.\n\n Shuffle a list of glance_api_servers and return an iterator\n that will cycle through the list, looping around to the beginning\n if necessary. If CONF.glance_api_servers is None then they will\n be retrieved from the catalog.\n '
api_servers = []
api_servers_info = []
if (CONF.glance_api_servers is None):
info = CONF.glance_catalog_info
try:
(service_type, service_name, endpoint_type) = info.split(':')
except ValueError:
raise exception.InvalidConfigurationValue(_("Failed to parse the configuration option 'glance_catalog_info', must be in the form <service_type>:<service_name>:<endpoint_type>"))
for entry in context.service_catalog:
if (entry.get('type') == service_type):
api_servers.append(entry.get('endpoints')[0].get(endpoint_type))
else:
for api_server in CONF.glance_api_servers:
api_servers.append(api_server)
for api_server in api_servers:
if ('//' not in api_server):
api_server = ('http://' + api_server)
url = urllib.parse.urlparse(api_server)
netloc = (url.netloc + url.path)
use_ssl = (url.scheme == 'https')
api_servers_info.append((netloc, use_ssl))
random.shuffle(api_servers_info)
return itertools.cycle(api_servers_info)
class GlanceClientWrapper(object, ):
'Glance client wrapper class that implements retries.'
def __init__(self, context=None, netloc=None, use_ssl=False, version=None):
if (netloc is not None):
self.client = self._create_static_client(context, netloc, use_ssl, version)
else:
self.client = None
self.api_servers = None
self.version = version
def _create_static_client(self, context, netloc, use_ssl, version):
"Create a client that we'll use for every call."
self.netloc = netloc
self.use_ssl = use_ssl
self.version = version
return _create_glance_client(context, self.netloc, self.use_ssl, self.version)
def _create_onetime_client(self, context, version):
'Create a client that will be used for one call.'
if (self.api_servers is None):
self.api_servers = get_api_servers(context)
(self.netloc, self.use_ssl) = next(self.api_servers)
return _create_glance_client(context, self.netloc, self.use_ssl, version)
def call(self, context, method, *args, **kwargs):
'Call a glance client method.\n\n If we get a connection error,\n retry the request according to CONF.glance_num_retries.\n '
version = kwargs.pop('version', self.version)
retry_excs = (glanceclient.exc.ServiceUnavailable, glanceclient.exc.InvalidEndpoint, glanceclient.exc.CommunicationError)
num_attempts = (1 + CONF.glance_num_retries)
for attempt in range(1, (num_attempts + 1)):
client = (self.client or self._create_onetime_client(context, version))
try:
controller = getattr(client, kwargs.pop('controller', 'images'))
return getattr(controller, method)(*args, **kwargs)
except retry_excs as e:
netloc = self.netloc
extra = 'retrying'
error_msg = _("Error contacting glance server '%(netloc)s' for '%(method)s', %(extra)s.")
if (attempt == num_attempts):
extra = 'done trying'
LOG.exception(error_msg, {'netloc': netloc, 'method': method, 'extra': extra})
raise exception.GlanceConnectionFailed(reason=e)
LOG.exception(error_msg, {'netloc': netloc, 'method': method, 'extra': extra})
time.sleep(1)
except glanceclient.exc.HTTPOverLimit as e:
raise exception.ImageLimitExceeded(e)
class GlanceImageService(object, ):
'Provides storage and retrieval of disk image objects within Glance.'
def __init__(self, client=None):
self._client = (client or GlanceClientWrapper())
self._image_schema = None
self.temp_images = None
def detail(self, context, **kwargs):
'Calls out to Glance for a list of detailed image information.'
params = self._extract_query_params(kwargs)
try:
images = self._client.call(context, 'list', **params)
except Exception:
_reraise_translated_exception()
_images = []
for image in images:
if self._is_image_available(context, image):
_images.append(self._translate_from_glance(context, image))
return _images
def _extract_query_params(self, params):
_params = {}
accepted_params = ('filters', 'marker', 'limit', 'sort_key', 'sort_dir')
for param in accepted_params:
if (param in params):
_params[param] = params.get(param)
if (CONF.glance_api_version <= 1):
_params.setdefault('filters', {})
_params['filters'].setdefault('is_public', 'none')
return _params
def show(self, context, image_id):
'Returns a dict with image data for the given opaque image id.'
try:
image = self._client.call(context, 'get', image_id)
except Exception:
_reraise_translated_image_exception()
if (not self._is_image_available(context, image)):
raise exception.ImageNotFound(image_id=image_id)
base_image_meta = self._translate_from_glance(context, image)
return base_image_meta
def get_location(self, context, image_id):
'Get backend storage location url.\n\n Returns a tuple containing the direct url and locations representing\n the backend storage location, or (None, None) if these attributes are\n not shown by Glance.\n '
if (CONF.glance_api_version == 1):
return (None, None)
try:
client = GlanceClientWrapper(version=2)
image_meta = client.call(context, 'get', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if (not self._is_image_available(context, image_meta)):
raise exception.ImageNotFound(image_id=image_id)
return (getattr(image_meta, 'direct_url', None), getattr(image_meta, 'locations', None))
def add_location(self, context, image_id, url, metadata):
'Add a backend location url to an image.\n\n Returns a dict containing image metadata on success.\n '
if (CONF.glance_api_version != 2):
raise exception.Invalid('Image API version 2 is disabled.')
client = GlanceClientWrapper(version=2)
try:
return client.call(context, 'add_location', image_id, url, metadata)
except Exception:
_reraise_translated_image_exception(image_id)
def download(self, context, image_id, data=None):
'Calls out to Glance for data and writes data.'
if (data and ('file' in CONF.allowed_direct_url_schemes)):
(direct_url, locations) = self.get_location(context, image_id)
urls = ([direct_url] + [loc.get('url') for loc in (locations or [])])
for url in urls:
if (url is None):
continue
parsed_url = urllib.parse.urlparse(url)
if (parsed_url.scheme == 'file'):
with open(parsed_url.path, 'r') as f:
shutil.copyfileobj(f, data)
return
try:
image_chunks = self._client.call(context, 'data', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if (not data):
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def create(self, context, image_meta, data=None):
'Store the image data and return the new image object.'
sent_service_image_meta = self._translate_to_glance(image_meta)
if data:
sent_service_image_meta['data'] = data
recv_service_image_meta = self._client.call(context, 'create', **sent_service_image_meta)
return self._translate_from_glance(context, recv_service_image_meta)
def update(self, context, image_id, image_meta, data=None, purge_props=True):
'Modify the given image with the new data.'
if ((CONF.glance_api_version > 1) and purge_props):
props_to_update = image_meta.get('properties', {}).keys()
image_meta = self._translate_to_glance(image_meta)
if (CONF.glance_api_version == 1):
image_meta['purge_props'] = purge_props
image_meta.pop('id', None)
try:
if (CONF.glance_api_version > 1):
if data:
self._client.call(context, 'upload', image_id, data)
if image_meta:
if purge_props:
cur_image_meta = self.show(context, image_id)
cur_props = cur_image_meta['properties'].keys()
remove_props = list((set(cur_props) - set(props_to_update)))
image_meta['remove_props'] = remove_props
image_meta = self._client.call(context, 'update', image_id, **image_meta)
else:
image_meta = self._client.call(context, 'get', image_id)
else:
if data:
image_meta['data'] = data
image_meta = self._client.call(context, 'update', image_id, **image_meta)
except Exception:
_reraise_translated_image_exception(image_id)
else:
return self._translate_from_glance(context, image_meta)
def delete(self, context, image_id):
'Delete the given image.\n\n :raises ImageNotFound: if the image does not exist.\n :raises NotAuthorized: if the user is not an owner.\n\n '
try:
self._client.call(context, 'delete', image_id)
except glanceclient.exc.NotFound:
raise exception.ImageNotFound(image_id=image_id)
return True
def _translate_from_glance(self, context, image):
"Get image metadata from glance image.\n\n Extract metadata from image and convert it's properties\n to type cinder expected.\n\n :param image: glance image object\n :return: image metadata dictionary\n "
if (CONF.glance_api_version == 2):
if (self._image_schema is None):
self._image_schema = self._client.call(context, 'get', controller='schemas', schema_name='image', version=2)
image_meta = {key: getattr(image, key) for key in image.keys() if ((self._image_schema.is_base_property(key) is True) and (key != 'schema'))}
image_meta['properties'] = {key: getattr(image, key) for key in image.keys() if (self._image_schema.is_base_property(key) is False)}
else:
image_meta = _extract_attributes(image)
image_meta = _convert_timestamps_to_datetimes(image_meta)
image_meta = _convert_from_string(image_meta)
return image_meta
@staticmethod
def _translate_to_glance(image_meta):
image_meta = _convert_to_string(image_meta)
image_meta = _remove_read_only(image_meta)
if (CONF.glance_api_version >= 2):
properties = image_meta.get('properties')
if properties:
image_meta.update(properties)
del image_meta['properties']
return image_meta
@staticmethod
def _is_image_available(context, image):
'Check image availability.\n\n This check is needed in case Nova and Glance are deployed\n without authentication turned on.\n '
if (hasattr(context, 'auth_token') and context.auth_token):
return True
if (image.is_public or context.is_admin):
return True
properties = image.properties
if (context.project_id and ('owner_id' in properties)):
return (str(properties['owner_id']) == str(context.project_id))
if (context.project_id and ('project_id' in properties)):
return (str(properties['project_id']) == str(context.project_id))
try:
user_id = properties['user_id']
except KeyError:
return False
return (str(user_id) == str(context.user_id))
def _convert_timestamps_to_datetimes(image_meta):
'Returns image with timestamp fields converted to datetime objects.'
for attr in ['created_at', 'updated_at', 'deleted_at']:
if image_meta.get(attr):
image_meta[attr] = timeutils.parse_isotime(image_meta[attr])
return image_meta
def _json_loads(properties, attr):
prop = properties[attr]
if isinstance(prop, six.string_types):
properties[attr] = jsonutils.loads(prop)
def _json_dumps(properties, attr):
prop = properties[attr]
if (not isinstance(prop, six.string_types)):
properties[attr] = jsonutils.dumps(prop)
_CONVERT_PROPS = ('block_device_mapping', 'mappings')
def _convert(method, metadata):
metadata = copy.deepcopy(metadata)
properties = metadata.get('properties')
if properties:
for attr in _CONVERT_PROPS:
if (attr in properties):
method(properties, attr)
return metadata
def _convert_from_string(metadata):
return _convert(_json_loads, metadata)
def _convert_to_string(metadata):
return _convert(_json_dumps, metadata)
def _extract_attributes(image):
IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner', 'container_format', 'status', 'id', 'name', 'created_at', 'updated_at', 'deleted', 'deleted_at', 'checksum', 'min_disk', 'min_ram', 'protected']
if (CONF.glance_api_version == 2):
IMAGE_ATTRIBUTES.append('visibility')
else:
IMAGE_ATTRIBUTES.append('is_public')
output = {}
for attr in IMAGE_ATTRIBUTES:
if ((attr == 'deleted_at') and (not output['deleted'])):
output[attr] = None
elif ((attr == 'checksum') and (output['status'] != 'active')):
output[attr] = None
else:
output[attr] = getattr(image, attr, None)
output['properties'] = getattr(image, 'properties', {})
return output
def _remove_read_only(image_meta):
IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at']
output = copy.deepcopy(image_meta)
for attr in IMAGE_ATTRIBUTES:
if (attr in output):
del output[attr]
return output
def _reraise_translated_image_exception(image_id):
'Transform the exception for the image but keep its traceback intact.'
(_exc_type, exc_value, exc_trace) = sys.exc_info()
new_exc = _translate_image_exception(image_id, exc_value)
six.reraise(type(new_exc), new_exc, exc_trace)
def _reraise_translated_exception():
'Transform the exception but keep its traceback intact.'
(_exc_type, exc_value, exc_trace) = sys.exc_info()
new_exc = _translate_plain_exception(exc_value)
six.reraise(type(new_exc), new_exc, exc_trace)
def _translate_image_exception(image_id, exc_value):
if isinstance(exc_value, (glanceclient.exc.Forbidden, glanceclient.exc.Unauthorized)):
return exception.ImageNotAuthorized(image_id=image_id)
if isinstance(exc_value, glanceclient.exc.NotFound):
return exception.ImageNotFound(image_id=image_id)
if isinstance(exc_value, glanceclient.exc.BadRequest):
return exception.Invalid(exc_value)
return exc_value
def _translate_plain_exception(exc_value):
if isinstance(exc_value, (glanceclient.exc.Forbidden, glanceclient.exc.Unauthorized)):
return exception.NotAuthorized(exc_value)
if isinstance(exc_value, glanceclient.exc.NotFound):
return exception.NotFound(exc_value)
if isinstance(exc_value, glanceclient.exc.BadRequest):
return exception.Invalid(exc_value)
return exc_value
def get_remote_image_service(context, image_href):
"Create an image_service and parse the id from the given image_href.\n\n The image_href param can be an href of the form\n 'http://example.com:9292/v1/images/b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3',\n or just an id such as 'b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3'. If the\n image_href is a standalone id, then the default image service is returned.\n\n :param image_href: href that describes the location of an image\n :returns: a tuple of the form (image_service, image_id)\n\n "
if ('/' not in str(image_href)):
image_service = get_default_image_service()
return (image_service, image_href)
try:
(image_id, glance_netloc, use_ssl) = _parse_image_ref(image_href)
glance_client = GlanceClientWrapper(context=context, netloc=glance_netloc, use_ssl=use_ssl)
except ValueError:
raise exception.InvalidImageRef(image_href=image_href)
image_service = GlanceImageService(client=glance_client)
return (image_service, image_id)
def get_default_image_service():
return GlanceImageService()

Xet Storage Details

Size:
19.9 kB
·
Xet hash:
4ce770ac685b82f3f5f9c3e821640dd002197ef8fef4fdff0b268a9a09571b81

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.