Cesar77RR's picture
download
raw
19.9 kB
'Implementation of an image service that uses Glance as the backend'
from __future__ import absolute_import
import copy
import itertools
import random
import shutil
import sys
import time
import glanceclient.exc
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_utils import timeutils
import six
from six.moves import range
from six.moves import urllib
from cinder import exception
from cinder.i18n import _
glance_opts = [cfg.ListOpt('allowed_direct_url_schemes', default=[], help='A list of url schemes that can be downloaded directly via the direct_url. Currently supported schemes: [file, cinder].'), cfg.StrOpt('glance_catalog_info', default='image:glance:publicURL', help='Info to match when looking for glance in the service catalog. Format is: separated values of the form: <service_type>:<service_name>:<endpoint_type> - Only used if glance_api_servers are not provided.')]
glance_core_properties_opts = [cfg.ListOpt('glance_core_properties', default=['checksum', 'container_format', 'disk_format', 'image_name', 'image_id', 'min_disk', 'min_ram', 'name', 'size'], help='Default core properties of image')]
CONF = cfg.CONF
CONF.register_opts(glance_opts)
CONF.register_opts(glance_core_properties_opts)
CONF.import_opt('glance_api_version', 'cinder.common.config')
LOG = logging.getLogger(__name__)
def _parse_image_ref(image_href):
'Parse an image href into composite parts.\n\n :param image_href: href of an image\n :returns: a tuple of the form (image_id, netloc, use_ssl)\n :raises ValueError:\n\n '
url = urllib.parse.urlparse(image_href)
netloc = url.netloc
image_id = url.path.split('/')[(-1)]
use_ssl = (url.scheme == 'https')
return (image_id, netloc, use_ssl)
def _create_glance_client(context, netloc, use_ssl, version=None):
'Instantiate a new glanceclient.Client object.'
if (version is None):
version = CONF.glance_api_version
params = {}
if use_ssl:
scheme = 'https'
params['insecure'] = CONF.glance_api_insecure
params['ssl_compression'] = CONF.glance_api_ssl_compression
params['cacert'] = CONF.glance_ca_certificates_file
else:
scheme = 'http'
if (CONF.auth_strategy == 'keystone'):
params['token'] = context.auth_token
if (CONF.glance_request_timeout is not None):
params['timeout'] = CONF.glance_request_timeout
endpoint = ('%s://%s' % (scheme, netloc))
params['global_request_id'] = context.global_id
return glanceclient.Client(str(version), endpoint, **params)
def get_api_servers(context):
'Return Iterable over shuffled api servers.\n\n Shuffle a list of glance_api_servers and return an iterator\n that will cycle through the list, looping around to the beginning\n if necessary. If CONF.glance_api_servers is None then they will\n be retrieved from the catalog.\n '
api_servers = []
api_servers_info = []
if (CONF.glance_api_servers is None):
info = CONF.glance_catalog_info
try:
(service_type, service_name, endpoint_type) = info.split(':')
except ValueError:
raise exception.InvalidConfigurationValue(_("Failed to parse the configuration option 'glance_catalog_info', must be in the form <service_type>:<service_name>:<endpoint_type>"))
for entry in context.service_catalog:
if (entry.get('type') == service_type):
api_servers.append(entry.get('endpoints')[0].get(endpoint_type))
else:
for api_server in CONF.glance_api_servers:
api_servers.append(api_server)
for api_server in api_servers:
if ('//' not in api_server):
api_server = ('http://' + api_server)
url = urllib.parse.urlparse(api_server)
netloc = (url.netloc + url.path)
use_ssl = (url.scheme == 'https')
api_servers_info.append((netloc, use_ssl))
random.shuffle(api_servers_info)
return itertools.cycle(api_servers_info)
class GlanceClientWrapper(object, ):
'Glance client wrapper class that implements retries.'
def __init__(self, context=None, netloc=None, use_ssl=False, version=None):
if (netloc is not None):
self.client = self._create_static_client(context, netloc, use_ssl, version)
else:
self.client = None
self.api_servers = None
self.version = version
def _create_static_client(self, context, netloc, use_ssl, version):
"Create a client that we'll use for every call."
self.netloc = netloc
self.use_ssl = use_ssl
self.version = version
return _create_glance_client(context, self.netloc, self.use_ssl, self.version)
def _create_onetime_client(self, context, version):
'Create a client that will be used for one call.'
if (self.api_servers is None):
self.api_servers = get_api_servers(context)
(self.netloc, self.use_ssl) = next(self.api_servers)
return _create_glance_client(context, self.netloc, self.use_ssl, version)
def call(self, context, method, *args, **kwargs):
'Call a glance client method.\n\n If we get a connection error,\n retry the request according to CONF.glance_num_retries.\n '
version = kwargs.pop('version', self.version)
retry_excs = (glanceclient.exc.ServiceUnavailable, glanceclient.exc.InvalidEndpoint, glanceclient.exc.CommunicationError)
num_attempts = (1 + CONF.glance_num_retries)
for attempt in range(1, (num_attempts + 1)):
client = (self.client or self._create_onetime_client(context, version))
try:
controller = getattr(client, kwargs.pop('controller', 'images'))
return getattr(controller, method)(*args, **kwargs)
except retry_excs as e:
netloc = self.netloc
extra = 'retrying'
error_msg = _("Error contacting glance server '%(netloc)s' for '%(method)s', %(extra)s.")
if (attempt == num_attempts):
extra = 'done trying'
LOG.exception(error_msg, {'netloc': netloc, 'method': method, 'extra': extra})
raise exception.GlanceConnectionFailed(reason=e)
LOG.exception(error_msg, {'netloc': netloc, 'method': method, 'extra': extra})
time.sleep(1)
except glanceclient.exc.HTTPOverLimit as e:
raise exception.ImageLimitExceeded(e)
class GlanceImageService(object, ):
'Provides storage and retrieval of disk image objects within Glance.'
def __init__(self, client=None):
self._client = (client or GlanceClientWrapper())
self._image_schema = None
self.temp_images = None
def detail(self, context, **kwargs):
'Calls out to Glance for a list of detailed image information.'
params = self._extract_query_params(kwargs)
try:
images = self._client.call(context, 'list', **params)
except Exception:
_reraise_translated_exception()
_images = []
for image in images:
if self._is_image_available(context, image):
_images.append(self._translate_from_glance(context, image))
return _images
def _extract_query_params(self, params):
_params = {}
accepted_params = ('filters', 'marker', 'limit', 'sort_key', 'sort_dir')
for param in accepted_params:
if (param in params):
_params[param] = params.get(param)
if (CONF.glance_api_version <= 1):
_params.setdefault('filters', {})
_params['filters'].setdefault('is_public', 'none')
return _params
def show(self, context, image_id):
'Returns a dict with image data for the given opaque image id.'
try:
image = self._client.call(context, 'get', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if (not self._is_image_available(context, image)):
raise exception.ImageNotFound(image_id=image_id)
base_image_meta = self._translate_from_glance(context, image)
return base_image_meta
def get_location(self, context, image_id):
'Get backend storage location url.\n\n Returns a tuple containing the direct url and locations representing\n the backend storage location, or (None, None) if these attributes are\n not shown by Glance.\n '
if (CONF.glance_api_version == 1):
return (None, None)
try:
client = GlanceClientWrapper(version=2)
image_meta = client.call(context, 'get', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if (not self._is_image_available(context, image_meta)):
raise exception.ImageNotFound(image_id=image_id)
return (getattr(image_meta, 'direct_url', None), getattr(image_meta, 'locations', None))
def add_location(self, context, image_id, url, metadata):
'Add a backend location url to an image.\n\n Returns a dict containing image metadata on success.\n '
if (CONF.glance_api_version != 2):
raise exception.Invalid('Image API version 2 is disabled.')
client = GlanceClientWrapper(version=2)
try:
return client.call(context, 'add_location', image_id, url, metadata)
except Exception:
_reraise_translated_image_exception(image_id)
def download(self, context, image_id, data=None):
'Calls out to Glance for data and writes data.'
if (data and ('file' in CONF.allowed_direct_url_schemes)):
(direct_url, locations) = self.get_location(context, image_id)
urls = ([direct_url] + [loc.get('url') for loc in (locations or [])])
for url in urls:
if (url is None):
continue
parsed_url = urllib.parse.urlparse(url)
if (parsed_url.scheme == 'file'):
with open(parsed_url.path, 'r') as f:
shutil.copyfileobj(f, data)
return
try:
image_chunks = self._client.call(context, 'data', image_id)
except Exception:
_reraise_translated_image_exception(image_id)
if (not data):
return image_chunks
else:
for chunk in image_chunks:
data.write(chunk)
def create(self, context, image_meta, data=None):
'Store the image data and return the new image object.'
sent_service_image_meta = self._translate_to_glance(image_meta)
if data:
sent_service_image_meta['data'] = data
recv_service_image_meta = self._client.call(context, 'create', **sent_service_image_meta)
return self._translate_from_glance(context, recv_service_image_meta)
def update(self, context, image_id, image_meta, data=None, purge_props=True):
'Modify the given image with the new data.'
if ((CONF.glance_api_version > 1) and purge_props):
props_to_update = image_meta.get('properties', {}).keys()
image_meta = self._translate_to_glance(image_meta)
if (CONF.glance_api_version == 1):
image_meta['purge_props'] = purge_props
image_meta.pop('id', None)
try:
if (CONF.glance_api_version > 1):
if data:
self._client.call(context, 'upload', image_id, data)
if image_meta:
if purge_props:
cur_image_meta = self.show(context, image_id)
cur_props = cur_image_meta['properties'].keys()
remove_props = list((set(cur_props) - set(props_to_update)))
image_meta['remove_props'] = remove_props
image_meta = self._client.call(context, 'update', image_id, **image_meta)
else:
image_meta = self._client.call(context, 'get', image_id)
else:
if data:
image_meta['data'] = data
image_meta = self._client.call(context, 'update', image_id, **image_meta)
except Exception:
_reraise_translated_image_exception(image_id)
else:
return self._translate_from_glance(context, image_meta)
def delete(self, context, image_id):
'Delete the given image.\n\n :raises ImageNotFound: if the image does not exist.\n :raises NotAuthorized: if the user is not an owner.\n\n '
try:
self._client.call(context, 'delete', image_id)
except glanceclient.exc.NotFound:
raise exception.ImageNotFound(image_id=image_id)
return True
def _translate_from_glance(self, context, image):
"Get image metadata from glance image.\n\n Extract metadata from image and convert it's properties\n to type cinder expected.\n\n :param image: glance image object\n :return: image metadata dictionary\n "
if (CONF.glance_api_version == 2):
if (self._image_schema is None):
self._image_schema = self._client.call(context, 'get', controller='schemas', schema_name='image', version=2)
image_meta = {key: getattr(image, key) for key in image.keys() if ((self._image_schema.is_base_property(key) is True) and (key != 'schema'))}
image_meta['properties'] = {key: getattr(image, key) for key in image.keys() if (self._image_schema.is_base_property(key) is False)}
else:
image_meta = _extract_attributes(image)
image_meta = _convert_timestamps_to_datetimes(image_meta)
image_meta = _convert_from_string(image_meta)
return image_meta
@staticmethod
def _translate_to_glance(image_meta):
image_meta = _convert_to_string(image_meta)
image_meta = _remove_read_only(image_meta)
if (CONF.glance_api_version >= 2):
properties = image_meta.get('properties')
if properties:
image_meta.update(properties)
del image_meta['properties']
return image_meta
@staticmethod
def _is_image_available(context, image):
'Check image availability.\n\n This check is needed in case Nova and Glance are deployed\n without authentication turned on.\n '
if (hasattr(context, 'auth_token') and context.auth_token):
return True
if (image.is_public or context.is_admin):
return True
properties = image.properties
if (context.project_id and ('owner_id' in properties)):
return (str(properties['owner_id']) == str(context.project_id))
if (context.project_id and ('project_id' in properties)):
return (str(properties['project_id']) == str(context.project_id))
try:
user_id = properties['user_id']
except KeyError:
return False
return (str(user_id) == str(context.user_id))
def _convert_timestamps_to_datetimes(image_meta):
'Returns image with timestamp fields converted to datetime objects.'
for attr in ['created_at', 'updated_at', 'deleted_at']:
if image_meta.get(attr):
image_meta[attr] = timeutils.parse_isotime(image_meta[attr])
return image_meta
def _json_loads(properties, attr):
prop = properties[attr]
if isinstance(prop, six.string_types):
properties[attr] = jsonutils.loads(prop)
def _json_dumps(properties, attr):
prop = properties[attr]
if (not isinstance(prop, six.string_types)):
properties[attr] = jsonutils.dumps(prop)
_CONVERT_PROPS = ('block_device_mapping', 'mappings')
def _convert(method, metadata):
metadata = copy.deepcopy(metadata)
properties = metadata.get('properties')
if properties:
for attr in _CONVERT_PROPS:
if (attr in properties):
method(properties, attr)
return metadata
def _convert_from_string(metadata):
return _convert(_json_loads, metadata)
def _convert_to_string(metadata):
return _convert(_json_dumps, metadata)
def _extract_attributes(image):
IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner', 'container_format', 'status', 'id', 'name', 'created_at', 'updated_at', 'deleted', 'deleted_at', 'checksum', 'min_disk', 'min_ram', 'protected']
if (CONF.glance_api_version == 2):
IMAGE_ATTRIBUTES.append('visibility')
else:
IMAGE_ATTRIBUTES.append('is_public')
output = {}
for attr in IMAGE_ATTRIBUTES:
if ((attr == 'deleted_at') and (not output['deleted'])):
output[attr] = None
elif ((attr == 'checksum') and (output['status'] != 'active')):
output[attr] = None
else:
output[attr] = getattr(image, attr, None)
output['properties'] = getattr(image, 'properties', {})
return output
def _remove_read_only(image_meta):
IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at']
output = copy.deepcopy(image_meta)
for attr in IMAGE_ATTRIBUTES:
if (attr in output):
del output[attr]
return output
def _reraise_translated_image_exception(image_id):
'Transform the exception for the image but keep its traceback intact.'
(_exc_type, exc_value, exc_trace) = sys.exc_info()
new_exc = _translate_image_exception(image_id, exc_value)
six.reraise(type(new_exc), new_exc, exc_trace)
def _reraise_translated_exception():
'Transform the exception but keep its traceback intact.'
(_exc_type, exc_value, exc_trace) = sys.exc_info()
new_exc = _translate_plain_exception(exc_value)
six.reraise(type(new_exc), new_exc, exc_trace)
def _translate_image_exception(image_id, exc_value):
if isinstance(exc_value, (glanceclient.exc.Forbidden, glanceclient.exc.Unauthorized)):
return exception.ImageNotAuthorized(image_id=image_id)
if isinstance(exc_value, glanceclient.exc.NotFound):
return exception.ImageNotFound(image_id=image_id)
if isinstance(exc_value, glanceclient.exc.BadRequest):
return exception.Invalid(exc_value)
return exc_value
def _translate_plain_exception(exc_value):
if isinstance(exc_value, (glanceclient.exc.Forbidden, glanceclient.exc.Unauthorized)):
return exception.NotAuthorized(exc_value)
if isinstance(exc_value, glanceclient.exc.NotFound):
return exception.NotFound(exc_value)
if isinstance(exc_value, glanceclient.exc.BadRequest):
return exception.Invalid(exc_value)
return exc_value
def get_remote_image_service(context, image_href):
"Create an image_service and parse the id from the given image_href.\n\n The image_href param can be an href of the form\n 'http://example.com:9292/v1/images/b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3',\n or just an id such as 'b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3'. If the\n image_href is a standalone id, then the default image service is returned.\n\n :param image_href: href that describes the location of an image\n :returns: a tuple of the form (image_service, image_id)\n\n "
if ('/' not in str(image_href)):
image_service = get_default_image_service()
return (image_service, image_href)
try:
(image_id, glance_netloc, use_ssl) = _parse_image_ref(image_href)
glance_client = GlanceClientWrapper(context=context, netloc=glance_netloc, use_ssl=use_ssl)
except ValueError:
raise exception.InvalidImageRef(image_href=image_href)
image_service = GlanceImageService(client=glance_client)
return (image_service, image_id)
def get_default_image_service():
return GlanceImageService()

Xet Storage Details

Size:
19.9 kB
·
Xet hash:
01069acd2e5ce82182f95f566752b82e2651ba1a4ff1d6208d211c2fb68388a6

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.