Buckets:
| 'Implementation of an image service that uses Glance as the backend' | |
| import copy | |
| import itertools | |
| import random | |
| import shutil | |
| import sys | |
| import time | |
| import glanceclient.exc | |
| from oslo_config import cfg | |
| from oslo_log import log as logging | |
| from oslo_serialization import jsonutils | |
| from oslo_utils import timeutils | |
| import six | |
| from six.moves import range | |
| from six.moves import urllib | |
| from cinder import exception | |
| from cinder.i18n import _ | |
| glance_opts = [cfg.ListOpt('allowed_direct_url_schemes', default=[], help='A list of url schemes that can be downloaded directly via the direct_url. Currently supported schemes: [file, cinder].'), cfg.StrOpt('glance_catalog_info', default='image:glance:publicURL', help='Info to match when looking for glance in the service catalog. Format is: separated values of the form: <service_type>:<service_name>:<endpoint_type> - Only used if glance_api_servers are not provided.')] | |
| glance_core_properties_opts = [cfg.ListOpt('glance_core_properties', default=['checksum', 'container_format', 'disk_format', 'image_name', 'image_id', 'min_disk', 'min_ram', 'name', 'size'], help='Default core properties of image')] | |
| CONF = cfg.CONF | |
| CONF.register_opts(glance_opts) | |
| CONF.register_opts(glance_core_properties_opts) | |
| CONF.import_opt('glance_api_version', 'cinder.common.config') | |
| LOG = logging.getLogger(__name__) | |
| def _parse_image_ref(image_href): | |
| 'Parse an image href into composite parts.\n\n :param image_href: href of an image\n :returns: a tuple of the form (image_id, netloc, use_ssl)\n :raises ValueError:\n\n ' | |
| url = urllib.parse.urlparse(image_href) | |
| netloc = url.netloc | |
| image_id = url.path.split('/')[(-1)] | |
| use_ssl = (url.scheme == 'https') | |
| return (image_id, netloc, use_ssl) | |
| def _create_glance_client(context, netloc, use_ssl, version=None): | |
| 'Instantiate a new glanceclient.Client object.' | |
| if (version is None): | |
| version = CONF.glance_api_version | |
| params = {} | |
| if use_ssl: | |
| scheme = 'https' | |
| params['insecure'] = CONF.glance_api_insecure | |
| params['ssl_compression'] = CONF.glance_api_ssl_compression | |
| params['cacert'] = CONF.glance_ca_certificates_file | |
| else: | |
| scheme = 'http' | |
| if (CONF.auth_strategy == 'keystone'): | |
| params['token'] = context.auth_token | |
| if (CONF.glance_request_timeout is not None): | |
| params['timeout'] = CONF.glance_request_timeout | |
| endpoint = ('%s://%s' % (scheme, netloc)) | |
| params['global_request_id'] = context.global_id | |
| return glanceclient.Client(str(version), endpoint, **params) | |
| def get_api_servers(context): | |
| 'Return Iterable over shuffled api servers.\n\n Shuffle a list of glance_api_servers and return an iterator\n that will cycle through the list, looping around to the beginning\n if necessary. If CONF.glance_api_servers is None then they will\n be retrieved from the catalog.\n ' | |
| api_servers = [] | |
| api_servers_info = [] | |
| if (CONF.glance_api_servers is None): | |
| info = CONF.glance_catalog_info | |
| try: | |
| (service_type, service_name, endpoint_type) = info.split(':') | |
| except ValueError: | |
| raise exception.InvalidConfigurationValue(_("Failed to parse the configuration option 'glance_catalog_info', must be in the form <service_type>:<service_name>:<endpoint_type>")) | |
| for entry in context.service_catalog: | |
| if (entry.get('type') == service_type): | |
| api_servers.append(entry.get('endpoints')[0].get(endpoint_type)) | |
| else: | |
| for api_server in CONF.glance_api_servers: | |
| api_servers.append(api_server) | |
| for api_server in api_servers: | |
| if ('//' not in api_server): | |
| api_server = ('http://' + api_server) | |
| url = urllib.parse.urlparse(api_server) | |
| netloc = (url.netloc + url.path) | |
| use_ssl = (url.scheme == 'https') | |
| api_servers_info.append((netloc, use_ssl)) | |
| random.shuffle(api_servers_info) | |
| return itertools.cycle(api_servers_info) | |
| class GlanceClientWrapper(object, ): | |
| 'Glance client wrapper class that implements retries.' | |
| def __init__(self, context=None, netloc=None, use_ssl=False, version=None): | |
| if (netloc is not None): | |
| self.client = self._create_static_client(context, netloc, use_ssl, version) | |
| else: | |
| self.client = None | |
| self.api_servers = None | |
| self.version = version | |
| def _create_static_client(self, context, netloc, use_ssl, version): | |
| "Create a client that we'll use for every call." | |
| self.netloc = netloc | |
| self.use_ssl = use_ssl | |
| self.version = version | |
| return _create_glance_client(context, self.netloc, self.use_ssl, self.version) | |
| def _create_onetime_client(self, context, version): | |
| 'Create a client that will be used for one call.' | |
| if (self.api_servers is None): | |
| self.api_servers = get_api_servers(context) | |
| (self.netloc, self.use_ssl) = next(self.api_servers) | |
| return _create_glance_client(context, self.netloc, self.use_ssl, version) | |
| def call(self, context, method, *args, **kwargs): | |
| 'Call a glance client method.\n\n If we get a connection error,\n retry the request according to CONF.glance_num_retries.\n ' | |
| version = kwargs.pop('version', self.version) | |
| retry_excs = (glanceclient.exc.ServiceUnavailable, glanceclient.exc.InvalidEndpoint, glanceclient.exc.CommunicationError) | |
| num_attempts = (1 + CONF.glance_num_retries) | |
| for attempt in range(1, (num_attempts + 1)): | |
| client = (self.client or self._create_onetime_client(context, version)) | |
| try: | |
| controller = getattr(client, kwargs.pop('controller', 'images')) | |
| return getattr(controller, method)(*args, **kwargs) | |
| except retry_excs as e: | |
| netloc = self.netloc | |
| extra = 'retrying' | |
| error_msg = _("Error contacting glance server '%(netloc)s' for '%(method)s', %(extra)s.") | |
| if (attempt == num_attempts): | |
| extra = 'done trying' | |
| LOG.exception(error_msg, {'netloc': netloc, 'method': method, 'extra': extra}) | |
| raise exception.GlanceConnectionFailed(reason=e) | |
| LOG.exception(error_msg, {'netloc': netloc, 'method': method, 'extra': extra}) | |
| time.sleep(1) | |
| except glanceclient.exc.HTTPOverLimit as e: | |
| raise exception.ImageLimitExceeded(e) | |
| class GlanceImageService(object, ): | |
| 'Provides storage and retrieval of disk image objects within Glance.' | |
| def __init__(self, client=None): | |
| self._client = (client or GlanceClientWrapper()) | |
| self._image_schema = None | |
| self.temp_images = None | |
| def detail(self, context, **kwargs): | |
| 'Calls out to Glance for a list of detailed image information.' | |
| params = self._extract_query_params(kwargs) | |
| try: | |
| images = self._client.call(context, 'list', **params) | |
| except Exception: | |
| _reraise_translated_exception() | |
| _images = [] | |
| for image in images: | |
| if self._is_image_available(context, image): | |
| _images.append(self._translate_from_glance(context, image)) | |
| return _images | |
| def _extract_query_params(self, params): | |
| _params = {} | |
| accepted_params = ('filters', 'marker', 'limit', 'sort_key', 'sort_dir') | |
| for param in accepted_params: | |
| if (param in params): | |
| _params[param] = params.get(param) | |
| if (CONF.glance_api_version <= 1): | |
| _params.setdefault('filters', {}) | |
| _params['filters'].setdefault('is_public', 'none') | |
| return _params | |
| def show(self, context, image_id): | |
| 'Returns a dict with image data for the given opaque image id.' | |
| try: | |
| image = self._client.call(context, 'get', image_id) | |
| except Exception: | |
| _reraise_translated_image_exception() | |
| if (not self._is_image_available(context, image)): | |
| raise exception.ImageNotFound(image_id=image_id) | |
| base_image_meta = self._translate_from_glance(context, image) | |
| return base_image_meta | |
| def get_location(self, context, image_id): | |
| 'Get backend storage location url.\n\n Returns a tuple containing the direct url and locations representing\n the backend storage location, or (None, None) if these attributes are\n not shown by Glance.\n ' | |
| if (CONF.glance_api_version == 1): | |
| return (None, None) | |
| try: | |
| client = GlanceClientWrapper(version=2) | |
| image_meta = client.call(context, 'get', image_id) | |
| except Exception: | |
| _reraise_translated_image_exception(image_id) | |
| if (not self._is_image_available(context, image_meta)): | |
| raise exception.ImageNotFound(image_id=image_id) | |
| return (getattr(image_meta, 'direct_url', None), getattr(image_meta, 'locations', None)) | |
| def add_location(self, context, image_id, url, metadata): | |
| 'Add a backend location url to an image.\n\n Returns a dict containing image metadata on success.\n ' | |
| if (CONF.glance_api_version != 2): | |
| raise exception.Invalid('Image API version 2 is disabled.') | |
| client = GlanceClientWrapper(version=2) | |
| try: | |
| return client.call(context, 'add_location', image_id, url, metadata) | |
| except Exception: | |
| _reraise_translated_image_exception(image_id) | |
| def download(self, context, image_id, data=None): | |
| 'Calls out to Glance for data and writes data.' | |
| if (data and ('file' in CONF.allowed_direct_url_schemes)): | |
| (direct_url, locations) = self.get_location(context, image_id) | |
| urls = ([direct_url] + [loc.get('url') for loc in (locations or [])]) | |
| for url in urls: | |
| if (url is None): | |
| continue | |
| parsed_url = urllib.parse.urlparse(url) | |
| if (parsed_url.scheme == 'file'): | |
| with open(parsed_url.path, 'r') as f: | |
| shutil.copyfileobj(f, data) | |
| return | |
| try: | |
| image_chunks = self._client.call(context, 'data', image_id) | |
| except Exception: | |
| _reraise_translated_image_exception(image_id) | |
| if (not data): | |
| return image_chunks | |
| else: | |
| for chunk in image_chunks: | |
| data.write(chunk) | |
| def create(self, context, image_meta, data=None): | |
| 'Store the image data and return the new image object.' | |
| sent_service_image_meta = self._translate_to_glance(image_meta) | |
| if data: | |
| sent_service_image_meta['data'] = data | |
| recv_service_image_meta = self._client.call(context, 'create', **sent_service_image_meta) | |
| return self._translate_from_glance(context, recv_service_image_meta) | |
| def update(self, context, image_id, image_meta, data=None, purge_props=True): | |
| 'Modify the given image with the new data.' | |
| if ((CONF.glance_api_version > 1) and purge_props): | |
| props_to_update = image_meta.get('properties', {}).keys() | |
| image_meta = self._translate_to_glance(image_meta) | |
| if (CONF.glance_api_version == 1): | |
| image_meta['purge_props'] = purge_props | |
| image_meta.pop('id', None) | |
| try: | |
| if (CONF.glance_api_version > 1): | |
| if data: | |
| self._client.call(context, 'upload', image_id, data) | |
| if image_meta: | |
| if purge_props: | |
| cur_image_meta = self.show(context, image_id) | |
| cur_props = cur_image_meta['properties'].keys() | |
| remove_props = list((set(cur_props) - set(props_to_update))) | |
| image_meta['remove_props'] = remove_props | |
| image_meta = self._client.call(context, 'update', image_id, **image_meta) | |
| else: | |
| image_meta = self._client.call(context, 'get', image_id) | |
| else: | |
| if data: | |
| image_meta['data'] = data | |
| image_meta = self._client.call(context, 'update', image_id, **image_meta) | |
| except Exception: | |
| _reraise_translated_image_exception(image_id) | |
| else: | |
| return self._translate_from_glance(context, image_meta) | |
| def delete(self, context, image_id): | |
| 'Delete the given image.\n\n :raises ImageNotFound: if the image does not exist.\n :raises NotAuthorized: if the user is not an owner.\n\n ' | |
| try: | |
| self._client.call(context, 'delete', image_id) | |
| except glanceclient.exc.NotFound: | |
| raise exception.ImageNotFound(image_id=image_id) | |
| return True | |
| def _translate_from_glance(self, context, image): | |
| "Get image metadata from glance image.\n\n Extract metadata from image and convert it's properties\n to type cinder expected.\n\n :param image: glance image object\n :return: image metadata dictionary\n " | |
| if (CONF.glance_api_version == 2): | |
| if (self._image_schema is None): | |
| self._image_schema = self._client.call(context, 'get', controller='schemas', schema_name='image', version=2) | |
| image_meta = {key: getattr(image, key) for key in image.keys() if ((self._image_schema.is_base_property(key) is True) and (key != 'schema'))} | |
| image_meta['properties'] = {key: getattr(image, key) for key in image.keys() if (self._image_schema.is_base_property(key) is False)} | |
| else: | |
| image_meta = _extract_attributes(image) | |
| image_meta = _convert_timestamps_to_datetimes(image_meta) | |
| image_meta = _convert_from_string(image_meta) | |
| return image_meta | |
| @staticmethod | |
| def _translate_to_glance(image_meta): | |
| image_meta = _convert_to_string(image_meta) | |
| image_meta = _remove_read_only(image_meta) | |
| if (CONF.glance_api_version >= 2): | |
| properties = image_meta.get('properties') | |
| if properties: | |
| image_meta.update(properties) | |
| del image_meta['properties'] | |
| return image_meta | |
| @staticmethod | |
| def _is_image_available(context, image): | |
| 'Check image availability.\n\n This check is needed in case Nova and Glance are deployed\n without authentication turned on.\n ' | |
| if (hasattr(context, 'auth_token') and context.auth_token): | |
| return True | |
| if (image.is_public or context.is_admin): | |
| return True | |
| properties = image.properties | |
| if (context.project_id and ('owner_id' in properties)): | |
| return (str(properties['owner_id']) == str(context.project_id)) | |
| if (context.project_id and ('project_id' in properties)): | |
| return (str(properties['project_id']) == str(context.project_id)) | |
| try: | |
| user_id = properties['user_id'] | |
| except KeyError: | |
| return False | |
| return (str(user_id) == str(context.user_id)) | |
| def _convert_timestamps_to_datetimes(image_meta): | |
| 'Returns image with timestamp fields converted to datetime objects.' | |
| for attr in ['created_at', 'updated_at', 'deleted_at']: | |
| if image_meta.get(attr): | |
| image_meta[attr] = timeutils.parse_isotime(image_meta[attr]) | |
| return image_meta | |
| def _json_loads(properties, attr): | |
| prop = properties[attr] | |
| if isinstance(prop, six.string_types): | |
| properties[attr] = jsonutils.loads(prop) | |
| def _json_dumps(properties, attr): | |
| prop = properties[attr] | |
| if (not isinstance(prop, six.string_types)): | |
| properties[attr] = jsonutils.dumps(prop) | |
| _CONVERT_PROPS = ('block_device_mapping', 'mappings') | |
| def _convert(method, metadata): | |
| metadata = copy.deepcopy(metadata) | |
| properties = metadata.get('properties') | |
| if properties: | |
| for attr in _CONVERT_PROPS: | |
| if (attr in properties): | |
| method(properties, attr) | |
| return metadata | |
| def _convert_from_string(metadata): | |
| return _convert(_json_loads, metadata) | |
| def _convert_to_string(metadata): | |
| return _convert(_json_dumps, metadata) | |
| def _extract_attributes(image): | |
| IMAGE_ATTRIBUTES = ['size', 'disk_format', 'owner', 'container_format', 'status', 'id', 'name', 'created_at', 'updated_at', 'deleted', 'deleted_at', 'checksum', 'min_disk', 'min_ram', 'protected'] | |
| if (CONF.glance_api_version == 2): | |
| IMAGE_ATTRIBUTES.append('visibility') | |
| else: | |
| IMAGE_ATTRIBUTES.append('is_public') | |
| output = {} | |
| for attr in IMAGE_ATTRIBUTES: | |
| if ((attr == 'deleted_at') and (not output['deleted'])): | |
| output[attr] = None | |
| elif ((attr == 'checksum') and (output['status'] != 'active')): | |
| output[attr] = None | |
| else: | |
| output[attr] = getattr(image, attr, None) | |
| output['properties'] = getattr(image, 'properties', {}) | |
| return output | |
| def _remove_read_only(image_meta): | |
| IMAGE_ATTRIBUTES = ['status', 'updated_at', 'created_at', 'deleted_at'] | |
| output = copy.deepcopy(image_meta) | |
| for attr in IMAGE_ATTRIBUTES: | |
| if (attr in output): | |
| del output[attr] | |
| return output | |
| def _reraise_translated_image_exception(image_id): | |
| 'Transform the exception for the image but keep its traceback intact.' | |
| (_exc_type, exc_value, exc_trace) = sys.exc_info() | |
| new_exc = _translate_image_exception(image_id, exc_value) | |
| six.reraise(type(new_exc), new_exc, exc_trace) | |
| def _reraise_translated_exception(): | |
| 'Transform the exception but keep its traceback intact.' | |
| (_exc_type, exc_value, exc_trace) = sys.exc_info() | |
| new_exc = _translate_plain_exception(exc_value) | |
| six.reraise(type(new_exc), new_exc, exc_trace) | |
| def _translate_image_exception(image_id, exc_value): | |
| if isinstance(exc_value, (glanceclient.exc.Forbidden, glanceclient.exc.Unauthorized)): | |
| return exception.ImageNotAuthorized(image_id=image_id) | |
| if isinstance(exc_value, glanceclient.exc.NotFound): | |
| return exception.ImageNotFound(image_id=image_id) | |
| if isinstance(exc_value, glanceclient.exc.BadRequest): | |
| return exception.Invalid(exc_value) | |
| return exc_value | |
| def _translate_plain_exception(exc_value): | |
| if isinstance(exc_value, (glanceclient.exc.Forbidden, glanceclient.exc.Unauthorized)): | |
| return exception.NotAuthorized(exc_value) | |
| if isinstance(exc_value, glanceclient.exc.NotFound): | |
| return exception.NotFound(exc_value) | |
| if isinstance(exc_value, glanceclient.exc.BadRequest): | |
| return exception.Invalid(exc_value) | |
| return exc_value | |
| def get_remote_image_service(context, image_href): | |
| "Create an image_service and parse the id from the given image_href.\n\n The image_href param can be an href of the form\n 'http://example.com:9292/v1/images/b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3',\n or just an id such as 'b8b2c6f7-7345-4e2f-afa2-eedaba9cbbe3'. If the\n image_href is a standalone id, then the default image service is returned.\n\n :param image_href: href that describes the location of an image\n :returns: a tuple of the form (image_service, image_id)\n\n " | |
| if ('/' not in str(image_href)): | |
| image_service = get_default_image_service() | |
| return (image_service, image_href) | |
| try: | |
| (image_id, glance_netloc, use_ssl) = _parse_image_ref(image_href) | |
| glance_client = GlanceClientWrapper(context=context, netloc=glance_netloc, use_ssl=use_ssl) | |
| except ValueError: | |
| raise exception.InvalidImageRef(image_href=image_href) | |
| image_service = GlanceImageService(client=glance_client) | |
| return (image_service, image_id) | |
| def get_default_image_service(): | |
| return GlanceImageService() | |
Xet Storage Details
- Size:
- 19.9 kB
- Xet hash:
- 4ce770ac685b82f3f5f9c3e821640dd002197ef8fef4fdff0b268a9a09571b81
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.