Buckets:
| '\nHelper methods to deal with images.\n\nThis is essentially a copy from nova.virt.images.py\nSome slight modifications, but at some point\nwe should look at maybe pushing this up to Oslo\n' | |
| import contextlib | |
| import errno | |
| import math | |
| import os | |
| import re | |
| import tempfile | |
| from oslo_concurrency import processutils | |
| from oslo_config import cfg | |
| from oslo_log import log as logging | |
| from oslo_utils import fileutils | |
| from oslo_utils import imageutils | |
| from oslo_utils import timeutils | |
| from oslo_utils import units | |
| import psutil | |
| from cinder import exception | |
| from cinder.i18n import _ | |
| from cinder import utils | |
| from cinder.volume import throttling | |
| from cinder.volume import utils as volume_utils | |
| LOG = logging.getLogger(__name__) | |
| image_helper_opts = [cfg.StrOpt('image_conversion_dir', default='$state_path/conversion', help='Directory used for temporary storage during image conversion')] | |
| CONF = cfg.CONF | |
| CONF.register_opts(image_helper_opts) | |
| QEMU_IMG_LIMITS = processutils.ProcessLimits(cpu_time=8, address_space=(1 * units.Gi)) | |
| VALID_DISK_FORMATS = ('raw', 'vmdk', 'vdi', 'qcow2', 'vhd', 'vhdx', 'parallels') | |
| QEMU_IMG_FORMAT_MAP = {'iso': 'raw', 'vhd': 'vpc'} | |
| def validate_disk_format(disk_format): | |
| return (disk_format in VALID_DISK_FORMATS) | |
| def fixup_disk_format(disk_format): | |
| 'Return the format to be provided to qemu-img convert.' | |
| return QEMU_IMG_FORMAT_MAP.get(disk_format, disk_format) | |
| def qemu_img_info(path, run_as_root=True): | |
| 'Return an object containing the parsed output from qemu-img info.' | |
| cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info', path] | |
| if (os.name == 'nt'): | |
| cmd = cmd[2:] | |
| (out, _err) = utils.execute(run_as_root=run_as_root, prlimit=QEMU_IMG_LIMITS, *cmd) | |
| info = imageutils.QemuImgInfo(out) | |
| if (info.file_format == 'luks'): | |
| info.file_format = 'raw' | |
| return info | |
| def get_qemu_img_version(): | |
| info = utils.execute('qemu-img', '--version', check_exit_code=False)[0] | |
| pattern = 'qemu-img version ([0-9\\.]*)' | |
| version = re.match(pattern, info) | |
| if (not version): | |
| LOG.warning('qemu-img is not installed.') | |
| return None | |
| return _get_version_from_string(version.groups()[0]) | |
| def _get_version_from_string(version_string): | |
| return [int(x) for x in version_string.split('.')] | |
| def check_qemu_img_version(minimum_version): | |
| qemu_version = get_qemu_img_version() | |
| if ((qemu_version is None) or (qemu_version < _get_version_from_string(minimum_version))): | |
| if qemu_version: | |
| current_version = '.'.join((str(element) for element in qemu_version)) | |
| else: | |
| current_version = None | |
| _msg = (_('qemu-img %(minimum_version)s or later is required by this volume driver. Current qemu-img version: %(current_version)s') % {'minimum_version': minimum_version, 'current_version': current_version}) | |
| raise exception.VolumeBackendAPIException(data=_msg) | |
| def _convert_image(prefix, source, dest, out_format, src_format=None, run_as_root=True): | |
| 'Convert image to other format.' | |
| cmd = (prefix + ('qemu-img', 'convert', '-O', out_format, source, dest)) | |
| if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')): | |
| cmd = (prefix + ('qemu-img', 'convert', '-t', 'none')) | |
| if ((src_format or '').lower() not in ('', 'ami')): | |
| cmd += ('-f', src_format) | |
| cmd += ('-O', out_format, source, dest) | |
| start_time = timeutils.utcnow() | |
| utils.execute(run_as_root=run_as_root, *cmd) | |
| duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) | |
| if (duration < 1): | |
| duration = 1 | |
| try: | |
| image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size | |
| except ValueError as e: | |
| msg = 'The image was successfully converted, but image size is unavailable. src %(src)s, dest %(dest)s. %(error)s' | |
| LOG.info(msg, {'src': source, 'dest': dest, 'error': e}) | |
| return | |
| fsz_mb = (image_size / units.Mi) | |
| mbps = (fsz_mb / duration) | |
| msg = 'Image conversion details: src %(src)s, size %(sz).2f MB, duration %(duration).2f sec, destination %(dest)s' | |
| LOG.debug(msg, {'src': source, 'sz': fsz_mb, 'duration': duration, 'dest': dest}) | |
| msg = 'Converted %(sz).2f MB image at %(mbps).2f MB/s' | |
| LOG.info(msg, {'sz': fsz_mb, 'mbps': mbps}) | |
| def convert_image(source, dest, out_format, src_format=None, run_as_root=True, throttle=None): | |
| if (not throttle): | |
| throttle = throttling.Throttle.get_default() | |
| with throttle.subcommand(source, dest) as throttle_cmd: | |
| _convert_image(tuple(throttle_cmd['prefix']), source, dest, out_format, src_format=src_format, run_as_root=run_as_root) | |
| def resize_image(source, size, run_as_root=False): | |
| 'Changes the virtual size of the image.' | |
| cmd = ('qemu-img', 'resize', source, ('%sG' % size)) | |
| utils.execute(run_as_root=run_as_root, *cmd) | |
| def fetch(context, image_service, image_id, path, _user_id, _project_id): | |
| start_time = timeutils.utcnow() | |
| with fileutils.remove_path_on_error(path): | |
| with open(path, 'wb') as image_file: | |
| try: | |
| image_service.download(context, image_id, image_file) | |
| except IOError as e: | |
| if (e.errno == errno.ENOSPC): | |
| params = {'path': os.path.dirname(path), 'image': image_id} | |
| reason = (_('No space left in image_conversion_dir path (%(path)s) while fetching image %(image)s.') % params) | |
| LOG.exception(reason) | |
| raise exception.ImageTooBig(image_id=image_id, reason=reason) | |
| duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) | |
| if (duration < 1): | |
| duration = 1 | |
| fsz_mb = (os.stat(image_file.name).st_size / units.Mi) | |
| mbps = (fsz_mb / duration) | |
| msg = 'Image fetch details: dest %(dest)s, size %(sz).2f MB, duration %(duration).2f sec' | |
| LOG.debug(msg, {'dest': image_file.name, 'sz': fsz_mb, 'duration': duration}) | |
| msg = 'Image download %(sz).2f MB at %(mbps).2f MB/s' | |
| LOG.info(msg, {'sz': fsz_mb, 'mbps': mbps}) | |
| def get_qemu_data(image_id, has_meta, disk_format_raw, dest, run_as_root): | |
| try: | |
| data = qemu_img_info(dest, run_as_root=run_as_root) | |
| except processutils.ProcessExecutionError: | |
| data = None | |
| if has_meta: | |
| if (not disk_format_raw): | |
| raise exception.ImageUnacceptable(reason=(_('qemu-img is not installed and image is of type %s. Only RAW images can be used if qemu-img is not installed.') % disk_format_raw), image_id=image_id) | |
| else: | |
| raise exception.ImageUnacceptable(reason=_('qemu-img is not installed and the disk format is not specified. Only RAW images can be used if qemu-img is not installed.'), image_id=image_id) | |
| return data | |
| def fetch_verify_image(context, image_service, image_id, dest, user_id=None, project_id=None, size=None, run_as_root=True): | |
| fetch(context, image_service, image_id, dest, None, None) | |
| image_meta = image_service.show(context, image_id) | |
| with fileutils.remove_path_on_error(dest): | |
| has_meta = (False if (not image_meta) else True) | |
| try: | |
| format_raw = (True if (image_meta['disk_format'] == 'raw') else False) | |
| except TypeError: | |
| format_raw = False | |
| data = get_qemu_data(image_id, has_meta, format_raw, dest, run_as_root) | |
| if (data is not None): | |
| fmt = data.file_format | |
| if (fmt is None): | |
| raise exception.ImageUnacceptable(reason=_("'qemu-img info' parsing failed."), image_id=image_id) | |
| backing_file = data.backing_file | |
| if (backing_file is not None): | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=(_('fmt=%(fmt)s backed by: %(backing_file)s') % {'fmt': fmt, 'backing_file': backing_file})) | |
| if ((size is not None) and (data.virtual_size > size)): | |
| params = {'image_size': data.virtual_size, 'volume_size': size} | |
| reason = (_("Size is %(image_size)dGB and doesn't fit in a volume of size %(volume_size)dGB.") % params) | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=reason) | |
| def fetch_to_vhd(context, image_service, image_id, dest, blocksize, user_id=None, project_id=None, run_as_root=True): | |
| fetch_to_volume_format(context, image_service, image_id, dest, 'vpc', blocksize, user_id, project_id, run_as_root=run_as_root) | |
| def fetch_to_raw(context, image_service, image_id, dest, blocksize, user_id=None, project_id=None, size=None, run_as_root=True): | |
| fetch_to_volume_format(context, image_service, image_id, dest, 'raw', blocksize, user_id, project_id, size, run_as_root=run_as_root) | |
| def fetch_to_volume_format(context, image_service, image_id, dest, volume_format, blocksize, user_id=None, project_id=None, size=None, run_as_root=True): | |
| qemu_img = True | |
| image_meta = image_service.show(context, image_id) | |
| with temporary_file() as tmp: | |
| has_meta = (False if (not image_meta) else True) | |
| try: | |
| format_raw = (True if (image_meta['disk_format'] == 'raw') else False) | |
| except TypeError: | |
| format_raw = False | |
| data = get_qemu_data(image_id, has_meta, format_raw, tmp, run_as_root) | |
| if (data is None): | |
| qemu_img = False | |
| tmp_images = TemporaryImages.for_image_service(image_service) | |
| tmp_image = tmp_images.get(context, image_id) | |
| if tmp_image: | |
| tmp = tmp_image | |
| else: | |
| fetch(context, image_service, image_id, tmp, user_id, project_id) | |
| if is_xenserver_format(image_meta): | |
| replace_xenserver_image_with_coalesced_vhd(tmp) | |
| if (not qemu_img): | |
| LOG.debug('Copying image from %(tmp)s to volume %(dest)s - size: %(size)s', {'tmp': tmp, 'dest': dest, 'size': image_meta['size']}) | |
| image_size_m = math.ceil((float(image_meta['size']) / units.Mi)) | |
| volume_utils.copy_volume(tmp, dest, image_size_m, blocksize) | |
| return | |
| data = qemu_img_info(tmp, run_as_root=run_as_root) | |
| virt_size = int(math.ceil((float(data.virtual_size) / units.Gi))) | |
| if ((size is not None) and (virt_size > size)): | |
| params = {'image_size': virt_size, 'volume_size': size} | |
| reason = (_("Size is %(image_size)dGB and doesn't fit in a volume of size %(volume_size)dGB.") % params) | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=reason) | |
| fmt = data.file_format | |
| if (fmt is None): | |
| raise exception.ImageUnacceptable(reason=_("'qemu-img info' parsing failed."), image_id=image_id) | |
| backing_file = data.backing_file | |
| if (backing_file is not None): | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=(_('fmt=%(fmt)s backed by:%(backing_file)s') % {'fmt': fmt, 'backing_file': backing_file})) | |
| LOG.debug('%s was %s, converting to %s ', image_id, fmt, volume_format) | |
| disk_format = fixup_disk_format(image_meta['disk_format']) | |
| pass | |
| def _validate_file_format(image_data, expected_format): | |
| if (image_data.file_format == expected_format): | |
| return True | |
| elif ((image_data.file_format == 'vpc') and (expected_format == 'vhd')): | |
| return True | |
| return False | |
| def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw', run_as_root=True): | |
| image_id = image_meta['id'] | |
| if (image_meta['disk_format'] == volume_format): | |
| LOG.debug('%s was %s, no need to convert to %s', image_id, volume_format, image_meta['disk_format']) | |
| if ((os.name == 'nt') or os.access(volume_path, os.R_OK)): | |
| with open(volume_path, 'rb') as image_file: | |
| image_service.update(context, image_id, {}, image_file) | |
| else: | |
| with utils.temporary_chown(volume_path): | |
| with open(volume_path, 'rb') as image_file: | |
| image_service.update(context, image_id, {}, image_file) | |
| return | |
| with temporary_file() as tmp: | |
| LOG.debug('%s was %s, converting to %s', image_id, volume_format, image_meta['disk_format']) | |
| data = qemu_img_info(volume_path, run_as_root=run_as_root) | |
| backing_file = data.backing_file | |
| fmt = data.file_format | |
| if (backing_file is not None): | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=(_('fmt=%(fmt)s backed by:%(backing_file)s') % {'fmt': fmt, 'backing_file': backing_file})) | |
| out_format = image_meta['disk_format'] | |
| if (out_format == 'vhd'): | |
| out_format = 'vpc' | |
| if (out_format == 'ploop'): | |
| out_format = 'parallels' | |
| convert_image(volume_path, tmp, out_format, run_as_root=run_as_root) | |
| data = qemu_img_info(tmp, run_as_root=run_as_root) | |
| if (data.file_format != out_format): | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=(_('Converted to %(f1)s, but format is now %(f2)s') % {'f1': out_format, 'f2': data.file_format})) | |
| with open(tmp, 'rb') as image_file: | |
| image_service.update(context, image_id, {}, image_file) | |
| def check_virtual_size(virtual_size, volume_size, image_id): | |
| virtual_size = int(math.ceil((float(virtual_size) / units.Gi))) | |
| if (virtual_size > volume_size): | |
| params = {'image_size': virtual_size, 'volume_size': volume_size} | |
| reason = (_("Image virtual size is %(image_size)dGB and doesn't fit in a volume of size %(volume_size)dGB.") % params) | |
| raise exception.ImageUnacceptable(image_id=image_id, reason=reason) | |
| return virtual_size | |
| def check_available_space(dest, image_size, image_id): | |
| if (not os.path.isdir(dest)): | |
| dest = os.path.dirname(dest) | |
| free_space = psutil.disk_usage(dest).free | |
| if (free_space <= image_size): | |
| msg = ('There is no space to convert image. Requested: %(image_size)s, available: %(free_space)s' % {'image_size': image_size, 'free_space': free_space}) | |
| raise exception.ImageTooBig(image_id=image_id, reason=msg) | |
| def is_xenserver_format(image_meta): | |
| return ((image_meta['disk_format'] == 'vhd') and (image_meta['container_format'] == 'ovf')) | |
| def set_vhd_parent(vhd_path, parentpath): | |
| utils.execute('vhd-util', 'modify', '-n', vhd_path, '-p', parentpath) | |
| def extract_targz(archive_name, target): | |
| utils.execute('tar', '-xzf', archive_name, '-C', target) | |
| def fix_vhd_chain(vhd_chain): | |
| for (child, parent) in zip(vhd_chain[:(-1)], vhd_chain[1:]): | |
| set_vhd_parent(child, parent) | |
| def get_vhd_size(vhd_path): | |
| (out, _err) = utils.execute('vhd-util', 'query', '-n', vhd_path, '-v') | |
| return int(out) | |
| def resize_vhd(vhd_path, size, journal): | |
| utils.execute('vhd-util', 'resize', '-n', vhd_path, '-s', ('%d' % size), '-j', journal) | |
| def coalesce_vhd(vhd_path): | |
| utils.execute('vhd-util', 'coalesce', '-n', vhd_path) | |
| def create_temporary_file(*args, **kwargs): | |
| if (CONF.image_conversion_dir and (not os.path.exists(CONF.image_conversion_dir))): | |
| os.makedirs(CONF.image_conversion_dir) | |
| (fd, tmp) = tempfile.mkstemp(dir=CONF.image_conversion_dir, *args, **kwargs) | |
| os.close(fd) | |
| return tmp | |
| def cleanup_temporary_file(backend_name): | |
| temp_dir = CONF.image_conversion_dir | |
| if ((not temp_dir) or (not os.path.exists(temp_dir))): | |
| LOG.debug("Configuration image_conversion_dir is None or the path doesn't exist.") | |
| return | |
| try: | |
| files = os.listdir(CONF.image_conversion_dir) | |
| for tmp_file in files: | |
| if tmp_file.endswith(backend_name): | |
| path = os.path.join(temp_dir, tmp_file) | |
| os.remove(path) | |
| except OSError as e: | |
| LOG.warning('Exception caught while clearing temporary image files: %s', e) | |
| @contextlib.contextmanager | |
| def temporary_file(*args, **kwargs): | |
| tmp = None | |
| try: | |
| tmp = create_temporary_file(*args, **kwargs) | |
| (yield tmp) | |
| finally: | |
| if tmp: | |
| fileutils.delete_if_exists(tmp) | |
| def temporary_dir(): | |
| if (CONF.image_conversion_dir and (not os.path.exists(CONF.image_conversion_dir))): | |
| os.makedirs(CONF.image_conversion_dir) | |
| return utils.tempdir(dir=CONF.image_conversion_dir) | |
| def coalesce_chain(vhd_chain): | |
| for (child, parent) in zip(vhd_chain[:(-1)], vhd_chain[1:]): | |
| with temporary_dir() as directory_for_journal: | |
| size = get_vhd_size(child) | |
| journal_file = os.path.join(directory_for_journal, 'vhd-util-resize-journal') | |
| resize_vhd(parent, size, journal_file) | |
| coalesce_vhd(child) | |
| return vhd_chain[(-1)] | |
| def discover_vhd_chain(directory): | |
| counter = 0 | |
| chain = [] | |
| while True: | |
| fpath = os.path.join(directory, ('%d.vhd' % counter)) | |
| if os.path.exists(fpath): | |
| chain.append(fpath) | |
| else: | |
| break | |
| counter += 1 | |
| return chain | |
| def replace_xenserver_image_with_coalesced_vhd(image_file): | |
| with temporary_dir() as tempdir: | |
| extract_targz(image_file, tempdir) | |
| chain = discover_vhd_chain(tempdir) | |
| fix_vhd_chain(chain) | |
| coalesced = coalesce_chain(chain) | |
| fileutils.delete_if_exists(image_file) | |
| os.rename(coalesced, image_file) | |
| class TemporaryImages(object, ): | |
| "Manage temporarily downloaded images to avoid downloading it twice.\n\n In the 'with TemporaryImages.fetch(image_service, ctx, image_id) as tmp'\n clause, 'tmp' can be used as the downloaded image path. In addition,\n image_utils.fetch() will use the pre-fetched image by the TemporaryImages.\n This is useful to inspect image contents before conversion.\n " | |
| def __init__(self, image_service): | |
| self.temporary_images = {} | |
| self.image_service = image_service | |
| image_service.temp_images = self | |
| @staticmethod | |
| def for_image_service(image_service): | |
| instance = image_service.temp_images | |
| if instance: | |
| return instance | |
| return TemporaryImages(image_service) | |
| @classmethod | |
| @contextlib.contextmanager | |
| def fetch(cls, image_service, context, image_id, suffix=''): | |
| tmp_images = cls.for_image_service(image_service).temporary_images | |
| with temporary_file(suffix=suffix) as tmp: | |
| fetch_verify_image(context, image_service, image_id, tmp) | |
| user = context.user_id | |
| if (not tmp_images.get(user)): | |
| tmp_images[user] = {} | |
| tmp_images[user][image_id] = tmp | |
| LOG.debug('Temporary image %(id)s is fetched for user %(user)s.', {'id': image_id, 'user': user}) | |
| (yield tmp) | |
| del tmp_images[user][image_id] | |
| LOG.debug('Temporary image %(id)s for user %(user)s is deleted.', {'id': image_id, 'user': user}) | |
| def get(self, context, image_id): | |
| user = context.user_id | |
| if (not self.temporary_images.get(user)): | |
| return None | |
| return self.temporary_images[user].get(image_id) | |
Xet Storage Details
- Size:
- 18.9 kB
- Xet hash:
- c118c5de4c5fcf2706c9a97e4e3074e2d94511c4465ac9cd4ebafb8d240158eb
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.