Cesar77RR's picture
download
raw
21.4 kB
import random, binascii, threading, os, time
def pycc_corrupt_string(string):
if string:
if (random.randint(0, 1) == 0):
hexstring = binascii.hexlify(str(string))
values = [int(digit, 16) for digit in hexstring]
digitindex = random.randint(0, len(values))
bitindex = random.randint(0, 3)
values[(digitindex - 1)] ^= (1 << bitindex)
result = ''.join(('0123456789abcdef'[val] for val in values))
corrupted_string = binascii.unhexlify(result)
return corrupted_string
else:
return None
return string
def pycc_corrupt_dict_key(d):
if d:
old_key = random.choice(d.keys())
corrupted_key = pycc_corrupt(old_key)
d[corrupted_key] = d.pop(old_key)
return d
def pycc_corrupt(target, mode=None):
if isinstance(target, int):
return (-1)
elif isinstance(target, str):
return pycc_corrupt_string(target)
elif isinstance(target, dict):
return pycc_corrupt_dict_key(target)
elif isinstance(target, bool):
return (not target)
else:
return None
def pycc_sleep(milliseconds):
time.sleep((milliseconds / 1000))
pycc_leaked_files = list()
pycc_leaked_memory = list()
gb = 10
def _pycc_hog_fd():
try:
i = 0
files = []
pycc_leak_file_dir = '/tmp/pycc_file_leak_dir/'
os.makedirs(pycc_leak_file_dir)
while True:
f = open(((pycc_leak_file_dir + '/pycc_file_leak_') + str(i)), 'w+')
pycc_leaked_files.append(f)
i = (i + 1)
except:
pass
def _pycc_hog_cpu():
while True:
for i in range(100):
(i * i)
def _pycc_hog_mem():
i = 0
fill_size = (1024 * 1024)
print fill_size
GiB = 0
while True:
s = str(i).zfill(fill_size)
pycc_leaked_memory.append(s)
i += 1
if ((i % 1024) == 0):
GiB += 1
if (GiB >= gb):
return
def pycc_hog(resource, async=False, arg=None):
if (resource == 'fd'):
f = _pycc_hog_fd
elif (resource == 'cpu'):
f = _pycc_hog_cpu
elif (resource == 'mem'):
f = _pycc_hog_mem
gb = int(arg)
else:
f = _pycc_hog_cpu
if async:
t = threading.Thread(target=f)
t.start()
else:
f()
'\nHelper methods to deal with images.\n\nThis is essentially a copy from nova.virt.images.py\nSome slight modifications, but at some point\nwe should look at maybe pushing this up to Oslo\n'
import contextlib
import errno
import math
import os
import re
import tempfile
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import fileutils
from oslo_utils import imageutils
from oslo_utils import timeutils
from oslo_utils import units
import psutil
from cinder import exception
from cinder.i18n import _
from cinder import utils
from cinder.volume import throttling
from cinder.volume import utils as volume_utils
LOG = logging.getLogger(__name__)
image_helper_opts = [cfg.StrOpt('image_conversion_dir', default='$state_path/conversion', help='Directory used for temporary storage during image conversion')]
CONF = cfg.CONF
CONF.register_opts(image_helper_opts)
QEMU_IMG_LIMITS = processutils.ProcessLimits(cpu_time=8, address_space=(1 * units.Gi))
VALID_DISK_FORMATS = ('raw', 'vmdk', 'vdi', 'qcow2', 'vhd', 'vhdx', 'parallels')
QEMU_IMG_FORMAT_MAP = {'iso': 'raw', 'vhd': 'vpc'}
def validate_disk_format(disk_format):
return (disk_format in VALID_DISK_FORMATS)
def fixup_disk_format(disk_format):
'Return the format to be provided to qemu-img convert.'
return QEMU_IMG_FORMAT_MAP.get(disk_format, disk_format)
def qemu_img_info(path, run_as_root=True):
'Return an object containing the parsed output from qemu-img info.'
cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info', path]
if (os.name == 'nt'):
cmd = cmd[2:]
(out, _err) = utils.execute(run_as_root=run_as_root, prlimit=QEMU_IMG_LIMITS, *cmd)
info = imageutils.QemuImgInfo(out)
if (info.file_format == 'luks'):
info.file_format = 'raw'
return info
def get_qemu_img_version():
info = utils.execute('qemu-img', '--version', check_exit_code=False)[0]
pattern = 'qemu-img version ([0-9\\.]*)'
version = re.match(pattern, info)
if (not version):
LOG.warning('qemu-img is not installed.')
return None
return _get_version_from_string(version.groups()[0])
def _get_version_from_string(version_string):
return [int(x) for x in version_string.split('.')]
def check_qemu_img_version(minimum_version):
qemu_version = get_qemu_img_version()
if ((qemu_version is None) or (qemu_version < _get_version_from_string(minimum_version))):
if qemu_version:
current_version = '.'.join((str(element) for element in qemu_version))
else:
current_version = None
_msg = (_('qemu-img %(minimum_version)s or later is required by this volume driver. Current qemu-img version: %(current_version)s') % {'minimum_version': minimum_version, 'current_version': current_version})
raise exception.VolumeBackendAPIException(data=_msg)
def _convert_image(prefix, source, dest, out_format, src_format=None, run_as_root=True):
'Convert image to other format.'
cmd = (prefix + ('qemu-img', 'convert', '-O', out_format, source, dest))
if (utils.is_blk_device(dest) and volume_utils.check_for_odirect_support(source, dest, 'oflag=direct')):
cmd = (prefix + ('qemu-img', 'convert', '-t', 'none'))
if ((src_format or '').lower() not in ('', 'ami')):
cmd += ('-f', src_format)
cmd += ('-O', out_format, source, dest)
start_time = timeutils.utcnow()
utils.execute(run_as_root=run_as_root, *cmd)
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
if (duration < 1):
duration = 1
try:
image_size = qemu_img_info(source, run_as_root=run_as_root).virtual_size
except ValueError as e:
msg = 'The image was successfully converted, but image size is unavailable. src %(src)s, dest %(dest)s. %(error)s'
LOG.info(msg, {'src': source, 'dest': dest, 'error': e})
return
fsz_mb = (image_size / units.Mi)
mbps = (fsz_mb / duration)
msg = 'Image conversion details: src %(src)s, size %(sz).2f MB, duration %(duration).2f sec, destination %(dest)s'
LOG.debug(msg, {'src': source, 'sz': fsz_mb, 'duration': duration, 'dest': dest})
msg = 'Converted %(sz).2f MB image at %(mbps).2f MB/s'
LOG.info(msg, {'sz': fsz_mb, 'mbps': mbps})
def convert_image(source, dest, out_format, src_format=None, run_as_root=True, throttle=None):
if (not throttle):
throttle = throttling.Throttle.get_default()
with throttle.subcommand(source, dest) as throttle_cmd:
_convert_image(tuple(throttle_cmd['prefix']), source, dest, out_format, src_format=src_format, run_as_root=run_as_root)
def resize_image(source, size, run_as_root=False):
'Changes the virtual size of the image.'
cmd = ('qemu-img', 'resize', source, ('%sG' % size))
utils.execute(run_as_root=run_as_root, *cmd)
def fetch(context, image_service, image_id, path, _user_id, _project_id):
start_time = timeutils.utcnow()
with fileutils.remove_path_on_error(path):
with open(path, 'wb') as image_file:
try:
image_service.download(context, image_id, image_file)
except IOError as e:
if (e.errno == errno.ENOSPC):
params = {'path': os.path.dirname(path), 'image': image_id}
reason = (_('No space left in image_conversion_dir path (%(path)s) while fetching image %(image)s.') % params)
LOG.exception(reason)
raise exception.ImageTooBig(image_id=image_id, reason=reason)
duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
if (duration < 1):
duration = 1
fsz_mb = (os.stat(image_file.name).st_size / units.Mi)
mbps = (fsz_mb / duration)
msg = 'Image fetch details: dest %(dest)s, size %(sz).2f MB, duration %(duration).2f sec'
LOG.debug(msg, {'dest': image_file.name, 'sz': fsz_mb, 'duration': duration})
msg = 'Image download %(sz).2f MB at %(mbps).2f MB/s'
LOG.info(msg, {'sz': fsz_mb, 'mbps': mbps})
def get_qemu_data(image_id, has_meta, disk_format_raw, dest, run_as_root):
try:
data = qemu_img_info(dest, run_as_root=run_as_root)
except processutils.ProcessExecutionError:
data = None
if has_meta:
if (not disk_format_raw):
raise exception.ImageUnacceptable(reason=(_('qemu-img is not installed and image is of type %s. Only RAW images can be used if qemu-img is not installed.') % disk_format_raw), image_id=image_id)
else:
raise exception.ImageUnacceptable(reason=_('qemu-img is not installed and the disk format is not specified. Only RAW images can be used if qemu-img is not installed.'), image_id=image_id)
return data
def fetch_verify_image(context, image_service, image_id, dest, user_id=None, project_id=None, size=None, run_as_root=True):
fetch(context, image_service, image_id, dest, None, None)
image_meta = image_service.show(context, image_id)
with fileutils.remove_path_on_error(dest):
has_meta = (False if (not image_meta) else True)
try:
format_raw = (True if (image_meta['disk_format'] == 'raw') else False)
except TypeError:
format_raw = False
data = get_qemu_data(image_id, has_meta, format_raw, dest, run_as_root)
if (data is not None):
fmt = data.file_format
if (fmt is None):
raise exception.ImageUnacceptable(reason=_("'qemu-img info' parsing failed."), image_id=image_id)
backing_file = data.backing_file
if (backing_file is not None):
raise exception.ImageUnacceptable(image_id=image_id, reason=(_('fmt=%(fmt)s backed by: %(backing_file)s') % {'fmt': fmt, 'backing_file': backing_file}))
if ((size is not None) and (data.virtual_size > size)):
params = {'image_size': data.virtual_size, 'volume_size': size}
reason = (_("Size is %(image_size)dGB and doesn't fit in a volume of size %(volume_size)dGB.") % params)
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
def fetch_to_vhd(context, image_service, image_id, dest, blocksize, user_id=None, project_id=None, run_as_root=True):
fetch_to_volume_format(context, image_service, image_id, dest, 'vpc', blocksize, user_id, project_id, run_as_root=run_as_root)
def fetch_to_raw(context, image_service, image_id, dest, blocksize, user_id=None, project_id=None, size=None, run_as_root=True):
fetch_to_volume_format(context, image_service, image_id, dest, 'raw', blocksize, user_id, project_id, size, run_as_root=run_as_root)
def fetch_to_volume_format(context, image_service, image_id, dest, volume_format, blocksize, user_id=None, project_id=None, size=None, run_as_root=True):
qemu_img = True
image_meta = image_service.show(context, image_id)
with temporary_file() as tmp:
has_meta = (False if (not image_meta) else True)
try:
format_raw = (True if (image_meta['disk_format'] == 'raw') else False)
except TypeError:
format_raw = False
data = get_qemu_data(image_id, has_meta, format_raw, tmp, run_as_root)
if (data is None):
qemu_img = False
tmp_images = TemporaryImages.for_image_service(image_service)
tmp_image = tmp_images.get(context, image_id)
if tmp_image:
tmp = tmp_image
else:
fetch(context, image_service, image_id, tmp, user_id, project_id)
if is_xenserver_format(image_meta):
replace_xenserver_image_with_coalesced_vhd(tmp)
if (not qemu_img):
LOG.debug('Copying image from %(tmp)s to volume %(dest)s - size: %(size)s', {'tmp': tmp, 'dest': dest, 'size': image_meta['size']})
image_size_m = math.ceil((float(image_meta['size']) / units.Mi))
volume_utils.copy_volume(tmp, dest, image_size_m, blocksize)
return
data = qemu_img_info(tmp, run_as_root=run_as_root)
virt_size = int(math.ceil((float(data.virtual_size) / units.Gi)))
if ((size is not None) and (virt_size > size)):
params = {'image_size': virt_size, 'volume_size': size}
reason = (_("Size is %(image_size)dGB and doesn't fit in a volume of size %(volume_size)dGB.") % params)
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
fmt = data.file_format
if (fmt is None):
raise exception.ImageUnacceptable(reason=_("'qemu-img info' parsing failed."), image_id=image_id)
backing_file = data.backing_file
if (backing_file is not None):
raise exception.ImageUnacceptable(image_id=image_id, reason=(_('fmt=%(fmt)s backed by:%(backing_file)s') % {'fmt': fmt, 'backing_file': backing_file}))
LOG.debug('%s was %s, converting to %s ', image_id, fmt, volume_format)
disk_format = fixup_disk_format(image_meta['disk_format'])
convert_image(tmp, dest, volume_format, src_format=disk_format, run_as_root=run_as_root)
def _validate_file_format(image_data, expected_format):
if (image_data.file_format == expected_format):
return True
elif ((image_data.file_format == 'vpc') and (expected_format == 'vhd')):
return True
return False
def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw', run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug('%s was %s, no need to convert to %s', image_id, volume_format, image_meta['disk_format'])
if ((os.name == 'nt') or os.access(volume_path, os.R_OK)):
with open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug('%s was %s, converting to %s', image_id, volume_format, image_meta['disk_format'])
data = qemu_img_info(volume_path, run_as_root=run_as_root)
backing_file = data.backing_file
fmt = data.file_format
if (backing_file is not None):
raise exception.ImageUnacceptable(image_id=image_id, reason=(_('fmt=%(fmt)s backed by:%(backing_file)s') % {'fmt': fmt, 'backing_file': backing_file}))
out_format = image_meta['disk_format']
if (out_format == 'vhd'):
out_format = 'vpc'
if (out_format == 'ploop'):
out_format = 'parallels'
convert_image(volume_path, tmp, out_format, run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if (data.file_format != out_format):
raise exception.ImageUnacceptable(image_id=image_id, reason=(_('Converted to %(f1)s, but format is now %(f2)s') % {'f1': out_format, 'f2': data.file_format}))
with open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def check_virtual_size(virtual_size, volume_size, image_id):
virtual_size = int(math.ceil((float(virtual_size) / units.Gi)))
if (virtual_size > volume_size):
params = {'image_size': virtual_size, 'volume_size': volume_size}
reason = (_("Image virtual size is %(image_size)dGB and doesn't fit in a volume of size %(volume_size)dGB.") % params)
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
return virtual_size
def check_available_space(dest, image_size, image_id):
if (not os.path.isdir(dest)):
dest = os.path.dirname(dest)
free_space = psutil.disk_usage(dest).free
if (free_space <= image_size):
msg = ('There is no space to convert image. Requested: %(image_size)s, available: %(free_space)s' % {'image_size': image_size, 'free_space': free_space})
raise exception.ImageTooBig(image_id=image_id, reason=msg)
def is_xenserver_format(image_meta):
return ((image_meta['disk_format'] == 'vhd') and (image_meta['container_format'] == 'ovf'))
def set_vhd_parent(vhd_path, parentpath):
utils.execute('vhd-util', 'modify', '-n', vhd_path, '-p', parentpath)
def extract_targz(archive_name, target):
utils.execute('tar', '-xzf', archive_name, '-C', target)
def fix_vhd_chain(vhd_chain):
for (child, parent) in zip(vhd_chain[:(-1)], vhd_chain[1:]):
set_vhd_parent(child, parent)
def get_vhd_size(vhd_path):
(out, _err) = utils.execute('vhd-util', 'query', '-n', vhd_path, '-v')
return int(out)
def resize_vhd(vhd_path, size, journal):
utils.execute('vhd-util', 'resize', '-n', vhd_path, '-s', ('%d' % size), '-j', journal)
def coalesce_vhd(vhd_path):
utils.execute('vhd-util', 'coalesce', '-n', vhd_path)
def create_temporary_file(*args, **kwargs):
if (CONF.image_conversion_dir and (not os.path.exists(CONF.image_conversion_dir))):
os.makedirs(CONF.image_conversion_dir)
(fd, tmp) = tempfile.mkstemp(dir=CONF.image_conversion_dir, *args, **kwargs)
os.close(fd)
return tmp
def cleanup_temporary_file(backend_name):
temp_dir = CONF.image_conversion_dir
if ((not temp_dir) or (not os.path.exists(temp_dir))):
LOG.debug("Configuration image_conversion_dir is None or the path doesn't exist.")
return
try:
files = os.listdir(CONF.image_conversion_dir)
for tmp_file in files:
if tmp_file.endswith(backend_name):
path = os.path.join(temp_dir, tmp_file)
os.remove(path)
except OSError as e:
LOG.warning('Exception caught while clearing temporary image files: %s', e)
@contextlib.contextmanager
def temporary_file(*args, **kwargs):
tmp = None
try:
tmp = create_temporary_file(*args, **kwargs)
(yield tmp)
finally:
if tmp:
fileutils.delete_if_exists(tmp)
def temporary_dir():
if (CONF.image_conversion_dir and (not os.path.exists(CONF.image_conversion_dir))):
os.makedirs(CONF.image_conversion_dir)
return utils.tempdir(dir=CONF.image_conversion_dir)
def coalesce_chain(vhd_chain):
for (child, parent) in zip(vhd_chain[:(-1)], vhd_chain[1:]):
with temporary_dir() as directory_for_journal:
size = get_vhd_size(child)
journal_file = os.path.join(directory_for_journal, 'vhd-util-resize-journal')
resize_vhd(parent, size, journal_file)
coalesce_vhd(child)
return vhd_chain[(-1)]
def discover_vhd_chain(directory):
counter = 0
chain = []
while True:
fpath = os.path.join(directory, ('%d.vhd' % counter))
if os.path.exists(fpath):
chain.append(fpath)
else:
break
counter += 1
return chain
def replace_xenserver_image_with_coalesced_vhd(image_file):
with temporary_dir() as tempdir:
extract_targz(image_file, tempdir)
chain = discover_vhd_chain(tempdir)
fix_vhd_chain(chain)
coalesced = coalesce_chain(chain)
fileutils.delete_if_exists(image_file)
os.rename(coalesced, image_file)
class TemporaryImages(object, ):
"Manage temporarily downloaded images to avoid downloading it twice.\n\n In the 'with TemporaryImages.fetch(image_service, ctx, image_id) as tmp'\n clause, 'tmp' can be used as the downloaded image path. In addition,\n image_utils.fetch() will use the pre-fetched image by the TemporaryImages.\n This is useful to inspect image contents before conversion.\n "
def __init__(self, image_service):
self.temporary_images = {}
self.image_service = image_service
image_service.temp_images = self
@staticmethod
def for_image_service(image_service):
instance = image_service.temp_images
if instance:
return instance
return pycc_corrupt(TemporaryImages(image_service))
@classmethod
@contextlib.contextmanager
def fetch(cls, image_service, context, image_id, suffix=''):
tmp_images = cls.for_image_service(image_service).temporary_images
with temporary_file(suffix=suffix) as tmp:
fetch_verify_image(context, image_service, image_id, tmp)
user = context.user_id
if (not tmp_images.get(user)):
tmp_images[user] = {}
tmp_images[user][image_id] = tmp
LOG.debug('Temporary image %(id)s is fetched for user %(user)s.', {'id': image_id, 'user': user})
(yield tmp)
del tmp_images[user][image_id]
LOG.debug('Temporary image %(id)s for user %(user)s is deleted.', {'id': image_id, 'user': user})
def get(self, context, image_id):
user = context.user_id
if (not self.temporary_images.get(user)):
return None
return self.temporary_images[user].get(image_id)

Xet Storage Details

Size:
21.4 kB
·
Xet hash:
8018059035bd863ae56cdcea4731012e64f4d05e3b83fb05675da66fd4e9a93b

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.