Instruction:
Can you write a function in Python 3 where it
applies a function to all values with a certain leaf key.

Response:
def apply(d, leaf_key, func, new_name=None, remove_lkey=True,
          list_of_dicts=False, unflatten_level=0, deepcopy=True, **kwargs):
    """ apply a function to all values with a certain leaf (terminal) key

    Parameters
    ----------
    d : dict
    leaf_key : str
        name of leaf key
    func : callable
        function to apply
    new_name : str
        if not None, rename leaf_key
    remove_lkey: bool
        whether to remove original leaf_key (if new_name is not None)
    list_of_dicts: bool
        treat list of dicts as additional branches
    unflatten_level : int or None
        the number of levels to leave unflattened before combining,
        for instance if you need dicts as inputs
    deepcopy: bool
        deepcopy values
    kwargs : dict
        additional keywords to parse to function

    Examples
    --------

    >>> from pprint import pprint
    >>> d = {'a':1,'b':1}
    >>> func = lambda x: x+1
    >>> pprint(apply(d,'a',func))
    {'a': 2, 'b': 1}
    >>> pprint(apply(d,'a',func,new_name='c'))
    {'b': 1, 'c': 2}
    >>> pprint(apply(d,'a',func,new_name='c', remove_lkey=False))
    {'a': 1, 'b': 1, 'c': 2}

    >>> test_dict = {"a":[{"b":[{"c":1, "d": 2}, {"e":3, "f": 4}]}, {"b":[{"c":5, "d": 6}, {"e":7, "f": 8}]}]}
    >>> pprint(apply(test_dict, "b", lambda x: x[-1], list_of_dicts=True, unflatten_level=2))
    {'a': [{'b': {'e': 3, 'f': 4}}, {'b': {'e': 7, 'f': 8}}]}

    """  # noqa: E501
    list_of_dicts = '__list__' if list_of_dicts else None
    if unflatten_level == 0:
        flatd = flatten(d, list_of_dicts=list_of_dicts)
    else:
        flatd = flattennd(d, unflatten_level, list_of_dicts=list_of_dicts)
    newd = {k: (func(v, **kwargs) if k[-1] == leaf_key else v)
            for k, v in flatd.items()}
    if new_name is not None:
        newd = {(tuple(list(k[:-1]) + [new_name]) if k[-1]
                 == leaf_key else k): v for k, v in newd.items()}
        if not remove_lkey:
            newd.update(flatd)

    return unflatten(newd, list_of_dicts=list_of_dicts, deepcopy=deepcopy)


Instruction:
How would you explain what the following Python 3 function does
def combine_apply(d, leaf_keys, func, new_name,
                  unflatten_level=1, remove_lkeys=True, overwrite=False,
                  list_of_dicts=False, deepcopy=True, **kwargs):
    """ combine values with certain leaf (terminal) keys by a function

    Parameters
    ----------
    d : dict
    leaf_keys : list
        names of leaf keys
    func : callable
        function to apply,
        must take at least len(leaf_keys) arguments
    new_name : str
        new key name
    unflatten_level : int or None
        the number of levels to leave unflattened before combining,
        for instance if you need dicts as inputs (None means all)
    remove_lkeys: bool
        whether to remove original leaf_keys
    overwrite: bool
        whether to overwrite any existing new_name key
    list_of_dicts: bool
        treat list of dicts as additional branches
    deepcopy: bool
        deepcopy values
    kwargs : dict
        additional keywords to parse to function

    Examples
    --------

    >>> from pprint import pprint
    >>> d = {'a':1,'b':2}
    >>> func = lambda x,y: x+y
    >>> pprint(combine_apply(d,['a','b'],func,'c'))
    {'c': 3}
    >>> pprint(combine_apply(d,['a','b'],func,'c',remove_lkeys=False))
    {'a': 1, 'b': 2, 'c': 3}

    >>> d = {1:{'a':1,'b':2},2:{'a':4,'b':5},3:{'a':1}}
    >>> pprint(combine_apply(d,['a','b'],func,'c'))
    {1: {'c': 3}, 2: {'c': 9}, 3: {'a': 1}}

    >>> func2 = lambda x: sorted(list(x.keys()))
    >>> d2 = {'d':{'a':{'b':1,'c':2}}}
    >>> pprint(combine_apply(d2,['a'],func2,'a',unflatten_level=2))
    {'d': {'a': ['b', 'c']}}

    """
    list_of_dicts = '__list__' if list_of_dicts else None
    if unflatten_level is not None:
        flatd = flattennd(d, levels=unflatten_level,
                          list_of_dicts=list_of_dicts)
    else:
        # TODO could do this better?
        flatd = unflatten(d, key_as_tuple=False,
                          delim='*@#$', deepcopy=deepcopy)

    for dic in flatd.values():
        if not is_dict_like(dic):
            continue
        if all([k in list(dic.keys()) for k in leaf_keys]):
            if remove_lkeys:
                vals = [dic.pop(k) for k in leaf_keys]
            else:
                vals = [dic[k] for k in leaf_keys]
            if new_name in dic and not overwrite:
                raise ValueError('{} already in sub-dict'.format(new_name))
            dic[new_name] = func(*vals, **kwargs)

    if unflatten_level is not None:
        return unflatten(flatd, list_of_dicts=list_of_dicts, deepcopy=deepcopy)
    else:
        return flatd

Response: combine values with certain leaf keys by a function


Instruction:
How would you explain what the following Python 3 function does
def split_lists(d, split_keys, new_name='split',
                check_length=True, deepcopy=True):
    """split_lists key:list pairs into dicts for each item in the lists
    NB: will only split if all split_keys are present

    Parameters
    ----------
    d : dict
    split_keys : list
        keys to split
    new_name : str
        top level key for split items
    check_length : bool
        if true, raise error if any lists are of a different length
    deepcopy: bool
        deepcopy values

    Examples
    --------

    >>> from pprint import pprint

    >>> d = {'path_key':{'x':[1,2],'y':[3,4],'a':1}}
    >>> new_d = split_lists(d,['x','y'])
    >>> pprint(new_d)
    {'path_key': {'a': 1, 'split': [{'x': 1, 'y': 3}, {'x': 2, 'y': 4}]}}

    >>> split_lists(d,['x','a'])
    Traceback (most recent call last):
    ...
    ValueError: "a" data at the following path is not a list ('path_key',)

    >>> d2 = {'path_key':{'x':[1,7],'y':[3,4,5]}}
    >>> split_lists(d2,['x','y'])
    Traceback (most recent call last):
    ...
    ValueError: lists at the following path do not have the same size ('path_key',)

    """  # noqa: E501
    flattened = flatten2d(d)

    new_d = {}
    for key, value in flattened.items():
        if set(split_keys).issubset(value.keys()):
            # combine_d = {}
            combine_d = []
            sub_d = {}
            length = None
            for subkey, subvalue in value.items():
                if subkey in split_keys:
                    if not isinstance(subvalue, list):
                        raise ValueError(
                            '"{0}" data at the following path is not a list '
                            '{1}'.format(subkey, key))

                    if check_length and length is not None:
                        if len(subvalue) != length:
                            raise ValueError(
                                'lists at the following path '
                                'do not have the same size {0}'.format(key))
                    if length is None:
                        combine_d = [{subkey: v} for v in subvalue]
                    else:
                        for item, val in zip(combine_d, subvalue):
                            item[subkey] = val

                    length = len(subvalue)
                    # new_combine = {k:{subkey:v}
                    # for k,v in enumerate(subvalue)}
                    # combine_d = merge([combine_d,new_combine])
                else:
                    sub_d[subkey] = subvalue
                try:
                    new_d[key] = merge([sub_d, {new_name: combine_d}])
                except ValueError:
                    raise ValueError(
                        'split data key: {0}, already exists at '
                        'this level for {1}'.format(new_name, key))
        else:
            new_d[key] = value

    return unflatten(new_d, deepcopy=deepcopy)

Response: split_lists key = list pairs into dicts for each item in the lists


Instruction:
Write a Python 3 script to
combine lists of dicts

Response:
def combine_lists(d, keys=None, deepcopy=True):
    """combine lists of dicts

    Parameters
    ----------
    d : dict or list[dict]
    keys : list
        keys to combine (all if None)
    deepcopy: bool
        deepcopy values

    Example
    -------
    >>> from pprint import pprint
    >>> d = {'path_key': {'a': 1, 'split': [{'x': 1, 'y': 3}, {'x': 2, 'y': 4}]}}
    >>> pprint(combine_lists(d,['split']))
    {'path_key': {'a': 1, 'split': {'x': [1, 2], 'y': [3, 4]}}}

    >>> combine_lists([{"a":2}, {"a":1}])
    {'a': [2, 1]}

    """  # noqa: E501
    if isinstance(d, list):
        init_list = True
        d = {'dummy_key843': d}
    else:
        init_list = False

    flattened = flatten(d, list_of_dicts=None)
    for key, value in list(flattened.items()):
        if keys is not None:
            try:
                if not key[-1] in keys:
                    continue
            except Exception:
                continue
        if not isinstance(value, list):
            continue
        if not all([is_dict_like(d) for d in value]):
            continue
        newd = {}
        for subdic in value:
            for subk, subv in subdic.items():
                if subk not in newd:
                    newd[subk] = []
                newd[subk].append(subv)
        flattened[key] = newd

    final = unflatten(flattened, list_of_dicts=None, deepcopy=deepcopy)

    if init_list:
        return list(final.values())[0]
    else:
        return final


Instruction:
Here you have a function in Python 3, explain what it does
def list_to_dict(lst, key=None, remove_key=True):

    assert all([is_dict_like(d) for d in lst])
    if key is not None:
        assert all([key in d for d in lst])
    new_dict = {}
    for i, d in enumerate(lst):
        d = unflatten(flatten(d))
        if key is None:
            new_dict[str(i)] = d
        else:
            if remove_key:
                k = d.pop(key)
            else:
                k = d[key]
            new_dict[k] = d

    return new_dict

Response: convert a list of dicts to a dict with root keys


Instruction:
Here you have a function in Python 3, explain what it does
def diff(new_dict, old_dict, iter_prefix='__iter__',
         np_allclose=False, **kwargs):
    """ return the difference between two dict_like objects

    Parameters
    ----------
    new_dict: dict
    old_dict: dict
    iter_prefix: str
        prefix to use for list and tuple indexes
    np_allclose: bool
        if True, try using numpy.allclose to assess differences
    **kwargs:
        keyword arguments to parse to numpy.allclose

    Returns
    -------
    outcome: dict
        Containing none or more of:

        - "insertions" : list of (path, val)
        - "deletions" : list of (path, val)
        - "changes" : list of (path, (val1, val2))
        - "uncomparable" : list of (path, (val1, val2))

    Examples
    --------
    >>> from pprint import pprint

    >>> diff({'a':1},{'a':1})
    {}

    >>> pprint(diff({'a': 1, 'b': 2, 'c': 5},{'b': 3, 'c': 4, 'd': 6}))
    {'changes': [(('b',), (2, 3)), (('c',), (5, 4))],
     'deletions': [(('d',), 6)],
     'insertions': [(('a',), 1)]}

    >>> pprint(diff({'a': [{"b":1}, {"c":2}, 1]},{'a': [{"b":1}, {"d":2}, 2]}))
    {'changes': [(('a', '__iter__2'), (1, 2))],
     'deletions': [(('a', '__iter__1', 'd'), 2)],
     'insertions': [(('a', '__iter__1', 'c'), 2)]}

    >>> diff({'a':1}, {'a':1+1e-10})
    {'changes': [(('a',), (1, 1.0000000001))]}

    >>> diff({'a':1}, {'a':1+1e-10}, np_allclose=True)
    {}

    """
    if np_allclose:
        try:
            import numpy
        except ImportError:
            raise ValueError("to use np_allclose, numpy must be installed")

    dct1_flat = flatten(new_dict, all_iters=iter_prefix)
    dct2_flat = flatten(old_dict, all_iters=iter_prefix)

    outcome = {'insertions': [], 'deletions': [],
               'changes': [], 'uncomparable': []}

    for path, val in dct1_flat.items():
        if path not in dct2_flat:
            outcome['insertions'].append((path, val))
            continue
        other_val = dct2_flat.pop(path)
        if np_allclose:
            try:
                if numpy.allclose(val, other_val, **kwargs):
                    continue
            except Exception:
                pass
        try:
            if val != other_val:
                outcome['changes'].append((path, (val, other_val)))
        except Exception:
            outcome['uncomparable'].append((path, (val, other_val)))

    for path2, val2 in dct2_flat.items():
        outcome['deletions'].append((path2, val2))

    # remove any empty lists and sort
    for key in list(outcome.keys()):
        if not outcome[key]:
            outcome.pop(key)
        try:
            outcome[key] = sorted(outcome[key])
        except Exception:
            pass

    return outcome

Response: Returns the difference between two dict_like objects


Instruction:
Implement a function in Python 3 to
output dict to json

Response:
def to_json(dct, jfile, overwrite=False, dirlevel=0, sort_keys=True, indent=2,
            default_name='root.json', **kwargs):
    """ output dict to json

    Parameters
    ----------
    dct : dict
    jfile : str or file_like
        if file_like, must have write method
    overwrite : bool
        whether to overwrite existing files
    dirlevel : int
        if jfile is path to folder,
        defines how many key levels to set as sub-folders
    sort_keys : bool
        if true then the output of dictionaries will be sorted by key
    indent : int
        if non-negative integer, then JSON array elements and object members
        will be pretty-printed on new lines with that indent level spacing.
    kwargs : dict
        keywords for json.dump

    Examples
    --------

    >>> from jsonextended.utils import MockPath
    >>> file_obj = MockPath('test.json',is_file=True,exists=False)
    >>> dct = {'a':{'b':1}}
    >>> to_json(dct, file_obj)
    >>> print(file_obj.to_string())
    File("test.json") Contents:
    {
      "a": {
        "b": 1
      }
    }

    >>> from jsonextended.utils import MockPath
    >>> folder_obj = MockPath()
    >>> dct = {'x':{'a':{'b':1},'c':{'d':3}}}
    >>> to_json(dct, folder_obj, dirlevel=0,indent=None)
    >>> print(folder_obj.to_string(file_content=True))
    Folder("root")
      File("x.json") Contents:
       {"a": {"b": 1}, "c": {"d": 3}}

    >>> folder_obj = MockPath()
    >>> to_json(dct, folder_obj, dirlevel=1,indent=None)
    >>> print(folder_obj.to_string(file_content=True))
    Folder("root")
      Folder("x")
        File("a.json") Contents:
         {"b": 1}
        File("c.json") Contents:
         {"d": 3}

    """
    if hasattr(jfile, 'write'):
        json.dump(dct, jfile, sort_keys=sort_keys,
                  indent=indent, default=encode)
        return

    if isinstance(jfile, basestring):
        path = pathlib.Path(jfile)
    else:
        path = jfile

    file_attrs = ['exists', 'is_dir', 'is_file', 'touch', 'open']
    if not all([hasattr(path, attr) for attr in file_attrs]):
        raise ValueError(
            'jfile should be a str or file_like object: {}'.format(jfile))

    if path.is_file() and path.exists() and not overwrite:
        raise IOError('jfile already exists and '
                      'overwrite is set to false: {}'.format(jfile))

    if not path.is_dir() and dirlevel <= 0:
        path.touch()  # try to create file if doesn't already exist
        with path.open('w') as outfile:
            outfile.write(unicode(json.dumps(
                dct, sort_keys=sort_keys,
                indent=indent, default=encode, **kwargs)))
            return

    if not path.is_dir():
        path.mkdir()
        dirlevel -= 1

    # if one or more values if not a nested dict
    if not all([hasattr(v, 'items') for v in dct.values()]):
        newpath = path.joinpath(default_name)
        newpath.touch()
        with newpath.open('w') as outfile:
            outfile.write(unicode(json.dumps(
                dct, sort_keys=sort_keys,
                indent=indent, default=encode, **kwargs)))
            return

    for key, val in dct.items():
        if dirlevel <= 0:
            newpath = path.joinpath('{}.json'.format(key))
            newpath.touch()
            with newpath.open('w') as outfile:
                outfile.write(unicode(json.dumps(
                    val, ensure_ascii=False, sort_keys=sort_keys,
                    indent=indent, default=encode, **kwargs)))
        else:
            newpath = path.joinpath('{}'.format(key))
            if not newpath.exists():
                newpath.mkdir()
            to_json(val, newpath, overwrite=overwrite, dirlevel=dirlevel - 1,
                    sort_keys=sort_keys, indent=indent,
                    default_name='{}.json'.format(key), **kwargs)


Instruction:
Can you tell what is the following Python 3 function doing
def dump(dct, jfile, overwrite=False, dirlevel=0, sort_keys=True,
         indent=2, default_name='root.json', **kwargs):
    """ output dict to json

    Parameters
    ----------
    dct : dict
    jfile : str or file_like
        if file_like, must have write method
    overwrite : bool
        whether to overwrite existing files
    dirlevel : int
        if jfile is path to folder,
        defines how many key levels to set as sub-folders
    sort_keys : bool
        if true then the output of dictionaries will be sorted by key
    indent : int
        if non-negative integer, then JSON array elements and object members
        will be pretty-printed on new lines with that indent level spacing.
    kwargs : dict
        keywords for json.dump
    """
    to_json(dct, jfile, overwrite=overwrite, dirlevel=dirlevel,
            sort_keys=sort_keys, indent=indent,
            default_name=default_name, **kwargs)

Response: dump dict to json


Instruction:
Can you write a function in Python 3 where it
expands a collection of items in a dict in which a key will be inserted or expanded.

Response:
def expand_collection_in_dict(d, key, new_items, no_duplicates=True):
    """
    Parameters
    d: dict
        dict in which a key will be inserted/expanded
    key: hashable
        key in d
    new_items: iterable
        d[key] will be extended with items in new_items
    no_duplicates: bool
        avoid inserting duplicates in d[key] (default: True)
    """
    if key in d:
        if no_duplicates:
            new_items = filter(lambda x: x not in d[key], new_items)
        if isinstance(d[key], set):
            map(d[key].add, new_items)
        elif isinstance(d[key], list):
            map(d[key].append, new_items)
        else:
            d[key] = d[key] + new_items
    else:
        d[key] = new_items


Instruction:
Implement a function in Python 3 to
augment shutil. copy with extra options and slightly slightly CTYPE modified behaviour

Response:
def copy(src, dst, only_update=False, copystat=True, cwd=None,
         dest_is_dir=False, create_dest_dirs=False, logger=None):
    """
    Augmented shutil.copy with extra options and slightly
    modified behaviour

    Parameters
    ==========
    src: string
        path to source file
    dst: string
        path to destingation
    only_update: bool
        only copy if source is newer than destination
        (returns None if it was newer), default: False
    copystat: bool
        See shutil.copystat. default: True
    cwd: string
        Path to working directory (root of relative paths)
    dest_is_dir: bool
        ensures that dst is treated as a directory. default: False
    create_dest_dirs: bool
        creates directories if needed.
    logger: logging.Looger
        debug level info emitted. Passed onto make_dirs.

    Returns
    =======
    Path to the copied file.

    """
    # Handle virtual working directory
    if cwd:
        if not os.path.isabs(src):
            src = os.path.join(cwd, src)
        if not os.path.isabs(dst):
            dst = os.path.join(cwd, dst)

    # Make sure source file extists
    if not os.path.exists(src):
        # Source needs to exist
        msg = "Source: `{}` does not exist".format(src)
        raise FileNotFoundError(msg)

    # We accept both (re)naming destination file _or_
    # passing a (possible non-existant) destination directory
    if dest_is_dir:
        if not dst[-1] == '/':
            dst = dst+'/'
    else:
        if os.path.exists(dst) and os.path.isdir(dst):
            dest_is_dir = True

    if dest_is_dir:
        dest_dir = dst
        dest_fname = os.path.basename(src)
        dst = os.path.join(dest_dir, dest_fname)
    else:
        dest_dir = os.path.dirname(dst)
        dest_fname = os.path.basename(dst)

    if not os.path.exists(dest_dir):
        if create_dest_dirs:
            make_dirs(dest_dir, logger=logger)
        else:
            msg = "You must create directory first."
            raise FileNotFoundError(msg)

    if only_update:
        if not missing_or_other_newer(dst, src):
            if logger:
                logger.debug(
                    "Did not copy {} to {} (source not newer)".format(
                        src, dst))
            return

    if os.path.islink(dst):
        if os.path.abspath(os.path.realpath(dst)) == \
           os.path.abspath(dst):
            pass  # destination is a symlic pointing to src
    else:
        if logger:
            logger.debug("Copying {} to {}".format(src, dst))
        shutil.copy(src, dst)
        if copystat:
            shutil.copystat(src, dst)
    return dst


Instruction:
Implement a Python 3 function for
computing the md5 hash of a file.

Response:
def md5_of_file(path, nblocks=128):
    """
    Computes the md5 hash of a file.

    Parameters
    ==========
    path: string
        path to file to compute hash of

    Returns
    =======
    hashlib md5 hash object. Use .digest() or .hexdigest()
    on returned object to get binary or hex encoded string.
    """
    md = md5()
    with open(path, 'rb') as f:
        for chunk in iter(lambda: f.read(nblocks*md.block_size), b''):
            md.update(chunk)
    return md


Instruction:
Can you tell what is the following Python 3 function doing
def missing_or_other_newer(path, other_path, cwd=None):
    cwd = cwd or '.'
    path = get_abspath(path, cwd=cwd)
    other_path = get_abspath(other_path, cwd=cwd)
    if not os.path.exists(path):
        return True
    if os.path.getmtime(other_path) - 1e-6 >= os.path.getmtime(path):
        # 1e-6 is needed beacuse http://stackoverflow.com/questions/17086426/
        return True
    return False

Response: Returns True if path is non - existant or older than other_path.


Instruction:
Write a Python 3 script to
import a module from a file.

Response:
def import_module_from_file(filename, only_if_newer_than=None):
    """
    Imports (cython generated) shared object file (.so)

    Provide a list of paths in `only_if_newer_than` to check
    timestamps of dependencies. import_ raises an ImportError
    if any is newer.

    Word of warning: Python's caching or the OS caching (unclear to author)
    is horrible for reimporting same path of an .so file. It will
    not detect the new time stamp nor new checksum but will use old
    module.

    Use unique names for this reason.

    Parameters
    ==========
    filename: string
        path to shared object
    only_if_newer_than: iterable of strings
        paths to dependencies of the shared object

    Raises
    ======
    ImportError if any of the files specified in only_if_newer_than are newer
    than the file given by filename.
    """
    import imp
    path, name = os.path.split(filename)
    name, ext = os.path.splitext(name)
    name = name.split('.')[0]
    fobj, filename, data = imp.find_module(name, [path])
    if only_if_newer_than:
        for dep in only_if_newer_than:
            if os.path.getmtime(filename) < os.path.getmtime(dep):
                raise ImportError("{} is newer than {}".format(dep, filename))
    mod = imp.load_module(name, fobj, filename, data)
    return mod


Instruction:
Implement a function in Python 3 to
find the first binary that can be used for the given list of candidates.

Response:
def find_binary_of_command(candidates):
    """
    Calls `find_executable` from distuils for
    provided candidates and returns first hit.
    If no candidate mathces, a RuntimeError is raised
    """
    from distutils.spawn import find_executable
    for c in candidates:
        binary_path = find_executable(c)
        if c and binary_path:
            return c, binary_path
    raise RuntimeError('No binary located for candidates: {}'.format(
        candidates))


Instruction:
Write a Python 3 function that can
check if a Cython source file is a C ++ file.

Response:
def pyx_is_cplus(path):
    """
    Inspect a Cython source file (.pyx) and look for comment line like:

    # distutils: language = c++

    Returns True if such a file is present in the file, else False.
    """
    for line in open(path, 'rt'):
        if line.startswith('#') and '=' in line:
            splitted = line.split('=')
            if len(splitted) != 2:
                continue
            lhs, rhs = splitted
            if lhs.strip().split()[-1].lower() == 'language' and \
               rhs.strip().split()[0].lower() == 'c++':
                    return True
    return False


Instruction:
Write a Python 3 function that can
get value of key in metadata file dict.

Response:
def get_from_metadata_file(cls, dirpath, key):
        """
        Get value of key in metadata file dict.
        """
        fullpath = os.path.join(dirpath, cls.metadata_filename)
        if os.path.exists(fullpath):
            d = pickle.load(open(fullpath, 'rb'))
            return d[key]
        else:
            raise FileNotFoundError(
                "No such file: {0}".format(fullpath))


Instruction:
Write a Python 3 function for
saving key value in metadata file dict.

Response:
def save_to_metadata_file(cls, dirpath, key, value):
        """
        Store `key: value` in metadata file dict.
        """
        fullpath = os.path.join(dirpath, cls.metadata_filename)
        if os.path.exists(fullpath):
            d = pickle.load(open(fullpath, 'rb'))
            d.update({key: value})
            pickle.dump(d, open(fullpath, 'wb'))
        else:
            pickle.dump({key: value}, open(fullpath, 'wb'))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def view_interfaces(category=None):
    if category is not None:
        return sorted(_plugins_interface[category][:])
    else:
        return {k: v[:] for k, v in _plugins_interface.items()}

Response: returns a view of the plugin minimal class attribute interface


Instruction:
Implement a Python 3 function for
returning a view of the loaded plugin names and descriptions for a single plugin category

Response:
def view_plugins(category=None):
    """ return a view of the loaded plugin names and descriptions

    Parameters
    ----------
    category : None or str
        if str, apply for single plugin category

    Examples
    --------

    >>> from pprint import pprint
    >>> pprint(view_plugins())
    {'decoders': {}, 'encoders': {}, 'parsers': {}}

    >>> class DecoderPlugin(object):
    ...     plugin_name = 'example'
    ...     plugin_descript = 'a decoder for dicts containing _example_ key'
    ...     dict_signature = ('_example_',)
    ...
    >>> errors = load_plugin_classes([DecoderPlugin])

    >>> pprint(view_plugins())
    {'decoders': {'example': 'a decoder for dicts containing _example_ key'},
     'encoders': {},
     'parsers': {}}

    >>> view_plugins('decoders')
    {'example': 'a decoder for dicts containing _example_ key'}

    >>> unload_all_plugins()

    """
    if category is not None:
        if category == 'parsers':
            return {
                name: {"descript": klass.plugin_descript,
                       "regex": klass.file_regex}
                for name, klass in _all_plugins[category].items()
            }
        return {
            name: klass.plugin_descript
            for name, klass in _all_plugins[category].items()
        }
    else:
        return {cat: {name: klass.plugin_descript
                      for name, klass in plugins.items()}
                for cat, plugins in _all_plugins.items()}


Instruction:
How would you implement a function in Python 3 that
unloads a single plugin from the _all_plugins list.

Response:
def unload_plugin(name, category=None):
    """ remove single plugin

    Parameters
    ----------
    name : str
        plugin name
    category : str
        plugin category

    Examples
    --------

    >>> from pprint import pprint
    >>> pprint(view_plugins())
    {'decoders': {}, 'encoders': {}, 'parsers': {}}

    >>> class DecoderPlugin(object):
    ...     plugin_name = 'example'
    ...     plugin_descript = 'a decoder for dicts containing _example_ key'
    ...     dict_signature = ('_example_',)
    ...
    >>> errors = load_plugin_classes([DecoderPlugin],category='decoders')

    >>> pprint(view_plugins())
    {'decoders': {'example': 'a decoder for dicts containing _example_ key'},
     'encoders': {},
     'parsers': {}}

    >>> unload_plugin('example','decoders')
    >>> pprint(view_plugins())
    {'decoders': {}, 'encoders': {}, 'parsers': {}}

    """
    if category is not None:
        _all_plugins[category].pop(name)
    else:
        for cat in _all_plugins:
            if name in _all_plugins[cat]:
                _all_plugins[cat].pop(name)


Instruction:
Write a Python 3 function for
loading all plugins from a list of classes.

Response:
def load_plugin_classes(classes, category=None, overwrite=False):
    """ load plugins from class objects

    Parameters
    ----------
    classes: list
        list of classes
    category : None or str
        if str, apply for single plugin category
    overwrite : bool
        if True, allow existing plugins to be overwritten

    Examples
    --------

    >>> from pprint import pprint
    >>> pprint(view_plugins())
    {'decoders': {}, 'encoders': {}, 'parsers': {}}

    >>> class DecoderPlugin(object):
    ...     plugin_name = 'example'
    ...     plugin_descript = 'a decoder for dicts containing _example_ key'
    ...     dict_signature = ('_example_',)
    ...
    >>> errors = load_plugin_classes([DecoderPlugin])

    >>> pprint(view_plugins())
    {'decoders': {'example': 'a decoder for dicts containing _example_ key'},
     'encoders': {},
     'parsers': {}}

    >>> unload_all_plugins()

    """
    load_errors = []
    for klass in classes:
        for pcat, pinterface in _plugins_interface.items():
            if category is not None and not pcat == category:
                continue
            if all([hasattr(klass, attr) for attr in pinterface]):
                if klass.plugin_name in _all_plugins[pcat] and not overwrite:
                    err = '{0} is already set for {1}'.format(
                        klass.plugin_name, pcat)
                    load_errors.append((klass.__name__, '{}'.format(err)))
                    continue
                _all_plugins[pcat][klass.plugin_name] = klass()
            else:
                load_errors.append((
                    klass.__name__,
                    'does not match {} interface: {}'.format(pcat, pinterface)
                ))
    return load_errors


Instruction:
How would you explain what the following Python 3 function does
def plugins_context(classes, category=None):
    original = {cat: list(_all_plugins[cat].keys()) for cat in _all_plugins}
    load_plugin_classes(classes, category, overwrite=True)
    # if errors:
    #     for cat in _all_plugins:
    #         for name, kls in list(_all_plugins[cat].items()):
    #             if name not in original[cat]:
    #                 _all_plugins[cat].pop(name)
    #     raise RuntimeError(
    # "errors occurred while loading plugins: {}".format(errors))
    yield
    for cat in _all_plugins:
        for name, kls in list(_all_plugins[cat].items()):
            if name not in original[cat]:
                _all_plugins[cat].pop(name)

Response: context manager to load plugin classes then unload on exit


Instruction:
Create a Python 3 function for
loading plugins from a directory.

Response:
def load_plugins_dir(path, category=None, overwrite=False):
    """ load plugins from a directory

    Parameters
    ----------
    path : str or path_like
    category : None or str
        if str, apply for single plugin category
    overwrite : bool
        if True, allow existing plugins to be overwritten

    """
    # get potential plugin python files
    if hasattr(path, 'glob'):
        pypaths = path.glob('*.py')
    else:
        pypaths = glob.glob(os.path.join(path, '*.py'))

    load_errors = []
    for pypath in pypaths:
        # use uuid to ensure no conflicts in name space
        mod_name = str(uuid.uuid4())
        try:
            if hasattr(pypath, 'resolve'):
                # Make the path absolute, resolving any symlinks
                pypath = pypath.resolve()

            with warnings.catch_warnings(record=True):
                warnings.filterwarnings("ignore", category=ImportWarning)

                # for MockPaths
                if hasattr(pypath, 'maketemp'):
                    with pypath.maketemp() as f:
                        module = load_source(mod_name, f.name)
                else:
                    module = load_source(mod_name, str(pypath))

        except Exception as err:
            load_errors.append((str(pypath), 'Load Error: {}'.format(err)))
            continue

        # only get classes that are local to the module
        class_members = inspect.getmembers(module, inspect.isclass)
        classes = [klass for klass_name, klass in class_members if
                   klass.__module__ == mod_name]
        load_errors += load_plugin_classes(classes, category, overwrite)

    return load_errors


Instruction:
Make a summary of the following Python 3 code
def load_builtin_plugins(category=None, overwrite=False):
    load_errors = []
    for cat, path in _plugins_builtin.items():
        if cat != category and category is not None:
            continue
        load_errors += load_plugins_dir(path, cat, overwrite=overwrite)
    return load_errors

Response: load plugins from builtin directories


Instruction:
Can you tell what is the following Python 3 function doing
def encode(obj, outtype='json', raise_error=False):
    for encoder in get_plugins('encoders').values():
        if (isinstance(obj, encoder.objclass)
                and hasattr(encoder, 'to_{}'.format(outtype))):
            return getattr(encoder, 'to_{}'.format(outtype))(obj)
            break

    if raise_error:
        raise ValueError(
            "No JSON serializer is available for"
            "{0} (of type {1})".format(obj, type(obj)))
    else:
        return obj

Response: Encodes the object obj to a new type in the specified outtype.


Instruction:
Given the following Python 3 function, write the documentation
def decode(dct, intype='json', raise_error=False):
    for decoder in get_plugins('decoders').values():
        if (set(list(decoder.dict_signature)).issubset(dct.keys())
            and hasattr(decoder, 'from_{}'.format(intype))
                and getattr(decoder, 'allow_other_keys', False)):
            return getattr(decoder, 'from_{}'.format(intype))(dct)
            break
        elif (sorted(list(decoder.dict_signature)) == sorted(dct.keys())
              and hasattr(decoder, 'from_{}'.format(intype))):
            return getattr(decoder, 'from_{}'.format(intype))(dct)
            break

    if raise_error:
        raise ValueError('no suitable plugin found for: {}'.format(dct))
    else:
        return dct

Response: decode dict objects via decoder plugins to new type


Instruction:
Can you implement a function in Python 3 that
tests if parser plugin available for fpath

Response:
def parser_available(fpath):
    """ test if parser plugin available for fpath

    Examples
    --------

    >>> load_builtin_plugins('parsers')
    []
    >>> test_file = StringIO('{"a":[1,2,3.4]}')
    >>> test_file.name = 'test.json'
    >>> parser_available(test_file)
    True
    >>> test_file.name = 'test.other'
    >>> parser_available(test_file)
    False

    >>> unload_all_plugins()

    """
    if isinstance(fpath, basestring):
        fname = fpath
    elif hasattr(fpath, 'open') and hasattr(fpath, 'name'):
        fname = fpath.name
    elif hasattr(fpath, 'readline') and hasattr(fpath, 'name'):
        fname = fpath.name
    else:
        raise ValueError(
            'fpath should be a str or file_like object: {}'.format(fpath))

    for parser in get_plugins('parsers').values():
        if fnmatch(fname, parser.file_regex):
            return True

    return False


Instruction:
Can you create a Python 3 function that
parses file contents via parser plugins to dict like object

Response:
def parse(fpath, **kwargs):
    """ parse file contents, via parser plugins, to dict like object
    NB: the longest file regex will be used from plugins

    Parameters
    ----------
    fpath : file_like
         string, object with 'open' and 'name' attributes, or
         object with 'readline' and 'name' attributes
    kwargs :
        to pass to parser plugin

    Examples
    --------

    >>> load_builtin_plugins('parsers')
    []

    >>> from pprint import pformat

    >>> json_file = StringIO('{"a":[1,2,3.4]}')
    >>> json_file.name = 'test.json'

    >>> dct = parse(json_file)
    >>> print(pformat(dct).replace("u'","'"))
    {'a': [1, 2, 3.4]}

    >>> reset = json_file.seek(0)
    >>> from decimal import Decimal
    >>> dct = parse(json_file, parse_float=Decimal,other=1)
    >>> print(pformat(dct).replace("u'","'"))
    {'a': [1, 2, Decimal('3.4')]}

    >>> class NewParser(object):
    ...     plugin_name = 'example'
    ...     plugin_descript = 'loads test.json files'
    ...     file_regex = 'test.json'
    ...     def read_file(self, file_obj, **kwargs):
    ...         return {'example':1}
    >>> load_plugin_classes([NewParser],'parsers')
    []
    >>> reset = json_file.seek(0)
    >>> parse(json_file)
    {'example': 1}

    >>> unload_all_plugins()

    """
    if isinstance(fpath, basestring):
        fname = fpath
    elif hasattr(fpath, 'open') and hasattr(fpath, 'name'):
        fname = fpath.name
    elif hasattr(fpath, 'readline') and hasattr(fpath, 'name'):
        fname = fpath.name
    else:
        raise ValueError(
            'fpath should be a str or file_like object: {}'.format(fpath))

    parser_dict = {
        plugin.file_regex: plugin
        for plugin in get_plugins('parsers').values()}

    # find longest match first
    for regex in sorted(parser_dict.keys(), key=len, reverse=True):
        parser = parser_dict[regex]
        if fnmatch(fname, regex):
            if isinstance(fpath, basestring):
                with open(fpath, 'r') as file_obj:
                    data = parser.read_file(file_obj, **kwargs)
            elif hasattr(fpath, 'open'):
                with fpath.open('r') as file_obj:
                    data = parser.read_file(file_obj, **kwargs)
            elif hasattr(fpath, 'readline'):
                data = parser.read_file(fpath, **kwargs)
            return data

    raise ValueError('{} does not match any regex'.format(fname))


Instruction:
Make a summary of the following Python 3 code
def colortxt(text, color=None, on_color=None, attrs=None):
    _RESET = '\033[0m'
    __ISON = True
    if __ISON and os.getenv('ANSI_COLORS_DISABLED') is None:
        fmt_str = '\033[%dm%s'
        if color is not None:
            text = fmt_str % (_COLORS[color], text)

        if on_color is not None:
            text = fmt_str % (_HIGHLIGHTS[on_color], text)

        if attrs is not None:
            for attr in attrs:
                text = fmt_str % (_ATTRIBUTES[attr], text)

        text += _RESET
    return text

Response:
Colorize text.

    Available text colors:
        red, green, yellow, blue, magenta, cyan, white.

    Available text highlights:
        on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white.

    Available attributes:
        bold, dark, underline, blink, reverse, concealed.

    Examples
    --------
    >>> txt = colortxt('Hello, World!', 'red', 'on_grey', ['bold'])
    >>> print(txt)
    \x1b[1m\x1b[40m\x1b[31mHello, World!\x1b[0m


Instruction:
Write a Python 3 script for
reading delimited text from zipfile.

Response:
def from_zipfile(cls, path, filename, encoding, dialect, fields, converters):
        """Read delimited text from zipfile."""

        stream = ZipReader(path, filename).readlines(encoding)
        return cls(stream, dialect, fields, converters)


Instruction:
How would you explain what the following Python 3 function does
def from_file(cls, path, encoding, dialect, fields, converters, field_index):

        return cls(open(path, 'r', encoding=encoding), dialect, fields, converters, field_index)

Response: Read delimited text from a text file.


Instruction:
How would you code a function in Python 3 to
read content into byte string.

Response:
def read_bytes(self):
        """Read content into byte string."""

        with ZipFile(self.path, mode='r') as archive:
            return archive.read(self.filename)


Instruction:
Can you implement a function in Python 3 that
reads content into byte str line iterator.

Response:
def readlines_bytes(self):
        """Read content into byte str line iterator."""

        with open_zipfile_archive(self.path, self.filename) as file:
            for line in file:
                yield line.rstrip(b'\r\n')


Instruction:
Write a Python 3 function for
parsing a XML file into a Tree object.

Response:
def xmlparse(self, file):
        """ Parse a XML file

        :param file: Opened File
        :return: Tree
        """
        if self.CACHE_FULL_TEI is True:
            return self.get_or(
                _cache_key("Nautilus", self.name, "File", "Tree", file.name),
                super(ProtoNautilusCtsResolver, self).xmlparse, file
            )
        return super(ProtoNautilusCtsResolver, self).xmlparse(file)


Instruction:
Can you generate the documentation for the following Python 3 function
def get_or(self, cache_key, callback, *args, **kwargs):
        cached = self.cache.get(cache_key)
        if cached is not None:
            return cached
        else:
            try:
                output = callback(*args, **kwargs)
            except MyCapytain.errors.UnknownCollection as E:
                raise UnknownCollection(str(E))
            except Exception as E:
                raise E
            self.cache.set(cache_key, output, self.TIMEOUT)
            return output

Response: Get or set the cache using callback and arguments


Instruction:
How would you implement a function in Python 3 that
reads a text object given an identifier and a path.

Response:
def read(self, identifier, path=None):
        """ Read a text object given an identifier and a path

        :param identifier: Identifier of the text
        :param path: Path of the text files
        :return: Text
        """
        if self.CACHE_FULL_TEI is True:
            o = self.cache.get(_cache_key(self.texts_parsed_cache_key, identifier))
            if o is not None:
                return o
            else:
                with open(path) as f:
                    o = Text(urn=identifier, resource=self.xmlparse(f))
                    self.cache.set(_cache_key(self.texts_parsed_cache_key, identifier), o)
        else:
            with open(path) as f:
                o = Text(urn=identifier, resource=self.xmlparse(f))
        return o


Instruction:
Given the following Python 3 function, write the documentation
def parse(self, resource=None):
        if resource is None:
            resource = self.__resources__

        self.inventory = self.dispatcher.collection

        try:
            self._parse(resource)
        except MyCapytain.errors.UndispatchedTextError as E:
            if self.RAISE_ON_UNDISPATCHED is True:
                raise UndispatchedTextError(E)

        self.inventory = self.dispatcher.collection
        return self.inventory

Response: Parse a list of directories ans


Instruction:
Write a Python 3 script to
retrieve the siblings of a textual node.

Response:
def getReffs(self, textId, level=1, subreference=None):
        """ Retrieve the siblings of a textual node

        :param textId: PrototypeText Identifier
        :type textId: str
        :param level: Depth for retrieval
        :type level: int
        :param subreference: Passage Reference
        :type subreference: str
        :return: List of references
        :rtype: [str]
        """
        return self.get_or(
            self.__cache_key_reffs__(textId, level, subreference),
            super(ProtoNautilusCtsResolver, self).getReffs, textId, level, subreference
        )


Instruction:
How would you explain what the following Python 3 function does
def getTextualNode(self, textId, subreference=None, prevnext=False, metadata=False):
        key = _cache_key("Nautilus", self.name, "Passage", textId, subreference)
        o = self.cache.get(key)
        if o is not None:
            return o
        text, text_metadata = self.__getText__(textId)
        if subreference is not None:
            subreference = Reference(subreference)

        passage = text.getTextualNode(subreference)
        passage.set_metadata_from_collection(text_metadata)
        self.cache.set(key, passage)
        return passage

Response: Retrieve a text node from the API and return it


Instruction:
How would you explain what the following Python 3 function does
def getSiblings(self, textId, subreference):
        key = _cache_key("Nautilus", self.name, "Siblings", textId, subreference)
        o = self.cache.get(key)
        if o is not None:
            return o
        passage = self.getTextualNode(textId, subreference, prevnext=True)
        siblings = passage.siblingsId
        self.cache.set(key, siblings)
        return siblings

Response: Retrieve the siblings of a textual node.


Instruction:
Here you have a function in Python 3, explain what it does
def getMetadata(self, objectId=None, **filters):
        return self.get_or(
            _cache_key("Nautilus", self.name, "GetMetadata", objectId),
            super(ProtoNautilusCtsResolver, self).getMetadata, objectId
        )

Response: Request metadata about a text or a collection of objects.


Instruction:
Given the following Python 3 function, write the documentation
def _dispatch(self, textgroup, directory):
        self.dispatcher.dispatch(textgroup, path=directory)

Response: Dispatches the object in the given directory to the appropriate handler


Instruction:
Implement a function in Python 3 to
ask for user input.

Response:
def ask(question, default_answer=False, default_answer_str="no"):
    """
    Ask for user input.

    This asks a yes/no question with a preset default.
    You can bypass the user-input and fetch the default answer, if
    you set

    Args:
        question: The question to ask on stdout.
        default_answer: The default value to return.
        default_answer_str:
            The default answer string that we present to the user.

    Tests:
        >>> os.putenv("TEST", "yes"); ask("Test?", default_answer=True)
        True
        >>> os.putenv("TEST", "yes"); ask("Test?", default_answer=False)
        False
    """
    response = default_answer

    def should_ignore_tty():
        """
        Check, if we want to ignore an opened tty result.
        """
        ret_to_bool = {"yes": True, "no": False, "true": True, "false": False}
        envs = [os.getenv("CI", default="no"), os.getenv("TEST", default="no")]
        vals = [ret_to_bool[val] for val in envs if val in ret_to_bool]
        return any(vals)

    ignore_stdin_istty = should_ignore_tty()
    has_tty = sys.stdin.isatty() and not ignore_stdin_istty
    if has_tty:
        response = query_yes_no(question, default_answer_str)
    else:
        LOG.debug("NoTTY: %s -> %s", question, response)

    return response


Instruction:
Make a summary of the following Python 3 code
def load_csv(path, delimiter=','):
    try:
        with open(path, 'rb') as csvfile:
            reader = DictReader(csvfile, delimiter=delimiter)
            for row in reader:
                yield row
    except (OSError, IOError):
        raise ClientException("File not found: {}".format(path))

Response: Load CSV file from path and yield CSV rows


Instruction:
Can you write a function in Python 3 where it
loads a JSON file from path

Response:
def load_json(path):
    """
    Load JSON file from path
    :param path: file path
    :return: A Python object (e.g. a dict)
    """
    try:
        with open(path, 'r') as json_file:
            return json.load(json_file)
    except (OSError, IOError):
        raise ClientException("File not found: {}".format(path))


Instruction:
Make a summary of the following Python 3 code
def partition_payload(data, key, thresh):
    data = data[key]
    for i in range(0, len(data), thresh):
        yield {key: data[i:i + thresh]}

Response: Yield partitions of a payload by a threshold


Instruction:
Can you tell what is the following Python 3 function doing
def search_auth_file(filename='dish.json'):
    if 'DHIS_HOME' in os.environ:
        return os.path.join(os.environ['DHIS_HOME'], filename)
    else:
        home_path = os.path.expanduser(os.path.join('~'))
        for root, dirs, files in os.walk(home_path):
            if filename in files:
                return os.path.join(root, filename)
    raise ClientException("'{}' not found - searched in $DHIS_HOME and in home folder".format(filename))

Response: Search filename in DHIS_HOME or user s home folder and return full path of the file


Instruction:
Here you have a function in Python 3, explain what it does
def version_to_int(value):
    # remove '-SNAPSHOT'
    value = value.replace('-SNAPSHOT', '')
    # remove '-RCx'
    if '-RC' in value:
        value = value.split('-RC', 1)[0]
    try:
        return int(value.split('.')[1])
    except (ValueError, IndexError):
        return

Response: Convert version info to integer


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_uid():
    # first must be a letter
    first = random.choice(string.ascii_letters)
    # rest must be letters or numbers
    rest = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(10))
    return first + rest

Response:
Create DHIS2 UID matching to Regex
    ^[A-Za-z][A-Za-z0-9]{10}$
    :return: UID string


Instruction:
Make a summary of the following Python 3 code
def is_valid_uid(uid):
    pattern = r'^[A-Za-z][A-Za-z0-9]{10}$'
    if not isinstance(uid, string_types):
        return False
    return bool(re.compile(pattern).match(uid))

Response: Check if the given UID is a valid DHIS2 UID.


Instruction:
Given the following Python 3 function, write the documentation
def pretty_json(obj):
    if isinstance(obj, string_types):
        try:
            obj = json.loads(obj)
        except ValueError:
            raise ClientException("`obj` is not a json string")
    json_str = json.dumps(obj, sort_keys=True, indent=2)
    print(highlight(json_str, JsonLexer(), TerminalFormatter()))

Response: Print JSON with indentation and colours


Instruction:
How would you explain what the following Python 3 function does
def commands(self):
        for child in self._children:
            with open(osp.join(child, YAML_REPORT_FILE)) as istr:
                command = yaml.safe_load(istr)['command']
                yield ' '.join(map(six.moves.shlex_quote, command))

Response: Get all commands of the benchmark category


Instruction:
Write a Python 3 script to
set current process environment according to execution environment and modules

Response:
def _module_env(self, execution):
        """Set current process environment according
        to execution `environment` and `modules`
        """
        env = copy.copy(os.environ)
        try:
            for mod in execution.get('modules') or []:
                Module.load(mod)
            os.environ.update(execution.get('environment') or {})
            yield
        finally:
            os.environ = env


Instruction:
Here you have a function in Python 3, explain what it does
def gather_metrics(self, runs):
        for run_dirs in runs.values():
            with open(JSON_METRICS_FILE, 'w') as ostr:
                ostr.write('[\n')
                for i in range(len(run_dirs)):
                    with open(osp.join(run_dirs[i], YAML_REPORT_FILE)) as istr:
                        data = yaml.safe_load(istr)
                        data.pop('category', None)
                        data.pop('command', None)
                        data['id'] = run_dirs[i]
                        json.dump(data, ostr, indent=2)
                    if i != len(run_dirs) - 1:
                        ostr.write(',')
                    ostr.write('\n')
                ostr.write(']\n')

Response:
Write a JSON file with the result of every runs
xid


Instruction:
How would you explain what the following Python 3 function does
def _check_metrics(cls, schema, metrics):
        for name, value in metrics.items():
            metric = schema.get(name)
            if not metric:
                message = "Unexpected metric '{}' returned".format(name)
                raise Exception(message)
            cls._check_metric(schema, metric, name, value)

Response: Ensure that returned metrics are properly exposed.


Instruction:
Given the following Python 3 function, write the documentation
def add_inputs_to_object(obj, values):
    for item in obj.inputs:
        if hasattr(obj, item):
            # print(item)
            setattr(obj, item, values[item])

Response: A generic function to load object parameters based on a dictionary list.


Instruction:
Can you write a function in Python 3 where it
loads sample data for the SoilStructureSystem object.

Response:
def load_sample_data(sss):
    """
    Sample data for the SoilStructureSystem object
    :param sss:
    :return:
    """

    load_soil_sample_data(sss.sp)  # soil
    load_foundation_sample_data(sss.fd)  # foundation
    load_structure_sample_data(sss.bd)  # structure
    load_hazard_sample_data(sss.hz)


Instruction:
Explain what the following Python 3 code does
def load_soil_sample_data(sp):
    # soil
    sp.g_mod = 60.0e6  # [Pa]
    sp.phi = 30  # [degrees]
    sp.relative_density = .40  # [decimal]
    sp.gwl = 2.  # [m], ground water level
    sp.unit_dry_weight = 17000  # [N/m3]
    sp.unit_sat_weight = 18000  # [N/m3]
    sp.unit_weight_water = 9800  # [N/m3]
    sp.cohesion = 10.0  # [Pa]
    sp.poissons_ratio = 0.22
    sp.e_min = 0.55
    sp.e_max = 0.95
    sp.e_critical0 = 0.79  # Jin et al. 2015
    sp.p_critical0 = 0.7  # Jin et al. 2015
    sp.lamb_crl = 0.015

Response: Load sample data for the Soil object.


Instruction:
Write a Python 3 function for
loading the sample data for the Foundation object

Response:
def load_foundation_sample_data(fd):
    """
    Sample data for the Foundation object
    :param fd: Foundation Object
    :return:
    """
    # foundation
    fd.width = 16.0  # m
    fd.length = 18.0  # m
    fd.depth = 0.0  # m
    fd.mass = 0.0


Instruction:
Write a Python 3 script for
loading the sample data for the structure object

Response:
def load_structure_sample_data(st):
    """
    Sample data for the Structure object
    :param st: Structure Object
    :return:
    """
    # structure
    st.h_eff = 9.0  # m
    st.mass_eff = 120e3  # kg
    st.t_eff = 1.0  # s
    st.mass_ratio = 1.0


Instruction:
Implement a function in Python 3 to
load sample data for the Hazard object.

Response:
def load_hazard_sample_data(hz):
    """
    Sample data for the Hazard object
    :param hz: Hazard Object
    :return:
    """
    # hazard
    hz.z_factor = 0.3  # Hazard factor
    hz.r_factor = 1.0  # Return period factor
    hz.n_factor = 1.0  # Near-fault factor
    hz.magnitude = 7.5  # Magnitude of earthquake
    hz.corner_period = 4.0  # s
    hz.corner_acc_factor = 0.55
    return hz


Instruction:
Make a summary of the following Python 3 code
def load_building_sample_data(bd):
    number_of_storeys = 6
    interstorey_height = 3.4  # m
    masses = 40.0e3  # kg

    bd.interstorey_heights = interstorey_height * np.ones(number_of_storeys)
    bd.floor_length = 18.0  # m
    bd.floor_width = 16.0  # m
    bd.storey_masses = masses * np.ones(number_of_storeys)

Response: Load the sample data for the Building object.


Instruction:
Given the following Python 3 function, write the documentation
def load_frame_building_sample_data():
    number_of_storeys = 6
    interstorey_height = 3.4  # m
    masses = 40.0e3  # kg
    n_bays = 3

    fb = models.BuildingFrame(number_of_storeys, n_bays)
    fb.interstorey_heights = interstorey_height * np.ones(number_of_storeys)
    fb.floor_length = 18.0  # m
    fb.floor_width = 16.0  # m
    fb.storey_masses = masses * np.ones(number_of_storeys)  # kg

    fb.bay_lengths = [6., 6.0, 6.0]
    fb.set_beam_prop("depth", [0.5, 0.5, 0.5], repeat="up")
    fb.set_beam_prop("width", [0.4, 0.4, 0.4], repeat="up")
    fb.set_column_prop("width", [0.5, 0.5, 0.5, 0.5], repeat="up")
    fb.set_column_prop("depth", [0.5, 0.5, 0.5, 0.5], repeat="up")
    fb.n_seismic_frames = 3
    fb.n_gravity_frames = 0
    return fb

Response: Load the sample data for the BuildingFrame object.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_random_fp_raw():
    '''
        生成随机的原始指纹列表
    '''
    fp_list = []
    fp_list.append(get_random_ua())  # ua
    fp_list.append('zh-CN')  # language
    fp_list.append('24')  # color depth
    fp_list.append(__get_random_screen_resolution())
    fp_list.append('-480')  # time zone offsite
    fp_list.append('true')  # session storage
    fp_list.append('true')  # local storage
    fp_list.append('true')  # indexed db
    fp_list.append('')  # add behavior
    fp_list.append('function')  # open database
    fp_list.append('')  # cpu class
    fp_list.append('MacIntel')  # platform
    fp_list.append('')  # do not track
    fp_list.append(
        'Widevine Content Decryption Module::Enables Widevine \
        licenses for playback of HTML audio/video content. \
        (version: 1.4.8.962)::application/x-ppapi-widevine-cdm~;'
    )  # plugin string

    return fp_list

Response: Generate a random FPGA - style list of random FPGA - style values.


Instruction:
Write a Python 3 function that can
get the raw version of the FPGA file.

Response:
def get_fp_raw():
    '''
        生成fp_raw_str
    '''
    fp_file_path = os.path.expanduser('~/.xunleipy_fp')
    fp_list = []
    try:
        with open(fp_file_path, 'r') as fp_file:
            fp_str = fp_file.readline()
            if len(fp_str) > 0:
                fp_list = fp_str.split('###')
    except IOError:
        pass

    if len(fp_list) < 14:
        fp_list = _get_random_fp_raw()
        fp_str = '###'.join(fp_list)
        with open(fp_file_path, 'w') as fp_file:
            fp_file.write(fp_str)

    source = fp_str.strip()
    if six.PY3:
        source = source.encode('utf-8')

    fp_raw = base64.b64encode(source)
    return fp_raw


Instruction:
How would you implement a function in Python 3 that
defines functions composition like f g

Response:
def compose(*functions):
    """Define functions composition like f ∘ g ∘ h
    :return: callable object that will perform
    function composition of callables given in argument.
    """

    def _compose2(f, g):  # pylint: disable=invalid-name
        return lambda x: f(g(x))

    return functools.reduce(_compose2, functions, lambda x: x)


Instruction:
Explain what the following Python 3 code does
def chunks(iterator, size):
    for item in iterator:
        yield [item] + list(islice(iterator, size - 1))

Response: Split an iterator into chunks with size elements each.


Instruction:
Can you implement a function in Python 3 that
yields the sub - group element dictionary that match a regex pattern.

Response:
def match_subgroup(sequence, pattern):
    """Yield the sub-group element dictionary that match a regex pattern."""

    for element in sequence:
        match = re.match(pattern, element)

        if match:
            yield match.groupdict()


Instruction:
Explain what the following Python 3 code does
def add_regex_start_end(pattern_function):

    @wraps(pattern_function)
    def func_wrapper(*args, **kwargs):
        return r'^{}$'.format(pattern_function(*args, **kwargs))
    return func_wrapper

Response: Decorator for adding regex start and end characters.


Instruction:
How would you implement a function in Python 3 that
converts a foundation stress to an equivalent mass.

Response:
def convert_stress_to_mass(q, width, length, gravity):
    """
    Converts a foundation stress to an equivalent mass.

    :param q: applied stress [Pa]
    :param width: foundation width [m]
    :param length: foundation length [m]
    :param gravity: applied gravitational acceleration [m/s2]
    :return:
    """
    mass = q * width * length / gravity
    return mass


Instruction:
Explain what the following Python 3 code does
def add_to_obj(obj, dictionary, objs=None, exceptions=None, verbose=0):
    if exceptions is None:
        exceptions = []
    for item in dictionary:
        if item in exceptions:
            continue
        if dictionary[item] is not None:
            if verbose:
                print("process: ", item, dictionary[item])
            key, value = get_key_value(dictionary[item], objs, key=item)
            if verbose:
                print("assign: ", key, value)
            try:
                setattr(obj, key, value)
            except AttributeError:
                raise AttributeError("Can't set {0}={1} on object: {2}".format(key, value, obj))

Response: Adds the key - value pairs to an object.


Instruction:
Explain what the following Python 3 code does
def create_quantiles(items: Sequence, lower_bound, upper_bound):

    interval = (upper_bound - lower_bound) / len(items)

    quantiles = ((g, (x - interval, x)) for g, x in
                 zip(items, accumulate(repeat(interval, len(items)))))

    return quantiles

Response: Create quantile start and end boundaries.


Instruction:
Here you have a function in Python 3, explain what it does
def tupleize(element, ignore_types=(str, bytes)):
    if hasattr(element, '__iter__') and not isinstance(element, ignore_types):
        return element
    else:
        return tuple((element,))

Response: Cast a single element to a tuple.


Instruction:
Write a Python 3 script for
creating dictionaries mapping fields to record data.

Response:
def dictionize(fields: Sequence, records: Sequence) -> Generator:
    """Create dictionaries mapping fields to record data."""

    return (dict(zip(fields, rec)) for rec in records)


Instruction:
How would you code a function in Python 3 to
transform dictionary to unpack values to map to respective key.

Response:
def flip_iterable_dict(d: dict) -> dict:
    """Transform dictionary to unpack values to map to respective key."""

    value_keys = disjoint_union((cartesian_product((v, k))
                                 for k, v in d.items()))
    return dict(value_keys)


Instruction:
Here you have a function in Python 3, explain what it does
def get_signed_area(self):
        accum = 0.0
        for i in range(len(self.verts)):
            j = (i + 1) % len(self.verts)
            accum += (
                self.verts[j][0] * self.verts[i][1] -
                self.verts[i][0] * self.verts[j][1])
        return accum / 2

Response: Return the area of a simple polygon.


Instruction:
Can you tell what is the following Python 3 function doing
def call_next(self, *args, **kwargs) -> t.List[run.RunInfo]:
        all_results = []
        for ext in self.next_extensions:
            LOG.debug("  %s ", ext)
            results = ext(*args, **kwargs)
            LOG.debug("  %s => %s", ext, results)
            if results is None:
                LOG.warning("No result from: %s", ext)
                continue
            result_list = []
            if isinstance(results, c.Iterable):
                result_list.extend(results)
            else:
                result_list.append(results)
            all_results.extend(result_list)

        return all_results

Response: Calls all child extensions with the given arguments and returns the results of all child extensions.


Instruction:
Create a Python 3 function for
printing a structural view of the registered extensions.

Response:
def print(self, indent=0):
        """Print a structural view of the registered extensions."""
        LOG.info("%s:: %s", indent * " ", self.__class__)
        for ext in self.next_extensions:
            ext.print(indent=indent + 2)


Instruction:
How would you explain what the following Python 3 function does
def print_results(matches):
    output = ""
    for n, match in enumerate(matches):
        matched_text = match[0][:40]+"..." if len(match[0]) > 40 else match[0]
        search_result = match[1]
        if search_result.uri==surf.ns.EFRBROO['F10_Person']:
            label = unicode(search_result)[:40]+"..." if len(unicode(search_result)) > 40 else unicode(search_result)
            print("\n{:5}) {:50} {:40} (Matched: \"{}\")\n".format(n+1, label, search_result.get_urn(), matched_text))
        elif search_result.uri==surf.ns.EFRBROO['F1_Work']:
            label = "{}, {}".format(search_result.author, search_result)
            label = label[:40]+"..." if len(label) > 40 else label
            print("\n{:5}) {:50} {:40} (Matched: \"{}\")\n".format(n+1, label, search_result.get_urn(), matched_text))

Response: Prints the results of the search.


Instruction:
Write a Python 3 script for
printing the result of a resource.

Response:
def show_result(resource, verbose=False):
    """
    TODO
    """
    if resource.uri == surf.ns.EFRBROO['F10_Person']:
        print("\n{} ({})\n".format(unicode(resource), resource.get_urn()))
        works = resource.get_works()
        print("Works by {} ({}):\n".format(resource, len(works)))
        [show_result(work) for work in works]
        print("\n")
    elif resource.uri == surf.ns.EFRBROO['F1_Work']:
        if verbose:
            print("\n{} ({})".format(unicode(resource), resource.get_urn()))
            print("\nTitles:")
            print("\n".join(["{:20} ({})".format(title, lang) for lang, title in resource.get_titles()]))
            if len(resource.get_abbreviations()) > 0:
                print("\nAbbreviations: {}\n".format(", ".join(["{}".format(abbr) for abbr in resource.get_abbreviations()])))
        else:
            print("{:50} {:40}".format(unicode(resource), resource.get_urn()))


Instruction:
Write a Python 3 function for
defining the CLI inteface / commands.

Response:
def main():
    """Define the CLI inteface/commands."""
    arguments = docopt(__doc__)
    cfg_filename = pkg_resources.resource_filename(
        'knowledge_base',
        'config/virtuoso.ini'
    )
    kb = KnowledgeBase(cfg_filename)

    # the user has issued a `find` command
    if arguments["find"]:
        search_string = arguments["<search_string>"]
        try:
            urn = CTS_URN(search_string)
            match = kb.get_resource_by_urn(str(urn))
            show_result(match, verbose=True)
            return
        except BadCtsUrnSyntax as e:
            pass
        except IndexError as e:
            raise e
            print("\nNo records with this CTS URN!\n")
            return
        try:
            matches = kb.search(search_string)
            print("\nSearching for \"%s\" yielded %s results" % (
                search_string,
                len(matches)
            ))
            print_results(matches)
            return
        except SparqlReaderException as e:
            print("\nWildcard word needs at least 4 leading characters")
    # the user has issued an `add` command
    elif arguments["add"]:
        input_urn = arguments["--to"]

        # first let's check if it's a valid URN
        try:
            urn = CTS_URN(input_urn)
        except Exception as e:
            print("The provided URN ({}) is invalid!".format(input_urn))
            return

        try:
            resource = kb.get_resource_by_urn(urn)
            assert resource is not None
        except ResourceNotFound:
            print("The KB does not contain a resource identified by {}".format(
                urn
            ))
            return

        print(arguments)
        #if arguments[""]
        pass


Instruction:
Write a Python 3 function that can
check if a container exists and is unpacked.

Response:
def is_valid(container, path):
    """
    Checks if a container exists and is unpacked.

    Args:
        path: The location where the container is expected.

    Returns:
        True if the container is valid, False if the container needs to
        unpacked or if the path does not exist yet.
    """
    try:
        tmp_hash_path = container.filename + ".hash"
        with open(tmp_hash_path, 'r') as tmp_file:
            tmp_hash = tmp_file.readline()
    except IOError:
        LOG.info("No .hash-file in the tmp-directory.")

    container_hash_path = local.path(path) / "gentoo.tar.bz2.hash"
    if container_hash_path.exists():
        with open(container_hash_path, 'r') as hash_file:
            container_hash = hash_file.readline()
            return container_hash == tmp_hash
    return False


Instruction:
Given the following Python 3 function, write the documentation
def unpack(container, path):
    from benchbuild.utils.run import run
    from benchbuild.utils.uchroot import no_args

    path = local.path(path)
    c_filename = local.path(container.filename)
    name = c_filename.basename

    if not path.exists():
        path.mkdir()

    with local.cwd(path):
        Wget(container.remote, name)

        uchroot = no_args()
        uchroot = uchroot["-E", "-A", "-C", "-r", "/", "-w",
                          os.path.abspath("."), "--"]

        # Check, if we need erlent support for this archive.
        has_erlent = bash[
            "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
                name)]
        has_erlent = (has_erlent & TF)

        untar = local["/bin/tar"]["xf", "./" + name]
        if not has_erlent:
            untar = uchroot[untar]

        run(untar["--exclude=dev/*"])
        if not os.path.samefile(name, container.filename):
            rm(name)
        else:
            LOG.warning("File contents do not match: %s != %s", name,
                        container.filename)
        cp(container.filename + ".hash", path)

Response: Unpack a container usable by uchroot.


Instruction:
Make a summary of the following Python 3 code
def local(self):
        assert self.name in CFG["container"]["images"].value
        tmp_dir = local.path(str(CFG["tmp_dir"]))
        target_dir = tmp_dir / self.name

        if not target_dir.exists() or not is_valid(self, target_dir):
            unpack(self, target_dir)

        return target_dir

Response: Find the current location of a container.


Instruction:
Write a Python 3 script for
getting the latest src_uri for a stage 3 tarball.

Response:
def src_file(self):
        """
        Get the latest src_uri for a stage 3 tarball.

        Returns (str):
            Latest src_uri from gentoo's distfiles mirror.
        """
        try:
            src_uri = (curl[Gentoo._LATEST_TXT] | tail["-n", "+3"]
                       | cut["-f1", "-d "])().strip()
        except ProcessExecutionError as proc_ex:
            src_uri = "NOT-FOUND"
            LOG.error("Could not determine latest stage3 src uri: %s",
                      str(proc_ex))
        return src_uri


Instruction:
Can you generate the documentation for the following Python 3 function
def version(self):
        try:
            _version = (curl[Gentoo._LATEST_TXT] | \
                    awk['NR==2{print}'] | \
                    cut["-f2", "-d="])().strip()
            _version = datetime.utcfromtimestamp(int(_version))\
                .strftime("%Y-%m-%d")
        except ProcessExecutionError as proc_ex:
            _version = "unknown"
            LOG.error("Could not determine timestamp: %s",
                      str(proc_ex))
        return _version

Response: Return the build date of the gentoo container.


Instruction:
Write a Python 3 function that can
determine the new start and end indices of the time series.

Response:
def time_indices(npts, dt, start, end, index):
    """
    Determine the new start and end indices of the time series.

    :param npts: Number of points in original time series
    :param dt: Time step of original time series
    :param start: int or float, optional, New start point
    :param end: int or float, optional, New end point
    :param index: bool, optional, if False then start and end are considered values in time.
    :return: tuple, start index, end index
    """
    if index is False:  # Convert time values into indices
        if end != -1:
            e_index = int(end / dt) + 1
        else:
            e_index = end
        s_index = int(start / dt)
    else:
        s_index = start
        e_index = end
    if e_index > npts:
        raise exceptions.ModelWarning("Cut point is greater than time series length")
    return s_index, e_index


Instruction:
Write a Python 3 function that can
cut the time series to reduce its length.

Response:
def cut(self, start=0, end=-1, index=False):
        """
        The method cuts the time series to reduce its length.
        :param start: int or float, optional, New start point
        :param end: int or float, optional, New end point
        :param index: bool, optional, if False then start and end are considered values in time.
        """
        s_index, e_index = time_indices(self.npts, self.dt, start, end, index)
        self._values = np.array(self.values[s_index:e_index])


Instruction:
Explain what the following Python 3 code does
def finished(cls, jobid):
        output = subprocess.check_output(
            [SACCT, '-n', '-X', '-o', "end", '-j', str(jobid)]
        )
        end = output.strip().decode()
        return end not in {'Unknown', ''}

Response: Check whether a SLURM job is finished or not


Instruction:
Write a Python 3 function for
beginning a run_group in the database.

Response:
def begin_run_group(project):
    """
    Begin a run_group in the database.

    A run_group groups a set of runs for a given project. This models a series
    of runs that form a complete binary runtime test.

    Args:
        project: The project we begin a new run_group for.

    Returns:
        ``(group, session)`` where group is the created group in the
        database and session is the database session this group lives in.
    """
    from benchbuild.utils.db import create_run_group
    from datetime import datetime

    group, session = create_run_group(project)
    group.begin = datetime.now()
    group.status = 'running'

    session.commit()
    return group, session


Instruction:
Can you implement a function in Python 3 that
ends the run_group successfully.

Response:
def end_run_group(group, session):
    """
    End the run_group successfully.

    Args:
        group: The run_group we want to complete.
        session: The database transaction we will finish.
    """
    from datetime import datetime

    group.end = datetime.now()
    group.status = 'completed'
    session.commit()


Instruction:
Write a Python 3 script to
end the run_group unsuccessfully.

Response:
def fail_run_group(group, session):
    """
    End the run_group unsuccessfully.

    Args:
        group: The run_group we want to complete.
        session: The database transaction we will finish.
    """
    from datetime import datetime

    group.end = datetime.now()
    group.status = 'failed'
    session.commit()


Instruction:
Can you generate the documentation for the following Python 3 function
def exit_code_from_run_infos(run_infos: t.List[RunInfo]) -> int:
    assert run_infos is not None

    if not hasattr(run_infos, "__iter__"):
        return run_infos.retcode

    rcs = [ri.retcode for ri in run_infos]
    max_rc = max(rcs)
    min_rc = min(rcs)
    if max_rc == 0:
        return min_rc
    return max_rc

Response: Generate a single exit code from a list of RunInfo objects.


Instruction:
Implement a function in Python 3 to
guard the execution of the given command. The given command (`cmd`) will be executed inside a database context. As soon as you leave the context we will commit the transaction. Any necessary modifications to the database can be identified inside the context with the RunInfo object. Args: cmd: The command we guard. project: The project we track for. experiment: The experiment we track for. Yields: RunInfo: A context object that carries the necessary database transaction.

Response:
def track_execution(cmd, project, experiment, **kwargs):
    """Guard the execution of the given command.

    The given command (`cmd`) will be executed inside a database context.
    As soon as you leave the context we will commit the transaction.
    Any necessary modifications to the database can be identified inside
    the context with the RunInfo object.

    Args:
        cmd: The command we guard.
        project: The project we track for.
        experiment: The experiment we track for.

    Yields:
        RunInfo: A context object that carries the necessary
            database transaction.
    """

    runner = RunInfo(cmd=cmd, project=project, experiment=experiment, **kwargs)
    yield runner
    runner.commit()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def with_env_recursive(cmd, **envvars):
    from plumbum.commands.base import BoundCommand, BoundEnvCommand
    if isinstance(cmd, BoundCommand):
        cmd.cmd = with_env_recursive(cmd.cmd, **envvars)
    elif isinstance(cmd, BoundEnvCommand):
        cmd.envvars.update(envvars)
        cmd.cmd = with_env_recursive(cmd.cmd, **envvars)
    return cmd

Response: Recursively updates the environment of the specified command and all its subcommands.


Instruction:
How would you explain what the following Python 3 function does
def in_builddir(sub='.'):
    from functools import wraps

    def wrap_in_builddir(func):
        """Wrap the function for the new build directory."""

        @wraps(func)
        def wrap_in_builddir_func(self, *args, **kwargs):
            """The actual function inside the wrapper for the new builddir."""
            p = local.path(self.builddir) / sub
            if not p.exists():
                LOG.error("%s does not exist.", p)

            if p == local.cwd:
                LOG.debug("CWD already is %s", p)
                return func(self, *args, *kwargs)
            with local.cwd(p):
                return func(self, *args, **kwargs)

        return wrap_in_builddir_func

    return wrap_in_builddir

Response: Decorator for creating a new build directory in a project phase with a local working directory change.


Instruction:
How would you explain what the following Python 3 function does
def store_config(func):
    from functools import wraps

    @wraps(func)
    def wrap_store_config(self, *args, **kwargs):
        """Wrapper that contains the actual storage call for the config."""
        CFG.store(local.path(self.builddir) / ".benchbuild.yml")
        return func(self, *args, **kwargs)

    return wrap_store_config

Response: Decorator for storing the configuration in the project s builddir.


Instruction:
Given the following Python 3 function, write the documentation
def clean_directories(builddir, in_dir=True, out_dir=True):
    container_in = local.path(builddir) / "container-in"
    container_out = local.path(builddir) / "container-out"

    if in_dir and container_in.exists():
        if ui.ask("Should I delete '{0}'?".format(container_in)):
            container_in.delete()
    if out_dir and container_out.exists():
        if ui.ask("Should I delete '{0}'?".format(container_out)):
            container_out.delete()

Response: Remove the in and out of the container if confirmed by the user.


Instruction:
Can you create a Python 3 function that
creates the in and out directories of the container.

Response:
def setup_directories(builddir):
    """Create the in and out directories of the container."""
    build_dir = local.path(builddir)
    in_dir = build_dir / "container-in"
    out_dir = build_dir / "container-out"

    if not in_dir.exists():
        in_dir.mkdir()
    if not out_dir.exists():
        out_dir.mkdir()


Instruction:
Here you have a function in Python 3, explain what it does
def setup_container(builddir, _container):
    build_dir = local.path(builddir)
    in_dir = build_dir / "container-in"
    container_path = local.path(_container)
    with local.cwd(builddir):
        container_bin = container_path.basename
        container_in = in_dir / container_bin
        download.Copy(_container, container_in)
        uchrt = uchroot.no_args()

        with local.cwd("container-in"):
            uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-r", "/",
                          "-w",
                          os.path.abspath("."), "--"]

        # Check, if we need erlent support for this archive.
        has_erlent = bash[
            "-c", "tar --list -f './{0}' | grep --silent '.erlent'".format(
                container_in)]
        has_erlent = (has_erlent & TF)

        # Unpack input container to: container-in
        if not has_erlent:
            cmd = local["/bin/tar"]["xf"]
            cmd = uchrt[cmd[container_bin]]
        else:
            cmd = tar["xf"]
            cmd = cmd[container_in]

        with local.cwd("container-in"):
            cmd("--exclude=dev/*")
        rm(container_in)
    return in_dir

Response: Prepare the container and returns the path where it can be found.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run_in_container(command, container_dir):
    container_p = local.path(container_dir)
    with local.cwd(container_p):
        uchrt = uchroot.with_mounts()
        uchrt = uchrt["-E", "-A", "-u", "0", "-g", "0", "-C", "-w", "/", "-r",
                      container_p]
        uchrt = uchrt["--"]

        cmd_path = container_p / command[0].lstrip('/')
        if not cmd_path.exists():
            LOG.error("The command does not exist inside the container! %s",
                      cmd_path)
            return

        cmd = uchrt[command]
        return cmd & FG

Response: Runs a given command inside a container.


Instruction:
Implement a Python 3 function for
packing a container image into a. tar. bz2 archive.

Response:
def pack_container(in_container, out_file):
    """
    Pack a container image into a .tar.bz2 archive.

    Args:
        in_container (str): Path string to the container image.
        out_file (str): Output file name.
    """
    container_filename = local.path(out_file).basename
    out_container = local.cwd / "container-out" / container_filename
    out_dir = out_container.dirname

    # Pack the results to: container-out
    with local.cwd(in_container):
        tar("cjf", out_container, ".")
    c_hash = download.update_hash(out_container)
    if out_dir.exists():
        mkdir("-p", out_dir)
    mv(out_container, out_file)
    mv(out_container + ".hash", out_file + ".hash")

    new_container = {"path": out_file, "hash": str(c_hash)}
    CFG["container"]["known"] += new_container


Instruction:
Make a summary of the following Python 3 code
def setup_bash_in_container(builddir, _container, outfile, shell):
    with local.cwd(builddir):
        # Switch to bash inside uchroot
        print("Entering bash inside User-Chroot. Prepare your image and "
              "type 'exit' when you are done. If bash exits with a non-zero"
              "exit code, no new container will be stored.")
        store_new_container = True
        try:
            run_in_container(shell, _container)
        except ProcessExecutionError:
            store_new_container = False

        if store_new_container:
            print("Packing new container image.")
            pack_container(_container, outfile)
            config_path = str(CFG["config_file"])
            CFG.store(config_path)
            print("Storing config in {0}".format(os.path.abspath(config_path)))

Response: Setup a bash environment inside a container.


Instruction:
How would you code a function in Python 3 to
save the input for the container in the configurations.

Response:
def set_input_container(_container, cfg):
    """Save the input for the container in the configurations."""
    if not _container:
        return False
    if _container.exists():
        cfg["container"]["input"] = str(_container)
        return True
    return False


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self, context):
        # Don't do something when running non-interactive.
        if not sys.stdout.isatty():
            return

        with local.cwd(context.in_container):
            from benchbuild.projects.gentoo import gentoo
            gentoo.setup_networking()
            gentoo.configure_portage()

            sed_in_chroot = uchroot.uchroot()["/bin/sed"]
            emerge_in_chroot = uchroot.uchroot()["/usr/bin/emerge"]
            has_pkg = uchroot.uchroot()["/usr/bin/qlist", "-I"]

            run.run(sed_in_chroot["-i", '/CC=/d', "/etc/portage/make.conf"])
            run.run(sed_in_chroot["-i", '/CXX=/d', "/etc/portage/make.conf"])

            want_sync = bool(CFG["container"]["strategy"]["polyjit"]["sync"])
            want_upgrade = bool(
                CFG["container"]["strategy"]["polyjit"]["upgrade"])

            packages = \
                CFG["container"]["strategy"]["polyjit"]["packages"].value
            with local.env(MAKEOPTS="-j{0}".format(int(CFG["jobs"]))):
                if want_sync:
                    LOG.debug("Synchronizing portage.")
                    run.run(emerge_in_chroot["--sync"])
                if want_upgrade:
                    LOG.debug("Upgrading world.")
                    run.run(emerge_in_chroot["--autounmask-only=y", "-uUDN",
                                             "--with-bdeps=y", "@world"])
                for pkg in packages:
                    if has_pkg[pkg["name"]] & TF:
                        continue
                    env = pkg["env"]
                    with local.env(**env):
                        run.run(emerge_in_chroot[pkg["name"]])

            gentoo.setup_benchbuild()

        print("Packing new container image.")
        with local.cwd(context.builddir):
            pack_container(context.in_container, context.out_container)

Response: Setup a gentoo container suitable for PolyJIT.


Instruction:
Can you tell what is the following Python 3 function doing
def input_file(self, _container):
        p = local.path(_container)
        if set_input_container(p, CFG):
            return

        p = find_hash(CFG["container"]["known"].value, container)
        if set_input_container(p, CFG):
            return

        raise ValueError("The path '{0}' does not exist.".format(p))

Response: Find the input path of a uchroot container.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def output_file(self, _container):
        p = local.path(_container)
        if p.exists():
            if not ui.ask("Path '{0}' already exists."
                          " Overwrite?".format(p)):
                sys.exit(0)
        CFG["container"]["output"] = str(p)

Response: Find and writes the output path of a chroot container.


Instruction:
Here you have a function in Python 3, explain what it does
def discretize_soil_profile(sp, incs=None, target=1.0):

    if incs is None:
        incs = np.ones(sp.n_layers) * target
    dd = {}
    dd["thickness"] = []
    dd["unit_mass"] = []
    dd["shear_vel"] = []
    cum_thickness = 0
    for i in range(sp.n_layers):
        sl = sp.layer(i + 1)
        thickness = sp.layer_height(i + 1)
        n_slices = max(int(thickness / incs[i]), 1)
        slice_thickness = float(thickness) / n_slices
        for j in range(n_slices):
            cum_thickness += slice_thickness
            if cum_thickness >= sp.gwl:
                rho = sl.unit_sat_mass
                saturation = True
            else:
                rho = sl.unit_dry_mass
                saturation = False
            if hasattr(sl, "get_shear_vel_at_v_eff_stress"):
                v_eff = sp.vertical_effective_stress(cum_thickness)
                vs = sl.get_shear_vel_at_v_eff_stress(v_eff, saturation)
            else:
                vs = sl.calc_shear_vel(saturation)
            dd["shear_vel"].append(vs)
            dd["unit_mass"].append(rho)
            dd["thickness"].append(slice_thickness)
    for item in dd:
        dd[item] = np.array(dd[item])
    return dd

Response: Discretizes the soil profile into slices and stores as dictionary


Instruction:
Here you have a function in Python 3, explain what it does
def reset_all(self):
        for item in self.inputs:
            setattr(self, "_%s" % item, None)
        self.stack = []

Response: Resets all parameters to None


Instruction:
Write a Python 3 function that can
add a parameter - value pair to the stack of parameters that have been set.

Response:
def _add_to_stack(self, item, value):
        """
        Add a parameter-value pair to the stack of parameters that have been set.
        :param item:
        :param value:
        :return:
        """
        p_value = (item, value)
        if p_value not in self.stack:
            self.stack.append(p_value)


Instruction:
Can you implement a function in Python 3 that
calculates the shear wave velocity of the user.

Response:
def get_shear_vel(self, saturated):
        """
        Calculate the shear wave velocity

        :param saturated: bool, if true then use saturated mass
        :return:
        """
        try:
            if saturated:
                return np.sqrt(self.g_mod / self.unit_sat_mass)
            else:
                return np.sqrt(self.g_mod / self.unit_dry_mass)
        except TypeError:
            return None


Instruction:
Write a Python 3 script for
voluming of water to volume of voids

Response:
def saturation(self, value):
        """Volume of water to volume of voids"""
        value = clean_float(value)
        if value is None:
            return
        try:
            unit_moisture_weight = self.unit_moist_weight - self.unit_dry_weight
            unit_moisture_volume = unit_moisture_weight / self._pw
            saturation = unit_moisture_volume / self._calc_unit_void_volume()
            if saturation is not None and not ct.isclose(saturation, value, rel_tol=self._tolerance):
                raise ModelError("New saturation (%.3f) is inconsistent "
                                 "with calculated value (%.3f)" % (value, saturation))
        except TypeError:
            pass
        old_value = self.saturation
        self._saturation = value
        try:
            self.recompute_all_weights_and_void()
            self._add_to_stack("saturation", value)
        except ModelError as e:
            self._saturation = old_value
            raise ModelError(e)


Instruction:
Can you tell what is the following Python 3 function doing
def specific_gravity(self, value):
        value = clean_float(value)
        if value is None:
            return
        specific_gravity = self._calc_specific_gravity()
        if specific_gravity is not None and not ct.isclose(specific_gravity, value, rel_tol=self._tolerance):
            raise ModelError("specific gravity is inconsistent with set unit_dry_weight and void_ratio")

        self._specific_gravity = float(value)
        self.stack.append(("specific_gravity", float(value)))
        self.recompute_all_weights_and_void()

Response: Set the relative weight of the solid


Instruction:
How would you explain what the following Python 3 function does
def add_layer(self, depth, soil):

        self._layers[depth] = soil
        self._sort_layers()
        if self.hydrostatic:
            if depth >= self.gwl:
                soil.saturation = 1.0
            else:
                li = self.get_layer_index_by_depth(depth)
                layer_height = self.layer_height(li)
                if layer_height is None:
                    soil.saturation = 0.0
                elif depth + layer_height <= self.gwl:
                    soil.saturation = 0.0
                else:
                    sat_height = depth + self.layer_height(li) - self.gwl
                    soil.saturation = sat_height / self.layer_height(li)

Response: Adds a soil to the SoilProfile at a set depth.


Instruction:
Implement a Python 3 function for
sorting the layers by depth.

Response:
def _sort_layers(self):
        """Sort the layers by depth."""
        self._layers = OrderedDict(sorted(self._layers.items(), key=lambda t: t[0]))


Instruction:
Here you have a function in Python 3, explain what it does
def layer_height(self, layer_int):
        if layer_int == self.n_layers:
            if self.height is None:
                return None
            return self.height - self.layer_depth(layer_int)
        else:
            return self.layer_depth(layer_int + 1) - self.layer_depth(layer_int)

Response: Get the layer height by layer id number.


Instruction:
Write a Python 3 function that can
calculate the equivalent crust cohesion strength according to Karamitros et al. 2013 sett pg 8 eq. 14

Response:
def equivalent_crust_cohesion(self):
        """
        Calculate the equivalent crust cohesion strength according to Karamitros et al. 2013 sett, pg 8 eq. 14

        :return: equivalent cohesion [Pa]
        """
        deprecation("Will be moved to a function")
        if len(self.layers) > 1:
            crust = self.layer(0)
            crust_phi_r = np.radians(crust.phi)
            equivalent_cohesion = crust.cohesion + crust.k_0 * self.crust_effective_unit_weight * \
                                                    self.layer_depth(1) / 2 * np.tan(crust_phi_r)
            return equivalent_cohesion


Instruction:
Write a Python 3 function for
returning the vertical total stress at depth z.

Response:
def get_v_total_stress_at_depth(self, z):
        """
        Determine the vertical total stress at depth z, where z can be a number or an array of numbers.
        """

        if not hasattr(z, "__len__"):
            return self.one_vertical_total_stress(z)
        else:
            sigma_v_effs = []
            for value in z:
                sigma_v_effs.append(self.one_vertical_total_stress(value))
            return np.array(sigma_v_effs)


Instruction:
Can you tell what is the following Python 3 function doing
def one_vertical_total_stress(self, z_c):
        total_stress = 0.0
        depths = self.depths
        end = 0
        for layer_int in range(1, len(depths) + 1):
            l_index = layer_int - 1
            if z_c > depths[layer_int - 1]:
                if l_index < len(depths) - 1 and z_c > depths[l_index + 1]:
                    height = depths[l_index + 1] - depths[l_index]
                    bottom_depth = depths[l_index + 1]
                else:
                    end = 1
                    height = z_c - depths[l_index]
                    bottom_depth = z_c

                if bottom_depth <= self.gwl:
                    total_stress += height * self.layer(layer_int).unit_dry_weight
                else:
                    if self.layer(layer_int).unit_sat_weight is None:
                        raise AnalysisError("Saturated unit weight not defined for layer %i." % layer_int)
                    sat_height = bottom_depth - max(self.gwl, depths[l_index])
                    dry_height = height - sat_height
                    total_stress += dry_height * self.layer(layer_int).unit_dry_weight + \
                                    sat_height * self.layer(layer_int).unit_sat_weight
            else:
                end = 1
            if end:
                break
        return total_stress

Response: Determines the vertical total stress at a single depth z_c.


Instruction:
Make a summary of the following Python 3 code
def get_v_eff_stress_at_depth(self, y_c):
        sigma_v_c = self.get_v_total_stress_at_depth(y_c)
        pp = self.get_hydrostatic_pressure_at_depth(y_c)
        sigma_veff_c = sigma_v_c - pp
        return sigma_veff_c

Response: Determine the vertical effective stress at a single depth z_c.


Instruction:
Implement a function in Python 3 to
get the shear wave velocity at a depth.

Response:
def shear_vel_at_depth(self, y_c):
        """
        Get the shear wave velocity at a depth.

        :param y_c: float, depth from surface
        :return:
        """
        sl = self.get_soil_at_depth(y_c)
        if y_c <= self.gwl:
            saturation = False
        else:
            saturation = True
        if hasattr(sl, "get_shear_vel_at_v_eff_stress"):
            v_eff = self.get_v_eff_stress_at_depth(y_c)
            vs = sl.get_shear_vel_at_v_eff_stress(v_eff, saturation)
        else:
            vs = sl.get_shear_vel(saturation)
        return vs


Instruction:
How would you explain what the following Python 3 function does
def create_package_file(root, master_package, subroot, py_files, opts, subs):
    package = os.path.split(root)[-1]
    text = format_heading(1, '%s Package' % package)
    # add each package's module
    for py_file in py_files:
        if shall_skip(os.path.join(root, py_file)):
            continue
        is_package = py_file == INIT
        py_file = os.path.splitext(py_file)[0]
        py_path = makename(subroot, py_file)
        if is_package:
            heading = ':mod:`%s` Package' % package
        else:
            heading = ':mod:`%s` Module' % py_file
        text += format_heading(2, heading)
        text += format_directive(is_package and subroot or py_path, master_package)
        text += '\n'

    # build a list of directories that are packages (they contain an INIT file)
    subs = [sub for sub in subs if os.path.isfile(os.path.join(root, sub, INIT))]
    # if there are some package directories, add a TOC for theses subpackages
    if subs:
        text += format_heading(2, 'Subpackages')
        text += '.. toctree::\n\n'
        for sub in subs:
            text += '    %s.%s\n' % (makename(master_package, subroot), sub)
        text += '\n'

    write_file(makename(master_package, subroot), text, opts)

Response: Create the text of the file and write it.