Instruction:
Create a Python 3 function for
removing all empty files from a directory.

Response:
def remove_empty_files(root_directory, dry_run=False, ignore_errors=True,
                       enable_scandir=False):
    """
    Remove all empty files from a path. Returns list of the empty files removed.

    :param root_directory: base directory to start at
    :param dry_run: just return a list of what would be removed
    :param ignore_errors: Permissions are a pain, just ignore if you blocked
    :param enable_scandir: on python < 3.5 enable external scandir package
    :return: list of removed files
    """
    file_list = []
    for root, directories, files in _walk(root_directory,
                                          enable_scandir=enable_scandir):
        for file_name in files:
            file_path = join_paths(root, file_name, strict=True)
            if os.path.isfile(file_path) and not os.path.getsize(file_path):
                if file_hash(file_path) == variables.hashes.empty_file.md5:
                    file_list.append(file_path)

    file_list = sorted(set(file_list))

    if not dry_run:
        for afile in file_list:
            try:
                os.unlink(afile)
            except OSError as err:
                if ignore_errors:
                    logger.info("File {0} could not be deleted".format(afile))
                else:
                    raise err

    return file_list


Instruction:
Given the following Python 3 function, write the documentation
def dup_finder(file_path, directory=".", enable_scandir=False):
    size = os.path.getsize(file_path)
    if size == 0:
        for empty_file in remove_empty_files(directory, dry_run=True):
            yield empty_file
    else:
        with open(file_path, 'rb') as f:
            first_twenty = f.read(20)
        file_sha256 = file_hash(file_path, "sha256")

        for root, directories, files in _walk(directory,
                                              enable_scandir=enable_scandir):
            for each_file in files:
                test_file = os.path.join(root, each_file)
                if os.path.getsize(test_file) == size:
                    try:
                        with open(test_file, 'rb') as f:
                            test_first_twenty = f.read(20)
                    except OSError:
                        logger.warning("Could not open file to compare - "
                                       "{0}".format(test_file))
                    else:
                        if first_twenty == test_first_twenty:
                            if file_hash(test_file, "sha256") == file_sha256:
                                yield os.path.abspath(test_file)

Response: A generator that returns the duplicate files of a file in a directory.


Instruction:
Write a Python 3 script for
finding all duplicates in a directory. Will return a list of duplicates in that list.

Response:
def directory_duplicates(directory, hash_type='md5', **kwargs):
    """
    Find all duplicates in a directory. Will return a list, in that list
    are lists of duplicate files.

    .. code: python

        dups = reusables.directory_duplicates('C:\\Users\\Me\\Pictures')

        print(len(dups))
        # 56
        print(dups)
        # [['C:\\Users\\Me\\Pictures\\IMG_20161127.jpg',
        # 'C:\\Users\\Me\\Pictures\\Phone\\IMG_20161127.jpg'], ...

    :param directory: Directory to search
    :param hash_type: Type of hash to perform
    :param kwargs: Arguments to pass to find_files to narrow file types
    :return: list of lists of dups"""
    size_map, hash_map = defaultdict(list), defaultdict(list)

    for item in find_files(directory, **kwargs):
        file_size = os.path.getsize(item)
        size_map[file_size].append(item)

    for possible_dups in (v for v in size_map.values() if len(v) > 1):
        for each_item in possible_dups:
            item_hash = file_hash(each_item, hash_type=hash_type)
            hash_map[item_hash].append(each_item)

    return [v for v in hash_map.values() if len(v) > 1]


Instruction:
Here you have a function in Python 3, explain what it does
def join_paths(*paths, **kwargs):
    path = os.path.abspath(paths[0])

    for next_path in paths[1:]:
        path = os.path.join(path, next_path.lstrip("\\").lstrip("/").strip())
    path.rstrip(os.sep)
    return path if not kwargs.get('safe') else safe_path(path)

Response: Join multiple paths together and return the absolute path of them.


Instruction:
Can you generate the documentation for the following Python 3 function
def join_here(*paths, **kwargs):
    path = os.path.abspath(".")
    for next_path in paths:
        next_path = next_path.lstrip("\\").lstrip("/").strip() if not \
            kwargs.get('strict') else next_path
        path = os.path.abspath(os.path.join(path, next_path))
    return path if not kwargs.get('safe') else safe_path(path)

Response: Join any path or paths as a sub directory of the current file s directory.


Instruction:
Create a Python 3 function to
check if the filename is safe to use or not.

Response:
def check_filename(filename):
    """
    Returns a boolean stating if the filename is safe to use or not. Note that
    this does not test for "legal" names accepted, but a more restricted set of:
    Letters, numbers, spaces, hyphens, underscores and periods.

    :param filename: name of a file as a string
    :return: boolean if it is a safe file name
    """
    if not isinstance(filename, str):
        raise TypeError("filename must be a string")
    if regex.path.linux.filename.search(filename):
        return True
    return False


Instruction:
Make a summary of the following Python 3 code
def safe_filename(filename, replacement="_"):
    if not isinstance(filename, str):
        raise TypeError("filename must be a string")
    if regex.path.linux.filename.search(filename):
        return filename
    safe_name = ""
    for char in filename:
        safe_name += char if regex.path.linux.filename.search(char) \
            else replacement
    return safe_name

Response: Returns a string that is safe for a given filename.


Instruction:
Write a Python 3 function for
returning a path that can be used as a safe version of the file.

Response:
def safe_path(path, replacement="_"):
    """
    Replace unsafe path characters with underscores. Do NOT use this
    with existing paths that cannot be modified, this to to help generate
    new, clean paths.

    Supports windows and *nix systems.

    :param path: path as a string
    :param replacement: character to use in place of bad characters
    :return: a safer path
    """
    if not isinstance(path, str):
        raise TypeError("path must be a string")
    if os.sep not in path:
        return safe_filename(path, replacement=replacement)
    filename = safe_filename(os.path.basename(path))
    dirname = os.path.dirname(path)
    safe_dirname = ""
    regexp = regex.path.windows.safe if win_based else regex.path.linux.safe
    if win_based and dirname.find(":\\") == 1 and dirname[0].isalpha():
        safe_dirname = dirname[0:3]
        dirname = dirname[3:]
    if regexp.search(dirname) and check_filename(filename):
        return path
    else:
        for char in dirname:
            safe_dirname += char if regexp.search(char) else replacement
    sanitized_path = os.path.normpath("{path}{sep}{filename}".format(
        path=safe_dirname,
        sep=os.sep if not safe_dirname.endswith(os.sep) else "",
        filename=filename))
    if (not filename and
            path.endswith(os.sep) and
            not sanitized_path.endswith(os.sep)):
        sanitized_path += os.sep
    return sanitized_path


Instruction:
How would you explain what the following Python 3 function does
def change_task_size(self, size):
        self._pause.value = True
        self.log.debug("About to change task size to {0}".format(size))
        try:
            size = int(size)
        except ValueError:
            self.log.error("Cannot change task size, non integer size provided")
            return False
        if size < 0:
            self.log.error("Cannot change task size, less than 0 size provided")
            return False
        self.max_tasks = size
        if size < self.max_tasks:
            diff = self.max_tasks - size
            self.log.debug("Reducing size offset by {0}".format(diff))
            while True:
                self._update_tasks()
                if len(self.free_tasks) >= diff:
                    for i in range(diff):
                        task_id = self.free_tasks.pop(0)
                        del self.current_tasks[task_id]
                    break
                time.sleep(0.5)
            if not size:
                self._reset_and_pause()
                return True
        elif size > self.max_tasks:
            diff = size - self.max_tasks
            for i in range(diff):
                task_id = str(uuid.uuid4())
                self.current_tasks[task_id] = {}
                self.free_tasks.append(task_id)
        self._pause.value = False
        self.log.debug("Task size changed to {0}".format(size))
        return True

Response: Blocking request to change the number of running tasks


Instruction:
Here you have a function in Python 3, explain what it does
def stop(self):
        self._end.value = True
        if self.background_process:
            try:
                self.background_process.terminate()
            except Exception:
                pass
        for task_id, values in self.current_tasks.items():
            try:
                values['proc'].terminate()
            except Exception:
                pass

Response: Hard stop the server and sub process


Instruction:
Can you tell what is the following Python 3 function doing
def get_state(self):
        return {"started": (True if self.background_process and
                            self.background_process.is_alive() else False),
                "paused": self._pause.value,
                "stopped": self._end.value,
                "tasks": len(self.current_tasks),
                "busy_tasks": len(self.busy_tasks),
                "free_tasks": len(self.free_tasks)}

Response: Get the state of the class


Instruction:
How would you implement a function in Python 3 that
blocks function that can be run directly and can be used to run a single process.

Response:
def main_loop(self, stop_at_empty=False):
        """Blocking function that can be run directly, if so would probably
        want to specify 'stop_at_empty' to true, or have a separate process
        adding items to the queue. """
        try:
            while True:
                self.hook_pre_command()
                self._check_command_queue()
                if self.run_until and self.run_until < datetime.datetime.now():
                    self.log.info("Time limit reached")
                    break
                if self._end.value:
                    break
                if self._pause.value:
                    time.sleep(.5)
                    continue
                self.hook_post_command()
                self._update_tasks()
                task_id = self._free_task()
                if task_id:
                    try:
                        task = self.task_queue.get(timeout=.1)
                    except queue.Empty:
                        if stop_at_empty:
                            break
                        self._return_task(task_id)
                    else:
                        self.hook_pre_task()
                        self.log.debug("Starting task on {0}".format(task_id))
                        try:
                            self._start_task(task_id, task)
                        except Exception as err:
                            self.log.exception("Could not start task {0} -"
                                               " {1}".format(task_id, err))
                        else:
                            self.hook_post_task()
        finally:
            self.log.info("Ending main loop")


Instruction:
How would you implement a function in Python 3 that
starts the main loop as a background process.

Response:
def run(self):
        """Start the main loop as a background process. *nix only"""
        if win_based:
            raise NotImplementedError("Please run main_loop, "
                                      "backgrounding not supported on Windows")
        self.background_process = mp.Process(target=self.main_loop)
        self.background_process.start()


Instruction:
Implement a function in Python 3 to
run a shell command and have it automatically decoded and printed

Response:
def cmd(command, ignore_stderr=False, raise_on_return=False, timeout=None,
        encoding="utf-8"):
    """ Run a shell command and have it automatically decoded and printed

    :param command: Command to run as str
    :param ignore_stderr: To not print stderr
    :param raise_on_return: Run CompletedProcess.check_returncode()
    :param timeout: timeout to pass to communicate if python 3
    :param encoding: How the output should be decoded
    """
    result = run(command, timeout=timeout, shell=True)
    if raise_on_return:
        result.check_returncode()
    print(result.stdout.decode(encoding))
    if not ignore_stderr and result.stderr:
        print(result.stderr.decode(encoding))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def pushd(directory):
    directory = os.path.expanduser(directory)
    _saved_paths.insert(0, os.path.abspath(os.getcwd()))
    os.chdir(directory)
    return [directory] + _saved_paths

Response: Change working directories in style and stay organized!


Instruction:
Can you tell what is the following Python 3 function doing
def popd():
    try:
        directory = _saved_paths.pop(0)
    except IndexError:
        return [os.getcwd()]
    os.chdir(directory)
    return [directory] + _saved_paths

Response: Go back to where you once were.


Instruction:
Given the following Python 3 function, write the documentation
def ls(params="", directory=".", printed=True):
    command = "{0} {1} {2}".format("ls" if not win_based else "dir",
                                   params, directory)
    response = run(command, shell=True)  # Shell required for windows
    response.check_returncode()
    if printed:
        print(response.stdout.decode("utf-8"))
    else:
        return response.stdout

Response: This function is just a python implementation of ls. It s just to subprocess ls.


Instruction:
Given the following Python 3 function, write the documentation
def find(name=None, ext=None, directory=".", match_case=False,
         disable_glob=False, depth=None):
    """ Designed for the interactive interpreter by making default order
    of find_files faster.

    :param name: Part of the file name
    :param ext: Extensions of the file you are looking for
    :param directory: Top location to recursively search for matching files
    :param match_case: If name has to be a direct match or not
    :param disable_glob: Do not look for globable names or use glob magic check
    :param depth: How many directories down to search
    :return: list of all files in the specified directory
    """
    return find_files_list(directory=directory, ext=ext, name=name,
                           match_case=match_case, disable_glob=disable_glob,
                           depth=depth)

Response: This function is used to find all files in a directory or not in a file system.


Instruction:
How would you implement a function in Python 3 that
reads the first N lines of a file

Response:
def head(file_path, lines=10, encoding="utf-8", printed=True,
         errors='strict'):
    """
    Read the first N lines of a file, defaults to 10

    :param file_path: Path to file to read
    :param lines: Number of lines to read in
    :param encoding: defaults to utf-8 to decode as, will fail on binary
    :param printed: Automatically print the lines instead of returning it
    :param errors: Decoding errors: 'strict', 'ignore' or 'replace'
    :return: if printed is false, the lines are returned as a list
    """
    data = []
    with open(file_path, "rb") as f:
        for _ in range(lines):
            try:
                if python_version >= (2, 7):
                    data.append(next(f).decode(encoding, errors=errors))
                else:
                    data.append(next(f).decode(encoding))
            except StopIteration:
                break
    if printed:
        print("".join(data))
    else:
        return data


Instruction:
Explain what the following Python 3 code does
def tail(file_path, lines=10, encoding="utf-8",
         printed=True, errors='strict'):
    """
    A really silly way to get the last N lines, defaults to 10.

    :param file_path: Path to file to read
    :param lines: Number of lines to read in
    :param encoding: defaults to utf-8 to decode as, will fail on binary
    :param printed: Automatically print the lines instead of returning it
    :param errors: Decoding errors: 'strict', 'ignore' or 'replace'
    :return: if printed is false, the lines are returned as a list
    """
    data = deque()

    with open(file_path, "rb") as f:
        for line in f:
            if python_version >= (2, 7):
                data.append(line.decode(encoding, errors=errors))
            else:
                data.append(line.decode(encoding))
            if len(data) > lines:
                data.popleft()
    if printed:
        print("".join(data))
    else:
        return data

Response: Returns the last N lines of a node in a node list.


Instruction:
Given the following Python 3 function, write the documentation
def cp(src, dst, overwrite=False):

    if not isinstance(src, list):
        src = [src]

    dst = os.path.expanduser(dst)
    dst_folder = os.path.isdir(dst)

    if len(src) > 1 and not dst_folder:
        raise OSError("Cannot copy multiple item to same file")

    for item in src:
        source = os.path.expanduser(item)
        destination = (dst if not dst_folder else
                       os.path.join(dst, os.path.basename(source)))
        if not overwrite and os.path.exists(destination):
            _logger.warning("Not replacing {0} with {1}, overwrite not enabled"
                            "".format(destination, source))
            continue

        shutil.copy(source, destination)

Response: Copy files from one folder to another.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cut(string, characters=2, trailing="normal"):
    split_str = [string[i:i + characters] for
                 i in range(0, len(string), characters)]

    if trailing != "normal" and len(split_str[-1]) != characters:
        if trailing.lower() == "remove":
            return split_str[:-1]
        if trailing.lower() == "combine" and len(split_str) >= 2:
            return split_str[:-2] + [split_str[-2] + split_str[-1]]
        if trailing.lower() == "error":
            raise IndexError("String of length {0} not divisible by {1} to"
                             " cut".format(len(string), characters))
    return split_str

Response: Returns a list of N characters from a string.


Instruction:
Write a Python 3 function for
converting an integer into a roman string.

Response:
def int_to_roman(integer):
    """
    Convert an integer into a string of roman numbers.

    .. code: python

        reusables.int_to_roman(445)
        # 'CDXLV'

    :param integer:
    :return: roman string
    """
    if not isinstance(integer, int):
        raise ValueError("Input integer must be of type int")
    output = []
    while integer > 0:
        for r, i in sorted(_roman_dict.items(),
                           key=lambda x: x[1], reverse=True):
            while integer >= i:
                output.append(r)
                integer -= i
    return "".join(output)


Instruction:
Make a summary of the following Python 3 code
def roman_to_int(roman_string):
    roman_string = roman_string.upper().strip()
    if "IIII" in roman_string:
        raise ValueError("Malformed roman string")
    value = 0
    skip_one = False
    last_number = None
    for i, letter in enumerate(roman_string):
        if letter not in _roman_dict:
            raise ValueError("Malformed roman string")
        if skip_one:
            skip_one = False
            continue
        if i < (len(roman_string) - 1):
            double_check = letter + roman_string[i + 1]
            if double_check in _roman_dict:
                if last_number and _roman_dict[double_check] > last_number:
                    raise ValueError("Malformed roman string")
                last_number = _roman_dict[double_check]
                value += _roman_dict[double_check]
                skip_one = True
                continue
        if last_number and _roman_dict[letter] > last_number:
            raise ValueError("Malformed roman string")
        last_number = _roman_dict[letter]
        value += _roman_dict[letter]
    return value

Response: Convert a string of roman numbers into an integer.


Instruction:
Here you have a function in Python 3, explain what it does
def int_to_words(number, european=False):
    def ones(n):
        return "" if n == 0 else _numbers[n]

    def tens(n):
        teen = int("{0}{1}".format(n[0], n[1]))

        if n[0] == 0:
            return ones(n[1])
        if teen in _numbers:
            return _numbers[teen]
        else:
            ten = _numbers[int("{0}0".format(n[0]))]
            one = _numbers[n[1]]
            return "{0}-{1}".format(ten, one)

    def hundreds(n):
        if n[0] == 0:
            return tens(n[1:])
        else:
            t = tens(n[1:])
            return "{0} hundred {1}".format(_numbers[n[0]], "" if not t else t)

    def comma_separated(list_of_strings):
        if len(list_of_strings) > 1:
            return "{0} ".format("" if len(list_of_strings) == 2
                                 else ",").join(list_of_strings)
        else:
            return list_of_strings[0]

    def while_loop(list_of_numbers, final_list):
        index = 0
        group_set = int(len(list_of_numbers) / 3)
        while group_set != 0:
            value = hundreds(list_of_numbers[index:index + 3])
            if value:
                final_list.append("{0} {1}".format(value, _places[group_set])
                                  if _places[group_set] else value)
            group_set -= 1
            index += 3

    number_list = []
    decimal_list = []

    decimal = ''
    number = str(number)
    group_delimiter, point_delimiter = (",", ".") \
        if not european else (".", ",")

    if point_delimiter in number:
        decimal = number.split(point_delimiter)[1]
        number = number.split(point_delimiter)[0].replace(
            group_delimiter, "")
    elif group_delimiter in number:
        number = number.replace(group_delimiter, "")

    if not number.isdigit():
        raise ValueError("Number is not numeric")

    if decimal and not decimal.isdigit():
        raise ValueError("Decimal is not numeric")

    if int(number) == 0:
        number_list.append("zero")

    r = len(number) % 3
    d_r = len(decimal) % 3
    number = number.zfill(len(number) + 3 - r if r else 0)
    f_decimal = decimal.zfill(len(decimal) + 3 - d_r if d_r else 0)

    d = [int(x) for x in f_decimal]
    n = [int(x) for x in number]

    while_loop(n, number_list)

    if decimal and int(decimal) != 0:
        while_loop(d, decimal_list)

        if decimal_list:
            name = ''
            if len(decimal) % 3 == 1:
                name = 'ten'
            elif len(decimal) % 3 == 2:
                name = 'hundred'

            place = int((str(len(decimal) / 3).split(".")[0]))
            number_list.append("and {0} {1}{2}{3}ths".format(
                comma_separated(decimal_list), name,
                "-" if name and _places[place+1] else "", _places[place+1]))

    return comma_separated(number_list)

Response: Converts an integer or float to words.


Instruction:
Given the following Python 3 function, write the documentation
def filter_osm_file():

    print_info('Filtering OSM file...')
    start_time = time.time()

    if check_osmfilter():
        # params = '--keep="highway=motorway =motorway_link =trunk =trunk_link =primary =primary_link =secondary' \
        # 		 ' =secondary_link =tertiary =tertiary_link =unclassified =unclassified_link =residential =residential_link' \
        # 		 ' =living_street" --drop="access=no"'
        params = config.osm_filter_params

        command = './osmfilter' if platform.system() == 'Linux' else 'osmfilter.exe'

        if platform.system() == 'Linux':
            filter_command = '%s "%s" %s | pv > "%s"' % (command, config.osm_map_filename, params,
                                                         config.filtered_osm_filename)
        else:
            filter_command = '%s "%s" %s > "%s"' % (
            command, config.osm_map_filename, params, config.filtered_osm_filename)
        os.system(filter_command)
    else:
        print_info('Osmfilter not available. Exiting.')
        exit(1)

    print_info('Filtering finished. (%.2f secs)' % (time.time() - start_time))

Response: Download and compiles the OSM file and filter out the road elements.


Instruction:
Can you tell what is the following Python 3 function doing
def task_list():
    try:
        jobs_module = settings.RQ_JOBS_MODULE
    except AttributeError:
        raise ImproperlyConfigured(_("You have to define RQ_JOBS_MODULE in settings.py"))

    if isinstance(jobs_module, string_types):
        jobs_modules = (jobs_module,)
    elif isinstance(jobs_module, (tuple, list)):
        jobs_modules = jobs_module
    else:
        raise ImproperlyConfigured(_("RQ_JOBS_MODULE must be a string or a tuple"))

    choices = []

    for module in jobs_modules:
        try:
            tasks = importlib.import_module(module)
        except ImportError:
            raise ImproperlyConfigured(_("Can not find module {}").format(module))

        module_choices = [('%s.%s' % (module, x), underscore_to_camelcase(x)) for x, y in list(tasks.__dict__.items())
                          if type(y) == FunctionType and hasattr(y, 'delay')]

        choices.extend(module_choices)

    choices.sort(key=lambda tup: tup[1])

    return choices

Response: Returns a list of all tasks that are available in the RQ_JOBS_MODULES


Instruction:
Explain what the following Python 3 code does
def rq_link(self):
        if self.rq_job:
            url = reverse('rq_job_detail',
                          kwargs={'job_id': self.rq_id, 'queue_index': queue_index_by_name(self.rq_origin)})
            return '{}'.format(url, self.rq_id)

Response: Link to Django - RQ status page for this job


Instruction:
How would you explain what the following Python 3 function does
def rq_task(self):
        task_path = self.task.split('.')
        module_name = '.'.join(task_path[:-1])
        task_name = task_path[-1]

        module = importlib.import_module(module_name)
        return getattr(module, task_name)

Response: Returns the class that is responsible for retrieving the current task s information.


Instruction:
Can you implement a function in Python 3 that
fixes for tasks without a module. Provides backwards compatibility with < 0. 1. 5

Response:
def fix_module(job):
    """
    Fix for tasks without a module. Provides backwards compatibility with < 0.1.5
    """
    modules = settings.RQ_JOBS_MODULE
    if not type(modules) == tuple:
        modules = [modules]
    for module in modules:
        try:
            module_match = importlib.import_module(module)
            if hasattr(module_match, job.task):
                job.task = '{}.{}'.format(module, job.task)
                break
        except ImportError:
            continue
    return job


Instruction:
Implement a function in Python 3 to
drop tables for all given models.

Response:
async def drop_model_tables(models, **drop_table_kwargs):
    """Drop tables for all given models (in the right order)."""
    for m in reversed(sort_models_topologically(models)):
        await m.drop_table(**drop_table_kwargs)


Instruction:
Make a summary of the following Python 3 code
async def model_to_dict(model, recurse=True, backrefs=False, only=None,
                        exclude=None, seen=None, extra_attrs=None,
                        fields_from_query=None, max_depth=None):
    """
    Convert a model instance (and any related objects) to a dictionary.
    :param bool recurse: Whether foreign-keys should be recursed.
    :param bool backrefs: Whether lists of related objects should be recursed.
    :param only: A list (or set) of field instances indicating which fields
        should be included.
    :param exclude: A list (or set) of field instances that should be
        excluded from the dictionary.
    :param list extra_attrs: Names of model instance attributes or methods
        that should be included.
    :param SelectQuery fields_from_query: Query that was source of model. Take
        fields explicitly selected by the query and serialize them.
    :param int max_depth: Maximum depth to recurse, value <= 0 means no max.
    """
    max_depth = -1 if max_depth is None else max_depth
    if max_depth == 0:
        recurse = False

    only = _clone_set(only)
    extra_attrs = _clone_set(extra_attrs)

    if fields_from_query is not None:
        for item in fields_from_query._select:
            if isinstance(item, Field):
                only.add(item)
            elif isinstance(item, Node) and item._alias:
                extra_attrs.add(item._alias)

    data = {}
    exclude = _clone_set(exclude)
    seen = _clone_set(seen)
    exclude |= seen
    model_class = type(model)

    for field in model._meta.declared_fields:
        if field in exclude or (only and (field not in only)):
            continue

        field_data = model._data.get(field.name)
        if isinstance(field, ForeignKeyField) and recurse:
            if field_data:
                seen.add(field)
                rel_obj = getattr(model, field.name)
                if iscoroutine(rel_obj):
                    rel_obj = await rel_obj
                field_data = await model_to_dict(
                    rel_obj,
                    recurse=recurse,
                    backrefs=backrefs,
                    only=only,
                    exclude=exclude,
                    seen=seen,
                    max_depth=max_depth - 1)
            else:
                field_data = None

        data[field.name] = field_data

    if extra_attrs:
        for attr_name in extra_attrs:
            attr = getattr(model, attr_name)
            if callable(attr):
                data[attr_name] = attr()
            else:
                data[attr_name] = attr

    if backrefs and recurse:
        for related_name, foreign_key in model._meta.reverse_rel.items():
            descriptor = getattr(model_class, related_name)
            if descriptor in exclude or foreign_key in exclude:
                continue
            if only and (descriptor not in only) and (foreign_key not in only):
                continue

            accum = []
            exclude.add(foreign_key)
            related_query = getattr(
                model,
                related_name + '_prefetch',
                getattr(model, related_name))

            async for rel_obj in related_query:
                accum.append(await model_to_dict(
                    rel_obj,
                    recurse=recurse,
                    backrefs=backrefs,
                    only=only,
                    exclude=exclude,
                    max_depth=max_depth - 1))

            data[related_name] = accum

    return data

Response: Convert a model instance to a dictionary.


Instruction:
Implement a function in Python 3 to
extract the rows from the CSV file and analyze their data - types.

Response:
async def extract_rows(self, file_or_name, **reader_kwargs):
        """
        Extract `self.sample_size` rows from the CSV file and analyze their
        data-types.
        :param str file_or_name: A string filename or a file handle.
        :param reader_kwargs: Arbitrary parameters to pass to the CSV reader.
        :returns: A 2-tuple containing a list of headers and list of rows
                  read from the CSV file.
        """
        rows = []
        rows_to_read = self.sample_size
        async with self.get_reader(file_or_name, **reader_kwargs) as reader:
            if self.has_header:
                rows_to_read += 1
            for i in range(self.sample_size):
                try:
                    row = await reader.__anext__()
                except AttributeError as te:
                    row = next(reader)
                except:
                    raise
                rows.append(row)

        if self.has_header:
            header, rows = rows[0], rows[1:]
        else:
            header = ['field_%d' % i for i in range(len(rows[0]))]
        return header, rows


Instruction:
Create a Python 3 function to
return a list of functions to use when testing values.

Response:
def get_checks(self):
        """Return a list of functions to use when testing values."""
        return [
            self.is_date,
            self.is_datetime,
            self.is_integer,
            self.is_float,
            self.default]


Instruction:
How would you code a function in Python 3 to
analyze the given rows and try to determine the type of value stored in the table.

Response:
def analyze(self, rows):
        """
        Analyze the given rows and try to determine the type of value stored.
        :param list rows: A list-of-lists containing one or more rows from a
                          csv file.
        :returns: A list of peewee Field objects for each column in the CSV.
        """
        transposed = zip(*rows)
        checks = self.get_checks()
        column_types = []
        for i, column in enumerate(transposed):
            # Remove any empty values.
            col_vals = [val for val in column if val != '']
            for check in checks:
                results = set(check(val) for val in col_vals)
                if all(results):
                    column_types.append(check.field())
                    break

        return column_types


Instruction:
Here you have a function in Python 3, explain what it does
def clean_text(self, text):
        '''Clean text using bleach.'''
        if text is None:
            return ''
        text = re.sub(ILLEGAL_CHARACTERS_RE, '', text)
        if '<' in text or '&lt' in text:
            text = clean(text, tags=self.tags, strip=self.strip)

        return unescape(text)

Response: Clean text using bleach.


Instruction:
How would you implement a function in Python 3 that
constructs the path to the image and feature of the unique key.

Response:
def path(self, category = None, image = None, feature = None):
        """
		Constructs the path to categories, images and features.

        This path function assumes that the following storage scheme is used on
        the hard disk to access categories, images and features:
            - categories: /impath/category
            - images:     /impath/category/category_image.png
            - features:   /ftrpath/category/feature/category_image.mat

        The path function is called to query the location of categories, images
        and features before they are loaded. Thus, if your features are organized
        in a different way, you can simply replace this method such that it returns
        appropriate paths' and the LoadFromDisk loader will use your naming
        scheme.
		"""
        filename = None
        if not category is None:
            filename = join(self.impath, str(category))
        if not image is None:
            assert not category is None, "The category has to be given if the image is given"
            filename = join(filename,
                '%s_%s.png' % (str(category), str(image)))
        if not feature is None:
            assert category != None and image != None, "If a feature name is given the category and image also have to be given."
            filename = join(self.ftrpath, str(category), feature,
                '%s_%s.mat' % (str(category), str(image)))
        return filename


Instruction:
How would you explain what the following Python 3 function does
def get_image(self, cat, img):
        filename = self.path(cat, img)
        data = []
        if filename.endswith('mat'):
            data = loadmat(filename)['output']
        else:
            data = imread(filename)
        if self.size is not None:
            return imresize(data, self.size)
        else:
            return data

Response: Loads an image from disk.


Instruction:
Given the following Python 3 function, write the documentation
def get_feature(self, cat, img, feature):
        filename = self.path(cat, img, feature)
        data = loadmat(filename)
        name = [k for k in list(data.keys()) if not k.startswith('__')]
        if self.size is not None:
            return imresize(data[name.pop()], self.size)
        return data[name.pop()]

Response: Load a feature from disk.


Instruction:
Can you generate the documentation for the following Python 3 function
def save_image(self, cat, img, data):
        filename = self.path(cat, img)
        mkdir(filename)
        if type(data) == np.ndarray:
            data = Image.fromarray(data).convert('RGB')
        data.save(filename)

Response: Saves a new image.


Instruction:
Here you have a function in Python 3, explain what it does
def save_feature(self, cat, img, feature, data):
        filename = self.path(cat, img, feature)
        mkdir(filename)
        savemat(filename, {'output':data})

Response: Saves a new feature.


Instruction:
Can you implement a function in Python 3 that
draws nr_samples random samples from vec.

Response:
def randsample(vec, nr_samples, with_replacement = False):
    """
    Draws nr_samples random samples from vec.
    """
    if not with_replacement:
        return np.random.permutation(vec)[0:nr_samples]
    else:
        return np.asarray(vec)[np.random.randint(0, len(vec), nr_samples)]


Instruction:
Can you implement a function in Python 3 that
calculates how much prediction. shape and image_size differ.

Response:
def calc_resize_factor(prediction, image_size):
    """
    Calculates how much prediction.shape and image_size differ.
    """
    resize_factor_x = prediction.shape[1] / float(image_size[1])
    resize_factor_y = prediction.shape[0] / float(image_size[0])
    if abs(resize_factor_x - resize_factor_y) > 1.0/image_size[1] :
        raise RuntimeError("""The aspect ratio of the fixations does not
                              match with the prediction: %f vs. %f"""
                              %(resize_factor_y, resize_factor_x))
    return (resize_factor_y, resize_factor_x)


Instruction:
Can you implement a function in Python 3 that
converts a dictionary to a NumPy matrix.

Response:
def dict_2_mat(data, fill = True):
    """
    Creates a NumPy array from a dictionary with only integers as keys and
    NumPy arrays as values. Dimension 0 of the resulting array is formed from
    data.keys(). Missing values in keys can be filled up with np.nan (default)
    or ignored.

    Parameters
    ----------
    data : dict
        a dictionary with integers as keys and array-likes of the same shape
        as values
    fill : boolean
        flag specifying if the resulting matrix will keep a correspondence
        between dictionary keys and matrix indices by filling up missing keys
        with matrices of NaNs. Defaults to True

    Returns
    -------
    numpy array with one more dimension than the values of the input dict
    """
    if any([type(k) != int for k in list(data.keys())]):
        raise RuntimeError("Dictionary cannot be converted to matrix, " +
                            "not all keys are ints")
    base_shape = np.array(list(data.values())[0]).shape
    result_shape = list(base_shape)
    if fill:
        result_shape.insert(0, max(data.keys()) + 1)
    else:
        result_shape.insert(0, len(list(data.keys())))
    result = np.empty(result_shape) + np.nan

    for (i, (k, v)) in enumerate(data.items()):
        v = np.array(v)
        if v.shape != base_shape:
            raise RuntimeError("Dictionary cannot be converted to matrix, " +
                                        "not all values have same dimensions")
        result[fill and [k][0] or [i][0]] = v
    return result


Instruction:
Can you tell what is the following Python 3 function doing
def dict_fun(data, function):
    return dict((k, function(v)) for k, v in list(data.items()))

Response: Apply a function to all values in a dictionary return a dictionary with the keys as data and the values as function.


Instruction:
Can you implement a function in Python 3 that
middles string of length max_len

Response:
def snip_string_middle(string, max_len=20, snip_string='...'):
    """
    >>> snip_string_middle('this is long', 8)
    'th...ong'
    >>> snip_string_middle('this is long', 12)
    'this is long'
    >>> snip_string_middle('this is long', 8, '~')
    'thi~long'

    """
    #warn('use snip_string() instead', DeprecationWarning)
    if len(string) <= max_len:
        new_string = string
    else:
        visible_len = (max_len - len(snip_string))
        start_len = visible_len//2
        end_len = visible_len-start_len

        new_string = string[0:start_len]+ snip_string + string[-end_len:]

    return new_string


Instruction:
Can you generate a brief explanation for the following Python 3 code
def snip_string(string, max_len=20, snip_string='...', snip_point=0.5):
    if len(string) <= max_len:
        new_string = string
    else:
        visible_len = (max_len - len(snip_string))
        start_len = int(visible_len*snip_point)
        end_len = visible_len-start_len

        new_string = string[0:start_len]+ snip_string
        if end_len > 0:
            new_string += string[-end_len:]

    return new_string

Response: Snips a string so that it is no longer than max_len replacing deleted characters with the snip_string.


Instruction:
Implement a Python 3 function for
giving a list of strings finds the longest string that is common to all strings in the list.

Response:
def find_common_beginning(string_list, boundary_char = None):
    """Given a list of strings, finds finds the longest string that is common
    to the *beginning* of all strings in the list.

    boundary_char defines a boundary that must be preserved, so that the
    common string removed must end with this char.
    """

    common=''

    # by definition there is nothing common to 1 item...
    if len(string_list) > 1:
        shortestLen = min([len(el) for el in string_list])

        for idx in range(shortestLen):
            chars = [s[idx] for s in string_list]
            if chars.count(chars[0]) != len(chars): # test if any chars differ
                break
            common+=chars[0]

    if boundary_char is not None:
        try:
            end_idx = common.rindex(boundary_char)
            common = common[0:end_idx+1]
        except ValueError:
            common = ''

    return common


Instruction:
How would you implement a function in Python 3 that
gives a list of strings finds the longest string that is common to all strings in the list and returns a new list of strings.

Response:
def factorise_strings (string_list, boundary_char=None):
    """Given a list of strings, finds the longest string that is common
    to the *beginning* of all strings in the list and
    returns a new list whose elements lack this common beginning.

    boundary_char defines a boundary that must be preserved, so that the
    common string removed must end with this char.

    >>> cmn='something/to/begin with?'
    >>> blah=[cmn+'yes',cmn+'no',cmn+'?maybe']
    >>> (blee, bleecmn) = factorise_strings(blah)
    >>> blee
    ['yes', 'no', '?maybe']
    >>> bleecmn == cmn
    True

    >>> blah = ['de.uos.nbp.senhance', 'de.uos.nbp.heartFelt']
    >>> (blee, bleecmn) = factorise_strings(blah, '.')
    >>> blee
    ['senhance', 'heartFelt']
    >>> bleecmn
    'de.uos.nbp.'

    >>> blah = ['/some/deep/dir/subdir', '/some/deep/other/dir', '/some/deep/other/dir2']
    >>> (blee, bleecmn) = factorise_strings(blah, '/')
    >>> blee
    ['dir/subdir', 'other/dir', 'other/dir2']
    >>> bleecmn
    '/some/deep/'

    >>> blah = ['/net/store/nbp/heartFelt/data/ecg/emotive_interoception/p20/2012-01-27T09.01.14-ecg.csv', '/net/store/nbp/heartFelt/data/ecg/emotive_interoception/p21/2012-01-27T11.03.08-ecg.csv', '/net/store/nbp/heartFelt/data/ecg/emotive_interoception/p23/2012-01-31T12.02.55-ecg.csv']
    >>> (blee, bleecmn) = factorise_strings(blah, '/')
    >>> bleecmn
    '/net/store/nbp/heartFelt/data/ecg/emotive_interoception/'

    rmuil 2012/02/01
    """

    cmn = find_common_beginning(string_list, boundary_char)

    new_list = [el[len(cmn):] for el in string_list]

    return (new_list, cmn)


Instruction:
Can you write a function in Python 3 where it
loads a datamat from a file.

Response:
def load(path, variable='Datamat'):
    """
    Load datamat at path.

    Parameters:
        path : string
            Absolute path of the file to load from.
    """
    f = h5py.File(path,'r')
    try:
        dm = fromhdf5(f[variable])
    finally:
        f.close()
    return dm


Instruction:
Explain what the following Python 3 code does
def VectorFactory(fields, parameters, categories = None):
    '''
    Creates a datamat from a dictionary that contains lists/arrays as values.

    Input:
        fields: Dictionary
            The values will be used as fields of the datamat and the keys
            as field names.
        parameters: Dictionary
            A dictionary whose values are added as parameters. Keys are used
            for parameter names.
    '''
    fm = Datamat(categories = categories)
    fm._fields = list(fields.keys())
    for (field, value) in list(fields.items()):
        try:
            fm.__dict__[field] = np.asarray(value)
        except ValueError:
            fm.__dict__[field] = np.asarray(value, dtype=np.object)

    fm._parameters = parameters
    for (field, value) in list(parameters.items()):
       fm.__dict__[field] = value
    fm._num_fix = len(fm.__dict__[list(fields.keys())[0]])
    return fm

Response: Returns a datamat from a dictionary that contains lists or arrays as values.


Instruction:
Make a summary of the following Python 3 code
def filter(self, index): #@ReservedAssignment
        return Datamat(categories=self._categories, datamat=self, index=index)

Response: This function filters a datamat by different aspects.


Instruction:
Here you have a function in Python 3, explain what it does
def copy(self):
        return self.filter(np.ones(self._num_fix).astype(bool))

Response: Returns a copy of the datamat.


Instruction:
Make a summary of the following Python 3 code
def save(self, path):
        f = h5py.File(path, 'w')
        try:
            fm_group = f.create_group('Datamat')
            for field in self.fieldnames():
                try:
                    fm_group.create_dataset(field, data = self.__dict__[field])
                except (TypeError,) as e:
                    # Assuming field is an object array that contains dicts which
                    # contain numpy arrays as values
                    sub_group = fm_group.create_group(field)
                    for i, d in enumerate(self.__dict__[field]):
                        index_group = sub_group.create_group(str(i))
                        print((field, d))
                        for key, value in list(d.items()):
                            index_group.create_dataset(key, data=value)

            for param in self.parameters():
                fm_group.attrs[param]=self.__dict__[param]
        finally:
            f.close()

Response: Saves the object to a file.


Instruction:
Create a Python 3 function for
setting the value of a parameter.

Response:
def set_param(self, key, value):
        """
        Set the value of a parameter.
        """
        self.__dict__[key] = value
        self._parameters[key] = value


Instruction:
How would you explain what the following Python 3 function does
def by_field(self, field):
        for value in np.unique(self.__dict__[field]):
            yield self.filter(self.__dict__[field] == value)

Response: Returns an iterator that iterates over unique values of field


Instruction:
Explain what the following Python 3 code does
def by_cat(self):
        for value in np.unique(self.category):
            cat_fm = self.filter(self.category == value)
            if self._categories:
                yield (cat_fm, self._categories[value])
            else:
                yield (cat_fm, None)

Response: Iterate over datamat and categories and returns a filtered datamat.


Instruction:
Write a Python 3 script for
iterating over datamat and returns a filtered datamat and associated categories objects.

Response:
def by_filenumber(self):
        """
        Iterates over categories and returns a filtered datamat.

        If a categories object is attached, the images object for the given
        category is returned as well (else None is returned).

        Returns:
            (datamat, categories) : A tuple that contains first the filtered
                datamat (has only one category) and second the associated
                categories object (if it is available, None otherwise)
        """
        for value in np.unique(self.filenumber):
            file_fm = self.filter(self.filenumber == value)
            if self._categories:
                yield (file_fm, self._categories[self.category[0]][value])
            else:
                yield (file_fm, None)


Instruction:
Make a summary of the following Python 3 code
def add_field(self, name, data):
        if name in self._fields:
            raise ValueError
        if not len(data) == self._num_fix:
            raise ValueError
        self._fields.append(name)
        self.__dict__[name] = data

Response: Add a new field to the datamat.


Instruction:
Make a summary of the following Python 3 code
def add_field_like(self, name, like_array):
        new_shape = list(like_array.shape)
        new_shape[0] = len(self)
        new_data = ma.empty(new_shape, like_array.dtype)
        new_data.mask = True
        self.add_field(name, new_data)

Response:
Add a new field to the Datamat with the dtype of the like_array and the shape of the like_array except for the first
        dimension.


Instruction:
How would you implement a function in Python 3 that
adds a new field (data_field) to the Datamat with data from the corresponding field of another Datamat (src_dm). This is accomplished through the use of a key_field, which is used to determine how the data is copied. This operation corresponds loosely to an SQL join operation. The two Datamats are essentially aligned by the unique values of key_field so that each block element of the new field of the target Datamat will consist of those elements of src_dm's data_field where the corresponding element in key_field matches. If 'take_first' is not true, and there is not only a single corresponding element (typical usage case) then the target element value will be a sequence (array) of all the matching elements. The target Datamat (self) must not have a field name data_field already, and both Datamats must have key_field. The new field in the target Datamat will be a masked array to handle non-existent data. TODO: Make example more generic, remove interoceptive reference TODO: Make standalone test Examples: >>> dm_intero = load_interoception_files ('test-ecg.csv', silent=True) >>> dm_emotiv = load_emotivestimuli_files ('test-bpm.csv', silent=True) >>> length(dm_intero) 4 >>> unique(dm_intero.subject_id) ['p05', 'p06'] >>> length(dm_emotiv) 3 >>> unique(dm_emotiv.subject_id) ['p04', 'p05', 'p06'] >>> 'interospective_awareness' in dm_intero.fieldnames() True >>> unique(dm_intero.interospective_awareness) == [0.5555, 0.6666] True >>> 'interospective_awareness' in dm_emotiv.fieldnames() False >>> dm_emotiv.copy_field(dm_intero, 'interospective_awareness', 'subject_id') >>> 'interospective_awareness' in dm_emotiv.fieldnames() True >>> unique(dm_emotiv.interospective_awareness) == [NaN, 0.5555, 0.6666] False

Response:
def annotate (self, src_dm, data_field, key_field, take_first=True):
        """
        Adds a new field (data_field) to the Datamat with data from the
        corresponding field of another Datamat (src_dm).

				This is accomplished through the use of a key_field, which is
        used to determine how the data is copied.

				This operation corresponds loosely to an SQL join operation.

        The two Datamats are essentially aligned by the unique values
        of key_field so that each block element of the new field of the target
        Datamat will consist of those elements of src_dm's data_field
        where the corresponding element in key_field matches.

        If 'take_first' is not true, and there is not
        only a single corresponding element (typical usage case) then the
        target element value will be
        a sequence (array) of all the matching elements.

        The target Datamat (self) must not have a field name data_field
        already, and both Datamats must have key_field.

        The new field in the target Datamat will be a masked array to handle
        non-existent data.

				TODO: Make example more generic, remove interoceptive reference
				TODO: Make standalone test
        Examples:

        >>> dm_intero = load_interoception_files ('test-ecg.csv', silent=True)
        >>> dm_emotiv = load_emotivestimuli_files ('test-bpm.csv', silent=True)
        >>> length(dm_intero)
        4
        >>> unique(dm_intero.subject_id)
        ['p05', 'p06']
        >>> length(dm_emotiv)
        3
        >>> unique(dm_emotiv.subject_id)
        ['p04', 'p05', 'p06']
        >>> 'interospective_awareness' in dm_intero.fieldnames()
        True
        >>> unique(dm_intero.interospective_awareness) == [0.5555, 0.6666]
        True
        >>> 'interospective_awareness' in dm_emotiv.fieldnames()
        False
        >>> dm_emotiv.copy_field(dm_intero, 'interospective_awareness', 'subject_id')
        >>> 'interospective_awareness' in dm_emotiv.fieldnames()
        True
        >>> unique(dm_emotiv.interospective_awareness) == [NaN, 0.5555, 0.6666]
        False
        """
        if key_field not in self._fields or key_field not in src_dm._fields:
            raise AttributeError('key field (%s) must exist in both Datamats'%(key_field))
        if data_field not in src_dm._fields:
            raise AttributeError('data field (%s) must exist in source Datamat' % (data_field))
        if data_field in self._fields:
            raise AttributeError('data field (%s) already exists in target Datamat' % (data_field))

        #Create a mapping of key_field value to data value.
        data_to_copy = dict([(x.field(key_field)[0], x.field(data_field)) for x in src_dm.by_field(key_field)])

        data_element = list(data_to_copy.values())[0]

        #Create the new data array of correct size.
        # We use a masked array because it is possible that for some elements
        # of the target Datamat, there exist simply no data in the source
        # Datamat. NaNs are fine as indication of this for floats, but if the
        # field happens to hold booleans or integers or something else, NaN
        # does not work.
        new_shape = [len(self)] + list(data_element.shape)
        new_data = ma.empty(new_shape, data_element.dtype)
        new_data.mask=True
        if np.issubdtype(new_data.dtype, np.float):
            new_data.fill(np.NaN) #For backwards compatibility, if mask not used

        #Now we copy the data. If the data to copy contains only a single value,
        # it is added to the target as a scalar (single value).
        # Otherwise, it is copied as is, i.e. as a sequence.
        for (key, val) in list(data_to_copy.items()):
            if take_first:
                new_data[self.field(key_field) == key] = val[0]
            else:
                new_data[self.field(key_field) == key] = val

        self.add_field(data_field, new_data)


Instruction:
Can you tell what is the following Python 3 function doing
def rm_field(self, name):
        if not name in self._fields:
            raise ValueError
        self._fields.remove(name)
        del self.__dict__[name]

Response: Removes a field from the datamat.


Instruction:
Can you generate the documentation for the following Python 3 function
def add_parameter(self, name, value):
        if name in self._parameters:
            raise ValueError("'%s' is already a parameter" % (name))
        elif name in self.__dict__:
            raise ValueError("'%s' conflicts with the Datamat name-space" % (name))

        self.__dict__[name] = value
        self._parameters[name] = self.__dict__[name]

Response: Adds a parameter to the existing Datamat.


Instruction:
Here you have a function in Python 3, explain what it does
def rm_parameter(self, name):
        if name not in self._parameters:
            raise ValueError("no '%s' parameter found" % (name))

        del self._parameters[name]
        del self.__dict__[name]

Response: Removes a parameter from the existing Datamat.


Instruction:
Implement a function in Python 3 to
promote a parameter to a field by creating a new array of same size as the other existing fields and filling it with the current value of the parameter.

Response:
def parameter_to_field(self, name):
        """
        Promotes a parameter to a field by creating a new array of same
        size as the other existing fields, filling it with the current
        value of the parameter, and then removing that parameter.
        """
        if name not in self._parameters:
            raise ValueError("no '%s' parameter found" % (name))
        if self._fields.count(name) > 0:
            raise ValueError("field with name '%s' already exists" % (name))

        data = np.array([self._parameters[name]]*self._num_fix)

        self.rm_parameter(name)
        self.add_field(name, data)


Instruction:
Write a Python 3 function that can
join the current instance of Datamat with the new instance of Datamat.

Response:
def join(self, fm_new, minimal_subset=True):
        """
        Adds content of a new Datamat to this Datamat.

        If a parameter of the Datamats is not equal or does not exist
        in one, it is promoted to a field.

        If the two Datamats have different fields then the elements for the
        Datamats that did not have the field will be NaN, unless
        'minimal_subset' is true, in which case the mismatching fields will
        simply be deleted.

        Parameters
        fm_new : instance of Datamat
            This Datamat is added to the current one.
        minimal_subset : if true, remove fields which don't exist in both,
        	instead of using NaNs for missing elements (defaults to False)

        Capacity to use superset of fields added by rmuil 2012/01/30

        """
        # Check if parameters are equal. If not, promote them to fields.
        '''
        for (nm, val) in fm_new._parameters.items():
            if self._parameters.has_key(nm):
                if (val != self._parameters[nm]):
                    self.parameter_to_field(nm)
                    fm_new.parameter_to_field(nm)
            else:
                fm_new.parameter_to_field(nm)
        '''
        # Deal with mismatch in the fields
        # First those in self that do not exist in new...
        orig_fields = self._fields[:]
        for field in orig_fields:
            if not field in fm_new._fields:
                if minimal_subset:
                    self.rm_field(field)
                else:
                    warnings.warn("This option is deprecated. Clean and Filter your data before it is joined.", DeprecationWarning)
                    fm_new.add_field_like(field, self.field(field))
        # ... then those in the new that do not exist in self.
        orig_fields = fm_new._fields[:]
        for field in orig_fields:
            if not field in self._fields:
                if minimal_subset:
                    fm_new.rm_field(field)
                else:
                    warnings.warn("This option is deprecated. Clean and Filter your data before it is joined.", DeprecationWarning)
                    self.add_field_like(field, fm_new.field(field))

        if 'SUBJECTINDEX' in self._fields[:]:
            if fm_new.SUBJECTINDEX[0] in self.SUBJECTINDEX:
                fm_new.SUBJECTINDEX[:] = self.SUBJECTINDEX.max()+1
        # Concatenate fields
        for field in self._fields:
            self.__dict__[field] = ma.hstack((self.__dict__[field],
                fm_new.__dict__[field]))

        # Update _num_fix
        self._num_fix += fm_new._num_fix


Instruction:
Implement a function in Python 3 to
generate a histogram of the given data along the x - axis and length differences.

Response:
def makeAngLenHist(ad, ld, fm = None, collapse=True, fit=spline_base.fit2d):
    """
    Histograms and performs a spline fit on the given data,
    usually angle and length differences.

    Parameters:
        ad : array
            The data to be histogrammed along the x-axis.
            May range from -180 to 180.
        ld : array
            The data to be histogrammed along the y-axis.
            May range from -36 to 36.
        collapse : boolean
            If true, the histogrammed data will include
            negative values on the x-axis. Else, the histogram
            will be collapsed along x = 0, and thus contain only
            positive angle differences
        fit : function or None, optional
            The function to use in order to fit the data.
            If no fit should be applied, set to None
        fm  : fixmat or None, optional
            If given, the angle and length differences are calculated
            from the fixmat and the previous parameters are overwritten.
    """

    if fm:
        ad,ld = anglendiff(fm, roll=2)
        ad, ld = ad[0], ld[0]

    ld = ld[~np.isnan(ld)]
    ad = reshift(ad[~np.isnan(ad)])

    if collapse:
        e_y = np.linspace(-36.5, 36.5, 74)
        e_x = np.linspace(0, 180, 181)
        H = makeHist(abs(ad), ld, fit=fit, bins=[e_y, e_x])

        H = H/H.sum()

        return H
    else:
        e_x = np.linspace(-180, 180, 361)
        e_y = np.linspace(-36.5, 36.5, 74)
        return makeHist(ad, ld, fit=fit, bins=[e_y, e_x])


Instruction:
Implement a function in Python 3 to
construct a 2D histogram of the given data.

Response:
def makeHist(x_val, y_val, fit=spline_base.fit2d,
            bins=[np.linspace(-36.5,36.5,74),np.linspace(-180,180,361)]):
    """
    Constructs a (fitted) histogram of the given data.

    Parameters:
        x_val : array
            The data to be histogrammed along the x-axis.
        y_val : array
            The data to be histogrammed along the y-axis.
        fit : function or None, optional
            The function to use in order to fit the data.
            If no fit should be applied, set to None
        bins : touple of arrays, giving the bin edges to be
            used in the histogram. (First value: y-axis, Second value: x-axis)
    """

    y_val = y_val[~np.isnan(y_val)]
    x_val = x_val[~np.isnan(x_val)]

    samples = list(zip(y_val, x_val))
    K, xedges, yedges = np.histogram2d(y_val, x_val, bins=bins)

    if (fit is None):
        return K/ K.sum()

    # Check if given attr is a function
    elif hasattr(fit, '__call__'):
        H = fit(np.array(samples), bins[0], bins[1], p_est=K)[0]
        return H/H.sum()
    else:
        raise TypeError("Not a valid argument, insert spline function or None")


Instruction:
Make a summary of the following Python 3 code
def firstSacDist(fm):
    ang, leng, ad, ld = anglendiff(fm, return_abs=True)
    y_arg = leng[0][np.roll(fm.fix == min(fm.fix), 1)]/fm.pixels_per_degree
    x_arg = reshift(ang[0][np.roll(fm.fix == min(fm.fix), 1)])
    bins = [list(range(int(ceil(np.nanmax(y_arg)))+1)), np.linspace(-180, 180, 361)]
    return makeHist(x_arg, y_arg, fit=None, bins = bins)

Response:
Computes the distribution of angle and length of the first saccades
           .


Instruction:
Implement a Python 3 function for
returning the distribution of trajectory lengths i. e. the number of saccades that were made as part of one trajectory ArcGIS record.

Response:
def trajLenDist(fm):
    """
        Computes the distribution of trajectory lengths, i.e.
        the number of saccades that were made as a part of one trajectory

        Parameters:
            fm : ocupy.fixmat
                The fixation data to be analysed

    """
    trajLen = np.roll(fm.fix, 1)[fm.fix == min(fm.fix)]
    val, borders = np.histogram(trajLen,
                    bins=np.linspace(-0.5, max(trajLen)+0.5, max(trajLen)+2))
    cumsum = np.cumsum(val.astype(float) / val.sum())
    return cumsum, borders


Instruction:
Write a Python 3 function that can
transform the given number element into a range of [-180, 180], which covers all possible angle differences. This method reshifts larger or smaller numbers that might be the output of other angular calculations into that range by adding or subtracting 360, respectively. To make sure that angular data ranges between -180 and 180 in order to be properly histogrammed, apply this method first. Parameters: I : array or list or int or float Number or numbers that shall be reshifted. Farell, Ludwig, Ellis, and Gilchrist Returns: numpy.ndarray : Reshifted number or numbers as array

Response:
def reshift(I):
    """
    Transforms the given number element into a range of [-180, 180],
    which covers all possible angle differences. This method reshifts larger or
    smaller numbers that might be the output of other angular calculations
    into that range by adding or subtracting 360, respectively.
    To make sure that angular data ranges between -180 and 180 in order to be
    properly histogrammed, apply this method first.

    Parameters:
        I : array or list or int or float
            Number or numbers that shall be reshifted.
    Farell, Ludwig, Ellis, and Gilchrist
    Returns:
        numpy.ndarray : Reshifted number or numbers as array
    """
    # Output -180 to +180
    if type(I)==list:
        I = np.array(I)

    return ((I-180)%360)-180


Instruction:
Here you have a function in Python 3, explain what it does
def initializeData(self, fit = None, full_H1=None, max_length = 40,
            in_deg = True):
        """
        Prepares the data to be replicated. Calculates the second-order length
        and angle dependencies between saccades and stores them in a fitted
        histogram.

        Parameters:
            fit : function, optional
                The method to use for fitting the histogram
            full_H1 : twodimensional numpy.ndarray, optional
                Where applicable, the distribution of angle and length
                differences to replicate with dimensions [73,361]
        """
        a, l, ad, ld = anglendiff(self.fm, roll=1, return_abs = True)
        if in_deg:
            self.fm.pixels_per_degree = 1

        samples = np.zeros([3, len(l[0])])
        samples[0] = l[0]/self.fm.pixels_per_degree
        samples[1] = np.roll(l[0]/self.fm.pixels_per_degree,-1)
        samples[2] = np.roll(reshift(ad[0]),-1)
        z = np.any(np.isnan(samples), axis=0)
        samples = samples[:,~np.isnan(samples).any(0)]

        if full_H1 is None:
            self.full_H1 = []
            for i in range(1, int(ceil(max_length+1))):
                idx = np.logical_and(samples[0]<=i, samples[0]>i-1)
                if idx.any():
                    self.full_H1.append(makeHist(samples[2][idx], samples[1][idx], fit=fit,
                                                bins=[np.linspace(0,max_length-1,max_length),np.linspace(-180,180,361)]))
                    # Sometimes if there's only one sample present there seems to occur a problem
                    # with histogram calculation and the hist is filled with nans. In this case, dismiss
                    # the hist.
                    if np.isnan(self.full_H1[-1]).any():
                        self.full_H1[-1] = np.array([])
                    self.nosamples.append(len(samples[2][idx]))
                else:
                    self.full_H1.append(np.array([]))
                    self.nosamples.append(0)
        else:
            self.full_H1 = full_H1

        self.firstLenAng_cumsum, self.firstLenAng_shape = (
                                        compute_cumsum(firstSacDist(self.fm)))
        self.probability_cumsum = []

        for i in range(len(self.full_H1)):
            if self.full_H1[i] == []:
                self.probability_cumsum.append(np.array([]))
            else:
                self.probability_cumsum.append(np.cumsum(self.full_H1[i].flat))

        self.trajLen_cumsum, self.trajLen_borders = trajLenDist(self.fm)

        min_distance = 1/np.array([min((np.unique(self.probability_cumsum[i]) \
                        -np.roll(np.unique(self.probability_cumsum[i]),1))[1:]) \
                        for i in range(len(self.probability_cumsum))])
        # Set a minimal resolution
        min_distance[min_distance<10] = 10

        self.linind = {}
        for i in range(len(self.probability_cumsum)):
            self.linind['self.probability_cumsum '+repr(i)] = np.linspace(0,1,min_distance[i])[0:-1]

        for elem in [self.firstLenAng_cumsum, self.trajLen_cumsum]:
            self.linind[elem] = np.linspace(0, 1, 1/min((np.unique((elem))-np.roll(np.unique((elem)),1))[1:]))[0:-1]

Response: Initializes the data for the given object.


Instruction:
Make a summary of the following Python 3 code
def _calc_xy(self, xxx_todo_changeme, angle, length):
        (x, y) = xxx_todo_changeme
        return (x+(cos(radians(angle))*length),
                y+(sin(radians(angle))*length))

Response: Calculates the coordinates after a specific saccade was made.


Instruction:
Explain what the following Python 3 code does
def _draw(self, prev_angle = None, prev_length = None):

        if (prev_angle is None) or (prev_length is None):
            (length, angle)= np.unravel_index(self.drawFrom('self.firstLenAng_cumsum', self.getrand('self.firstLenAng_cumsum')),
                                                self.firstLenAng_shape)
            angle = angle-((self.firstLenAng_shape[1]-1)/2)
            angle += 0.5
            length += 0.5
            length *= self.fm.pixels_per_degree
        else:
            ind = int(floor(prev_length/self.fm.pixels_per_degree))
            while ind >= len(self.probability_cumsum):
                ind -= 1

            while not(self.probability_cumsum[ind]).any():
                ind -= 1

            J, I = np.unravel_index(self.drawFrom('self.probability_cumsum '+repr(ind),self.getrand('self.probability_cumsum '+repr(ind))),
                                    self.full_H1[ind].shape)
            angle = reshift((I-self.full_H1[ind].shape[1]/2) + prev_angle)
            angle += 0.5
            length = J+0.5
            length *= self.fm.pixels_per_degree
        return angle, length

Response: Draw a new length - and angle - difference pair and calculates the last saccade drawn.


Instruction:
How would you explain what the following Python 3 function does
def sample_many(self, num_samples = 2000):
        x = []
        y = []
        fix = []
        sample = []

        # XXX: Delete ProgressBar
        pbar = ProgressBar(widgets=[Percentage(),Bar()], maxval=num_samples).start()

        for s in range(0, num_samples):
            for i, (xs, ys) in enumerate(self.sample()):
                x.append(xs)
                y.append(ys)
                fix.append(i+1)
                sample.append(s)
            pbar.update(s+1)

        fields = {'fix':np.array(fix), 'y':np.array(y), 'x':np.array(x)}
        param = {'pixels_per_degree':self.fm.pixels_per_degree}
        out =  fixmat.VectorFixmatFactory(fields, param)
        return out

Response: Generates a given number of trajectories using the method sample.


Instruction:
Explain what the following Python 3 code does
def sample(self):
        lenghts = []
        angles = []
        coordinates = []
        fix = []
        sample_size = int(round(self.trajLen_borders[self.drawFrom('self.trajLen_cumsum', self.getrand('self.trajLen_cumsum'))]))

        coordinates.append([0, 0])
        fix.append(1)

        while len(coordinates) < sample_size:
            if len(lenghts) == 0 and len(angles) == 0:
                angle, length = self._draw(self)
            else:
                angle, length = self._draw(prev_angle = angles[-1],
                                            prev_length = lenghts[-1])

            x, y = self._calc_xy(coordinates[-1], angle, length)

            coordinates.append([x, y])
            lenghts.append(length)
            angles.append(angle)
            fix.append(fix[-1]+1)
        return coordinates

Response: Draw a trajectory length first coordinates lengths angles and length - angle - difference pairs according to the empirical distribution. Each call creates one complete trajectory.


Instruction:
How would you implement a function in Python 3 that
draws a value from a cumulative sum.

Response:
def drawFrom(self, cumsum, r):
        """
        Draws a value from a cumulative sum.

        Parameters:
            cumsum : array
                Cumulative sum from which shall be drawn.

        Returns:
            int : Index of the cumulative sum element drawn.
        """
        a = cumsum.rsplit()
        if len(a)>1:
            b = eval(a[0])[int(a[1])]
        else:
            b = eval(a[0])

        return np.nonzero(b>=r)[0][0]


Instruction:
Can you create a Python 3 function that
loads a fixmat from a file.

Response:
def load(path):
    """
    Load fixmat at path.

    Parameters:
        path : string
            Absolute path of the file to load from.
    """
    f = h5py.File(path,'r')
    if 'Fixmat' in f:
      fm_group = f['Fixmat']
    else:
      fm_group = f['Datamat']
    fields = {}
    params = {}
    for field, value in list(fm_group.items()):
        fields[field] = np.array(value)
    for param, value in list(fm_group.attrs.items()):
        params[param] = value
    f.close()
    return VectorFixmatFactory(fields, params)


Instruction:
Can you write a function in Python 3 where it
computes the fixation density map for the calling fixmat.

Response:
def compute_fdm(fixmat, fwhm=2, scale_factor=1):
    """
    Computes a fixation density map for the calling fixmat.

    Creates a map the size of the image fixations were recorded on.
    Every pixel contains the frequency of fixations
    for this image. The fixation map is smoothed by convolution with a
    Gaussian kernel to approximate the area with highest processing
    (usually 2 deg. visual angle).

    Note: The function does not check whether the fixmat contains
    fixations from different images as it might be desirable to compute
    fdms over fixations from more than one image.

    Parameters:
        fwhm :  float
            the full width at half maximum of the Gaussian kernel used
            for convolution of the fixation frequency map.

        scale_factor : float
            scale factor for the resulting fdm. Default is 1. Scale_factor
            must be a float specifying the fraction of the current size.

    Returns:
        fdm  : numpy.array
            a numpy.array of size fixmat.image_size containing
            the fixation probability for every location on the image.
    """
    # image category must exist (>-1) and image_size must be non-empty
    assert (len(fixmat.image_size) == 2 and (fixmat.image_size[0] > 0) and
        (fixmat.image_size[1] > 0)), 'The image_size is either 0, or not 2D'
    # check whether fixmat contains fixations
    if fixmat._num_fix == 0 or len(fixmat.x) == 0 or len(fixmat.y) == 0 :
        raise RuntimeError('There are no fixations in the fixmat.')
    assert not scale_factor <= 0, "scale_factor has to be > 0"
    # this specifies left edges of the histogram bins, i.e. fixations between
    # ]0 binedge[0]] are included. --> fixations are ceiled
    e_y = np.arange(0, np.round(scale_factor*fixmat.image_size[0]+1))
    e_x = np.arange(0, np.round(scale_factor*fixmat.image_size[1]+1))
    samples = np.array(list(zip((scale_factor*fixmat.y), (scale_factor*fixmat.x))))
    (hist, _) = np.histogramdd(samples, (e_y, e_x))
    kernel_sigma = fwhm * fixmat.pixels_per_degree * scale_factor
    kernel_sigma = kernel_sigma / (2 * (2 * np.log(2)) ** .5)
    fdm = gaussian_filter(hist, kernel_sigma, order=0, mode='constant')
    return fdm / fdm.sum()


Instruction:
How would you explain what the following Python 3 function does
def relative_bias(fm,  scale_factor = 1, estimator = None):
    assert 'fix' in fm.fieldnames(), "Can not work without fixation  numbers"
    excl = fm.fix - np.roll(fm.fix, 1) != 1

    # Now calculate the direction where the NEXT fixation goes to
    diff_x = (np.roll(fm.x, 1) - fm.x)[~excl]
    diff_y = (np.roll(fm.y, 1) - fm.y)[~excl]

    # Make a histogram of diff values
    # this specifies left edges of the histogram bins, i.e. fixations between
    # ]0 binedge[0]] are included. --> fixations are ceiled
    ylim =  np.round(scale_factor * fm.image_size[0])
    xlim =  np.round(scale_factor * fm.image_size[1])
    x_steps = np.ceil(2*xlim) +1
    if x_steps % 2 != 0: x_steps+=1
    y_steps = np.ceil(2*ylim)+1
    if y_steps % 2 != 0: y_steps+=1
    e_x = np.linspace(-xlim,xlim,x_steps)
    e_y = np.linspace(-ylim,ylim,y_steps)

    #e_y = np.arange(-ylim, ylim+1)
    #e_x = np.arange(-xlim, xlim+1)
    samples = np.array(list(zip((scale_factor * diff_y),
                             (scale_factor* diff_x))))
    if estimator == None:
        (hist, _) = np.histogramdd(samples, (e_y, e_x))
    else:
        hist = estimator(samples, e_y, e_x)
    return hist

Response: Computes the relative bias of a single class hierarchy.


Instruction:
Can you tell what is the following Python 3 function doing
def DirectoryFixmatFactory(directory, categories = None, glob_str = '*.mat', var_name = 'fixmat'):
    files = glob(join(directory,glob_str))
    if len(files) == 0:
        raise ValueError("Could not find any fixmats in " +
            join(directory, glob_str))
    f_all = FixmatFactory(files.pop(), categories, var_name)
    for fname in files:
        f_current = FixmatFactory(fname, categories, var_name)
        f_all.join(f_current)
    return f_all

Response: Returns a new fixmat object that contains all fixmats in a directory.


Instruction:
Given the following Python 3 function, write the documentation
def FixmatFactory(fixmatfile, categories = None, var_name = 'fixmat', field_name='x'):
    try:
        data = loadmat(fixmatfile, struct_as_record = False)
        keys = list(data.keys())
        data = data[var_name][0][0]
    except KeyError:
        raise RuntimeError('%s is not a field of the matlab structure. Possible'+
                'Keys are %s'%str(keys))

    num_fix = data.__getattribute__(field_name).size

    # Get a list with fieldnames and a list with parameters
    fields = {}
    parameters = {}
    for field in data._fieldnames:
        if data.__getattribute__(field).size == num_fix:
            fields[field] = data.__getattribute__(field)
        else:
            parameters[field] = data.__getattribute__(field)[0].tolist()
            if len(parameters[field]) == 1:
                parameters[field] = parameters[field][0]

    # Generate FixMat
    fixmat = FixMat(categories = categories)
    fixmat._fields = list(fields.keys())
    for (field, value) in list(fields.items()):
        fixmat.__dict__[field] = value.reshape(-1,)

    fixmat._parameters = parameters
    fixmat._subjects = None
    for (field, value) in list(parameters.items()):
        fixmat.__dict__[field] = value
    fixmat._num_fix = num_fix
    return fixmat

Response: This function creates a new fixmat object from a matlab fixmat file.


Instruction:
Given the following Python 3 function, write the documentation
def add_feature_values(self, features):
        if not 'x' in self.fieldnames():
            raise RuntimeError("""add_feature_values expects to find
        (x,y) locations in self.x and self.y. But self.x does not exist""")

        if not self._categories:
            raise RuntimeError(
            '''"%s" does not exist as a fieldname and the
            fixmat does not have a Categories object (no features
            available. The fixmat has these fields: %s''' \
            %(features, str(self._fields)))
        for feature in features:
            # initialize new field with NaNs
            feat_vals = np.zeros([len(self.x)]) * np.nan
            for (cat_mat, imgs) in self.by_cat():
                for img in np.unique(cat_mat.filenumber).astype(int):
                    fmap = imgs[img][feature]
                    on_image = (self.x >= 0) & (self.x <= self.image_size[1])
                    on_image = on_image & (self.y >= 0) & (self.y <= self.image_size[0])
                    idx = (self.category == imgs.category) & \
                          (self.filenumber == img) & \
                          (on_image.astype('bool'))
                    feat_vals[idx] = fmap[self.y[idx].astype('int'),
                        self.x[idx].astype('int')]
            # setattr(self, feature, feat_vals)
            self.add_field(feature, feat_vals)

Response:
This function adds feature values of feature to all fixations in
 .


Instruction:
Write a Python 3 function for
generating two M x N matrices with M feature values at fixations for N features. Controls are a random sample out of all non-fixated regions of an image or fixations of the same subject group on a randomly chosen image. Fixations are pooled over all subjects in the calling fixmat. Parameters : all_controls : bool if True, all non-fixated points on a feature map are takes as control values. If False, controls are fixations from the same subjects but on one other randomly chosen image of the same category feature_list : list of strings contains names of all features that are used to generate the feature value matrix (--> number of dimensions in the model). ...note: this list has to be sorted ! Returns : N x M matrix of N control feature values per feature (M). Rows = Feature number /type Columns = Feature values

Response:
def make_reg_data(self, feature_list=None, all_controls=False):
        """
        Generates two M x N matrices with M feature values at fixations for
        N features. Controls are a random sample out of all non-fixated regions
        of an image or fixations of the same subject group on a randomly chosen
        image. Fixations are pooled over all subjects in the calling fixmat.

        Parameters :
            all_controls : bool
                if True, all non-fixated points on a feature map are takes as
                control values. If False, controls are fixations from the same
                subjects but on one other randomly chosen image of the same
                category
            feature_list : list of strings
                contains names of all features that are used to generate
                the feature value matrix (--> number of dimensions in the
                model).
                ...note: this list has to be sorted !

        Returns :
            N x M matrix of N control feature values per feature (M).
            Rows = Feature number /type
            Columns = Feature values
        """
        if not 'x' in self.fieldnames():
            raise RuntimeError("""make_reg_data expects to find
        (x,y) locations in self.x and self.y. But self.x does not exist""")

        on_image = (self.x >= 0) & (self.x <= self.image_size[1])
        on_image = on_image & (self.y >= 0) & (self.y <= self.image_size[0])
        assert on_image.all(), "All Fixations need to be on the image"
        assert len(np.unique(self.filenumber) > 1), "Fixmat has to have more than one filenumber"
        self.x = self.x.astype(int)
        self.y = self.y.astype(int)

        if feature_list == None:
            feature_list = np.sort(self._categories._features)
        all_act = np.zeros((len(feature_list), 1)) * np.nan
        all_ctrls = all_act.copy()

        for (cfm, imgs) in self.by_cat():
            # make a list of all filenumbers in this category and then
            # choose one random filenumber without replacement
            imfiles = np.array(imgs.images()) # array makes a copy of the list
            ctrl_imgs = imfiles.copy()
            np.random.shuffle(ctrl_imgs)
            while (imfiles == ctrl_imgs).any():
                np.random.shuffle(ctrl_imgs)
            for (imidx, img) in enumerate(imfiles):
                xact = cfm.x[cfm.filenumber == img]
                yact = cfm.y[cfm.filenumber == img]
                if all_controls:
                # take a sample the same length as the actuals out of every
                # non-fixated point in the feature map
                    idx = np.ones(self.image_size)
                    idx[cfm.y[cfm.filenumber == img],
                        cfm.x[cfm.filenumber == img]] = 0
                    yctrl, xctrl = idx.nonzero()
                    idx = np.random.randint(0, len(yctrl), len(xact))
                    yctrl = yctrl[idx]
                    xctrl = xctrl[idx]
                    del idx
                else:
                    xctrl = cfm.x[cfm.filenumber == ctrl_imgs[imidx]]
                    yctrl = cfm.y[cfm.filenumber == ctrl_imgs[imidx]]
                # initialize arrays for this filenumber
                actuals = np.zeros((1, len(xact))) * np.nan
                controls = np.zeros((1, len(xctrl))) * np.nan

                for feature in feature_list:
                    # get the feature map
                    fmap = imgs[img][feature]
                    actuals = np.vstack((actuals, fmap[yact, xact]))
                    controls = np.vstack((controls, fmap[yctrl, xctrl]))
                all_act = np.hstack((all_act, actuals[1:, :]))
                all_ctrls = np.hstack((all_ctrls, controls[1:, :]))
        return (all_act[:, 1:], all_ctrls[:, 1:])


Instruction:
Write a Python 3 script to
compute velocity of eye - movements.

Response:
def get_velocity(samplemat, Hz, blinks=None):
    '''
    Compute velocity of eye-movements.

    Samplemat must contain fields 'x' and 'y', specifying the x,y coordinates
    of gaze location. The function assumes that the values in x,y are sampled
    continously at a rate specified by 'Hz'.
    '''
    Hz = float(Hz)
    distance = ((np.diff(samplemat.x) ** 2) +
                (np.diff(samplemat.y) ** 2)) ** .5
    distance = np.hstack(([distance[0]], distance))
    if blinks is not None:
        distance[blinks[1:]] = np.nan
    win = np.ones((velocity_window_size)) / float(velocity_window_size)
    velocity = np.convolve(distance, win, mode='same')
    velocity = velocity / (velocity_window_size / Hz)
    acceleration = np.diff(velocity) / (1. / Hz)
    acceleration = abs(np.hstack(([acceleration[0]], acceleration)))
    return velocity, acceleration


Instruction:
Can you write a function in Python 3 where it
detects saccades in a stream of gaze location samples.

Response:
def saccade_detection(samplemat, Hz=200, threshold=30,
                      acc_thresh=2000, min_duration=21, min_movement=.35,
                      ignore_blinks=False):
    '''
    Detect saccades in a stream of gaze location samples.

    Coordinates in samplemat are assumed to be in degrees.

    Saccades are detect by a velocity/acceleration threshold approach.
    A saccade starts when a) the velocity is above threshold, b) the
    acceleration is above acc_thresh at least once during the interval
    defined by the velocity threshold, c) the saccade lasts at least min_duration
    ms and d) the distance between saccade start and enpoint is at least
    min_movement degrees.
    '''
    if ignore_blinks:
        velocity, acceleration = get_velocity(samplemat, float(Hz), blinks=samplemat.blinks)
    else:
        velocity, acceleration = get_velocity(samplemat, float(Hz))

    saccades = (velocity > threshold)
    #print velocity[samplemat.blinks[1:]]
    #print saccades[samplemat.blinks[1:]]

    borders = np.where(np.diff(saccades.astype(int)))[0] + 1
    if velocity[1] > threshold:
        borders = np.hstack(([0], borders))
    saccade = 0 * np.ones(samplemat.x.shape)

    # Only count saccades when acceleration also surpasses threshold
    for i, (start, end) in enumerate(zip(borders[0::2], borders[1::2])):
        if sum(acceleration[start:end] > acc_thresh) >= 1:
            saccade[start:end] = 1

    borders = np.where(np.diff(saccade.astype(int)))[0] + 1
    if saccade[0] == 0:
        borders = np.hstack(([0], borders))
    for i, (start, end) in enumerate(zip(borders[0::2], borders[1::2])):
        if (1000*(end - start) / float(Hz)) < (min_duration):
            saccade[start:end] = 1

    # Delete saccade between fixations that are too close together.
    dists_ok = False
    while not dists_ok:
        dists_ok = True
        num_merges = 0
        for i, (lfixstart, lfixend, start, end, nfixstart, nfixend) in enumerate(zip(
                borders[0::2], borders[1::2],
                borders[1::2], borders[2::2],
                borders[2::2], borders[3::2])):
            lastx = samplemat.x[lfixstart:lfixend].mean()
            lasty = samplemat.y[lfixstart:lfixend].mean()
            nextx = samplemat.x[nfixstart:nfixend].mean()
            nexty = samplemat.y[nfixstart:nfixend].mean()
            if (1000*(lfixend - lfixstart) / float(Hz)) < (min_duration):
                saccade[lfixstart:lfixend] = 1
                continue
            distance = ((nextx - lastx) ** 2 + (nexty - lasty) ** 2) ** .5
            if distance < min_movement:
                num_merges += 1
                dists_ok = False
                saccade[start:end] = 0
        borders = np.where(np.diff(saccade.astype(int)))[0] + 1
        if saccade[0] == 0:
            borders = np.hstack(([0], borders))
    return saccade.astype(bool)


Instruction:
Implement a function in Python 3 to
detect fixation from saccades.

Response:
def fixation_detection(samplemat, saccades, Hz=200, samples2fix=None,
                       respect_trial_borders=False, sample_times=None):
    '''
    Detect Fixation from saccades.

    Fixations are defined as intervals between saccades. This function
    also calcuates start and end times (in ms) for each fixation.
    Input:
        samplemat: datamat
            Contains the recorded samples and associated metadata.
        saccades: ndarray
            Logical vector that is True for samples that belong to a saccade.
        Hz: Float
            Number of samples per second.
        samples2fix: Dict
            There is usually metadata associated with the samples (e.g. the
            trial number). This dictionary can be used to specify how the
            metadata should be collapsed for one fixation. It contains
            field names from samplemat as keys and functions as values that
            return one value when they are called with all samples for one
            fixation. In addition the function can raise an 'InvalidFixation'
            exception to signal that the fixation should be discarded.
    '''
    if samples2fix is None:
        samples2fix = {}
    fixations = ~saccades
    acc = AccumulatorFactory()
    if not respect_trial_borders:
        borders = np.where(np.diff(fixations.astype(int)))[0] + 1
    else:
        borders = np.where(
            ~(np.diff(fixations.astype(int)) == 0) |
            ~(np.diff(samplemat.trial.astype(int)) == 0))[0] + 1

    fixations = 0 * saccades.copy()
    if not saccades[0]:
        borders = np.hstack(([0], borders))
    #lasts,laste = borders[0], borders[1]
    for i, (start, end) in enumerate(zip(borders[0::2], borders[1::2])):

        current = {}
        for k in samplemat.fieldnames():
            if k in list(samples2fix.keys()):
                current[k] = samples2fix[k](samplemat, k, start, end)
            else:
                current[k] = np.mean(samplemat.field(k)[start:end])
        current['start_sample'] = start
        current['end_sample'] = end
        fixations[start:end] = 1
        # Calculate start and end time in ms
        if sample_times is None:
            current['start'] = 1000 * start / Hz
            current['end'] = 1000 * end / Hz
        else:
            current['start'] = sample_times[start]
            current['end'] = sample_times[end]

        #lasts, laste = start,end
        acc.update(current)

    return acc.get_dm(params=samplemat.parameters()), fixations.astype(bool)


Instruction:
Make a summary of the following Python 3 code
def parse(parse_obj, agent=None, etag=None, modified=None, inject=False):

    guarantees = common.SuperDict({
        'bozo': 0,
        'feeds': [],
        'lists': [],
        'opportunities': [],
        'meta': common.SuperDict(),
        'version': '',
    })
    fileobj, info = _mkfile(parse_obj, (agent or USER_AGENT), etag, modified)
    guarantees.update(info)
    if not fileobj:
        return guarantees

    handler = Handler()
    handler.harvest.update(guarantees)
    parser = xml.sax.make_parser()
    parser.setFeature(xml.sax.handler.feature_namespaces, True)
    parser.setContentHandler(handler)
    parser.setErrorHandler(handler)
    if inject:
        fileobj = Injector(fileobj)
    try:
        parser.parse(fileobj)
    except (SAXParseException, MalformedByteSequenceException):  # noqa: E501  # pragma: no cover
        # Jython propagates exceptions past the ErrorHandler.
        err = sys.exc_info()[1]
        handler.harvest.bozo = 1
        handler.harvest.bozo_exception = err
    finally:
        fileobj.close()

    # Test if a DOCTYPE injection is needed
    if hasattr(handler.harvest, 'bozo_exception'):
        if 'entity' in handler.harvest.bozo_exception.__str__():
            if not inject:
                return parse(parse_obj, agent, etag, modified, True)
    # Make it clear that the XML file is broken
    # (if no other exception has been assigned)
    if inject and not handler.harvest.bozo:
        handler.harvest.bozo = 1
        handler.harvest.bozo_exception = ListError('undefined entity found')
    return handler.harvest

Response: Parse a subscription list and return a dict containing the results.


Instruction:
Here you have a function in Python 3, explain what it does
def FixmatStimuliFactory(fm, loader):
    # Find all feature names
    features = []
    if loader.ftrpath:
        assert os.access(loader.ftrpath, os.R_OK)
        features = os.listdir(os.path.join(loader.ftrpath, str(fm.category[0])))
    # Find all images in all categories
    img_per_cat = {}
    for cat in np.unique(fm.category):
        if not loader.test_for_category(cat):
            raise ValueError('Category %s is specified in fixmat but '%(
                                str(cat) + 'can not be located by loader'))
        img_per_cat[cat] = []
        for img in np.unique(fm[(fm.category == cat)].filenumber):
            if not loader.test_for_image(cat, img):
                raise ValueError('Image %s in category %s is '%(str(cat),
                    str(img)) +
                    'specified in fixmat but can be located by loader')
            img_per_cat[cat].append(img)
            if loader.ftrpath:
                for feature in features:
                    if not loader.test_for_feature(cat, img, feature):
                        raise RuntimeError(
                            'Feature %s for image %s' %(str(feature),str(img)) +
                            ' in category %s ' %str(cat) +
                            'can not be located by loader')
    return Categories(loader, img_per_cat = img_per_cat,
         features = features, fixations = fm)

Response: Returns a new object for all image and category combinations in a fixmat.


Instruction:
Can you tell what is the following Python 3 function doing
def DirectoryStimuliFactory(loader):
    impath = loader.impath
    ftrpath = loader.ftrpath
    # checks whether user has reading permission for the path
    assert os.access(impath, os.R_OK)
    assert os.access(ftrpath, os.R_OK)

    # EXTRACTING IMAGE NAMES
    img_per_cat = {}
    # extract only directories in the given folder
    subfolders = [name for name in os.listdir(impath) if os.path.isdir(
        os.path.join(impath, name))]
    # if there are no subfolders, walk through files. Take 1 as key for the
    # categories object
    if not subfolders:
        [_, _, files] = next(os.walk(os.path.join(impath)))
        # this only takes entries that end with '.png'
        entries = {1:
            [int(cur_file[cur_file.find('_')+1:-4]) for cur_file
            in files if cur_file.endswith('.png')]}
        img_per_cat.update(entries)
        subfolders = ['']
    # if there are subfolders, walk through them
    else:
        for directory in subfolders:
            [_, _, files] = next(os.walk(os.path.join(impath, directory)))
            # this only takes entries that end with '.png'. Strips ending and
            # considers everything after the first '_' as the imagenumber
            imagenumbers = [int(cur_file[cur_file.find('_')+1:-4])
                    for cur_file in files
                        if (cur_file.endswith('.png') & (len(cur_file) > 4))]
            entries = {int(directory): imagenumbers}
            img_per_cat.update(entries)
            del directory
    del imagenumbers

    # in case subfolders do not exist, '' is appended here.
    _, features, files = next(os.walk(os.path.join(ftrpath,
                                            subfolders[0])))
    return Categories(loader, img_per_cat = img_per_cat, features = features)

Response: This function creates a directory - independent structure for the given folder - and image - files.


Instruction:
Can you tell what is the following Python 3 function doing
def fixations(self):
        ''' Filter the fixmat such that it only contains fixations on images
        in categories that are also in the categories object'''
        if not self._fixations:
            raise RuntimeError('This Images object does not have'
                +' an associated fixmat')
        if len(list(self._categories.keys())) == 0:
            return None
        else:
            idx = np.zeros(self._fixations.x.shape, dtype='bool')
            for (cat, _) in list(self._categories.items()):
                idx = idx | ((self._fixations.category == cat))
            return self._fixations[idx]

Response:
Filter the fixmat such that it only contains fixations on images
        in categories that are also in the categories object.


Instruction:
Make a summary of the following Python 3 code
def data(self, value):
        self.loader.save_image(self.category, self.image, value)

Response: Saves a new image to disk