Instruction:
Can you implement a function in Python 3 that
returns the memory usage of a process or piece of code.

Response:
def memory_usage(proc=-1, interval=.1, timeout=None, timestamps=False,
                 include_children=False, max_usage=False, retval=False,
                 stream=None):
    """
    Return the memory usage of a process or piece of code

    Parameters
    ----------
    proc : {int, string, tuple, subprocess.Popen}, optional
        The process to monitor. Can be given by an integer/string
        representing a PID, by a Popen object or by a tuple
        representing a Python function. The tuple contains three
        values (f, args, kw) and specifies to run the function
        f(*args, **kw).
        Set to -1 (default) for current process.

    interval : float, optional
        Interval at which measurements are collected.

    timeout : float, optional
        Maximum amount of time (in seconds) to wait before returning.

    max_usage : bool, optional
        Only return the maximum memory usage (default False)

    retval : bool, optional
        For profiling python functions. Save the return value of the profiled
        function. Return value of memory_usage becomes a tuple:
        (mem_usage, retval)

    timestamps : bool, optional
        if True, timestamps of memory usage measurement are collected as well.

    stream : File
        if stream is a File opened with write access, then results are written
        to this file instead of stored in memory and returned at the end of
        the subprocess. Useful for long-running processes.
        Implies timestamps=True.

    Returns
    -------
    mem_usage : list of floating-poing values
        memory usage, in MiB. It's length is always < timeout / interval
        if max_usage is given, returns the two elements maximum memory and
        number of measurements effectuated
    ret : return value of the profiled function
        Only returned if retval is set to True
    """
    if stream is not None:
        timestamps = True

    if not max_usage:
        ret = []
    else:
        ret = -1

    if timeout is not None:
        max_iter = int(timeout / interval)
    elif isinstance(proc, int):
        # external process and no timeout
        max_iter = 1
    else:
        # for a Python function wait until it finishes
        max_iter = float('inf')

    if hasattr(proc, '__call__'):
        proc = (proc, (), {})
    if isinstance(proc, (list, tuple)):
        if len(proc) == 1:
            f, args, kw = (proc[0], (), {})
        elif len(proc) == 2:
            f, args, kw = (proc[0], proc[1], {})
        elif len(proc) == 3:
            f, args, kw = (proc[0], proc[1], proc[2])
        else:
            raise ValueError

        while True:
            child_conn, parent_conn = Pipe()  # this will store MemTimer's results
            p = MemTimer(os.getpid(), interval, child_conn, timestamps=timestamps,
                      max_usage=max_usage, include_children=include_children)
            p.start()
            parent_conn.recv()  # wait until we start getting memory
            returned = f(*args, **kw)
            parent_conn.send(0)  # finish timing
            ret = parent_conn.recv()
            n_measurements = parent_conn.recv()
            if retval:
                ret = ret, returned
            p.join(5 * interval)
            if n_measurements > 4 or interval < 1e-6:
                break
            interval /= 10.
    elif isinstance(proc, subprocess.Popen):
        # external process, launched from Python
        line_count = 0
        while True:
            if not max_usage:
                mem_usage = _get_memory(proc.pid, timestamps=timestamps,
                                        include_children=include_children)
                if stream is not None:
                    stream.write("MEM {0:.6f} {1:.4f}\n".format(*mem_usage))
                else:
                    ret.append(mem_usage)
            else:
                ret = max([ret,
                           _get_memory(proc.pid,
                                       include_children=include_children)])
            time.sleep(interval)
            line_count += 1
            # flush every 50 lines. Make 'tail -f' usable on profile file
            if line_count > 50:
                line_count = 0
                if stream is not None:
                    stream.flush()
            if timeout is not None:
                max_iter -= 1
                if max_iter == 0:
                    break
            if proc.poll() is not None:
                break
    else:
        # external process
        if max_iter == -1:
            max_iter = 1
        counter = 0
        while counter < max_iter:
            counter += 1
            if not max_usage:
                mem_usage = _get_memory(proc, timestamps=timestamps,
                                        include_children=include_children)
                if stream is not None:
                    stream.write("MEM {0:.6f} {1:.4f}\n".format(*mem_usage))
                else:
                    ret.append(mem_usage)
            else:
                ret = max([ret,
                           _get_memory(proc, include_children=include_children)
                           ])

            time.sleep(interval)
            # Flush every 50 lines.
            if counter % 50 == 0 and stream is not None:
                stream.flush()
    if stream:
        return None
    return ret


Instruction:
Make a summary of the following Python 3 code
def _find_script(script_name):
    if os.path.isfile(script_name):
        return script_name
    path = os.getenv('PATH', os.defpath).split(os.pathsep)
    for folder in path:
        if not folder:
            continue
        fn = os.path.join(folder, script_name)
        if os.path.isfile(fn):
            return fn

    sys.stderr.write('Could not find script {0}\n'.format(script_name))
    raise SystemExit(1)

Response: Find the script in the system path.


Instruction:
Can you tell what is the following Python 3 function doing
def magic_mprun(self, parameter_s=''):
    try:
        from StringIO import StringIO
    except ImportError:  # Python 3.x
        from io import StringIO

    # Local imports to avoid hard dependency.
    from distutils.version import LooseVersion
    import IPython
    ipython_version = LooseVersion(IPython.__version__)
    if ipython_version < '0.11':
        from IPython.genutils import page
        from IPython.ipstruct import Struct
        from IPython.ipapi import UsageError
    else:
        from IPython.core.page import page
        from IPython.utils.ipstruct import Struct
        from IPython.core.error import UsageError

    # Escape quote markers.
    opts_def = Struct(T=[''], f=[])
    parameter_s = parameter_s.replace('"', r'\"').replace("'", r"\'")
    opts, arg_str = self.parse_options(parameter_s, 'rf:T:c', list_all=True)
    opts.merge(opts_def)
    global_ns = self.shell.user_global_ns
    local_ns = self.shell.user_ns

    # Get the requested functions.
    funcs = []
    for name in opts.f:
        try:
            funcs.append(eval(name, global_ns, local_ns))
        except Exception as e:
            raise UsageError('Could not find function %r.\n%s: %s' % (name,
                             e.__class__.__name__, e))

    include_children = 'c' in opts
    profile = LineProfiler(include_children=include_children)
    for func in funcs:
        profile(func)

    # Add the profiler to the builtins for @profile.
    try:
        import builtins
    except ImportError:  # Python 3x
        import __builtin__ as builtins

    if 'profile' in builtins.__dict__:
        had_profile = True
        old_profile = builtins.__dict__['profile']
    else:
        had_profile = False
        old_profile = None
    builtins.__dict__['profile'] = profile

    try:
        try:
            profile.runctx(arg_str, global_ns, local_ns)
            message = ''
        except SystemExit:
            message = "*** SystemExit exception caught in code being profiled."
        except KeyboardInterrupt:
            message = ("*** KeyboardInterrupt exception caught in code being "
                       "profiled.")
    finally:
        if had_profile:
            builtins.__dict__['profile'] = old_profile

    # Trap text output.
    stdout_trap = StringIO()
    show_results(profile, stdout_trap)
    output = stdout_trap.getvalue()
    output = output.rstrip()

    if ipython_version < '0.11':
        page(output, screen_lines=self.shell.rc.screen_length)
    else:
        page(output)
    print(message,)

    text_file = opts.T[0]
    if text_file:
        with open(text_file, 'w') as pfile:
            pfile.write(output)
        print('\n*** Profile printout saved to text file %s. %s' % (text_file,
                                                                    message))

    return_value = None
    if 'r' in opts:
        return_value = profile

    return return_value

Response: Execute a magic statement under the line - by - line memory profiler module.


Instruction:
Can you write a function in Python 3 where it
measures memory usage of a Python statement.

Response:
def magic_memit(self, line=''):
    """Measure memory usage of a Python statement

    Usage, in line mode:
      %memit [-r<R>t<T>i<I>] statement

    Options:
    -r<R>: repeat the loop iteration <R> times and take the best result.
    Default: 1

    -t<T>: timeout after <T> seconds. Default: None

    -i<I>: Get time information at an interval of I times per second.
        Defaults to 0.1 so that there is ten measurements per second.

    -c: If present, add the memory usage of any children process to the report.

    Examples
    --------
    ::

      In [1]: import numpy as np

      In [2]: %memit np.zeros(1e7)
      maximum of 1: 76.402344 MiB per loop

      In [3]: %memit np.ones(1e6)
      maximum of 1: 7.820312 MiB per loop

      In [4]: %memit -r 10 np.empty(1e8)
      maximum of 10: 0.101562 MiB per loop

    """
    opts, stmt = self.parse_options(line, 'r:t:i:c', posix=False, strict=False)
    repeat = int(getattr(opts, 'r', 1))
    if repeat < 1:
        repeat == 1
    timeout = int(getattr(opts, 't', 0))
    if timeout <= 0:
        timeout = None
    interval = float(getattr(opts, 'i', 0.1))
    include_children = 'c' in opts

    # I've noticed we get less noisier measurements if we run
    # a garbage collection first
    import gc
    gc.collect()

    mem_usage = 0
    counter = 0
    baseline = memory_usage()[0]
    while counter < repeat:
        counter += 1
        tmp = memory_usage((_func_exec, (stmt, self.shell.user_ns)),
                           timeout=timeout, interval=interval, max_usage=True,
                           include_children=include_children)
        mem_usage = max(mem_usage, tmp[0])

    if mem_usage:
        print('peak memory: %.02f MiB, increment: %.02f MiB' %
              (mem_usage, mem_usage - baseline))
    else:
        print('ERROR: could not read memory usage, try with a lower interval '
              'or more iterations')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def timestamp(self, name="<block>"):
        # Make a fake function
        func = lambda x: x
        func.__module__ = ""
        func.__name__ = name
        self.add_function(func)
        timestamps = []
        self.functions[func].append(timestamps)
        # A new object is required each time, since there can be several
        # nested context managers.
        return _TimeStamperCM(timestamps)

Response: Returns a context manager for timestamping a block of code.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def wrap_function(self, func):
        def f(*args, **kwds):
            # Start time
            timestamps = [_get_memory(os.getpid(), timestamps=True)]
            self.functions[func].append(timestamps)
            try:
                result = func(*args, **kwds)
            finally:
                # end time
                timestamps.append(_get_memory(os.getpid(), timestamps=True))
            return result
        return f

Response: Wrap a function to timestamp it.


Instruction:
Write a Python 3 script for
adding a Python function to the line profiling information.

Response:
def add_function(self, func):
        """ Record line profiling information for the given Python function.
        """
        try:
            # func_code does not exist in Python3
            code = func.__code__
        except AttributeError:
            warnings.warn("Could not extract a code object for the object %r"
                          % func)
        else:
            self.add_code(code)


Instruction:
Can you generate the documentation for the following Python 3 function
def run(self, cmd):
        # TODO: can this be removed ?
        import __main__
        main_dict = __main__.__dict__
        return self.runctx(cmd, main_dict, main_dict)

Response: Profile a single executable statement in the main namespace.


Instruction:
Given the following Python 3 function, write the documentation
def trace_memory_usage(self, frame, event, arg):
        if (event in ('call', 'line', 'return')
                and frame.f_code in self.code_map):
            if event != 'call':
                # "call" event just saves the lineno but not the memory
                mem = _get_memory(-1, include_children=self.include_children)
                # if there is already a measurement for that line get the max
                old_mem = self.code_map[frame.f_code].get(self.prevline, 0)
                self.code_map[frame.f_code][self.prevline] = max(mem, old_mem)
            self.prevline = frame.f_lineno

        if self._original_trace_function is not None:
            (self._original_trace_function)(frame, event, arg)

        return self.trace_memory_usage

Response: Callback for sys. settrace


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_root( self, key ):
        if key not in self.roots:
            function = getattr( self, 'load_%s'%(key,) )()
            self.roots[key] = function
        return self.roots[key]

Response: Retrieve a given declared root by root - type - key


Instruction:
Write a Python 3 script to
get the set of rows for the type - key

Response:
def get_rows( self, key ):
        """Get the set of rows for the type-key"""
        if key not in self.roots:
            self.get_root( key )
        if key == 'location':
            return self.location_rows
        else:
            return self.rows


Instruction:
Implement a Python 3 function for
building a squaremap - compatible model from a pstats class

Response:
def load( self, stats ):
        """Build a squaremap-compatible model from a pstats class"""
        rows = self.rows
        for func, raw in stats.iteritems():
            try:
                rows[func] = row = PStatRow( func,raw )
            except ValueError, err:
                log.info( 'Null row: %s', func )
        for row in rows.itervalues():
            row.weave( rows )
        return self.find_root( rows )


Instruction:
Can you tell what is the following Python 3 function doing
def find_root( self, rows ):
        maxes = sorted( rows.values(), key = lambda x: x.cumulative )
        if not maxes:
            raise RuntimeError( """Null results!""" )
        root = maxes[-1]
        roots = [root]
        for key,value in rows.items():
            if not value.parents:
                log.debug( 'Found node root: %s', value )
                if value not in roots:
                    roots.append( value )
        if len(roots) > 1:
            root = PStatGroup(
                directory='*',
                filename='*',
                name=_("<profiling run>"),
                children= roots,
            )
            root.finalize()
            self.rows[ root.key ] = root
        self.roots['functions'] = root
        return root

Response: Attempt to find a reasonable root node from a list of rows.


Instruction:
Can you tell what is the following Python 3 function doing
def _load_location( self ):
        directories = {}
        files = {}
        root = PStatLocation( '/', 'PYTHONPATH' )
        self.location_rows = self.rows.copy()
        for child in self.rows.values():
            current = directories.get( child.directory )
            directory, filename = child.directory, child.filename
            if current is None:
                if directory == '':
                    current = root
                else:
                    current = PStatLocation( directory, '' )
                    self.location_rows[ current.key ] = current
                directories[ directory ] = current
            if filename == '~':
                filename = '<built-in>'
            file_current = files.get( (directory,filename) )
            if file_current is None:
                file_current = PStatLocation( directory, filename )
                self.location_rows[ file_current.key ] = file_current
                files[ (directory,filename) ] = file_current
                current.children.append( file_current )
            file_current.children.append( child )
        # now link the directories...
        for key,value in directories.items():
            if value is root:
                continue
            found = False
            while key:
                new_key,rest = os.path.split( key )
                if new_key == key:
                    break
                key = new_key
                parent = directories.get( key )
                if parent:
                    if value is not parent:
                        parent.children.append( value )
                        found = True
                        break
            if not found:
                root.children.append( value )
        # lastly, finalize all of the directory records...
        root.finalize()
        return root

Response: Build a squaremap - compatible model for location - based hierarchy of directories and files.


Instruction:
Given the following Python 3 function, write the documentation
def finalize( self, already_done=None ):
        if already_done is None:
            already_done = {}
        if already_done.has_key( self ):
            return True
        already_done[self] = True
        self.filter_children()
        children = self.children
        for child in children:
            if hasattr( child, 'finalize' ):
                child.finalize( already_done)
            child.parents.append( self )
        self.calculate_totals( self.children, self.local_children )

Response: Finalize our values taken from our children


Instruction:
Make a summary of the following Python 3 code
def calculate_totals( self, children, local_children=None ):
        for field,local_field in (('recursive','calls'),('cumulative','local')):
            values = []
            for child in children:
                if isinstance( child, PStatGroup ) or not self.LOCAL_ONLY:
                    values.append( getattr( child, field, 0 ) )
                elif isinstance( child, PStatRow ) and self.LOCAL_ONLY:
                    values.append( getattr( child, local_field, 0 ) )
            value = sum( values )
            setattr( self, field, value )
        if self.recursive:
            self.cumulativePer = self.cumulative/float(self.recursive)
        else:
            self.recursive = 0
        if local_children:
            for field in ('local','calls'):
                value = sum([ getattr( child, field, 0 ) for child in children] )
                setattr( self, field, value )
            if self.calls:
                self.localPer = self.local / self.calls
        else:
            self.local = 0
            self.calls = 0
            self.localPer = 0

Response: Calculate our cumulative totals from children and local children


Instruction:
Implement a Python 3 function for
filtering our children into regular and local children sets

Response:
def filter_children( self ):
        """Filter our children into regular and local children sets"""
        real_children = []
        for child in self.children:
            if child.name == '<module>':
                self.local_children.append( child )
            else:
                real_children.append( child )
        self.children = real_children


Instruction:
Create a Python 3 function to
return set of two boxes where first is the fraction given

Response:
def split_box( fraction, x,y, w,h ):
    """Return set of two boxes where first is the fraction given"""
    if w >= h:
        new_w = int(w*fraction)
        if new_w:
            return (x,y,new_w,h),(x+new_w,y,w-new_w,h)
        else:
            return None,None
    else:
        new_h = int(h*fraction)
        if new_h:
            return (x,y,w,new_h),(x,y+new_h,w,h-new_h)
        else:
            return None,None


Instruction:
Explain what the following Python 3 code does
def split_by_value( total, nodes, headdivisor=2.0 ):
    head_sum,tail_sum = 0,0
    divider = 0
    for node in nodes[::-1]:
        if head_sum < total/headdivisor:
            head_sum += node[0]
            divider -= 1
        else:
            break
    return (head_sum,nodes[divider:]),(total-head_sum,nodes[:divider])

Response: Produce a tuple of head and tail for a binary partition of the total number of nodes.


Instruction:
Given the following Python 3 function, write the documentation
def findNode(class_, hot_map, targetNode, parentNode=None):
        ''' Find the target node in the hot_map. '''
        for index, (rect, node, children) in enumerate(hot_map):
            if node == targetNode:
                return parentNode, hot_map, index
            result = class_.findNode(children, targetNode, node)
            if result:
                return result
        return None

Response: Find the target node in the hot_map.


Instruction:
Explain what the following Python 3 code does
def firstChild(hot_map, index):
        ''' Return the first child of the node indicated by index. '''
        children = hot_map[index][2]
        if children:
            return children[0][1]
        else:
            return hot_map[index][1]

Response: Return the first child of the node indicated by index.


Instruction:
How would you explain what the following Python 3 function does
def nextChild(hotmap, index):
        ''' Return the next sibling of the node indicated by index. '''
        nextChildIndex = min(index + 1, len(hotmap) - 1)
        return hotmap[nextChildIndex][1]

Response: Return the next sibling of the node indicated by index.


Instruction:
Implement a function in Python 3 to
return the very last node in the hot map.

Response:
def lastNode(class_, hot_map):
        ''' Return the very last node (recursively) in the hot map. '''
        children = hot_map[-1][2]
        if children:
            return class_.lastNode(children)
        else:
            return hot_map[-1][1]


Instruction:
Implement a Python 3 function for
handling mouse - move event by selecting a given element

Response:
def OnMouse( self, event ):
        """Handle mouse-move event by selecting a given element"""
        node = HotMapNavigator.findNodeAtPosition(self.hot_map, event.GetPosition())
        self.SetHighlight( node, event.GetPosition() )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def OnClickRelease( self, event ):
        node = HotMapNavigator.findNodeAtPosition(self.hot_map, event.GetPosition())
        self.SetSelected( node, event.GetPosition() )

Response: Release over a given square in the map


Instruction:
Can you generate a brief explanation for the following Python 3 code
def OnDoubleClick(self, event):
        node = HotMapNavigator.findNodeAtPosition(self.hot_map, event.GetPosition())
        if node:
            wx.PostEvent( self, SquareActivationEvent( node=node, point=event.GetPosition(), map=self ) )

Response: Double click on a given square in the map


Instruction:
Can you generate a brief explanation for the following Python 3 code
def SetSelected( self, node, point=None, propagate=True ):
        if node == self.selectedNode:
            return
        self.selectedNode = node
        self.UpdateDrawing()
        if node:
            wx.PostEvent( self, SquareSelectionEvent( node=node, point=point, map=self ) )

Response: Set the given node selected in the square - map


Instruction:
Here you have a function in Python 3, explain what it does
def SetHighlight( self, node, point=None, propagate=True ):
        if node == self.highlightedNode:
            return
        self.highlightedNode = node
        # TODO: restrict refresh to the squares for previous node and new node...
        self.UpdateDrawing()
        if node and propagate:
            wx.PostEvent( self, SquareHighlightEvent( node=node, point=point, map=self ) )

Response: Set the currently - highlighted node


Instruction:
How would you implement a function in Python 3 that
sets our model object.

Response:
def SetModel( self, model, adapter=None ):
        """Set our model object (root of the tree)"""
        self.model = model
        if adapter is not None:
            self.adapter = adapter
        self.UpdateDrawing()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def Draw(self, dc):
        ''' Draw the tree map on the device context. '''
        self.hot_map = []
        dc.BeginDrawing()
        brush = wx.Brush( self.BackgroundColour  )
        dc.SetBackground( brush )
        dc.Clear()
        if self.model:
            self.max_depth_seen = 0
            font = self.FontForLabels(dc)
            dc.SetFont(font)
            self._em_size_ = dc.GetFullTextExtent( 'm', font )[0]
            w, h = dc.GetSize()
            self.DrawBox( dc, self.model, 0,0,w,h, hot_map = self.hot_map )
        dc.EndDrawing()

Response: Draw the tree map on the device context.


Instruction:
Write a Python 3 script to
return the default GUI font scaled for printing if necessary.

Response:
def FontForLabels(self, dc):
        ''' Return the default GUI font, scaled for printing if necessary. '''
        font = wx.SystemSettings_GetFont(wx.SYS_DEFAULT_GUI_FONT)
        scale = dc.GetPPI()[0] / wx.ScreenDC().GetPPI()[0]
        font.SetPointSize(scale*font.GetPointSize())
        return font


Instruction:
How would you explain what the following Python 3 function does
def BrushForNode( self, node, depth=0 ):
        if node == self.selectedNode:
            color = wx.SystemSettings_GetColour(wx.SYS_COLOUR_HIGHLIGHT)
        elif node == self.highlightedNode:
            color = wx.Colour( red=0, green=255, blue=0 )
        else:
            color = self.adapter.background_color(node, depth)
            if not color:
                red = (depth * 10)%255
                green = 255-((depth * 5)%255)
                blue = (depth * 25)%255
                color = wx.Colour( red, green, blue )
        return wx.Brush( color  )

Response: Create brush to use to display the given node


Instruction:
How would you explain what the following Python 3 function does
def PenForNode( self, node, depth=0 ):
        if node == self.selectedNode:
            return self.SELECTED_PEN
        return self.DEFAULT_PEN

Response: Determine the pen to use to display the given node


Instruction:
How would you implement a function in Python 3 that
determines the text foreground color to use to display the label of the given node

Response:
def TextForegroundForNode(self, node, depth=0):
        """Determine the text foreground color to use to display the label of
           the given node"""
        if node == self.selectedNode:
            fg_color = wx.SystemSettings_GetColour(wx.SYS_COLOUR_HIGHLIGHTTEXT)
        else:
            fg_color = self.adapter.foreground_color(node, depth)
            if not fg_color:
                fg_color = wx.SystemSettings_GetColour(wx.SYS_COLOUR_WINDOWTEXT)
        return fg_color


Instruction:
How would you code a function in Python 3 to
draw a model - node s box and all children nodes.

Response:
def DrawBox( self, dc, node, x,y,w,h, hot_map, depth=0 ):
        """Draw a model-node's box and all children nodes"""
        log.debug( 'Draw: %s to (%s,%s,%s,%s) depth %s',
            node, x,y,w,h, depth,
        )
        if self.max_depth and depth > self.max_depth:
            return
        self.max_depth_seen = max( (self.max_depth_seen,depth))
        dc.SetBrush( self.BrushForNode( node, depth ) )
        dc.SetPen( self.PenForNode( node, depth ) )
        # drawing offset by margin within the square...
        dx,dy,dw,dh = x+self.margin,y+self.margin,w-(self.margin*2),h-(self.margin*2)
        if sys.platform == 'darwin':
            # Macs don't like drawing small rounded rects...
            if w < self.padding*2 or h < self.padding*2:
                dc.DrawRectangle( dx,dy,dw,dh )
            else:
                dc.DrawRoundedRectangle( dx,dy,dw,dh, self.padding )
        else:
            dc.DrawRoundedRectangle( dx,dy,dw,dh, self.padding*3 )
#        self.DrawIconAndLabel(dc, node, x, y, w, h, depth)
        children_hot_map = []
        hot_map.append( (wx.Rect( int(x),int(y),int(w),int(h)), node, children_hot_map ) )
        x += self.padding
        y += self.padding
        w -= self.padding*2
        h -= self.padding*2

        empty = self.adapter.empty( node )
        icon_drawn = False
        if self.max_depth and depth == self.max_depth:
            self.DrawIconAndLabel(dc, node, x, y, w, h, depth)
            icon_drawn = True
        elif empty:
            # is a fraction of the space which is empty...
            log.debug( '  empty space fraction: %s', empty )
            new_h = h * (1.0-empty)
            self.DrawIconAndLabel(dc, node, x, y, w, h-new_h, depth)
            icon_drawn = True
            y += (h-new_h)
            h = new_h

        if w >self.padding*2 and h> self.padding*2:
            children = self.adapter.children( node )
            if children:
                log.debug( '  children: %s', children )
                self.LayoutChildren( dc, children, node, x,y,w,h, children_hot_map, depth+1 )
            else:
                log.debug( '  no children' )
                if not icon_drawn:
                    self.DrawIconAndLabel(dc, node, x, y, w, h, depth)
        else:
            log.debug( '  not enough space: children skipped' )


Instruction:
Here you have a function in Python 3, explain what it does
def DrawIconAndLabel(self, dc, node, x, y, w, h, depth):
        ''' Draw the icon, if any, and the label, if any, of the node. '''
        if w-2 < self._em_size_//2 or h-2 < self._em_size_ //2:
            return
        dc.SetClippingRegion(x+1, y+1, w-2, h-2) # Don't draw outside the box
        try:
            icon = self.adapter.icon(node, node==self.selectedNode)
            if icon and h >= icon.GetHeight() and w >= icon.GetWidth():
                iconWidth = icon.GetWidth() + 2
                dc.DrawIcon(icon, x+2, y+2)
            else:
                iconWidth = 0
            if self.labels and h >= dc.GetTextExtent('ABC')[1]:
                dc.SetTextForeground(self.TextForegroundForNode(node, depth))
                dc.DrawText(self.adapter.label(node), x + iconWidth + 2, y+2)
        finally:
            dc.DestroyClippingRegion()

Response: Draw the icon and label of the selected node.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def overall( self, node ):
        return sum( [self.value(value,node) for value in self.children(node)] )

Response: Calculate overall size of the node including children and empty space


Instruction:
Given the following Python 3 function, write the documentation
def children_sum( self, children,node ):
        return sum( [self.value(value,node) for value in children] )

Response: Calculate children s total sum


Instruction:
Create a Python 3 function for
calculating empty space as a fraction of total space

Response:
def empty( self, node ):
        """Calculate empty space as a fraction of total space"""
        overall = self.overall( node )
        if overall:
            return (overall - self.children_sum( self.children(node), node))/float(overall)
        return 0


Instruction:
How would you code a function in Python 3 to
update the internal counter of the internal counter of the internal counter.

Response:
def update(self, n=1):
        """
        Manually update the progress bar, useful for streams such as reading files (set init(total=filesize) and then in the reading loop, use update(len(current_buffer)) )

        Parameters
        ----------
        n  : int
            Increment to add to the internal counter of iterations.
        """
        if n < 1:
            n = 1
        self.n += n

        delta_it = self.n - self.last_print_n
        if delta >= self.miniters:
            # We check the counter first, to reduce the overhead of time.time()
            cur_t = time.time()
            if cur_t - self.last_print_t >= self.mininterval:
                self.sp.print_status(format_meter(self.n, self.total, cur_t-self.start_t, self.ncols, self.prefix, self.unit, self.unit_format, self.ascii))
                if self.dynamic_miniters: self.miniters = max(self.miniters, delta_it)
                self.last_print_n = self.n
                self.last_print_t = cur_t


Instruction:
Write a Python 3 function that can
call this method to force print the last progress bar update based on the latest n value

Response:
def close(self):
        """
        Call this method to force print the last progress bar update based on the latest n value
        """
        if self.leave:
            if self.last_print_n < self.n:
                cur_t = time.time()
                self.sp.print_status(format_meter(self.n, self.total, cur_t-self.start_t, self.ncols, self.prefix, self.unit, self.unit_format, self.ascii))
            self.file.write('\n')
        else:
            self.sp.print_status('')
            self.file.write('\r')


Instruction:
Can you generate the documentation for the following Python 3 function
def add(self, value):
        if value not in self._set:
            self._set.add(value)
            self._list.add(value)

Response: Add the element value to the set.


Instruction:
Make a summary of the following Python 3 code
def copy(self):
        return self.__class__(key=self._key, load=self._load, _set=set(self._set))

Response: Create a shallow copy of the sorted set.


Instruction:
Create a Python 3 function for
returning a new set with elements in the set that are not in the given iterable.

Response:
def difference(self, *iterables):
        """
        Return a new set with elements in the set that are not in the
        *iterables*.
        """
        diff = self._set.difference(*iterables)
        new_set = self.__class__(key=self._key, load=self._load, _set=diff)
        return new_set


Instruction:
How would you explain what the following Python 3 function does
def intersection(self, *iterables):
        comb = self._set.intersection(*iterables)
        new_set = self.__class__(key=self._key, load=self._load, _set=comb)
        return new_set

Response: Return a new set with elements common to the set and all iterables.


Instruction:
Create a Python 3 function for
returning a new set with elements in either self or that but not both.

Response:
def symmetric_difference(self, that):
        """
        Return a new set with elements in either *self* or *that* but not both.
        """
        diff = self._set.symmetric_difference(that)
        new_set = self.__class__(key=self._key, load=self._load, _set=diff)
        return new_set


Instruction:
Given the following Python 3 function, write the documentation
def cookie_decode(data, key):
    ''' Verify and decode an encoded string. Return an object or None'''
    if isinstance(data, unicode): data = data.encode('ascii') #2to3 hack
    if cookie_is_encoded(data):
        sig, msg = data.split(u'?'.encode('ascii'),1) #2to3 hack
        if sig[1:] == base64.b64encode(hmac.new(key, msg).digest()):
           return pickle.loads(base64.b64decode(msg))
    return None

Response: Verify and decode an encoded string. Return an object or None.


Instruction:
Make a summary of the following Python 3 code
def cookie_is_encoded(data):
    ''' Verify and decode an encoded string. Return an object or None'''
    return bool(data.startswith(u'!'.encode('ascii')) and u'?'.encode('ascii') in data)

Response: Verify and decode an encoded string. Return an object or None


Instruction:
Given the following Python 3 function, write the documentation
def tonativefunc(enc='utf-8'):
    ''' Returns a function that turns everything into 'native' strings using enc '''
    if sys.version_info >= (3,0,0):
        return lambda x: x.decode(enc) if isinstance(x, bytes) else str(x)
    return lambda x: x.encode(enc) if isinstance(x, unicode) else str(x)

Response: Returns a function that turns everything into native strings using enc


Instruction:
Here you have a function in Python 3, explain what it does
def add(self, *a, **ka):
        route = a[0] if a and isinstance(a[0], Route) else Route(*a, **ka)
        self.routes.append(route)
        if route.name:
            self.named[route.name] = route.format_str()
        if route.static:
            self.static[route.route] = route.target
            return
        gpatt = route.group_re()
        fpatt = route.flat_re()
        try:
            gregexp = re.compile('^(%s)$' % gpatt) if '(?P' in gpatt else None
            combined = '%s|(^%s$)' % (self.dynamic[-1][0].pattern, fpatt)
            self.dynamic[-1] = (re.compile(combined), self.dynamic[-1][1])
            self.dynamic[-1][1].append((route.target, gregexp))
        except (AssertionError, IndexError), e: # AssertionError: Too many groups
            self.dynamic.append((re.compile('(^%s$)'%fpatt),[(route.target, gregexp)]))
        except re.error, e:
            raise RouteSyntaxError("Could not add Route: %s (%s)" % (route, e))

Response: Adds a route - > target pair or a Route object to the Router.


Instruction:
Create a Python 3 function for
mounting a Bottle application to a specific URL prefix.

Response:
def mount(self, app, script_path):
        ''' Mount a Bottle application to a specific URL prefix '''
        if not isinstance(app, Bottle):
            raise TypeError('Only Bottle instances are supported for now.')
        script_path = '/'.join(filter(None, script_path.split('/')))
        path_depth = script_path.count('/') + 1
        if not script_path:
            raise TypeError('Empty script_path. Perhaps you want a merge()?')
        for other in self.mounts:
            if other.startswith(script_path):
                raise TypeError('Conflict with existing mount: %s' % other)
        @self.route('/%s/:#.*#' % script_path, method="ANY")
        def mountpoint():
            request.path_shift(path_depth)
            return app.handle(request.path, request.method)
        self.mounts[script_path] = app


Instruction:
Implement a Python 3 function for
executing the handler bound to the specified url and method and return the result as a new HTTP response object.

Response:
def handle(self, url, method):
        """ Execute the handler bound to the specified url and method and return
        its output. If catchall is true, exceptions are catched and returned as
        HTTPError(500) objects. """
        if not self.serve:
            return HTTPError(503, "Server stopped")

        handler, args = self.match_url(url, method)
        if not handler:
            return HTTPError(404, "Not found:" + url)

        try:
            return handler(**args)
        except HTTPResponse, e:
            return e
        except Exception, e:
            if isinstance(e, (KeyboardInterrupt, SystemExit, MemoryError))\
            or not self.catchall:
                raise
            return HTTPError(500, 'Unhandled exception', e, format_exc(10))


Instruction:
Create a Python 3 function to
try to convert the output of a WSGI request into something WSGI compatible and set correct HTTP headers when possible.

Response:
def _cast(self, out, request, response, peek=None):
        """ Try to convert the parameter into something WSGI compatible and set
        correct HTTP headers when possible.
        Support: False, str, unicode, dict, HTTPResponse, HTTPError, file-like,
        iterable of strings and iterable of unicodes
        """
        # Filtered types (recursive, because they may return anything)
        for testtype, filterfunc in self.castfilter:
            if isinstance(out, testtype):
                return self._cast(filterfunc(out), request, response)

        # Empty output is done here
        if not out:
            response.headers['Content-Length'] = 0
            return []
        # Join lists of byte or unicode strings. Mixed lists are NOT supported
        if isinstance(out, list) and isinstance(out[0], (StringType, unicode)):
            out = out[0][0:0].join(out) # b'abc'[0:0] -> b''
        # Encode unicode strings
        if isinstance(out, unicode):
            out = out.encode(response.charset)
        # Byte Strings are just returned
        if isinstance(out, StringType):
            response.headers['Content-Length'] = str(len(out))
            return [out]
        # HTTPError or HTTPException (recursive, because they may wrap anything)
        if isinstance(out, HTTPError):
            out.apply(response)
            return self._cast(self.error_handler.get(out.status, repr)(out), request, response)
        if isinstance(out, HTTPResponse):
            out.apply(response)
            return self._cast(out.output, request, response)

        # Cast Files into iterables
        if hasattr(out, 'read') and 'wsgi.file_wrapper' in request.environ:
            out = request.environ.get('wsgi.file_wrapper',
            lambda x, y: iter(lambda: x.read(y), ''))(out, 1024*64)

        # Handle Iterables. We peek into them to detect their inner type.
        try:
            out = iter(out)
            first = out.next()
            while not first:
                first = out.next()
        except StopIteration:
            return self._cast('', request, response)
        except HTTPResponse, e:
            first = e
        except Exception, e:
            first = HTTPError(500, 'Unhandled exception', e, format_exc(10))
            if isinstance(e, (KeyboardInterrupt, SystemExit, MemoryError))\
            or not self.catchall:
                raise
        # These are the inner types allowed in iterator or generator objects.
        if isinstance(first, HTTPResponse):
            return self._cast(first, request, response)
        if isinstance(first, StringType):
            return itertools.chain([first], out)
        if isinstance(first, unicode):
            return itertools.imap(lambda x: x.encode(response.charset),
                                  itertools.chain([first], out))
        return self._cast(HTTPError(500, 'Unsupported response type: %s'\
                                         % type(first)), request, response)


Instruction:
Create a Python 3 function to
return a HeaderDict filled with request headers.

Response:
def header(self):
        ''' :class:`HeaderDict` filled with request headers.

            HeaderDict keys are case insensitive str.title()d
        '''
        if self._header is None:
            self._header = HeaderDict()
            for key, value in self.environ.iteritems():
                if key.startswith('HTTP_'):
                    key = key[5:].replace('_','-').title()
                    self._header[key] = value
        return self._header


Instruction:
Make a summary of the following Python 3 code
def COOKIES(self):
        if self._COOKIES is None:
            raw_dict = SimpleCookie(self.environ.get('HTTP_COOKIE',''))
            self._COOKIES = {}
            for cookie in raw_dict.itervalues():
                self._COOKIES[cookie.key] = cookie.value
        return self._COOKIES

Response: Returns a dictionary of cookie information parsed into a dictionary.


Instruction:
Given the following Python 3 function, write the documentation
def set_cookie(self, key, value, **kargs):
        if not isinstance(value, basestring):
            sec = self.app.config['securecookie.key']
            value = cookie_encode(value, sec).decode('ascii') #2to3 hack
        self.COOKIES[key] = value
        for k, v in kargs.iteritems():
            self.COOKIES[key][k.replace('_', '-')] = v

Response: Add a new cookie with various options.


Instruction:
How would you explain what the following Python 3 function does
def tamper_file_at(path, pos=0, replace_str=None):
    if not replace_str:
        replace_str = "\x00"
    try:
        with open(path, "r+b") as fh:
            if pos < 0: # if negative, we calculate the position backward from the end of file
                fsize = os.fstat(fh.fileno()).st_size
                pos = fsize + pos
            fh.seek(pos)
            fh.write(replace_str)
    except IOError:
        return False
    finally:
        try:
            fh.close()
        except Exception:
            pass
    return True

Response: Tamper a file at the given position and using the given string


Instruction:
Can you generate a brief explanation for the following Python 3 code
def tamper_dir(inputpath, *args, **kwargs):
    silent = kwargs.get('silent', False)
    if 'silent' in kwargs: del kwargs['silent']

    filescount = 0
    for _ in tqdm(recwalk(inputpath), desc='Precomputing', disable=silent):
        filescount += 1

    files_tampered = 0
    tamper_count = 0
    total_size = 0
    for dirname, filepath in tqdm(recwalk(inputpath), total=filescount, leave=True, desc='Tamper file n.', disable=silent):
        tcount, tsize = tamper_file(os.path.join(dirname, filepath), *args, **kwargs)
        if tcount > 0:
            tamper_count += tcount
            files_tampered += 1
        total_size += tsize
    return [files_tampered, filescount, tamper_count, total_size]

Response: Randomly tamper the files content in a directory tree recursively.


Instruction:
Implement a Python 3 function for
saving current stack trace as formatted string.

Response:
def _save_trace(self):
        """
        Save current stack trace as formatted string.
        """
        stack_trace = stack()
        try:
            self.trace = []
            for frm in stack_trace[5:]: # eliminate our own overhead
                self.trace.insert(0, frm[1:])
        finally:
            del stack_trace


Instruction:
How would you code a function in Python 3 to
store timestamp and current size for later evaluation.

Response:
def track_size(self, ts, sizer):
        """
        Store timestamp and current size for later evaluation.
        The 'sizer' is a stateful sizing facility that excludes other tracked
        objects.
        """
        obj = self.ref()
        self.snapshots.append(
            (ts, sizer.asized(obj, detail=self._resolution_level))
        )
        if obj is not None:
            self.repr = safe_repr(obj, clip=128)


Instruction:
Here you have a function in Python 3, explain what it does
def get_size_at_time(self, timestamp):
        size = 0
        for (t, s) in self.snapshots:
            if t == timestamp:
                size = s.size
        return size

Response: Get the size of the object at a specific time.


Instruction:
Can you generate the documentation for the following Python 3 function
def run(self):
        self.stop = False
        while not self.stop:
            self.tracker.create_snapshot()
            sleep(self.interval)

Response: Loop until a stop signal is set.


Instruction:
Can you implement a function in Python 3 that
returns the total size of the process in bytes.

Response:
def total(self):
        """
        Return the total (virtual) size of the process in bytes. If process
        information is not available, get the best number available, even if it
        is a poor approximation of reality.
        """
        if self.system_total.available:
            return self.system_total.vsz
        elif self.asizeof_total: # pragma: no cover
            return self.asizeof_total
        else: # pragma: no cover
            return self.tracked_total


Instruction:
Create a Python 3 function for
returning timestamped label for this snapshot or a raw timestamp.

Response:
def label(self):
        """Return timestamped label for this snapshot, or a raw timestamp."""
        if not self.desc:
            return "%.3fs" % self.timestamp
        return "%s (%.3fs)" % (self.desc, self.timestamp)


Instruction:
Write a Python 3 function for
injecting constructor for tracked classes. Call the actual constructor of the object and track the object. Attach to the object before calling the constructor to track the object with the parameters of the most specialized class.

Response:
def _tracker(self, _observer_, _self_, *args, **kwds):
        """
        Injected constructor for tracked classes.
        Call the actual constructor of the object and track the object.
        Attach to the object before calling the constructor to track the object with
        the parameters of the most specialized class.
        """
        self.track_object(_self_,
                          name=_observer_.name,
                          resolution_level=_observer_.detail,
                          keep=_observer_.keep,
                          trace=_observer_.trace)
        _observer_.init(_self_, *args, **kwds)


Instruction:
Can you implement a function in Python 3 that
injects a constructor into the class.

Response:
def _inject_constructor(self, cls, func, name, resolution_level, keep,
                            trace):
        """
        Modifying Methods in Place - after the recipe 15.7 in the Python
        Cookbook by Ken Seehof. The original constructors may be restored
        later.
        """
        try:
            constructor = cls.__init__
        except AttributeError:
            def constructor(self, *_args, **_kwargs):
                pass

        # Possible name clash between keyword arguments of the tracked class'
        # constructor and the curried arguments of the injected constructor.
        # Therefore, the additional argument has a 'magic' name to make it less
        # likely that an argument name clash occurs.
        self._observers[cls] = _ClassObserver(constructor,
                                              name,
                                              resolution_level,
                                              keep,
                                              trace)
        cls.__init__ = instancemethod(
            lambda *args, **kwds: func(self._observers[cls], *args, **kwds),
            None,
            cls
        )


Instruction:
Create a Python 3 function for
modifying settings of a tracked class

Response:
def _track_modify(self, cls, name, detail, keep, trace):
        """
        Modify settings of a tracked class
        """
        self._observers[cls].modify(name, detail, keep, trace)


Instruction:
Can you generate the documentation for the following Python 3 function
def _restore_constructor(self, cls):
        cls.__init__ = self._observers[cls].init
        del self._observers[cls]

Response: Restores the original constructor lose track of class.


Instruction:
Can you create a Python 3 function that
changes tracking options for the already tracked object instance.

Response:
def track_change(self, instance, resolution_level=0):
        """
        Change tracking options for the already tracked object 'instance'.
        If instance is not tracked, a KeyError will be raised.
        """
        tobj = self.objects[id(instance)]
        tobj.set_resolution_level(resolution_level)


Instruction:
Here you have a function in Python 3, explain what it does
def track_object(self, instance, name=None, resolution_level=0, keep=False, trace=False):

        # Check if object is already tracked. This happens if track_object is
        # called multiple times for the same object or if an object inherits
        # from multiple tracked classes. In the latter case, the most
        # specialized class wins.  To detect id recycling, the weak reference
        # is checked. If it is 'None' a tracked object is dead and another one
        # takes the same 'id'.
        if id(instance) in self.objects and \
            self.objects[id(instance)].ref() is not None:
            return

        tobj = TrackedObject(instance, resolution_level=resolution_level, trace=trace)

        if name is None:
            name = instance.__class__.__name__
        if not name in self.index:
            self.index[name] = []
        self.index[name].append(tobj)
        self.objects[id(instance)] = tobj

        if keep:
            self._keepalive.append(instance)

Response: Track an object and sample size and lifetime information.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def track_class(self, cls, name=None, resolution_level=0, keep=False, trace=False):
        if not isclass(cls):
            raise TypeError("only class objects can be tracked")
        if name is None:
            name = cls.__module__ + '.' + cls.__name__
        if self._is_tracked(cls):
            self._track_modify(cls, name, resolution_level, keep, trace)
        else:
            self._inject_constructor(cls, self._tracker, name, resolution_level, keep, trace)

Response: Track all objects of the given class.


Instruction:
Explain what the following Python 3 code does
def detach_all_classes(self):
        classes = list(self._observers.keys())
        for cls in classes:
            self.detach_class(cls)

Response: Detach from all tracked classes.


Instruction:
Can you implement a function in Python 3 that
detaches from all tracked classes and objects and removes the original constructor and cleans up the tracking lists.

Response:
def detach_all(self):
        """
        Detach from all tracked classes and objects.
        Restore the original constructors and cleanse the tracking lists.
        """
        self.detach_all_classes()
        self.objects.clear()
        self.index.clear()
        self._keepalive[:] = []


Instruction:
Write a Python 3 function for
starting a thread which takes snapshots periodically.

Response:
def start_periodic_snapshots(self, interval=1.0):
        """
        Start a thread which takes snapshots periodically. The `interval` specifies
        the time in seconds the thread waits between taking snapshots. The thread is
        started as a daemon allowing the program to exit. If periodic snapshots are
        already active, the interval is updated.
        """
        if not self._periodic_thread:
            self._periodic_thread = PeriodicThread(self, interval, name='BackgroundMonitor')
            self._periodic_thread.setDaemon(True)
            self._periodic_thread.start()
        else:
            self._periodic_thread.interval = interval


Instruction:
Can you generate the documentation for the following Python 3 function
def stop_periodic_snapshots(self):
        if self._periodic_thread and self._periodic_thread.isAlive():
            self._periodic_thread.stop = True
            self._periodic_thread.join()
            self._periodic_thread = None

Response: Stop the periodic snapshots of the current resource.


Instruction:
Create a Python 3 function for
creating a snapshot of the current state of the class tracker.

Response:
def create_snapshot(self, description='', compute_total=False):
        """
        Collect current per instance statistics and saves total amount of
        memory associated with the Python process.

        If `compute_total` is `True`, the total consumption of all objects
        known to *asizeof* is computed. The latter might be very slow if many
        objects are mapped into memory at the time the snapshot is taken.
        Therefore, `compute_total` is set to `False` by default.

        The overhead of the `ClassTracker` structure is also computed.

        Snapshots can be taken asynchronously. The function is protected with a
        lock to prevent race conditions.
        """

        try:
            # TODO: It is not clear what happens when memory is allocated or
            # released while this function is executed but it will likely lead
            # to inconsistencies. Either pause all other threads or don't size
            # individual objects in asynchronous mode.
            self.snapshot_lock.acquire()

            timestamp = _get_time()

            sizer = asizeof.Asizer()
            objs = [tobj.ref() for tobj in list(self.objects.values())]
            sizer.exclude_refs(*objs)

            # The objects need to be sized in a deterministic order. Sort the
            # objects by its creation date which should at least work for non-parallel
            # execution. The "proper" fix would be to handle shared data separately.
            tracked_objects = list(self.objects.values())
            tracked_objects.sort(key=lambda x: x.birth)
            for tobj in tracked_objects:
                tobj.track_size(timestamp, sizer)

            snapshot = Snapshot()

            snapshot.timestamp = timestamp
            snapshot.tracked_total = sizer.total
            if compute_total:
                snapshot.asizeof_total = asizeof.asizeof(all=True, code=True)
            snapshot.system_total = pympler.process.ProcessMemoryInfo()
            snapshot.desc = str(description)

            # Compute overhead of all structures, use sizer to exclude tracked objects(!)
            snapshot.overhead = 0
            if snapshot.tracked_total:
                snapshot.overhead = sizer.asizeof(self)
                if snapshot.asizeof_total:
                    snapshot.asizeof_total -= snapshot.overhead

            self.snapshots.append(snapshot)

        finally:
            self.snapshot_lock.release()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def is_required(action):
  '''_actions which are positional or possessing the `required` flag '''
  return not action.option_strings and not isinstance(action, _SubParsersAction) or action.required == True

Response: Return True if the action is required.


Instruction:
Can you write a function in Python 3 where it
extracts the metadata fields from a raw ecc entry.

Response:
def entry_fields(entry, field_delim="\xFF"):
    '''From a raw ecc entry (a string), extract the metadata fields (filename, filesize, ecc for both), and the rest being blocks of hash and ecc per blocks of the original file's header'''
    entry = entry.lstrip(field_delim) # if there was some slight adjustment error (example: the last ecc block of the last file was the field_delim, then we will start with a field_delim, and thus we need to remove the trailing field_delim which is useless and will make the field detection buggy). This is not really a big problem for the previous file's ecc block: the missing ecc characters (which were mistaken for a field_delim), will just be missing (so we will lose a bit of resiliency for the last block of the previous file, but that's not a huge issue, the correction can still rely on the other characters).

    # Find metadata fields delimiters positions
    # TODO: automate this part, just give in argument the number of field_delim to find, and the func will find the x field_delims (the number needs to be specified in argument because the field_delim can maybe be found wrongly inside the ecc stream, which we don't want)
    first = entry.find(field_delim)
    second = entry.find(field_delim, first+len(field_delim))
    third = entry.find(field_delim, second+len(field_delim))
    fourth = entry.find(field_delim, third+len(field_delim))
    # Note: we do not try to find all the field delimiters because we optimize here: we just walk the string to find the exact number of field_delim we are looking for, and after we stop, no need to walk through the whole string.

    # Extract the content of the fields
    # Metadata fields
    relfilepath = entry[:first]
    filesize = entry[first+len(field_delim):second]
    relfilepath_ecc = entry[second+len(field_delim):third]
    filesize_ecc = entry[third+len(field_delim):fourth]
    # Ecc stream field (aka ecc blocks)
    ecc_field = entry[fourth+len(field_delim):]

    # Try to convert to an int, an error may happen
    try:
        filesize = int(filesize)
    except Exception, e:
        print("Exception when trying to detect the filesize in ecc field (it may be corrupted), skipping: ")
        print(e)
        #filesize = 0 # avoid setting to 0, we keep as an int so that we can try to fix using intra-ecc

    # entries = [ {"message":, "ecc":, "hash":}, etc.]
    #print(entry)
    #print(len(entry))
    return {"relfilepath": relfilepath, "relfilepath_ecc": relfilepath_ecc, "filesize": filesize, "filesize_ecc": filesize_ecc, "ecc_field": ecc_field}


Instruction:
Here you have a function in Python 3, explain what it does
def entry_assemble(entry_fields, ecc_params, header_size, filepath, fileheader=None):
    '''From an entry with its parameters (filename, filesize), assemble a list of each block from the original file along with the relative hash and ecc for easy processing later.'''
    # Extract the header from the file
    if fileheader is None:
        with open(filepath, 'rb') as file: # filepath is the absolute path to the original file (the one with maybe corruptions, NOT the output repaired file!)
            # Compute the size of the buffer to read: either header_size if possible, but if the file is smaller than that then we will read the whole file.
            if entry_fields["filesize"] > 0 and entry_fields["filesize"] < header_size:
                fileheader = file.read(entry_fields["filesize"])
            else:
                fileheader = file.read(header_size)

    # Cut the header and the ecc entry into blocks, and then assemble them so that we can easily process block by block
    entry_asm = []
    for i, j in itertools.izip(xrange(0, len(fileheader), ecc_params["message_size"]), xrange(0, len(entry_fields["ecc_field"]), ecc_params["hash_size"] + ecc_params["ecc_size"])):
        # Extract each fields from each block
        mes = fileheader[i:i+ecc_params["message_size"]]
        hash = entry_fields["ecc_field"][j:j+ecc_params["hash_size"]]
        ecc = entry_fields["ecc_field"][j+ecc_params["hash_size"]:j+ecc_params["hash_size"]+ecc_params["ecc_size"]]
        entry_asm.append({"message": mes, "hash": hash, "ecc": ecc})

    # Return a list of fields for each block
    return entry_asm

Response: Assemble a list of each block from the original file along with the relative hash and ecc for easy processing later.


Instruction:
Can you tell what is the following Python 3 function doing
def compute_ecc_hash(ecc_manager, hasher, buf, max_block_size, rate, message_size=None, as_string=False):
    '''Split a string in blocks given max_block_size and compute the hash and ecc for each block, and then return a nice list with both for easy processing.'''
    result = []
    # If required parameters were not provided, we compute them
    if not message_size:
        ecc_params = compute_ecc_params(max_block_size, rate, hasher)
        message_size = ecc_params["message_size"]
    # Split the buffer string in blocks (necessary for Reed-Solomon encoding because it's limited to 255 characters max)
    for i in xrange(0, len(buf), message_size):
        # Compute the message block
        mes = buf[i:i+message_size]
        # Compute the ecc
        ecc = ecc_manager.encode(mes)
        # Compute the hash
        hash = hasher.hash(mes)
        #crc = zlib.crc32(mes) # DEPRECATED: CRC is not resilient enough
        #print("mes %i (%i) - ecc %i (%i) - hash %i (%i)" % (len(mes), message_size, len(ecc), ecc_params["ecc_size"], len(hash), ecc_params["hash_size"])) # DEBUGLINE

        # Return the result (either in string for easy writing into a file, or in a list for easy post-processing)
        if as_string:
            result.append("%s%s" % (str(hash),str(ecc)))
        else:
            result.append([hash, ecc])
    return result

Response: Split a string in blocks given max_block_size and compute the hash and ecc for each block and then return a nice list with both for easy writing into a file.


Instruction:
Write a Python 3 script for
correcting an intra - field with its corresponding intra - ecc if necessary

Response:
def ecc_correct_intra(ecc_manager_intra, ecc_params_intra, field, ecc, enable_erasures=False, erasures_char="\x00", only_erasures=False):
    """ Correct an intra-field with its corresponding intra-ecc if necessary """
    fentry_fields = {"ecc_field": ecc}
    field_correct = [] # will store each block of the corrected (or already correct) filepath
    fcorrupted = False # check if field was corrupted
    fcorrected = True # check if field was corrected (if it was corrupted)
    errmsg = ''
    # Decode each block of the filepath
    for e in entry_assemble(fentry_fields, ecc_params_intra, len(field), '', field):
        # Check if this block of the filepath is OK, if yes then we just copy it over
        if ecc_manager_intra.check(e["message"], e["ecc"]):
            field_correct.append(e["message"])
        else: # Else this block is corrupted, we will try to fix it using the ecc
            fcorrupted = True
            # Repair the message block and the ecc
            try:
                repaired_block, repaired_ecc = ecc_manager_intra.decode(e["message"], e["ecc"], enable_erasures=enable_erasures, erasures_char=erasures_char, only_erasures=only_erasures)
            except (ReedSolomonError, RSCodecError), exc: # the reedsolo lib may raise an exception when it can't decode. We ensure that we can still continue to decode the rest of the file, and the other files.
                repaired_block = None
                repaired_ecc = None
                errmsg += "- Error: metadata field at offset %i: %s\n" % (entry_pos[0], exc)
            # Check if the block was successfully repaired: if yes then we copy the repaired block...
            if repaired_block is not None and ecc_manager_intra.check(repaired_block, repaired_ecc):
                field_correct.append(repaired_block)
            else: # ... else it failed, then we copy the original corrupted block and report an error later
                field_correct.append(e["message"])
                fcorrected = False
    # Join all the blocks into one string to build the final filepath
    if isinstance(field_correct[0], bytearray): field_correct = [str(x) for x in field_correct] # workaround when using --ecc_algo 3 or 4, because we get a list of bytearrays instead of str
    field = ''.join(field_correct)
    # Report errors
    return (field, fcorrupted, fcorrected, errmsg)


Instruction:
Explain what the following Python 3 code does
def _eliminate_leafs(self, graph):
        result = []
        idset = set([id(x) for x in graph])
        for n in graph:
            refset = set([id(x) for x in get_referents(n)])
            if refset.intersection(idset):
                result.append(n)
        return result

Response:
Eliminate leaf objects - that are objects not referencing any other
        objects in the list graph. Returns the list of objects without the leafs.


Instruction:
How would you explain what the following Python 3 function does
def _reduce_to_cycles(self):
        cycles = self.objects[:]
        cnt = 0
        while cnt != len(cycles):
            cnt = len(cycles)
            cycles = self._eliminate_leafs(cycles)
        self.objects = cycles
        return len(self.objects)

Response:
Reduce the set of objects to only those
        that build cycles. Return the number of objects involved in reference
       .


Instruction:
Write a Python 3 function for
reducing the set of objects to only those those that build cycles. Return the reduced graph.

Response:
def reduce_to_cycles(self):
        """
        Iteratively eliminate leafs to reduce the set of objects to only those
        that build cycles. Return the reduced graph. If there are no cycles,
        None is returned.
        """
        if not self._reduced:
            reduced = copy(self)
            reduced.objects = self.objects[:]
            reduced.metadata = []
            reduced.edges = []
            self.num_in_cycles = reduced._reduce_to_cycles()
            reduced.num_in_cycles = self.num_in_cycles
            if self.num_in_cycles:
                reduced._get_edges()
                reduced._annotate_objects()
                for meta in reduced.metadata:
                    meta.cycle = True
            else:
                reduced = None
            self._reduced = reduced
        return self._reduced


Instruction:
Make a summary of the following Python 3 code
def _get_edges(self):
        idset = set([id(x) for x in self.objects])
        self.edges = set([])
        for n in self.objects:
            refset = set([id(x) for x in get_referents(n)])
            for ref in refset.intersection(idset):
                label = ''
                members = None
                if isinstance(n, dict):
                    members = n.items()
                if not members:
                    members = named_refs(n)
                for (k, v) in members:
                    if id(v) == ref:
                        label = k
                        break
                self.edges.add(_Edge(id(n), ref, label))

Response: Compute the edges for the reference graph.


Instruction:
How would you implement a function in Python 3 that
annotates the objects belonging to separate graphs with their respective groups.

Response:
def _annotate_groups(self):
        """
        Annotate the objects belonging to separate (non-connected) graphs with
        individual indices.
        """
        g = {}
        for x in self.metadata:
            g[x.id] = x

        idx = 0
        for x in self.metadata:
            if not hasattr(x, 'group'):
                x.group = idx
                idx += 1
            neighbors = set()
            for e in self.edges:
                if e.src == x.id:
                    neighbors.add(e.dst)
                if e.dst == x.id:
                    neighbors.add(e.src)
            for nb in neighbors:
                g[nb].group = min(x.group, getattr(g[nb], 'group', idx))

        # Assign the edges to the respective groups. Both "ends" of the edge
        # should share the same group so just use the first object's group.
        for e in self.edges:
            e.group = g[e.src].group

        self._max_group = idx


Instruction:
Write a Python 3 function that can
return True if the group is non - empty. Otherwise returns False.

Response:
def _filter_group(self, group):
        """
        Eliminate all objects but those which belong to `group`.
        ``self.objects``, ``self.metadata`` and ``self.edges`` are modified.
        Returns `True` if the group is non-empty. Otherwise returns `False`.
        """
        self.metadata = [x for x in self.metadata if x.group == group]
        group_set = set([x.id for x in self.metadata])
        self.objects = [obj for obj in self.objects if id(obj) in group_set]
        self.count = len(self.metadata)
        if self.metadata == []:
            return False

        self.edges = [e for e in self.edges if e.group == group]

        del self._max_group

        return True


Instruction:
Create a Python 3 function for
splitting the reference graph into sub - graphs.

Response:
def split(self):
        """
        Split the graph into sub-graphs. Only connected objects belong to the
        same graph. `split` yields copies of the Graph object. Shallow copies
        are used that only replicate the meta-information, but share the same
        object list ``self.objects``.

        >>> from pympler.refgraph import ReferenceGraph
        >>> a = 42
        >>> b = 'spam'
        >>> c = {a: b}
        >>> t = (1,2,3)
        >>> rg = ReferenceGraph([a,b,c,t])
        >>> for subgraph in rg.split():
        ...   print subgraph.index
        0
        1
        """
        self._annotate_groups()
        index = 0

        for group in range(self._max_group):
            subgraph = copy(self)
            subgraph.metadata = self.metadata[:]
            subgraph.edges = self.edges.copy()

            if subgraph._filter_group(group):
                subgraph.total_size = sum([x.size for x in subgraph.metadata])
                subgraph.index = index
                index += 1
                yield subgraph


Instruction:
Here you have a function in Python 3, explain what it does
def split_and_sort(self):
        graphs = list(self.split())
        graphs.sort(key=lambda x: -len(x.metadata))
        for index, graph in enumerate(graphs):
            graph.index = index
        return graphs

Response: Split the graphs into sub graphs and sort the graphs by the number of nodes.


Instruction:
Make a summary of the following Python 3 code
def _annotate_objects(self):
        self.metadata = []
        sizer = Asizer()
        sizes = sizer.asizesof(*self.objects)
        self.total_size = sizer.total
        for obj, sz in zip(self.objects, sizes):
            md = _MetaObject()
            md.size = sz
            md.id = id(obj)
            try:
                md.type = obj.__class__.__name__
            except (AttributeError, ReferenceError): # pragma: no cover
                md.type = type(obj).__name__
            md.str = safe_repr(obj, clip=128)
            self.metadata.append(md)

Response: Extract meta - data describing the stored objects.


Instruction:
Can you implement a function in Python 3 that
returns a string representation of the object with graphviz.

Response:
def _get_graphviz_data(self):
        """
        Emit a graph representing the connections between the objects described
        within the metadata list. The text representation can be transformed to
        a graph with graphviz. Returns a string.
        """
        s = []
        header = '// Process this file with graphviz\n'
        s.append( header)
        s.append('digraph G {\n')
        s.append('    node [shape=box];\n')
        for md in self.metadata:
            label = trunc(md.str, 48).replace('"', "'")
            extra = ''
            if md.type == 'instancemethod':
                extra = ', color=red'
            elif md.type == 'frame':
                extra = ', color=orange'
            s.append('    "X%s" [ label = "%s\\n%s" %s ];\n' % \
                (hex(md.id)[1:], label, md.type, extra))
        for e in self.edges:
            extra = ''
            if e.label == '__dict__':
                extra = ',weight=100'
            s.append('    X%s -> X%s [label="%s"%s];\n' % \
                (hex(e.src)[1:], hex(e.dst)[1:], e.label, extra))

        s.append('}\n')
        return "".join(s)


Instruction:
Given the following Python 3 function, write the documentation
def render(self, filename, cmd='dot', format='ps', unflatten=False):
        if self.objects == []:
            return False

        data = self._get_graphviz_data()

        options = ('-Nfontsize=10',
                   '-Efontsize=10',
                   '-Nstyle=filled',
                   '-Nfillcolor=#E5EDB8',
                   '-Ncolor=#CCCCCC')
        cmdline = (cmd, '-T%s' % format, '-o', filename) + options

        if unflatten:
            p1 = Popen(('unflatten', '-l7'), stdin=PIPE, stdout=PIPE,
                **popen_flags)
            p2 = Popen(cmdline, stdin=p1.stdout, **popen_flags)
            p1.communicate(encode4pipe(data))
            p2.communicate()
            return p2.returncode == 0
        else:
            p = Popen(cmdline, stdin=PIPE, **popen_flags)
            p.communicate(encode4pipe(data))
            return p.returncode == 0

Response: Render the object to filename using graphviz.


Instruction:
Can you tell what is the following Python 3 function doing
def write_graph(self, filename):
        f = open(filename, 'w')
        f.write(self._get_graphviz_data())
        f.close()

Response: Write raw graph data to a file


Instruction:
How would you implement a function in Python 3 that
returns the parsed result in the form of a tree of Frame objects where the first element is the name of the frame and the second is the time of the frame.

Response:
def root_frame(self):
        """
        Returns the parsed results in the form of a tree of Frame objects
        """
        if not hasattr(self, '_root_frame'):
            self._root_frame = Frame()

            # define a recursive function that builds the hierarchy of frames given the
            # stack of frame identifiers
            def frame_for_stack(stack):
                if len(stack) == 0:
                    return self._root_frame

                parent = frame_for_stack(stack[:-1])
                frame_name = stack[-1]

                if not frame_name in parent.children_dict:
                    parent.add_child(Frame(frame_name, parent))

                return parent.children_dict[frame_name]

            for stack, self_time in self.stack_self_time.items():
                frame_for_stack(stack).self_time = self_time

        return self._root_frame


Instruction:
Here you have a function in Python 3, explain what it does
def reset_trace():
    global call_dict
    global call_stack
    global func_count
    global func_count_max
    global func_time
    global func_time_max
    global call_stack_timer

    call_dict = {}

    # current call stack
    call_stack = ['__main__']

    # counters for each function
    func_count = {}
    func_count_max = 0

    # accumative time per function
    func_time = {}
    func_time_max = 0

    # keeps track of the start time of each call on the stack
    call_stack_timer = []

Response: Resets all collected statistics. This is run automatically by the module load_module and reset_trace.


Instruction:
Write a Python 3 script to
return True if the file_name is in the lib directory.

Response:
def is_module_stdlib(file_name):
    """Returns True if the file_name is in the lib directory."""
    # TODO: Move these calls away from this function so it doesn't have to run
    # every time.
    lib_path = sysconfig.get_python_lib()
    path = os.path.split(lib_path)
    if path[1] == 'site-packages':
        lib_path = path[0]
    return file_name.lower().startswith(lib_path.lower())


Instruction:
Here you have a function in Python 3, explain what it does
def start_trace(reset=True, filter_func=None, time_filter_func=None):
    global trace_filter
    global time_filter
    if reset:
        reset_trace()

    if filter_func:
        trace_filter = filter_func
    else:
        trace_filter = GlobbingFilter(exclude=['pycallgraph.*'])

    if time_filter_func:
        time_filter = time_filter_func
    else:
        time_filter = GlobbingFilter()

    sys.settrace(tracer)

Response: Starts a trace for the current call graph.


Instruction:
Here you have a function in Python 3, explain what it does
def tracer(frame, event, arg):
    global func_count_max
    global func_count
    global trace_filter
    global time_filter
    global call_stack
    global func_time
    global func_time_max

    if event == 'call':
        keep = True
        code = frame.f_code

        # Stores all the parts of a human readable name of the current call.
        full_name_list = []

        # Work out the module name
        module = inspect.getmodule(code)
        if module:
            module_name = module.__name__
            module_path = module.__file__
            if not settings['include_stdlib'] \
                and is_module_stdlib(module_path):
                keep = False
            if module_name == '__main__':
                module_name = ''
        else:
            module_name = ''
        if module_name:
            full_name_list.append(module_name)

        # Work out the class name.
        try:
            class_name = frame.f_locals['self'].__class__.__name__
            full_name_list.append(class_name)
        except (KeyError, AttributeError):
            class_name = ''

        # Work out the current function or method
        func_name = code.co_name
        if func_name == '?':
            func_name = '__main__'
        full_name_list.append(func_name)

        # Create a readable representation of the current call
        full_name = '.'.join(full_name_list)

        # Load the trace filter, if any. 'keep' determines if we should ignore
        # this call
        if keep and trace_filter:
            keep = trace_filter(call_stack, module_name, class_name,
                func_name, full_name)

        # Store the call information
        if keep:

            if call_stack:
                fr = call_stack[-1]
            else:
                fr = None
            if fr not in call_dict:
                call_dict[fr] = {}
            if full_name not in call_dict[fr]:
                call_dict[fr][full_name] = 0
            call_dict[fr][full_name] += 1

            if full_name not in func_count:
                func_count[full_name] = 0
            func_count[full_name] += 1
            if func_count[full_name] > func_count_max:
                func_count_max = func_count[full_name]

            call_stack.append(full_name)
            call_stack_timer.append(time.time())

        else:
            call_stack.append('')
            call_stack_timer.append(None)

    if event == 'return':
        if call_stack:
            full_name = call_stack.pop(-1)
            if call_stack_timer:
                t = call_stack_timer.pop(-1)
            else:
                t = None
            if t and time_filter(stack=call_stack, full_name=full_name):
                if full_name not in func_time:
                    func_time[full_name] = 0
                call_time = (time.time() - t)
                func_time[full_name] += call_time
                if func_time[full_name] > func_time_max:
                    func_time_max = func_time[full_name]

    return tracer

Response: This is the internal function that is called every time a call is made. It is called every time a call is made. It keeps track of relationships between calls between calls.


Instruction:
How would you implement a function in Python 3 that
returns a string containing a DOT file.

Response:
def get_dot(stop=True):
    """Returns a string containing a DOT file. Setting stop to True will cause
    the trace to stop.
    """
    defaults = []
    nodes    = []
    edges    = []

    # define default attributes
    for comp, comp_attr in graph_attributes.items():
        attr = ', '.join( '%s = "%s"' % (attr, val)
                         for attr, val in comp_attr.items() )
        defaults.append( '\t%(comp)s [ %(attr)s ];\n' % locals() )

    # define nodes
    for func, hits in func_count.items():
        calls_frac, total_time_frac, total_time = _frac_calculation(func, hits)
        col = settings['node_colour'](calls_frac, total_time_frac)
        attribs = ['%s="%s"' % a for a in settings['node_attributes'].items()]
        node_str = '"%s" [%s];' % (func, ', '.join(attribs))
        nodes.append( node_str % locals() )

    # define edges
    for fr_key, fr_val in call_dict.items():
        if not fr_key: continue
        for to_key, to_val in fr_val.items():
            calls_frac, total_time_frac, totla_time = \
                _frac_calculation(to_key, to_val)
            col = settings['edge_colour'](calls_frac, total_time_frac)
            edge = '[ color = "%s", label="%s" ]' % (col, to_val)
            edges.append('"%s"->"%s" %s;' % (fr_key, to_key, edge))

    defaults = '\n\t'.join( defaults )
    nodes    = '\n\t'.join( nodes )
    edges    = '\n\t'.join( edges )

    dot_fmt = ("digraph G {\n"
               "	%(defaults)s\n\n"
               "	%(nodes)s\n\n"
               "	%(edges)s\n}\n"
              )
    return dot_fmt % locals()


Instruction:
Explain what the following Python 3 code does
def get_gdf(stop=True):
    ret = ['nodedef>name VARCHAR, label VARCHAR, hits INTEGER, ' + \
            'calls_frac DOUBLE, total_time_frac DOUBLE, ' + \
            'total_time DOUBLE, color VARCHAR, width DOUBLE']
    for func, hits in func_count.items():
        calls_frac, total_time_frac, total_time = _frac_calculation(func, hits)
        col = settings['node_colour'](calls_frac, total_time_frac)
        color = ','.join([str(round(float(c) * 255)) for c in col.split()])
        ret.append('%s,%s,%s,%s,%s,%s,\'%s\',%s' % (func, func, hits, \
                    calls_frac, total_time_frac, total_time, color, \
                    math.log(hits * 10)))
    ret.append('edgedef>node1 VARCHAR, node2 VARCHAR, color VARCHAR')
    for fr_key, fr_val in call_dict.items():
        if fr_key == '':
            continue
        for to_key, to_val in fr_val.items():
            calls_frac, total_time_frac, total_time = \
                _frac_calculation(to_key, to_val)
            col = settings['edge_colour'](calls_frac, total_time_frac)
            color = ','.join([str(round(float(c) * 255)) for c in col.split()])
            ret.append('%s,%s,\'%s\'' % (fr_key, to_key, color))
    ret = '\n'.join(ret)
    return ret

Response: Returns a string containing a GDF file.


Instruction:
How would you implement a function in Python 3 that
creates a graph from the dot language file specified by filename.

Response:
def make_dot_graph(filename, format='png', tool='dot', stop=True):
    """Creates a graph using a Graphviz tool that supports the dot language. It
    will output into a file specified by filename with the format specified.
    Setting stop to True will stop the current trace.
    """
    if stop:
        stop_trace()

    dot_data = get_dot()

    # normalize filename
    regex_user_expand = re.compile('\A~')
    if regex_user_expand.match(filename):
        filename = os.path.expanduser(filename)
    else:
        filename = os.path.expandvars(filename)  # expand, just in case

    if format == 'dot':
        f = open(filename, 'w')
        f.write(dot_data)
        f.close()

    else:
        # create a temporary file to be used for the dot data
        fd, tempname = tempfile.mkstemp()
        with os.fdopen(fd, 'w') as f:
            f.write(dot_data)

        cmd = '%(tool)s -T%(format)s -o%(filename)s %(tempname)s' % locals()
        try:
            ret = os.system(cmd)
            if ret:
                raise PyCallGraphException( \
                    'The command "%(cmd)s" failed with error ' \
                    'code %(ret)i.' % locals())
        finally:
            os.unlink(tempname)


Instruction:
Can you generate the documentation for the following Python 3 function
def make_gdf_graph(filename, stop=True):
    if stop:
        stop_trace()

    try:
        f = open(filename, 'w')
        f.write(get_gdf())
    finally:
        if f: f.close()

Response: Create a graph in simple GDF format suitable for feeding into Gephi


Instruction:
How would you code a function in Python 3 to
shim to run 32 - bit on 64 - bit mac as a sub - process

Response:
def macshim():
    """Shim to run 32-bit on 64-bit mac as a sub-process"""
    import subprocess, sys
    subprocess.call([
        sys.argv[0] + '32'
    ]+sys.argv[1:],
        env={"VERSIONER_PYTHON_PREFER_32_BIT":"yes"}
    )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def file_supports_color(file_obj):
    plat = sys.platform
    supported_platform = plat != 'Pocket PC' and (plat != 'win32' or
                                                  'ANSICON' in os.environ)

    is_a_tty = hasattr(file_obj, 'isatty') and file_obj.isatty()
    if not supported_platform or not is_a_tty:
        return False
    return True

Response: Returns True if the running system s terminal supports color and False otherwise.


Instruction:
Can you implement a function in Python 3 that
computes the ecc parameters for the ECCMan object.

Response:
def compute_ecc_params(max_block_size, rate, hasher):
    '''Compute the ecc parameters (size of the message, size of the hash, size of the ecc). This is an helper function to easily compute the parameters from a resilience rate to instanciate an ECCMan object.'''
    #message_size = max_block_size - int(round(max_block_size * rate * 2, 0)) # old way to compute, wasn't really correct because we applied the rate on the total message+ecc size, when we should apply the rate to the message size only (that is not known beforehand, but we want the ecc size (k) = 2*rate*message_size or in other words that k + k * 2 * rate = n)
    message_size = int(round(float(max_block_size) / (1 + 2*rate), 0))
    ecc_size = max_block_size - message_size
    hash_size = len(hasher) # 32 when we use MD5
    return {"message_size": message_size, "ecc_size": ecc_size, "hash_size": hash_size}


Instruction:
Explain what the following Python 3 code does
def detect_reedsolomon_parameters(message, mesecc_orig, gen_list=[2, 3, 5], c_exp=8):
    '''Use an exhaustive search to automatically find the correct parameters for the ReedSolomon codec from a sample message and its encoded RS code.
    Arguments: message is the sample message, eg, "hello world" ; mesecc_orig is the message variable encoded with RS block appended at the end.
    '''
    # Description: this is basically an exhaustive search where we will try every possible RS parameter, then try to encode the sample message, and see if the resulting RS code is close to the supplied code.
    # All variables except the Galois Field's exponent are automatically generated and searched.
    # To compare with the supplied RS code, we compute the Hamming distance, so that even if the RS code is tampered, we can still find the closest set of RS parameters to decode this message.
    # The goal is to provide users a function so that they can use the "hello world" sample string in generated ECC files to recover their RS parameters in case they forget them. But users can use any sample message: for example, if they have an untampered file and its relative ecc track, they can use the ecc track as the mesecc_orig and their original file as the sample message.

    from .reedsolomon import reedsolo as reedsolop # need to import another time the reedsolo library for detect_reedsolomon_parameters to work (because we need to reinit all the tables, and they are declared module-wide, so this would conflict with decoding)

    # Init the variables
    n = len(mesecc_orig)
    k = len(message)
    field_charac = int((2**c_exp) - 1)
    maxval1 = max([ord(x) if isinstance(x, basestring) else x for x in message ])
    maxval2 = max([ord(x) if isinstance(x, basestring) else x for x in mesecc_orig])
    maxval = max([maxval1, maxval2])
    if (maxval > field_charac):
        raise ValueError("The specified field's exponent is wrong, the message contains values (%i) above the field's cardinality (%i)!" % (maxval, field_charac))

    # Prepare the variable that will store the result
    best_match = {"hscore": -1, "params": [{"gen_nb": 0, "prim": 0, "fcr": 0}]}

    # Exhaustively search by generating every combination of values for the RS parameters and test the Hamming distance
    for gen_nb in gen_list:
        prim_list = reedsolop.find_prime_polys(generator=gen_nb, c_exp=c_exp, fast_primes=False, single=False)
        for prim in prim_list:
            reedsolop.init_tables(prim)
            for fcr in xrange(field_charac):
                #g = reedsolop.rs_generator_poly_all(n, fcr=fcr, generator=gen_nb)
                # Generate a RS code from the sample message using the current combination of RS parameters
                mesecc = reedsolop.rs_encode_msg(message, n-k, fcr=fcr)
                # Compute the Hamming distance
                h = hamming(mesecc, mesecc_orig)
                # If the Hamming distance is lower than the previous best match (or if it's the first try), save this set of parameters
                if best_match["hscore"] == -1 or h <= best_match["hscore"]:
                    # If the distance is strictly lower than for the previous match, then we replace the previous match with the current one
                    if best_match["hscore"] == -1 or h < best_match["hscore"]:
                        best_match["hscore"] = h
                        best_match["params"] = [{"gen_nb": gen_nb, "prim": prim, "fcr": fcr}]
                    # Else there is an ambiguity: the Hamming distance is the same as for the previous best match, so we keep the previous set of parameters but we append the current set
                    elif h == best_match["hscore"]:
                        best_match["params"].append({"gen_nb": gen_nb, "prim": prim, "fcr": fcr})
                    # If Hamming distance is 0, then we have found a perfect match (the current set of parameters allow to generate the exact same RS code from the sample message), so we stop here
                    if h == 0: break

    # Printing the results to the user
    if best_match["hscore"] >= 0 and best_match["hscore"] < len(mesecc_orig):
        perfect_match_str = " (0=perfect match)" if best_match["hscore"]==0 else ""
        result = ''
        result += "Found closest set of parameters, with Hamming distance %i%s:\n" % (best_match["hscore"], perfect_match_str)
        for param in best_match["params"]:
            result += "gen_nb=%s prim=%s(%s) fcr=%s\n" % (param["gen_nb"], param["prim"], hex(param["prim"]), param["fcr"])
        return result
    else:
        return "Parameters could not be automatically detected..."

Response: Detects the correct parameters for the ReedSolomon codec from a sample message and its encoded RS code.


Instruction:
Here you have a function in Python 3, explain what it does
def encode(self, message, k=None):
        '''Encode one message block (up to 255) into an ecc'''
        if not k: k = self.k
        message, _ = self.pad(message, k=k)
        if self.algo == 1:
            mesecc = self.ecc_manager.encode(message, k=k)
        elif self.algo == 2:
            mesecc = self.ecc_manager.encode_fast(message, k=k)
        elif self.algo == 3 or self.algo == 4:
            mesecc = rs_encode_msg(message, self.n-k, fcr=self.fcr, gen=self.g[self.n-k])
            #mesecc = rs_encode_msg_precomp(message, self.n-k, fcr=self.fcr, gen=self.g[self.n-k])

        ecc = mesecc[len(message):]
        return ecc

Response: Encode one message block into an ecc


Instruction:
Write a Python 3 script for
repairing a message and its ecc also given the message and its ecc

Response:
def decode(self, message, ecc, k=None, enable_erasures=False, erasures_char="\x00", only_erasures=False):
        '''Repair a message and its ecc also, given the message and its ecc (both can be corrupted, we will still try to fix both of them)'''
        if not k: k = self.k

        # Optimization, use bytearray
        if isinstance(message, _str):
            message = bytearray([ord(x) for x in message])
            ecc = bytearray([ord(x) for x in ecc])

        # Detect erasures positions and replace with null bytes (replacing erasures with null bytes is necessary for correct syndrome computation)
        # Note that this must be done before padding, else we risk counting the padded null bytes as erasures!
        erasures_pos = None
        if enable_erasures:
            # Concatenate to find erasures in the whole codeword
            mesecc = message + ecc
            # Convert char to a int (because we use a bytearray)
            if isinstance(erasures_char, _str): erasures_char = ord(erasures_char)
            # Find the positions of the erased characters
            erasures_pos = [i for i in xrange(len(mesecc)) if mesecc[i] == erasures_char]
            # Failing case: no erasures could be found and we want to only correct erasures, then we return the message as-is
            if only_erasures and not erasures_pos: return message, ecc

        # Pad with null bytes if necessary
        message, pad = self.pad(message, k=k)
        ecc, _ = self.rpad(ecc, k=k) # fill ecc with null bytes if too small (maybe the field delimiters were misdetected and this truncated the ecc? But we maybe still can correct if the truncation is less than the resilience rate)

        # If the message was left padded, then we need to update the positions of the erasures
        if erasures_pos and pad:
            len_pad = len(pad)
            erasures_pos = [x+len_pad for x in erasures_pos]

        # Decoding
        if self.algo == 1:
            msg_repaired, ecc_repaired = self.ecc_manager.decode(message + ecc, nostrip=True, k=k, erasures_pos=erasures_pos, only_erasures=only_erasures) # Avoid automatic stripping because we are working with binary streams, thus we should manually strip padding only when we know we padded
        elif self.algo == 2:
            msg_repaired, ecc_repaired = self.ecc_manager.decode_fast(message + ecc, nostrip=True, k=k, erasures_pos=erasures_pos, only_erasures=only_erasures)
        elif self.algo == 3:
            #msg_repaired, ecc_repaired = self.ecc_manager.decode_fast(message + ecc, nostrip=True, k=k, erasures_pos=erasures_pos, only_erasures=only_erasures)
            msg_repaired, ecc_repaired = reedsolo.rs_correct_msg_nofsynd(bytearray(message + ecc), self.n-k, fcr=self.fcr, generator=self.gen_nb, erase_pos=erasures_pos, only_erasures=only_erasures)
            msg_repaired = bytearray(msg_repaired)
            ecc_repaired = bytearray(ecc_repaired)
        elif self.algo == 4:
            msg_repaired, ecc_repaired = reedsolo.rs_correct_msg(bytearray(message + ecc), self.n-k, fcr=self.fcr, generator=self.gen_nb, erase_pos=erasures_pos, only_erasures=only_erasures)
            msg_repaired = bytearray(msg_repaired)
            ecc_repaired = bytearray(ecc_repaired)

        if pad: # Strip the null bytes if we padded the message before decoding
            msg_repaired = msg_repaired[len(pad):len(msg_repaired)]
        return msg_repaired, ecc_repaired


Instruction:
Explain what the following Python 3 code does
def pad(self, message, k=None):
        '''Automatically left pad with null bytes a message if too small, or leave unchanged if not necessary. This allows to keep track of padding and strip the null bytes after decoding reliably with binary data. Equivalent to shortening (shortened reed-solomon code).'''
        if not k: k = self.k
        pad = None
        if len(message) < k:
            #pad = "\x00" * (k-len(message))
            pad = bytearray(k-len(message))
            message = pad + message
        return [message, pad]

Response: Automatically left pad with null bytes a message if too small or leave unchanged if not necessary.