Instruction:
Implement a Python 3 function for
setting the value of nested items in the archive.

Response:
def set_nested(self, klist, value):
        """D.set_nested((k1, k2,k3, ...), v) -> D[k1][k2][k3] ... = v"""
        keys = list(klist)
        if len(keys) > 0:
            curr_dict = self
            last_key = keys.pop()
            for key in keys:
                if not curr_dict.has_key(key) or not isinstance(curr_dict[key],
                                                                NestedDict):
                    curr_dict[key] = type(self)()
                curr_dict = curr_dict[key]
            curr_dict[last_key] = value


Instruction:
Can you tell what is the following Python 3 function doing
def registerFilter(self, column, patterns, is_regex=False,
                       ignore_case=False):
        """Register filter on a column of table.

        @param column:      The column name.
        @param patterns:    A single pattern or a list of patterns used for
                            matching column values.
        @param is_regex:    The patterns will be treated as regex if True, the
                            column values will be tested for equality with the
                            patterns otherwise.
        @param ignore_case: Case insensitive matching will be used if True.

        """
        if isinstance(patterns, basestring):
            patt_list = (patterns,)
        elif isinstance(patterns, (tuple, list)):
            patt_list = list(patterns)
        else:
            raise ValueError("The patterns parameter must either be as string "
                             "or a tuple / list of strings.")
        if is_regex:
            if ignore_case:
                flags = re.IGNORECASE
            else:
                flags = 0
            patt_exprs = [re.compile(pattern, flags) for pattern in patt_list]
        else:
            if ignore_case:
                patt_exprs = [pattern.lower() for pattern in patt_list]
            else:
                patt_exprs = patt_list
        self._filters[column] = (patt_exprs, is_regex, ignore_case)

Response: Register a filter on a column of table.


Instruction:
How would you explain what the following Python 3 function does
def unregisterFilter(self, column):
        if self._filters.has_key(column):
            del self._filters[column]

Response: Unregister a filter on a column of the table.


Instruction:
Can you generate the documentation for the following Python 3 function
def registerFilters(self, **kwargs):
        for (key, patterns) in kwargs.items():
            if key.endswith('_regex'):
                col = key[:-len('_regex')]
                is_regex = True
            else:
                col = key
                is_regex = False
            if col.endswith('_ic'):
                col = col[:-len('_ic')]
                ignore_case = True
            else:
                ignore_case = False
            self.registerFilter(col, patterns, is_regex, ignore_case)

Response: Register multiple filters at once.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def applyFilters(self, headers, table):
        result = []
        column_idxs = {}
        for column in self._filters.keys():
            try:
                column_idxs[column] = headers.index(column)
            except ValueError:
                raise ValueError('Invalid column name %s in filter.' % column)
        for row in table:
            for (column, (patterns,
                          is_regex,
                          ignore_case)) in self._filters.items():
                col_idx = column_idxs[column]
                col_val = row[col_idx]
                if is_regex:
                    for pattern in patterns:
                        if pattern.search(col_val):
                            break
                    else:
                        break
                else:
                    if ignore_case:
                        col_val = col_val.lower()
                    if col_val in patterns:
                        pass
                    else:
                        break
            else:
                result.append(row)
        return result

Response: Apply filters on ps command result.


Instruction:
Can you tell what is the following Python 3 function doing
def open(self, host=None, port=0, socket_file=None,
             timeout=socket.getdefaulttimeout()):
        """Connect to a host.

        With a host argument, it connects the instance using TCP; port number
        and timeout are optional, socket_file must be None. The port number
        defaults to the standard telnet port (23).

        With a socket_file argument, it connects the instance using
        named socket; timeout is optional and host must be None.

        Don't try to reopen an already connected instance.

        """
        self.socket_file = socket_file
        if host is not None:
            if sys.version_info[:2] >= (2,6):
                telnetlib.Telnet.open(self, host, port, timeout)
            else:
                telnetlib.Telnet.open(self, host, port)
        elif socket_file is not None:
            self.eof = 0
            self.host = host
            self.port = port
            self.timeout = timeout
            self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            self.sock.settimeout(timeout)
            self.sock.connect(socket_file)
        else:
            raise TypeError("Either host or socket_file argument is required.")

Response: Open an instance of a Telnet instance.


Instruction:
Explain what the following Python 3 code does
def analyze(egg, subjgroup=None, listgroup=None, subjname='Subject',
            listname='List', analysis=None, position=0, permute=False,
            n_perms=1000, parallel=False, match='exact',
            distance='euclidean', features=None, ts=None):
    """
    General analysis function that groups data by subject/list number and performs analysis.

    Parameters
    ----------
    egg : Egg data object
        The data to be analyzed

    subjgroup : list of strings or ints
        String/int variables indicating how to group over subjects.  Must be
        the length of the number of subjects

    subjname : string
        Name of the subject grouping variable

    listgroup : list of strings or ints
        String/int variables indicating how to group over list.  Must be
        the length of the number of lists

    listname : string
        Name of the list grouping variable

    analysis : string
        This is the analysis you want to run.  Can be accuracy, spc, pfr,
        temporal or fingerprint

    position : int
        Optional argument for pnr analysis.  Defines encoding position of item
        to run pnr.  Default is 0, and it is zero indexed

    permute : bool
        Optional argument for fingerprint/temporal cluster analyses. Determines
        whether to correct clustering scores by shuffling recall order for each list
        to create a distribution of clustering scores (for each feature). The
        "corrected" clustering score is the proportion of clustering scores in
        that random distribution that were lower than the clustering score for
        the observed recall sequence. Default is False.

    n_perms : int
        Optional argument for fingerprint/temporal cluster analyses. Number of
        permutations to run for "corrected" clustering scores. Default is 1000 (
        per recall list).

    parallel : bool
        Option to use multiprocessing (this can help speed up the permutations
        tests in the clustering calculations)

    match : str (exact, best or smooth)
        Matching approach to compute recall matrix.  If exact, the presented and
        recalled items must be identical (default).  If best, the recalled item
        that is most similar to the presented items will be selected. If smooth,
        a weighted average of all presented items will be used, where the
        weights are derived from the similarity between the recalled item and
        each presented item.

    distance : str
        The distance function used to compare presented and recalled items.
        Applies only to 'best' and 'smooth' matching approaches.  Can be any
        distance function supported by numpy.spatial.distance.cdist.

    Returns
    ----------
    result : quail.FriedEgg
        Class instance containing the analysis results

    """
    if analysis is None:
        raise ValueError('You must pass an analysis type.')

    if analysis not in analyses.keys():
        raise ValueError('Analysis not recognized. Choose one of the following: '
                        'accuracy, spc, pfr, lag-crp, fingerprint, temporal')

    from ..egg import FriedEgg

    if hasattr(egg, 'subjgroup'):
        if egg.subjgroup is not None:
            subjgroup = egg.subjgroup

    if hasattr(egg, 'subjname'):
        if egg.subjname is not None:
            subjname = egg.subjname

    if hasattr(egg, 'listgroup'):
        if egg.listgroup is not None:
            listgroup = egg.listgroup

    if hasattr(egg, 'listname'):
        if egg.listname is not None:
            listname = egg.listname

    if features is None:
        features = egg.feature_names

    opts = {
        'subjgroup' : subjgroup,
        'listgroup' : listgroup,
        'subjname' : subjname,
        'parallel' : parallel,
        'match' : match,
        'distance' : distance,
        'features' : features,
        'analysis_type' : analysis,
        'analysis' : analyses[analysis]
    }

    if analysis is 'pfr':
        opts.update({'position' : 0})
    elif analysis is 'pnr':
        opts.update({'position' : position})
    if analysis is 'temporal':
        opts.update({'features' : ['temporal']})
    if analysis in ['temporal', 'fingerprint']:
        opts.update({'permute' : permute, 'n_perms' : n_perms})
    if analysis is 'lagcrp':
        opts.update({'ts' : ts})

    return FriedEgg(data=_analyze_chunk(egg, **opts), analysis=analysis,
                    list_length=egg.list_length, n_lists=egg.n_lists,
                    n_subjects=egg.n_subjects, position=position)

Response: Analyze the data of a specific object and returns a tree of items that are part of the tree.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _analyze_chunk(data, subjgroup=None, subjname='Subject', listgroup=None,
                   listname='List', analysis=None, analysis_type=None,
                   pass_features=False, features=None, parallel=False,
                   **kwargs):
    """
    Private function that groups data by subject/list number and performs
    analysis for a chunk of data.

    Parameters
    ----------
    data : Egg data object
        The data to be analyzed

    subjgroup : list of strings or ints
        String/int variables indicating how to group over subjects.  Must be
        the length of the number of subjects

    subjname : string
        Name of the subject grouping variable

    listgroup : list of strings or ints
        String/int variables indicating how to group over list.  Must be
        the length of the number of lists

    listname : string
        Name of the list grouping variable

    analysis : function
        This function analyzes data and returns it.

    pass_features : bool
        Logical indicating whether the analyses uses the features field of the
        Egg

    Returns
    ----------
    analyzed_data : Pandas DataFrame
        DataFrame containing the analysis results

    """

    # perform the analysis
    def _analysis(c):
        subj, lst = c
        subjects = [s for s in subjdict[subj]]
        lists = [l for l in listdict[subj][lst]]
        s = data.crack(lists=lists, subjects=subjects)
        index = pd.MultiIndex.from_arrays([[subj],[lst]], names=[subjname, listname])
        opts = dict()
        if analysis_type is 'fingerprint':
                opts.update({'columns' : features})
        elif analysis_type is 'lagcrp':
            if kwargs['ts']:
                opts.update({'columns' : range(-kwargs['ts'],kwargs['ts']+1)})
            else:
                opts.update({'columns' : range(-data.list_length,data.list_length+1)})
        return pd.DataFrame([analysis(s, features=features, **kwargs)],
                            index=index, **opts)

    subjgroup = subjgroup if subjgroup else data.pres.index.levels[0].values
    listgroup = listgroup if listgroup else data.pres.index.levels[1].values

    subjdict = {subj : data.pres.index.levels[0].values[subj==np.array(subjgroup)] for subj in set(subjgroup)}

    if all(isinstance(el, list) for el in listgroup):
        listdict = [{lst : data.pres.index.levels[1].values[lst==np.array(listgrpsub)] for lst in set(listgrpsub)} for listgrpsub in listgroup]
    else:
        listdict = [{lst : data.pres.index.levels[1].values[lst==np.array(listgroup)] for lst in set(listgroup)} for subj in subjdict]

    chunks = [(subj, lst) for subj in subjdict for lst in listdict[0]]

    if parallel:
        import multiprocessing
        from pathos.multiprocessing import ProcessingPool as Pool
        p = Pool(multiprocessing.cpu_count())
        res = p.map(_analysis, chunks)
    else:
        res = [_analysis(c) for c in chunks]

    return pd.concat(res)

Response: Private function that performs the analysis of a single chunk of data.


Instruction:
Implement a Python 3 function for
retrieving values for graphs.

Response:
def retrieveVals(self):
        """Retrieve values for graphs."""
        if self.hasGraph('tomcat_memory'):
            stats = self._tomcatInfo.getMemoryStats()
            self.setGraphVal('tomcat_memory', 'used',
                             stats['total'] - stats['free'])
            self.setGraphVal('tomcat_memory', 'free', stats['free'])
            self.setGraphVal('tomcat_memory', 'max', stats['max'])
        for (port, stats) in self._tomcatInfo.getConnectorStats().iteritems():
            thrstats = stats['threadInfo']
            reqstats = stats['requestInfo']
            if self.portIncluded(port):
                name = "tomcat_threads_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'busy',
                                     thrstats['currentThreadsBusy'])
                    self.setGraphVal(name, 'idle',
                                     thrstats['currentThreadCount']
                                     - thrstats['currentThreadsBusy'])
                    self.setGraphVal(name, 'max', thrstats['maxThreads'])
                name = "tomcat_access_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'reqs', reqstats['requestCount'])
                name = "tomcat_error_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'errors', reqstats['errorCount'])
                name = "tomcat_traffic_%d" % port
                if self.hasGraph(name):
                    self.setGraphVal(name, 'rx', reqstats['bytesReceived'])
                    self.setGraphVal(name, 'tx', reqstats['bytesSent'])


Instruction:
Implement a function in Python 3 to
calculate the autocorrelation function for a 1D time series.

Response:
def function(data, maxt=None):
    """
    Calculate the autocorrelation function for a 1D time series.

    Parameters
    ----------
    data : numpy.ndarray (N,)
        The time series.

    Returns
    -------
    rho : numpy.ndarray (N,)
        An autocorrelation function.

    """
    data = np.atleast_1d(data)
    assert len(np.shape(data)) == 1, \
        "The autocorrelation function can only by computed " \
        + "on a 1D time series."
    if maxt is None:
        maxt = len(data)
    result = np.zeros(maxt, dtype=float)
    _acor.function(np.array(data, dtype=float), result)
    return result / result[0]


Instruction:
Here you have a function in Python 3, explain what it does
def retrieveVals(self):
        nginxInfo = NginxInfo(self._host, self._port,
                                self._user, self._password,
                                self._statuspath, self._ssl)
        stats = nginxInfo.getServerStats()
        if stats:
            if self.hasGraph('nginx_activeconn'):
                self.setGraphVal('nginx_activeconn', 'proc', stats['writing'])
                self.setGraphVal('nginx_activeconn', 'read', stats['reading'])
                self.setGraphVal('nginx_activeconn', 'wait', stats['waiting'])
                self.setGraphVal('nginx_activeconn', 'total',
                                 stats['connections'])
            if self.hasGraph('nginx_connections'):
                self.setGraphVal('nginx_connections', 'handled', stats['handled'])
                self.setGraphVal('nginx_connections', 'nothandled',
                                 stats['accepts'] - stats['handled'])
            if self.hasGraph('nginx_requests'):
                self.setGraphVal('nginx_requests', 'requests', stats['requests'])
            if self.hasGraph('nginx_requestsperconn'):
                curr_stats = (stats['handled'], stats['requests'])
                hist_stats = self.restoreState()
                if hist_stats:
                    prev_stats = hist_stats[0]
                else:
                    hist_stats = []
                    prev_stats = (0,0)
                conns = max(curr_stats[0] - prev_stats[0], 0)
                reqs = max(curr_stats[1] - prev_stats[1], 0)
                if conns > 0:
                    self.setGraphVal('nginx_requestsperconn', 'requests',
                                     float(reqs) / float(conns))
                else:
                    self.setGraphVal('nginx_requestsperconn', 'requests', 0)
                hist_stats.append(curr_stats)
                self.saveState(hist_stats[-self._numSamples:])

Response: Retrieve values for graphs.


Instruction:
How would you code a function in Python 3 to
implement Munin Plugin Auto - Configuration Option.

Response:
def autoconf(self):
        """Implements Munin Plugin Auto-Configuration Option.

        @return: True if plugin can be  auto-configured, False otherwise.

        """
        nginxInfo = NginxInfo(self._host, self._port,
                                self._user, self._password,
                                self._statuspath, self._ssl)
        return nginxInfo is not None


Instruction:
Can you generate the documentation for the following Python 3 function
def getStats(self):
        url = "%s://%s:%d/%s" % (self._proto, self._host, self._port,
                                 self._monpath)
        response = util.get_url(url, self._user, self._password)
        stats = {}
        for line in response.splitlines():
            mobj = re.match('([\w\s]+):\s+(\w+)$', line)
            if mobj:
                stats[mobj.group(1)] = util.parse_value(mobj.group(2))
        return stats

Response: Query and parse Web Server Status Page.


Instruction:
Given the following Python 3 function, write the documentation
def retrieveVals(self):
        apcinfo = APCinfo(self._host, self._port, self._user, self._password,
                          self._monpath, self._ssl, self._extras)
        stats = apcinfo.getAllStats()

        if self.hasGraph('php_apc_memory') and stats:
            filecache = stats['cache_sys']['mem_size']
            usercache = stats['cache_user']['mem_size']
            total = stats['memory']['seg_size'] * stats['memory']['num_seg']
            free = stats['memory']['avail_mem']
            other = total - free - filecache - usercache
            self.setGraphVal('php_apc_memory', 'filecache', filecache)
            self.setGraphVal('php_apc_memory', 'usercache', usercache)
            self.setGraphVal('php_apc_memory', 'other', other)
            self.setGraphVal('php_apc_memory', 'free', free)
        if self.hasGraph('php_apc_items') and stats:
            self.setGraphVal('php_apc_items', 'filecache',
                             stats['cache_sys']['num_entries'])
            self.setGraphVal('php_apc_items', 'usercache',
                             stats['cache_user']['num_entries'])
        if self.hasGraph('php_apc_reqs_filecache') and stats:
            self.setGraphVal('php_apc_reqs_filecache', 'hits',
                             stats['cache_sys']['num_hits'])
            self.setGraphVal('php_apc_reqs_filecache', 'misses',
                             stats['cache_sys']['num_misses'])
            self.setGraphVal('php_apc_reqs_filecache', 'inserts',
                             stats['cache_sys']['num_inserts'])
        if self.hasGraph('php_apc_reqs_usercache') and stats:
            self.setGraphVal('php_apc_reqs_usercache', 'hits',
                             stats['cache_user']['num_hits'])
            self.setGraphVal('php_apc_reqs_usercache', 'misses',
                             stats['cache_user']['num_misses'])
            self.setGraphVal('php_apc_reqs_usercache', 'inserts',
                             stats['cache_user']['num_inserts'])
        if self.hasGraph('php_apc_expunge') and stats:
            self.setGraphVal('php_apc_expunge', 'filecache',
                             stats['cache_sys']['expunges'])
            self.setGraphVal('php_apc_expunge', 'usercache',
                             stats['cache_user']['expunges'])
        if self.hasGraph('php_apc_mem_util_frag'):
            self.setGraphVal('php_apc_mem_util_frag', 'util',
                             stats['memory']['utilization_ratio'] * 100)
            self.setGraphVal('php_apc_mem_util_frag', 'frag',
                             stats['memory']['fragmentation_ratio'] * 100)
        if self.hasGraph('php_apc_mem_frag_count'):
            self.setGraphVal('php_apc_mem_frag_count', 'num',
                             stats['memory']['fragment_count'])
        if self.hasGraph('php_apc_mem_frag_avgsize'):
            self.setGraphVal('php_apc_mem_frag_avgsize', 'size',
                             stats['memory']['fragment_avg_size'])

Response: Retrieve values for graphs.


Instruction:
How would you code a function in Python 3 to
implement Munin Plugin Auto - Configuration Option.

Response:
def autoconf(self):
        """Implements Munin Plugin Auto-Configuration Option.

        @return: True if plugin can be  auto-configured, False otherwise.

        """
        apcinfo = APCinfo(self._host, self._port, self._user, self._password,
                          self._monpath, self._ssl)
        return apcinfo is not None


Instruction:
Implement a function in Python 3 to
run varnishstats command to get stats from Varnish Cache.

Response:
def getStats(self):
        """Runs varnishstats command to get stats from Varnish Cache.

        @return: Dictionary of stats.

        """
        info_dict = {}
        args = [varnishstatCmd, '-1']
        if self._instance is not None:
            args.extend(['-n', self._instance])
        output = util.exec_command(args)
        if self._descDict is None:
            self._descDict = {}
        for line in output.splitlines():
            mobj = re.match('(\S+)\s+(\d+)\s+(\d+\.\d+|\.)\s+(\S.*\S)\s*$',
                            line)
            if mobj:
                fname = mobj.group(1).replace('.', '_')
                info_dict[fname] = util.parse_value(mobj.group(2))
                self._descDict[fname] = mobj.group(4)
        return info_dict


Instruction:
Write a Python 3 function for
returning description for entry.

Response:
def getDesc(self, entry):
        """Returns description for stat entry.

        @param entry: Entry name.
        @return:      Description for entry.

        """
        if len(self._descDict) == 0:
            self.getStats()
        return self._descDict.get(entry)


Instruction:
How would you explain what the following Python 3 function does
def retrieveVals(self):
        opcinfo = OPCinfo(self._host, self._port, self._user, self._password,
                          self._monpath, self._ssl)
        stats = opcinfo.getAllStats()

        if self.hasGraph('php_opc_memory') and stats:
            mem = stats['memory_usage']
            keys = ('used_memory', 'wasted_memory', 'free_memory')
            map(lambda k:self.setGraphVal('php_opc_memory',k,mem[k]), keys)

        if self.hasGraph('php_opc_opcache_statistics') and stats:
            st = stats['opcache_statistics']
            self.setGraphVal('php_opc_opcache_statistics', 'hits',
                             st['hits'])
            self.setGraphVal('php_opc_opcache_statistics', 'misses',
                             st['misses'])

        if self.hasGraph('php_opc_opcache_hitrate') and stats:
            st = stats['opcache_statistics']
            self.setGraphVal('php_opc_opcache_hitrate', 'opcache_hit_rate',
                             st['opcache_hit_rate'])

        if self.hasGraph('php_opc_key_status') and stats:
            st = stats['opcache_statistics']
            wasted = st['num_cached_keys'] - st['num_cached_scripts']
            free = st['max_cached_keys'] - st['num_cached_keys']
            self.setGraphVal('php_opc_key_status', 'num_cached_scripts', st['num_cached_scripts'])
            self.setGraphVal('php_opc_key_status', 'num_wasted_keys', wasted)
            self.setGraphVal('php_opc_key_status', 'num_free_keys', free)

Response: Retrieve values for graphs.


Instruction:
Can you tell what is the following Python 3 function doing
def autoconf(self):
        opcinfo = OPCinfo(self._host, self._port, self._user, self._password,
                          self._monpath, self._ssl)
        return opcinfo is not None

Response: Implements Munin Plugin Auto - Configuration Option.


Instruction:
Make a summary of the following Python 3 code
def fingerprint_helper(egg, permute=False, n_perms=1000,
                       match='exact', distance='euclidean', features=None):
    """
    Computes clustering along a set of feature dimensions

    Parameters
    ----------
    egg : quail.Egg
        Data to analyze

    dist_funcs : dict
        Dictionary of distance functions for feature clustering analyses

    Returns
    ----------
    probabilities : Numpy array
      Each number represents clustering along a different feature dimension

    """

    if features is None:
        features = egg.dist_funcs.keys()

    inds = egg.pres.index.tolist()
    slices = [egg.crack(subjects=[i], lists=[j]) for i, j in inds]

    weights = _get_weights(slices, features, distdict, permute, n_perms, match,
                            distance)
    return np.nanmean(weights, axis=0)

Response: This function is used by the fingerprint_helper function to compute the fingerprint of a set of features.


Instruction:
Implement a Python 3 function for
computing the clustering scores along a set of feature dimensions and the distance vector of the next word in the encoding list.

Response:
def compute_feature_weights(pres_list, rec_list, feature_list, distances):
    """
    Compute clustering scores along a set of feature dimensions

    Parameters
    ----------
    pres_list : list
        list of presented words

    rec_list : list
        list of recalled words

    feature_list : list
        list of feature dicts for presented words

    distances : dict
        dict of distance matrices for each feature

    Returns
    ----------
    weights : list
        list of clustering scores for each feature dimension
    """

    # initialize the weights object for just this list
    weights = {}
    for feature in feature_list[0]:
        weights[feature] = []

    # return default list if there is not enough data to compute the fingerprint
    if len(rec_list) <= 2:
        print('Not enough recalls to compute fingerprint, returning default'
              'fingerprint.. (everything is .5)')
        for feature in feature_list[0]:
            weights[feature] = .5
        return [weights[key] for key in weights]

    # initialize past word list
    past_words = []
    past_idxs = []

    # loop over words
    for i in range(len(rec_list)-1):

        # grab current word
        c = rec_list[i]

        # grab the next word
        n = rec_list[i + 1]

        # if both recalled words are in the encoding list and haven't been recalled before
        if (c in pres_list and n in pres_list) and (c not in past_words and n not in past_words):

            # for each feature
            for feature in feature_list[0]:

                # get the distance vector for the current word
                dists = distances[feature][pres_list.index(c),:]

                # distance between current and next word
                cdist = dists[pres_list.index(n)]

                # filter dists removing the words that have already been recalled
                dists_filt = np.array([dist for idx, dist in enumerate(dists) if idx not in past_idxs])

                # get indices
                avg_rank = np.mean(np.where(np.sort(dists_filt)[::-1] == cdist)[0]+1)

                # compute the weight
                weights[feature].append(avg_rank / len(dists_filt))

            # keep track of what has been recalled already
            past_idxs.append(pres_list.index(c))
            past_words.append(c)

    # average over the cluster scores for a particular dimension
    for feature in weights:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", category=RuntimeWarning)
            weights[feature] = np.nanmean(weights[feature])

    return [weights[key] for key in weights]


Instruction:
Write a Python 3 function that can
compute probabilities for each transition distance (probability that a word recalled will be a given distance--in presentation order--from the previous recalled word). Parameters ---------- egg : quail.Egg Data to analyze match : str (exact, best or smooth) Matching approach to compute recall matrix. If exact, the presented and recalled items must be identical (default). If best, the recalled item that is most similar to the presented items will be selected. If smooth, a weighted average of all presented items will be used, where the weights are derived from the similarity between the recalled item and each presented item. distance : str The distance function used to compare presented and recalled items. Applies only to 'best' and 'smooth' matching approaches. Can be any distance function supported by numpy.spatial.distance.cdist. Returns ---------- prec : numpy array each float is the probability of transition distance (distnaces indexed by position, from -(n-1) to (n-1), excluding zero

Response:
def lagcrp_helper(egg, match='exact', distance='euclidean',
                  ts=None, features=None):
    """
    Computes probabilities for each transition distance (probability that a word
    recalled will be a given distance--in presentation order--from the previous
    recalled word).

    Parameters
    ----------
    egg : quail.Egg
        Data to analyze

    match : str (exact, best or smooth)
        Matching approach to compute recall matrix.  If exact, the presented and
        recalled items must be identical (default).  If best, the recalled item
        that is most similar to the presented items will be selected. If smooth,
        a weighted average of all presented items will be used, where the
        weights are derived from the similarity between the recalled item and
        each presented item.

    distance : str
        The distance function used to compare presented and recalled items.
        Applies only to 'best' and 'smooth' matching approaches.  Can be any
        distance function supported by numpy.spatial.distance.cdist.

    Returns
    ----------
    prec : numpy array
      each float is the probability of transition distance (distnaces indexed by
      position, from -(n-1) to (n-1), excluding zero

    """

    def lagcrp(rec, lstlen):
        """Computes lag-crp for a given recall list"""

        def check_pair(a, b):
            if (a>0 and b>0) and (a!=b):
                return True
            else:
                return False

        def compute_actual(rec, lstlen):
            arr=pd.Series(data=np.zeros((lstlen)*2),
                          index=list(range(-lstlen,0))+list(range(1,lstlen+1)))
            recalled=[]
            for trial in range(0,len(rec)-1):
                a=rec[trial]
                b=rec[trial+1]
                if check_pair(a, b) and (a not in recalled) and (b not in recalled):
                    arr[b-a]+=1
                recalled.append(a)
            return arr

        def compute_possible(rec, lstlen):
            arr=pd.Series(data=np.zeros((lstlen)*2),
                          index=list(range(-lstlen,0))+list(range(1,lstlen+1)))
            recalled=[]
            for trial in rec:
                if np.isnan(trial):
                    pass
                else:
                    lbound=int(1-trial)
                    ubound=int(lstlen-trial)
                    chances=list(range(lbound,0))+list(range(1,ubound+1))
                    for each in recalled:
                        if each-trial in chances:
                            chances.remove(each-trial)
                    arr[chances]+=1
                    recalled.append(trial)
            return arr

        actual = compute_actual(rec, lstlen)
        possible = compute_possible(rec, lstlen)
        crp = [0.0 if j == 0 else i / j for i, j in zip(actual, possible)]
        crp.insert(int(len(crp) / 2), np.nan)
        return crp

    def nlagcrp(distmat, ts=None):

        def lagcrp_model(s):
            idx = list(range(0, -s, -1))
            return np.array([list(range(i, i+s)) for i in idx])

        # remove nan columns
        distmat = distmat[:,~np.all(np.isnan(distmat), axis=0)].T

        model = lagcrp_model(distmat.shape[1])
        lagcrp = np.zeros(ts * 2)
        for rdx in range(len(distmat)-1):
            item = distmat[rdx, :]
            next_item = distmat[rdx+1, :]
            if not np.isnan(item).any() and not np.isnan(next_item).any():
                outer = np.outer(item, next_item)
                lagcrp += np.array(list(map(lambda lag: np.mean(outer[model==lag]), range(-ts, ts))))
        lagcrp /= ts
        lagcrp = list(lagcrp)
        lagcrp.insert(int(len(lagcrp) / 2), np.nan)
        return np.array(lagcrp)

    def _format(p, r):
        p = np.matrix([np.array(i) for i in p])
        if p.shape[0]==1:
            p=p.T
        r = map(lambda x: [np.nan]*p.shape[1] if check_nan(x) else x, r)
        r = np.matrix([np.array(i) for i in r])
        if r.shape[0]==1:
            r=r.T
        return p, r

    opts = dict(match=match, distance=distance, features=features)
    if match is 'exact':
        opts.update({'features' : 'item'})
    recmat = recall_matrix(egg, **opts)
    if not ts:
        ts = egg.pres.shape[1]
    if match in ['exact', 'best']:
        lagcrp = [lagcrp(lst, egg.list_length) for lst in recmat]
    elif match is 'smooth':
        lagcrp = np.atleast_2d(np.mean([nlagcrp(r, ts=ts) for r in recmat], 0))
    else:
        raise ValueError('Match must be set to exact, best or smooth.')
    return np.nanmean(lagcrp, axis=0)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def retrieveVals(self):
        if self._diskList:
            self._fetchDevAll('disk', self._diskList,
                              self._info.getDiskStats)
        if self._mdList:
            self._fetchDevAll('md', self._mdList,
                              self._info.getMDstats)
        if self._partList:
            self._fetchDevAll('part', self._partList,
                              self._info.getPartitionStats)
        if self._lvList:
            self._fetchDevAll('lv', self._lvList,
                              self._info.getLVstats)
        self._fetchDevAll('fs', self._fsList,
                          self._info.getFilesystemStats)

Response: Retrieve values for graphs.


Instruction:
Make a summary of the following Python 3 code
def _configDevRequests(self, namestr, titlestr, devlist):
        name = 'diskio_%s_requests' % namestr
        if self.graphEnabled(name):
            graph = MuninGraph('Disk I/O - %s - Requests' % titlestr, self._category,
                info='Disk I/O - %s Throughput, Read / write requests per second.'
                     % titlestr,
                args='--base 1000 --lower-limit 0',
                vlabel='reqs/sec read (-) / write (+)', printf='%6.1lf',
                autoFixNames = True)
            for dev in devlist:
                graph.addField(dev + '_read',
                               fixLabel(dev, maxLabelLenGraphDual,
                                        repl = '..', truncend=False,
                                        delim = self._labelDelim.get(namestr)),
                               draw='LINE2', type='DERIVE', min=0, graph=False)
                graph.addField(dev + '_write',
                               fixLabel(dev, maxLabelLenGraphDual,
                                        repl = '..', truncend=False,
                                        delim = self._labelDelim.get(namestr)),
                               draw='LINE2', type='DERIVE', min=0,
                               negative=(dev + '_read'),info=dev)
            self.appendGraph(name, graph)

Response: Generate configuration for disk I / O Request stats.


Instruction:
How would you explain what the following Python 3 function does
def _configDevActive(self, namestr, titlestr, devlist):
        name = 'diskio_%s_active' % namestr
        if self.graphEnabled(name):
            graph = MuninGraph('Disk I/O - %s - Queue Length' % titlestr,
                self._category,
                info='Disk I/O - Number  of I/O Operations in Progress for every %s.'
                     % titlestr,
                args='--base 1000 --lower-limit 0', printf='%6.1lf',
                autoFixNames = True)
            for dev in devlist:
                graph.addField(dev,
                               fixLabel(dev, maxLabelLenGraphSimple,
                                        repl = '..', truncend=False,
                                        delim = self._labelDelim.get(namestr)),
                               draw='AREASTACK', type='GAUGE', info=dev)
            self.appendGraph(name, graph)

Response: Generate configuration for the active disk i / o queue length.


Instruction:
Here you have a function in Python 3, explain what it does
def _fetchDevAll(self, namestr, devlist, statsfunc):
        for dev in devlist:
            stats = statsfunc(dev)
            name = 'diskio_%s_requests' % namestr
            if self.hasGraph(name):
                self.setGraphVal(name, dev + '_read', stats['rios'])
                self.setGraphVal(name, dev + '_write', stats['wios'])
            name = 'diskio_%s_bytes' % namestr
            if self.hasGraph(name):
                self.setGraphVal(name, dev + '_read', stats['rbytes'])
                self.setGraphVal(name, dev + '_write', stats['wbytes'])
            name = 'diskio_%s_active' % namestr
            if self.hasGraph(name):
                self.setGraphVal(name, dev, stats['ios_active'])

Response: Initialize I / O stats for devices.


Instruction:
Implement a function in Python 3 to
retrieve values for graphs.

Response:
def retrieveVals(self):
        """Retrieve values for graphs."""
        for graph_name in self.getGraphList():
            for field_name in self.getGraphFieldList(graph_name):
                self.setGraphVal(graph_name, field_name, self._stats.get(field_name))


Instruction:
Write a Python 3 script to
retrieve values for graphs.

Response:
def retrieveVals(self):
        """Retrieve values for graphs."""
        if self.hasGraph('sys_loadavg'):
            self._loadstats = self._sysinfo.getLoadAvg()
            if self._loadstats:
                self.setGraphVal('sys_loadavg', 'load15min', self._loadstats[2])
                self.setGraphVal('sys_loadavg', 'load5min', self._loadstats[1])
                self.setGraphVal('sys_loadavg', 'load1min', self._loadstats[0])
        if self._cpustats and self.hasGraph('sys_cpu_util'):
            for field in self.getGraphFieldList('sys_cpu_util'):
                self.setGraphVal('sys_cpu_util',
                                 field, int(self._cpustats[field] * 1000))
        if self._memstats:
            if self.hasGraph('sys_mem_util'):
                for field in self.getGraphFieldList('sys_mem_util'):
                    self.setGraphVal('sys_mem_util',
                                     field, self._memstats[field])
            if self.hasGraph('sys_mem_avail'):
                for field in self.getGraphFieldList('sys_mem_avail'):
                    self.setGraphVal('sys_mem_avail',
                                     field, self._memstats[field])
            if self.hasGraph('sys_mem_huge'):
                for field in ['Rsvd', 'Surp', 'Free']:
                    fkey = 'HugePages_' + field
                    if self._memstats.has_key(fkey):
                        self.setGraphVal('sys_mem_huge', field,
                            self._memstats[fkey] * self._memstats['Hugepagesize'])
        if self.hasGraph('sys_processes'):
            if self._procstats is None:
                self._procstats = self._sysinfo.getProcessStats()
            if self._procstats:
                self.setGraphVal('sys_processes', 'running',
                                 self._procstats['procs_running'])
                self.setGraphVal('sys_processes', 'blocked',
                                 self._procstats['procs_blocked'])
        if self.hasGraph('sys_forks'):
            if self._procstats is None:
                self._procstats = self._sysinfo.getProcessStats()
            if self._procstats:
                self.setGraphVal('sys_forks', 'forks',
                                 self._procstats['processes'])
        if self.hasGraph('sys_intr_ctxt'):
            if self._procstats is None:
                self._procstats = self._sysinfo.getProcessStats()
            if self._procstats:
                for field in self.getGraphFieldList('sys_intr_ctxt'):
                    self.setGraphVal('sys_intr_ctxt', field,
                                     self._procstats[field])
        if self.hasGraph('sys_vm_paging'):
            if self._vmstats is None:
                self._vmstats = self._sysinfo.getVMstats()
            if self._vmstats:
                self.setGraphVal('sys_vm_paging', 'in',
                                 self._vmstats['pgpgin'])
                self.setGraphVal('sys_vm_paging', 'out',
                                 self._vmstats['pgpgout'])
        if self.hasGraph('sys_vm_swapping'):
            if self._vmstats is None:
                self._vmstats = self._sysinfo.getVMstats()
            if self._vmstats:
                self.setGraphVal('sys_vm_swapping', 'in',
                                 self._vmstats['pswpin'])
                self.setGraphVal('sys_vm_swapping', 'out',
                                 self._vmstats['pswpout'])


Instruction:
Can you write a function in Python 3 where it
searches os. environ and returns the string value of the given key.

Response:
def get(key, default=None):
    """
        Searches os.environ. If a key is found try evaluating its type else;
        return the string.

        returns: k->value (type as defined by ast.literal_eval)
    """
    try:
        # Attempt to evaluate into python literal
        return ast.literal_eval(os.environ.get(key.upper(), default))
    except (ValueError, SyntaxError):
        return os.environ.get(key.upper(), default)


Instruction:
Write a Python 3 script to
save a list of keyword arguments as environment variables to a file.

Response:
def save(filepath=None, **kwargs):
    """
        Saves a list of keyword arguments as environment variables to a file.
        If no filepath given will default to the default `.env` file.
    """
    if filepath is None:
        filepath = os.path.join('.env')

    with open(filepath, 'wb') as file_handle:
        file_handle.writelines(
            '{0}={1}\n'.format(key.upper(), val)
            for key, val in kwargs.items()
        )


Instruction:
Make a summary of the following Python 3 code
def load(filepath=None):
    if filepath and os.path.exists(filepath):
        pass
    else:
        if not os.path.exists('.env'):
            return False
        filepath = os.path.join('.env')

    for key, value in _get_line_(filepath):
        # set the key, value in the python environment vars dictionary
        # does not make modifications system wide.
        os.environ.setdefault(key, str(value))
    return True

Response: Reads a. env file into the dictionary that contains the current Python environment vars.


Instruction:
Create a Python 3 function for
getting each line from the file and parses it into a list of tuples.

Response:
def _get_line_(filepath):
    """
    Gets each line from the file and parse the data.
    Attempt to translate the value into a python type is possible
    (falls back to string).
    """
    for line in open(filepath):
        line = line.strip()
        # allows for comments in the file
        if line.startswith('#') or '=' not in line:
            continue
        # split on the first =, allows for subsiquent `=` in strings
        key, value = line.split('=', 1)
        key = key.strip().upper()
        value = value.strip()

        if not (key and value):
            continue

        try:
            # evaluate the string before adding into environment
            # resolves any hanging (') characters
            value = ast.literal_eval(value)
        except (ValueError, SyntaxError):
            pass

        #return line
        yield (key, value)


Instruction:
Can you tell what is the following Python 3 function doing
def initStats(self):
        url = "%s://%s:%d/%s?auto"  % (self._proto, self._host, self._port,
                                       self._statuspath)
        response = util.get_url(url, self._user, self._password)
        self._statusDict = {}
        for line in response.splitlines():
            mobj = re.match('(\S.*\S)\s*:\s*(\S+)\s*$', line)
            if mobj:
                self._statusDict[mobj.group(1)] = util.parse_value(mobj.group(2))
        if self._statusDict.has_key('Scoreboard'):
            self._statusDict['MaxWorkers'] = len(self._statusDict['Scoreboard'])

Response: Query and parse Apache Web Server Status Page and initialize self. _statusDict.


Instruction:
Explain what the following Python 3 code does
def get_pres_features(self, features=None):
        if features is None:
            features = self.dist_funcs.keys()
        elif not isinstance(features, list):
            features = [features]
        return self.pres.applymap(lambda x: {k:v for k,v in x.items() if k in features} if x is not None else None)

Response: Returns a df of features for presented items


Instruction:
Can you generate the documentation for the following Python 3 function
def get_rec_features(self, features=None):
        if features is None:
            features = self.dist_funcs.keys()
        elif not isinstance(features, list):
            features = [features]
        return self.rec.applymap(lambda x: {k:v for k,v in x.items() if k != 'item'} if x is not None else None)

Response: Returns a df of features for recalled items


Instruction:
Implement a function in Python 3 to
print info about the data egg

Response:
def info(self):
        """
        Print info about the data egg
        """
        print('Number of subjects: ' + str(self.n_subjects))
        print('Number of lists per subject: ' + str(self.n_lists))
        print('Number of words per list: ' + str(self.list_length))
        print('Date created: ' + str(self.date_created))
        print('Meta data: ' + str(self.meta))


Instruction:
How would you code a function in Python 3 to
save the object to a file.

Response:
def save(self, fname, compression='blosc'):
        """
        Save method for the Egg object

        The data will be saved as a 'egg' file, which is a dictionary containing
        the elements of a Egg saved in the hd5 format using
        `deepdish`.

        Parameters
        ----------

        fname : str
            A name for the file.  If the file extension (.egg) is not specified,
            it will be appended.

        compression : str
            The kind of compression to use.  See the deepdish documentation for
            options: http://deepdish.readthedocs.io/en/latest/api_io.html#deepdish.io.save

        """

        # put egg vars into a dict
        egg = {
            'pres' : df2list(self.pres),
            'rec' : df2list(self.rec),
            'dist_funcs' : self.dist_funcs,
            'subjgroup' : self.subjgroup,
            'subjname' : self.subjname,
            'listgroup' : self.listgroup,
            'listname' : self.listname,
            'date_created' : self.date_created,
            'meta' : self.meta
        }

        # if extension wasn't included, add it
        if fname[-4:]!='.egg':
            fname+='.egg'

        # save
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            dd.io.save(fname, egg, compression=compression)


Instruction:
How would you implement a function in Python 3 that
saves the object to a file.

Response:
def save(self, fname, compression='blosc'):
        """
        Save method for the FriedEgg object

        The data will be saved as a 'fegg' file, which is a dictionary containing
        the elements of a FriedEgg saved in the hd5 format using
        `deepdish`.

        Parameters
        ----------

        fname : str
            A name for the file.  If the file extension (.fegg) is not specified,
            it will be appended.

        compression : str
            The kind of compression to use.  See the deepdish documentation for
            options: http://deepdish.readthedocs.io/en/latest/api_io.html#deepdish.io.save

        """

        egg = {
            'data' : self.data,
            'analysis' : self.analysis,
            'list_length' : self.list_length,
            'n_lists' : self.n_lists,
            'n_subjects' : self.n_subjects,
            'position' : self.position,
            'date_created' : self.date_created,
            'meta' : self.meta
        }

        if fname[-4:]!='.fegg':
            fname+='.fegg'

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            dd.io.save(fname, egg, compression=compression)


Instruction:
How would you explain what the following Python 3 function does
def pnr_helper(egg, position, match='exact',
               distance='euclidean', features=None):

    """
    Computes probability of a word being recalled nth (in the appropriate recall
    list), given its presentation position.  Note: zero indexed

    Parameters
    ----------
    egg : quail.Egg
        Data to analyze

    position : int
        Position of item to be analyzed

    match : str (exact, best or smooth)
        Matching approach to compute recall matrix.  If exact, the presented and
        recalled items must be identical (default).  If best, the recalled item
        that is most similar to the presented items will be selected. If smooth,
        a weighted average of all presented items will be used, where the
        weights are derived from the similarity between the recalled item and
        each presented item.

    distance : str
        The distance function used to compare presented and recalled items.
        Applies only to 'best' and 'smooth' matching approaches.  Can be any
        distance function supported by numpy.spatial.distance.cdist.

    Returns
    ----------
    prob_recalled : numpy array
      each number represents the probability of nth recall for a word presented in given position/index

    """
    def pnr(lst, position):
        return [1 if pos==lst[position] else 0 for pos in range(1,egg.list_length+1)]

    opts = dict(match=match, distance=distance, features=features)
    if match is 'exact':
        opts.update({'features' : 'item'})
    recmat = recall_matrix(egg, **opts)

    if match in ['exact', 'best']:
        result = [pnr(lst, position) for lst in recmat]
    elif match is 'smooth':
        result = np.atleast_2d(recmat[:, :, 0])
    else:
        raise ValueError('Match must be set to exact, best or smooth.')
    return np.nanmean(result, axis=0)

Response: Returns the probability of a word being recalled nth for a given item in the given presentation position.


Instruction:
How would you code a function in Python 3 to
retrieve values for graphs.

Response:
def retrieveVals(self):
        """Retrieve values for graphs."""
        apacheInfo = ApacheInfo(self._host, self._port,
                                self._user, self._password,
                                self._statuspath, self._ssl)
        stats = apacheInfo.getServerStats()
        if self.hasGraph('apache_access'):
            self.setGraphVal('apache_access', 'reqs', stats['Total Accesses'])
        if self.hasGraph('apache_bytes'):
            self.setGraphVal('apache_bytes', 'bytes',
                             stats['Total kBytes'] * 1000)
        if self.hasGraph('apache_workers'):
            self.setGraphVal('apache_workers', 'busy', stats['BusyWorkers'])
            self.setGraphVal('apache_workers', 'idle', stats['IdleWorkers'])
            self.setGraphVal('apache_workers', 'max', stats['MaxWorkers'])


Instruction:
Write a Python 3 function that can
implement Munin Plugin Auto - Configuration Option.

Response:
def autoconf(self):
        """Implements Munin Plugin Auto-Configuration Option.

        @return: True if plugin can be  auto-configured, False otherwise.

        """
        apacheInfo = ApacheInfo(self._host, self._port,
                                self._user, self._password,
                                self._statuspath, self._ssl)
        return apacheInfo is not None


Instruction:
How would you implement a function in Python 3 that
retrieves values for graphs.

Response:
def retrieveVals(self):
        """Retrieve values for graphs."""
        ntpinfo = NTPinfo()
        ntpstats = ntpinfo.getHostOffsets(self._remoteHosts)
        if ntpstats:
            for host in self._remoteHosts:
                hostkey = re.sub('\.', '_', host)
                hoststats = ntpstats.get(host)
                if hoststats:
                    if self.hasGraph('ntp_host_stratums'):
                        self.setGraphVal('ntp_host_stratums', hostkey,
                                         hoststats.get('stratum'))
                    if self.hasGraph('ntp_host_offsets'):
                        self.setGraphVal('ntp_host_offsets', hostkey,
                                         hoststats.get('offset'))
                    if self.hasGraph('ntp_host_delays'):
                        self.setGraphVal('ntp_host_delays', hostkey,
                                         hoststats.get('delay'))


Instruction:
Implement a Python 3 function for
implementing Munin Plugin Auto - Configuration Option.

Response:
def autoconf(self):
        """Implements Munin Plugin Auto-Configuration Option.

        @return: True if plugin can be  auto-configured, False otherwise.

        """
        ntpinfo = NTPinfo()
        ntpstats = ntpinfo.getHostOffsets(self._remoteHosts)
        return len(ntpstats) > 0


Instruction:
Make a summary of the following Python 3 code
def retrieveVals(self):
        lighttpdInfo = LighttpdInfo(self._host, self._port,
                                self._user, self._password,
                                self._statuspath, self._ssl)
        stats = lighttpdInfo.getServerStats()
        if self.hasGraph('lighttpd_access'):
            self.setGraphVal('lighttpd_access', 'reqs', stats['Total Accesses'])
        if self.hasGraph('lighttpd_bytes'):
            self.setGraphVal('lighttpd_bytes', 'bytes',
                             stats['Total kBytes'] * 1000)
        if self.hasGraph('lighttpd_servers'):
            self.setGraphVal('lighttpd_servers', 'busy', stats['BusyServers'])
            self.setGraphVal('lighttpd_servers', 'idle', stats['IdleServers'])
            self.setGraphVal('lighttpd_servers', 'max', stats['MaxServers'])

Response: Retrieve values for graphs.


Instruction:
Make a summary of the following Python 3 code
def autoconf(self):
        lighttpdInfo = LighttpdInfo(self._host, self._port,
                                self._user, self._password,
                                self._statuspath, self._ssl)
        return lighttpdInfo is not None

Response: Implements Munin Plugin Auto - Configuration Option.


Instruction:
Make a summary of the following Python 3 code
def retrieveVals(self):
        name = 'diskspace'
        if self.hasGraph(name):
            for fspath in self._fslist:
                if self._statsSpace.has_key(fspath):
                    self.setGraphVal(name, fspath,
                                     self._statsSpace[fspath]['inuse_pcent'])
        name = 'diskinode'
        if self.hasGraph(name):
            for fspath in self._fslist:
                if self._statsInode.has_key(fspath):
                    self.setGraphVal(name, fspath,
                                     self._statsInode[fspath]['inuse_pcent'])

Response: Retrieve values for graphs.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def symbolic(self, mtx):
        self.free_symbolic()

        indx = self._getIndx(mtx)

        if not assumeSortedIndices:
            # row/column indices cannot be assumed to be sorted
            mtx.sort_indices()

        if self.isReal:
            status, self._symbolic\
                    = self.funs.symbolic(mtx.shape[0], mtx.shape[1],
                                          mtx.indptr,
                                          indx,
                                          mtx.data,
                                          self.control, self.info)
        else:
            real, imag = mtx.data.real.copy(), mtx.data.imag.copy()
            status, self._symbolic\
                    = self.funs.symbolic(mtx.shape[0], mtx.shape[1],
                                          mtx.indptr,
                                          indx,
                                          real, imag,
                                          self.control, self.info)

        if status != UMFPACK_OK:
            raise RuntimeError('%s failed with %s' % (self.funs.symbolic,
                                                       umfStatus[status]))

        self.mtx = mtx

Response: Perform a symbolic LU decomposition of a given object.


Instruction:
Make a summary of the following Python 3 code
def numeric(self, mtx):

        self.free_numeric()

        if self._symbolic is None:
            self.symbolic(mtx)

        indx = self._getIndx(mtx)

        failCount = 0
        while 1:
            if self.isReal:
                status, self._numeric\
                        = self.funs.numeric(mtx.indptr, indx, mtx.data,
                                             self._symbolic,
                                             self.control, self.info)
            else:
                real, imag = mtx.data.real.copy(), mtx.data.imag.copy()
                status, self._numeric\
                        = self.funs.numeric(mtx.indptr, indx,
                                             real, imag,
                                             self._symbolic,
                                             self.control, self.info)

            if status != UMFPACK_OK:
                if status == UMFPACK_WARNING_singular_matrix:
                    warnings.warn('Singular matrix', UmfpackWarning)
                    break
                elif status in (UMFPACK_ERROR_different_pattern,
                                UMFPACK_ERROR_invalid_Symbolic_object):
                    # Try again.
                    warnings.warn('Recomputing symbolic', UmfpackWarning)
                    self.symbolic(mtx)
                    failCount += 1
                else:
                    failCount += 100
            else:
                break
            if failCount >= 2:
                raise RuntimeError('%s failed with %s' % (self.funs.numeric,
                                                           umfStatus[status]))

Response: Perform numeric object decomposition using the symbolic decomposition.


Instruction:
Can you implement a function in Python 3 that
solves the system of linear equation using the Numeric object.

Response:
def solve(self, sys, mtx, rhs, autoTranspose=False):
        """
        Solution of system of linear equation using the Numeric object.

        Parameters
        ----------
        sys : constant
            one of UMFPACK system description constants, like
            UMFPACK_A, UMFPACK_At, see umfSys list and UMFPACK docs
        mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix
            Input.
        rhs : ndarray
            Right Hand Side
        autoTranspose : bool
            Automatically changes `sys` to the transposed type, if `mtx` is in CSR,
            since UMFPACK assumes CSC internally

        Returns
        -------
        sol : ndarray
            Solution to the equation system.

        """
        if sys not in umfSys:
            raise ValueError('sys must be in' % umfSys)

        if autoTranspose and self.isCSR:
            ##
            # UMFPACK uses CSC internally...
            if self.family in umfRealTypes:
                ii = 0
            else:
                ii = 1
            if sys in umfSys_transposeMap[ii]:
                sys = umfSys_transposeMap[ii][sys]
            else:
                raise RuntimeError('autoTranspose ambiguous, switch it off')

        if self._numeric is not None:
            if self.mtx is not mtx:
                raise ValueError('must be called with same matrix as numeric()')
        else:
            raise RuntimeError('numeric() not called')

        indx = self._getIndx(mtx)

        if self.isReal:
            rhs = rhs.astype(np.float64)
            sol = np.zeros((mtx.shape[1],), dtype=np.float64)
            status = self.funs.solve(sys, mtx.indptr, indx, mtx.data, sol, rhs,
                                      self._numeric, self.control, self.info)
        else:
            rhs = rhs.astype(np.complex128)
            sol = np.zeros((mtx.shape[1],), dtype=np.complex128)
            mreal, mimag = mtx.data.real.copy(), mtx.data.imag.copy()
            sreal, simag = sol.real.copy(), sol.imag.copy()
            rreal, rimag = rhs.real.copy(), rhs.imag.copy()
            status = self.funs.solve(sys, mtx.indptr, indx,
                                      mreal, mimag, sreal, simag, rreal, rimag,
                                      self._numeric, self.control, self.info)
            sol.real, sol.imag = sreal, simag

        # self.funs.report_info( self.control, self.info )
        # pause()
        if status != UMFPACK_OK:
            if status == UMFPACK_WARNING_singular_matrix:
                ## Change inf, nan to zeros.
                warnings.warn('Zeroing nan and inf entries...', UmfpackWarning)
                sol[~np.isfinite(sol)] = 0.0
            else:
                raise RuntimeError('%s failed with %s' % (self.funs.solve,
                                                           umfStatus[status]))
        econd = 1.0 / self.info[UMFPACK_RCOND]
        if econd > self.maxCond:
            msg = '(almost) singular matrix! '\
                  + '(estimated cond. number: %.2e)' % econd
            warnings.warn(msg, UmfpackWarning)

        return sol


Instruction:
Can you write a function in Python 3 where it
returns a matrix of one - shot solution of system of linear equation.

Response:
def linsolve(self, sys, mtx, rhs, autoTranspose=False):
        """
        One-shot solution of system of linear equation. Reuses Numeric object
        if possible.

        Parameters
        ----------
        sys : constant
            one of UMFPACK system description constants, like
            UMFPACK_A, UMFPACK_At, see umfSys list and UMFPACK docs
        mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix
            Input.
        rhs : ndarray
            Right Hand Side
        autoTranspose : bool
            Automatically changes `sys` to the transposed type, if `mtx` is in CSR,
            since UMFPACK assumes CSC internally

        Returns
        -------
        sol : ndarray
            Solution to the equation system.
        """

        if sys not in umfSys:
            raise ValueError('sys must be in' % umfSys)

        if self._numeric is None:
            self.numeric(mtx)
        else:
            if self.mtx is not mtx:
                self.numeric(mtx)

        sol = self.solve(sys, mtx, rhs, autoTranspose)
        self.free_numeric()

        return sol


Instruction:
How would you code a function in Python 3 to
perform LU decomposition. For a given matrix A, the decomposition satisfies:: LU = PRAQ when do_recip is true LU = P(R^-1)AQ when do_recip is false Parameters ---------- mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix Input. Returns ------- L : csr_matrix Lower triangular m-by-min(m,n) CSR matrix U : csc_matrix Upper triangular min(m,n)-by-n CSC matrix P : ndarray Vector of row permutations Q : ndarray Vector of column permutations R : ndarray Vector of diagonal row scalings do_recip : bool Whether R is R^-1 or R

Response:
def lu(self, mtx):
        """
        Perform LU decomposition.

        For a given matrix A, the decomposition satisfies::

                LU = PRAQ        when do_recip is true
                LU = P(R^-1)AQ   when do_recip is false

        Parameters
        ----------
        mtx : scipy.sparse.csc_matrix or scipy.sparse.csr_matrix
            Input.

        Returns
        -------
        L : csr_matrix
            Lower triangular m-by-min(m,n) CSR matrix
        U : csc_matrix
            Upper triangular min(m,n)-by-n CSC matrix
        P : ndarray
            Vector of row permutations
        Q : ndarray
            Vector of column permutations
        R : ndarray
            Vector of diagonal row scalings
        do_recip : bool
            Whether R is R^-1 or R

        """

        # this should probably be changed
        mtx = mtx.tocsc()
        self.numeric(mtx)

        # first find out how much space to reserve
        (status, lnz, unz, n_row, n_col, nz_udiag)\
                 = self.funs.get_lunz(self._numeric)

        if status != UMFPACK_OK:
            raise RuntimeError('%s failed with %s' % (self.funs.get_lunz,
                                                       umfStatus[status]))

        # allocate storage for decomposition data
        i_type = mtx.indptr.dtype

        Lp = np.zeros((n_row+1,), dtype=i_type)
        Lj = np.zeros((lnz,), dtype=i_type)
        Lx = np.zeros((lnz,), dtype=np.double)

        Up = np.zeros((n_col+1,), dtype=i_type)
        Ui = np.zeros((unz,), dtype=i_type)
        Ux = np.zeros((unz,), dtype=np.double)

        P = np.zeros((n_row,), dtype=i_type)
        Q = np.zeros((n_col,), dtype=i_type)

        Dx = np.zeros((min(n_row,n_col),), dtype=np.double)

        Rs = np.zeros((n_row,), dtype=np.double)

        if self.isReal:
            (status,do_recip) = self.funs.get_numeric(Lp,Lj,Lx,Up,Ui,Ux,
                                                       P,Q,Dx,Rs,
                                                       self._numeric)

            if status != UMFPACK_OK:
                raise RuntimeError('%s failed with %s'
                        % (self.funs.get_numeric, umfStatus[status]))

            L = sp.csr_matrix((Lx,Lj,Lp),(n_row,min(n_row,n_col)))
            U = sp.csc_matrix((Ux,Ui,Up),(min(n_row,n_col),n_col))
            R = Rs

            return (L,U,P,Q,R,bool(do_recip))

        else:
            # allocate additional storage for imaginary parts
            Lz = np.zeros((lnz,), dtype=np.double)
            Uz = np.zeros((unz,), dtype=np.double)
            Dz = np.zeros((min(n_row,n_col),), dtype=np.double)

            (status,do_recip) = self.funs.get_numeric(Lp,Lj,Lx,Lz,Up,Ui,Ux,Uz,
                                                      P,Q,Dx,Dz,Rs,
                                                      self._numeric)

            if status != UMFPACK_OK:
                raise RuntimeError('%s failed with %s'
                        % (self.funs.get_numeric, umfStatus[status]))

            Lxz = np.zeros((lnz,), dtype=np.complex128)
            Uxz = np.zeros((unz,), dtype=np.complex128)
            Dxz = np.zeros((min(n_row,n_col),), dtype=np.complex128)

            Lxz.real,Lxz.imag = Lx,Lz
            Uxz.real,Uxz.imag = Ux,Uz
            Dxz.real,Dxz.imag = Dx,Dz

            L = sp.csr_matrix((Lxz,Lj,Lp),(n_row,min(n_row,n_col)))
            U = sp.csc_matrix((Uxz,Ui,Up),(min(n_row,n_col),n_col))
            R = Rs

            return (L,U,P,Q,R,bool(do_recip))


Instruction:
Here you have a function in Python 3, explain what it does
def order_stick(presenter, egg, dist_dict, strategy, fingerprint):

    def compute_feature_stick(features, weights, alpha):
        '''create a 'stick' of feature weights'''

        feature_stick = []
        for f, w in zip(features, weights):
            feature_stick+=[f]*int(np.power(w,alpha)*100)

        return feature_stick

    def reorder_list(egg, feature_stick, dist_dict, tau):

        def compute_stimulus_stick(s, tau):
            '''create a 'stick' of feature weights'''

            feature_stick = [[weights[feature]]*round(weights[feature]**alpha)*100 for feature in w]
            return [item for sublist in feature_stick for item in sublist]

        # parse egg
        pres, rec, features, dist_funcs = parse_egg(egg)

        # turn pres and features into np arrays
        pres_arr = np.array(pres)
        features_arr = np.array(features)

        # starting with a random word
        reordered_list = []
        reordered_features = []

        # start with a random choice
        idx = np.random.choice(len(pres), 1)[0]

        # original inds
        inds = list(range(len(pres)))

        # keep track of the indices
        inds_used = [idx]

        # get the word
        current_word = pres[idx]

        # get the features dict
        current_features = features[idx]

        # append that word to the reordered list
        reordered_list.append(current_word)

        # append the features to the reordered list
        reordered_features.append(current_features)

        # loop over the word list
        for i in range(len(pres)-1):

            # sample from the stick
            feature_sample = feature_stick[np.random.choice(len(feature_stick), 1)[0]]

            # indices left
            inds_left = [ind for ind in inds if ind not in inds_used]

            # make a copy of the words filtering out the already used ones
            words_left = pres[inds_left]

            # get word distances for the word
            dists_left = np.array([dist_dict[current_word][word][feature_sample] for word in words_left])

            # features left
            features_left = features[inds_left]

            # normalize distances
            dists_left_max = np.max(dists_left)
            if dists_left_max>0:
                dists_left_norm = dists_left/np.max(dists_left)
            else:
                dists_left_norm = dists_left

            # get the min
            dists_left_min = np.min(-dists_left_norm)

            # invert the word distances to turn distance->similarity
            dists_left_inv = - dists_left_norm - dists_left_min + .01

            # create a word stick
            words_stick = []
            for word, dist in zip(words_left, dists_left_inv):
                words_stick+=[word]*int(np.power(dist,tau)*100)

            next_word = np.random.choice(words_stick)

            next_word_idx = np.where(pres==next_word)[0]

            inds_used.append(next_word_idx)

            reordered_list.append(next_word)
            reordered_features.append(features[next_word_idx][0])

        return Egg(pres=[reordered_list], rec=[reordered_list], features=[[reordered_features]], dist_funcs=dist_funcs)

    # parse egg
    pres, rec, features, dist_funcs = parse_egg(egg)

    # get params needed for list reordering
    features = presenter.get_params('fingerprint').get_features()
    alpha = presenter.get_params('alpha')
    tau = presenter.get_params('tau')
    weights = fingerprint

    # invert the weights if strategy is destabilize
    if strategy=='destabilize':
        weights = 1 - weights

    # compute feature stick
    feature_stick = compute_feature_stick(features, weights, alpha)

    # reorder list
    return reorder_list(egg, feature_stick, dist_dict, tau)

Response: Reorders a list according to a given strategy


Instruction:
Can you generate a brief explanation for the following Python 3 code
def stick_perm(presenter, egg, dist_dict, strategy):

    # seed RNG
    np.random.seed()

    # unpack egg
    egg_pres, egg_rec, egg_features, egg_dist_funcs = parse_egg(egg)

    # reorder
    regg = order_stick(presenter, egg, dist_dict, strategy)

    # unpack regg
    regg_pres, regg_rec, regg_features, regg_dist_funcs = parse_egg(regg)

    # # get the order
    regg_pres = list(regg_pres)
    egg_pres = list(egg_pres)
    idx = [egg_pres.index(r) for r in regg_pres]

    # compute weights
    weights = compute_feature_weights_dict(list(regg_pres), list(regg_pres), list(regg_features), dist_dict)

    # save out the order
    orders = idx

    return weights, orders

Response: Computes weights for one reordering using stick - breaking method


Instruction:
Write a Python 3 function that can
compute the distances between two sets of words in the archive.

Response:
def compute_distances_dict(egg):
    """ Creates a nested dict of distances """
    pres, rec, features, dist_funcs = parse_egg(egg)
    pres_list = list(pres)
    features_list = list(features)

    # initialize dist dict
    distances = {}

    # for each word in the list
    for idx1, item1 in enumerate(pres_list):

        distances[item1]={}

        # for each word in the list
        for idx2, item2 in enumerate(pres_list):

            distances[item1][item2]={}

            # for each feature in dist_funcs
            for feature in dist_funcs:
                distances[item1][item2][feature] = builtin_dist_funcs[dist_funcs[feature]](features_list[idx1][feature],features_list[idx2][feature])

    return distances


Instruction:
Create a Python 3 function to
compute the clustering scores along a set of feature dimensions and the distance matrix of each feature.

Response:
def compute_feature_weights_dict(pres_list, rec_list, feature_list, dist_dict):
    """
    Compute clustering scores along a set of feature dimensions

    Parameters
    ----------
    pres_list : list
        list of presented words
    rec_list : list
        list of recalled words
    feature_list : list
        list of feature dicts for presented words
    distances : dict
        dict of distance matrices for each feature

    Returns
    ----------
    weights : list
        list of clustering scores for each feature dimension
    """

    # initialize the weights object for just this list
    weights = {}
    for feature in feature_list[0]:
        weights[feature] = []

    # return default list if there is not enough data to compute the fingerprint
    if len(rec_list) < 2:
        print('Not enough recalls to compute fingerprint, returning default fingerprint.. (everything is .5)')
        for feature in feature_list[0]:
            weights[feature] = .5
        return [weights[key] for key in weights]

    # initialize past word list
    past_words = []
    past_idxs = []

    # loop over words
    for i in range(len(rec_list)-1):

        # grab current word
        c = rec_list[i]

        # grab the next word
        n = rec_list[i + 1]

        # if both recalled words are in the encoding list and haven't been recalled before
        if (c in pres_list and n in pres_list) and (c not in past_words and n not in past_words):

            # for each feature
            for feature in feature_list[0]:

                # get the distance vector for the current word
                # dists = [dist_dict[c][j][feature] for j in dist_dict[c]]
                # distance between current and next word
                c_dist = dist_dict[c][n][feature]

                # filter dists removing the words that have already been recalled
                # dists_filt = np.array([dist for idx, dist in enumerate(dists) if idx not in past_idxs])
                dists_filt = [dist_dict[c][j][feature] for j in dist_dict[c] if j not in past_words]

                # get indices
                avg_rank = np.mean(np.where(np.sort(dists_filt)[::-1] == c_dist)[0]+1)

                # compute the weight
                weights[feature].append(avg_rank / len(dists_filt))

            # keep track of what has been recalled already
            past_idxs.append(pres_list.index(c))
            past_words.append(c)

    # average over the cluster scores for a particular dimension
    for feature in weights:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", category=RuntimeWarning)
            weights[feature] = np.nanmean(weights[feature])

    return [weights[key] for key in weights]


Instruction:
Can you create a Python 3 function that
reorders a list of stimuli to match a fingerprint Parameters ---------- egg : quail.Egg Data to compute fingerprint method : str Method to re-sort list. Can be 'stick' or 'permute' (default: permute) nperms : int Number of permutations to use. Only used if method='permute'. (default: 2500) strategy : str or None The strategy to use to reorder the list. This can be 'stabilize', 'destabilize', 'random' or None. If None, the self.strategy field will be used. (default: None) distfun : str or function The distance function to reorder the list fingerprint to the target fingerprint. Can be any distance function supported by scipy.spatial.distance.cdist. For more info, see: https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html (default: euclidean) fingerprint : quail.Fingerprint or np.array Fingerprint (or just the state of a fingerprint) to reorder by. If None, the list will be reordered according to the fingerprint attached to the presenter object. Returns ---------- egg : quail.Egg Egg re-sorted to match fingerprint

Response:
def order(self, egg, method='permute', nperms=2500, strategy=None,
              distfun='correlation', fingerprint=None):
        """
        Reorders a list of stimuli to match a fingerprint

        Parameters
        ----------
        egg : quail.Egg
            Data to compute fingerprint

        method : str
            Method to re-sort list. Can be 'stick' or 'permute' (default: permute)

        nperms : int
            Number of permutations to use. Only used if method='permute'. (default:
            2500)

        strategy : str or None
            The strategy to use to reorder the list.  This can be 'stabilize',
            'destabilize', 'random' or None.  If None, the self.strategy field
            will be used. (default: None)

        distfun : str or function
            The distance function to reorder the list fingerprint to the target
            fingerprint.  Can be any distance function supported by
            scipy.spatial.distance.cdist. For more info, see:
            https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html
            (default: euclidean)

        fingerprint : quail.Fingerprint or np.array
            Fingerprint (or just the state of a fingerprint) to reorder by. If
            None, the list will be reordered according to the fingerprint
            attached to the presenter object.

        Returns
        ----------
        egg : quail.Egg
            Egg re-sorted to match fingerprint
        """

        def order_perm(self, egg, dist_dict, strategy, nperm, distperm,
                       fingerprint):
            """
            This function re-sorts a list by computing permutations of a given
            list and choosing the one that maximizes/minimizes variance.
            """

            # parse egg
            pres, rec, features, dist_funcs = parse_egg(egg)

            # length of list
            pres_len = len(pres)

            weights = []
            orders = []
            for i in range(nperms):
                x = rand_perm(pres, features, dist_dict, dist_funcs)
                weights.append(x[0])
                orders.append(x[1])
            weights = np.array(weights)
            orders = np.array(orders)

            # find the closest (or farthest)
            if strategy=='stabilize':
                closest = orders[np.nanargmin(cdist(np.array(fingerprint, ndmin=2), weights, distperm)),:].astype(int).tolist()
            elif strategy=='destabilize':
                closest = orders[np.nanargmax(cdist(np.array(fingerprint, ndmin=2), weights, distperm)),:].astype(int).tolist()

            # return a re-sorted egg
            return Egg(pres=[list(pres[closest])], rec=[list(pres[closest])], features=[list(features[closest])])

        def order_best_stick(self, egg, dist_dict, strategy, nperms, distfun,
                             fingerprint):

            # parse egg
            pres, rec, features, dist_funcs = parse_egg(egg)

            results = Parallel(n_jobs=multiprocessing.cpu_count())(
            delayed(stick_perm)(self, egg, dist_dict, strategy) for i in range(nperms))

            weights = np.array([x[0] for x in results])
            orders = np.array([x[1] for x in results])

            # find the closest (or farthest)
            if strategy=='stabilize':
                closest = orders[np.nanargmin(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()
            elif strategy=='destabilize':
                closest = orders[np.nanargmax(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()

            # return a re-sorted egg
            return Egg(pres=[list(pres[closest])], rec=[list(pres[closest])], features=[list(features[closest])], dist_funcs=dist_funcs)

        def order_best_choice(self, egg, dist_dict, nperms, distfun,
                              fingerprint):

            # get strategy
            strategy = self.strategy

            # parse egg
            pres, rec, features, dist_funcs = parse_egg(egg)

            results = Parallel(n_jobs=multiprocessing.cpu_count())(
            delayed(choice_perm)(self, egg, dist_dict) for i in range(nperms))

            weights = np.array([x[0] for x in results])
            orders = np.array([x[1] for x in results])

            # find the closest (or farthest)
            if strategy=='stabilize':
                closest = orders[np.nanargmin(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()
            elif strategy=='destabilize':
                closest = orders[np.nanargmax(cdist(np.array(fingerprint, ndmin=2), weights, distfun)),:].astype(int).tolist()

            # return a re-sorted egg
            return Egg(pres=[list(pres[closest])], rec=[list(pres[closest])], features=[list(features[closest])], dist_funcs=dist_funcs)

        # if strategy is not set explicitly, default to the class strategy
        if strategy is None:
            strategy = self.strategy

        dist_dict = compute_distances_dict(egg)

        if fingerprint is None:
            fingerprint = self.get_params('fingerprint').state
        elif isinstance(fingerprint, Fingerprint):
            fingerprint = fingerprint.state
        else:
            print('using custom fingerprint')

        if (strategy=='random') or (method=='random'):
            return shuffle_egg(egg)
        elif method=='permute':
            return order_perm(self, egg, dist_dict, strategy, nperms, distfun,
                              fingerprint) #
        elif method=='stick':
            return order_stick(self, egg, dist_dict, strategy, fingerprint) #
        elif method=='best_stick':
            return order_best_stick(self, egg, dist_dict, strategy, nperms,
                                    distfun, fingerprint) #
        elif method=='best_choice':
            return order_best_choice(self, egg, dist_dict, nperms,
                                     fingerprint)


Instruction:
Create a Python 3 function to
query and parse Web Server Status Page.

Response:
def initStats(self, extras=None):
        """Query and parse Web Server Status Page.

        @param extras: Include extra metrics, which can be computationally more
                       expensive.

        """
        url = "%s://%s:%d/%s" % (self._proto, self._host, self._port, self._monpath)
        response = util.get_url(url, self._user, self._password)
        #with open('/tmp/opcinfo.json') as f:
        #    response = f.read()
        self._statusDict = json.loads(response)


Instruction:
Can you implement a function in Python 3 that
returns major and minor device number for a given block device path.

Response:
def _getDevMajorMinor(self, devpath):
        """Return major and minor device number for block device path devpath.
        @param devpath: Full path for block device.
        @return:        Tuple (major, minor).

        """
        fstat = os.stat(devpath)
        if stat.S_ISBLK(fstat.st_mode):
            return(os.major(fstat.st_rdev), os.minor(fstat.st_rdev))
        else:
            raise ValueError("The file %s is not a valid block device." % devpath)


Instruction:
Write a Python 3 script to
return unique device for any block device path.

Response:
def _getUniqueDev(self, devpath):
        """Return unique device for any block device path.

        @param devpath: Full path for block device.
        @return:        Unique device string without the /dev prefix.

        """
        realpath = os.path.realpath(devpath)
        mobj = re.match('\/dev\/(.*)$', realpath)
        if mobj:
            dev = mobj.group(1)
            if dev in self._diskStats:
                return dev
            else:
                try:
                    (major, minor) = self._getDevMajorMinor(realpath)
                except:
                    return None
                return self._mapMajorMinor2dev.get((major, minor))
        return None


Instruction:
Can you tell what is the following Python 3 function doing
def _initBlockMajorMap(self):
        self._mapMajorDevclass = {}
        try:
            fp = open(devicesFile, 'r')
            data = fp.read()
            fp.close()
        except:
            raise IOError('Failed reading device information from file: %s'
                          % devicesFile)
        skip = True
        for line in data.splitlines():
            if skip:
                if re.match('block.*:', line, re.IGNORECASE):
                    skip = False
            else:
                mobj = re.match('\s*(\d+)\s+([\w\-]+)$', line)
                if mobj:
                    major = int(mobj.group(1))
                    devtype = mobj.group(2)
                    self._mapMajorDevclass[major] = devtype
                    if devtype == 'device-mapper':
                        self._dmMajorNum = major

Response: Parses the block devices file to initialize the device class - major number map.


Instruction:
Can you tell what is the following Python 3 function doing
def _initDMinfo(self):
        self._mapLVtuple2dm = {}
        self._mapLVname2dm = {}
        self._vgTree = {}
        if self._dmMajorNum is None:
            self._initBlockMajorMap()
        for file in os.listdir(devmapperDir):
            mobj = re.match('([a-zA-Z0-9+_.\-]*[a-zA-Z0-9+_.])-([a-zA-Z0-9+_.][a-zA-Z0-9+_.\-]*)$', file)
            if mobj:
                path = os.path.join(devmapperDir, file)
                (major, minor) = self._getDevMajorMinor(path)
                if major == self._dmMajorNum:
                    vg = mobj.group(1).replace('--', '-')
                    lv = mobj.group(2).replace('--', '-')
                    dmdev = "dm-%d" % minor
                    self._mapLVtuple2dm[(vg,lv)] = dmdev
                    self._mapLVname2dm[file] = dmdev
                    if not vg in self._vgTree:
                        self._vgTree[vg] = []
                    self._vgTree[vg].append(lv)

Response:
Initialize the internal data structures for the VGs
        and LVs.


Instruction:
Make a summary of the following Python 3 code
def _initFilesystemInfo(self):
        self._mapFSpathDev = {}
        fsinfo = FilesystemInfo()
        for fs in fsinfo.getFSlist():
            devpath = fsinfo.getFSdev(fs)
            dev = self._getUniqueDev(devpath)
            if dev is not None:
                self._mapFSpathDev[fs] = dev

Response: Initialize filesystem to device mappings.


Instruction:
Can you write a function in Python 3 where it
initializes swap partition to device mappings.

Response:
def _initSwapInfo(self):
        """Initialize swap partition to device mappings."""
        self._swapList = []
        sysinfo = SystemInfo()
        for (swap,attrs) in sysinfo.getSwapStats().iteritems():
            if attrs['type'] == 'partition':
                dev = self._getUniqueDev(swap)
                if dev is not None:
                    self._swapList.append(dev)


Instruction:
How would you explain what the following Python 3 function does
def _initDiskStats(self):
        self._diskStats = {}
        self._mapMajorMinor2dev = {}
        try:
            fp = open(diskStatsFile, 'r')
            data = fp.read()
            fp.close()
        except:
            raise IOError('Failed reading disk stats from file: %s'
                          % diskStatsFile)
        for line in data.splitlines():
            cols = line.split()
            dev = cols.pop(2)
            if len(cols) == 13:
                self._diskStats[dev] = dict(zip(
                    ('major', 'minor',
                     'rios', 'rmerges', 'rsect', 'rticks',
                     'wios', 'wmerges', 'wsect', 'wticks',
                     'ios_active', 'totticks', 'rqticks'),
                    [int(x) for x in cols]))
            elif len(cols) == 6:
                self._diskStats[dev] = dict(zip(
                    ('major', 'minor',
                     'rios', 'rsect',
                     'wios', 'wsect'),
                    [int(x) for x in cols]))
            else:
                continue
            self._diskStats[dev]['rbytes'] = (
                self._diskStats[dev]['rsect'] * sectorSize)
            self._diskStats[dev]['wbytes'] = (
                self._diskStats[dev]['wsect'] * sectorSize)
            self._mapMajorMinor2dev[(int(cols[0]), int(cols[1]))] = dev

Response: Parse and initialize block device I / O stats in diskstats file.


Instruction:
How would you code a function in Python 3 to
sort block devices into lists depending on device class and corresponding partition type and initialize device type map and partition map.

Response:
def _initDevClasses(self):
        """Sort block devices into lists depending on device class and
        initialize device type map and partition map."""
        self._devClassTree = {}
        self._partitionTree = {}
        self._mapDevType = {}
        basedevs = []
        otherdevs = []
        if self._mapMajorDevclass is None:
            self._initBlockMajorMap()
        for dev in self._diskStats:
            stats = self._diskStats[dev]
            devclass = self._mapMajorDevclass.get(stats['major'])
            if devclass is not None:
                devdir = os.path.join(sysfsBlockdevDir, dev)
                if os.path.isdir(devdir):
                    if not self._devClassTree.has_key(devclass):
                        self._devClassTree[devclass] = []
                    self._devClassTree[devclass].append(dev)
                    self._mapDevType[dev] = devclass
                    basedevs.append(dev)
                else:
                    otherdevs.append(dev)
        basedevs.sort(key=len, reverse=True)
        otherdevs.sort(key=len, reverse=True)
        idx = 0
        for partdev in otherdevs:
            while len(basedevs[idx]) > partdev:
                idx += 1
            for dev in basedevs[idx:]:
                if re.match("%s(\d+|p\d+)$" % dev, partdev):
                    if not self._partitionTree.has_key(dev):
                        self._partitionTree[dev] = []
                    self._partitionTree[dev].append(partdev)
                    self._mapDevType[partdev] = 'part'


Instruction:
Write a Python 3 script to
return type of device dev.

Response:
def getDevType(self, dev):
        """Returns type of device dev.

        @return: Device type as string.

        """
        if self._devClassTree is None:
            self._initDevClasses()
        return self._mapDevType.get(dev)


Instruction:
Here you have a function in Python 3, explain what it does
def getPartitionList(self):
        if self._partList is None:
            self._partList = []
            for (disk,parts) in self.getPartitionDict().iteritems():
                for part in parts:
                    self._partList.append((disk,part))
        return self._partList

Response: Returns list of partitions.


Instruction:
Implement a Python 3 function for
returning I / O stats for the specified device.

Response:
def getDevStats(self, dev, devtype = None):
        """Returns I/O stats for block device.

        @param dev:     Device name
        @param devtype: Device type. (Ignored if None.)
        @return:        Dict of stats.

        """
        if devtype is not None:
            if self._devClassTree is None:
                self._initDevClasses()
            if devtype <> self._mapDevType.get(dev):
                return None
        return self._diskStats.get(dev)


Instruction:
Can you implement a function in Python 3 that
returns I / O stats for a given device name.

Response:
def getSwapStats(self, dev):
        """Returns I/O stats for swap partition.

        @param dev: Device name for swap partition.
        @return: Dict of stats.

        """
        if self._swapList is None:
            self._initSwapInfo()
        if dev in self._swapList:
            return self.getDevStats(dev)
        else:
            return None


Instruction:
Explain what the following Python 3 code does
def getLVstats(self, *args):
        if not len(args) in (1, 2):
            raise TypeError("The getLVstats must be called with either "
                            "one or two arguments.")
        if self._vgTree is None:
            self._initDMinfo()
        if len(args) == 1:
            dmdev = self._mapLVname2dm.get(args[0])
        else:
            dmdev = self._mapLVtuple2dm.get(args)
        if dmdev is not None:
            return self.getDevStats(dmdev)
        else:
            return None

Response: Returns I / O stats for LV.


Instruction:
Write a Python 3 script to
return I / O stats for filesystem.

Response:
def getFilesystemStats(self, fs):
        """Returns I/O stats for filesystem.

        @param fs: Filesystem path.
        @return: Dict of stats.

        """
        if self._mapFSpathDev is None:
            self._initFilesystemInfo()
        return self._diskStats.get(self._mapFSpathDev.get(fs))


Instruction:
Write a Python 3 script to
retrieve values for graphs.

Response:
def retrieveVals(self):
        """Retrieve values for graphs."""
        ntpinfo = NTPinfo()
        stats = ntpinfo.getHostOffset(self._remoteHost)
        if stats:
            graph_name = 'ntp_host_stratum_%s' % self._remoteHost
            if self.hasGraph(graph_name):
                self.setGraphVal(graph_name, 'stratum', stats.get('stratum'))
            graph_name = 'ntp_host_stat_%s' % self._remoteHost
            if self.hasGraph(graph_name):
                self.setGraphVal(graph_name, 'offset', stats.get('offset'))
                self.setGraphVal(graph_name, 'delay', stats.get('delay'))


Instruction:
Here you have a function in Python 3, explain what it does
def getContainerStats(self, limit=None, marker=None):
        stats = {}
        for row in self._conn.list_containers_info(limit, marker):
            stats[row['name']] = {'count': row['count'], 'size': row['bytes']}
        return stats

Response: Returns Rackspace Cloud Files usage stats for containers.