Instruction:
Explain what the following Python 3 code does
def frame_times(self):
        frame_times = None
        # Make sure we have already computed the features
        self.features
        if self.feat_type is FeatureTypes.framesync:
            self._compute_framesync_times()
            frame_times = self._framesync_times
        elif self.feat_type is FeatureTypes.est_beatsync:
            frame_times = self._est_beatsync_times
        elif self.feat_type is FeatureTypes.ann_beatsync:
            frame_times = self._ann_beatsync_times

        return frame_times

Response:
This getter returns the frame times for the corresponding type of
        features.


Instruction:
Implement a Python 3 function for
selecting the features from the given parameters.

Response:
def select_features(cls, features_id, file_struct, annot_beats, framesync):
        """Selects the features from the given parameters.

        Parameters
        ----------
        features_id: str
            The identifier of the features (it must be a key inside the
            `features_registry`)
        file_struct: msaf.io.FileStruct
            The file struct containing the files to extract the features from
        annot_beats: boolean
            Whether to use annotated (`True`) or estimated (`False`) beats
        framesync: boolean
            Whether to use framesync (`True`) or beatsync (`False`) features

        Returns
        -------
        features: obj
            The actual features object that inherits from `msaf.Features`
        """
        if not annot_beats and framesync:
            feat_type = FeatureTypes.framesync
        elif annot_beats and not framesync:
            feat_type = FeatureTypes.ann_beatsync
        elif not annot_beats and not framesync:
            feat_type = FeatureTypes.est_beatsync
        else:
            raise FeatureTypeNotFound("Type of features not valid.")

        # Select features with default parameters
        if features_id not in features_registry.keys():
            raise FeaturesNotFound(
                "The features '%s' are invalid (valid features are %s)"
                % (features_id, features_registry.keys()))

        return features_registry[features_id](file_struct, feat_type)


Instruction:
Can you generate the documentation for the following Python 3 function
def _preprocess(self, valid_features=["pcp", "tonnetz", "mfcc",
                                          "cqt", "tempogram"]):
        """This method obtains the actual features."""
        # Use specific feature
        if self.feature_str not in valid_features:
            raise RuntimeError("Feature %s in not valid for algorithm: %s "
                               "(valid features are %s)." %
                               (self.feature_str, __name__, valid_features))
        else:
            try:
                F = self.features.features
            except KeyError:
                raise RuntimeError("Feature %s in not supported by MSAF" %
                                   (self.feature_str))

        return F

Response: This method obtains the actual features.


Instruction:
Can you generate the documentation for the following Python 3 function
def _postprocess(self, est_idxs, est_labels):
        # Make sure we are using the previously input bounds, if any
        if self.in_bound_idxs is not None:
            F = self._preprocess()
            est_labels = U.synchronize_labels(self.in_bound_idxs, est_idxs,
                                              est_labels, F.shape[0])
            est_idxs = self.in_bound_idxs

        # Remove empty segments if needed
        est_idxs, est_labels = U.remove_empty_segments(est_idxs, est_labels)

        assert len(est_idxs) - 1 == len(est_labels), "Number of boundaries " \
            "(%d) and number of labels(%d) don't match" % (len(est_idxs),
                                                           len(est_labels))

        # Make sure the indeces are integers
        est_idxs = np.asarray(est_idxs, dtype=int)

        return est_idxs, est_labels

Response: Post processes the estimations from the algorithm removing empty segments and making sure the lenghts of the boundaries and labels match.


Instruction:
Here you have a function in Python 3, explain what it does
def process(in_path, annot_beats=False, feature="mfcc", framesync=False,
            boundaries_id="gt", labels_id=None, n_jobs=4, config=None):
    """Sweeps parameters across the specified algorithm."""

    results_file = "results_sweep_boundsE%s_labelsE%s.csv" % (boundaries_id,
                                                              labels_id)

    if labels_id == "cnmf3" or boundaries_id == "cnmf3":
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

        hh = range(15, 33)
        RR = range(15, 40)
        ranks = range(3, 6)
        RR_labels = range(11, 12)
        ranks_labels = range(6, 7)
        all_results = pd.DataFrame()
        for rank in ranks:
            for h in hh:
                for R in RR:
                    for rank_labels in ranks_labels:
                        for R_labels in RR_labels:
                            config["h"] = h
                            config["R"] = R
                            config["rank"] = rank
                            config["rank_labels"] = rank_labels
                            config["R_labels"] = R_labels
                            config["features"] = None

                            # Run process
                            msaf.run.process(
                                in_path, n_jobs=n_jobs,
                                boundaries_id=boundaries_id,
                                labels_id=labels_id, config=config)

                            # Compute evaluations
                            results = msaf.eval.process(
                                in_path, boundaries_id, labels_id,
                                save=True, n_jobs=n_jobs, config=config)

                            # Save avg results
                            new_columns = {"config_h": h, "config_R": R,
                                           "config_rank": rank,
                                           "config_R_labels": R_labels,
                                           "config_rank_labels": rank_labels}
                            results = results.append([new_columns],
                                                     ignore_index=True)
                            all_results = all_results.append(results.mean(),
                                                             ignore_index=True)
                            all_results.to_csv(results_file)

    elif labels_id is None and boundaries_id == "sf":
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

        MM = range(20, 32)
        mm = range(3, 4)
        kk = np.arange(0.03, 0.1, 0.01)
        Mpp = range(16, 32)
        ott = np.arange(0.02, 0.1, 0.01)
        all_results = pd.DataFrame()
        for k in kk:
            for ot in ott:
                for m in mm:
                    for M in MM:
                        for Mp in Mpp:
                            config["M_gaussian"] = M
                            config["m_embedded"] = m
                            config["k_nearest"] = k
                            config["Mp_adaptive"] = Mp
                            config["offset_thres"] = ot
                            config["features"] = None

                            # Run process
                            msaf.run.process(
                                in_path, n_jobs=n_jobs,
                                boundaries_id=boundaries_id,
                                labels_id=labels_id, config=config)

                            # Compute evaluations
                            results = msaf.eval.process(
                                in_path, boundaries_id, labels_id,
                                save=True, n_jobs=n_jobs, config=config)

                            # Save avg results
                            new_columns = {"config_M": M, "config_m": m,
                                           "config_k": k, "config_Mp": Mp,
                                           "config_ot": ot}
                            results = results.append([new_columns],
                                                     ignore_index=True)
                            all_results = all_results.append(results.mean(),
                                                             ignore_index=True)
                            all_results.to_csv(results_file)

    else:
        logging.error("Can't sweep parameters for %s algorithm. "
                      "Implement me! :D")

Response: Process the input file and return a pandas DataFrame of the results.


Instruction:
Explain what the following Python 3 code does
def main():
    parser = argparse.ArgumentParser(
        description="Runs the speficied algorithm(s) on the MSAF "
        "formatted dataset.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("in_path",
                        action="store",
                        help="Input dataset")
    parser.add_argument("-f",
                        action="store",
                        dest="feature",
                        default="pcp",
                        type=str,
                        help="Type of features",
                        choices=["pcp", "tonnetz", "mfcc", "cqt", "tempogram"])
    parser.add_argument("-b",
                        action="store_true",
                        dest="annot_beats",
                        help="Use annotated beats",
                        default=False)
    parser.add_argument("-fs",
                        action="store_true",
                        dest="framesync",
                        help="Use frame-synchronous features",
                        default=False)
    parser.add_argument("-bid",
                        action="store",
                        help="Boundary algorithm identifier",
                        dest="boundaries_id",
                        default="gt",
                        choices=["gt"] +
                        io.get_all_boundary_algorithms())
    parser.add_argument("-lid",
                        action="store",
                        help="Label algorithm identifier",
                        dest="labels_id",
                        default=None,
                        choices=io.get_all_label_algorithms())
    parser.add_argument("-j",
                        action="store",
                        dest="n_jobs",
                        default=4,
                        type=int,
                        help="The number of threads to use")
    args = parser.parse_args()
    start_time = time.time()

    # Run the algorithm(s)
    process(args.in_path, annot_beats=args.annot_beats, feature=args.feature,
            framesync=args.framesync, boundaries_id=args.boundaries_id,
            labels_id=args.labels_id, n_jobs=args.n_jobs)

    # Done!
    logging.info("Done! Took %.2f seconds." % (time.time() - start_time))

Response: Main function to sweep parameters of a certain algorithm.


Instruction:
Can you generate the documentation for the following Python 3 function
def main():
    parser = argparse.ArgumentParser(
        description="Runs the speficied algorithm(s) on the input file and "
        "the results using the MIREX format.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-bid",
                        action="store",
                        help="Boundary algorithm identifier",
                        dest="boundaries_id",
                        default=msaf.config.default_bound_id,
                        choices=["gt"] +
                        msaf.io.get_all_boundary_algorithms())
    parser.add_argument("-lid",
                        action="store",
                        help="Label algorithm identifier",
                        dest="labels_id",
                        default=msaf.config.default_label_id,
                        choices=msaf.io.get_all_label_algorithms())
    parser.add_argument("-i",
                        action="store",
                        dest="in_file",
                        help="Input audio file")
    parser.add_argument("-o",
                        action="store",
                        dest="out_file",
                        help="Output file with the results",
                        default="out.txt")

    args = parser.parse_args()
    start_time = time.time()

    # Setup the logger
    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',
                        level=logging.INFO)

    # Run MSAF
    params = {
        "annot_beats": False,
        "feature": "cqt",
        "framesync": False,
        "boundaries_id": args.boundaries_id,
        "labels_id": args.labels_id,
        "n_jobs": 1,
        "hier": False,
        "sonify_bounds": False,
        "plot": False
    }
    res = msaf.run.process(args.in_file, **params)
    msaf.io.write_mirex(res[0], res[1], args.out_file)

    # Done!
    logging.info("Done! Took %.2f seconds." % (time.time() - start_time))

Response: Main function to parse the arguments and call the main process.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def print_results(results):
    if len(results) == 0:
        logging.warning("No results to print!")
        return
    res = results.mean()
    logging.info("Results:\n%s" % res)

Response: Print all the results.


Instruction:
Here you have a function in Python 3, explain what it does
def compute_results(ann_inter, est_inter, ann_labels, est_labels, bins,
                    est_file, weight=0.58):
    """Compute the results using all the available evaluations.

    Parameters
    ----------
    ann_inter : np.array
        Annotated intervals in seconds.
    est_inter : np.array
        Estimated intervals in seconds.
    ann_labels : np.array
        Annotated labels.
    est_labels : np.array
        Estimated labels
    bins : int
        Number of bins for the information gain.
    est_file : str
        Path to the output file to store results.
    weight: float
        Weight the Precision and Recall values of the hit rate boundaries
        differently (<1 will weight Precision higher, >1 will weight Recall
        higher).
        The default parameter (0.58) is the one proposed in (Nieto et al. 2014)

    Return
    ------
    results : dict
        Contains the results of all the evaluations for the given file.
        Keys are the following:
            track_id: Name of the track
            HitRate_3F: F-measure of hit rate at 3 seconds
            HitRate_3P: Precision of hit rate at 3 seconds
            HitRate_3R: Recall of hit rate at 3 seconds
            HitRate_0.5F: F-measure of hit rate at 0.5 seconds
            HitRate_0.5P: Precision of hit rate at 0.5 seconds
            HitRate_0.5R: Recall of hit rate at 0.5 seconds
            HitRate_w3F: F-measure of hit rate at 3 seconds weighted
            HitRate_w0.5F: F-measure of hit rate at 0.5 seconds weighted
            HitRate_wt3F: F-measure of hit rate at 3 seconds weighted and
                          trimmed
            HitRate_wt0.5F: F-measure of hit rate at 0.5 seconds weighted
                            and trimmed
            HitRate_t3F: F-measure of hit rate at 3 seconds (trimmed)
            HitRate_t3P: Precision of hit rate at 3 seconds (trimmed)
            HitRate_t3F: Recall of hit rate at 3 seconds (trimmed)
            HitRate_t0.5F: F-measure of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5P: Precision of hit rate at 0.5 seconds (trimmed)
            HitRate_t0.5R: Recall of hit rate at 0.5 seconds (trimmed)
            DevA2E: Median deviation of annotation to estimation
            DevE2A: Median deviation of estimation to annotation
            D: Information gain
            PWF: F-measure of pair-wise frame clustering
            PWP: Precision of pair-wise frame clustering
            PWR: Recall of pair-wise frame clustering
            Sf: F-measure normalized entropy score
            So: Oversegmentation normalized entropy score
            Su: Undersegmentation normalized entropy score
    """
    res = {}

    # --Boundaries-- #
    # Hit Rate standard
    res["HitRate_3P"], res["HitRate_3R"], res["HitRate_3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=False)
    res["HitRate_0.5P"], res["HitRate_0.5R"], res["HitRate_0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=False)

    # Hit rate trimmed
    res["HitRate_t3P"], res["HitRate_t3R"], res["HitRate_t3F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=3, trim=True)
    res["HitRate_t0.5P"], res["HitRate_t0.5R"], res["HitRate_t0.5F"] = \
        mir_eval.segment.detection(ann_inter, est_inter, window=.5, trim=True)

    # Hit rate weighted
    _, _, res["HitRate_w3F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=3, trim=False, beta=weight)
    _, _, res["HitRate_w0.5F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=.5, trim=False, beta=weight)

    # Hit rate weighted and trimmed
    _, _, res["HitRate_wt3F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=3, trim=True, beta=weight)
    _, _, res["HitRate_wt0.5F"] = mir_eval.segment.detection(
        ann_inter, est_inter, window=.5, trim=True, beta=weight)

    # Information gain
    res["D"] = compute_information_gain(ann_inter, est_inter, est_file,
                                        bins=bins)

    # Median Deviations
    res["DevR2E"], res["DevE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=False)
    res["DevtR2E"], res["DevtE2R"] = mir_eval.segment.deviation(
        ann_inter, est_inter, trim=True)

    # --Labels-- #
    if est_labels is not None and ("-1" in est_labels or "@" in est_labels):
        est_labels = None
    if est_labels is not None and len(est_labels) != 0:
        # Align labels with intervals
        ann_labels = list(ann_labels)
        est_labels = list(est_labels)
        ann_inter, ann_labels = mir_eval.util.adjust_intervals(ann_inter,
                                                               ann_labels)
        est_inter, est_labels = mir_eval.util.adjust_intervals(
            est_inter, est_labels, t_min=0.0, t_max=ann_inter.max())

        # Pair-wise frame clustering
        res["PWP"], res["PWR"], res["PWF"] = mir_eval.segment.pairwise(
            ann_inter, ann_labels, est_inter, est_labels)

        # Normalized Conditional Entropies
        res["So"], res["Su"], res["Sf"] = mir_eval.segment.nce(
            ann_inter, ann_labels, est_inter, est_labels)

    # Names
    base = os.path.basename(est_file)
    res["track_id"] = base[:-5]
    res["ds_name"] = base.split("_")[0]

    return res

Response: Compute the results of all the available evaluations for the given file.


Instruction:
How would you code a function in Python 3 to
compute the results by using the ground truth dataset identified by by the annotator parameter.

Response:
def compute_gt_results(est_file, ref_file, boundaries_id, labels_id, config,
                       bins=251, annotator_id=0):
    """Computes the results by using the ground truth dataset identified by
    the annotator parameter.

    Return
    ------
    results : dict
        Dictionary of the results (see function compute_results).
    """
    if config["hier"]:
        ref_times, ref_labels, ref_levels = \
            msaf.io.read_hier_references(
                ref_file, annotation_id=annotator_id,
                exclude_levels=["segment_salami_function"])
    else:
        jam = jams.load(ref_file, validate=False)
        ann = jam.search(namespace='segment_.*')[annotator_id]
        ref_inter, ref_labels = ann.to_interval_values()

    # Read estimations with correct configuration
    est_inter, est_labels = io.read_estimations(est_file, boundaries_id,
                                                labels_id, **config)

    # Compute the results and return
    logging.info("Evaluating %s" % os.path.basename(est_file))
    if config["hier"]:
        # Hierarchical
        assert len(est_inter) == len(est_labels), "Same number of levels " \
            "are required in the boundaries and labels for the hierarchical " \
            "evaluation."
        est_times = []
        est_labels = []

        # Sort based on how many segments per level
        est_inter = sorted(est_inter, key=lambda level: len(level))

        for inter in est_inter:
            est_times.append(msaf.utils.intervals_to_times(inter))
            # Add fake labels (hierarchical eval does not use labels --yet--)
            est_labels.append(np.ones(len(est_times[-1]) - 1) * -1)

        # Align the times
        utils.align_end_hierarchies(est_times, ref_times, thres=1)

        # To intervals
        est_hier = [utils.times_to_intervals(times) for times in est_times]
        ref_hier = [utils.times_to_intervals(times) for times in ref_times]

        # Compute evaluations
        res = {}
        res["t_recall10"], res["t_precision10"], res["t_measure10"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=10)
        res["t_recall15"], res["t_precision15"], res["t_measure15"] = \
            mir_eval.hierarchy.tmeasure(ref_hier, est_hier, window=15)

        res["track_id"] = os.path.basename(est_file)[:-5]
        return res
    else:
        # Flat
        return compute_results(ref_inter, est_inter, ref_labels, est_labels,
                               bins, est_file)


Instruction:
Can you implement a function in Python 3 that
computes the information gain of the est_file from the annotated intervals and the estimated intervals.

Response:
def compute_information_gain(ann_inter, est_inter, est_file, bins):
    """Computes the information gain of the est_file from the annotated
    intervals and the estimated intervals."""
    ann_times = utils.intervals_to_times(ann_inter)
    est_times = utils.intervals_to_times(est_inter)
    return mir_eval.beat.information_gain(ann_times, est_times, bins=bins)


Instruction:
Given the following Python 3 function, write the documentation
def process_track(file_struct, boundaries_id, labels_id, config,
                  annotator_id=0):
    """Processes a single track.

    Parameters
    ----------
    file_struct : object (FileStruct) or str
        File struct or full path of the audio file to be evaluated.
    boundaries_id : str
        Identifier of the boundaries algorithm.
    labels_id : str
        Identifier of the labels algorithm.
    config : dict
        Configuration of the algorithms to be evaluated.
    annotator_id : int
        Number identifiying the annotator.

    Returns
    -------
    one_res : dict
        Dictionary of the results (see function compute_results).
    """
    # Convert to file_struct if string is passed
    if isinstance(file_struct, six.string_types):
        file_struct = io.FileStruct(file_struct)

    est_file = file_struct.est_file
    ref_file = file_struct.ref_file

    # Sanity check
    assert os.path.basename(est_file)[:-4] == \
        os.path.basename(ref_file)[:-4], "File names are different %s --- %s" \
        % (os.path.basename(est_file)[:-4], os.path.basename(ref_file)[:-4])

    if not os.path.isfile(ref_file):
        raise NoReferencesError("Reference file %s does not exist. You must "
                                "have annotated references to run "
                                "evaluations." % ref_file)

    one_res = compute_gt_results(est_file, ref_file, boundaries_id, labels_id,
                                 config, annotator_id=annotator_id)

    return one_res

Response: Processes a single audio file and returns a dictionary of the results.


Instruction:
Implement a Python 3 function for
getting the file name to store the results.

Response:
def get_results_file_name(boundaries_id, labels_id, config,
                          annotator_id):
    """Based on the config and the dataset, get the file name to store the
    results."""
    utils.ensure_dir(msaf.config.results_dir)
    file_name = os.path.join(msaf.config.results_dir, "results")
    file_name += "_boundsE%s_labelsE%s" % (boundaries_id, labels_id)
    file_name += "_annotatorE%d" % (annotator_id)
    sorted_keys = sorted(config.keys(), key=str.lower)
    for key in sorted_keys:
        file_name += "_%sE%s" % (key, str(config[key]).replace("/", "_"))

    # Check for max file length
    if len(file_name) > 255 - len(msaf.config.results_ext):
        file_name = file_name[:255 - len(msaf.config.results_ext)]

    return file_name + msaf.config.results_ext


Instruction:
Here you have a function in Python 3, explain what it does
def process(in_path, boundaries_id=msaf.config.default_bound_id,
            labels_id=msaf.config.default_label_id, annot_beats=False,
            framesync=False, feature="pcp", hier=False, save=False,
            out_file=None, n_jobs=4, annotator_id=0, config=None):
    """Main process to evaluate algorithms' results.

    Parameters
    ----------
    in_path : str
        Path to the dataset root folder.
    boundaries_id : str
        Boundaries algorithm identifier (e.g. siplca, cnmf)
    labels_id : str
        Labels algorithm identifier (e.g. siplca, cnmf)
    ds_name : str
        Name of the dataset to be evaluated (e.g. SALAMI). * stands for all.
    annot_beats : boolean
        Whether to use the annotated beats or not.
    framesync: str
        Whether to use framesync features or not (default: False -> beatsync)
    feature: str
        String representing the feature to be used (e.g. pcp, mfcc, tonnetz)
    hier : bool
        Whether to compute a hierarchical or flat segmentation.
    save: boolean
        Whether to save the results into the `out_file` csv file.
    out_file: str
        Path to the csv file to save the results (if `None` and `save = True`
        it will save the results in the default file name obtained by
        calling `get_results_file_name`).
    n_jobs: int
        Number of processes to run in parallel. Only available in collection
        mode.
    annotator_id : int
        Number identifiying the annotator.
    config: dict
        Dictionary containing custom configuration parameters for the
        algorithms.  If None, the default parameters are used.

    Return
    ------
    results : pd.DataFrame
        DataFrame containing the evaluations for each file.
    """

    # Set up configuration based on algorithms parameters
    if config is None:
        config = io.get_configuration(feature, annot_beats, framesync,
                                      boundaries_id, labels_id)

    # Hierarchical segmentation
    config["hier"] = hier

    # Remove actual features
    config.pop("features", None)

    # Get out file in case we want to save results
    if out_file is None:
        out_file = get_results_file_name(boundaries_id, labels_id, config,
                                         annotator_id)

    # If out_file already exists, read and return them
    if os.path.exists(out_file):
        logging.warning("Results already exists, reading from file %s" %
                        out_file)
        results = pd.read_csv(out_file)
        print_results(results)
        return results

    # Perform actual evaluations
    if os.path.isfile(in_path):
        # Single File mode
        evals = [process_track(in_path, boundaries_id, labels_id, config,
                               annotator_id=annotator_id)]
    else:
        # Collection mode
        # Get files
        file_structs = io.get_dataset_files(in_path)

        # Evaluate in parallel
        logging.info("Evaluating %d tracks..." % len(file_structs))
        evals = Parallel(n_jobs=n_jobs)(delayed(process_track)(
            file_struct, boundaries_id, labels_id, config,
            annotator_id=annotator_id) for file_struct in file_structs[:])

    # Aggregate evaluations in pandas format
    results = pd.DataFrame()
    for e in evals:
        if e != []:
            results = results.append(e, ignore_index=True)
    logging.info("%d tracks analyzed" % len(results))

    # Print results
    print_results(results)

    # Save all results
    if save:
        logging.info("Writing results in %s" % out_file)
        results.to_csv(out_file)

    return results

Response: This function processes the algorithms in the dataset root folder and returns the results as a pandas DataFrame.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_config_string(config_string, issue_warnings=True):
    config_dict = {}
    my_splitter = shlex.shlex(config_string, posix=True)
    my_splitter.whitespace = ','
    my_splitter.whitespace_split = True
    for kv_pair in my_splitter:
        kv_pair = kv_pair.strip()
        if not kv_pair:
            continue
        kv_tuple = kv_pair.split('=', 1)
        if len(kv_tuple) == 1:
            if issue_warnings:
                MsafConfigWarning.warn(
                    ("Config key '%s' has no value, ignoring it" %
                     kv_tuple[0]), stacklevel=1)
        else:
            k, v = kv_tuple
            # subsequent values for k will override earlier ones
            config_dict[k] = v
    return config_dict

Response: Parses a config string into a dict.


Instruction:
How would you code a function in Python 3 to
fetch the overriding config value for a key.

Response:
def fetch_val_for_key(key, delete_key=False):
    """Return the overriding config value for a key.
    A successful search returns a string value.
    An unsuccessful search raises a KeyError
    The (decreasing) priority order is:
    - MSAF_FLAGS
    - ~./msafrc
    """

    # first try to find it in the FLAGS
    try:
        if delete_key:
            return MSAF_FLAGS_DICT.pop(key)
        return MSAF_FLAGS_DICT[key]
    except KeyError:
        pass

    # next try to find it in the config file

    # config file keys can be of form option, or section.option
    key_tokens = key.rsplit('.', 1)
    if len(key_tokens) == 2:
        section, option = key_tokens
    else:
        section, option = 'global', key
    try:
        try:
            return msaf_cfg.get(section, option)
        except InterpolationError:
            return msaf_raw_cfg.get(section, option)
    except (NoOptionError, NoSectionError):
        raise KeyError(key)


Instruction:
Make a summary of the following Python 3 code
def AddConfigVar(name, doc, configparam, root=config):

    # This method also performs some of the work of initializing ConfigParam
    # instances

    if root is config:
        # only set the name in the first call, not the recursive ones
        configparam.fullname = name
    sections = name.split('.')
    if len(sections) > 1:
        # set up a subobject
        if not hasattr(root, sections[0]):
            # every internal node in the config tree is an instance of its own
            # unique class
            class SubObj(object):
                _i_am_a_config_class = True
            setattr(root.__class__, sections[0], SubObj())
        newroot = getattr(root, sections[0])
        if (not getattr(newroot, '_i_am_a_config_class', False) or
                isinstance(newroot, type)):
            raise TypeError(
                'Internal config nodes must be config class instances',
                newroot)
        return AddConfigVar('.'.join(sections[1:]), doc, configparam,
                            root=newroot)
    else:
        if hasattr(root, name):
            raise AttributeError('This name is already taken',
                                 configparam.fullname)
        configparam.doc = doc
        # Trigger a read of the value from config files and env vars
        # This allow to filter wrong value from the user.
        if not callable(configparam.default):
            configparam.__get__(root, type(root), delete_key=True)
        else:
            # We do not want to evaluate now the default value
            # when it is a callable.
            try:
                fetch_val_for_key(configparam.fullname)
                # The user provided a value, filter it now.
                configparam.__get__(root, type(root), delete_key=True)
            except KeyError:
                pass
        setattr(root.__class__, sections[0], configparam)
        _config_var_list.append(configparam)

Response: Add a new variable to the msaf. config object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def processFlat(self):
        # Preprocess to obtain features (array(n_frames, n_features))

        F = self._preprocess()
        F = librosa.util.normalize(F, axis=0)
        F = librosa.feature.stack_memory(F.T).T

        self.config["hier"] = False
        my_bounds, my_labels, _ = main.scluster_segment(F, self.config, self.in_bound_idxs)

        # Post process estimations
        est_idxs, est_labels = self._postprocess(my_bounds, my_labels)

        assert est_idxs[0] == 0 and est_idxs[-1] == F.shape[0] - 1
        # We're done!
        return est_idxs, est_labels

Response:
Main process.
        Returns
        -------
        est_idxs : np.array(N)
            Estimated indeces the segment boundaries in frame indeces.
        est_labels : np.array(N-1)
            Estimated labels for the segments.


Instruction:
Create a Python 3 function to
compute all features for the given file.

Response:
def compute_all_features(file_struct, framesync):
    """Computes all features for the given file."""
    for feature_id in msaf.features_registry:
        logging.info("Computing %s for file %s" % (feature_id,
                                                   file_struct.audio_file))
        feats = Features.select_features(feature_id, file_struct, False, framesync)
        feats.features


Instruction:
Can you write a function in Python 3 where it
computes the features for the selected dataset or file.

Response:
def process(in_path, out_file, n_jobs, framesync):
    """Computes the features for the selected dataset or file."""
    if os.path.isfile(in_path):
        # Single file mode
        # Get (if they exitst) or compute features
        file_struct = msaf.io.FileStruct(in_path)
        file_struct.features_file = out_file
        compute_all_features(file_struct, framesync)
    else:
        # Collection mode
        file_structs = msaf.io.get_dataset_files(in_path)

        # Call in parallel
        return Parallel(n_jobs=n_jobs)(delayed(compute_all_features)(
            file_struct, framesync) for file_struct in file_structs)


Instruction:
Given the following Python 3 function, write the documentation
def features(file_struct, annot_beats=False, framesync=False):
    '''Feature-extraction for audio segmentation
    Arguments:
        file_struct -- msaf.io.FileStruct
        paths to the input files in the Segmentation dataset

    Returns:
        - X -- ndarray

            beat-synchronous feature matrix:
            MFCC (mean-aggregated)
            Chroma (median-aggregated)
            Latent timbre repetition
            Latent chroma repetition
            Time index
            Beat index

        - dur -- float
            duration of the track in seconds

    '''
    def compress_data(X, k):
        Xtemp = X.dot(X.T)
        if len(Xtemp) == 0:
            return None
        e_vals, e_vecs = np.linalg.eig(Xtemp)

        e_vals = np.maximum(0.0, np.real(e_vals))
        e_vecs = np.real(e_vecs)

        idx = np.argsort(e_vals)[::-1]

        e_vals = e_vals[idx]
        e_vecs = e_vecs[:, idx]

        # Truncate to k dimensions
        if k < len(e_vals):
            e_vals = e_vals[:k]
            e_vecs = e_vecs[:, :k]

        # Normalize by the leading singular value of X
        Z = np.sqrt(e_vals.max())

        if Z > 0:
            e_vecs = e_vecs / Z

        return e_vecs.T.dot(X)

    # Latent factor repetition features
    def repetition(X, metric='euclidean'):
        R = librosa.segment.recurrence_matrix(
            X, k=2 * int(np.ceil(np.sqrt(X.shape[1]))),
            width=REP_WIDTH, metric=metric, sym=False).astype(np.float32)

        P = scipy.signal.medfilt2d(librosa.segment.recurrence_to_lag(R),
                                   [1, REP_FILTER])

        # Discard empty rows.
        # This should give an equivalent SVD, but resolves some numerical
        # instabilities.
        P = P[P.any(axis=1)]

        return compress_data(P, N_REP)

    #########
    # '\tloading annotations and features of ', audio_path
    pcp_obj = Features.select_features("pcp", file_struct, annot_beats,
                                       framesync)
    mfcc_obj = Features.select_features("mfcc", file_struct, annot_beats,
                                        framesync)
    chroma = pcp_obj.features
    mfcc = mfcc_obj.features
    beats = pcp_obj.frame_times
    dur = pcp_obj.dur

    # Sampling Rate
    sr = msaf.config.sample_rate

    ##########
    # print '\treading beats'
    B = beats[:chroma.shape[0]]
    # beat_frames = librosa.time_to_frames(B, sr=sr,
                                         #hop_length=msaf.config.hop_size)
    #print beat_frames, len(beat_frames), uidx

    #########
    M = mfcc.T
    #plt.imshow(M, interpolation="nearest", aspect="auto"); plt.show()

    #########
    # Get the beat-sync chroma
    C = chroma.T
    C += C.min() + 0.1
    C = C / C.max(axis=0)
    C = 80 * np.log10(C)  # Normalize from -80 to 0
    #plt.imshow(C, interpolation="nearest", aspect="auto"); plt.show()

    # Time-stamp features
    N = np.arange(float(chroma.shape[0]))

    #########
    #print '\tgenerating structure features'

    # TODO:  This might fail if audio file (or number of beats) is too small
    R_timbre = repetition(librosa.feature.stack_memory(M))
    R_chroma = repetition(librosa.feature.stack_memory(C))
    if R_timbre is None or R_chroma is None:
        return None, dur

    R_timbre += R_timbre.min()
    R_timbre /= R_timbre.max()
    R_chroma += R_chroma.min()
    R_chroma /= R_chroma.max()
    #plt.imshow(R_chroma, interpolation="nearest", aspect="auto"); plt.show()

    # Stack it all up
    #print M.shape, C.shape, R_timbre.shape, R_chroma.shape, len(B), len(N)
    X = np.vstack([M, C, R_timbre, R_chroma, B, B / dur, N,
                   N / float(chroma.shape[0])])

    #plt.imshow(X, interpolation="nearest", aspect="auto"); plt.show()

    return X, dur

Response: Feature - extraction for audio segmentation


Instruction:
Given the following Python 3 function, write the documentation
def gaussian_cost(X):
    '''Return the average log-likelihood of data under a standard normal
    '''

    d, n = X.shape

    if n < 2:
        return 0

    sigma = np.var(X, axis=1, ddof=1)

    cost = -0.5 * d * n * np.log(2. * np.pi) - 0.5 * (n - 1.) * np.sum(sigma)
    return cost

Response: Return the average log - likelihood of data under a standard normal


Instruction:
Can you implement a function in Python 3 that
logs - normalizes features such that each vector is between min_db to 0.

Response:
def lognormalize(F, floor=0.1, min_db=-80):
    """Log-normalizes features such that each vector is between min_db to 0."""
    assert min_db < 0
    F = min_max_normalize(F, floor=floor)
    F = np.abs(min_db) * np.log10(F)  # Normalize from min_db to 0
    return F


Instruction:
Explain what the following Python 3 code does
def min_max_normalize(F, floor=0.001):
    F += -F.min() + floor
    F = F / F.max(axis=0)
    return F

Response: Normalizes features such that each vector is between floor to 1.


Instruction:
Given the following Python 3 function, write the documentation
def normalize(X, norm_type, floor=0.0, min_db=-80):
    if isinstance(norm_type, six.string_types):
        if norm_type == "min_max":
            return min_max_normalize(X, floor=floor)
        if norm_type == "log":
            return lognormalize(X, floor=floor, min_db=min_db)
    return librosa.util.normalize(X, norm=norm_type, axis=1)

Response: Normalizes the given matrix of features.


Instruction:
Explain what the following Python 3 code does
def get_time_frames(dur, anal):
    n_frames = get_num_frames(dur, anal)
    return np.linspace(0, dur, num=n_frames)

Response: Gets the time frames and puts them in a numpy array.


Instruction:
Can you generate the documentation for the following Python 3 function
def remove_empty_segments(times, labels):
    assert len(times) - 1 == len(labels)
    inters = times_to_intervals(times)
    new_inters = []
    new_labels = []
    for inter, label in zip(inters, labels):
        if inter[0] < inter[1]:
            new_inters.append(inter)
            new_labels.append(label)
    return intervals_to_times(np.asarray(new_inters)), new_labels

Response: Removes empty segments if needed.


Instruction:
Write a Python 3 function for
synchronizing the labels from the new_bound_idxs to the old_labels.

Response:
def synchronize_labels(new_bound_idxs, old_bound_idxs, old_labels, N):
    """Synchronizes the labels from the old_bound_idxs to the new_bound_idxs.

    Parameters
    ----------
    new_bound_idxs: np.array
        New indeces to synchronize with.
    old_bound_idxs: np.array
        Old indeces, same shape as labels + 1.
    old_labels: np.array
        Labels associated to the old_bound_idxs.
    N: int
        Total number of frames.

    Returns
    -------
    new_labels: np.array
        New labels, synchronized to the new boundary indeces.
    """
    assert len(old_bound_idxs) - 1 == len(old_labels)

    # Construct unfolded labels array
    unfold_labels = np.zeros(N)
    for i, (bound_idx, label) in enumerate(
            zip(old_bound_idxs[:-1], old_labels)):
        unfold_labels[bound_idx:old_bound_idxs[i + 1]] = label

    # Constuct new labels
    new_labels = np.zeros(len(new_bound_idxs) - 1)
    for i, bound_idx in enumerate(new_bound_idxs[:-1]):
        new_labels[i] = np.median(
            unfold_labels[bound_idx:new_bound_idxs[i + 1]])

    return new_labels


Instruction:
Can you create a Python 3 function that
processes a level of segmentation and converts it into times.

Response:
def process_segmentation_level(est_idxs, est_labels, N, frame_times, dur):
    """Processes a level of segmentation, and converts it into times.

    Parameters
    ----------
    est_idxs: np.array
        Estimated boundaries in frame indeces.
    est_labels: np.array
        Estimated labels.
    N: int
        Number of frames in the whole track.
    frame_times: np.array
        Time stamp for each frame.
    dur: float
        Duration of the audio track.

    Returns
    -------
    est_times: np.array
        Estimated segment boundaries in seconds.
    est_labels: np.array
        Estimated labels for each segment.
    """
    assert est_idxs[0] == 0 and est_idxs[-1] == N - 1
    assert len(est_idxs) - 1 == len(est_labels)

    # Add silences, if needed
    est_times = np.concatenate(([0], frame_times[est_idxs], [dur]))
    silence_label = np.max(est_labels) + 1
    est_labels = np.concatenate(([silence_label], est_labels, [silence_label]))

    # Remove empty segments if needed
    est_times, est_labels = remove_empty_segments(est_times, est_labels)

    # Make sure that the first and last times are 0 and duration, respectively
    assert np.allclose([est_times[0]], [0]) and \
        np.allclose([est_times[-1]], [dur])

    return est_times, est_labels


Instruction:
Can you implement a function in Python 3 that
aligns the end of the hierarchies such that they end at the same exact value and they have the same duration within a certain threshold.

Response:
def align_end_hierarchies(hier1, hier2, thres=0.5):
    """Align the end of the hierarchies such that they end at the same exact
    second as long they have the same duration within a certain threshold.

    Parameters
    ----------
    hier1: list
        List containing hierarchical segment boundaries.
    hier2: list
        List containing hierarchical segment boundaries.
    thres: float > 0
        Threshold to decide whether two values are the same.
    """
    # Make sure we have correctly formatted hierarchies
    dur_h1 = hier1[0][-1]
    for hier in hier1:
        assert hier[-1] == dur_h1, "hier1 is not correctly " \
            "formatted {} {}".format(hier[-1], dur_h1)
    dur_h2 = hier2[0][-1]
    for hier in hier2:
        assert hier[-1] == dur_h2, "hier2 is not correctly formatted"

    # If durations are different, do nothing
    if abs(dur_h1 - dur_h2) > thres:
        return

    # Align h1 with h2
    for hier in hier1:
        hier[-1] = dur_h2


Instruction:
Make a summary of the following Python 3 code
def _distance(self, idx):

        if scipy.sparse.issparse(self.data):
            step = self.data.shape[1]
        else:
            step = 50000

        d = np.zeros((self.data.shape[1]))
        if idx == -1:
            # set vec to origin if idx=-1
            vec = np.zeros((self.data.shape[0], 1))
            if scipy.sparse.issparse(self.data):
                vec = scipy.sparse.csc_matrix(vec)
        else:
            vec = self.data[:, idx:idx+1]

        self._logger.info('compute distance to node ' + str(idx))

        # slice data into smaller chunks
        for idx_start in range(0, self.data.shape[1], step):
            if idx_start + step > self.data.shape[1]:
                idx_end = self.data.shape[1]
            else:
                idx_end = idx_start + step

            d[idx_start:idx_end] = self._distfunc(
                self.data[:,idx_start:idx_end], vec)
            self._logger.info('completed:' +
                str(idx_end/(self.data.shape[1]/100.0)) + "%")
        return d

Response: compute distances of a specific data point to all other samples


Instruction:
Explain what the following Python 3 code does
def estimate_K_xmeans(self, th=0.2, maxK = 10):

        # Run initial K-means
        means, labels = self.run_kmeans(self.X, self.init_K)

        # Run X-means algorithm
        stop = False
        curr_K = self.init_K
        while not stop:
            stop = True
            final_means = []
            for k in range(curr_K):
                # Find the data that corresponds to the k-th cluster
                D = self.get_clustered_data(self.X, labels, k)
                if len(D) == 0 or D.shape[0] == 1:
                    continue

                # Whiten and find whitened mean
                stdD = np.std(D, axis=0)
                #D = vq.whiten(D)
                D /= float(stdD)  # Same as line above
                mean = D.mean(axis=0)

                # Cluster this subspace by half (K=2)
                half_means, half_labels = self.run_kmeans(D, K=2)

                # Compute BICs
                bic1 = self.compute_bic(D, [mean], K=1,
                                        labels=np.zeros(D.shape[0]),
                                        R=D.shape[0])
                bic2 = self.compute_bic(D, half_means, K=2,
                                        labels=half_labels, R=D.shape[0])

                # Split or not
                max_bic = np.max([np.abs(bic1), np.abs(bic2)])
                norm_bic1 = bic1 / float(max_bic)
                norm_bic2 = bic2 / float(max_bic)
                diff_bic = np.abs(norm_bic1 - norm_bic2)

                # Split!
                #print "diff_bic", diff_bic
                if diff_bic > th:
                    final_means.append(half_means[0] * stdD)
                    final_means.append(half_means[1] * stdD)
                    curr_K += 1
                    stop = False
                # Don't split
                else:
                    final_means.append(mean * stdD)

            final_means = np.asarray(final_means)

            #print "Estimated K: ", curr_K
            if self.plot:
                plt.scatter(self.X[:, 0], self.X[:, 1])
                plt.scatter(final_means[:, 0], final_means[:, 1], color="y")
                plt.show()

            if curr_K >= maxK or self.X.shape[-1] != final_means.shape[-1]:
                stop = True
            else:
                labels, dist = vq.vq(self.X, final_means)

        return curr_K

Response: Estimates K running X - means algorithm.


Instruction:
Create a Python 3 function for
estimating the K using K - means and BIC by sweeping various K and choosing the optimal BIC.

Response:
def estimate_K_knee(self, th=.015, maxK=12):
        """Estimates the K using K-means and BIC, by sweeping various K and
            choosing the optimal BIC."""
        # Sweep K-means
        if self.X.shape[0] < maxK:
            maxK = self.X.shape[0]
        if maxK < 2:
            maxK = 2
        K = np.arange(1, maxK)
        bics = []
        for k in K:
            means, labels = self.run_kmeans(self.X, k)
            bic = self.compute_bic(self.X, means, labels, K=k,
                                   R=self.X.shape[0])
            bics.append(bic)
        diff_bics = np.diff(bics)
        finalK = K[-1]

        if len(bics) == 1:
            finalK = 2
        else:
            # Normalize
            bics = np.asarray(bics)
            bics -= bics.min()
            #bics /= bics.max()
            diff_bics -= diff_bics.min()
            #diff_bics /= diff_bics.max()

            #print bics, diff_bics

            # Find optimum K
            for i in range(len(K[:-1])):
                #if bics[i] > diff_bics[i]:
                if diff_bics[i] < th and K[i] != 1:
                    finalK = K[i]
                    break

        #print "Estimated K: ", finalK
        if self.plot:
            plt.subplot(2, 1, 1)
            plt.plot(K, bics, label="BIC")
            plt.plot(K[:-1], diff_bics, label="BIC diff")
            plt.legend(loc=2)
            plt.subplot(2, 1, 2)
            plt.scatter(self.X[:, 0], self.X[:, 1])
            plt.show()

        return finalK


Instruction:
Can you write a function in Python 3 where it
returns the data with a specific label_index using the previously splitted labels.

Response:
def get_clustered_data(self, X, labels, label_index):
        """Returns the data with a specific label_index, using the previously
         learned labels."""
        D = X[np.argwhere(labels == label_index)]
        return D.reshape((D.shape[0], D.shape[-1]))


Instruction:
How would you implement a function in Python 3 that
runs k - means and returns the labels assigned to the data.

Response:
def run_kmeans(self, X, K):
        """Runs k-means and returns the labels assigned to the data."""
        wX = vq.whiten(X)
        means, dist = vq.kmeans(wX, K, iter=100)
        labels, dist = vq.vq(wX, means)
        return means, labels


Instruction:
Make a summary of the following Python 3 code
def compute_bic(self, D, means, labels, K, R):
        D = vq.whiten(D)
        Rn = D.shape[0]
        M = D.shape[1]

        if R == K:
            return 1

        # Maximum likelihood estimate (MLE)
        mle_var = 0
        for k in range(len(means)):
            X = D[np.argwhere(labels == k)]
            X = X.reshape((X.shape[0], X.shape[-1]))
            for x in X:
                mle_var += distance.euclidean(x, means[k])
                #print x, means[k], mle_var
        mle_var /= float(R - K)

        # Log-likelihood of the data
        l_D = - Rn/2. * np.log(2*np.pi) - (Rn * M)/2. * np.log(mle_var) - \
            (Rn - K) / 2. + Rn * np.log(Rn) - Rn * np.log(R)

        # Params of BIC
        p = (K-1) + M * K + mle_var

        #print "BIC:", l_D, p, R, K

        # Return the bic
        return l_D - p / 2. * np.log(R)

Response: Computes the Bayesian Information Criterion.


Instruction:
Write a Python 3 function that can
generate N * K 2D data points with K means and N data points for each mean and N data points for each mean.

Response:
def generate_2d_data(self, N=100, K=5):
        """Generates N*K 2D data points with K means and N data points
            for each mean."""
        # Seed the random
        np.random.seed(seed=int(time.time()))

        # Amount of spread of the centroids
        spread = 30

        # Generate random data
        X = np.empty((0, 2))
        for i in range(K):
            mean = np.array([np.random.random()*spread,
                             np.random.random()*spread])
            x = np.random.normal(0.0, scale=1.0, size=(N, 2)) + mean
            X = np.append(X, x, axis=0)

        return X


Instruction:
Can you generate a brief explanation for the following Python 3 code
def factorize(self):
        # compute new coefficients for reconstructing data points
        self.update_w()

        # for CHNMF it is sometimes useful to only compute
        # the basis vectors
        if self._compute_h:
            self.update_h()

        self.W = self.mdl.W
        self.H = self.mdl.H

        self.ferr = np.zeros(1)
        self.ferr[0] = self.mdl.frobenius_norm()
        self._print_cur_status(' Fro:' + str(self.ferr[0]))

Response: Do factorization s. t. data = dot product of data beta and H


Instruction:
Make a summary of the following Python 3 code
def resample_mx(X, incolpos, outcolpos):
    noutcols = len(outcolpos)
    Y = np.zeros((X.shape[0], noutcols))
    # assign 'end times' to final columns
    if outcolpos.max() > incolpos.max():
        incolpos = np.concatenate([incolpos,[outcolpos.max()]])
        X = np.concatenate([X, X[:,-1].reshape(X.shape[0],1)], axis=1)
    outcolpos = np.concatenate([outcolpos, [outcolpos[-1]]])
    # durations (default weights) of input columns)
    incoldurs = np.concatenate([np.diff(incolpos), [1]])

    for c in range(noutcols):
        firstincol = np.where(incolpos <= outcolpos[c])[0][-1]
        firstincolnext = np.where(incolpos < outcolpos[c+1])[0][-1]
        lastincol = max(firstincol,firstincolnext)
        # default weights
        wts = copy.deepcopy(incoldurs[firstincol:lastincol+1])
        # now fix up by partial overlap at ends
        if len(wts) > 1:
            wts[0] = wts[0] - (outcolpos[c] - incolpos[firstincol])
            wts[-1] = wts[-1] - (incolpos[lastincol+1] - outcolpos[c+1])
        wts = wts * 1. / float(sum(wts))
        Y[:,c] = np.dot(X[:,firstincol:lastincol+1], wts)
    # done
    return Y

Response: Resample X to the given time boundaries incolpos and outcolpos.


Instruction:
Explain what the following Python 3 code does
def magnitude(X):
    r = np.real(X)
    i = np.imag(X)
    return np.sqrt(r * r + i * i);

Response: Magnitude of a complex matrix.


Instruction:
Given the following Python 3 function, write the documentation
def json_to_bounds(segments_json):
    f = open(segments_json)
    segments = json.load(f)["segments"]
    bounds = []
    for segment in segments:
        bounds.append(segment["start"])
    bounds.append(bounds[-1] + segments[-1]["duration"]) # Add last boundary
    f.close()
    return np.asarray(bounds)

Response:
Extracts the boundaries from a json file and puts them into
        an np array.


Instruction:
Write a Python 3 script for
extracting the boundaries from a json file and puts them into an np array.

Response:
def json_bounds_to_bounds(bounds_json):
    """Extracts the boundaries from a bounds json file and puts them into
        an np array."""
    f = open(bounds_json)
    segments = json.load(f)["bounds"]
    bounds = []
    for segment in segments:
        bounds.append(segment["start"])
    f.close()
    return np.asarray(bounds)


Instruction:
Make a summary of the following Python 3 code
def json_to_labels(segments_json):
    f = open(segments_json)
    segments = json.load(f)["segments"]
    labels = []
    str_labels = []
    for segment in segments:
        if not segment["label"] in str_labels:
            str_labels.append(segment["label"])
            labels.append(len(str_labels)-1)
        else:
            label_idx = np.where(np.asarray(str_labels) == segment["label"])[0][0]
            labels.append(label_idx)
    f.close()
    return np.asarray(labels)

Response: Extracts the labels from a json file and puts them into a np array.


Instruction:
Can you write a function in Python 3 where it
extracts the beats from the beats_json_file and puts them into an np array.

Response:
def json_to_beats(beats_json_file):
    """Extracts the beats from the beats_json_file and puts them into
        an np array."""
    f = open(beats_json_file, "r")
    beats_json = json.load(f)
    beats = []
    for beat in beats_json["beats"]:
        beats.append(beat["start"])
    f.close()
    return np.asarray(beats)


Instruction:
Given the following Python 3 function, write the documentation
def compute_ffmc2d(X):
    # 2d-fft
    fft2 = scipy.fftpack.fft2(X)

    # Magnitude
    fft2m = magnitude(fft2)

    # FFTshift and flatten
    fftshift = scipy.fftpack.fftshift(fft2m).flatten()

    #cmap = plt.cm.get_cmap('hot')
    #plt.imshow(np.log1p(scipy.fftpack.fftshift(fft2m)).T, interpolation="nearest",
    #    aspect="auto", cmap=cmap)
    #plt.show()

    # Take out redundant components
    return fftshift[:fftshift.shape[0] // 2 + 1]

Response: Computes the 2D - Fourier Magnitude Coefficients.


Instruction:
Write a Python 3 script to
factorize s. t. WH.

Response:
def factorize(self, niter=10, compute_w=True, compute_h=True,
                  compute_err=True, show_progress=False):
        """ Factorize s.t. WH = data

            Parameters
            ----------
            niter : int
                    number of iterations.
            show_progress : bool
                    print some extra information to stdout.
            compute_h : bool
                    iteratively update values for H.
            compute_w : bool
                    iteratively update values for W.
            compute_err : bool
                    compute Frobenius norm |data-WH| after each update and store
                    it to .ferr[k].

            Updated Values
            --------------
            .W : updated values for W.
            .H : updated values for H.
            .ferr : Frobenius norm |data-WH| for each iteration.
        """

        if not hasattr(self,'W'):
               self.init_w()

        if not hasattr(self,'H'):
                self.init_h()

        def separate_positive(m):
            return (np.abs(m) + m)/2.0

        def separate_negative(m):
            return (np.abs(m) - m)/2.0

        if show_progress:
            self._logger.setLevel(logging.INFO)
        else:
            self._logger.setLevel(logging.ERROR)

        XtX = np.dot(self.data[:,:].T, self.data[:,:])
        XtX_pos = separate_positive(XtX)
        XtX_neg = separate_negative(XtX)

        self.ferr = np.zeros(niter)
        # iterate over W and H

        for i in range(niter):
            # update H
            XtX_neg_x_W = np.dot(XtX_neg, self.G)
            XtX_pos_x_W = np.dot(XtX_pos, self.G)

            if compute_h:
                H_x_WT = np.dot(self.H.T, self.G.T)
                ha = XtX_pos_x_W + np.dot(H_x_WT, XtX_neg_x_W)
                hb = XtX_neg_x_W + np.dot(H_x_WT, XtX_pos_x_W) + 10**-9
                self.H = (self.H.T*np.sqrt(ha/hb)).T

            # update W
            if compute_w:
                HT_x_H = np.dot(self.H, self.H.T)
                wa = np.dot(XtX_pos, self.H.T) + np.dot(XtX_neg_x_W, HT_x_H)
                wb = np.dot(XtX_neg, self.H.T) + np.dot(XtX_pos_x_W, HT_x_H) + 10**-9

                self.G *= np.sqrt(wa/wb)
                self.W = np.dot(self.data[:,:], self.G)

            if compute_err:
                self.ferr[i] = self.frobenius_norm()
                self._logger.info('Iteration ' + str(i+1) + '/' + str(niter) +
                ' FN:' + str(self.ferr[i]))
            else:
                self._logger.info('Iteration ' + str(i+1) + '/' + str(niter))

            if i > 1 and compute_err:
                if self.converged(i):
                    self.ferr = self.ferr[:i]
                    break


Instruction:
Can you tell what is the following Python 3 function doing
def cnmf(S, rank, niter=500, hull=False):
    if hull:
        nmf_mdl = pymf.CHNMF(S, num_bases=rank)
    else:
        nmf_mdl = pymf.CNMF(S, num_bases=rank)
    nmf_mdl.factorize(niter=niter)
    F = np.asarray(nmf_mdl.W)
    G = np.asarray(nmf_mdl.H)
    return F, G

Response: Convex Non - Negative Matrix Factorization.


Instruction:
Here you have a function in Python 3, explain what it does
def compute_labels(X, rank, R, bound_idxs, niter=300):

    try:
        F, G = cnmf(X, rank, niter=niter, hull=False)
    except:
        return [1]

    label_frames = filter_activation_matrix(G.T, R)
    label_frames = np.asarray(label_frames, dtype=int)

    #labels = [label_frames[0]]
    labels = []
    bound_inters = zip(bound_idxs[:-1], bound_idxs[1:])
    for bound_inter in bound_inters:
        if bound_inter[1] - bound_inter[0] <= 0:
            labels.append(np.max(label_frames) + 1)
        else:
            labels.append(most_frequent(
                label_frames[bound_inter[0]: bound_inter[1]]))
        #print bound_inter, labels[-1]
    #labels.append(label_frames[-1])

    return labels

Response: Computes the labels using the bounds.


Instruction:
Write a Python 3 script for
filtering the activation matrix G and returns a flattened copy.

Response:
def filter_activation_matrix(G, R):
    """Filters the activation matrix G, and returns a flattened copy."""

    #import pylab as plt
    #plt.imshow(G, interpolation="nearest", aspect="auto")
    #plt.show()

    idx = np.argmax(G, axis=1)
    max_idx = np.arange(G.shape[0])
    max_idx = (max_idx, idx.flatten())
    G[:, :] = 0
    G[max_idx] = idx + 1

    # TODO: Order matters?
    G = np.sum(G, axis=1)
    G = median_filter(G[:, np.newaxis], R)

    return G.flatten()


Instruction:
Can you generate the documentation for the following Python 3 function
def get_segmentation(X, rank, R, rank_labels, R_labels, niter=300,
                     bound_idxs=None, in_labels=None):
    """
    Gets the segmentation (boundaries and labels) from the factorization
    matrices.

    Parameters
    ----------
    X: np.array()
        Features matrix (e.g. chromagram)
    rank: int
        Rank of decomposition
    R: int
        Size of the median filter for activation matrix
    niter: int
        Number of iterations for k-means
    bound_idxs : list
        Use previously found boundaries (None to detect them)
    in_labels : np.array()
        List of input labels (None to compute them)

    Returns
    -------
    bounds_idx: np.array
        Bound indeces found
    labels: np.array
        Indeces of the labels representing the similarity between segments.
    """

    #import pylab as plt
    #plt.imshow(X, interpolation="nearest", aspect="auto")
    #plt.show()

    # Find non filtered boundaries
    compute_bounds = True if bound_idxs is None else False
    while True:
        if bound_idxs is None:
            try:
                F, G = cnmf(X, rank, niter=niter, hull=False)
            except:
                return np.empty(0), [1]

            # Filter G
            G = filter_activation_matrix(G.T, R)
            if bound_idxs is None:
                bound_idxs = np.where(np.diff(G) != 0)[0] + 1

        # Increase rank if we found too few boundaries
        if compute_bounds and len(np.unique(bound_idxs)) <= 2:
            rank += 1
            bound_idxs = None
        else:
            break

    # Add first and last boundary
    bound_idxs = np.concatenate(([0], bound_idxs, [X.shape[1] - 1]))
    bound_idxs = np.asarray(bound_idxs, dtype=int)
    if in_labels is None:
        labels = compute_labels(X, rank_labels, R_labels, bound_idxs,
                                niter=niter)
    else:
        labels = np.ones(len(bound_idxs) - 1)

    #plt.imshow(G[:, np.newaxis], interpolation="nearest", aspect="auto")
    #for b in bound_idxs:
        #plt.axvline(b, linewidth=2.0, color="k")
    #plt.show()

    return bound_idxs, labels

Response: Returns the segmentation of a factorization matrix X with rank rank_labels R_labels and labels in in_labels.


Instruction:
Create a Python 3 function to
process the flat data.

Response:
def processFlat(self):
        """Main process.
        Returns
        -------
        est_idxs : np.array(N)
            Estimated indeces for the segment boundaries in frames.
        est_labels : np.array(N-1)
            Estimated labels for the segments.
        """
        # C-NMF params
        niter = self.config["niters"]  # Iterations for the MF and clustering

        # Preprocess to obtain features, times, and input boundary indeces
        F = self._preprocess()

        # Normalize
        F = U.normalize(F, norm_type=self.config["norm_feats"])

        if F.shape[0] >= self.config["h"]:
            # Median filter
            F = median_filter(F, M=self.config["h"])
            #plt.imshow(F.T, interpolation="nearest", aspect="auto"); plt.show()

            # Find the boundary indices and labels using matrix factorization
            est_idxs, est_labels = get_segmentation(
                F.T, self.config["rank"], self.config["R"],
                self.config["rank_labels"], self.config["R_labels"],
                niter=niter, bound_idxs=self.in_bound_idxs, in_labels=None)

            # Remove empty segments if needed
            est_idxs, est_labels = U.remove_empty_segments(est_idxs, est_labels)
        else:
            # The track is too short. We will only output the first and last
            # time stamps
            if self.in_bound_idxs is None:
                est_idxs = np.array([0, F.shape[0] - 1])
                est_labels = [1]
            else:
                est_idxs = self.in_bound_idxs
                est_labels = [1] * (len(est_idxs) + 1)

        # Make sure that the first and last boundaries are included
        assert est_idxs[0] == 0 and est_idxs[-1] == F.shape[0] - 1

        # Post process estimations
        est_idxs, est_labels = self._postprocess(est_idxs, est_labels)

        return est_idxs, est_labels


Instruction:
Here you have a function in Python 3, explain what it does
def get_boundaries_module(boundaries_id):
    if boundaries_id == "gt":
        return None
    try:
        module = eval(algorithms.__name__ + "." + boundaries_id)
    except AttributeError:
        raise RuntimeError("Algorithm %s can not be found in msaf!" %
                           boundaries_id)
    if not module.is_boundary_type:
        raise RuntimeError("Algorithm %s can not identify boundaries!" %
                           boundaries_id)
    return module

Response: Returns the object containing the selected boundary module.


Instruction:
Create a Python 3 function for
returning the label module given a label algorithm identificator.

Response:
def get_labels_module(labels_id):
    """Obtains the label module given a label algorithm identificator.

    Parameters
    ----------
    labels_id: str
        Label algorithm identificator (e.g., fmc2d, cnmf).

    Returns
    -------
    module: object
        Object containing the selected label module.
        None for not computing the labeling part of music segmentation.
    """
    if labels_id is None:
        return None
    try:
        module = eval(algorithms.__name__ + "." + labels_id)
    except AttributeError:
        raise RuntimeError("Algorithm %s can not be found in msaf!" %
                           labels_id)
    if not module.is_label_type:
        raise RuntimeError("Algorithm %s can not label segments!" %
                           labels_id)
    return module


Instruction:
Implement a function in Python 3 to
run hierarchical algorithms with the specified identifiers on the audio_file.

Response:
def run_hierarchical(audio_file, bounds_module, labels_module, frame_times,
                     config, annotator_id=0):
    """Runs hierarchical algorithms with the specified identifiers on the
    audio_file. See run_algorithm for more information.
    """
    # Sanity check
    if bounds_module is None:
        raise NoHierBoundaryError("A boundary algorithm is needed when using "
                                  "hierarchical segmentation.")

    # Get features to make code nicer
    features = config["features"].features

    # Compute boundaries
    S = bounds_module.Segmenter(audio_file, **config)
    est_idxs, est_labels = S.processHierarchical()

    # Compute labels if needed
    if labels_module is not None and \
            bounds_module.__name__ != labels_module.__name__:
        # Compute labels for each level in the hierarchy
        flat_config = deepcopy(config)
        flat_config["hier"] = False
        for i, level_idxs in enumerate(est_idxs):
            S = labels_module.Segmenter(audio_file,
                                        in_bound_idxs=level_idxs,
                                        **flat_config)
            est_labels[i] = S.processFlat()[1]

    # Make sure the first and last boundaries are included for each
    # level in the hierarchy
    est_times = []
    cleaned_est_labels = []
    for level in range(len(est_idxs)):
        est_level_times, est_level_labels = \
            utils.process_segmentation_level(
                est_idxs[level], est_labels[level], features.shape[0],
                frame_times, config["features"].dur)
        est_times.append(est_level_times)
        cleaned_est_labels.append(est_level_labels)
    est_labels = cleaned_est_labels

    return est_times, est_labels


Instruction:
Can you create a Python 3 function that
runs the flat algorithm with the specified identifiers on the the audio_file.

Response:
def run_flat(file_struct, bounds_module, labels_module, frame_times, config,
             annotator_id):
    """Runs the flat algorithms with the specified identifiers on the
    audio_file. See run_algorithm for more information.
    """
    # Get features to make code nicer
    features = config["features"].features

    # Segment using the specified boundaries and labels
    # Case when boundaries and labels algorithms are the same
    if bounds_module is not None and labels_module is not None and \
            bounds_module.__name__ == labels_module.__name__:
        S = bounds_module.Segmenter(file_struct, **config)
        est_idxs, est_labels = S.processFlat()
    # Different boundary and label algorithms
    else:
        # Identify segment boundaries
        if bounds_module is not None:
            S = bounds_module.Segmenter(file_struct, in_labels=[], **config)
            est_idxs, est_labels = S.processFlat()
        else:
            try:
                # Ground-truth boundaries
                est_times, est_labels = io.read_references(
                    file_struct.audio_file, annotator_id=annotator_id)
                est_idxs = io.align_times(est_times, frame_times)
                if est_idxs[0] != 0:
                    est_idxs = np.concatenate(([0], est_idxs))
            except IOError:
                logging.warning("No references found for file: %s" %
                                file_struct.audio_file)
                return [], []

        # Label segments
        if labels_module is not None:
            if len(est_idxs) == 2:
                est_labels = np.array([0])
            else:
                S = labels_module.Segmenter(file_struct,
                                            in_bound_idxs=est_idxs,
                                            **config)
                est_labels = S.processFlat()[1]

    # Make sure the first and last boundaries are included
    est_times, est_labels = utils.process_segmentation_level(
        est_idxs, est_labels, features.shape[0], frame_times,
        config["features"].dur)

    return est_times, est_labels


Instruction:
Given the following Python 3 function, write the documentation
def run_algorithms(file_struct, boundaries_id, labels_id, config,
                   annotator_id=0):
    """Runs the algorithms with the specified identifiers on the audio_file.

    Parameters
    ----------
    file_struct: `msaf.io.FileStruct`
        Object with the file paths.
    boundaries_id: str
        Identifier of the boundaries algorithm to use ("gt" for ground truth).
    labels_id: str
        Identifier of the labels algorithm to use (None for not labeling).
    config: dict
        Dictionary containing the custom parameters of the algorithms to use.
    annotator_id: int
        Annotator identificator in the ground truth.

    Returns
    -------
    est_times: np.array or list
        List of estimated times for the segment boundaries.
        If `list`, it will be a list of np.arrays, sorted by segmentation
        layer.
    est_labels: np.array or list
        List of all the labels associated segments.
        If `list`, it will be a list of np.arrays, sorted by segmentation
        layer.
    """
    # Check that there are enough audio frames
    if config["features"].features.shape[0] <= msaf.config.minimum_frames:
        logging.warning("Audio file too short, or too many few beats "
                        "estimated. Returning empty estimations.")
        return np.asarray([0, config["features"].dur]), \
            np.asarray([0], dtype=int)

    # Get the corresponding modules
    bounds_module = get_boundaries_module(boundaries_id)
    labels_module = get_labels_module(labels_id)

    # Get the correct frame times
    frame_times = config["features"].frame_times

    # Segment audio based on type of segmentation
    run_fun = run_hierarchical if config["hier"] else run_flat
    est_times, est_labels = run_fun(file_struct, bounds_module, labels_module,
                                    frame_times, config, annotator_id)

    return est_times, est_labels

Response: Runs the algorithms with the specified identifiers on the audio_file.


Instruction:
Implement a Python 3 function for
preparing the parameters runs the algorithms and saves the results in the estimator file.

Response:
def process_track(file_struct, boundaries_id, labels_id, config,
                  annotator_id=0):
    """Prepares the parameters, runs the algorithms, and saves results.

    Parameters
    ----------
    file_struct: `msaf.io.FileStruct`
        FileStruct containing the paths of the input files (audio file,
        features file, reference file, output estimation file).
    boundaries_id: str
        Identifier of the boundaries algorithm to use ("gt" for ground truth).
    labels_id: str
        Identifier of the labels algorithm to use (None for not labeling).
    config: dict
        Dictionary containing the custom parameters of the algorithms to use.
    annotator_id: int
        Annotator identificator in the ground truth.

    Returns
    -------
    est_times: np.array
        List of estimated times for the segment boundaries.
    est_labels: np.array
        List of all the labels associated segments.
    """
    logging.info("Segmenting %s" % file_struct.audio_file)

    # Get features
    config["features"] = Features.select_features(
        config["feature"], file_struct, config["annot_beats"],
        config["framesync"])

    # Get estimations
    est_times, est_labels = run_algorithms(file_struct,
                                           boundaries_id, labels_id, config,
                                           annotator_id=annotator_id)

    # Save
    logging.info("Writing results in: %s" % file_struct.est_file)
    io.save_estimations(file_struct, est_times, est_labels,
                        boundaries_id, labels_id, **config)

    return est_times, est_labels


Instruction:
Create a Python 3 function to
update W under the convexity of the class

Response:
def update_w(self):
        """ alternating least squares step, update W under the convexity
        constraint """
        def update_single_w(i):
            """ compute single W[:,i] """
            # optimize beta     using qp solver from cvxopt
            FB = base.matrix(np.float64(np.dot(-self.data.T, W_hat[:,i])))
            be = solvers.qp(HB, FB, INQa, INQb, EQa, EQb)
            self.beta[i,:] = np.array(be['x']).reshape((1, self._num_samples))

        # float64 required for cvxopt
        HB = base.matrix(np.float64(np.dot(self.data[:,:].T, self.data[:,:])))
        EQb = base.matrix(1.0, (1, 1))
        W_hat = np.dot(self.data, pinv(self.H))
        INQa = base.matrix(-np.eye(self._num_samples))
        INQb = base.matrix(0.0, (self._num_samples, 1))
        EQa = base.matrix(1.0, (1, self._num_samples))

        for i in range(self._num_bases):
            update_single_w(i)

        self.W = np.dot(self.beta, self.data.T).T


Instruction:
Can you tell what is the following Python 3 function doing
def main():
    '''
    Main Entry point for translator and argument parser
    '''
    args      = command_line()
    translate = partial(translator, args.source, args.dest,
                        version=' '.join([__version__, __build__]))

    return source(spool(set_task(translate, translit=args.translit)), args.text)

Response: Main entry point for translator and argument parser


Instruction:
Implement a Python 3 function for
initializing coroutine essentially priming it to the yield statement. Used as a decorator over functions that generate coroutines. .. code-block:: python # Basic coroutine producer/consumer pattern from translate import coroutine @coroutine def coroutine_foo(bar): try: while True: baz = (yield) bar.send(baz) except GeneratorExit: bar.close() :param func: Unprimed Generator :type func: Function :return: Initialized Coroutine :rtype: Function

Response:
def coroutine(func):
    """
    Initializes coroutine essentially priming it to the yield statement.
    Used as a decorator over functions that generate coroutines.

    .. code-block:: python

        # Basic coroutine producer/consumer pattern
        from translate import coroutine

        @coroutine
        def coroutine_foo(bar):
            try:
                while True:
                    baz = (yield)
                    bar.send(baz)

            except GeneratorExit:
                bar.close()

    :param func: Unprimed Generator
    :type func: Function

    :return: Initialized Coroutine
    :rtype: Function
    """

    @wraps(func)
    def initialization(*args, **kwargs):

        start = func(*args, **kwargs)
        next(start)

        return start

    return initialization


Instruction:
Explain what the following Python 3 code does
def accumulator(init, update):
    return (
        init + len(update)
            if isinstance(init, int) else
        init + update
    )

Response:
Generic accumulator function.

    .. code-block:: python

        # Simplest Form
        >>> a = 'this' + ' '
        >>> b = 'that'
        >>> c = functools.reduce(accumulator, a, b)
        >>> c
        'this that'

        # The type of the initial value determines output type.
        >>> a = 5
        >>> b = Hello
        >>> c = functools.reduce(accumulator, a, b)
        >>> c
        10

    :param init:  Initial Value
    :param update: Value to accumulate

    :return: Combined Values


Instruction:
Write a Python 3 script for
writing a stream of text to stdout.

Response:
def write_stream(script, output='trans'):
    """
    :param script: Translated Text
    :type script: Iterable

    :param output: Output Type (either 'trans' or 'translit')
    :type output: String
    """
    first    = operator.itemgetter(0)
    sentence, _ = script
    printer  = partial(print, file=sys.stdout, end='')

    for line in sentence:
        if isinstance(first(line), str):
            printer(first(line))
        else:
            printer(first(line).encode('UTF-8'))

    printer('\n')

    return sys.stdout.flush()


Instruction:
Can you tell what is the following Python 3 function doing
def set_task(translator, translit=False):
    # Initialize Task Queue
    task    = str()
    queue   = list()

    # Function Partial
    output  = ('translit' if translit else 'trans')
    stream  = partial(write_stream, output=output)
    workers = ThreadPoolExecutor(max_workers=8)

    try:
        while True:

            task = yield
            queue.append(task)

    except GeneratorExit:
        list(map(stream, workers.map(translator, queue)))

Response: This function sets the Task of the next available language.


Instruction:
Implement a Python 3 function for
reading the entire sequence of words from the given iterable and spools them together for more io efficient processes.

Response:
def spool(iterable, maxlen=1250):
    """
    Consumes text streams and spools them together for more io
    efficient processes.

    :param iterable: Sends text stream for further processing
    :type iterable: Coroutine

    :param maxlen: Maximum query string size
    :type maxlen: Integer
    """
    words = int()
    text  = str()

    try:
        while True:

            while words < maxlen:
                stream = yield
                text   = reduce(accumulator, stream, text)
                words  = reduce(accumulator, stream, words)

            iterable.send(text)
            words = int()
            text  = str()

    except GeneratorExit:
        iterable.send(text)
        iterable.close()


Instruction:
Make a summary of the following Python 3 code
def source(target, inputstream=sys.stdin):
    for line in inputstream:

        while len(line) > 600:
            init, sep, line = line.partition(' ')
            assert len(init) <= 600
            target.send(''.join([init, sep]))

        target.send(line)

    inputstream.close()

    return target.close()

Response: Coroutine starting point. Produces text stream and forwards to consumers


Instruction:
Make a summary of the following Python 3 code
def push_url(interface):
    '''
    Decorates a function returning the url of translation API.
    Creates and maintains HTTP connection state

    Returns a dict response object from the server containing the translated
    text and metadata of the request body

    :param interface: Callable Request Interface
    :type interface: Function
    '''

    @functools.wraps(interface)
    def connection(*args, **kwargs):
        """
        Extends and wraps a HTTP interface.

        :return: Response Content
        :rtype: Dictionary
        """
        session = Session()
        session.mount('http://',  HTTPAdapter(max_retries=2))
        session.mount('https://', HTTPAdapter(max_retries=2))

        request  = Request(**interface(*args, **kwargs))
        prepare  = session.prepare_request(request)
        response = session.send(prepare, verify=True)

        if response.status_code != requests.codes.ok:
            response.raise_for_status()

        cleanup = re.subn(r',(?=,)', '', response.content.decode('utf-8'))[0]

        return json.loads(cleanup.replace(r'\xA0', r' ').replace('[,', '[1,'), encoding='UTF-8')

    return connection

Response: Decorator that creates a HTTP connection to the translation API and returns the url of the translation API.


Instruction:
How would you code a function in Python 3 to
translate a source language code into a target language code.

Response:
def translator(source, target, phrase, version='0.0 test', charset='utf-8'):
    """
    Returns the url encoded string that will be pushed to the translation
    server for parsing.

    List of acceptable language codes for source and target languages
    can be found as a JSON file in the etc directory.

    Some source languages are limited in scope of the possible target languages
    that are available.

    .. code-block:: python

        >>> from translate import translator
        >>> translator('en', 'zh-TW', 'Hello World!')
            '你好世界！'

    :param source: Language code for translation source
    :type source: String

    :param target: Language code that source will be translate into
    :type target: String

    :param phrase: Text body string that will be url encoded and translated
    :type phrase: String

    :return: Request Interface
    :rtype: Dictionary
    """

    url     = 'https://translate.google.com/translate_a/single'
    agent   = 'User-Agent',   'py-translate v{}'.format(version)
    content = 'Content-Type', 'application/json; charset={}'.format(charset)

    params  = {'client': 'a', 'ie': charset, 'oe': charset,
                   'dt': 't', 'sl':  source, 'tl':  target,  'q': phrase}

    request = {'method': 'GET',
                  'url': url,
               'params': params,
              'headers': dict([agent, content])}

    return request


Instruction:
Can you write a function in Python 3 where it
reads in the language codes and prints them out.

Response:
def translation_table(language, filepath='supported_translations.json'):
    '''
    Opens up file located under the etc directory containing language
    codes and prints them out.

    :param file: Path to location of json file
    :type file: str

    :return: language codes
    :rtype: dict
    '''
    fullpath = abspath(join(dirname(__file__), 'etc', filepath))

    if not isfile(fullpath):
        raise IOError('File does not exist at {0}'.format(fullpath))

    with open(fullpath, 'rt') as fp:
        raw_data = json.load(fp).get(language, None)
        assert(raw_data is not None)

    return dict((code['language'], code['name']) for code in raw_data)


Instruction:
Write a Python 3 function for
generating a formatted table of language codes

Response:
def print_table(language):
    '''
    Generates a formatted table of language codes
    '''
    table = translation_table(language)

    for code, name in sorted(table.items(), key=operator.itemgetter(0)):
        print(u'{language:<8} {name:\u3000<20}'.format(
            name=name, language=code
        ))

    return None


Instruction:
Here you have a function in Python 3, explain what it does
def remove_nodes(network, rm_nodes):
    rm_nodes = set(rm_nodes)
    ndf = network.nodes_df
    edf = network.edges_df

    nodes_to_keep = ~ndf.index.isin(rm_nodes)
    edges_to_keep = ~(edf['from'].isin(rm_nodes) | edf['to'].isin(rm_nodes))

    return ndf.loc[nodes_to_keep], edf.loc[edges_to_keep]

Response: Create DataFrames of nodes and edges that do not include specified nodes.


Instruction:
Can you tell what is the following Python 3 function doing
def network_to_pandas_hdf5(network, filename, rm_nodes=None):
    if rm_nodes is not None:
        nodes, edges = remove_nodes(network, rm_nodes)
    else:
        nodes, edges = network.nodes_df, network.edges_df

    with pd.HDFStore(filename, mode='w') as store:
        store['nodes'] = nodes
        store['edges'] = edges

        store['two_way'] = pd.Series([network._twoway])
        store['impedance_names'] = pd.Series(network.impedance_names)

Response: Save a Network to a Pandas HDFStore.


Instruction:
Create a Python 3 function for
building a Network from data in a Pandas HDF5 file.

Response:
def network_from_pandas_hdf5(cls, filename):
    """
    Build a Network from data in a Pandas HDFStore.

    Parameters
    ----------
    cls : class
        Class to instantiate, usually pandana.Network.
    filename : str

    Returns
    -------
    network : pandana.Network

    """
    with pd.HDFStore(filename) as store:
        nodes = store['nodes']
        edges = store['edges']
        two_way = store['two_way'][0]
        imp_names = store['impedance_names'].tolist()

    return cls(
        nodes['x'], nodes['y'], edges['from'], edges['to'], edges[imp_names],
        twoway=two_way)


Instruction:
Implement a function in Python 3 to
return the bounding box for the nodes in this network

Response:
def bbox(self):
        """
        The bounding box for nodes in this network [xmin, ymin, xmax, ymax]
        """
        return [self.nodes_df.x.min(), self.nodes_df.y.min(),
                self.nodes_df.x.max(), self.nodes_df.y.max()]


Instruction:
Write a Python 3 script for
characterizing urban space with a variable that is related to nodes in the network. Parameters ---------- node_ids : Pandas Series, int A series of node_ids which are usually computed using get_node_ids on this object. variable : Pandas Series, numeric, optional A series which represents some variable defined in urban space. It could be the location of buildings, or the income of all households - just about anything can be aggregated using the network queries provided here and this provides the api to set the variable at its disaggregate locations. Note that node_id and variable should have the same index (although the index is not actually used). If variable is not set, then it is assumed that the variable is all "ones" at the location specified by node_ids. This could be, for instance, the location of all coffee shops which don't really have a variable to aggregate. The variable is connected to the closest node in the Pandana network which assumes no impedance between the location of the variable and the location of the closest network node. name : string, optional Name the variable. This is optional in the sense that if you don't specify it, the default name will be used. Since the same default name is used by aggregate on this object, you can alternate between characterize and aggregate calls without setting names. Returns ------- Nothing

Response:
def set(self, node_ids, variable=None, name="tmp"):
        """
        Characterize urban space with a variable that is related to nodes in
        the network.

        Parameters
        ----------
        node_ids : Pandas Series, int
            A series of node_ids which are usually computed using
            get_node_ids on this object.
        variable : Pandas Series, numeric, optional
            A series which represents some variable defined in urban space.
            It could be the location of buildings, or the income of all
            households - just about anything can be aggregated using the
            network queries provided here and this provides the api to set
            the variable at its disaggregate locations.  Note that node_id
            and variable should have the same index (although the index is
            not actually used).  If variable is not set, then it is assumed
            that the variable is all "ones" at the location specified by
            node_ids.  This could be, for instance, the location of all
            coffee shops which don't really have a variable to aggregate. The
            variable is connected to the closest node in the Pandana network
            which assumes no impedance between the location of the variable
            and the location of the closest network node.
        name : string, optional
            Name the variable.  This is optional in the sense that if you don't
            specify it, the default name will be used.  Since the same
            default name is used by aggregate on this object, you can
            alternate between characterize and aggregate calls without
            setting names.

        Returns
        -------
        Nothing
        """

        if variable is None:
            variable = pd.Series(np.ones(len(node_ids)), index=node_ids.index)

        df = pd.DataFrame({name: variable,
                           "node_idx": self._node_indexes(node_ids)})

        length = len(df)
        df = df.dropna(how="any")
        newl = len(df)
        if length-newl > 0:
            print(
                "Removed %d rows because they contain missing values" %
                (length-newl))

        self.variable_names.add(name)

        self.net.initialize_access_var(name.encode('utf-8'),
                                       df.node_idx.values.astype('int'),
                                       df[name].values.astype('double'))


Instruction:
Make a summary of the following Python 3 code
def aggregate(self, distance, type="sum", decay="linear", imp_name=None,
                  name="tmp"):
        """
        Aggregate information for every source node in the network - this is
        really the main purpose of this library.  This allows you to touch
        the data specified by calling set and perform some aggregation on it
        within the specified distance.  For instance, summing the population
        within 1000 meters.

        Parameters
        ----------
        distance : float
            The maximum distance to aggregate data within. 'distance' can
            represent any impedance unit that you have set as your edge
            weight. This will usually be a distance unit in meters however
            if you have customized the impedance this could be in other
            units such as utility or time etc.
        type : string
            The type of aggregation, can be one of "ave", "sum", "std",
            "count", and now "min", "25pct", "median", "75pct", and "max" will
            compute the associated quantiles.  (Quantiles are computed by
            sorting so might be slower than the others.)
        decay : string
            The type of decay to apply, which makes things that are further
            away count less in the aggregation - must be one of "linear",
            "exponential" or "flat" (which means no decay).  Linear is the
            fastest computation to perform.  When performing an "ave",
            the decay is typically "flat"
        imp_name : string, optional
            The impedance name to use for the aggregation on this network.
            Must be one of the impedance names passed in the constructor of
            this object.  If not specified, there must be only one impedance
            passed in the constructor, which will be used.
        name : string, optional
            The variable to aggregate.  This variable will have been created
            and named by a call to set.  If not specified, the default
            variable name will be used so that the most recent call to set
            without giving a name will be the variable used.

        Returns
        -------
        agg : Pandas Series
            Returns a Pandas Series for every origin node in the network,
            with the index which is the same as the node_ids passed to the
            init method and the values are the aggregations for each source
            node in the network.
        """

        imp_num = self._imp_name_to_num(imp_name)
        type = type.lower()
        if type == "ave":
            type = "mean"  # changed generic ave to mean

        assert name in self.variable_names, "A variable with that name " \
                                            "has not yet been initialized"

        res = self.net.get_all_aggregate_accessibility_variables(distance,
                                                                 name.encode('utf-8'),
                                                                 type.encode('utf-8'),
                                                                 decay.encode('utf-8'),
                                                                 imp_num)

        return pd.Series(res, index=self.node_ids)

Response: Aggregate the data for every source node in the network within the specified distance.


Instruction:
Write a Python 3 function for
assigning node_ids to data specified by x_col and y_col Parameters ---------- x_col : Pandas series (float) A Pandas Series where values specify the x (e.g. longitude) location of dataset. y_col : Pandas series (float) A Pandas Series where values specify the y (e.g. latitude) location of dataset. x_col and y_col should use the same index. mapping_distance : float, optional The maximum distance that will be considered a match between the x, y data and the nearest node in the network. This will usually be a distance unit in meters however if you have customized the impedance this could be in other units such as utility or time etc. If not specified, every x, y coordinate will be mapped to the nearest node. Returns ------- node_ids : Pandas series (int) Returns a Pandas Series of node_ids for each x, y in the input data. The index is the same as the indexes of the x, y input data, and the values are the mapped node_ids. If mapping distance is not passed and if there are no nans in the x, y data, this will be the same length as the x, y data. If the mapping is imperfect, this function returns all the input x, y's that were successfully mapped to node_ids.

Response:
def get_node_ids(self, x_col, y_col, mapping_distance=None):
        """
        Assign node_ids to data specified by x_col and y_col

        Parameters
        ----------
        x_col : Pandas series (float)
            A Pandas Series where values specify the x (e.g. longitude)
            location of dataset.
        y_col : Pandas series (float)
            A Pandas Series where values specify the y (e.g. latitude)
            location of dataset.  x_col and y_col should use the same index.
        mapping_distance : float, optional
            The maximum distance that will be considered a match between the
            x, y data and the nearest node in the network.  This will usually
            be a distance unit in meters however if you have customized the
            impedance this could be in other units such as utility or time
            etc. If not specified, every x, y coordinate will be mapped to
            the nearest node.

        Returns
        -------
        node_ids : Pandas series (int)
            Returns a Pandas Series of node_ids for each x, y in the
            input data. The index is the same as the indexes of the
            x, y input data, and the values are the mapped node_ids.
            If mapping distance is not passed and if there are no nans in the
            x, y data, this will be the same length as the x, y data.
            If the mapping is imperfect, this function returns all the
            input x, y's that were successfully mapped to node_ids.
        """
        xys = pd.DataFrame({'x': x_col, 'y': y_col})

        distances, indexes = self.kdtree.query(xys.as_matrix())
        indexes = np.transpose(indexes)[0]
        distances = np.transpose(distances)[0]

        node_ids = self.nodes_df.iloc[indexes].index

        df = pd.DataFrame({"node_id": node_ids, "distance": distances},
                          index=xys.index)

        if mapping_distance is not None:
            df = df[df.distance <= mapping_distance]

        return df.node_id