Instruction:
How would you code a function in Python 3 to
take PDF output from PDFExporter and uploads to One Codex Documents portal.

Response:
def from_notebook_node(self, nb, resources=None, **kw):
        """Takes PDF output from PDFExporter and uploads to One Codex Documents portal."""
        output, resources = super(OneCodexDocumentExporter, self).from_notebook_node(
            nb, resources=resources, **kw
        )

        from onecodex import Api
        from onecodex.lib.upload import upload_document_fileobj

        ocx = Api()

        default_filename = "Analysis Report - {dt:%B} {dt.day}, {dt:%Y}".format(
            dt=datetime.datetime.now()
        )

        file_name = resources["metadata"].get("one_codex_doc_portal_filename", default_filename)

        try:
            document_id = upload_document_fileobj(
                BytesIO(output), file_name, ocx._client.session, ocx.Documents._resource
            )
        except UploadException as exc:
            resp = json.dumps({"status": 500, "message": str(exc)})
            return resp, resources
        except Exception:
            resp = json.dumps(
                {
                    "status": 500,
                    "message": "Upload failed. Please contact help@onecodex.com for assistance.",
                }
            )
            return resp, resources

        resp = json.dumps({"status": 200, "document_id": document_id})
        return resp, resources


Instruction:
How would you implement a function in Python 3 that
generates a tree structure of strings for the current language of the current language.

Response:
def generate_strings(project_base_dir, localization_bundle_path, tmp_directory, exclude_dirs, include_strings_file,
                     special_ui_components_prefix):
    """
    Calls the builtin 'genstrings' command with JTLocalizedString as the string to search for,
    and adds strings extracted from UI elements internationalized with 'JTL' + removes duplications.
    """

    localization_directory = os.path.join(localization_bundle_path, DEFAULT_LANGUAGE_DIRECTORY_NAME)
    if not os.path.exists(localization_directory):
        os.makedirs(localization_directory)

    localization_file = os.path.join(localization_directory, LOCALIZATION_FILENAME)

    # Creating the same directory tree structure in the tmp directory
    tmp_localization_directory = os.path.join(tmp_directory, DEFAULT_LANGUAGE_DIRECTORY_NAME)
    tmp_localization_file = os.path.join(tmp_localization_directory, LOCALIZATION_FILENAME)

    if os.path.isdir(tmp_localization_directory):
        shutil.rmtree(tmp_localization_directory)
    os.mkdir(tmp_localization_directory)

    logging.info("Running genstrings")

    source_files = extract_source_files(project_base_dir, exclude_dirs)

    genstrings_cmd = 'genstrings -s JTLocalizedString -o %s %s' % (tmp_localization_directory, " ".join(
            ['"%s"' % (source_file,) for source_file in source_files]))

    genstrings_process = subprocess.Popen(genstrings_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                          stdin=subprocess.PIPE, shell=True)

    genstrings_out, genstrings_err = genstrings_process.communicate()

    remove_empty_comments_from_file(tmp_localization_file)
    add_genstrings_comments_to_file(tmp_localization_file, genstrings_err)

    genstrings_rc = genstrings_process.returncode
    if genstrings_rc != 0:
        logging.fatal("genstrings returned %d, aborting run!", genstrings_rc)
        sys.exit(genstrings_rc)

    create_localized_strings_from_ib_files(project_base_dir, exclude_dirs, tmp_localization_file,
                                           special_ui_components_prefix)

    if include_strings_file:
        target = open_strings_file(tmp_localization_file, "a")
        source = open_strings_file(include_strings_file, "r")
        target.write(source.read())
        source.close()
        target.close()

    handle_duplications(tmp_localization_file)

    if os.path.isfile(localization_file):
        logging.info("Merging old localizable with new one...")
        merge_strings_files(localization_file, tmp_localization_file)
    else:
        logging.info("No Localizable yet, moving the created file...")
        shutil.move(tmp_localization_file, localization_file)


Instruction:
Can you write a function in Python 3 where it
returns a new SampleCollection containing only samples meeting the filter criteria.

Response:
def filter(self, filter_func):
        """Return a new SampleCollection containing only samples meeting the filter criteria.

        Will pass any kwargs (e.g., field or skip_missing) used when instantiating the current class
        on to the new SampleCollection that is returned.

        Parameters
        ----------
        filter_func : `callable`
            A function that will be evaluated on every object in the collection. The function must
            return a `bool`. If True, the object will be kept. If False, it will be removed from the
            SampleCollection that is returned.

        Returns
        -------
        `onecodex.models.SampleCollection` containing only objects `filter_func` returned True on.

        Examples
        --------
        Generate a new collection of Samples that have a specific filename extension:

            new_collection = samples.filter(lambda s: s.filename.endswith('.fastq.gz'))
        """
        if callable(filter_func):
            return self.__class__([obj for obj in self if filter_func(obj) is True], **self._kwargs)
        else:
            raise OneCodexException(
                "Expected callable for filter, got: {}".format(type(filter_func).__name__)
            )


Instruction:
Here you have a function in Python 3, explain what it does
def _classification_fetch(self, skip_missing=None):
        skip_missing = skip_missing if skip_missing else self._kwargs["skip_missing"]

        new_classifications = []

        for a in self._res_list:
            if a.__class__.__name__ == "Samples":
                c = a.primary_classification
            elif a.__class__.__name__ == "Classifications":
                c = a
            else:
                raise OneCodexException(
                    "Objects in SampleCollection must be one of: Classifications, Samples"
                )

            if skip_missing and not c.success:
                warnings.warn("Classification {} not successful. Skipping.".format(c.id))
                continue

            new_classifications.append(c)

        self._cached["classifications"] = new_classifications

Response: Turns a list of objects associated with a classification result into a list of Classifications objects.


Instruction:
Write a Python 3 function that can
turn a list of objects associated with a classification result into a DataFrame of metadata. Returns None if no metadata is found.

Response:
def _collate_metadata(self):
        """Turns a list of objects associated with a classification result into a DataFrame of
        metadata.

        Returns
        -------
        None, but stores a result in self._cached.
        """
        import pandas as pd

        DEFAULT_FIELDS = None
        metadata = []

        for c in self._classifications:
            m = c.sample.metadata

            if DEFAULT_FIELDS is None:
                DEFAULT_FIELDS = list(m._resource._schema["properties"].keys())
                DEFAULT_FIELDS.remove("$uri")
                DEFAULT_FIELDS.remove("sample")

            metadatum = {f: getattr(m, f) for f in DEFAULT_FIELDS}
            metadatum["classification_id"] = c.id
            metadatum["sample_id"] = m.sample.id
            metadatum["metadata_id"] = m.id
            metadatum["created_at"] = m.sample.created_at
            metadatum["filename"] = c.sample.filename

            metadatum.update(m.custom)
            metadata.append(metadatum)

        if metadata:
            metadata = pd.DataFrame(metadata).set_index("classification_id")
        else:
            metadata = pd.DataFrame(
                columns=["classification_id", "sample_id", "metadata_id", "created_at"]
            )

        self._cached["metadata"] = metadata


Instruction:
Write a Python 3 script to
return the results of a classification result.

Response:
def _collate_results(self, field=None):
        """For a list of objects associated with a classification result, return the results as a
        DataFrame and dict of taxa info.

        Parameters
        ----------
        field : {'readcount_w_children', 'readcount', 'abundance'}
            Which field to use for the abundance/count of a particular taxon in a sample.

            - 'readcount_w_children': total reads of this taxon and all its descendants
            - 'readcount': total reads of this taxon
            - 'abundance': genome size-normalized relative abundances, from shotgun sequencing

        Returns
        -------
        None, but stores a result in self._cached.
        """
        import pandas as pd

        field = field if field else self._kwargs["field"]

        if field not in ("auto", "abundance", "readcount", "readcount_w_children"):
            raise OneCodexException("Specified field ({}) not valid.".format(field))

        # we'll fill these dicts that eventually turn into DataFrames
        df = {"classification_id": [c.id for c in self._classifications]}

        tax_info = {"tax_id": [], "name": [], "rank": [], "parent_tax_id": []}

        if field == "auto":
            field = "readcount_w_children"

        self._cached["field"] = field

        for c_idx, c in enumerate(self._classifications):
            # pulling results from mainline is the slowest part of the function
            result = c.results()["table"]

            # d contains info about a taxon in result, including name, id, counts, rank, etc.
            for d in result:
                d_tax_id = d["tax_id"]

                if d_tax_id not in tax_info["tax_id"]:
                    for k in ("tax_id", "name", "rank", "parent_tax_id"):
                        tax_info[k].append(d[k])

                    # first time we've seen this taxon, so make a vector for it
                    df[d_tax_id] = [0] * len(self._classifications)

                df[d_tax_id][c_idx] = d[field]

        # format as a Pandas DataFrame
        df = pd.DataFrame(df).set_index("classification_id").fillna(0)

        df.columns.name = "tax_id"

        tax_info = pd.DataFrame(tax_info).set_index("tax_id")

        self._cached["results"] = df
        self._cached["taxonomy"] = tax_info


Instruction:
Here you have a function in Python 3, explain what it does
def to_otu(self, biom_id=None):
        otu_format = "Biological Observation Matrix 1.0.0"

        # Note: This is exact format URL is required by https://github.com/biocore/biom-format
        otu_url = "http://biom-format.org"

        otu = OrderedDict(
            {
                "id": biom_id,
                "format": otu_format,
                "format_url": otu_url,
                "type": "OTU table",
                "generated_by": "One Codex API V1",
                "date": datetime.now().isoformat(),
                "rows": [],
                "columns": [],
                "matrix_type": "sparse",
                "matrix_element_type": "int",
            }
        )

        rows = defaultdict(dict)

        tax_ids_to_names = {}
        for classification in self._classifications:
            col_id = len(otu["columns"])  # 0 index

            # Re-encoding the JSON is a bit of a hack, but
            # we need a ._to_dict() method that properly
            # resolves references and don't have one at the moment
            columns_entry = {
                "id": str(classification.id),
                "sample_id": str(classification.sample.id),
                "sample_filename": classification.sample.filename,
                "metadata": json.loads(
                    classification.sample.metadata._to_json(include_references=False)
                ),
            }

            otu["columns"].append(columns_entry)
            sample_df = classification.table()

            for row in sample_df.iterrows():
                tax_id = row[1]["tax_id"]
                tax_ids_to_names[tax_id] = row[1]["name"]
                rows[tax_id][col_id] = int(row[1]["readcount"])

        num_rows = len(rows)
        num_cols = len(otu["columns"])

        otu["shape"] = [num_rows, num_cols]
        otu["data"] = []

        for present_taxa in sorted(rows):
            # add the row entry
            row_id = len(otu["rows"])
            otu["rows"].append(
                {"id": present_taxa, "metadata": {"taxonomy": tax_ids_to_names[present_taxa]}}
            )

            for sample_with_hit in rows[present_taxa]:
                counts = rows[present_taxa][sample_with_hit]
                otu["data"].append([row_id, sample_with_hit, counts])

        return otu

Response: Converts a list of objects associated with a classification result into a Python OrderedDict that can be dumped to JSON.


Instruction:
Can you tell what is the following Python 3 function doing
def localization_merge_back(updated_localizable_file, old_translated_file, new_translated_file, merged_translated_file):
    output_file_elements = []
    old_translated_file_dict = generate_localization_key_to_entry_dictionary_from_file(old_translated_file)
    new_translated_file_dict = generate_localization_key_to_entry_dictionary_from_file(new_translated_file)

    f = open_strings_file(updated_localizable_file, "r")

    for header_comment, comments, key, value in extract_header_comment_key_value_tuples_from_file(f):
        translation_value = None
        if len(header_comment) > 0:
            output_file_elements.append(Comment(header_comment))

        if value in new_translated_file_dict:
            translation_value = new_translated_file_dict[value].value
        elif key in old_translated_file_dict:
            translation_value = old_translated_file_dict[key].value
        elif key in new_translated_file_dict:
            translation_value = new_translated_file_dict[key].value

        if translation_value is not None:
            output_file_elements.append(LocalizationEntry(comments, key, translation_value))

    f.close()

    write_file_elements_to_strings_file(merged_translated_file, output_file_elements)

Response: This function generates a new file that contains the updated localization strings file and the new ones.


Instruction:
Here you have a function in Python 3, explain what it does
def boxplot(df, category, quantity, category_type="N", title=None, xlabel=None, ylabel=None):
    # must be one of Nominal, Ordinal, Time per altair
    if category_type not in ("N", "O", "T"):
        raise OneCodexException("If specifying category_type, must be N, O, or T")

    # adapted from https://altair-viz.github.io/gallery/boxplot_max_min.html
    lower_box = "q1({}):Q".format(quantity)
    lower_whisker = "min({}):Q".format(quantity)
    upper_box = "q3({}):Q".format(quantity)
    upper_whisker = "max({}):Q".format(quantity)

    if category_type == "T":
        x_format = "hoursminutes({}):{}".format(category, category_type)
    else:
        x_format = "{}:{}".format(category, category_type)

    lower_plot = (
        alt.Chart(df)
        .mark_rule()
        .encode(y=alt.Y(lower_whisker, axis=alt.Axis(title=ylabel)), y2=lower_box, x=x_format)
    )

    middle_plot = alt.Chart(df).mark_bar(size=35).encode(y=lower_box, y2=upper_box, x=x_format)

    upper_plot = alt.Chart(df).mark_rule().encode(y=upper_whisker, y2=upper_box, x=x_format)

    middle_tick = (
        alt.Chart(df)
        .mark_tick(color="black", size=35)
        .encode(
            y="median({}):Q".format(quantity),
            x=alt.X(x_format, axis=alt.Axis(title=xlabel), scale=alt.Scale(rangeStep=45)),
            tooltip="median({}):Q".format(quantity),
        )
    )

    chart = lower_plot + middle_plot + upper_plot + middle_tick

    if title:
        chart = chart.properties(title=title)

    return chart

Response: Plot a simple boxplot using Altair.


Instruction:
Implement a function in Python 3 to
plot a simple square dendrogram using Altair.

Response:
def dendrogram(tree):
    """Plot a simple square dendrogram using Altair.

    Parameters
    ----------
    tree : `dict` returned by `scipy.cluster.hierarchy.dendrogram`
    Contains, at a minimum, 'icoord', 'dcoord', and 'leaves' keys. Scipy does all the work of
    determining where the lines in the tree should go. All we have to do is draw them.

    Returns
    -------
    `altair.Chart`
    """
    plot_data = {
        "x": [],
        "y": [],
        "o": [],  # order these points should be connected in
        "b": [],  # one number per branch
    }

    for idx, (i, d) in enumerate(zip(tree["icoord"], tree["dcoord"])):
        plot_data["x"].extend(map(lambda x: -x, d))
        plot_data["y"].extend(map(lambda x: -x, i))
        plot_data["o"].extend([0, 1, 2, 3])
        plot_data["b"].extend([idx] * 4)

    plot_data = pd.DataFrame(plot_data)

    chart = (
        alt.Chart(plot_data, width=100, height=15 * len(tree["leaves"]) - 7.5)
        .mark_line(point=False, opacity=0.5)
        .encode(
            x=alt.X("x", axis=None),
            y=alt.Y("y", axis=None, scale=alt.Scale(zero=True, nice=False)),
            order="o",
            color=alt.Color(
                "b:N",
                scale=alt.Scale(domain=list(range(idx + 1)), range=["black"] * (idx + 1)),
                legend=None,
            ),
        )
    )

    return chart


Instruction:
Create a Python 3 function for
adding the comments produced by the genstrings script to the strings file.

Response:
def add_genstrings_comments_to_file(localization_file, genstrings_err):
    """ Adds the comments produced by the genstrings script for duplicate keys.

    Args:
        localization_file (str): The path to the strings file.

    """

    errors_to_log = [line for line in genstrings_err.splitlines() if "used with multiple comments" not in line]

    if len(errors_to_log) > 0:
        logging.warning("genstrings warnings:\n%s", "\n".join(errors_to_log))

    loc_file = open_strings_file(localization_file, "a")

    regex_matches = re.findall(r'Warning: Key "(.*?)" used with multiple comments ("[^"]*" (& "[^"]*")+)',
                               genstrings_err)

    logging.info("Adding multiple comments from genstrings output")
    for regex_match in regex_matches:
        if len(regex_match) == 3:
            key = regex_match[0]
            comments = [comment.strip()[1:-1] for comment in regex_match[1].split("&")]

            logging.info("Found key with %d comments: %s", len(comments), key)

            loc_key = LocalizationEntry(comments, key, key)

            loc_file.write(unicode(loc_key))
            loc_file.write(u"\n")

    loc_file.close()


Instruction:
Write a Python 3 script for
plotting beta diversity distance matrix for a specific species level.

Response:
def plot_distance(
        self,
        rank="auto",
        metric="braycurtis",
        title=None,
        xlabel=None,
        ylabel=None,
        tooltip=None,
        return_chart=False,
        linkage="average",
        label=None,
    ):
        """Plot beta diversity distance matrix as a heatmap and dendrogram.

        Parameters
        ----------
        rank : {'auto', 'kingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species'}, optional
            Analysis will be restricted to abundances of taxa at the specified level.
        metric : {'braycurtis', 'manhattan', 'jaccard', 'unifrac', 'unweighted_unifrac}, optional
            Function to use when calculating the distance between two samples.
        linkage : {'average', 'single', 'complete', 'weighted', 'centroid', 'median'}
            The type of linkage to use when clustering axes.
        title : `string`, optional
            Text label at the top of the plot.
        xlabel : `string`, optional
            Text label along the horizontal axis.
        ylabel : `string`, optional
            Text label along the vertical axis.
        tooltip : `string` or `list`, optional
            A string or list containing strings representing metadata fields. When a point in the
            plot is hovered over, the value of the metadata associated with that sample will be
            displayed in a modal.
        label : `string` or `callable`, optional
            A metadata field (or function) used to label each analysis. If passing a function, a
            dict containing the metadata for each analysis is passed as the first and only
            positional argument. The callable function must return a string.

        Examples
        --------
        Plot the weighted UniFrac distance between all our samples, using counts at the genus level.

        >>> plot_distance(rank='genus', metric='unifrac')
        """
        if len(self._results) < 2:
            raise OneCodexException(
                "`plot_distance` requires 2 or more valid classification results."
            )

        # this will be passed to the heatmap chart as a dataframe eventually
        plot_data = {"1) Label": [], "2) Label": [], "Distance": [], "classification_id": []}

        # here we figure out what to put in the tooltips and get the appropriate data
        if tooltip:
            if not isinstance(tooltip, list):
                tooltip = [tooltip]
        else:
            tooltip = []

        tooltip.insert(0, "Label")

        magic_metadata, magic_fields = self._metadata_fetch(tooltip, label=label)
        formatted_fields = []

        for _, magic_field in magic_fields.items():
            field_group = []

            for i in (1, 2):
                field = "{}) {}".format(i, magic_field)
                plot_data[field] = []
                field_group.append(field)

            formatted_fields.append(field_group)

        clust = self._cluster_by_sample(rank=rank, metric=metric, linkage=linkage)

        # must convert to long format for heatmap plotting
        for idx1, id1 in enumerate(clust["dist_matrix"].index):
            for idx2, id2 in enumerate(clust["dist_matrix"].index):
                if idx1 == idx2:
                    plot_data["Distance"].append(np.nan)
                else:
                    plot_data["Distance"].append(clust["dist_matrix"].iloc[idx1, idx2])

                plot_data["classification_id"].append(id1)

                for field_group, magic_field in zip(formatted_fields, magic_fields.values()):
                    plot_data[field_group[0]].append(magic_metadata[magic_field][id1])
                    plot_data[field_group[1]].append(magic_metadata[magic_field][id2])

        plot_data = pd.DataFrame(data=plot_data)

        labels_in_order = magic_metadata["Label"][clust["ids_in_order"]].tolist()

        # it's important to tell altair to order the cells in the heatmap according to the clustering
        # obtained from scipy
        alt_kwargs = dict(
            x=alt.X("1) Label:N", axis=alt.Axis(title=xlabel), sort=labels_in_order),
            y=alt.Y(
                "2) Label:N", axis=alt.Axis(title=ylabel, orient="right"), sort=labels_in_order
            ),
            color="Distance:Q",
            tooltip=list(chain.from_iterable(formatted_fields)) + ["Distance:Q"],
            href="url:N",
            url="https://app.onecodex.com/classification/" + alt.datum.classification_id,
        )

        chart = (
            alt.Chart(
                plot_data,
                width=15 * len(clust["dist_matrix"].index),
                height=15 * len(clust["dist_matrix"].index),
            )
            .transform_calculate(url=alt_kwargs.pop("url"))
            .mark_rect()
            .encode(**alt_kwargs)
        )

        if title:
            chart = chart.properties(title=title)

        dendro_chart = dendrogram(clust["scipy_tree"])

        if return_chart:
            return dendro_chart | chart
        else:
            (dendro_chart | chart).display()


Instruction:
Create a Python 3 function to
plot beta diversity distance matrix for a specific metadata class.

Response:
def plot_mds(
        self,
        rank="auto",
        metric="braycurtis",
        method="pcoa",
        title=None,
        xlabel=None,
        ylabel=None,
        color=None,
        size=None,
        tooltip=None,
        return_chart=False,
        label=None,
    ):
        """Plot beta diversity distance matrix using multidimensional scaling (MDS).

        Parameters
        ----------
        rank : {'auto', 'kingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species'}, optional
            Analysis will be restricted to abundances of taxa at the specified level.
        metric : {'braycurtis', 'manhattan', 'jaccard', 'unifrac', 'unweighted_unifrac}, optional
            Function to use when calculating the distance between two samples.
        method : {'pcoa', 'smacof'}
            Algorithm to use for ordination. PCoA uses eigenvalue decomposition and is not well
            suited to non-euclidean distance functions. SMACOF is an iterative optimization strategy
            that can be used as an alternative.
        title : `string`, optional
            Text label at the top of the plot.
        xlabel : `string`, optional
            Text label along the horizontal axis.
        ylabel : `string`, optional
            Text label along the vertical axis.
        size : `string` or `tuple`, optional
            A string or a tuple containing strings representing metadata fields. The size of points
            in the resulting plot will change based on the metadata associated with each sample.
        color : `string` or `tuple`, optional
            A string or a tuple containing strings representing metadata fields. The color of points
            in the resulting plot will change based on the metadata associated with each sample.
        tooltip : `string` or `list`, optional
            A string or list containing strings representing metadata fields. When a point in the
            plot is hovered over, the value of the metadata associated with that sample will be
            displayed in a modal.
        label : `string` or `callable`, optional
            A metadata field (or function) used to label each analysis. If passing a function, a
            dict containing the metadata for each analysis is passed as the first and only
            positional argument. The callable function must return a string.

        Examples
        --------
        Scatter plot of weighted UniFrac distance between all our samples, using counts at the genus
        level.

        >>> plot_mds(rank='genus', metric='unifrac')

        Notes
        -----
        **For `smacof`**: The values reported on the axis labels are Pearson's correlations between
        the distances between points on each axis alone, and the corresponding distances in the
        distance matrix calculated using the user-specified metric. These values are related to the
        effectiveness of the MDS algorithm in placing points on the scatter plot in such a way that
        they truly represent the calculated distances. They do not reflect how well the distance
        metric captures similarities between the underlying data (in this case, an OTU table).
        """
        if len(self._results) < 2:
            raise OneCodexException("`plot_mds` requires 2 or more valid classification results.")

        dists = self._compute_distance(rank, metric).to_data_frame()

        # here we figure out what to put in the tooltips and get the appropriate data
        if tooltip:
            if not isinstance(tooltip, list):
                tooltip = [tooltip]
        else:
            tooltip = []

        tooltip.insert(0, "Label")

        if color and color not in tooltip:
            tooltip.insert(1, color)

        if size and size not in tooltip:
            tooltip.insert(2, size)

        magic_metadata, magic_fields = self._metadata_fetch(tooltip, label=label)

        if method == "smacof":
            # adapted from https://scikit-learn.org/stable/auto_examples/manifold/plot_mds.html
            x_field = "MDS1"
            y_field = "MDS2"

            seed = np.random.RandomState(seed=3)
            mds = manifold.MDS(
                max_iter=3000, eps=1e-12, random_state=seed, dissimilarity="precomputed", n_jobs=1
            )
            pos = mds.fit(dists).embedding_
            plot_data = pd.DataFrame(pos, columns=[x_field, y_field], index=dists.index)
            plot_data = plot_data.div(plot_data.abs().max(axis=0), axis=1)  # normalize to [0,1]

            # determine how much of the original distance is captured by each of the axes after MDS.
            # this implementation of MDS does not use eigen decomposition and so there's no simple
            # way of returning a 'percent of variance explained' value
            r_squared = []

            for axis in [0, 1]:
                mds_dist = pos.copy()
                mds_dist[::, axis] = 0
                mds_dist = squareform(euclidean_distances(mds_dist).round(6))
                r_squared.append(pearsonr(mds_dist, squareform(dists))[0])

            # label the axes
            x_extra_label = "r² = %.02f" % (r_squared[0],)
            y_extra_label = "r² = %.02f" % (r_squared[1],)
        elif method == "pcoa":
            # suppress eigenvalue warning from skbio--not because it's an invalid warning, but
            # because lots of folks in the field run pcoa on these distances functions, even if
            # statistically inappropriate. perhaps this will change if we ever become more
            # opinionated about the analyses that we allow our users to do (roo)
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                ord_result = ordination.pcoa(
                    dists.round(6)
                )  # round to avoid float precision errors

            plot_data = ord_result.samples.iloc[:, [0, 1]]  # get first two components
            plot_data = plot_data.div(plot_data.abs().max(axis=0), axis=1)  # normalize to [0,1]
            plot_data.index = dists.index
            x_field, y_field = plot_data.columns.tolist()  # name of first two components

            x_extra_label = "%0.02f%%" % (ord_result.proportion_explained[0] * 100,)
            y_extra_label = "%0.02f%%" % (ord_result.proportion_explained[1] * 100,)
        else:
            raise OneCodexException("MDS method must be one of: smacof, pcoa")

        # label the axes
        if xlabel is None:
            xlabel = "{} ({})".format(x_field, x_extra_label)
        if ylabel is None:
            ylabel = "{} ({})".format(y_field, y_extra_label)

        plot_data = pd.concat([plot_data, magic_metadata], axis=1).reset_index()

        alt_kwargs = dict(
            x=alt.X(x_field, axis=alt.Axis(title=xlabel)),
            y=alt.Y(y_field, axis=alt.Axis(title=ylabel)),
            tooltip=[magic_fields[t] for t in tooltip],
            href="url:N",
            url="https://app.onecodex.com/classification/" + alt.datum.classification_id,
        )

        # only add these parameters if they are in use
        if color:
            alt_kwargs["color"] = magic_fields[color]
        if size:
            alt_kwargs["size"] = magic_fields[size]

        chart = (
            alt.Chart(plot_data)
            .transform_calculate(url=alt_kwargs.pop("url"))
            .mark_circle()
            .encode(**alt_kwargs)
        )

        if title:
            chart = chart.properties(title=title)

        if return_chart:
            return chart
        else:
            chart.interactive().display()


Instruction:
Write a Python 3 function for
downloading the file from One Codex and saves it to a file - like object.

Response:
def download(self, path=None, file_obj=None, progressbar=False):
        """Downloads files from One Codex.

        Parameters
        ----------
        path : `string`, optional
            Full path to save the file to. If omitted, defaults to the original filename
            in the current working directory.
        file_obj : file-like object, optional
            Rather than save the file to a path, write it to this file-like object.
        progressbar : `bool`
            Display a progress bar using Click for the download?

        Returns
        -------
        `string`
            The path the file was downloaded to, if applicable. Otherwise, None.

        Notes
        -----
        If no arguments specified, defaults to download the file as the original filename
        in the current working directory. If `file_obj` given, will write data into the
        passed file-like object. If `path` given, will download the file to the path provided,
        but will not overwrite any existing files.
        """
        if path and file_obj:
            raise OneCodexException("Please specify only one of: path, file_obj")

        if path is None and file_obj is None:
            path = os.path.join(os.getcwd(), self.filename)

        if path and os.path.exists(path):
            raise OneCodexException("{} already exists! Will not overwrite.".format(path))

        try:
            url_data = self._resource.download_uri()
            resp = requests.get(url_data["download_uri"], stream=True)

            with (open(path, "wb") if path else file_obj) as f_out:
                if progressbar:
                    with click.progressbar(length=self.size, label=self.filename) as bar:
                        for data in resp.iter_content(chunk_size=1024):
                            bar.update(len(data))
                            f_out.write(data)
                else:
                    for data in resp.iter_content(chunk_size=1024):
                        f_out.write(data)
        except KeyboardInterrupt:
            if path:
                os.remove(path)
            raise
        except requests.exceptions.HTTPError as exc:
            if exc.response.status_code == 401:
                raise OneCodexException("You must be logged in to download files.")
            elif exc.response.status_code == 402:
                raise OneCodexException(
                    "You must either have a premium platform account or be in "
                    "a notebook environment to download files."
                )
            elif exc.response.status_code == 403:
                raise OneCodexException("You are not authorized to download this file.")
            else:
                raise OneCodexException(
                    "Download failed with an HTTP status code {}.".format(exc.response.status_code)
                )

        return path


Instruction:
Given the following Python 3 function, write the documentation
def upload(cls, file_path, progressbar=None):
        res = cls._resource
        doc_id = upload_document(file_path, res._client.session, res, progressbar=progressbar)

        return cls.get(doc_id)

Response: Uploads a series of files to One Codex server.


Instruction:
Can you write a function in Python 3 where it
prepares the localization bundle for translation.

Response:
def prepare_for_translation(localization_bundle_path):
    """ Prepares the localization bundle for translation.

    This means, after creating the strings files using genstrings.sh, this will produce '.pending' files, that contain
    the files that are yet to be translated.

    Args:
        localization_bundle_path (str): The path to the localization bundle.

    """

    logging.info("Preparing for translation..")

    for strings_file in os.listdir(os.path.join(localization_bundle_path, DEFAULT_LANGUAGE_DIRECTORY_NAME)):
        if not strings_file.endswith(".strings"):
            continue

        strings_path = os.path.join(localization_bundle_path, DEFAULT_LANGUAGE_DIRECTORY_NAME, strings_file)
        for lang_dir in os.listdir(localization_bundle_path):
            if lang_dir == DEFAULT_LANGUAGE_DIRECTORY_NAME or lang_dir.startswith("."):
                continue

            dest_strings_path = os.path.join(localization_bundle_path, lang_dir, strings_file)
            pending_path = dest_strings_path + ".pending"
            excluded_path = dest_strings_path + ".excluded"
            if not os.path.exists(dest_strings_path):
                open_strings_file(dest_strings_path, "a").close()

            logging.info("Preparing diff for %s in %s", lang_dir, pending_path)
            localization_diff(strings_path, dest_strings_path, excluded_path, pending_path)


Instruction:
Make a summary of the following Python 3 code
def documents_upload(ctx, max_threads, files):
    if len(files) == 0:
        click.echo(ctx.get_help())
        return

    files = list(files)

    bar = click.progressbar(length=sum([_file_size(x) for x in files]), label="Uploading... ")
    run_via_threadpool(
        ctx.obj["API"].Documents.upload,
        files,
        {"progressbar": bar},
        max_threads=max_threads,
        graceful_exit=False,
    )

Response: Upload a document to One Codex


Instruction:
Create a Python 3 function for
retrieving performed metagenomic classifications

Response:
def classifications(ctx, classifications, results, readlevel, readlevel_path):
    """Retrieve performed metagenomic classifications"""

    # basic operation -- just print
    if not readlevel and not results:
        cli_resource_fetcher(ctx, "classifications", classifications)

    # fetch the results
    elif not readlevel and results:
        if len(classifications) != 1:
            log.error("Can only request results data on one Classification at a time")
        else:
            classification = ctx.obj["API"].Classifications.get(classifications[0])
            if not classification:
                log.error(
                    "Could not find classification {} (404 status code)".format(classifications[0])
                )
                return
            results = classification.results(json=True)
            pprint(results, ctx.obj["NOPPRINT"])

    # fetch the readlevel
    elif readlevel is not None and not results:
        if len(classifications) != 1:
            log.error("Can only request read-level data on one Classification at a time")
        else:
            classification = ctx.obj["API"].Classifications.get(classifications[0])
            if not classification:
                log.error(
                    "Could not find classification {} (404 status code)".format(classifications[0])
                )
                return
            tsv_url = classification._readlevel()["url"]
            log.info("Downloading tsv data from: {}".format(tsv_url))
            download_file_helper(tsv_url, readlevel_path)

    # both given -- complain
    else:
        log.error("Can only request one of read-level data or results data at a time")


Instruction:
Can you create a Python 3 function that
uploads a FASTA or FASTQ file to One Codex.

Response:
def upload(
    ctx, files, max_threads, prompt, forward, reverse, tags, metadata, project_id, coerce_ascii
):
    """Upload a FASTA or FASTQ (optionally gzip'd) to One Codex"""

    appendables = {}
    if tags:
        appendables["tags"] = []
        for tag in tags:
            appendables["tags"].append(tag)

    if metadata:
        appendables["metadata"] = {}
        for metadata_kv in metadata:
            split_metadata = metadata_kv.split("=", 1)
            if len(split_metadata) > 1:
                metadata_value = split_metadata[1]
                appendables["metadata"][split_metadata[0]] = metadata_value

    appendables = validate_appendables(appendables, ctx.obj["API"])

    if (forward or reverse) and not (forward and reverse):
        click.echo("You must specify both forward and reverse files", err=True)
        ctx.exit(1)

    if forward and reverse:
        if len(files) > 0:
            click.echo(
                "You may not pass a FILES argument when using the "
                " --forward and --reverse options.",
                err=True,
            )
            ctx.exit(1)
        files = [(forward, reverse)]
    elif len(files) == 0:
        click.echo(ctx.get_help())
        return
    else:
        files = list(files)

        # "intelligently" find paired files and tuple them
        paired_files = []
        single_files = set(files)

        for filename in files:
            # convert "read 1" filenames into "read 2" and check that they exist; if they do
            # upload the files as a pair, autointerleaving them
            pair = re.sub("[._][Rr]1[._]", lambda x: x.group().replace("1", "2"), filename)

            # we don't necessary need the R2 to have been passed in; we infer it anyways
            if pair != filename and os.path.exists(pair):
                if not prompt and pair not in single_files:
                    # if we're not prompting, don't automatically pull in files
                    # not in the list the user passed in
                    continue

                paired_files.append((filename, pair))

                if pair in single_files:
                    single_files.remove(pair)

                single_files.remove(filename)

        auto_pair = True

        if prompt and len(paired_files) > 0:
            pair_list = ""
            for p in paired_files:
                pair_list += "\n  {}  &  {}".format(os.path.basename(p[0]), os.path.basename(p[1]))

            answer = click.confirm(
                "It appears there are paired files:{}\nInterleave them after upload?".format(
                    pair_list
                ),
                default="Y",
            )

            if not answer:
                auto_pair = False

        if auto_pair:
            files = paired_files + list(single_files)

    total_size = sum(
        [
            (_file_size(x[0], uncompressed=True) + _file_size(x[1], uncompressed=True))
            if isinstance(x, tuple)
            else _file_size(x, uncompressed=False)
            for x in files
        ]
    )

    upload_kwargs = {
        "metadata": appendables["valid_metadata"],
        "tags": appendables["valid_tags"],
        "project": project_id,
        "coerce_ascii": coerce_ascii,
        "progressbar": progressbar(length=total_size, label="Uploading..."),
    }

    run_via_threadpool(
        ctx.obj["API"].Samples.upload,
        files,
        upload_kwargs,
        max_threads=max_threads,
        graceful_exit=False,
    )


Instruction:
Can you implement a function in Python 3 that
logs-in to One Codex and check if the credentials match.

Response:
def login(ctx):
    """Add an API key (saved in ~/.onecodex)"""
    base_url = os.environ.get("ONE_CODEX_API_BASE", "https://app.onecodex.com")
    if not ctx.obj["API_KEY"]:
        _login(base_url)
    else:
        email = _login(base_url, api_key=ctx.obj["API_KEY"])
        ocx = Api(api_key=ctx.obj["API_KEY"], telemetry=ctx.obj["TELEMETRY"])

        # TODO: This should be protected or built in as a first class resource
        # with, e.g., connection error catching (it's not part of our formally documeted API at the moment)
        if ocx._client.Account.instances()["email"] != email:
            click.echo("Your login credentials do not match the provided email!", err=True)
            _remove_creds()
            ctx.exit(1)


Instruction:
Can you generate the documentation for the following Python 3 function
def make_taxonomy_dict(classification, parent=False):

    tax_id_map = {}

    if parent:
        for row in classification.results()["table"]:
            if row["parent_tax_id"] is not None and row["tax_id"] is not None:
                tax_id_map[row["tax_id"]] = row["parent_tax_id"]
    else:
        for row in classification.results()["table"]:
            if row["parent_tax_id"] is not None and row["tax_id"] is not None:
                try:
                    tax_id_map[row["parent_tax_id"]].add(row["tax_id"])
                except KeyError:
                    tax_id_map[row["parent_tax_id"]] = set([row["tax_id"]])

    return tax_id_map

Response: Takes a classification data frame and parses it into a dictionary mapping a tax_id to its children or parent.


Instruction:
Here you have a function in Python 3, explain what it does
def recurse_taxonomy_map(tax_id_map, tax_id, parent=False):

    if parent:
        # TODO: allow filtering on tax_id and its parents, too
        pass
    else:

        def _child_recurse(tax_id, visited):
            try:
                children = [tax_id] + list(tax_id_map[tax_id])
            except KeyError:
                children = [tax_id]

            for child in children:
                if child not in visited:
                    visited.append(child)
                    children.extend(_child_recurse(child, visited))

            return children

        return list(set(_child_recurse(tax_id, [])))

Response: Recursively walks the taxonomy tree to get all the children of the given tax_id.


Instruction:
Can you tell what is the following Python 3 function doing
def reference(text=None, label=None):
    if text is None and label is None:
        raise OneCodexException("Please specify at least one of: text, label")

    try:
        ipy = get_ipython()
        ref_list = ipy.meta.get("references", {})
    except NameError:
        raise OneCodexException("Must be run from within IPython")

    def to_html(ref_num):
        return '{}'.format(ref_num)

    if text is not None:
        # has this reference already been cited?
        for ref_label, (ref_num, ref_text) in ref_list.items():
            if text == ref_text:
                if label is not None and label != ref_label:
                    raise OneCodexException(
                        "Citation already in use with label={}".format(ref_label)
                    )
                else:
                    break
        else:
            # reference has not been cited. is the label already in use?
            if label is not None and label in ref_list.keys():
                raise OneCodexException("Citation label={} already in use".format(label))

            # create the citation and assign next number
            if not ref_list:
                ref_num = 1
            else:
                ref_num = max([x[0] for x in ref_list.values()]) + 1

            if label is None:
                ref_label = ref_num
            else:
                ref_label = label

            ref_list[ref_label] = (ref_num, text)
            ipy.meta["references"] = ref_list

        return to_html(ref_num)
    elif label is not None:
        if label not in ref_list.keys():
            raise OneCodexException("Cannot find citation with label={}".format(label))

        return to_html(ref_list[label][0])

Response: Add a reference to the bibliography and insert a superscript number.


Instruction:
Implement a function in Python 3 to
create a new resource with a given schema.

Response:
def resource_factory(self, name, schema, resource_cls=None):
        """
        Registers a new resource with a given schema. The schema must not have any unresolved references
        (such as `{"$ref": "#"}` for self-references, or otherwise). A subclass of :class:`Resource`
        may be provided to add specific functionality to the resulting :class:`Resource`.

        :param str name:
        :param dict schema:
        :param Resource resource_cls: a subclass of :class:`Resource` or None
        :return: The new :class:`Resource`.
        """
        cls = type(str(upper_camel_case(name)), (resource_cls or Resource, collections.MutableMapping), {
            '__doc__': schema.get('description', '')
        })

        cls._schema = schema
        cls._client = self
        cls._links = links = {}

        for link_schema in schema['links']:
            link = Link(self,
                        rel=link_schema['rel'],
                        href=link_schema['href'],
                        method=link_schema['method'],
                        schema=link_schema.get('schema', None),
                        target_schema=link_schema.get('targetSchema', None))

            # Set Resource._self, etc. for the special methods as they are managed by the Resource class
            if link.rel in ('self', 'instances', 'create', 'update', 'destroy'):
                setattr(cls, '_{}'.format(link.rel), link)
            links[link.rel] = link

            if link.rel != 'update':  # 'update' is a special case because of MutableMapping.update()
                setattr(cls, snake_case(link.rel), link)

        # TODO routes (instance & non-instance)

        for property_name, property_schema in schema.get('properties', {}).items():
            # skip $uri and $id as these are already implemented in Resource and overriding them causes unnecessary
            # fetches.
            if property_name.startswith('$'):
                continue

            if property_schema.get('readOnly', False):
                # TODO better error message. Raises AttributeError("can't set attribute")
                setattr(cls,
                        property_name,
                        property(fget=partial((lambda name, obj: getitem(obj, name)), property_name),
                                 doc=property_schema.get('description', None)))
            else:
                setattr(cls,
                        property_name,
                        property(fget=partial((lambda name, obj: getitem(obj, name)), property_name),
                                 fset=partial((lambda name, obj, value: setitem(obj, name, value)), property_name),
                                 fdel=partial((lambda name, obj: delitem(obj, name)), property_name),
                                 doc=property_schema.get('description', None)))

        root = None
        if 'instances' in links:
            root = cls._instances.href
        elif 'self' in links:
            root = cls._self.href[:cls._self.href.rfind('/')]
        else:
            root = self._root_path + '/' + name.replace('_', '-')

        self._resources[root] = cls
        return cls


Instruction:
Create a Python 3 function to
extract all string pairs in the given directory.

Response:
def extract_string_pairs_in_dir(directory, exclude_dirs, special_ui_components_prefix):
    """ Extract string pairs in the given directory's xib/storyboard files.

    Args:
        directory (str): The path to the directory.
        exclude_dirs (str): A list of directories to exclude from extraction.
        special_ui_components_prefix (str):
            If not None, extraction will not warn about internationalized UI components with this class prefix.

    Returns:
        list: The extracted string pairs for all IB files in the directory.

    """
    result = []
    for ib_file_path in find_files(directory, [".xib", ".storyboard"], exclude_dirs):
        result += extract_string_pairs_in_ib_file(ib_file_path, special_ui_components_prefix)

    return result


Instruction:
Can you generate the documentation for the following Python 3 function
def extract_element_internationalized_comment(element):
    element_entry_comment = get_element_attribute_or_empty(element, 'userLabel')
    if element_entry_comment == "":
        try:
            element_entry_comment = element.getElementsByTagName('string')[0].firstChild.nodeValue
        except Exception:
            element_entry_comment = ""
    if not element_entry_comment.lower().startswith(JT_INTERNATIONALIZED_COMMENT_PREFIX):
        return None
    else:
        return element_entry_comment[len(JT_INTERNATIONALIZED_COMMENT_PREFIX):]

Response: Extracts the element s internationalized comment.


Instruction:
Create a Python 3 function for
logging a warning if the element is not of the given type.

Response:
def warn_if_element_not_of_class(element, class_suffix, special_ui_components_prefix):
    """ Log a warning if the element is not of the given type (indicating that it is not internationalized).

    Args:
        element: The xib's XML element.
        class_name: The type the element should be, but is missing.
        special_ui_components_prefix: If provided, will not warn about class with this prefix (default is only 'JT')
    """
    valid_class_names = ["%s%s" % (DEFAULT_UI_COMPONENTS_PREFIX, class_suffix)]
    if special_ui_components_prefix is not None:
        valid_class_names.append("%s%s" % (special_ui_components_prefix, class_suffix))

    if (not element.hasAttribute('customClass')) or element.attributes['customClass'].value not in valid_class_names:
        logging.warn("WARNING: %s is internationalized but isn't one of %s",
                     extract_element_internationalized_comment(element), valid_class_names)


Instruction:
Make a summary of the following Python 3 code
def add_string_pairs_from_attributed_ui_element(results, ui_element, comment_prefix):
    attributed_strings = ui_element.getElementsByTagName('attributedString')
    if attributed_strings.length == 0:
        return False

    attributed_element = attributed_strings[0]
    fragment_index = 1
    for fragment in attributed_element.getElementsByTagName('fragment'):
        # The fragment text is either as an attribute <fragment content="TEXT">
        # or a child in the format <string key='content'>TEXT</string>
        try:
            label_entry_key = fragment.attributes['content'].value
        except KeyError:
            label_entry_key = fragment.getElementsByTagName('string')[0].firstChild.nodeValue

        comment = "%s Part %d" % (comment_prefix, fragment_index)
        results.append((label_entry_key, comment))
        fragment_index += 1

    return fragment_index > 1

Response: Adds string pairs from an attributed UI element with attributed text as a list of strings.


Instruction:
Explain what the following Python 3 code does
def add_string_pairs_from_label_element(xib_file, results, label, special_ui_components_prefix):
    label_entry_comment = extract_element_internationalized_comment(label)
    if label_entry_comment is None:
        return

    warn_if_element_not_of_class(label, 'Label', special_ui_components_prefix)

    if label.hasAttribute('usesAttributedText') and label.attributes['usesAttributedText'].value == 'YES':
        add_string_pairs_from_attributed_ui_element(results, label, label_entry_comment)
    else:
        try:
            label_entry_key = label.attributes['text'].value
        except KeyError:
            try:
                label_entry_key = label.getElementsByTagName('string')[0].firstChild.nodeValue
            except Exception:
                label_entry_key = 'N/A'
                logging.warn("%s: Missing text entry in %s", xib_file, label.toxml('UTF8'))
        results.append((label_entry_key, label_entry_comment))

Response: Adds string pairs from a label element.


Instruction:
How would you explain what the following Python 3 function does
def add_string_pairs_from_text_field_element(xib_file, results, text_field, special_ui_components_prefix):
    text_field_entry_comment = extract_element_internationalized_comment(text_field)
    if text_field_entry_comment is None:
        return

    if text_field.hasAttribute('usesAttributedText') and text_field.attributes['usesAttributedText'].value == 'YES':
        add_string_pairs_from_attributed_ui_element(results, text_field, text_field_entry_comment)
    else:
        try:
            text_field_entry_key = text_field.attributes['text'].value
            results.append((text_field_entry_key, text_field_entry_comment + ' default text value'))
        except KeyError:
            pass
    try:
        text_field_entry_key = text_field.attributes['placeholder'].value
        results.append((text_field_entry_key, text_field_entry_comment + ' placeholder text value'))
    except KeyError:
        pass
    warn_if_element_not_of_class(text_field, 'TextField', special_ui_components_prefix)

Response: Adds string pairs from a textfield element to the list of results.


Instruction:
Given the following Python 3 function, write the documentation
def add_string_pairs_from_text_view_element(xib_file, results, text_view, special_ui_components_prefix):
    text_view_entry_comment = extract_element_internationalized_comment(text_view)
    if text_view_entry_comment is None:
        return

    if text_view.hasAttribute('usesAttributedText') and text_view.attributes['usesAttributedText'].value == 'YES':
        add_string_pairs_from_attributed_ui_element(results, text_view, text_view_entry_comment)
    else:
        try:
            text_view_entry_key = text_view.attributes['text'].value
            results.append((text_view_entry_key, text_view_entry_comment + ' default text value'))
        except KeyError:
            pass
    warn_if_element_not_of_class(text_view, 'TextView', special_ui_components_prefix)

Response: Adds string pairs from a textview element to the list of results.


Instruction:
Can you generate the documentation for the following Python 3 function
def add_string_pairs_from_button_element(xib_file, results, button, special_ui_components_prefix):
    button_entry_comment = extract_element_internationalized_comment(button)
    if button_entry_comment is None:
        return

    for state in button.getElementsByTagName('state'):
        state_name = state.attributes['key'].value
        state_entry_comment = button_entry_comment + " - " + state_name + " state of button"
        if not add_string_pairs_from_attributed_ui_element(results, state, state_entry_comment):
            try:
                button_entry_key = state.attributes['title'].value
            except KeyError:
                try:
                    button_entry_key = state.getElementsByTagName('string')[0].firstChild.nodeValue
                except Exception:
                    continue

            results.append((button_entry_key, state_entry_comment))

    warn_if_element_not_of_class(button, 'Button', special_ui_components_prefix)

Response: Adds strings pairs from a button element to the list.


Instruction:
How would you explain what the following Python 3 function does
def extract_string_pairs_in_ib_file(file_path, special_ui_components_prefix):
    try:
        results = []
        xmldoc = minidom.parse(file_path)

        element_name_to_add_func = {'label': add_string_pairs_from_label_element,
                                    'button': add_string_pairs_from_button_element,
                                    'textField': add_string_pairs_from_text_field_element,
                                    'textView': add_string_pairs_from_text_view_element}

        for element_name in element_name_to_add_func:
            add_func = element_name_to_add_func[element_name]
            elements = xmldoc.getElementsByTagName(element_name)
            for element in elements:
                add_func(file_path, results, element, special_ui_components_prefix)

        # Find strings of format JTL('Key Name', 'Key Comment') and add them to the results
        jtl_brackets_find_results = re.findall(JTL_REGEX, open(file_path).read())
        unescaped_jtl_brackets_find_results = [(unescape(x), unescape(y)) for (x, y) in jtl_brackets_find_results]
        results += unescaped_jtl_brackets_find_results

        if len(results) > 0:
            results = [(None, os.path.basename(file_path))] + results
        return results

    except Exception, e:
        logging.warn("ERROR: Error processing %s (%s: %s)", file_path, type(e), str(e))
        return []

Response: Extracts the strings pairs from an xib file.


Instruction:
Make a summary of the following Python 3 code
def localization_diff(localizable_file, translated_file, excluded_strings_file, output_translation_file):
    old_translated_file_dictionary = generate_localization_key_to_entry_dictionary_from_file(translated_file)
    if excluded_strings_file is not None and os.path.isfile(excluded_strings_file):
        excluded_file_dictionary = generate_localization_key_to_entry_dictionary_from_file(excluded_strings_file)
    else:
        excluded_file_dictionary = {}

    # The reason we keep a list of the keys, and not just pop is because values can repeat themselves.
    translated_list = old_translated_file_dictionary.keys()
    output_dictionary = {}
    output_file_elements = []
    f = open_strings_file(localizable_file, "r")

    output_file_elements.append(Comment(u"""
/**
 * This file contains all the strings that were extracted from our app and that need to be translated.
 * Each entry may or may not have a comment explaining context, and a "key" = "%s" equation.
 * To localize, you need to fill the right side of the equation with the translation of the left side.
 * Please keep special expressions such as '%%@' or '%%1$@' as is. Usually the comment will explain their context.
 */
""" % (VALUE_PLACEHOLDER,)))

    for _header_comment, comments, key, value in extract_header_comment_key_value_tuples_from_file(f):
        if key in translated_list or key in excluded_file_dictionary:
            if key in old_translated_file_dictionary:
                old_translated_file_dictionary.pop(key)
        elif value in output_dictionary:
            output_dictionary[value].add_comments(comments)
            output_file_elements.append(Comment(
                u"/* There was a value '%s' here but it was a duplicate of an older value and removed. */\n" % value))
        else:
            loc_obj = LocalizationEntry(comments, value, VALUE_PLACEHOLDER)
            output_dictionary[value] = loc_obj
            output_file_elements.append(loc_obj)

    for key, removed_trans in old_translated_file_dictionary.items():
        output_file_elements.append(Comment(u"""
/*
 * Entry removed from previous translation file:
 * %s
 * "%s" = "%s";
 */
""" % (", ".join(removed_trans.comments), removed_trans.key, removed_trans.value)))

    write_file_elements_to_strings_file(output_translation_file, output_file_elements)

Response: Generates a new string file that contains all the strings that need to be translated and the original strings that need to be translated.


Instruction:
Write a Python 3 function that can
plot an arbitrary metadata field versus an arbitrary quantity as a boxplot or scatter plot.

Response:
def plot_metadata(
        self,
        rank="auto",
        haxis="Label",
        vaxis="simpson",
        title=None,
        xlabel=None,
        ylabel=None,
        return_chart=False,
        plot_type="auto",
        label=None,
    ):
        """Plot an arbitrary metadata field versus an arbitrary quantity as a boxplot or scatter plot.

        Parameters
        ----------
        rank : {'auto', 'kingdom', 'phylum', 'class', 'order', 'family', 'genus', 'species'}, optional
            Analysis will be restricted to abundances of taxa at the specified level.

        haxis : `string`, optional
            The metadata field (or tuple containing multiple categorical fields) to be plotted on
            the horizontal axis.

        vaxis : `string`, optional
            Data to be plotted on the vertical axis. Can be any one of the following:

            - A metadata field: the name of a metadata field containing numerical data
            - {'simpson', 'chao1', 'shannon'}: an alpha diversity statistic to calculate for each sample
            - A taxon name: the name of a taxon in the analysis
            - A taxon ID: the ID of a taxon in the analysis

        title : `string`, optional
            Text label at the top of the plot.

        xlabel : `string`, optional
            Text label along the horizontal axis.

        ylabel : `string`, optional
            Text label along the vertical axis.

        plot_type : {'auto', 'boxplot', 'scatter'}
            By default, will determine plot type automatically based on the data. Otherwise, specify
            one of 'boxplot' or 'scatter' to set the type of plot manually.

        label : `string` or `callable`, optional
            A metadata field (or function) used to label each analysis. If passing a function, a
            dict containing the metadata for each analysis is passed as the first and only
            positional argument. The callable function must return a string.

        Examples
        --------
        Generate a boxplot of the abundance of Bacteroides (genus) of samples grouped by whether the
        individuals are allergy to dogs, cats, both, or neither.

        >>> plot_metadata(haxis=('allergy_dogs', 'allergy_cats'), vaxis='Bacteroides')
        """
        if rank is None:
            raise OneCodexException("Please specify a rank or 'auto' to choose automatically")

        if plot_type not in ("auto", "boxplot", "scatter"):
            raise OneCodexException("Plot type must be one of: auto, boxplot, scatter")

        # alpha diversity is only allowed on vertical axis--horizontal can be magically mapped
        df, magic_fields = self._metadata_fetch([haxis, "Label"], label=label)

        if vaxis in ("simpson", "chao1", "shannon"):
            df.loc[:, vaxis] = self.alpha_diversity(vaxis, rank=rank)
            magic_fields[vaxis] = vaxis
        else:
            # if it's not alpha diversity, vertical axis can also be magically mapped
            vert_df, vert_magic_fields = self._metadata_fetch([vaxis])

            # we require the vertical axis to be numerical otherwise plots get weird
            if (
                pd.api.types.is_bool_dtype(vert_df[vert_magic_fields[vaxis]])
                or pd.api.types.is_categorical_dtype(vert_df[vert_magic_fields[vaxis]])
                or pd.api.types.is_object_dtype(vert_df[vert_magic_fields[vaxis]])
                or not pd.api.types.is_numeric_dtype(vert_df[vert_magic_fields[vaxis]])
            ):  # noqa
                raise OneCodexException("Metadata field on vertical axis must be numerical")

            df = pd.concat([df, vert_df], axis=1).dropna(subset=[vert_magic_fields[vaxis]])
            magic_fields.update(vert_magic_fields)

        # plots can look different depending on what the horizontal axis contains
        if pd.api.types.is_datetime64_any_dtype(df[magic_fields[haxis]]):
            category_type = "T"

            if plot_type == "auto":
                plot_type = "boxplot"
        elif "date" in magic_fields[haxis].split("_"):
            df.loc[:, magic_fields[haxis]] = df.loc[:, magic_fields[haxis]].apply(
                pd.to_datetime, utc=True
            )

            category_type = "T"

            if plot_type == "auto":
                plot_type = "boxplot"
        elif (
            pd.api.types.is_bool_dtype(df[magic_fields[haxis]])
            or pd.api.types.is_categorical_dtype(df[magic_fields[haxis]])
            or pd.api.types.is_object_dtype(df[magic_fields[haxis]])
        ):  # noqa
            df = df.fillna({field: "N/A" for field in df.columns})

            category_type = "N"

            if plot_type == "auto":
                # if data is categorical but there is only one value per sample, scatter plot instead
                if len(df[magic_fields[haxis]].unique()) == len(df[magic_fields[haxis]]):
                    plot_type = "scatter"
                else:
                    plot_type = "boxplot"
        elif pd.api.types.is_numeric_dtype(df[magic_fields[haxis]]):
            df = df.dropna(subset=[magic_fields[vaxis]])

            category_type = "O"

            if plot_type == "auto":
                plot_type = "scatter"
        else:
            raise OneCodexException(
                "Unplottable column type for horizontal axis ({})".format(haxis)
            )

        if xlabel is None:
            xlabel = magic_fields[haxis]

        if ylabel is None:
            ylabel = magic_fields[vaxis]

        if plot_type == "scatter":
            df = df.reset_index()

            alt_kwargs = dict(
                x=alt.X(magic_fields[haxis], axis=alt.Axis(title=xlabel)),
                y=alt.Y(magic_fields[vaxis], axis=alt.Axis(title=ylabel)),
                tooltip=["Label", "{}:Q".format(vaxis)],
                href="url:N",
                url="https://app.onecodex.com/classification/" + alt.datum.classification_id,
            )

            chart = (
                alt.Chart(df)
                .transform_calculate(url=alt_kwargs.pop("url"))
                .mark_circle()
                .encode(**alt_kwargs)
            )

            if title:
                chart = chart.properties(title=title)
        elif plot_type == "boxplot":
            chart = boxplot(
                df,
                magic_fields[haxis],
                magic_fields[vaxis],
                category_type=category_type,
                title=title,
                xlabel=xlabel,
                ylabel=ylabel,
            )

        if return_chart:
            return chart
        else:
            chart.interactive().display()


Instruction:
Here you have a function in Python 3, explain what it does
def add_404_page(app):
    is_epub = isinstance(app.builder, EpubBuilder)
    config_pages = app.config.html_additional_pages

    if not is_epub and "404" not in config_pages:
        yield ("404", {}, "404.html")

Response:
Add a 404. html page if no key is in the the
   .


Instruction:
Given the following Python 3 function, write the documentation
def canonical_url(app, pagename, templatename, context, doctree):
    base = context.get("canonical_url")

    if not base:
        return

    target = app.builder.get_target_uri(pagename)
    context["page_canonical_url"] = base + target

Response: Build the canonical URL for a page.


Instruction:
Create a Python 3 function for
skipping rendering internal modules.

Response:
def skip_internal(app, what, name, obj, skip, options):
    """Skip rendering autodoc when the docstring contains a line with
    only the string `:internal:`.
    """
    docstring = inspect.getdoc(obj) or ""

    if skip or re.search(r"^\s*:internal:\s*$", docstring, re.M) is not None:
        return True


Instruction:
Can you implement a function in Python 3 that
cuts module meta information from source code.

Response:
def cut_module_meta(app, what, name, obj, options, lines):
    """Don't render lines that start with ``:copyright:`` or
    ``:license:`` when rendering module autodoc. These lines are useful
    meta information in the source code, but are noisy in the docs.
    """
    if what != "module":
        return

    lines[:] = [
        line for line in lines if not line.startswith((":copyright:", ":license:"))
    ]


Instruction:
Write a Python 3 script for
ensuring that the named package is installed and returns version strings to be used by Sphinx.

Response:
def get_version(name, version_length=2, placeholder="x"):
    """Ensures that the named package is installed and returns version
    strings to be used by Sphinx.

    Sphinx uses ``version`` to mean an abbreviated form of the full
    version string, which is called ``release``. In ``conf.py``::

        release, version = get_version("Flask")
        # release = 1.0.x, version = 1.0.3.dev0

    :param name: Name of package to get.
    :param version_length: How many values from ``release`` to use for
        ``version``.
    :param placeholder: Extra suffix to add to the version. The default
        produces versions like ``1.2.x``.
    :return: ``(release, version)`` tuple.
    """
    try:
        release = pkg_resources.get_distribution(name).version
    except ImportError:
        print(
            textwrap.fill(
                "'{name}' must be installed to build the documentation."
                " Install from source using `pip install -e .` in a"
                " virtualenv.".format(name=name)
            )
        )
        sys.exit(1)

    version = ".".join(release.split(".", version_length)[:version_length])

    if placeholder:
        version = "{}.{}".format(version, placeholder)

    return release, version


Instruction:
Write a Python 3 function that can
set the is_pallets_theme config to True if the current theme is a decedent of the pocoo theme.

Response:
def set_is_pallets_theme(app):
    """Set the ``is_pallets_theme`` config to ``True`` if the current
    theme is a decedent of the ``pocoo`` theme.
    """
    if app.config.is_pallets_theme is not None:
        return

    theme = getattr(app.builder, "theme", None)

    while theme is not None:
        if theme.name == "pocoo":
            app.config.is_pallets_theme = True
            break

        theme = theme.base
    else:
        app.config.is_pallets_theme = False


Instruction:
Can you create a Python 3 function that
creates a decorator that calls a function only if the the is_pallets_theme config is True.

Response:
def only_pallets_theme(default=None):
    """Create a decorator that calls a function only if the
    ``is_pallets_theme`` config is ``True``.

    Used to prevent Sphinx event callbacks from doing anything if the
    Pallets themes are installed but not used. ::

        @only_pallets_theme()
        def inject_value(app):
            ...

        app.connect("builder-inited", inject_value)

    :param default: Value to return if a Pallets theme is not in use.
    :return: A decorator.
    """

    def decorator(f):
        @wraps(f)
        def wrapped(app, *args, **kwargs):
            if not app.config.is_pallets_theme:
                return default

            return f(app, *args, **kwargs)

        return wrapped

    return decorator


Instruction:
Can you generate the documentation for the following Python 3 function
def patch_modules():
    old_call = subprocess.call

    def dummy_call(*args, **kwargs):
        with tempfile.TemporaryFile("wb+") as f:
            kwargs["stdout"] = f
            kwargs["stderr"] = f
            rv = subprocess.Popen(*args, **kwargs).wait()
            f.seek(0)
            click.echo(f.read().decode("utf-8", "replace").rstrip())
        return rv

    subprocess.call = dummy_call

    try:
        yield
    finally:
        subprocess.call = old_call

Response: Patch subprocess. call to work better with example runner. invoke.


Instruction:
How would you explain what the following Python 3 function does
def get_example_runner(document):
    runner = getattr(document, "click_example_runner", None)
    if runner is None:
        runner = document.click_example_runner = ExampleRunner()
    return runner

Response:
Get or create the ExampleRunner instance associated with
    a document.


Instruction:
How would you implement a function in Python 3 that
likes CLIRunner. invoke but displays what the user would enter in the terminal for the specified command args and environment variables.

Response:
def invoke(
        self,
        cli,
        args=None,
        prog_name=None,
        input=None,
        terminate_input=False,
        env=None,
        _output_lines=None,
        **extra
    ):
        """Like :meth:`CliRunner.invoke` but displays what the user
        would enter in the terminal for env vars, command args, and
        prompts.

        :param terminate_input: Whether to display "^D" after a list of
            input.
        :param _output_lines: A list used internally to collect lines to
            be displayed.
        """
        output_lines = _output_lines if _output_lines is not None else []

        if env:
            for key, value in sorted(env.items()):
                value = shlex.quote(value)
                output_lines.append("$ export {}={}".format(key, value))

        args = args or []

        if prog_name is None:
            prog_name = cli.name.replace("_", "-")

        output_lines.append(
            "$ {} {}".format(prog_name, " ".join(shlex.quote(x) for x in args)).rstrip()
        )
        # remove "python" from command
        prog_name = prog_name.rsplit(" ", 1)[-1]

        if isinstance(input, (tuple, list)):
            input = "\n".join(input) + "\n"

            if terminate_input:
                input += "\x04"

        result = super(ExampleRunner, self).invoke(
            cli=cli, args=args, input=input, env=env, prog_name=prog_name, **extra
        )
        output_lines.extend(result.output.splitlines())
        return result


Instruction:
Create a Python 3 function for
executing the given code adding it to the runner s namespace.

Response:
def declare_example(self, source):
        """Execute the given code, adding it to the runner's namespace."""
        with patch_modules():
            code = compile(source, "<docs>", "exec")
            exec(code, self.namespace)


Instruction:
Given the following Python 3 function, write the documentation
def run_example(self, source):
        code = compile(source, "<docs>", "exec")
        buffer = []
        invoke = partial(self.invoke, _output_lines=buffer)

        def println(text=""):
            buffer.append(text)

        exec(
            code,
            self.namespace,
            {
                "invoke": invoke,
                "println": println,
                "isolated_filesystem": self.isolated_filesystem,
            },
        )
        return buffer

Response: Runs the given code and returns the lines of input and output.


Instruction:
Here you have a function in Python 3, explain what it does
def setup(
    *,
    verbose: bool = False,
    quiet: bool = False,
    color: str = "auto",
    title: str = "auto",
    timestamp: bool = False
) -> None:
    """ Configure behavior of message functions.

    :param verbose: Whether :func:`debug` messages should get printed
    :param quiet: Hide every message except :func:`warning`, :func:`error`, and
                  :func:`fatal`
    :param color: Choices: 'auto', 'always', or 'never'. Whether to color output.
                  By default ('auto'), only use color when output is a terminal.
    :param title: Ditto for setting terminal title
    :param timestamp: Whether to prefix every message with a time stamp
    """
    _setup(verbose=verbose, quiet=quiet, color=color, title=title, timestamp=timestamp)

Response: Configure behavior of message functions.


Instruction:
Implement a function in Python 3 to
return two strings from a list of tokens. One containing ASCII escape codes the other only the normal characters

Response:
def process_tokens(
    tokens: Sequence[Token], *, end: str = "\n", sep: str = " "
) -> Tuple[str, str]:
    """ Returns two strings from a list of tokens.
    One containing ASCII escape codes, the other
    only the 'normal' characters

    """
    # Flatten the list of tokens in case some of them are of
    # class UnicodeSequence:
    flat_tokens = list()  # type: List[Token]
    for token in tokens:
        if isinstance(token, UnicodeSequence):
            flat_tokens.extend(token.tuple())
        else:
            flat_tokens.append(token)

    with_color = _process_tokens(flat_tokens, end=end, sep=sep, color=True)
    without_color = _process_tokens(flat_tokens, end=end, sep=sep, color=False)
    return (with_color, without_color)


Instruction:
Make a summary of the following Python 3 code
def message(
    *tokens: Token,
    end: str = "\n",
    sep: str = " ",
    fileobj: FileObj = sys.stdout,
    update_title: bool = False
) -> None:
    """ Helper method for error, warning, info, debug

    """
    if using_colorama():
        global _INITIALIZED
        if not _INITIALIZED:
            colorama.init()
            _INITIALIZED = True
    with_color, without_color = process_tokens(tokens, end=end, sep=sep)
    if CONFIG["record"]:
        _MESSAGES.append(without_color)
    if update_title and with_color:
        write_title_string(without_color, fileobj)
    to_write = with_color if config_color(fileobj) else without_color
    write_and_flush(fileobj, to_write)

Response: Write a message to the fileobj.


Instruction:
Can you write a function in Python 3 where it
prints an error message and call sys. exit.

Response:
def fatal(*tokens: Token, **kwargs: Any) -> None:
    """ Print an error message and call ``sys.exit`` """
    error(*tokens, **kwargs)
    sys.exit(1)


Instruction:
Explain what the following Python 3 code does
def error(*tokens: Token, **kwargs: Any) -> None:
    tokens = [bold, red, "Error:"] + list(tokens)  # type: ignore
    kwargs["fileobj"] = sys.stderr
    message(*tokens, **kwargs)

Response: Print an error message.


Instruction:
How would you explain what the following Python 3 function does
def warning(*tokens: Token, **kwargs: Any) -> None:
    tokens = [brown, "Warning:"] + list(tokens)  # type: ignore
    kwargs["fileobj"] = sys.stderr
    message(*tokens, **kwargs)

Response: Print a warning message.


Instruction:
How would you explain what the following Python 3 function does
def info_section(*tokens: Token, **kwargs: Any) -> None:
    # We need to know the length of the section:
    process_tokens_kwargs = kwargs.copy()
    process_tokens_kwargs["color"] = False
    no_color = _process_tokens(tokens, **process_tokens_kwargs)
    info(*tokens, **kwargs)
    info("-" * len(no_color), end="\n\n")

Response: Print an underlined section name.


Instruction:
Create a Python 3 function for
printing an important informative message

Response:
def info_1(*tokens: Token, **kwargs: Any) -> None:
    """ Print an important informative message """
    info(bold, blue, "::", reset, *tokens, **kwargs)


Instruction:
Make a summary of the following Python 3 code
def dot(*, last: bool = False, fileobj: Any = None) -> None:
    end = "\n" if last else ""
    info(".", end=end, fileobj=fileobj)

Response: Print a dot without a newline unless it is the last one.


Instruction:
Explain what the following Python 3 code does
def info_count(i: int, n: int, *rest: Token, **kwargs: Any) -> None:
    num_digits = len(str(n))
    counter_format = "(%{}d/%d)".format(num_digits)
    counter_str = counter_format % (i + 1, n)
    info(green, "*", reset, counter_str, reset, *rest, **kwargs)

Response: Display a counter before the rest of the message.


Instruction:
Can you write a function in Python 3 where it
displays info progress in percent.

Response:
def info_progress(prefix: str, value: float, max_value: float) -> None:
    """ Display info progress in percent.

    :param value: the current value
    :param max_value: the max value
    :param prefix: the prefix message to print

    """
    if sys.stdout.isatty():
        percent = float(value) / max_value * 100
        sys.stdout.write(prefix + ": %.0f%%\r" % percent)
        sys.stdout.flush()


Instruction:
Can you create a Python 3 function that
prints a debug message.

Response:
def debug(*tokens: Token, **kwargs: Any) -> None:
    """ Print a debug message.

    Messages are shown only when ``CONFIG["verbose"]`` is true
    """
    if not CONFIG["verbose"] or CONFIG["record"]:
        return
    message(*tokens, **kwargs)


Instruction:
How would you code a function in Python 3 to
indent a piece of text.

Response:
def indent(text: str, num: int = 2) -> str:
    """Indent a piece of text."""
    lines = text.splitlines()
    return "\n".join(indent_iterable(lines, num=num))


Instruction:
Can you tell what is the following Python 3 function doing
def message_for_exception(exception: Exception, message: str) -> Sequence[Token]:
    tb = sys.exc_info()[2]
    buffer = io.StringIO()
    traceback.print_tb(tb, file=io)  # type: ignore
    # fmt: off
    return (
        red, message + "\n",
        exception.__class__.__name__,
        str(exception), "\n",
        reset, buffer.getvalue()
    )

Response: Returns a tuple suitable for cli_ui. error


Instruction:
Can you generate the documentation for the following Python 3 function
def ask_string(*question: Token, default: Optional[str] = None) -> Optional[str]:
    tokens = get_ask_tokens(question)
    if default:
        tokens.append("(%s)" % default)
    info(*tokens)
    answer = read_input()
    if not answer:
        return default
    return answer

Response: Ask the user to enter a string.


Instruction:
Can you create a Python 3 function that
asks the user to enter a password.

Response:
def ask_password(*question: Token) -> str:
    """Ask the user to enter a password.
    """
    tokens = get_ask_tokens(question)
    info(*tokens)
    answer = read_password()
    return answer


Instruction:
Create a Python 3 function for
asking the user to choose from a list of choices.

Response:
def ask_choice(
    *prompt: Token, choices: List[Any], func_desc: Optional[FuncDesc] = None
) -> Any:
    """Ask the user to choose from a list of choices.

    :return: the selected choice

    ``func_desc`` will be called on every list item for displaying
    and sorting the list. If not given, will default to
    the identity function.

    Will loop until:
        * the user enters a valid index
        * or leaves the prompt empty

    In the last case, None will be returned
    """
    if func_desc is None:
        func_desc = lambda x: str(x)
    tokens = get_ask_tokens(prompt)
    info(*tokens)
    choices.sort(key=func_desc)
    for i, choice in enumerate(choices, start=1):
        choice_desc = func_desc(choice)
        info("  ", blue, "%i" % i, reset, choice_desc)
    keep_asking = True
    res = None
    while keep_asking:
        answer = read_input()
        if not answer:
            return None
        try:
            index = int(answer)
        except ValueError:
            info("Please enter a valid number")
            continue
        if index not in range(1, len(choices) + 1):
            info(str(index), "is out of range")
            continue
        res = choices[index - 1]
        keep_asking = False

    return res


Instruction:
Create a Python 3 function for
asking the user to answer by yes or no.

Response:
def ask_yes_no(*question: Token, default: bool = False) -> bool:
    """Ask the user to answer by yes or no"""
    while True:
        tokens = [green, "::", reset] + list(question) + [reset]
        if default:
            tokens.append("(Y/n)")
        else:
            tokens.append("(y/N)")
        info(*tokens)
        answer = read_input()
        if answer.lower() in ["y", "yes"]:
            return True
        if answer.lower() in ["n", "no"]:
            return False
        if not answer:
            return default
        warning("Please answer by 'y' (yes) or 'n' (no) ")


Instruction:
How would you implement a function in Python 3 that
gives a list of choices and an invalid user input display the closest available item in the list that match the input.

Response:
def did_you_mean(message: str, user_input: str, choices: Sequence[str]) -> str:
    """ Given a list of choices and an invalid user input, display the closest
    items in the list that match the input.

    """
    if not choices:
        return message
    else:
        result = {
            difflib.SequenceMatcher(a=user_input, b=choice).ratio(): choice
            for choice in choices
        }
        message += "\nDid you mean: %s?" % result[max(result)]
        return message


Instruction:
Here you have a function in Python 3, explain what it does
def stop(self) -> None:
        end_time = datetime.datetime.now()
        elapsed_time = end_time - self.start_time
        elapsed_seconds = elapsed_time.seconds
        hours, remainder = divmod(int(elapsed_seconds), 3600)
        minutes, seconds = divmod(remainder, 60)
        as_str = "%sh %sm %ss %dms" % (
            hours,
            minutes,
            seconds,
            elapsed_time.microseconds / 1000,
        )
        info("%s took %s" % (self.description, as_str))

Response: Stop the timer and emit a nice log


Instruction:
Write a Python 3 script for
downloading a file from the workspace to a local file.

Response:
def download_to_directory(self, directory, url, basename=None, overwrite=False, subdir=None):
        """
        Download a file to the workspace.

        Early Shortcut: If url is a file://-URL and that file is already in the directory, keep it there.

        If basename is not given but subdir is, assume user knows what she's doing and use last URL segment as the basename.
        If basename is not given and no subdir is given, use the alnum characters in the URL as the basename.

        Args:
            directory (string): Directory to download files to
            basename (string, None): basename part of the filename on disk.
            url (string): URL to download from
            overwrite (boolean): Whether to overwrite existing files with that name
            subdir (string, None): Subdirectory to create within the directory. Think fileGrp.

        Returns:
            Local filename
        """
        log = getLogger('ocrd.resolver.download_to_directory') # pylint: disable=redefined-outer-name
        log.debug("directory=|%s| url=|%s| basename=|%s| overwrite=|%s| subdir=|%s|", directory, url, basename, overwrite, subdir)

        if url is None:
            raise Exception("'url' must be a string")
        if directory is None:
            raise Exception("'directory' must be a string")

        if basename is None:
            if (subdir is not None) or \
                (directory and url.startswith('file://%s' % directory)): # in case downloading a url 'file:///tmp/foo/bar' to directory '/tmp/foo'
                basename = url.rsplit('/', 1)[-1]
            else:
                basename = safe_filename(url)

        if subdir is not None:
            basename = join(subdir, basename)

        outfilename = join(directory, basename)

        if exists(outfilename) and not overwrite:
            log.debug("File already exists and overwrite=False: %s", outfilename)
            return outfilename

        outfiledir = outfilename.rsplit('/', 1)[0]
        #  print(outfiledir)
        if not isdir(outfiledir):
            makedirs(outfiledir)

        log.debug("Downloading <%s> to '%s'", url, outfilename)

        # de-scheme file:// URL
        if url.startswith('file://'):
            url = url[len('file://'):]

        # Copy files or download remote assets
        if '://' not in url:
            copyfile(url, outfilename)
        else:
            response = requests.get(url)
            if response.status_code != 200:
                raise Exception("Not found: %s (HTTP %d)" % (url, response.status_code))
            with open(outfilename, 'wb') as outfile:
                outfile.write(response.content)

        return outfilename


Instruction:
How would you code a function in Python 3 to
create a workspace from a METS file.

Response:
def workspace_from_url(self, mets_url, dst_dir=None, clobber_mets=False, mets_basename=None, download=False, baseurl=None):
        """
        Create a workspace from a METS by URL.

        Sets the mets.xml file

        Arguments:
            mets_url (string): Source mets URL
            dst_dir (string, None): Target directory for the workspace
            clobber_mets (boolean, False): Whether to overwrite existing mets.xml. By default existing mets.xml will raise an exception.
            download (boolean, False): Whether to download all the files
            baseurl (string, None): Base URL for resolving relative file locations

        Returns:
            Workspace
        """
        if dst_dir and not dst_dir.startswith('/'):
            dst_dir = abspath(dst_dir)

        if mets_url is None:
            if baseurl is None:
                raise Exception("Must pass mets_url and/or baseurl to workspace_from_url")
            else:
                mets_url = 'file://%s/%s' % (baseurl, mets_basename if mets_basename else 'mets.xml')
        if baseurl is None:
            baseurl = mets_url.rsplit('/', 1)[0]
        log.debug("workspace_from_url\nmets_url='%s'\nbaseurl='%s'\ndst_dir='%s'", mets_url, baseurl, dst_dir)

        # resolve to absolute
        if '://' not in mets_url:
            mets_url = 'file://%s' % abspath(mets_url)

        if dst_dir is None:
            # if mets_url is a file-url assume working directory is source directory
            if mets_url.startswith('file://'):
                # if dst_dir was not given and mets_url is a file assume that
                # dst_dir should be the directory where the mets.xml resides
                dst_dir = dirname(mets_url[len('file://'):])
            else:
                dst_dir = tempfile.mkdtemp(prefix=TMP_PREFIX)
                log.debug("Creating workspace '%s' for METS @ <%s>", dst_dir, mets_url)

        # if mets_basename is not given, use the last URL segment of the mets_url
        if mets_basename is None:
            mets_basename = mets_url \
                .rsplit('/', 1)[-1] \
                .split('?')[0] \
                .split('#')[0]

        dst_mets = join(dst_dir, mets_basename)
        log.debug("Copying mets url '%s' to '%s'", mets_url, dst_mets)
        if 'file://' + dst_mets == mets_url:
            log.debug("Target and source mets are identical")
        else:
            if exists(dst_mets) and not clobber_mets:
                raise Exception("File '%s' already exists but clobber_mets is false" % dst_mets)
            else:
                self.download_to_directory(dst_dir, mets_url, basename=mets_basename)

        workspace = Workspace(self, dst_dir, mets_basename=mets_basename, baseurl=baseurl)

        if download:
            for f in workspace.mets.find_files():
                workspace.download_file(f)

        return workspace


Instruction:
How would you implement a function in Python 3 that
creates an empty workspace.

Response:
def workspace_from_nothing(self, directory, mets_basename='mets.xml', clobber_mets=False):
        """
        Create an empty workspace.
        """
        if directory is None:
            directory = tempfile.mkdtemp(prefix=TMP_PREFIX)
        if not exists(directory):
            makedirs(directory)

        mets_fpath = join(directory, mets_basename)
        if not clobber_mets and exists(mets_fpath):
            raise Exception("Not clobbering existing mets.xml in '%s'." % directory)
        mets = OcrdMets.empty_mets()
        with open(mets_fpath, 'wb') as fmets:
            log.info("Writing %s", mets_fpath)
            fmets.write(mets.to_xml(xmllint=True))

        return Workspace(self, directory, mets)


Instruction:
Implement a function in Python 3 to
serialize all properties as XML

Response:
def to_xml(self, xmllint=False):
        """
        Serialize all properties as pretty-printed XML

        Args:
            xmllint (boolean): Format with ``xmllint`` in addition to pretty-printing
        """
        root = self._tree.getroot()
        ret = ET.tostring(ET.ElementTree(root), pretty_print=True)
        if xmllint:
            ret = xmllint_format(ret)
        return ret


Instruction:
Can you create a Python 3 function that
creates a workspace from a METS_URL and return the directory containing the METS_URL.

Response:
def workspace_clone(ctx, clobber_mets, download, mets_url, workspace_dir):
    """
    Create a workspace from a METS_URL and return the directory

    METS_URL can be a URL, an absolute path or a path relative to $PWD.

    If WORKSPACE_DIR is not provided, creates a temporary directory.
    """
    workspace = ctx.resolver.workspace_from_url(
        mets_url,
        dst_dir=os.path.abspath(workspace_dir if workspace_dir else mkdtemp(prefix=TMP_PREFIX)),
        mets_basename=ctx.mets_basename,
        clobber_mets=clobber_mets,
        download=download,
    )
    workspace.save_mets()
    print(workspace.directory)


Instruction:
Can you create a Python 3 function that
creates a workspace with an empty METS file in DIRECTORY.

Response:
def workspace_create(ctx, clobber_mets, directory):
    """
    Create a workspace with an empty METS file in DIRECTORY.

    Use '.' for $PWD"
    """
    workspace = ctx.resolver.workspace_from_nothing(
        directory=os.path.abspath(directory),
        mets_basename=ctx.mets_basename,
        clobber_mets=clobber_mets
    )
    workspace.save_mets()
    print(workspace.directory)


Instruction:
Create a Python 3 function for
adding a file to METS in a workspace.

Response:
def workspace_add_file(ctx, file_grp, file_id, mimetype, page_id, force, local_filename):
    """
    Add a file LOCAL_FILENAME to METS in a workspace.
    """
    workspace = Workspace(ctx.resolver, directory=ctx.directory, mets_basename=ctx.mets_basename, automatic_backup=ctx.automatic_backup)

    if not local_filename.startswith(ctx.directory):
        log.debug("File '%s' is not in workspace, copying", local_filename)
        local_filename = ctx.resolver.download_to_directory(ctx.directory, "file://" + local_filename, subdir=file_grp)

    url = "file://" + local_filename

    workspace.mets.add_file(
        fileGrp=file_grp,
        ID=file_id,
        mimetype=mimetype,
        url=url,
        pageId=page_id,
        force=force,
        local_filename=local_filename
    )
    workspace.save_mets()


Instruction:
How would you explain what the following Python 3 function does
def workspace_find(ctx, file_grp, mimetype, page_id, file_id, output_field, download):
    workspace = Workspace(ctx.resolver, directory=ctx.directory, mets_basename=ctx.mets_basename)
    for f in workspace.mets.find_files(
            ID=file_id,
            fileGrp=file_grp,
            mimetype=mimetype,
            pageId=page_id,
        ):
        if download:
            workspace.download_file(f)
            workspace.save_mets()
        ret = '\t'.join([getattr(f, field) or '' for field in output_field])
        print(ret)

Response: Find files in a workspace.


Instruction:
Can you write a function in Python 3 where it
creates a new backup

Response:
def workspace_backup_add(ctx):
    """
    Create a new backup
    """
    backup_manager = WorkspaceBackupManager(Workspace(ctx.resolver, directory=ctx.directory, mets_basename=ctx.mets_basename, automatic_backup=ctx.automatic_backup))
    backup_manager.add()


Instruction:
Given the following Python 3 function, write the documentation
def workspace_backup_list(ctx):
    backup_manager = WorkspaceBackupManager(Workspace(ctx.resolver, directory=ctx.directory, mets_basename=ctx.mets_basename, automatic_backup=ctx.automatic_backup))
    for b in backup_manager.list():
        print(b)

Response: List backups in workspace


Instruction:
Given the following Python 3 function, write the documentation
def workspace_backup_restore(ctx, choose_first, bak):
    backup_manager = WorkspaceBackupManager(Workspace(ctx.resolver, directory=ctx.directory, mets_basename=ctx.mets_basename, automatic_backup=ctx.automatic_backup))
    backup_manager.restore(bak, choose_first)

Response: Restore a backup of a workspace.


Instruction:
How would you code a function in Python 3 to
restore the last backup

Response:
def workspace_backup_undo(ctx):
    """
    Restore the last backup
    """
    backup_manager = WorkspaceBackupManager(Workspace(ctx.resolver, directory=ctx.directory, mets_basename=ctx.mets_basename, automatic_backup=ctx.automatic_backup))
    backup_manager.undo()