Instruction:
How would you implement a function in Python 3 that
returns the n - grams in data that do not contain any other n - gram in data.

Response:
def _generate_filter_ngrams(self, data, min_size):
        """Returns the n-grams in `data` that do not contain any other n-gram
        in `data`.

        :param data: n-gram results data
        :type data: `pandas.DataFrame`
        :param min_size: minimum n-gram size in `data`
        :type min_size: `int`
        :rtype: `list` of `str`

        """
        max_size = data[constants.SIZE_FIELDNAME].max()
        kept_ngrams = list(data[data[constants.SIZE_FIELDNAME] == min_size][
            constants.NGRAM_FIELDNAME])
        for size in range(min_size+1, max_size+1):
            pattern = FilteredWitnessText.get_filter_ngrams_pattern(
                kept_ngrams)
            potential_ngrams = list(data[data[constants.SIZE_FIELDNAME] ==
                                         size][constants.NGRAM_FIELDNAME])
            kept_ngrams.extend([ngram for ngram in potential_ngrams if
                                pattern.search(ngram) is None])
        return kept_ngrams


Instruction:
Can you tell what is the following Python 3 function doing
def _generate_substrings(self, ngram, size):
        text = Text(ngram, self._tokenizer)
        substrings = []
        for sub_size, ngrams in text.get_ngrams(1, size-1):
            for sub_ngram, count in ngrams.items():
                substrings.extend([sub_ngram] * count)
        return substrings

Response: Returns a list of all substrings of ngram.


Instruction:
Can you implement a function in Python 3 that
groups the result rows by n - gram and label.

Response:
def group_by_ngram(self, labels):
        """Groups result rows by n-gram and label, providing a single summary
        field giving the range of occurrences across each work's
        witnesses. Results are sorted by n-gram then by label (in the
        order given in `labels`).

        :param labels: labels to order on
        :type labels: `list` of `str`

        """
        if self._matches.empty:
            # Ensure that the right columns are used, even though the
            # results are empty.
            self._matches = pd.DataFrame(
                {}, columns=[
                    constants.NGRAM_FIELDNAME,
                    constants.SIZE_FIELDNAME,
                    constants.LABEL_FIELDNAME,
                    constants.WORK_COUNTS_FIELDNAME])
            return
        label_order_col = 'label order'

        def work_summary(group):
            match = group.iloc[0]
            work = match[constants.WORK_FIELDNAME]
            minimum = group[constants.COUNT_FIELDNAME].min()
            maximum = group[constants.COUNT_FIELDNAME].max()
            if minimum == maximum:
                work_count = '{}({})'.format(work, minimum)
            else:
                work_count = '{}({}-{})'.format(work, minimum, maximum)
            match['work count'] = work_count
            return match

        def ngram_label_summary(group):
            match = group.iloc[0]
            summary = group.groupby(constants.WORK_FIELDNAME).apply(
                work_summary)
            work_counts = ', '.join(list(summary['work count']))
            match[constants.WORK_COUNTS_FIELDNAME] = work_counts
            return match

        def add_label_order(row, labels):
            row[label_order_col] = labels.index(row[constants.LABEL_FIELDNAME])
            return row

        group_cols = [constants.NGRAM_FIELDNAME, constants.LABEL_FIELDNAME]
        matches = self._matches.groupby(group_cols, sort=False).apply(
            ngram_label_summary)
        del matches[constants.WORK_FIELDNAME]
        del matches[constants.SIGLUM_FIELDNAME]
        del matches[constants.COUNT_FIELDNAME]
        matches = matches.apply(add_label_order, axis=1, args=(labels,))
        matches.reset_index(drop=True, inplace=True)
        matches.sort_values(by=[constants.NGRAM_FIELDNAME, label_order_col],
                            ascending=True, inplace=True)
        del matches[label_order_col]
        self._matches = matches


Instruction:
Can you generate a brief explanation for the following Python 3 code
def group_by_witness(self):
        if self._matches.empty:
            # Ensure that the right columns are used, even though the
            # results are empty.
            self._matches = pd.DataFrame(
                {}, columns=[constants.WORK_FIELDNAME,
                             constants.SIGLUM_FIELDNAME,
                             constants.LABEL_FIELDNAME,
                             constants.NGRAMS_FIELDNAME,
                             constants.NUMBER_FIELDNAME,
                             constants.TOTAL_COUNT_FIELDNAME])
            return

        def witness_summary(group):
            matches = group.sort_values(by=[constants.NGRAM_FIELDNAME],
                                        ascending=[True])
            match = matches.iloc[0]
            ngrams = ', '.join(list(matches[constants.NGRAM_FIELDNAME]))
            match[constants.NGRAMS_FIELDNAME] = ngrams
            match[constants.NUMBER_FIELDNAME] = len(matches)
            match[constants.TOTAL_COUNT_FIELDNAME] = matches[
                constants.COUNT_FIELDNAME].sum()
            return match

        group_cols = [constants.WORK_FIELDNAME, constants.SIGLUM_FIELDNAME]
        # Remove zero-count results.
        self._matches = self._matches[
            self._matches[constants.COUNT_FIELDNAME] != 0]
        self._matches = self._matches.groupby(group_cols, sort=False).apply(
            witness_summary)
        del self._matches[constants.NGRAM_FIELDNAME]
        del self._matches[constants.SIZE_FIELDNAME]
        del self._matches[constants.COUNT_FIELDNAME]

Response:
Groups results by witness providing a single summary field giving
        the n - grams found in it a count of their number and the count of their combined occurrences.


Instruction:
Implement a function in Python 3 to
return False if results has an n - gram that exists in only one label True otherwise.

Response:
def _is_intersect_results(results):
        """Returns False if `results` has an n-gram that exists in only one
        label, True otherwise.

        :param results: results to analyze
        :type results: `pandas.DataFrame`
        :rtype: `bool`

        """
        sample = results.iloc[0]
        ngram = sample[constants.NGRAM_FIELDNAME]
        label = sample[constants.LABEL_FIELDNAME]
        return not(results[
            (results[constants.NGRAM_FIELDNAME] == ngram) &
            (results[constants.LABEL_FIELDNAME] != label)].empty)


Instruction:
Here you have a function in Python 3, explain what it does
def prune_by_ngram(self, ngrams):
        self._logger.info('Pruning results by n-gram')
        self._matches = self._matches[
            ~self._matches[constants.NGRAM_FIELDNAME].isin(ngrams)]

Response: Removes results rows whose n - gram is in ngrams.


Instruction:
Write a Python 3 function that can
remove results rows whose total n - gram count is outside the range specified by minimum and maximum.

Response:
def prune_by_ngram_count(self, minimum=None, maximum=None, label=None):
        """Removes results rows whose total n-gram count (across all
        works bearing this n-gram) is outside the range specified by
        `minimum` and `maximum`.

        For each text, the count used as part of the sum across all
        works is the maximum count across the witnesses for that work.

        If `label` is specified, the works checked are restricted to
        those associated with `label`.

        :param minimum: minimum n-gram count
        :type minimum: `int`
        :param maximum: maximum n-gram count
        :type maximum: `int`
        :param label: optional label to restrict requirement to
        :type label: `str`

        """
        self._logger.info('Pruning results by n-gram count')

        def calculate_total(group):
            work_grouped = group.groupby(constants.WORK_FIELDNAME, sort=False)
            total_count = work_grouped[constants.COUNT_FIELDNAME].max().sum()
            group['total_count'] = pd.Series([total_count] * len(group.index),
                                             index=group.index)
            return group

        # self._matches may be empty, in which case not only is there
        # no point trying to do the pruning, but it will raise an
        # exception due to referencing the column 'total_count' which
        # won't have been added. Therefore just return immediately.
        if self._matches.empty:
            return
        matches = self._matches
        if label is not None:
            matches = matches[matches[constants.LABEL_FIELDNAME] == label]
        matches = matches.groupby(
            constants.NGRAM_FIELDNAME, sort=False).apply(calculate_total)
        ngrams = None
        if minimum:
            ngrams = matches[matches['total_count'] >= minimum][
                constants.NGRAM_FIELDNAME].unique()
        if maximum:
            max_ngrams = matches[matches['total_count'] <= maximum][
                constants.NGRAM_FIELDNAME].unique()
            if ngrams is None:
                ngrams = max_ngrams
            else:
                ngrams = list(set(ngrams) & set(max_ngrams))
        self._matches = self._matches[
            self._matches[constants.NGRAM_FIELDNAME].isin(ngrams)]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def prune_by_ngram_count_per_work(self, minimum=None, maximum=None,
                                      label=None):
        """Removes results rows if the n-gram count for all works bearing that
        n-gram is outside the range specified by `minimum` and
        `maximum`.

        That is, if a single witness of a single work has an n-gram
        count that falls within the specified range, all result rows
        for that n-gram are kept.

        If `label` is specified, the works checked are restricted to
        those associated with `label`.

        :param minimum: minimum n-gram count
        :type minimum: `int`
        :param maximum: maximum n-gram count
        :type maximum: `int`
        :param label: optional label to restrict requirement to
        :type label: `str`

        """
        self._logger.info('Pruning results by n-gram count per work')
        matches = self._matches
        keep_ngrams = matches[constants.NGRAM_FIELDNAME].unique()
        if label is not None:
            matches = matches[matches[constants.LABEL_FIELDNAME] == label]
        if minimum and maximum:
            keep_ngrams = matches[
                (matches[constants.COUNT_FIELDNAME] >= minimum) &
                (matches[constants.COUNT_FIELDNAME] <= maximum)][
                    constants.NGRAM_FIELDNAME].unique()
        elif minimum:
            keep_ngrams = matches[
                matches[constants.COUNT_FIELDNAME] >= minimum][
                    constants.NGRAM_FIELDNAME].unique()
        elif maximum:
            keep_ngrams = matches[
                self._matches[constants.COUNT_FIELDNAME] <= maximum][
                    constants.NGRAM_FIELDNAME].unique()
        self._matches = self._matches[self._matches[
            constants.NGRAM_FIELDNAME].isin(keep_ngrams)]

Response:
Removes the results rows if the n - gram count for all works bearing that
        n - gram is outside the specified range.


Instruction:
How would you explain what the following Python 3 function does
def prune_by_ngram_size(self, minimum=None, maximum=None):
        self._logger.info('Pruning results by n-gram size')
        if minimum:
            self._matches = self._matches[
                self._matches[constants.SIZE_FIELDNAME] >= minimum]
        if maximum:
            self._matches = self._matches[
                self._matches[constants.SIZE_FIELDNAME] <= maximum]

Response: Removes results rows whose n - gram size is outside the specified minimum and maximum.


Instruction:
Create a Python 3 function for
removing rows for n - grams that are not attested in a specific n - gram with a number of works in the range specified by minimum and maximum.

Response:
def prune_by_work_count(self, minimum=None, maximum=None, label=None):
        """Removes results rows for n-grams that are not attested in a
        number of works in the range specified by `minimum` and
        `maximum`.

        Work here encompasses all witnesses, so that the same n-gram
        appearing in multiple witnesses of the same work are counted
        as a single work.

        If `label` is specified, the works counted are restricted to
        those associated with `label`.

        :param minimum: minimum number of works
        :type minimum: `int`
        :param maximum: maximum number of works
        :type maximum: `int`
        :param label: optional label to restrict requirement to
        :type label: `str`

        """
        self._logger.info('Pruning results by work count')
        count_fieldname = 'tmp_count'
        matches = self._matches
        if label is not None:
            matches = matches[matches[constants.LABEL_FIELDNAME] == label]
        filtered = matches[matches[constants.COUNT_FIELDNAME] > 0]
        grouped = filtered.groupby(constants.NGRAM_FIELDNAME, sort=False)
        counts = pd.DataFrame(grouped[constants.WORK_FIELDNAME].nunique())
        counts.rename(columns={constants.WORK_FIELDNAME: count_fieldname},
                      inplace=True)
        if minimum:
            counts = counts[counts[count_fieldname] >= minimum]
        if maximum:
            counts = counts[counts[count_fieldname] <= maximum]
        self._matches = pd.merge(self._matches, counts,
                                 left_on=constants.NGRAM_FIELDNAME,
                                 right_index=True)
        del self._matches[count_fieldname]


Instruction:
Make a summary of the following Python 3 code
def reciprocal_remove(self):
        self._logger.info(
            'Removing n-grams that are not attested in all labels')
        self._matches = self._reciprocal_remove(self._matches)

Response:
Removes results rows for which the n - gram is not present in
            at least one text in each labelled set of texts.


Instruction:
Can you create a Python 3 function that
removes rows whose n - grams are contained in larger n - grams.

Response:
def reduce(self):
        """Removes results rows whose n-grams are contained in larger
        n-grams."""
        self._logger.info('Reducing the n-grams')
        # This does not make use of any pandas functionality; it
        # probably could, and if so ought to.
        data = {}
        labels = {}
        # Derive a convenient data structure from the rows.
        for row_index, row in self._matches.iterrows():
            work = row[constants.WORK_FIELDNAME]
            siglum = row[constants.SIGLUM_FIELDNAME]
            labels[work] = row[constants.LABEL_FIELDNAME]
            witness_data = data.setdefault((work, siglum), {})
            witness_data[row[constants.NGRAM_FIELDNAME]] = {
                'count': int(row[constants.COUNT_FIELDNAME]),
                'size': int(row[constants.SIZE_FIELDNAME])}
        for witness_data in data.values():
            ngrams = list(witness_data.keys())
            ngrams.sort(key=lambda ngram: witness_data[ngram]['size'],
                        reverse=True)
            for ngram in ngrams:
                if witness_data[ngram]['count'] > 0:
                    self._reduce_by_ngram(witness_data, ngram)
        # Recreate rows from the modified data structure.
        rows = []
        for (work, siglum), witness_data in data.items():
            for ngram, ngram_data in witness_data.items():
                count = ngram_data['count']
                if count > 0:
                    rows.append(
                        {constants.NGRAM_FIELDNAME: ngram,
                         constants.SIZE_FIELDNAME: ngram_data['size'],
                         constants.WORK_FIELDNAME: work,
                         constants.SIGLUM_FIELDNAME: siglum,
                         constants.COUNT_FIELDNAME: count,
                         constants.LABEL_FIELDNAME: labels[work]})
        self._matches = pd.DataFrame(
            rows, columns=constants.QUERY_FIELDNAMES)


Instruction:
Implement a Python 3 function for
reducing the counts of all n - grams in data by the count of n - grams of ngram.

Response:
def _reduce_by_ngram(self, data, ngram):
        """Lowers the counts of all n-grams in `data` that are
        substrings of `ngram` by `ngram`\'s count.

        Modifies `data` in place.

        :param data: row data dictionary for the current text
        :type data: `dict`
        :param ngram: n-gram being reduced
        :type ngram: `str`

        """
        # Find all substrings of `ngram` and reduce their count by the
        # count of `ngram`. Substrings may not exist in `data`.
        count = data[ngram]['count']
        for substring in self._generate_substrings(ngram, data[ngram]['size']):
            try:
                substring_data = data[substring]
            except KeyError:
                continue
            else:
                substring_data['count'] -= count


Instruction:
Given the following Python 3 function, write the documentation
def relabel(self, catalogue):
        for work, label in catalogue.items():
            self._matches.loc[self._matches[constants.WORK_FIELDNAME] == work,
                              constants.LABEL_FIELDNAME] = label

Response: Relabels results rows according to catalogue.


Instruction:
Here you have a function in Python 3, explain what it does
def remove_label(self, label):
        self._logger.info('Removing label "{}"'.format(label))
        count = self._matches[constants.LABEL_FIELDNAME].value_counts().get(
            label, 0)
        self._matches = self._matches[
            self._matches[constants.LABEL_FIELDNAME] != label]
        self._logger.info('Removed {} labelled results'.format(count))

Response: Removes all results rows associated with label.


Instruction:
Explain what the following Python 3 code does
def sort(self):
        self._matches.sort_values(
            by=[constants.SIZE_FIELDNAME, constants.NGRAM_FIELDNAME,
                constants.COUNT_FIELDNAME, constants.LABEL_FIELDNAME,
                constants.WORK_FIELDNAME, constants.SIGLUM_FIELDNAME],
            ascending=[False, True, False, True, True, True], inplace=True)

Response: Sorts all results rows.


Instruction:
Write a Python 3 function for
adding rows to the results to ensure that every n - gram that is attested in at least one witness for that text is not present in at least one witness for that text.

Response:
def zero_fill(self, corpus):
        """Adds rows to the results to ensure that, for every n-gram that is
        attested in at least one witness, every witness for that text
        has a row, with added rows having a count of zero.

        :param corpus: corpus containing the texts appearing in the results
        :type corpus: `Corpus`

        """
        self._logger.info('Zero-filling results')
        zero_rows = []
        work_sigla = {}
        grouping_cols = [constants.LABEL_FIELDNAME, constants.NGRAM_FIELDNAME,
                         constants.SIZE_FIELDNAME, constants.WORK_FIELDNAME]
        grouped = self._matches.groupby(grouping_cols, sort=False)
        for (label, ngram, size, work), group in grouped:
            row_data = {
                constants.NGRAM_FIELDNAME: ngram,
                constants.LABEL_FIELDNAME: label,
                constants.SIZE_FIELDNAME: size,
                constants.COUNT_FIELDNAME: 0,
                constants.WORK_FIELDNAME: work,
            }
            if work not in work_sigla:
                work_sigla[work] = corpus.get_sigla(work)
            for siglum in work_sigla[work]:
                if group[group[constants.SIGLUM_FIELDNAME] == siglum].empty:
                    row_data[constants.SIGLUM_FIELDNAME] = siglum
                    zero_rows.append(row_data)
        zero_df = pd.DataFrame(zero_rows, columns=constants.QUERY_FIELDNAMES)
        self._matches = pd.concat([self._matches, zero_df], ignore_index=True,
                                  sort=False)


Instruction:
Can you generate the documentation for the following Python 3 function
def get_logger():

    # Grab the logger
    if not hasattr(get_logger, 'logger'):

        # Setup the default logging config
        get_logger.logger = logging.getLogger('chains')
        format_str = '%(asctime)s [%(levelname)s] - %(module)s: %(message)s'
        logging.basicConfig(datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO, format=format_str)

    # Return the logger
    return get_logger.logger

Response: Setup logging output defaults


Instruction:
Write a Python 3 function for
reading Packets from the packet capture interface.

Response:
def read_interface(self):
        """Read Packets from the packet capture interface"""

        # Spin up the packet capture
        if self._iface_is_file():
            self.pcap = pcapy.open_offline(self.iface_name)
        else:
            try:
                # self.pcap = pcap.pcap(name=self.iface_name, promisc=True, immediate=True)
                #   snaplen (maximum number of bytes to capture _per_packet_)
                #   promiscious mode (1 for true)
                #   timeout (in milliseconds)
                self.pcap = pcapy.open_live(self.iface_name, 65536 , 1 , 0)
            except OSError:
                try:
                    logger.warning('Could not get promisc mode, turning flag off')
                    self.pcap = pcapy.open_live(self.iface_name, 65536 , 0 , 0)
                except OSError:
                    log_utils.panic('Could no open interface with any options (may need to be sudo)')

        # Add the BPF if it's specified
        if self.bpf:
            self.pcap.setfilter(self.bpf)
        print('listening on %s: %s' % (self.iface_name, self.bpf))

        # For each packet in the pcap process the contents
        _packets = 0
        while True:
            # Grab the next header and packet buffer
            header, raw_buf = self.pcap.next()

            # If we don't get a packet header break out of the loop
            if not header:
                break;

            # Extract the timestamp from the header and yield the packet
            seconds, micro_sec = header.getts()
            timestamp = seconds + micro_sec * 10**-6
            yield {'timestamp': timestamp, 'raw_buf': raw_buf, 'packet_num': _packets}
            _packets += 1

            # Is there a max packets set if so break on it
            if self.max_packets and _packets >= self.max_packets:
                break

        # All done so report and raise a StopIteration
        try:
            print('Packet stats: %d  received, %d dropped,  %d dropped by interface' % self.pcap.stats())
        except pcapy.PcapError:
            print('No stats available...')
        raise StopIteration


Instruction:
How would you explain what the following Python 3 function does
def generate_parser():
    parser = argparse.ArgumentParser(
        description=constants.TACL_DESCRIPTION,
        formatter_class=ParagraphFormatter)
    subparsers = parser.add_subparsers(title='subcommands')
    generate_align_subparser(subparsers)
    generate_catalogue_subparser(subparsers)
    generate_counts_subparser(subparsers)
    generate_diff_subparser(subparsers)
    generate_excise_subparser(subparsers)
    generate_highlight_subparser(subparsers)
    generate_intersect_subparser(subparsers)
    generate_lifetime_subparser(subparsers)
    generate_ngrams_subparser(subparsers)
    generate_prepare_subparser(subparsers)
    generate_results_subparser(subparsers)
    generate_supplied_diff_subparser(subparsers)
    generate_search_subparser(subparsers)
    generate_supplied_intersect_subparser(subparsers)
    generate_statistics_subparser(subparsers)
    generate_strip_subparser(subparsers)
    return parser

Response: Generates a parser configured with sub - commands and arguments.


Instruction:
Given the following Python 3 function, write the documentation
def generate_align_subparser(subparsers):
    parser = subparsers.add_parser(
        'align', description=constants.ALIGN_DESCRIPTION,
        epilog=constants.ALIGN_EPILOG,
        formatter_class=ParagraphFormatter, help=constants.ALIGN_HELP)
    parser.set_defaults(func=align_results)
    utils.add_common_arguments(parser)
    parser.add_argument('-m', '--minimum', default=20,
                        help=constants.ALIGN_MINIMUM_SIZE_HELP, type=int)
    utils.add_corpus_arguments(parser)
    parser.add_argument('output', help=constants.ALIGN_OUTPUT_HELP,
                        metavar='OUTPUT')
    parser.add_argument('results', help=constants.RESULTS_RESULTS_HELP,
                        metavar='RESULTS')

Response:
Adds a sub - command parser to subparsers to generate aligned
    sequences from a set of results.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_catalogue(args, parser):
    catalogue = tacl.Catalogue()
    catalogue.generate(args.corpus, args.label)
    catalogue.save(args.catalogue)

Response: Generates and saves a catalogue file.


Instruction:
Create a Python 3 function to
add a sub - command parser to subparsers to generate and save a catalogue file.

Response:
def generate_catalogue_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to generate and save
    a catalogue file."""
    parser = subparsers.add_parser(
        'catalogue', description=constants.CATALOGUE_DESCRIPTION,
        epilog=constants.CATALOGUE_EPILOG,
        formatter_class=ParagraphFormatter, help=constants.CATALOGUE_HELP)
    utils.add_common_arguments(parser)
    parser.set_defaults(func=generate_catalogue)
    parser.add_argument('corpus', help=constants.DB_CORPUS_HELP,
                        metavar='CORPUS')
    utils.add_query_arguments(parser)
    parser.add_argument('-l', '--label', default='',
                        help=constants.CATALOGUE_LABEL_HELP)


Instruction:
Create a Python 3 function for
adding a sub - command parser to subparsers to make a counts query.

Response:
def generate_counts_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to make a counts
    query."""
    parser = subparsers.add_parser(
        'counts', description=constants.COUNTS_DESCRIPTION,
        epilog=constants.COUNTS_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.COUNTS_HELP)
    parser.set_defaults(func=ngram_counts)
    utils.add_common_arguments(parser)
    utils.add_db_arguments(parser)
    utils.add_corpus_arguments(parser)
    utils.add_query_arguments(parser)


Instruction:
Here you have a function in Python 3, explain what it does
def generate_diff_subparser(subparsers):
    parser = subparsers.add_parser(
        'diff', description=constants.DIFF_DESCRIPTION,
        epilog=constants.DIFF_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.DIFF_HELP)
    parser.set_defaults(func=ngram_diff)
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-a', '--asymmetric', help=constants.ASYMMETRIC_HELP,
                       metavar='LABEL')
    utils.add_common_arguments(parser)
    utils.add_db_arguments(parser)
    utils.add_corpus_arguments(parser)
    utils.add_query_arguments(parser)

Response:
Adds a sub - command parser to subparsers to make a diff
    query.


Instruction:
Write a Python 3 function for
adding a sub - command parser to subparsers to excise n - grams from witnesses.

Response:
def generate_excise_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to excise n-grams from
    witnesses."""
    parser = subparsers.add_parser(
        'excise', description=constants.EXCISE_DESCRIPTION,
        help=constants.EXCISE_HELP)
    parser.set_defaults(func=excise)
    utils.add_common_arguments(parser)
    parser.add_argument('ngrams', metavar='NGRAMS',
                        help=constants.EXCISE_NGRAMS_HELP)
    parser.add_argument('replacement', metavar='REPLACEMENT',
                        help=constants.EXCISE_REPLACEMENT_HELP)
    parser.add_argument('output', metavar='OUTPUT',
                        help=constants.EXCISE_OUTPUT_HELP)
    utils.add_corpus_arguments(parser)
    parser.add_argument('works', metavar='WORK',
                        help=constants.EXCISE_WORKS_HELP, nargs='+')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_highlight_subparser(subparsers):
    parser = subparsers.add_parser(
        'highlight', description=constants.HIGHLIGHT_DESCRIPTION,
        epilog=constants.HIGHLIGHT_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.HIGHLIGHT_HELP)
    parser.set_defaults(func=highlight_text)
    utils.add_common_arguments(parser)
    parser.add_argument('-m', '--minus-ngrams', metavar='NGRAMS',
                        help=constants.HIGHLIGHT_MINUS_NGRAMS_HELP)
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-n', '--ngrams', action='append', metavar='NGRAMS',
                       help=constants.HIGHLIGHT_NGRAMS_HELP)
    group.add_argument('-r', '--results', metavar='RESULTS',
                       help=constants.HIGHLIGHT_RESULTS_HELP)
    parser.add_argument('-l', '--label', action='append', metavar='LABEL',
                        help=constants.HIGHLIGHT_LABEL_HELP)
    utils.add_corpus_arguments(parser)
    parser.add_argument('base_name', help=constants.HIGHLIGHT_BASE_NAME_HELP,
                        metavar='BASE_NAME')
    parser.add_argument('output', metavar='OUTPUT',
                        help=constants.REPORT_OUTPUT_HELP)

Response:
Adds a sub - command parser to subparsers to highlight a witness'
    text with its matches in a result.


Instruction:
Can you write a function in Python 3 where it
adds a sub - command parser to subparsers to make an intersection query.

Response:
def generate_intersect_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to make an
    intersection query."""
    parser = subparsers.add_parser(
        'intersect', description=constants.INTERSECT_DESCRIPTION,
        epilog=constants.INTERSECT_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.INTERSECT_HELP)
    parser.set_defaults(func=ngram_intersection)
    utils.add_common_arguments(parser)
    utils.add_db_arguments(parser)
    utils.add_corpus_arguments(parser)
    utils.add_query_arguments(parser)


Instruction:
Can you generate the documentation for the following Python 3 function
def generate_lifetime_subparser(subparsers):
    parser = subparsers.add_parser(
        'lifetime', description=constants.LIFETIME_DESCRIPTION,
        epilog=constants.LIFETIME_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.LIFETIME_HELP)
    parser.set_defaults(func=lifetime_report)
    utils.add_tokenizer_argument(parser)
    utils.add_common_arguments(parser)
    utils.add_query_arguments(parser)
    parser.add_argument('results', help=constants.LIFETIME_RESULTS_HELP,
                        metavar='RESULTS')
    parser.add_argument('label', help=constants.LIFETIME_LABEL_HELP,
                        metavar='LABEL')
    parser.add_argument('output', help=constants.REPORT_OUTPUT_HELP,
                        metavar='OUTPUT')

Response: Adds a sub - command parser to subparsers to make a lifetime report.


Instruction:
Create a Python 3 function to
add n - grams data to the data store.

Response:
def generate_ngrams(args, parser):
    """Adds n-grams data to the data store."""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    if args.catalogue:
        catalogue = utils.get_catalogue(args)
    else:
        catalogue = None
    store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)


Instruction:
Create a Python 3 function to
add a sub - command parser to subparsers to add n - grams data to the data store.

Response:
def generate_ngrams_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to add n-grams data
    to the data store."""
    parser = subparsers.add_parser(
        'ngrams', description=constants.NGRAMS_DESCRIPTION,
        epilog=constants.NGRAMS_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.NGRAMS_HELP)
    parser.set_defaults(func=generate_ngrams)
    utils.add_common_arguments(parser)
    parser.add_argument('-c', '--catalogue', dest='catalogue',
                        help=constants.NGRAMS_CATALOGUE_HELP,
                        metavar='CATALOGUE')
    utils.add_db_arguments(parser)
    utils.add_corpus_arguments(parser)
    parser.add_argument('min_size', help=constants.NGRAMS_MINIMUM_HELP,
                        metavar='MINIMUM', type=int)
    parser.add_argument('max_size', help=constants.NGRAMS_MAXIMUM_HELP,
                        metavar='MAXIMUM', type=int)


Instruction:
How would you implement a function in Python 3 that
adds a sub - command parser to subparsers to prepare source XML files for stripping.

Response:
def generate_prepare_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to prepare source XML
    files for stripping."""
    parser = subparsers.add_parser(
        'prepare', description=constants.PREPARE_DESCRIPTION,
        epilog=constants.PREPARE_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.PREPARE_HELP)
    parser.set_defaults(func=prepare_xml)
    utils.add_common_arguments(parser)
    parser.add_argument('-s', '--source', dest='source',
                        choices=constants.TEI_SOURCE_CHOICES,
                        default=constants.TEI_SOURCE_CBETA_GITHUB,
                        help=constants.PREPARE_SOURCE_HELP,
                        metavar='SOURCE')
    parser.add_argument('input', help=constants.PREPARE_INPUT_HELP,
                        metavar='INPUT')
    parser.add_argument('output', help=constants.PREPARE_OUTPUT_HELP,
                        metavar='OUTPUT')


Instruction:
Can you write a function in Python 3 where it
adds a sub - command parser to manipulate CSV results data.

Response:
def generate_results_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to manipulate CSV
    results data."""
    parser = subparsers.add_parser(
        'results', description=constants.RESULTS_DESCRIPTION,
        epilog=constants.RESULTS_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.RESULTS_HELP)
    utils.add_common_arguments(parser)
    parser.set_defaults(func=results)
    be_group = parser.add_argument_group('bifurcated extend')
    be_group.add_argument('-b', '--bifurcated-extend',
                          dest='bifurcated_extend', metavar='CORPUS',
                          help=constants.RESULTS_BIFURCATED_EXTEND_HELP)
    be_group.add_argument('--max-be-count', dest='bifurcated_extend_size',
                          help=constants.RESULTS_BIFURCATED_EXTEND_MAX_HELP,
                          metavar='COUNT', type=int)
    parser.add_argument('-e', '--extend', dest='extend',
                        help=constants.RESULTS_EXTEND_HELP, metavar='CORPUS')
    parser.add_argument('--excise', help=constants.RESULTS_EXCISE_HELP,
                        metavar='NGRAM', type=str)
    parser.add_argument('-l', '--label', dest='label',
                        help=constants.RESULTS_LABEL_HELP, metavar='LABEL')
    parser.add_argument('--min-count', dest='min_count',
                        help=constants.RESULTS_MINIMUM_COUNT_HELP,
                        metavar='COUNT', type=int)
    parser.add_argument('--max-count', dest='max_count',
                        help=constants.RESULTS_MAXIMUM_COUNT_HELP,
                        metavar='COUNT', type=int)
    parser.add_argument('--min-count-work', dest='min_count_work',
                        help=constants.RESULTS_MINIMUM_COUNT_WORK_HELP,
                        metavar='COUNT', type=int)
    parser.add_argument('--max-count-work', dest='max_count_work',
                        help=constants.RESULTS_MAXIMUM_COUNT_WORK_HELP,
                        metavar='COUNT', type=int)
    parser.add_argument('--min-size', dest='min_size',
                        help=constants.RESULTS_MINIMUM_SIZE_HELP,
                        metavar='SIZE', type=int)
    parser.add_argument('--max-size', dest='max_size',
                        help=constants.RESULTS_MAXIMUM_SIZE_HELP,
                        metavar='SIZE', type=int)
    parser.add_argument('--min-works', dest='min_works',
                        help=constants.RESULTS_MINIMUM_WORK_HELP,
                        metavar='COUNT', type=int)
    parser.add_argument('--max-works', dest='max_works',
                        help=constants.RESULTS_MAXIMUM_WORK_HELP,
                        metavar='COUNT', type=int)
    parser.add_argument('--ngrams', dest='ngrams',
                        help=constants.RESULTS_NGRAMS_HELP, metavar='NGRAMS')
    parser.add_argument('--reciprocal', action='store_true',
                        help=constants.RESULTS_RECIPROCAL_HELP)
    parser.add_argument('--reduce', action='store_true',
                        help=constants.RESULTS_REDUCE_HELP)
    parser.add_argument('--relabel', help=constants.RESULTS_RELABEL_HELP,
                        metavar='CATALOGUE')
    parser.add_argument('--remove', help=constants.RESULTS_REMOVE_HELP,
                        metavar='LABEL', type=str)
    parser.add_argument('--sort', action='store_true',
                        help=constants.RESULTS_SORT_HELP)
    utils.add_tokenizer_argument(parser)
    parser.add_argument('-z', '--zero-fill', dest='zero_fill',
                        help=constants.RESULTS_ZERO_FILL_HELP,
                        metavar='CORPUS')
    parser.add_argument('results', help=constants.RESULTS_RESULTS_HELP,
                        metavar='RESULTS')
    unsafe_group = parser.add_argument_group(
        constants.RESULTS_UNSAFE_GROUP_TITLE,
        constants.RESULTS_UNSAFE_GROUP_DESCRIPTION)
    unsafe_group.add_argument('--add-label-count', action='store_true',
                              help=constants.RESULTS_ADD_LABEL_COUNT_HELP)
    unsafe_group.add_argument('--add-label-work-count', action='store_true',
                              help=constants.RESULTS_ADD_LABEL_WORK_COUNT_HELP)
    unsafe_group.add_argument('--collapse-witnesses', action='store_true',
                              help=constants.RESULTS_COLLAPSE_WITNESSES_HELP)
    unsafe_group.add_argument('--group-by-ngram', dest='group_by_ngram',
                              help=constants.RESULTS_GROUP_BY_NGRAM_HELP,
                              metavar='CATALOGUE')
    unsafe_group.add_argument('--group-by-witness', action='store_true',
                              help=constants.RESULTS_GROUP_BY_WITNESS_HELP)


Instruction:
Can you tell what is the following Python 3 function doing
def generate_search_subparser(subparsers):
    parser = subparsers.add_parser(
        'search', description=constants.SEARCH_DESCRIPTION,
        epilog=constants.SEARCH_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.SEARCH_HELP)
    parser.set_defaults(func=search_texts)
    utils.add_common_arguments(parser)
    utils.add_db_arguments(parser)
    utils.add_corpus_arguments(parser)
    utils.add_query_arguments(parser)
    parser.add_argument('ngrams', help=constants.SEARCH_NGRAMS_HELP,
                        nargs='*', metavar='NGRAMS')

Response:
Adds a sub - command parser to subparsers to generate search
    results for a set of n - grams.


Instruction:
Can you write a function in Python 3 where it
adds a sub - command parser to subparsers to generate statistics from a set of results.

Response:
def generate_statistics_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to generate statistics
    from a set of results."""
    parser = subparsers.add_parser(
        'stats', description=constants.STATISTICS_DESCRIPTION,
        formatter_class=ParagraphFormatter, help=constants.STATISTICS_HELP)
    parser.set_defaults(func=generate_statistics)
    utils.add_common_arguments(parser)
    utils.add_corpus_arguments(parser)
    parser.add_argument('results', help=constants.STATISTICS_RESULTS_HELP,
                        metavar='RESULTS')


Instruction:
How would you code a function in Python 3 to
add a sub - command parser to subparsers to process prepared files for use with the tacl ngrams command.

Response:
def generate_strip_subparser(subparsers):
    """Adds a sub-command parser to `subparsers` to process prepared files
    for use with the tacl ngrams command."""
    parser = subparsers.add_parser(
        'strip', description=constants.STRIP_DESCRIPTION,
        epilog=constants.STRIP_EPILOG, formatter_class=ParagraphFormatter,
        help=constants.STRIP_HELP)
    parser.set_defaults(func=strip_files)
    utils.add_common_arguments(parser)
    parser.add_argument('input', help=constants.STRIP_INPUT_HELP,
                        metavar='INPUT')
    parser.add_argument('output', help=constants.STRIP_OUTPUT_HELP,
                        metavar='OUTPUT')


Instruction:
Can you tell what is the following Python 3 function doing
def generate_supplied_diff_subparser(subparsers):
    parser = subparsers.add_parser(
        'sdiff', description=constants.SUPPLIED_DIFF_DESCRIPTION,
        epilog=constants.SUPPLIED_DIFF_EPILOG,
        formatter_class=ParagraphFormatter, help=constants.SUPPLIED_DIFF_HELP)
    parser.set_defaults(func=supplied_diff)
    utils.add_common_arguments(parser)
    utils.add_tokenizer_argument(parser)
    utils.add_db_arguments(parser, True)
    utils.add_supplied_query_arguments(parser)

Response:
Adds a sub - command parser to subparsers to run a diff query using
    the supplied results sets.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate_supplied_intersect_subparser(subparsers):
    parser = subparsers.add_parser(
        'sintersect', description=constants.SUPPLIED_INTERSECT_DESCRIPTION,
        epilog=constants.SUPPLIED_INTERSECT_EPILOG,
        formatter_class=ParagraphFormatter,
        help=constants.SUPPLIED_INTERSECT_HELP)
    parser.set_defaults(func=supplied_intersect)
    utils.add_common_arguments(parser)
    utils.add_db_arguments(parser, True)
    utils.add_supplied_query_arguments(parser)

Response: Adds a sub - command parser to subparsers to run an intersect query on the supplied results sets.


Instruction:
Can you generate the documentation for the following Python 3 function
def highlight_text(args, parser):
    tokenizer = utils.get_tokenizer(args)
    corpus = utils.get_corpus(args)
    output_dir = os.path.abspath(args.output)
    if os.path.exists(output_dir):
        parser.exit(status=3, message='Output directory already exists, '
                    'aborting.\n')
    os.makedirs(output_dir, exist_ok=True)
    if args.ngrams:
        if args.label is None or len(args.label) != len(args.ngrams):
            parser.error('There must be as many labels as there are files '
                         'of n-grams')
        report = tacl.NgramHighlightReport(corpus, tokenizer)
        ngrams = []
        for ngram_file in args.ngrams:
            ngrams.append(utils.get_ngrams(ngram_file))
        minus_ngrams = []
        if args.minus_ngrams:
            minus_ngrams = utils.get_ngrams(args.minus_ngrams)
        report.generate(args.output, args.base_name, ngrams, args.label,
                        minus_ngrams)
    else:
        report = tacl.ResultsHighlightReport(corpus, tokenizer)
        report.generate(args.output, args.base_name, args.results)

Response: Outputs the result of highlighting a text.


Instruction:
Make a summary of the following Python 3 code
def lifetime_report(args, parser):
    catalogue = utils.get_catalogue(args)
    tokenizer = utils.get_tokenizer(args)
    results = tacl.Results(args.results, tokenizer)
    output_dir = os.path.abspath(args.output)
    os.makedirs(output_dir, exist_ok=True)
    report = tacl.LifetimeReport()
    report.generate(output_dir, catalogue, results, args.label)

Response: Generates a lifetime report.


Instruction:
Can you generate the documentation for the following Python 3 function
def ngram_counts(args, parser):
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    store.validate(corpus, catalogue)
    store.counts(catalogue, sys.stdout)

Response: Outputs the results of performing a counts query.


Instruction:
Implement a Python 3 function for
outputting the results of performing a diff query.

Response:
def ngram_diff(args, parser):
    """Outputs the results of performing a diff query."""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    tokenizer = utils.get_tokenizer(args)
    store.validate(corpus, catalogue)
    if args.asymmetric:
        store.diff_asymmetric(catalogue, args.asymmetric, tokenizer,
                              sys.stdout)
    else:
        store.diff(catalogue, tokenizer, sys.stdout)


Instruction:
Make a summary of the following Python 3 code
def ngram_intersection(args, parser):
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    store.validate(corpus, catalogue)
    store.intersection(catalogue, sys.stdout)

Response: Outputs the results of performing an intersection query.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def prepare_xml(args, parser):
    if args.source == constants.TEI_SOURCE_CBETA_GITHUB:
        corpus_class = tacl.TEICorpusCBETAGitHub
    else:
        raise Exception('Unsupported TEI source option provided')
    corpus = corpus_class(args.input, args.output)
    corpus.tidy()

Response: Prepares XML files for stripping.


Instruction:
Create a Python 3 function for
searching texts for presence of n - grams.

Response:
def search_texts(args, parser):
    """Searches texts for presence of n-grams."""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    catalogue = utils.get_catalogue(args)
    store.validate(corpus, catalogue)
    ngrams = []
    for ngram_file in args.ngrams:
        ngrams.extend(utils.get_ngrams(ngram_file))
    store.search(catalogue, ngrams, sys.stdout)


Instruction:
Implement a function in Python 3 to
process prepared XML files for use with the tacl ngrams command.

Response:
def strip_files(args, parser):
    """Processes prepared XML files for use with the tacl ngrams
    command."""
    stripper = tacl.Stripper(args.input, args.output)
    stripper.strip_files()


Instruction:
How would you explain what the following Python 3 function does
def http_meta_data(self):

        # For each flow process the contents
        for flow in self.input_stream:

            # Client to Server
            if flow['direction'] == 'CTS':
                try:
                    request = dpkt.http.Request(flow['payload'])
                    request_data = data_utils.make_dict(request)
                    request_data['uri'] = self._clean_uri(request['uri'])
                    flow['http'] = {'type':'HTTP_REQUEST', 'data':request_data}
                except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
                    flow['http'] = None

            # Server to Client
            else:
                try:
                    response = dpkt.http.Response(flow['payload'])
                    flow['http'] = {'type': 'HTTP_RESPONSE', 'data': data_utils.make_dict(response)}
                except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
                    flow['http'] = None

            # Mark non-TCP HTTP
            if flow['http'] and flow['protocol'] != 'TCP':
                flow['http'].update({'weird': 'UDP-HTTP'})

            # All done
            yield flow

Response: Pull out the application metadata for each flow in the input_stream and yield the flow data as a generator of the flow data.


Instruction:
Here you have a function in Python 3, explain what it does
def transport_meta_data(self):

        # For each packet in the pcap process the contents
        for item in self.input_stream:

            # Get the transport data and type
            trans_data = item['packet']['data']
            trans_type = self._get_transport_type(trans_data)
            if trans_type and trans_data:
                item['transport'] = data_utils.make_dict(trans_data)
                item['transport']['type'] = trans_type
                item['transport']['flags'] = self._readable_flags(item['transport'])
                item['transport']['data'] = trans_data['data']

            # All done
            yield item

Response: Yields the metadata for each packet in the input_stream


Instruction:
Can you tell what is the following Python 3 function doing
def _create_breakdown_chart(self, data, work, output_dir):
        chart_data = data.loc[work].sort_values(by=SHARED, ascending=False)[
            [SHARED, UNIQUE, COMMON]]
        csv_path = os.path.join(output_dir, 'breakdown_{}.csv'.format(
            work))
        chart_data.to_csv(csv_path)

Response: Generates and writes a CSV file in output_dir the data used to get the breakdown chart.


Instruction:
Explain what the following Python 3 code does
def _create_chord_chart(self, data, works, output_dir):
        matrix = []
        chord_data = data.unstack(BASE_WORK)[SHARED]
        for index, row_data in chord_data.fillna(value=0).iterrows():
            matrix.append([value / 100 for value in row_data])
        colours = generate_colours(len(works))
        colour_works = [{'work': work, 'colour': colour} for work, colour
                        in zip(chord_data, colours)]
        json_data = json.dumps({'works': colour_works, 'matrix': matrix})
        with open(os.path.join(output_dir, 'chord_data.js'), 'w') as fh:
            fh.write('var chordData = {}'.format(json_data))

Response: Generates and writes a file in output_dir the data used to display a chord chart.


Instruction:
How would you code a function in Python 3 to
generate and writes a JSON file that contains the data used to display a matrix chart.

Response:
def _create_matrix_chart(self, data, works, output_dir):
        """Generates and writes to a file in `output_dir` the data used to
        display a matrix chart.

        :param data: data to derive the matrix data from
        :type data: `pandas.DataFrame`
        :param works: works to display
        :type works: `list`
        :param output_dir: directory to output data file to
        :type output_dir: `str`

        """
        nodes = [{'work': work, 'group': 1} for work in works]
        weights = data.stack().unstack(RELATED_WORK).max()
        seen = []
        links = []
        for (source, target), weight in weights.iteritems():
            if target not in seen and target != source:
                seen.append(source)
                links.append({'source': works.index(source),
                              'target': works.index(target),
                              'value': weight})
        json_data = json.dumps({'nodes': nodes, 'links': links})
        with open(os.path.join(output_dir, 'matrix_data.js'), 'w') as fh:
            fh.write('var matrixData = {}'.format(json_data))


Instruction:
Here you have a function in Python 3, explain what it does
def _create_related_chart(self, data, work, output_dir):
        chart_data = data[work].dropna().sort_values(by=SHARED_RELATED_WORK,
                                                     ascending=False)
        csv_path = os.path.join(output_dir, 'related_{}.csv'.format(work))
        chart_data.to_csv(csv_path)

Response: Generates and writes a CSV file in output_dir the data used to be shown for each maybe work that is shared with the maybe work.


Instruction:
Implement a Python 3 function for
writing the results to fh minus those results associated with the no label.

Response:
def _drop_no_label_results(self, results, fh):
        """Writes `results` to `fh` minus those results associated with the
        'no' label.

        :param results: results to be manipulated
        :type results: file-like object
        :param fh: output destination
        :type fh: file-like object

        """
        results.seek(0)
        results = Results(results, self._tokenizer)
        results.remove_label(self._no_label)
        results.csv(fh)


Instruction:
How would you code a function in Python 3 to
write a statistics report for the results at results_path to out_path.

Response:
def _generate_statistics(self, out_path, results_path):
        """Writes a statistics report for the results at `results_path` to
        `out_path`.

        Reuses an existing statistics report if one exists at
        `out_path`.

        :param out_path: path to output statistics report to
        :type out_path: `str`
        :param results_path: path of results to generate statistics for
        :type results_path: `str`

        """
        if not os.path.exists(out_path):
            report = StatisticsReport(self._corpus, self._tokenizer,
                                      results_path)
            report.generate_statistics()
            with open(out_path, mode='w', encoding='utf-8', newline='') as fh:
                report.csv(fh)


Instruction:
Implement a Python 3 function for
processing the diff between two sets of yes_work and maybe_work and returns the statistics on the difference between the intersection of yes_work and maybe_work and the intersection of ym_results_path and yn_results_path.

Response:
def _process_diff(self, yes_work, maybe_work, work_dir, ym_results_path,
                      yn_results_path, stats):
        """Returns statistics on the difference between the intersection of
        `yes_work` and `maybe_work` and the intersection of `yes_work`
        and "no" works.

        :param yes_work: name of work for which stats are collected
        :type yes_work: `str`
        :param maybe_work: name of work being compared with `yes_work`
        :type maybe_work: `str`
        :param work_dir: directory where generated files are saved
        :type work_dir: `str`
        :param ym_results_path: path to results intersecting
                                `yes_work` with `maybe_work`
        :type yn_results_path: `str`
        :param yn_results_path: path to results intersecting
                                `yes_work` with "no" works
        :type yn_results_path: `str`
        :param stats: data structure to hold the statistical data
        :type stats: `dict`
        :rtype: `dict`

        """
        distinct_results_path = os.path.join(
            work_dir, 'distinct_{}.csv'.format(maybe_work))
        results = [yn_results_path, ym_results_path]
        labels = [self._no_label, self._maybe_label]
        self._run_query(distinct_results_path, self._store.diff_supplied,
                        [results, labels, self._tokenizer])
        return self._update_stats('diff', work_dir, distinct_results_path,
                                  yes_work, maybe_work, stats, SHARED, COMMON)


Instruction:
How would you code a function in Python 3 to
return statistics on the intersection between yes_work and maybe_work.

Response:
def _process_intersection(self, yes_work, maybe_work, work_dir,
                              ym_results_path, stats):
        """Returns statistics on the intersection between `yes_work` and
        `maybe_work`.

        :param yes_work: name of work for which stats are collected
        :type yes_work: `str`
        :param maybe_work: name of work being compared with `yes_work`
        :type maybe_work: `str`
        :param work_dir: directory where generated files are saved
        :type work_dir: `str`
        :param ym_results_path: path to results intersecting
                                `yes_work` with `maybe_work`
        :type ym_results_path: `str`
        :param stats: data structure to hold the statistical data
        :type stats: `dict`
        :rtype: `dict`

        """
        catalogue = {yes_work: self._no_label, maybe_work: self._maybe_label}
        self._run_query(ym_results_path, self._store.intersection, [catalogue],
                        False)
        # Though this is the intersection only between "yes" and
        # "maybe", the percentage of overlap is added to the "common"
        # stat rather than "shared". Then, in _process_diff, the
        # percentage of difference between "yes" and "no" can be
        # removed from "common" and added to "shared".
        return self._update_stats('intersect', work_dir, ym_results_path,
                                  yes_work, maybe_work, stats, COMMON, UNIQUE)


Instruction:
How would you explain what the following Python 3 function does
def _process_maybe_work(self, yes_work, maybe_work, work_dir,
                            yn_results_path, stats):
        """Returns statistics of how `yes_work` compares with `maybe_work`.

        :param yes_work: name of work for which stats are collected
        :type yes_work: `str`
        :param maybe_work: name of work being compared with `yes_work`
        :type maybe_work: `str`
        :param work_dir: directory where generated files are saved
        :type work_dir: `str`
        :param yn_results_path: path to results intersecting
                                `yes_work` with "no" works
        :type yn_results_path: `str`
        :param stats: data structure to hold statistical data of the
                      comparison
        :type stats: `dict`
        :rtype: `dict`

        """
        if maybe_work == yes_work:
            return stats
        self._logger.info(
            'Processing "maybe" work {} against "yes" work {}.'.format(
                maybe_work, yes_work))
        # Set base values for each statistic of interest, for each
        # witness.
        for siglum in self._corpus.get_sigla(maybe_work):
            witness = (maybe_work, siglum)
            stats[COMMON][witness] = 0
            stats[SHARED][witness] = 0
            stats[UNIQUE][witness] = 100
        works = [yes_work, maybe_work]
        # Sort the works to have a single filename for the
        # intersection each pair of works, whether they are yes or
        # maybe. This saves repeating the intersection with the roles
        # switched, since _run_query will use a found file rather than
        # rerun the query.
        works.sort()
        ym_results_path = os.path.join(
            self._ym_intersects_dir, '{}_intersect_{}.csv'.format(*works))
        stats = self._process_intersection(yes_work, maybe_work, work_dir,
                                           ym_results_path, stats)
        stats = self._process_diff(yes_work, maybe_work, work_dir,
                                   ym_results_path, yn_results_path, stats)
        return stats

Response: Processes the maybe_work and maybe_work and returns statistics of how the yes_work compares with maybe_work.


Instruction:
Write a Python 3 function for
collecting and return the data of how each work in maybe_works relates to each other work.

Response:
def _process_works(self, maybe_works, no_works, output_dir):
        """Collect and return the data of how each work in `maybe_works`
        relates to each other work.

        :param maybe_works:
        :type maybe_works: `list` of `str`
        :param no_works:
        :type no_works: `list` of `str`
        :param output_dir: base output directory
        :type output_dir: `str`
        :rtype: `pandas.DataFrame`

        """
        output_data_dir = os.path.join(output_dir, 'data')
        no_catalogue = {work: self._no_label for work in no_works}
        self._ym_intersects_dir = os.path.join(output_data_dir,
                                               'ym_intersects')
        data = {}
        os.makedirs(self._ym_intersects_dir, exist_ok=True)
        for yes_work in maybe_works:
            no_catalogue[yes_work] = self._maybe_label
            stats = self._process_yes_work(yes_work, no_catalogue,
                                           maybe_works, output_data_dir)
            no_catalogue.pop(yes_work)
            for scope in (SHARED, COMMON, UNIQUE):
                work_data = stats[scope]
                index = pd.MultiIndex.from_tuples(
                    list(work_data.keys()), names=[RELATED_WORK, SIGLUM])
                data[(yes_work, scope)] = pd.Series(list(work_data.values()),
                                                    index=index)
        df = pd.DataFrame(data)
        df.columns.names = [BASE_WORK, SCOPE]
        df = df.stack(BASE_WORK).swaplevel(
            BASE_WORK, SIGLUM).swaplevel(RELATED_WORK, BASE_WORK)
        return df


Instruction:
Write a Python 3 script for
processing a yes_work and maybe_work and returns a dictionary of how the yes_work compares with the other works in no_catalogue and the maybe_works.

Response:
def _process_yes_work(self, yes_work, no_catalogue, maybe_works,
                          output_dir):
        """Returns statistics of how `yes_work` compares with the other works
        in `no_catalogue` and the "maybe" works.

        :param yes_work: name of work being processed
        :type yes_work: `str`
        :param no_catalogue: catalogue of containing `yes_work` and the "no"
                             works
        :type no_catalogue: `Catalogue`
        :param maybe_works: names of "maybe" works
        :type maybe_works: `list` of `str`
        :param output_dir: directory where generated files are saved
        :type output_dir: `str`
        :rtype: `dict`

        """
        self._logger.info('Processing "maybe" work {} as "yes".'.format(
            yes_work))
        stats = {COMMON: {}, SHARED: {}, UNIQUE: {}}
        yes_work_dir = os.path.join(output_dir, yes_work)
        os.makedirs(yes_work_dir, exist_ok=True)
        results_path = os.path.join(yes_work_dir, 'intersect_with_no.csv')
        self._run_query(results_path, self._store.intersection, [no_catalogue])
        for maybe_work in maybe_works:
            stats = self._process_maybe_work(
                yes_work, maybe_work, yes_work_dir, results_path, stats)
        return stats


Instruction:
Write a Python 3 script for
running a query and outputs results to a file at path.

Response:
def _run_query(self, path, query, query_args, drop_no=True):
        """Runs `query` and outputs results to a file at `path`.

        If `path` exists, the query is not run.

        :param path: path to output results to
        :type path: `str`
        :param query: query to run
        :type query: `method`
        :param query_args: arguments to supply to the query
        :type query_args: `list`
        :param drop_no: whether to drop results from the No corpus
        :type drop_no: `bool`

        """
        if os.path.exists(path):
            return
        output_results = io.StringIO(newline='')
        query(*query_args, output_fh=output_results)
        with open(path, mode='w', encoding='utf-8', newline='') as fh:
            if drop_no:
                self._drop_no_label_results(output_results, fh)
            else:
                fh.write(output_results.getvalue())


Instruction:
Implement a Python 3 function for
adding the indices relating to n - grams.

Response:
def _add_indices(self):
        """Adds the database indices relating to n-grams."""
        self._logger.info('Adding database indices')
        self._conn.execute(constants.CREATE_INDEX_TEXTNGRAM_SQL)
        self._logger.info('Indices added')


Instruction:
Can you implement a function in Python 3 that
adds n - gram data from a corpus to the data store.

Response:
def add_ngrams(self, corpus, minimum, maximum, catalogue=None):
        """Adds n-gram data from `corpus` to the data store.

        :param corpus: corpus of works
        :type corpus: `Corpus`
        :param minimum: minimum n-gram size
        :type minimum: `int`
        :param maximum: maximum n-gram size
        :type maximum: `int`
        :param catalogue: optional catalogue to limit corpus to
        :type catalogue: `Catalogue`

        """
        self._initialise_database()
        if catalogue:
            for work in catalogue:
                for witness in corpus.get_witnesses(work):
                    self._add_text_ngrams(witness, minimum, maximum)
        else:
            for witness in corpus.get_witnesses():
                self._add_text_ngrams(witness, minimum, maximum)
        self._add_indices()
        self._analyse()


Instruction:
How would you explain what the following Python 3 function does
def _add_temporary_ngrams(self, ngrams):
        # Remove duplicate n-grams, empty n-grams, and non-string n-grams.
        ngrams = [ngram for ngram in ngrams if ngram and
                  isinstance(ngram, str)]
        # Deduplicate while preserving order (useful for testing).
        seen = {}
        ngrams = [seen.setdefault(x, x) for x in ngrams if x not in seen]
        self._conn.execute(constants.DROP_TEMPORARY_NGRAMS_TABLE_SQL)
        self._conn.execute(constants.CREATE_TEMPORARY_NGRAMS_TABLE_SQL)
        self._conn.executemany(constants.INSERT_TEMPORARY_NGRAM_SQL,
                               [(ngram,) for ngram in ngrams])

Response: Adds n - grams to a temporary table.


Instruction:
Make a summary of the following Python 3 code
def _add_temporary_results(self, results, label):
        NGRAM, SIZE, NAME, SIGLUM, COUNT, LABEL = constants.QUERY_FIELDNAMES
        reader = csv.DictReader(results)
        data = [(row[NGRAM], row[SIZE], row[NAME], row[SIGLUM], row[COUNT],
                 label) for row in reader]
        self._conn.executemany(constants.INSERT_TEMPORARY_RESULTS_SQL, data)

Response: Adds the results to a temporary table with label.


Instruction:
Explain what the following Python 3 code does
def _add_text_ngrams(self, witness, minimum, maximum):
        text_id = self._get_text_id(witness)
        self._logger.info('Adding n-grams ({} <= n <= {}) for {}'.format(
            minimum, maximum, witness.get_filename()))
        skip_sizes = []
        for size in range(minimum, maximum + 1):
            if self._has_ngrams(text_id, size):
                self._logger.info(
                    '{}-grams are already in the database'.format(size))
                skip_sizes.append(size)
        for size, ngrams in witness.get_ngrams(minimum, maximum, skip_sizes):
            self._add_text_size_ngrams(text_id, size, ngrams)

Response: Adds n - gram data from witness to the data store.


Instruction:
Given the following Python 3 function, write the documentation
def _add_text_record(self, witness):
        filename = witness.get_filename()
        name, siglum = witness.get_names()
        self._logger.info('Adding record for text {}'.format(filename))
        checksum = witness.get_checksum()
        token_count = len(witness.get_tokens())
        with self._conn:
            cursor = self._conn.execute(
                constants.INSERT_TEXT_SQL,
                [name, siglum, checksum, token_count, ''])
        return cursor.lastrowid

Response: Adds a Text record for witness.


Instruction:
Write a Python 3 function that can
add n - grams that are of size to the data store.

Response:
def _add_text_size_ngrams(self, text_id, size, ngrams):
        """Adds `ngrams`, that are of size `size`, to the data store.

        The added `ngrams` are associated with `text_id`.

        :param text_id: database ID of text associated with `ngrams`
        :type text_id: `int`
        :param size: size of n-grams
        :type size: `int`
        :param ngrams: n-grams to be added
        :type ngrams: `collections.Counter`

        """
        unique_ngrams = len(ngrams)
        self._logger.info('Adding {} unique {}-grams'.format(
            unique_ngrams, size))
        parameters = [[text_id, ngram, size, count]
                      for ngram, count in ngrams.items()]
        with self._conn:
            self._conn.execute(constants.INSERT_TEXT_HAS_NGRAM_SQL,
                               [text_id, size, unique_ngrams])
            self._conn.executemany(constants.INSERT_NGRAM_SQL, parameters)


Instruction:
Write a Python 3 function for
analysing the database or table.

Response:
def _analyse(self, table=''):
        """Analyses the database, or `table` if it is supplied.

        :param table: optional name of table to analyse
        :type table: `str`

        """
        self._logger.info('Starting analysis of database')
        self._conn.execute(constants.ANALYSE_SQL.format(table))
        self._logger.info('Analysis of database complete')


Instruction:
Can you tell what is the following Python 3 function doing
def _check_diff_result(row, matches, tokenize, join):
        ngram_tokens = tokenize(row[constants.NGRAM_FIELDNAME])
        sub_ngram1 = join(ngram_tokens[:-1])
        sub_ngram2 = join(ngram_tokens[1:])
        count = constants.COUNT_FIELDNAME
        discard = False
        # For performance reasons, avoid searching through matches
        # unless necessary.
        status1 = matches.get(sub_ngram1)
        if status1 == 0:
            discard = True
        else:
            status2 = matches.get(sub_ngram2)
            if status2 == 0:
                discard = True
            elif (status1 is None) ^ (status2 is None):
                discard = True
        if discard:
            row[count] = 0
        return row

Response: Checks the n - grams in a result row and updates the count of the n - grams that compose it.


Instruction:
Write a Python 3 script for
returning output_fh populated with CSV results giving n - gram counts of the witnesses of the works in catalogue.

Response:
def counts(self, catalogue, output_fh):
        """Returns `output_fh` populated with CSV results giving
        n-gram counts of the witnesses of the works in `catalogue`.

        :param catalogue: catalogue matching filenames to labels
        :type catalogue: `Catalogue`
        :param output_fh: object to output results to
        :type output_fh: file-like object
        :rtype: file-like object

        """
        labels = list(self._set_labels(catalogue))
        label_placeholders = self._get_placeholders(labels)
        query = constants.SELECT_COUNTS_SQL.format(label_placeholders)
        self._logger.info('Running counts query')
        self._logger.debug('Query: {}\nLabels: {}'.format(query, labels))
        cursor = self._conn.execute(query, labels)
        return self._csv(cursor, constants.COUNTS_FIELDNAMES, output_fh)


Instruction:
How would you explain what the following Python 3 function does
def _csv(self, cursor, fieldnames, output_fh):
        self._logger.info('Finished query; outputting results in CSV format')
        # Specify a lineterminator to avoid an extra \r being added on
        # Windows; see
        # https://stackoverflow.com/questions/3191528/csv-in-python-adding-extra-carriage-return
        if sys.platform in ('win32', 'cygwin') and output_fh is sys.stdout:
            writer = csv.writer(output_fh, lineterminator='\n')
        else:
            writer = csv.writer(output_fh)
        writer.writerow(fieldnames)
        for row in cursor:
            writer.writerow(row)
        self._logger.info('Finished outputting results')
        return output_fh

Response: Writes the rows of cursor in CSV format to output_fh and returns it.


Instruction:
Implement a Python 3 function for
writing the rows of cursor in CSV format and returns the path to that file.

Response:
def _csv_temp(self, cursor, fieldnames):
        """Writes the rows of `cursor` in CSV format to a temporary file and
        returns the path to that file.

        :param cursor: database cursor containing data to be output
        :type cursor: `sqlite3.Cursor`
        :param fieldnames: row headings
        :type fieldnames: `list`
        :rtype: `str`

        """
        temp_fd, temp_path = tempfile.mkstemp(text=True)
        with open(temp_fd, 'w', encoding='utf-8', newline='') as results_fh:
            self._csv(cursor, fieldnames, results_fh)
        return temp_path


Instruction:
Here you have a function in Python 3, explain what it does
def _delete_text_ngrams(self, text_id):
        with self._conn:
            self._conn.execute(constants.DELETE_TEXT_NGRAMS_SQL, [text_id])
            self._conn.execute(constants.DELETE_TEXT_HAS_NGRAMS_SQL, [text_id])

Response:
Deletes all n - grams associated with text_id from the data
        store.


Instruction:
Make a summary of the following Python 3 code
def _diff(self, cursor, tokenizer, output_fh):
        temp_path = self._csv_temp(cursor, constants.QUERY_FIELDNAMES)
        output_fh = self._reduce_diff_results(temp_path, tokenizer, output_fh)
        try:
            os.remove(temp_path)
        except OSError as e:
            self._logger.error('Failed to remove temporary file containing '
                               'unreduced results: {}'.format(e))
        return output_fh

Response: Returns output_fh with diff results that have been reduced.


Instruction:
Can you tell what is the following Python 3 function doing
def diff(self, catalogue, tokenizer, output_fh):
        labels = self._sort_labels(self._set_labels(catalogue))
        if len(labels) < 2:
            raise MalformedQueryError(
                constants.INSUFFICIENT_LABELS_QUERY_ERROR)
        label_placeholders = self._get_placeholders(labels)
        query = constants.SELECT_DIFF_SQL.format(label_placeholders,
                                                 label_placeholders)
        parameters = labels + labels
        self._logger.info('Running diff query')
        self._logger.debug('Query: {}\nLabels: {}'.format(query, labels))
        self._log_query_plan(query, parameters)
        cursor = self._conn.execute(query, parameters)
        return self._diff(cursor, tokenizer, output_fh)

Response:
Returns output_fh populated with CSV results giving the n - grams that are unique to each set of works
        in catalogue.


Instruction:
Can you create a Python 3 function that
returns an output_fh populated with CSV results giving the difference in n - grams between the witnesses of labelled sets of works in catalogue with prime_label.

Response:
def diff_asymmetric(self, catalogue, prime_label, tokenizer, output_fh):
        """Returns `output_fh` populated with CSV results giving the
        difference in n-grams between the witnesses of labelled sets
        of works in `catalogue`, limited to those works labelled with
        `prime_label`.

        :param catalogue: catalogue matching filenames to labels
        :type catalogue: `Catalogue`
        :param prime_label: label to limit results to
        :type prime_label: `str`
        :param tokenizer: tokenizer for the n-grams
        :type tokenizer: `Tokenizer`
        :param output_fh: object to output results to
        :type output_fh: file-like object
        :rtype: file-like object

        """
        labels = list(self._set_labels(catalogue))
        if len(labels) < 2:
            raise MalformedQueryError(
                constants.INSUFFICIENT_LABELS_QUERY_ERROR)
        try:
            labels.remove(prime_label)
        except ValueError:
            raise MalformedQueryError(constants.LABEL_NOT_IN_CATALOGUE_ERROR)
        label_placeholders = self._get_placeholders(labels)
        query = constants.SELECT_DIFF_ASYMMETRIC_SQL.format(label_placeholders)
        parameters = [prime_label, prime_label] + labels
        self._logger.info('Running asymmetric diff query')
        self._logger.debug('Query: {}\nLabels: {}\nPrime label: {}'.format(
            query, labels, prime_label))
        self._log_query_plan(query, parameters)
        cursor = self._conn.execute(query, parameters)
        return self._diff(cursor, tokenizer, output_fh)


Instruction:
Can you tell what is the following Python 3 function doing
def diff_supplied(self, results_filenames, labels, tokenizer, output_fh):
        self._add_temporary_results_sets(results_filenames, labels)
        query = constants.SELECT_DIFF_SUPPLIED_SQL
        self._logger.info('Running supplied diff query')
        self._logger.debug('Query: {}'.format(query))
        self._log_query_plan(query, [])
        cursor = self._conn.execute(query)
        return self._diff(cursor, tokenizer, output_fh)

Response: Returns output_fh populated with CSV results giving the n - grams that are unique to each set of works in results_sets and the n - grams that are unique to each set of works in results_sets.


Instruction:
Make a summary of the following Python 3 code
def _drop_indices(self):
        self._logger.info('Dropping database indices')
        self._conn.execute(constants.DROP_TEXTNGRAM_INDEX_SQL)
        self._logger.info('Finished dropping database indices')

Response: Drops the database indices relating to n - grams.


Instruction:
Write a Python 3 function that can
return the database ID of the Text record for witness.

Response:
def _get_text_id(self, witness):
        """Returns the database ID of the Text record for `witness`.

        This may require creating such a record.

        If `text`\'s checksum does not match an existing record's
        checksum, the record's checksum is updated and all associated
        TextNGram and TextHasNGram records are deleted.

        :param witness: witness to add a record for
        :type witness: `WitnessText`
        :rtype: `int`

        """
        name, siglum = witness.get_names()
        text_record = self._conn.execute(constants.SELECT_TEXT_SQL,
                                         [name, siglum]).fetchone()
        if text_record is None:
            text_id = self._add_text_record(witness)
        else:
            text_id = text_record['id']
            if text_record['checksum'] != witness.get_checksum():
                filename = witness.get_filename()
                self._logger.info('Text {} has changed since it was added to '
                                  'the database'.format(filename))
                self._update_text_record(witness, text_id)
                self._logger.info('Deleting potentially out-of-date n-grams')
                self._delete_text_ngrams(text_id)
        return text_id


Instruction:
How would you explain what the following Python 3 function does
def _has_ngrams(self, text_id, size):
        if self._conn.execute(constants.SELECT_HAS_NGRAMS_SQL,
                              [text_id, size]).fetchone() is None:
            return False
        return True

Response: Returns True if a text has existing records for n - grams of size size.


Instruction:
Given the following Python 3 function, write the documentation
def _initialise_database(self):
        self._logger.info('Creating database schema, if necessary')
        self._conn.execute(constants.CREATE_TABLE_TEXT_SQL)
        self._conn.execute(constants.CREATE_TABLE_TEXTNGRAM_SQL)
        self._conn.execute(constants.CREATE_TABLE_TEXTHASNGRAM_SQL)
        self._conn.execute(constants.CREATE_INDEX_TEXTHASNGRAM_SQL)
        self._conn.execute(constants.CREATE_INDEX_TEXT_SQL)

Response: Creates the database schema.


Instruction:
Given the following Python 3 function, write the documentation
def intersection(self, catalogue, output_fh):
        labels = self._sort_labels(self._set_labels(catalogue))
        if len(labels) < 2:
            raise MalformedQueryError(
                constants.INSUFFICIENT_LABELS_QUERY_ERROR)
        label_placeholders = self._get_placeholders(labels)
        subquery = self._get_intersection_subquery(labels)
        query = constants.SELECT_INTERSECT_SQL.format(label_placeholders,
                                                      subquery)
        parameters = labels + labels
        self._logger.info('Running intersection query')
        self._logger.debug('Query: {}\nLabels: {}'.format(query, labels))
        self._log_query_plan(query, parameters)
        cursor = self._conn.execute(query, parameters)
        return self._csv(cursor, constants.QUERY_FIELDNAMES, output_fh)

Response:
Returns output_fh populated with CSV results giving the intersection in n - grams of the witnesses of labelled sets of
        works in catalogue.


Instruction:
Write a Python 3 function for
returning output_fh populated with CSV results giving the n - grams that are common to witnesses in every set of works in results_sets with the labels in labels.

Response:
def intersection_supplied(self, results_filenames, labels, output_fh):
        """Returns `output_fh` populated with CSV results giving the n-grams
        that are common to witnesses in every set of works in
        `results_sets`, using the labels in `labels`.

        :param results_filenames: list of results to be diffed
        :type results_filenames: `list` of `str`
        :param labels: labels to be applied to the results_sets
        :type labels: `list`
        :param output_fh: object to output results to
        :type output_fh: file-like object
        :rtype: file-like object

        """
        self._add_temporary_results_sets(results_filenames, labels)
        query = constants.SELECT_INTERSECT_SUPPLIED_SQL
        parameters = [len(labels)]
        self._logger.info('Running supplied intersect query')
        self._logger.debug('Query: {}\nNumber of labels: {}'.format(
            query, parameters[0]))
        self._log_query_plan(query, parameters)
        cursor = self._conn.execute(query, parameters)
        return self._csv(cursor, constants.QUERY_FIELDNAMES, output_fh)


Instruction:
Create a Python 3 function for
reducing the diff results from one set of n - grams to another.

Response:
def _reduce_diff_results(self, matches_path, tokenizer, output_fh):
        """Returns `output_fh` populated with a reduced set of data from
        `matches_fh`.

        Diff results typically contain a lot of filler results that
        serve only to hide real differences. If one text has a single
        extra token than another, the diff between them will have
        results for every n-gram containing that extra token, which is
        not helpful. This method removes these filler results by
        'reducing down' the results.

        :param matches_path: filepath or buffer of CSV results to be reduced
        :type matches_path: `str` or file-like object
        :param tokenizer: tokenizer for the n-grams
        :type tokenizer: `Tokenizer`
        :param output_fh: object to write results to
        :type output_fh: file-like object
        :rtype: file-like object

        """
        self._logger.info('Removing filler results')
        # For performance, perform the attribute accesses once.
        tokenize = tokenizer.tokenize
        join = tokenizer.joiner.join
        results = []
        previous_witness = (None, None)
        previous_data = {}
        # Calculate the index of ngram and count columns in a Pandas
        # named tuple row, as used below. The +1 is due to the tuple
        # having the row index as the first element.
        ngram_index = constants.QUERY_FIELDNAMES.index(
            constants.NGRAM_FIELDNAME) + 1
        count_index = constants.QUERY_FIELDNAMES.index(
            constants.COUNT_FIELDNAME) + 1
        # Operate over individual witnesses and sizes, so that there
        # is no possible results pollution between them.
        grouped = pd.read_csv(matches_path, encoding='utf-8',
                              na_filter=False).groupby(
            [constants.WORK_FIELDNAME, constants.SIGLUM_FIELDNAME,
             constants.SIZE_FIELDNAME])
        for (work, siglum, size), group in grouped:
            if (work, siglum) != previous_witness:
                previous_matches = group
                previous_witness = (work, siglum)
            else:
                self._logger.debug(
                    'Reducing down {} {}-grams for {} {}'.format(
                        len(group.index), size, work, siglum))
                if previous_matches.empty:
                    reduced_count = 0
                else:
                    previous_matches = group.apply(
                        self._check_diff_result, axis=1,
                        args=(previous_data, tokenize, join))
                    reduced_count = len(previous_matches[previous_matches[
                                        constants.COUNT_FIELDNAME] != 0].index)
                self._logger.debug('Reduced down to {} grams'.format(
                    reduced_count))
            # Put the previous matches into a form that is more
            # performant for the lookups made in _check_diff_result.
            previous_data = {}
            for row in previous_matches.itertuples():
                previous_data[row[ngram_index]] = row[count_index]
            if not previous_matches.empty:
                results.append(previous_matches[previous_matches[
                    constants.COUNT_FIELDNAME] != 0])
        reduced_results = pd.concat(results, ignore_index=True).reindex(
            columns=constants.QUERY_FIELDNAMES)
        reduced_results.to_csv(output_fh, encoding='utf-8', float_format='%d',
                               index=False)
        return output_fh


Instruction:
Write a Python 3 script for
returning output_fh populated with CSV results for each n - gram inn - gram inn - gram in catalogue that occurs within labelled witnesses in catalogue.

Response:
def search(self, catalogue, ngrams, output_fh):
        """Returns `output_fh` populated with CSV results for each n-gram in
        `ngrams` that occurs within labelled witnesses in `catalogue`.

        If `ngrams` is empty, include all n-grams.

        :param catalogue: catalogue matching filenames to labels
        :type catalogue: `Catalogue`
        :param ngrams: n-grams to search for
        :type ngrams: `list` of `str`
        :param output_fh: object to write results to
        :type output_fh: file-like object
        :rtype: file-like object

        """
        labels = list(self._set_labels(catalogue))
        label_placeholders = self._get_placeholders(labels)
        if ngrams:
            self._add_temporary_ngrams(ngrams)
            query = constants.SELECT_SEARCH_SQL.format(label_placeholders)
        else:
            query = constants.SELECT_SEARCH_ALL_SQL.format(label_placeholders)
        self._logger.info('Running search query')
        self._logger.debug('Query: {}\nN-grams: {}'.format(
            query, ', '.join(ngrams)))
        self._log_query_plan(query, labels)
        cursor = self._conn.execute(query, labels)
        return self._csv(cursor, constants.QUERY_FIELDNAMES, output_fh)


Instruction:
How would you code a function in Python 3 to
return a dictionary of the unique labels in catalogue and the count of all tokens associated with each Text.

Response:
def _set_labels(self, catalogue):
        """Returns a dictionary of the unique labels in `catalogue` and the
        count of all tokens associated with each, and sets the record
        of each Text to its corresponding label.

        Texts that do not have a label specified are set to the empty
        string.

        Token counts are included in the results to allow for
        semi-accurate sorting based on corpora size.

        :param catalogue: catalogue matching filenames to labels
        :type catalogue: `Catalogue`
        :rtype: `dict`

        """
        with self._conn:
            self._conn.execute(constants.UPDATE_LABELS_SQL, [''])
            labels = {}
            for work, label in catalogue.items():
                self._conn.execute(constants.UPDATE_LABEL_SQL, [label, work])
                cursor = self._conn.execute(
                    constants.SELECT_TEXT_TOKEN_COUNT_SQL, [work])
                token_count = cursor.fetchone()['token_count']
                labels[label] = labels.get(label, 0) + token_count
        return labels


Instruction:
Can you generate the documentation for the following Python 3 function
def _sort_labels(label_data):
        labels = list(label_data)
        labels.sort(key=label_data.get, reverse=True)
        return labels

Response:
Returns the labels in label_data sorted in descending order
        according to the size of their token counts.


Instruction:
Can you implement a function in Python 3 that
updates the record with the given text_id with the given witness.

Response:
def _update_text_record(self, witness, text_id):
        """Updates the record with `text_id` with `witness`\'s checksum and
        token count.

        :param withness: witness to update from
        :type witness: `WitnessText`
        :param text_id: database ID of Text record
        :type text_id: `int`

        """
        checksum = witness.get_checksum()
        token_count = len(witness.get_tokens())
        with self._conn:
            self._conn.execute(constants.UPDATE_TEXT_SQL,
                               [checksum, token_count, text_id])


Instruction:
Make a summary of the following Python 3 code
def validate(self, corpus, catalogue):
        is_valid = True
        for name in catalogue:
            count = 0
            # It is unfortunate that this creates WitnessText objects
            # for each work, since that involves reading the file.
            for witness in corpus.get_witnesses(name):
                count += 1
                name, siglum = witness.get_names()
                filename = witness.get_filename()
                row = self._conn.execute(constants.SELECT_TEXT_SQL,
                                         [name, siglum]).fetchone()
                if row is None:
                    is_valid = False
                    self._logger.warning(
                        'No record (or n-grams) exists for {} in '
                        'the database'.format(filename))
                elif row['checksum'] != witness.get_checksum():
                    is_valid = False
                    self._logger.warning(
                        '{} has changed since its n-grams were '
                        'added to the database'.format(filename))
            if count == 0:
                raise FileNotFoundError(
                    constants.CATALOGUE_WORK_NOT_IN_CORPUS_ERROR.format(
                        name))
        return is_valid

Response: Returns True if all of the files labelled in catalogue are up - to - date in the database.


Instruction:
Implement a Python 3 function for
formatting the alignment a1 with HTML spans around characters that are also at the same index in a2.

Response:
def _format_alignment(self, a1, a2):
        """Returns `a1` marked up with HTML spans around characters that are
        also at the same index in `a2`.

        :param a1: text sequence from one witness
        :type a1: `str`
        :param a2: text sequence from another witness
        :type a2: `str`
        :rtype: `str`

        """
        html = []
        for index, char in enumerate(a1):
            output = self._substitutes.get(char, char)
            if a2[index] == char:
                html.append('{}'.format(output))
            elif char != '-':
                html.append(output)
        return ''.join(html)


Instruction:
How would you implement a function in Python 3 that
returns a tuple of HTML fragments rendering each element of the sequence.

Response:
def render(self):
        """Returns a tuple of HTML fragments rendering each element of the
        sequence."""
        f1 = self._format_alignment(self._alignment[0], self._alignment[1])
        f2 = self._format_alignment(self._alignment[1], self._alignment[0])
        return f1, f2


Instruction:
Write a Python 3 script for
generating sequences for the n - grams in the n - grams of the n - grams in the n - grams of the n - grams in the n - grams of the n - grams in the n - grams of the n - grams in the n - grams of the n - grams in the n - grams of the n - grams in the n - grams of the n - grams in the n - grams.

Response:
def generate(self, output_dir, minimum_size):
        """Generates sequence reports and writes them to the output directory.

        :param output_dir: directory to output reports to
        :type output_dir: `str`
        :param minimum_size: minimum size of n-grams to create sequences for
        :type minimum_size: `int`

        """
        self._output_dir = output_dir
        # Get a list of the files in the matches, grouped by label
        # (ordered by number of works).
        labels = list(self._matches.groupby([constants.LABEL_FIELDNAME])[
            constants.WORK_FIELDNAME].nunique().index)
        original_ngrams = self._matches[
            self._matches[
                constants.SIZE_FIELDNAME] >= minimum_size].sort_values(
                by=constants.SIZE_FIELDNAME, ascending=False)[
                    constants.NGRAM_FIELDNAME].unique()
        ngrams = []
        for original_ngram in original_ngrams:
            ngrams.append(self._get_text(Text(original_ngram,
                                              self._tokenizer)))
        # Generate sequences for each witness in every combination of
        # (different) labels.
        for index, primary_label in enumerate(labels):
            for secondary_label in labels[index+1:]:
                self._generate_sequences(primary_label, secondary_label,
                                         ngrams)


Instruction:
Can you generate the documentation for the following Python 3 function
def _generate_sequence(self, t1, t1_span, t2, t2_span, context_length,
                           covered_spans):
        """Returns an aligned sequence (or None) between extract `t1_span` of
        witness text `t1` and extract `t2_span` of witness text `t2`.

        Adds the span of the aligned sequence, if any, to
        `covered_spans` (which, being mutable, is modified in place
        without needing to be returned).

        This method repeats the alignment process, increasing the
        context length until the alignment score (the measure of how
        much the two sequences align) drops below a certain point or
        it is not possible to increase the context length.

        :param t1: text content of first witness
        :type t1: `str`
        :param t1_span: start and end indices within `t1` to align
        :type t1_span: 2-`tuple` of `int`
        :param t2: text content of second witness
        :type t2: `str`
        :param t2_span: start and end indices within `t2` to align
        :type t2_span: 2-`tuple` of `int`
        :param context_length: length of context on either side of
                               the spans to include in the sequence
        :type context_length: `int`
        :param covered_spans: lists of start and end indices for parts
                              of the texts already covered by a sequence
        :type covered_spans: `list` of two `list`s of 2-`tuple` of `int`

        """
        old_length = 0
        self._logger.debug('Match found; generating new sequence')
        while True:
            s1, span1 = self._get_text_sequence(t1, t1_span, context_length)
            s2, span2 = self._get_text_sequence(t2, t2_span, context_length)
            length = len(s1)
            alignment = pairwise2.align.globalms(
                s1, s2, constants.IDENTICAL_CHARACTER_SCORE,
                constants.DIFFERENT_CHARACTER_SCORE,
                constants.OPEN_GAP_PENALTY, constants.EXTEND_GAP_PENALTY)[0]
            context_length = length
            score = alignment[2] / length
            if not alignment:
                return None
            elif score < constants.SCORE_THRESHOLD or length == old_length:
                break
            else:
                self._logger.debug('Score: {}'.format(score))
            old_length = length
        covered_spans[0].append(span1)
        covered_spans[1].append(span2)
        return Sequence(alignment, self._reverse_substitutes, t1_span[0])

Response: Returns an aligned sequence between extract t1_span and extract t2_span of witness text t1 and extract t2_span of witness text t2.


Instruction:
Implement a Python 3 function for
generating aligned sequences between each witness labelled primary_label and each witness labelled secondary_label based around ngrams.

Response:
def _generate_sequences(self, primary_label, secondary_label, ngrams):
        """Generates aligned sequences between each witness labelled
        `primary_label` and each witness labelled `secondary_label`,
        based around `ngrams`.

        :param primary_label: label for one side of the pairs of
                              witnesses to align
        :type primary_label: `str`
        :param secondary_label: label for the other side of the pairs
                                of witnesses to align
        :type secondary_label: `str`
        :param ngrams: n-grams to base sequences off
        :type ngrams: `list` of `str`

        """
        cols = [constants.WORK_FIELDNAME, constants.SIGLUM_FIELDNAME]
        primary_works = self._matches[self._matches[
            constants.LABEL_FIELDNAME] == primary_label][
                cols].drop_duplicates()
        secondary_works = self._matches[self._matches[
            constants.LABEL_FIELDNAME] == secondary_label][
                cols].drop_duplicates()
        for index, (work1, siglum1) in primary_works.iterrows():
            text1 = self._get_text(self._corpus.get_witness(work1, siglum1))
            label1 = '{}_{}'.format(work1, siglum1)
            for index, (work2, siglum2) in secondary_works.iterrows():
                text2 = self._get_text(self._corpus.get_witness(
                    work2, siglum2))
                label2 = '{}_{}'.format(work2, siglum2)
                self._generate_sequences_for_texts(label1, text1, label2,
                                                   text2, ngrams)