Instruction:
How would you code a function in Python 3 to
apply contextual rules to the given list of tokens.

Response:
def apply(self, tokens):
        """ Applies contextual rules to the given list of tokens,
            where each token is a [word, tag] list.
        """
        o = [("STAART", "STAART")] * 3 # Empty delimiters for look ahead/back.
        t = o + tokens + o
        for i, token in enumerate(t):
            for r in self:
                if token[1] == "STAART":
                    continue
                if token[1] != r[0] and r[0] != "*":
                    continue
                cmd, x, y = r[2], r[3], r[4] if len(r) > 4 else ""
                cmd = cmd.lower()
                if (cmd == "prevtag"        and x ==  t[i-1][1]) \
                or (cmd == "nexttag"        and x ==  t[i+1][1]) \
                or (cmd == "prev2tag"       and x ==  t[i-2][1]) \
                or (cmd == "next2tag"       and x ==  t[i+2][1]) \
                or (cmd == "prev1or2tag"    and x in (t[i-1][1], t[i-2][1])) \
                or (cmd == "next1or2tag"    and x in (t[i+1][1], t[i+2][1])) \
                or (cmd == "prev1or2or3tag" and x in (t[i-1][1], t[i-2][1], t[i-3][1])) \
                or (cmd == "next1or2or3tag" and x in (t[i+1][1], t[i+2][1], t[i+3][1])) \
                or (cmd == "surroundtag"    and x ==  t[i-1][1] and y == t[i+1][1]) \
                or (cmd == "curwd"          and x ==  t[i+0][0]) \
                or (cmd == "prevwd"         and x ==  t[i-1][0]) \
                or (cmd == "nextwd"         and x ==  t[i+1][0]) \
                or (cmd == "prev1or2wd"     and x in (t[i-1][0], t[i-2][0])) \
                or (cmd == "next1or2wd"     and x in (t[i+1][0], t[i+2][0])) \
                or (cmd == "prevwdtag"      and x ==  t[i-1][0] and y == t[i-1][1]) \
                or (cmd == "nextwdtag"      and x ==  t[i+1][0] and y == t[i+1][1]) \
                or (cmd == "wdprevtag"      and x ==  t[i-1][1] and y == t[i+0][0]) \
                or (cmd == "wdnexttag"      and x ==  t[i+0][0] and y == t[i+1][1]) \
                or (cmd == "wdand2aft"      and x ==  t[i+0][0] and y == t[i+2][0]) \
                or (cmd == "wdand2tagbfr"   and x ==  t[i-2][1] and y == t[i+0][0]) \
                or (cmd == "wdand2tagaft"   and x ==  t[i+0][0] and y == t[i+2][1]) \
                or (cmd == "lbigram"        and x ==  t[i-1][0] and y == t[i+0][0]) \
                or (cmd == "rbigram"        and x ==  t[i+0][0] and y == t[i+1][0]) \
                or (cmd == "prevbigram"     and x ==  t[i-2][1] and y == t[i-1][1]) \
                or (cmd == "nextbigram"     and x ==  t[i+1][1] and y == t[i+2][1]):
                    t[i] = [t[i][0], r[1]]
        return t[len(o):-len(o)]


Instruction:
Given the following Python 3 function, write the documentation
def insert(self, i, tag1, tag2, cmd="prevtag", x=None, y=None):
        if " < " in tag1 and not x and not y:
            tag1, x = tag1.split(" < "); cmd="prevtag"
        if " > " in tag1 and not x and not y:
            x, tag1 = tag1.split(" > "); cmd="nexttag"
        lazylist.insert(self, i, [tag1, tag2, cmd, x or "", y or ""])

Response: Inserts a new rule that updates words with tag1 to tag2 given constraints x and y.


Instruction:
Can you create a Python 3 function that
applies the named entity recognizer to the given list of tokens.

Response:
def apply(self, tokens):
        """ Applies the named entity recognizer to the given list of tokens,
            where each token is a [word, tag] list.
        """
        # Note: we could also scan for patterns, e.g.,
        # "my|his|her name is|was *" => NNP-PERS.
        i = 0
        while i < len(tokens):
            w = tokens[i][0].lower()
            if RE_ENTITY1.match(w) \
            or RE_ENTITY2.match(w) \
            or RE_ENTITY3.match(w):
                tokens[i][1] = self.tag
            if w in self:
                for e in self[w]:
                    # Look ahead to see if successive words match the named entity.
                    e, tag = (e[:-1], "-"+e[-1].upper()) if e[-1] in self._cmd else (e, "")
                    b = True
                    for j, e in enumerate(e):
                        if i + j >= len(tokens) or tokens[i+j][0].lower() != e:
                            b = False; break
                    if b:
                        for token in tokens[i:i+j+1]:
                            token[1] = token[1] if token[1].startswith(self.tag) else self.tag
                            token[1] += tag
                        i += j
                        break
            i += 1
        return tokens


Instruction:
Write a Python 3 function that can
append a named entity to the lexicon.

Response:
def append(self, entity, name="pers"):
        """ Appends a named entity to the lexicon,
            e.g., Entities.append("Hooloovoo", "PERS")
        """
        e = map(lambda s: s.lower(), entity.split(" ") + [name])
        self.setdefault(e[0], []).append(e)


Instruction:
Write a Python 3 script to
return a sorted list of keywords in the given string.

Response:
def find_keywords(self, string, **kwargs):
        """ Returns a sorted list of keywords in the given string.
        """
        return find_keywords(string,
                     parser = self,
                        top = kwargs.pop("top", 10),
                  frequency = kwargs.pop("frequency", {}), **kwargs
        )


Instruction:
Can you generate the documentation for the following Python 3 function
def find_tokens(self, string, **kwargs):
        # "The cat purs." => ["The cat purs ."]
        return find_tokens(string,
                punctuation = kwargs.get(  "punctuation", PUNCTUATION),
              abbreviations = kwargs.get("abbreviations", ABBREVIATIONS),
                    replace = kwargs.get(      "replace", replacements),
                  linebreak = r"\n{2,}")

Response: Returns a list of sentences from the given string.


Instruction:
How would you code a function in Python 3 to
annotate the given list of tokens with part - of - speech tags.

Response:
def find_tags(self, tokens, **kwargs):
        """ Annotates the given list of tokens with part-of-speech tags.
            Returns a list of tokens, where each token is now a [word, tag]-list.
        """
        # ["The", "cat", "purs"] => [["The", "DT"], ["cat", "NN"], ["purs", "VB"]]
        return find_tags(tokens,
                    lexicon = kwargs.get(   "lexicon", self.lexicon or {}),
                      model = kwargs.get(     "model", self.model),
                 morphology = kwargs.get("morphology", self.morphology),
                    context = kwargs.get(   "context", self.context),
                   entities = kwargs.get(  "entities", self.entities),
                   language = kwargs.get(  "language", self.language),
                    default = kwargs.get(   "default", self.default),
                        map = kwargs.get(       "map", None))


Instruction:
Create a Python 3 function for
annotating the given list of tokens with chunk tags. Several tags can be added for example chunk + preposition tags.

Response:
def find_chunks(self, tokens, **kwargs):
        """ Annotates the given list of tokens with chunk tags.
            Several tags can be added, for example chunk + preposition tags.
        """
        # [["The", "DT"], ["cat", "NN"], ["purs", "VB"]] =>
        # [["The", "DT", "B-NP"], ["cat", "NN", "I-NP"], ["purs", "VB", "B-VP"]]
        return find_prepositions(
               find_chunks(tokens,
                   language = kwargs.get("language", self.language)))


Instruction:
How would you explain what the following Python 3 function does
def parse(self, s, tokenize=True, tags=True, chunks=True, relations=False, lemmata=False, encoding="utf-8", **kwargs):
        # Tokenizer.
        if tokenize is True:
            s = self.find_tokens(s, **kwargs)
        if isinstance(s, (list, tuple)):
            s = [isinstance(s, basestring) and s.split(" ") or s for s in s]
        if isinstance(s, basestring):
            s = [s.split(" ") for s in s.split("\n")]
        # Unicode.
        for i in range(len(s)):
            for j in range(len(s[i])):
                if isinstance(s[i][j], str):
                    s[i][j] = decode_string(s[i][j], encoding)
            # Tagger (required by chunker, labeler & lemmatizer).
            if tags or chunks or relations or lemmata:
                s[i] = self.find_tags(s[i], **kwargs)
            else:
                s[i] = [[w] for w in s[i]]
            # Chunker.
            if chunks or relations:
                s[i] = self.find_chunks(s[i], **kwargs)
            # Labeler.
            if relations:
                s[i] = self.find_labels(s[i], **kwargs)
            # Lemmatizer.
            if lemmata:
                s[i] = self.find_lemmata(s[i], **kwargs)
        # Slash-formatted tagged string.
        # With collapse=False (or split=True), returns raw list
        # (this output is not usable by tree.Text).
        if not kwargs.get("collapse", True) \
            or kwargs.get("split", False):
            return s
        # Construct TaggedString.format.
        # (this output is usable by tree.Text).
        format = ["word"]
        if tags:
            format.append("part-of-speech")
        if chunks:
            format.extend(("chunk", "preposition"))
        if relations:
            format.append("relation")
        if lemmata:
            format.append("lemma")
        # Collapse raw list.
        # Sentences are separated by newlines, tokens by spaces, tags by slashes.
        # Slashes in words are encoded with &slash;
        for i in range(len(s)):
            for j in range(len(s[i])):
                s[i][j][0] = s[i][j][0].replace("/", "&slash;")
                s[i][j] = "/".join(s[i][j])
            s[i] = " ".join(s[i])
        s = "\n".join(s)
        s = TaggedString(s, format, language=kwargs.get("language", self.language))
        return s

Response: Takes a string and returns a tagged Unicode string.


Instruction:
Make a summary of the following Python 3 code
def split(self, sep=TOKENS):
        if sep != TOKENS:
            return unicode.split(self, sep)
        if len(self) == 0:
            return []
        return [[[x.replace("&slash;", "/") for x in token.split("/")]
            for token in sentence.split(" ")]
                for sentence in unicode.split(self, "\n")]

Response: Returns a list of sentences where each sentence is a list of tokens where each token is a list of word + tags.


Instruction:
Make a summary of the following Python 3 code
def TENSES(self):
        a = set(TENSES[id] for id in self._format)
        a = a.union(set(TENSES[id] for id in self._default.keys()))
        a = a.union(set(TENSES[id] for id in self._default.values()))
        a = sorted(x[:-2] for x in a if x[-2] is False) # Exclude negation.
        return a

Response: Yields a list of tenses for this language excluding negations.


Instruction:
Write a Python 3 function that can
return the infinitive form of the given verb or None if no such verb exists.

Response:
def lemma(self, verb, parse=True):
        """ Returns the infinitive form of the given verb, or None.
        """
        if dict.__len__(self) == 0:
            self.load()
        if verb.lower() in self._inverse:
            return self._inverse[verb.lower()]
        if verb in self._inverse:
            return self._inverse[verb]
        if parse is True: # rule-based
            return self.find_lemma(verb)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def lexeme(self, verb, parse=True):
        a = []
        b = self.lemma(verb, parse=parse)
        if b in self:
            a = [x for x in self[b] if x != ""]
        elif parse is True: # rule-based
            a = self.find_lexeme(b)
        u = []; [u.append(x) for x in a if x not in u]
        return u

Response: Returns a list of all possible inflections of the given verb.


Instruction:
Write a Python 3 script to
inflect the verb and returns the given tense.

Response:
def conjugate(self, verb, *args, **kwargs):
        """ Inflects the verb and returns the given tense (or None).
            For example: be
            - Verbs.conjugate("is", INFINITVE) => be
            - Verbs.conjugate("be", PRESENT, 1, SINGULAR) => I am
            - Verbs.conjugate("be", PRESENT, 1, PLURAL) => we are
            - Verbs.conjugate("be", PAST, 3, SINGULAR) => he was
            - Verbs.conjugate("be", PAST, aspect=PROGRESSIVE) => been
            - Verbs.conjugate("be", PAST, person=1, negated=True) => I wasn't
        """
        id = tense_id(*args, **kwargs)
        # Get the tense index from the format description (or a default).
        i1 = self._format.get(id)
        i2 = self._format.get(self._default.get(id))
        i3 = self._format.get(self._default.get(self._default.get(id)))
        b = self.lemma(verb, parse=kwargs.get("parse", True))
        v = []
        # Get the verb lexeme and return the requested index.
        if b in self:
            v = self[b]
            for i in (i1, i2, i3):
                if i is not None and 0 <= i < len(v) and v[i]:
                    return v[i]
        if kwargs.get("parse", True) is True: # rule-based
            v = self.find_lexeme(b)
            for i in (i1, i2, i3):
                if i is not None and 0 <= i < len(v) and v[i]:
                    return v[i]


Instruction:
Implement a function in Python 3 to
return a list of possible tenses for the given verb.

Response:
def tenses(self, verb, parse=True):
        """ Returns a list of possible tenses for the given inflected verb.
        """
        verb = verb.lower()
        a = set()
        b = self.lemma(verb, parse=parse)
        v = []
        if b in self:
            v = self[b]
        elif parse is True: # rule-based
            v = self.find_lexeme(b)
        # For each tense in the verb lexeme that matches the given tense,
        # 1) retrieve the tense tuple,
        # 2) retrieve the tense tuples for which that tense is a default.
        for i, tense in enumerate(v):
            if tense == verb:
                for id, index in self._format.items():
                    if i == index:
                        a.add(id)
                for id1, id2 in self._default.items():
                    if id2 in a:
                        a.add(id1)
                for id1, id2 in self._default.items():
                    if id2 in a:
                        a.add(id1)
        a = (TENSES[id][:-2] for id in a)
        a = Tenses(sorted(a))
        return a


Instruction:
How would you explain what the following Python 3 function does
def load(self, path=None):
        # <word form="great" wordnet_id="a-01123879" pos="JJ" polarity="1.0" subjectivity="1.0" intensity="1.0" />
        # <word form="damnmit" polarity="-0.75" subjectivity="1.0" label="profanity" />
        if not path:
            path = self._path
        if not os.path.exists(path):
            return
        words, synsets, labels = {}, {}, {}
        xml = cElementTree.parse(path)
        xml = xml.getroot()
        for w in xml.findall("word"):
            if self._confidence is None \
            or self._confidence <= float(w.attrib.get("confidence", 0.0)):
                w, pos, p, s, i, label, synset = (
                    w.attrib.get("form"),
                    w.attrib.get("pos"),
                    w.attrib.get("polarity", 0.0),
                    w.attrib.get("subjectivity", 0.0),
                    w.attrib.get("intensity", 1.0),
                    w.attrib.get("label"),
                    w.attrib.get(self._synset) # wordnet_id, cornetto_id, ...
                )
                psi = (float(p), float(s), float(i))
                if w:
                    words.setdefault(w, {}).setdefault(pos, []).append(psi)
                if w and label:
                    labels[w] = label
                if synset:
                    synsets.setdefault(synset, []).append(psi)
        self._language = xml.attrib.get("language", self._language)
        # Average scores of all word senses per part-of-speech tag.
        for w in words:
            words[w] = dict((pos, [avg(each) for each in zip(*psi)]) for pos, psi in words[w].items())
        # Average scores of all part-of-speech tags.
        for w, pos in list(words.items()):
            words[w][None] = [avg(each) for each in zip(*pos.values())]
        # Average scores of all synonyms per synset.
        for id, psi in synsets.items():
            synsets[id] = [avg(each) for each in zip(*psi)]
        dict.update(self, words)
        dict.update(self.labeler, labels)
        dict.update(self._synsets, synsets)

Response: Loads the Sentiment. xml file with sentiment annotations.


Instruction:
How would you code a function in Python 3 to
return a tuple of polarity subjectivity - tuple for the given synset id.

Response:
def synset(self, id, pos=ADJECTIVE):
        """ Returns a (polarity, subjectivity)-tuple for the given synset id.
            For example, the adjective "horrible" has id 193480 in WordNet:
            Sentiment.synset(193480, pos="JJ") => (-0.6, 1.0, 1.0).
        """
        id = str(id).zfill(8)
        if not id.startswith(("n-", "v-", "a-", "r-")):
            if pos == NOUN:
                id = "n-" + id
            if pos == VERB:
                id = "v-" + id
            if pos == ADJECTIVE:
                id = "a-" + id
            if pos == ADVERB:
                id = "r-" + id
        if dict.__len__(self) == 0:
            self.load()
        try:
            return tuple(self._synsets[id])[:2]
        except KeyError: # Some WordNet id's are not zero padded.
            return tuple(self._synsets.get(re.sub(r"-0+", "-", id), (0.0, 0.0))[:2])


Instruction:
Can you tell what is the following Python 3 function doing
def assessments(self, words=[], negation=True):
        a = []
        m = None # Preceding modifier (i.e., adverb or adjective).
        n = None # Preceding negation (e.g., "not beautiful").
        for w, pos in words:
            # Only assess known words, preferably by part-of-speech tag.
            # Including unknown words (polarity 0.0 and subjectivity 0.0) lowers the average.
            if w is None:
                continue
            if w in self and pos in self[w]:
                p, s, i = self[w][pos]
                # Known word not preceded by a modifier ("good").
                if m is None:
                    a.append(dict(w=[w], p=p, s=s, i=i, n=1, x=self.labeler.get(w)))
                # Known word preceded by a modifier ("really good").
                if m is not None:
                    a[-1]["w"].append(w)
                    a[-1]["p"] = max(-1.0, min(p * a[-1]["i"], +1.0))
                    a[-1]["s"] = max(-1.0, min(s * a[-1]["i"], +1.0))
                    a[-1]["i"] = i
                    a[-1]["x"] = self.labeler.get(w)
                # Known word preceded by a negation ("not really good").
                if n is not None:
                    a[-1]["w"].insert(0, n)
                    a[-1]["i"] = 1.0 / a[-1]["i"]
                    a[-1]["n"] = -1
                # Known word may be a negation.
                # Known word may be modifying the next word (i.e., it is a known adverb).
                m = None
                n = None
                if pos and pos in self.modifiers or any(map(self[w].__contains__, self.modifiers)):
                    m = (w, pos)
                if negation and w in self.negations:
                    n = w
            else:
                # Unknown word may be a negation ("not good").
                if negation and w in self.negations:
                    n = w
                # Unknown word. Retain negation across small words ("not a good").
                elif n and len(w.strip("'")) > 1:
                    n = None
                # Unknown word may be a negation preceded by a modifier ("really not good").
                if n is not None and m is not None and (pos in self.modifiers or self.modifier(m[0])):
                    a[-1]["w"].append(n)
                    a[-1]["n"] = -1
                    n = None
                # Unknown word. Retain modifier across small words ("really is a good").
                elif m and len(w) > 2:
                    m = None
                # Exclamation marks boost previous word.
                if w == "!" and len(a) > 0:
                    a[-1]["w"].append("!")
                    a[-1]["p"] = max(-1.0, min(a[-1]["p"] * 1.25, +1.0))
                # Exclamation marks in parentheses indicate sarcasm.
                if w == "(!)":
                    a.append(dict(w=[w], p=0.0, s=1.0, i=1.0, n=1, x=IRONY))
                # EMOTICONS: {("grin", +1.0): set((":-D", ":D"))}
                if w.isalpha() is False and len(w) <= 5 and w not in PUNCTUATION: # speedup
                    for (type, p), e in EMOTICONS.items():
                        if w in map(lambda e: e.lower(), e):
                            a.append(dict(w=[w], p=p, s=1.0, i=1.0, n=1, x=MOOD))
                            break
        for i in range(len(a)):
            w = a[i]["w"]
            p = a[i]["p"]
            s = a[i]["s"]
            n = a[i]["n"]
            x = a[i]["x"]
            # "not good" = slightly bad, "not bad" = slightly good.
            a[i] = (w, p * -0.5 if n < 0 else p, s, x)
        return a

Response: Returns a list of tuples for the given list of words.


Instruction:
Can you tell what is the following Python 3 function doing
def annotate(self, word, pos=None, polarity=0.0, subjectivity=0.0, intensity=1.0, label=None):
        w = self.setdefault(word, {})
        w[pos] = w[None] = (polarity, subjectivity, intensity)
        if label:
            self.labeler[word] = label

Response: Annotates the given word with polarity subjectivity and intensity scores.


Instruction:
Can you tell what is the following Python 3 function doing
def train(self, s, path="spelling.txt"):
        model = {}
        for w in re.findall("[a-z]+", s.lower()):
            model[w] = w in model and model[w] + 1 or 1
        model = ("%s %s" % (k, v) for k, v in sorted(model.items()))
        model = "\n".join(model)
        f = open(path, "w")
        f.write(model)
        f.close()

Response: Counts the words in the given string and saves the probabilities at the given path.


Instruction:
Write a Python 3 script to
return a set of words with edit distance 1 from the given word.

Response:
def _edit1(self, w):
        """ Returns a set of words with edit distance 1 from the given word.
        """
        # Of all spelling errors, 80% is covered by edit distance 1.
        # Edit distance 1 = one character deleted, swapped, replaced or inserted.
        split = [(w[:i], w[i:]) for i in range(len(w) + 1)]
        delete, transpose, replace, insert = (
            [a + b[1:] for a, b in split if b],
            [a + b[1] + b[0] + b[2:] for a, b in split if len(b) > 1],
            [a + c + b[1:] for a, b in split for c in Spelling.ALPHA if b],
            [a + c + b[0:] for a, b in split for c in Spelling.ALPHA]
        )
        return set(delete + transpose + replace + insert)


Instruction:
Write a Python 3 function that can
return a set of words with edit distance 2 from the given word.

Response:
def _edit2(self, w):
        """ Returns a set of words with edit distance 2 from the given word
        """
        # Of all spelling errors, 99% is covered by edit distance 2.
        # Only keep candidates that are actually known words (20% speedup).
        return set(e2 for e1 in self._edit1(w) for e2 in self._edit1(e1) if e2 in self)


Instruction:
Can you generate the documentation for the following Python 3 function
def suggest(self, w):
        if len(self) == 0:
            self.load()
        if len(w) == 1:
            return [(w, 1.0)] # I
        if w in PUNCTUATION:
            return [(w, 1.0)] # .?!
        if w.replace(".", "").isdigit():
            return [(w, 1.0)] # 1.5
        candidates = self._known([w]) \
                  or self._known(self._edit1(w)) \
                  or self._known(self._edit2(w)) \
                  or [w]
        candidates = [(self.get(c, 0.0), c) for c in candidates]
        s = float(sum(p for p, w in candidates) or 1)
        candidates = sorted(((p / s, w) for p, w in candidates), reverse=True)
        candidates = [(w.istitle() and x.title() or x, p) for p, x in candidates] # case-sensitive
        return candidates

Response: Return a list of ( word confidence ) spelling corrections for the given word.


Instruction:
Can you implement a function in Python 3 that
returns a list of tuples where the i - th tuple contains the i - th element from each of the argument sequences or iterables.

Response:
def zip(*args, **kwargs):
    """ Returns a list of tuples, where the i-th tuple contains the i-th element
        from each of the argument sequences or iterables (or default if too short).
    """
    args = [list(iterable) for iterable in args]
    n = max(map(len, args))
    v = kwargs.get("default", None)
    return _zip(*[i + [v] * (n - len(i)) for i in args])


Instruction:
Explain what the following Python 3 code does
def chunked(sentence):
    # For example, to construct a training vector with the head of previous chunks as a feature.
    # Doing this with Sentence.chunks would discard the punctuation marks and conjunctions
    # (Sentence.chunks only yields Chunk objects), which amy be useful features.
    chunks = []
    for word in sentence:
        if word.chunk is not None:
            if len(chunks) == 0 or chunks[-1] != word.chunk:
                chunks.append(word.chunk)
        else:
            ch = Chink(sentence)
            ch.append(word.copy(ch))
            chunks.append(ch)
    return chunks

Response: Returns a list of Chunk and Chink objects from the given sentence.


Instruction:
Given the following Python 3 function, write the documentation
def tree(string, token=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):
    return Text(string, token)

Response: Transforms the output of parse into a Text object.


Instruction:
How would you code a function in Python 3 to
return the string with XML - safe special characters.

Response:
def xml_encode(string):
    """ Returns the string with XML-safe special characters.
    """
    string = string.replace("&", "&")
    string = string.replace("<", "<")
    string = string.replace(">", ">")
    string = string.replace("\"",""")
    string = string.replace(SLASH, "/")
    return string


Instruction:
How would you explain what the following Python 3 function does
def xml_decode(string):
    string = string.replace("&", "&")
    string = string.replace("<",  "<")
    string = string.replace(">",  ">")
    string = string.replace(""","\"")
    string = string.replace("/", SLASH)
    return string

Response: Returns the string with special characters decoded.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_xml(sentence, tab="\t", id=""):
    uid  = lambda *parts: "".join([str(id), _UID_SEPARATOR ]+[str(x) for x in parts]).lstrip(_UID_SEPARATOR)
    push = lambda indent: indent+tab         # push() increases the indentation.
    pop  = lambda indent: indent[:-len(tab)] # pop() decreases the indentation.
    indent = tab
    xml = []
    # Start the sentence element:
    # <sentence token="word, part-of-speech, chunk, preposition, relation, anchor, lemma">
    xml.append('<%s%s %s="%s" %s="%s">' % (
        XML_SENTENCE,
        XML_ID and " %s=\"%s\"" % (XML_ID, str(id)) or "",
        XML_TOKEN, ", ".join(sentence.token),
        XML_LANGUAGE, sentence.language
    ))
    # Collect chunks that are PNP anchors and assign id.
    anchors = {}
    for chunk in sentence.chunks:
        if chunk.attachments:
            anchors[chunk.start] = len(anchors) + 1
    # Traverse all words in the sentence.
    for word in sentence.words:
        chunk = word.chunk
        pnp   = word.chunk and word.chunk.pnp or None
        # Start the PNP element if the chunk is the first chunk in PNP:
        # <chunk type="PNP" of="A1">
        if pnp and pnp.start == chunk.start:
            a = pnp.anchor and ' %s="%s"' % (XML_OF, uid("A", anchors.get(pnp.anchor.start, ""))) or ""
            xml.append(indent + '<%s %s="PNP"%s>' % (XML_CHUNK, XML_TYPE, a))
            indent = push(indent)
        # Start the chunk element if the word is the first word in the chunk:
        # <chunk type="VP" relation="VP" id="1" anchor="A1">
        if chunk and chunk.start == word.index:
            if chunk.relations:
                # Create the shortest possible attribute values for multiple relations,
                # e.g., [(1,"OBJ"),(2,"OBJ")]) => relation="OBJ" id="1|2"
                r1 = unzip(0, chunk.relations) # Relation id's.
                r2 = unzip(1, chunk.relations) # Relation roles.
                r1 = [x is None and "-" or uid(x) for x in r1]
                r2 = [x is None and "-" or x for x in r2]
                r1 = not len(unique(r1)) == 1 and "|".join(r1) or (r1+[None])[0]
                r2 = not len(unique(r2)) == 1 and "|".join(r2) or (r2+[None])[0]
            xml.append(indent + '<%s%s%s%s%s%s>' % (
                XML_CHUNK,
                chunk.type and ' %s="%s"' % (XML_TYPE, chunk.type) or "",
                chunk.relations and chunk.role != None and ' %s="%s"' % (XML_RELATION, r2) or "",
                chunk.relation  and chunk.type == "VP" and ' %s="%s"' % (XML_ID, uid(chunk.relation)) or "",
                chunk.relation  and chunk.type != "VP" and ' %s="%s"' % (XML_OF, r1) or "",
                chunk.attachments and ' %s="%s"' % (XML_ANCHOR, uid("A",anchors[chunk.start])) or ""
            ))
            indent = push(indent)
        # Words outside of a chunk are wrapped in a <chink> tag:
        # <chink>
        if not chunk:
            xml.append(indent + '<%s>' % XML_CHINK)
            indent = push(indent)
        # Add the word element:
        # <word type="VBP" lemma="eat">eat</word>
        xml.append(indent + '<%s%s%s%s>%s</%s>' % (
            XML_WORD,
            word.type and ' %s="%s"' % (XML_TYPE, xml_encode(word.type)) or '',
            word.lemma and ' %s="%s"' % (XML_LEMMA, xml_encode(word.lemma)) or '',
            (" "+" ".join(['%s="%s"' % (k,v) for k,v in word.custom_tags.items() if v != None])).rstrip(),
            xml_encode(unicode(word)),
            XML_WORD
        ))
        if not chunk:
            # Close the <chink> element if outside of a chunk.
            indent = pop(indent); xml.append(indent + "</%s>" % XML_CHINK)
        if chunk and chunk.stop-1 == word.index:
            # Close the <chunk> element if this is the last word in the chunk.
            indent = pop(indent); xml.append(indent + "</%s>" % XML_CHUNK)
        if pnp and pnp.stop-1 == word.index:
            # Close the PNP element if this is the last word in the PNP.
            indent = pop(indent); xml.append(indent + "</%s>" % XML_CHUNK)
    xml.append("</%s>" % XML_SENTENCE)
    # Return as a plain str.
    return "\n".join(xml).encode("utf-8")

Response: Parses the given sentence as an XML - string.


Instruction:
How would you explain what the following Python 3 function does
def parse_string(xml):
    string = ""
    # Traverse all the <sentence> elements in the XML.
    dom = XML(xml)
    for sentence in dom(XML_SENTENCE):
        _anchors.clear()     # Populated by calling _parse_tokens().
        _attachments.clear() # Populated by calling _parse_tokens().
        # Parse the language from <sentence language="">.
        language = sentence.get(XML_LANGUAGE, "en")
        # Parse the token tag format from <sentence token="">.
        # This information is returned in TokenString.tags,
        # so the format and order of the token tags is retained when exporting/importing as XML.
        format = sentence.get(XML_TOKEN, [WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA])
        format = not isinstance(format, basestring) and format or format.replace(" ","").split(",")
        # Traverse all <chunk> and <chink> elements in the sentence.
        # Find the <word> elements inside and create tokens.
        tokens = []
        for chunk in sentence:
            tokens.extend(_parse_tokens(chunk, format))
        # Attach PNP's to their anchors.
        # Keys in _anchors have linked anchor chunks (each chunk is a list of tokens).
        # The keys correspond to the keys in _attachments, which have linked PNP chunks.
        if ANCHOR in format:
            A, P, a, i = _anchors, _attachments, 1, format.index(ANCHOR)
            for id in sorted(A.keys()):
                for token in A[id]:
                    token[i] += "-"+"-".join(["A"+str(a+p) for p in range(len(P[id]))])
                    token[i]  = token[i].strip("O-")
                for p, pnp in enumerate(P[id]):
                    for token in pnp:
                        token[i] += "-"+"P"+str(a+p)
                        token[i]  = token[i].strip("O-")
                a += len(P[id])
        # Collapse the tokens to string.
        # Separate multiple sentences with a new line.
        tokens = ["/".join([tag for tag in token]) for token in tokens]
        tokens = " ".join(tokens)
        string += tokens + "\n"
    # Return a TokenString, which is a unicode string that transforms easily
    # into a plain str, a list of tokens, or a Sentence.
    try:
        if MBSP: from mbsp import TokenString
        return TokenString(string.strip(), tags=format, language=language)
    except:
        return TaggedString(string.strip(), tags=format, language=language)

Response: Parses the given XML representation into a TokenString or TaggedString.


Instruction:
Make a summary of the following Python 3 code
def _parse_tokens(chunk, format=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):
    tokens = []
    # Only process <chunk> and <chink> elements,
    # text nodes in between return an empty list.
    if not (chunk.tag == XML_CHUNK or chunk.tag == XML_CHINK):
        return []
    type = chunk.get(XML_TYPE, "O")
    if type == "PNP":
        # For, <chunk type="PNP">, recurse all the child chunks inside the PNP.
        for ch in chunk:
            tokens.extend(_parse_tokens(ch, format))
        # Tag each of them as part of the PNP.
        if PNP in format:
            i = format.index(PNP)
            for j, token in enumerate(tokens):
                token[i] = (j==0 and "B-" or "I-") + "PNP"
        # Store attachments so we can construct anchor id's in parse_string().
        # This has to be done at the end, when all the chunks have been found.
        a = chunk.get(XML_OF).split(_UID_SEPARATOR)[-1]
        if a:
            _attachments.setdefault(a, [])
            _attachments[a].append(tokens)
        return tokens
    # For <chunk type-"VP" id="1">, the relation is VP-1.
    # For <chunk type="NP" relation="OBJ" of="1">, the relation is NP-OBJ-1.
    relation = _parse_relation(chunk, type)
    # Process all of the <word> elements in the chunk, for example:
    # <word type="NN" lemma="pizza">pizza</word> => [pizza, NN, I-NP, O, NP-OBJ-1, O, pizza]
    for word in filter(lambda n: n.tag == XML_WORD, chunk):
        tokens.append(_parse_token(word, chunk=type, relation=relation, format=format))
    # Add the IOB chunk tags:
    # words at the start of a chunk are marked with B-, words inside with I-.
    if CHUNK in format:
        i = format.index(CHUNK)
        for j, token in enumerate(tokens):
            token[i] = token[i] != "O" and ((j==0 and "B-" or "I-") + token[i]) or "O"
    # The chunk can be the anchor of one or more PNP chunks.
    # Store anchors so we can construct anchor id's in parse_string().
    a = chunk.get(XML_ANCHOR, "").split(_UID_SEPARATOR)[-1]
    if a:
        _anchors[a] = tokens
    return tokens

Response: Parses a list of tokens from the given XML chunk element.


Instruction:
Can you implement a function in Python 3 that
parses the relation string from the given chunk element.

Response:
def _parse_relation(chunk, type="O"):
    """ Returns a string of the roles and relations parsed from the given <chunk> element.
        The chunk type (which is part of the relation string) can be given as parameter.
    """
    r1 = chunk.get(XML_RELATION)
    r2 = chunk.get(XML_ID, chunk.get(XML_OF))
    r1 = [x != "-" and x or None for x in r1.split("|")] or [None]
    r2 = [x != "-" and x or None for x in r2.split("|")] or [None]
    r2 = [x is not None and x.split(_UID_SEPARATOR )[-1] or x for x in r2]
    if len(r1) < len(r2): r1 = r1 + r1 * (len(r2)-len(r1)) # [1] ["SBJ", "OBJ"] => "SBJ-1;OBJ-1"
    if len(r2) < len(r1): r2 = r2 + r2 * (len(r1)-len(r2)) # [2,4] ["OBJ"] => "OBJ-2;OBJ-4"
    return ";".join(["-".join([x for x in (type, r1, r2) if x]) for r1, r2 in zip(r1, r2)])


Instruction:
How would you implement a function in Python 3 that
parses a word element into a list of token tags.

Response:
def _parse_token(word, chunk="O", pnp="O", relation="O", anchor="O",
                 format=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):
    """ Returns a list of token tags parsed from the given <word> element.
        Tags that are not attributes in a <word> (e.g., relation) can be given as parameters.
    """
    tags = []
    for tag in format:
        if   tag == WORD   : tags.append(xml_decode(word.value))
        elif tag == POS    : tags.append(xml_decode(word.get(XML_TYPE, "O")))
        elif tag == CHUNK  : tags.append(chunk)
        elif tag == PNP    : tags.append(pnp)
        elif tag == REL    : tags.append(relation)
        elif tag == ANCHOR : tags.append(anchor)
        elif tag == LEMMA  : tags.append(xml_decode(word.get(XML_LEMMA, "")))
        else:
            # Custom tags when the parser has been extended, see also Word.custom_tags{}.
            tags.append(xml_decode(word.get(tag, "O")))
    return tags


Instruction:
Can you tell what is the following Python 3 function doing
def nltk_tree(sentence):
    from nltk import tree
    def do_pnp(pnp):
        # Returns the PNPChunk (and the contained Chunk objects) in NLTK bracket format.
        s = ' '.join([do_chunk(ch) for ch in pnp.chunks])
        return '(PNP %s)' % s

    def do_chunk(ch):
        # Returns the Chunk in NLTK bracket format. Recurse attached PNP's.
        s = ' '.join(['(%s %s)' % (w.pos, w.string) for w in ch.words])
        s+= ' '.join([do_pnp(pnp) for pnp in ch.attachments])
        return '(%s %s)' % (ch.type, s)

    T = ['(S']
    v = [] # PNP's already visited.
    for ch in sentence.chunked():
        if not ch.pnp and isinstance(ch, Chink):
            T.append('(%s %s)' % (ch.words[0].pos, ch.words[0].string))
        elif not ch.pnp:
            T.append(do_chunk(ch))
        #elif ch.pnp not in v:
        elif ch.pnp.anchor is None and ch.pnp not in v:
            # The chunk is part of a PNP without an anchor.
            T.append(do_pnp(ch.pnp))
            v.append(ch.pnp)
    T.append(')')
    return tree.bracket_parse(' '.join(T))

Response: Returns an NLTK nltk. tree. Tree object from the given Sentence.


Instruction:
Write a Python 3 function for
returning a DOT - formatted string that can be visualized as a graph in GraphViz.

Response:
def graphviz_dot(sentence, font="Arial", colors=BLUE):
    """ Returns a dot-formatted string that can be visualized as a graph in GraphViz.
    """
    s  = 'digraph sentence {\n'
    s += '\tranksep=0.75;\n'
    s += '\tnodesep=0.15;\n'
    s += '\tnode [penwidth=1, fontname="%s", shape=record, margin=0.1, height=0.35];\n' % font
    s += '\tedge [penwidth=1];\n'
    s += '\t{ rank=same;\n'
    # Create node groups for words, chunks and PNP chunks.
    for w in sentence.words:
        s += '\t\tword%s [label="<f0>%s|<f1>%s"%s];\n' % (w.index, w.string, w.type, _colorize(w, colors))
    for w in sentence.words[:-1]:
        # Invisible edges forces the words into the right order:
        s += '\t\tword%s -> word%s [color=none];\n' % (w.index, w.index+1)
    s += '\t}\n'
    s += '\t{ rank=same;\n'
    for i, ch in enumerate(sentence.chunks):
        s += '\t\tchunk%s [label="<f0>%s"%s];\n' % (i+1, "-".join([x for x in (
            ch.type, ch.role, str(ch.relation or '')) if x]) or '-', _colorize(ch, colors))
    for i, ch in enumerate(sentence.chunks[:-1]):
        # Invisible edges forces the chunks into the right order:
        s += '\t\tchunk%s -> chunk%s [color=none];\n' % (i+1, i+2)
    s += '}\n'
    s += '\t{ rank=same;\n'
    for i, ch in enumerate(sentence.pnp):
        s += '\t\tpnp%s [label="<f0>PNP"%s];\n' % (i+1, _colorize(ch, colors))
    s += '\t}\n'
    s += '\t{ rank=same;\n S [shape=circle, margin=0.25, penwidth=2]; }\n'
    # Connect words to chunks.
    # Connect chunks to PNP or S.
    for i, ch in enumerate(sentence.chunks):
        for w in ch:
            s += '\tword%s -> chunk%s;\n' % (w.index, i+1)
        if ch.pnp:
            s += '\tchunk%s -> pnp%s;\n' % (i+1, sentence.pnp.index(ch.pnp)+1)
        else:
            s += '\tchunk%s -> S;\n' % (i+1)
        if ch.type == 'VP':
            # Indicate related chunks with a dotted
            for r in ch.related:
                s += '\tchunk%s -> chunk%s [style=dotted, arrowhead=none];\n' % (
                    i+1, sentence.chunks.index(r)+1)
    # Connect PNP to anchor chunk or S.
    for i, ch in enumerate(sentence.pnp):
        if ch.anchor:
            s += '\tpnp%s -> chunk%s;\n' % (i+1, sentence.chunks.index(ch.anchor)+1)
            s += '\tpnp%s -> S [color=none];\n' % (i+1)
        else:
            s += '\tpnp%s -> S;\n' % (i+1)
    s += "}"
    return s


Instruction:
Write a Python 3 script to
return a string where the tags of tokens in the sentence are organized in outlined columns.

Response:
def table(sentence, fill=1, placeholder="-"):
    """ Returns a string where the tags of tokens in the sentence are organized in outlined columns.
    """
    tags  = [WORD, POS, IOB, CHUNK, ROLE, REL, PNP, ANCHOR, LEMMA]
    tags += [tag for tag in sentence.token if tag not in tags]
    def format(token, tag):
        # Returns the token tag as a string.
        if   tag == WORD   : s = token.string
        elif tag == POS    : s = token.type
        elif tag == IOB    : s = token.chunk and (token.index == token.chunk.start and "B" or "I")
        elif tag == CHUNK  : s = token.chunk and token.chunk.type
        elif tag == ROLE   : s = token.chunk and token.chunk.role
        elif tag == REL    : s = token.chunk and token.chunk.relation and str(token.chunk.relation)
        elif tag == PNP    : s = token.chunk and token.chunk.pnp and token.chunk.pnp.type
        elif tag == ANCHOR : s = token.chunk and token.chunk.anchor_id
        elif tag == LEMMA  : s = token.lemma
        else               : s = token.custom_tags.get(tag)
        return s or placeholder
    def outline(column, fill=1, padding=3, align="left"):
        # Add spaces to each string in the column so they line out to the highest width.
        n = max([len(x) for x in column]+[fill])
        if align == "left"  : return [x+" "*(n-len(x))+" "*padding for x in column]
        if align == "right" : return [" "*(n-len(x))+x+" "*padding for x in column]

    # Gather the tags of the tokens in the sentece per column.
    # If the IOB-tag is I-, mark the chunk tag with "^".
    # Add the tag names as headers in each column.
    columns = [[format(token, tag) for token in sentence] for tag in tags]
    columns[3] = [columns[3][i]+(iob == "I" and " ^" or "") for i, iob in enumerate(columns[2])]
    del columns[2]
    for i, header in enumerate(['word', 'tag', 'chunk', 'role', 'id', 'pnp', 'anchor', 'lemma']+tags[9:]):
        columns[i].insert(0, "")
        columns[i].insert(0, header.upper())
    # The left column (the word itself) is outlined to the right,
    # and has extra spacing so that words across sentences line out nicely below each other.
    for i, column in enumerate(columns):
        columns[i] = outline(column, fill+10*(i==0), align=("left","right")[i==0])
    # Anchor column is useful in MBSP but not in pattern.en.
    if not MBSP:
        del columns[6]
    # Create a string with one row (i.e., one token) per line.
    return "\n".join(["".join([x[i] for x in columns]) for i in range(len(columns[0]))])


Instruction:
How would you explain what the following Python 3 function does
def tags(self):
        # See also. Sentence.__repr__().
        ch, I,O,B = self.chunk, INSIDE+"-", OUTSIDE, BEGIN+"-"
        tags = [OUTSIDE for i in range(len(self.sentence.token))]
        for i, tag in enumerate(self.sentence.token): # Default: [WORD, POS, CHUNK, PNP, RELATION, ANCHOR, LEMMA]
            if tag == WORD:
                tags[i] = encode_entities(self.string)
            elif tag == POS and self.type:
                tags[i] = self.type
            elif tag == CHUNK and ch and ch.type:
                tags[i] = (self == ch[0] and B or I) + ch.type
            elif tag == PNP and self.pnp:
                tags[i] = (self == self.pnp[0] and B or I) + "PNP"
            elif tag == REL and ch and len(ch.relations) > 0:
                tags[i] = ["-".join([str(x) for x in [ch.type]+list(reversed(r)) if x]) for r in ch.relations]
                tags[i] = "*".join(tags[i])
            elif tag == ANCHOR and ch:
                tags[i] = ch.anchor_id or OUTSIDE
            elif tag == LEMMA:
                tags[i] = encode_entities(self.lemma or "")
            elif tag in self.custom_tags:
                tags[i] = self.custom_tags.get(tag) or OUTSIDE
        return tags

Response: Yields a list of all the token tags as they appeared when the word was parsed.


Instruction:
Write a Python 3 script for
returning the next word in the sentence with the given type.

Response:
def next(self, type=None):
        """ Returns the next word in the sentence with the given type.
        """
        i = self.index + 1
        s = self.sentence
        while i < len(s):
            if type in (s[i].type, None):
                return s[i]
            i += 1


Instruction:
Can you create a Python 3 function that
returns the previous word in the sentence with the given type.

Response:
def previous(self, type=None):
        """ Returns the next previous word in the sentence with the given type.
        """
        i = self.index - 1
        s = self.sentence
        while i > 0:
            if type in (s[i].type, None):
                return s[i]
            i -= 1


Instruction:
Write a Python 3 script for
yielding the first entry in the list.

Response:
def head(self):
        """ Yields the head of the chunk (usually, the last word in the chunk).
        """
        if self.type == "NP" and any(w.type.startswith("NNP") for w in self):
            w = find(lambda w: w.type.startswith("NNP"), reversed(self))
        elif self.type == "NP":  # "the cat" => "cat"
            w = find(lambda w: w.type.startswith("NN"), reversed(self))
        elif self.type == "VP":  # "is watching" => "watching"
            w = find(lambda w: w.type.startswith("VB"), reversed(self))
        elif self.type == "PP":  # "from up on" => "from"
            w = find(lambda w: w.type.startswith(("IN", "PP")), self)
        elif self.type == "PNP": # "from up on the roof" => "roof"
            w = find(lambda w: w.type.startswith("NN"), reversed(self))
        else:
            w = None
        if w is None:
            w = self[-1]
        return w


Instruction:
Can you implement a function in Python 3 that
yields a list of all chunks in the sentence with the same relation id.

Response:
def related(self):
        """ Yields a list of all chunks in the sentence with the same relation id.
        """
        return [ch for ch in self.sentence.chunks
                    if ch != self and intersects(unzip(0, ch.relations), unzip(0, self.relations))]


Instruction:
Implement a function in Python 3 to
yield the anchor tag as parsed from the original token.

Response:
def anchor_id(self):
        """ Yields the anchor tag as parsed from the original token.
            Chunks that are anchors have a tag with an "A" prefix (e.g., "A1").
            Chunks that are PNP attachmens (or chunks inside a PNP) have "P" (e.g., "P1").
            Chunks inside a PNP can be both anchor and attachment (e.g., "P1-A2"),
            as in: "clawed/A1 at/P1 mice/P1-A2 in/P2 the/P2 wall/P2"
        """
        id = ""
        f = lambda ch: filter(lambda k: self.sentence._anchors[k] == ch, self.sentence._anchors)
        if self.pnp and self.pnp.anchor:
            id += "-" + "-".join(f(self.pnp))
        if self.anchor:
            id += "-" + "-".join(f(self))
        if self.attachments:
            id += "-" + "-".join(f(self))
        return id.strip("-") or None


Instruction:
Can you implement a function in Python 3 that
returns a list of all the modifiers that are part of the VP.

Response:
def modifiers(self):
        """ For verb phrases (VP), yields a list of the nearest adjectives and adverbs.
        """
        if self._modifiers is None:
            # Iterate over all the chunks and attach modifiers to their VP-anchor.
            is_modifier = lambda ch: ch.type in ("ADJP", "ADVP") and ch.relation is None
            for chunk in self.sentence.chunks:
                chunk._modifiers = []
            for chunk in filter(is_modifier, self.sentence.chunks):
                anchor = chunk.nearest("VP")
                if anchor: anchor._modifiers.append(chunk)
        return self._modifiers


Instruction:
Given the following Python 3 function, write the documentation
def nearest(self, type="VP"):
        candidate, d = None, len(self.sentence.chunks)
        if isinstance(self, PNPChunk):
            i = self.sentence.chunks.index(self.chunks[0])
        else:
            i = self.sentence.chunks.index(self)
        for j, chunk in enumerate(self.sentence.chunks):
            if chunk.type.startswith(type) and abs(i-j) < d:
                candidate, d = chunk, abs(i-j)
        return candidate

Response: Returns the nearest chunk in the sentence with the given type.


Instruction:
Given the following Python 3 function, write the documentation
def next(self, type=None):
        i = self.stop
        s = self.sentence
        while i < len(s):
            if s[i].chunk is not None and type in (s[i].chunk.type, None):
                return s[i].chunk
            i += 1

Response: Returns the next chunk in the sentence with the given type.


Instruction:
Can you create a Python 3 function that
returns the previous chunk in the sentence with the given type.

Response:
def previous(self, type=None):
        """ Returns the next previous chunk in the sentence with the given type.
        """
        i = self.start - 1
        s = self.sentence
        while i > 0:
            if s[i].chunk is not None and type in (s[i].chunk.type, None):
                return s[i].chunk
            i -= 1


Instruction:
Implement a function in Python 3 to
append a new word to the internal list of words.

Response:
def append(self, word, lemma=None, type=None, chunk=None, role=None, relation=None, pnp=None, anchor=None, iob=None, custom={}):
        """ Appends the next word to the sentence / chunk / preposition.
            For example: Sentence.append("clawed", "claw", "VB", "VP", role=None, relation=1)
            - word     : the current word,
            - lemma    : the canonical form of the word,
            - type     : part-of-speech tag for the word (NN, JJ, ...),
            - chunk    : part-of-speech tag for the chunk this word is part of (NP, VP, ...),
            - role     : the chunk's grammatical role (SBJ, OBJ, ...),
            - relation : an id shared by other related chunks (e.g., SBJ-1 <=> VP-1),
            - pnp      : PNP if this word is in a prepositional noun phrase (B- prefix optional),
            - iob      : BEGIN if the word marks the start of a new chunk,
                         INSIDE (optional) if the word is part of the previous chunk,
            - custom   : a dictionary of (tag, value)-items for user-defined word tags.
        """
        self._do_word(word, lemma, type)            # Append Word object.
        self._do_chunk(chunk, role, relation, iob)  # Append Chunk, or add last word to last chunk.
        self._do_conjunction()
        self._do_relation()
        self._do_pnp(pnp, anchor)
        self._do_anchor(anchor)
        self._do_custom(custom)


Instruction:
Write a Python 3 function for
parsing a tagged token representation into a list of arguments.

Response:
def parse_token(self, token, tags=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):
        """ Returns the arguments for Sentence.append() from a tagged token representation.
            The order in which token tags appear can be specified.
            The default order is (separated by slashes):
            - word,
            - part-of-speech,
            - (IOB-)chunk,
            - (IOB-)preposition,
            - chunk(-relation)(-role),
            - anchor,
            - lemma.
            Examples:
            The/DT/B-NP/O/NP-SBJ-1/O/the
            cats/NNS/I-NP/O/NP-SBJ-1/O/cat
            clawed/VBD/B-VP/O/VP-1/A1/claw
            at/IN/B-PP/B-PNP/PP/P1/at
            the/DT/B-NP/I-PNP/NP/P1/the
            sofa/NN/I-NP/I-PNP/NP/P1/sofa
            ././O/O/O/O/.
            Returns a (word, lemma, type, chunk, role, relation, preposition, anchor, iob, custom)-tuple,
            which can be passed to Sentence.append(): Sentence.append(*Sentence.parse_token("cats/NNS/NP"))
            The custom value is a dictionary of (tag, value)-items of unrecognized tags in the token.
        """
        p = { WORD: "",
               POS: None,
               IOB: None,
             CHUNK: None,
               PNP: None,
               REL: None,
              ROLE: None,
            ANCHOR: None,
             LEMMA: None }
        # Split the slash-formatted token into separate tags in the given order.
        # Decode &slash; characters (usually in words and lemmata).
        # Assume None for missing tags (except the word itself, which defaults to an empty string).
        custom = {}
        for k, v in izip(tags, token.split("/")):
            if SLASH0 in v:
                v = v.replace(SLASH, "/")
            if k not in p:
                custom[k] = None
            if v != OUTSIDE or k == WORD or k == LEMMA: # "type O negative" => "O" != OUTSIDE.
                (p if k not in custom else custom)[k] = v
        # Split IOB-prefix from the chunk tag:
        # B- marks the start of a new chunk,
        # I- marks inside of a chunk.
        ch = p[CHUNK]
        if ch is not None and ch.startswith(("B-", "I-")):
            p[IOB], p[CHUNK] = ch[:1], ch[2:] # B-NP
        # Split the role from the relation:
        # NP-SBJ-1 => relation id is 1 and role is SBJ,
        # VP-1 => relation id is 1 with no role.
        # Tokens may be tagged with multiple relations (e.g., NP-OBJ-1*NP-OBJ-3).
        if p[REL] is not None:
            ch, p[REL], p[ROLE] = self._parse_relation(p[REL])
            # Infer a missing chunk tag from the relation tag (e.g., NP-SBJ-1 => NP).
            # For PP relation tags (e.g., PP-CLR-1), the first chunk is PP, the following chunks NP.
            if ch == "PP" \
             and self._previous \
             and self._previous[REL] == p[REL] \
             and self._previous[ROLE] == p[ROLE]:
                ch = "NP"
            if p[CHUNK] is None and ch != OUTSIDE:
                p[CHUNK] = ch
        self._previous = p
        # Return the tags in the right order for Sentence.append().
        return p[WORD], p[LEMMA], p[POS], p[CHUNK], p[ROLE], p[REL], p[PNP], p[ANCHOR], p[IOB], custom


Instruction:
Implement a Python 3 function for
parsing the token relation tag and returns the chunk relation role and relation id.

Response:
def _parse_relation(self, tag):
        """ Parses the chunk tag, role and relation id from the token relation tag.
            - VP                => VP, [], []
            - VP-1              => VP, [1], [None]
            - ADJP-PRD          => ADJP, [None], [PRD]
            - NP-SBJ-1          => NP, [1], [SBJ]
            - NP-OBJ-1*NP-OBJ-2 => NP, [1,2], [OBJ,OBJ]
            - NP-SBJ;NP-OBJ-1   => NP, [1,1], [SBJ,OBJ]
        """
        chunk, relation, role = None, [], []
        if ";" in tag:
            # NP-SBJ;NP-OBJ-1 => 1 relates to both SBJ and OBJ.
            id = tag.split("*")[0][-2:]
            id = id if id.startswith("-") else ""
            tag = tag.replace(";", id + "*")
        if "*" in tag:
            tag = tag.split("*")
        else:
            tag = [tag]
        for s in tag:
            s = s.split("-")
            n = len(s)
            if n == 1:
                chunk = s[0]
            if n == 2:
                chunk = s[0]; relation.append(s[1]); role.append(None)
            if n >= 3:
                chunk = s[0]; relation.append(s[2]); role.append(s[1])
            if n > 1:
                id = relation[-1]
                if id.isdigit():
                    relation[-1] = int(id)
                else:
                    # Correct "ADJP-PRD":
                    # (ADJP, [PRD], [None]) => (ADJP, [None], [PRD])
                    relation[-1], role[-1] = None, id
        return chunk, relation, role


Instruction:
Implement a Python 3 function for
adding a new Word to the sentence.

Response:
def _do_word(self, word, lemma=None, type=None):
        """ Adds a new Word to the sentence.
            Other Sentence._do_[tag] functions assume a new word has just been appended.
        """
        # Improve 3rd person singular "'s" lemma to "be", e.g., as in "he's fine".
        if lemma == "'s" and type in ("VB", "VBZ"):
            lemma = "be"
        self.words.append(Word(self, word, lemma, type, index=len(self.words)))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _do_chunk(self, type, role=None, relation=None, iob=None):
        if (type is None or type == OUTSIDE) and \
           (role is None or role == OUTSIDE) and (relation is None or relation == OUTSIDE):
            return
        if iob != BEGIN \
         and self.chunks \
         and self.chunks[-1].type == type \
         and self._relation == (relation, role) \
         and self.words[-2].chunk is not None: # "one, two" => "one" & "two" different chunks.
            self.chunks[-1].append(self.words[-1])
        else:
            ch = Chunk(self, [self.words[-1]], type, role, relation)
            self.chunks.append(ch)
            self._relation = (relation, role)

Response: Add a new Chunk to the sentence or adds the last word to the previous chunk.


Instruction:
Explain what the following Python 3 code does
def _do_relation(self):
        if self.chunks:
            ch = self.chunks[-1]
            for relation, role in ch.relations:
                if role == "SBJ" or role == "OBJ":
                    self.relations[role][relation] = ch
            if ch.type in ("VP",):
                self.relations[ch.type][ch.relation] = ch

Response: Attaches subjects objects and verbs to the last chunk of the list of related resources.


Instruction:
Implement a Python 3 function for
attaching prepositional noun phrases. Identifies PNP's from either the PNP tag or the P-attachment tag. This does not determine the PP-anchor, it only groups words in a PNP chunk.

Response:
def _do_pnp(self, pnp, anchor=None):
        """ Attaches prepositional noun phrases.
            Identifies PNP's from either the PNP tag or the P-attachment tag.
            This does not determine the PP-anchor, it only groups words in a PNP chunk.
        """
        if anchor or pnp and pnp.endswith("PNP"):
            if anchor is not None:
                m = find(lambda x: x.startswith("P"), anchor)
            else:
                m = None
            if self.pnp \
             and pnp \
             and pnp != OUTSIDE \
             and pnp.startswith("B-") is False \
             and self.words[-2].pnp is not None:
                self.pnp[-1].append(self.words[-1])
            elif m is not None and m == self._attachment:
                self.pnp[-1].append(self.words[-1])
            else:
                ch = PNPChunk(self, [self.words[-1]], type="PNP")
                self.pnp.append(ch)
            self._attachment = m


Instruction:
Explain what the following Python 3 code does
def _do_anchor(self, anchor):
        if anchor:
            for x in anchor.split("-"):
                A, P = None, None
                if x.startswith("A") and len(self.chunks) > 0: # anchor
                    A, P = x, x.replace("A","P")
                    self._anchors[A] = self.chunks[-1]
                if x.startswith("P") and len(self.pnp) > 0:    # attachment (PNP)
                    A, P = x.replace("P","A"), x
                    self._anchors[P] = self.pnp[-1]
                if A in self._anchors and P in self._anchors and not self._anchors[P].anchor:
                    pnp = self._anchors[P]
                    pnp.anchor = self._anchors[A]
                    pnp.anchor.attachments.append(pnp)

Response: Collects anchors and attachments in a dictionary.


Instruction:
How would you explain what the following Python 3 function does
def _do_conjunction(self, _and=("and", "e", "en", "et", "und", "y")):
        w = self.words
        if len(w) > 2 and w[-2].type == "CC" and w[-2].chunk is None:
            cc  = w[-2].string.lower() in _and and AND or OR
            ch1 = w[-3].chunk
            ch2 = w[-1].chunk
            if ch1 is not None and \
               ch2 is not None:
                ch1.conjunctions.append(ch2, cc)
                ch2.conjunctions.append(ch1, cc)

Response: Add a conjunction to the list of possible conjunctions.


Instruction:
How would you explain what the following Python 3 function does
def get(self, index, tag=LEMMA):
        if tag == WORD:
            return self.words[index]
        if tag == LEMMA:
            return self.words[index].lemma
        if tag == POS:
            return self.words[index].type
        if tag == CHUNK:
            return self.words[index].chunk
        if tag == PNP:
            return self.words[index].pnp
        if tag == REL:
            ch = self.words[index].chunk; return ch and ch.relation
        if tag == ROLE:
            ch = self.words[index].chunk; return ch and ch.role
        if tag == ANCHOR:
            ch = self.words[index].pnp; return ch and ch.anchor
        if tag in self.words[index].custom_tags:
            return self.words[index].custom_tags[tag]
        return None

Response: Returns a tag for the word at the given index.


Instruction:
Can you implement a function in Python 3 that
yields the words in the Sentence and lemmata.

Response:
def loop(self, *tags):
        """ Iterates over the tags in the entire Sentence,
            For example, Sentence.loop(POS, LEMMA) yields tuples of the part-of-speech tags and lemmata.
            Possible tags: WORD, LEMMA, POS, CHUNK, PNP, RELATION, ROLE, ANCHOR or a custom word tag.
            Any order or combination of tags can be supplied.
        """
        for i in range(len(self.words)):
            yield tuple([self.get(i, tag=tag) for tag in tags])


Instruction:
Explain what the following Python 3 code does
def indexof(self, value, tag=WORD):
        match = lambda a, b: a.endswith("*") and b.startswith(a[:-1]) or a==b
        indices = []
        for i in range(len(self.words)):
            if match(value, unicode(self.get(i, tag))):
                indices.append(i)
        return indices

Response: Returns the indices of the words in the sentence where the given token tag equals the string.


Instruction:
How would you code a function in Python 3 to
return a portion of the sentence from word start index to word stop index.

Response:
def slice(self, start, stop):
        """ Returns a portion of the sentence from word start index to word stop index.
            The returned slice is a subclass of Sentence and a deep copy.
        """
        s = Slice(token=self.token, language=self.language)
        for i, word in enumerate(self.words[start:stop]):
            # The easiest way to copy (part of) a sentence
            # is by unpacking all of the token tags and passing them to Sentence.append().
            p0 = word.string                                                       # WORD
            p1 = word.lemma                                                        # LEMMA
            p2 = word.type                                                         # POS
            p3 = word.chunk is not None and word.chunk.type or None                # CHUNK
            p4 = word.pnp is not None and "PNP" or None                            # PNP
            p5 = word.chunk is not None and unzip(0, word.chunk.relations) or None # REL
            p6 = word.chunk is not None and unzip(1, word.chunk.relations) or None # ROLE
            p7 = word.chunk and word.chunk.anchor_id or None                       # ANCHOR
            p8 = word.chunk and word.chunk.start == start+i and BEGIN or None      # IOB
            p9 = word.custom_tags                                                  # User-defined tags.
            # If the given range does not contain the chunk head, remove the chunk tags.
            if word.chunk is not None and (word.chunk.stop > stop):
                p3, p4, p5, p6, p7, p8 = None, None, None, None, None, None
            # If the word starts the preposition, add the IOB B-prefix (i.e., B-PNP).
            if word.pnp is not None and word.pnp.start == start+i:
                p4 = BEGIN+"-"+"PNP"
            # If the given range does not contain the entire PNP, remove the PNP tags.
            # The range must contain the entire PNP,
            # since it starts with the PP and ends with the chunk head (and is meaningless without these).
            if word.pnp is not None and (word.pnp.start < start or word.chunk.stop > stop):
                p4, p7 = None, None
            s.append(word=p0, lemma=p1, type=p2, chunk=p3, pnp=p4, relation=p5, role=p6, anchor=p7, iob=p8, custom=p9)
        s.parent = self
        s._start = start
        return s


Instruction:
Can you generate a brief explanation for the following Python 3 code
def constituents(self, pnp=False):
        a = []
        for word in self.words:
            if pnp and word.pnp is not None:
                if len(a) == 0 or a[-1] != word.pnp:
                    a.append(word.pnp)
            elif word.chunk is not None:
                if len(a) == 0 or a[-1] != word.chunk:
                    a.append(word.chunk)
            else:
                a.append(word)
        return a

Response: Returns an in - order list of mixed Chunk and Word objects. With pnp = True also contains PNPChunk objects whenever possible.


Instruction:
Explain what the following Python 3 code does
def from_xml(cls, xml):
        s = parse_string(xml)
        return Sentence(s.split("\n")[0], token=s.tags, language=s.language)

Response: Returns a new Sentence from the given XML string.


Instruction:
Can you write a function in Python 3 where it
yields the sentence as an XML - formatted string.

Response:
def xml(self):
        """ Yields the sentence as an XML-formatted string (plain bytestring, UTF-8 encoded).
            All the sentences in the XML are wrapped in a <text> element.
        """
        xml = []
        xml.append('<?xml version="1.0" encoding="%s"?>' % XML_ENCODING.get(self.encoding, self.encoding))
        xml.append("<%s>" % XML_TEXT)
        xml.extend([sentence.xml for sentence in self])
        xml.append("</%s>" % XML_TEXT)
        return "\n".join(xml)


Instruction:
Create a Python 3 function to
extract credits from AUTHORS. rst

Response:
def get_credits():
    """Extract credits from `AUTHORS.rst`"""
    credits = read(os.path.join(_HERE, "AUTHORS.rst")).split("\n")
    from_index = credits.index("Active Contributors")
    credits = "\n".join(credits[from_index + 2:])
    return credits


Instruction:
Here you have a function in Python 3, explain what it does
def rst2markdown_github(path_to_rst, path_to_md, pandoc="pandoc"):
    _proc = subprocess.Popen([pandoc, "-f", "rst",
                              "-t", "markdown_github",
                              #"-o", path_to_md,
                              path_to_rst],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
    print("Converting README.rst to markdown_github, "
          "using 'pandoc' ...")
    _stdout, _stderr = _proc.communicate()
    with _open(path_to_md, "w", encoding="utf-8") as _md:
        _skip_newline = False
        for line in _stdout.decode('utf-8').split(os.linesep):
            line = re.sub("``` sourceCode", "``` python", line)
            if line.startswith("[!["):
                _md.write(line)
                _md.write("\n")
                if not line.startswith(("[![LICENSE")):
                    _skip_newline = True
            elif _skip_newline and line == "":
                _skip_newline = False
                continue
            else:
                _md.write(line)
                _md.write("\n")

    if _stderr:
        print("pandoc.exe STDERR: ", _stderr)
    if os.path.isfile(path_to_md) and os.stat(path_to_md).st_size > 0:
        print("README.rst converted and saved as: {}".format(path_to_md))

Response: Converts README. rst to markdown_github.


Instruction:
How would you implement a function in Python 3 that
extracts HELP information from ``<program> -h | --help`` message **Input** * ``$ <program> -h | --help`` * ``$ cd <cwd> && make help`` **Output** * ``docs/src/console_help_xy.rst``

Response:
def console_help2rst(cwd, help_cmd, path_to_rst, rst_title,
                     format_as_code=False):
    """
    Extract HELP information from ``<program> -h | --help`` message

    **Input**

        * ``$ <program> -h | --help``
        * ``$ cd <cwd> && make help``

    **Output**

        * ``docs/src/console_help_xy.rst``

    """
    generated_time_str = """

    ::

     generated: {0}

""".format(time.strftime("%d %B %Y - %H:%M"))

    with _open(path_to_rst, "w", encoding='utf-8') as f:
        print("File", f)
        print("cwd", cwd)
        print("help_cmd", help_cmd)
        os.chdir(cwd)
        _proc = subprocess.Popen(
            help_cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            universal_newlines=True,)
        help_msg = _proc.stdout.readlines()
        f.write(get_rst_title(
                rst_title,
                "-",
                overline=True))
        f.write(generated_time_str)
        if "README" in path_to_rst:
            help_msg = "".join(help_msg[10:])
            #help_msg = PACKAGE_DOCSTRING + help_msg
        for line in help_msg:
            # exclude directory walk messages of 'make'
            if line.strip().startswith("make[1]:"):
                print("skipped line: {}".format(line))
            # exclude warning messages
            elif line.strip().startswith("\x1b[1m"):
                print("skipped line: {}".format(line))
            # exclude warning messages on Windows (without ``colorama``)
            elif line.strip().startswith("Using fallback version of '"):
                print("skipped line: {}".format(line))
            else:
                # correctly indent tips in 'make help'
                if line.strip().startswith("-->"):
                    f.write(3 * "\t")
                if format_as_code:
                    f.write("\t" + line.strip())
                    f.write("\n")
                else:
                    f.write(line)

        f.write("\n")
        if "README" in path_to_rst:
            f.write(get_rst_title("Credits", "^"))
            f.write(get_credits())

    print("\ncmd:{} in dir:{} --> RST generated:\n\t{}\n\n".format(
        help_cmd, cwd, path_to_rst))


Instruction:
Implement a function in Python 3 to
update the documentation for the current project.

Response:
def update_docs(readme=True, makefiles=True):
    """Update documentation (ready for publishing new release)

    Usually called by ``make docs``

    :param bool make_doc: generate DOC page from Makefile help messages

    """
    if readme:
        _pandoc = get_external_executable("pandoc")
        rst2markdown_github(os.path.join(_HERE, "README.rst"),
                            os.path.join(_HERE, "README.md"),
                            pandoc=_pandoc)

    if makefiles:
        _make = get_external_executable("make")
        project_makefile_dir = os.path.abspath(_HERE)
        project_makefile_rst = os.path.join(
            _HERE,
            'docs',
            'src',
            'project_makefile.rst')
        docs_makefile_dir = os.path.join(_HERE, 'docs', 'src')
        docs_makefile_rst = os.path.join(
            _HERE,
            'docs',
            'src',
            'docs_makefile.rst')

        #: ``help2rst_queue`` stores tuples of
        #: ``(cwd, help_cmd, path_to_rst_file, rst_title_of_new_file)``
        help2rst_queue = [
            (project_makefile_dir, [_make, "help"], project_makefile_rst,
             "Project ``Makefile``"),

            (docs_makefile_dir, [_make, "help"], docs_makefile_rst,
             "Documentation ``Makefile``")]

        for cwd, help_cmd, outfile, title in help2rst_queue:
            console_help2rst(
                cwd,
                help_cmd,
                outfile,
                title,
                format_as_code=True)


Instruction:
Can you create a Python 3 function that
returns root mean square value of f

Response:
def rms(self, x, params=()):
        """ Returns root mean square value of f(x, params) """
        internal_x, internal_params = self.pre_process(np.asarray(x),
                                                       np.asarray(params))
        if internal_params.ndim > 1:
            raise NotImplementedError("Parameters should be constant.")
        result = np.empty(internal_x.size//self.nx)
        for idx in range(internal_x.shape[0]):
            result[idx] = np.sqrt(np.mean(np.square(self.f_cb(
                internal_x[idx, :], internal_params))))
        return result


Instruction:
Can you generate the documentation for the following Python 3 function
def solve_series(self, x0, params, varied_data, varied_idx,
                     internal_x0=None, solver=None, propagate=True, **kwargs):
        """ Solve system for a set of parameters in which one is varied

        Parameters
        ----------
        x0 : array_like
            Guess (subject to ``self.post_processors``)
        params : array_like
            Parameter values
        vaired_data : array_like
            Numerical values of the varied parameter.
        varied_idx : int or str
            Index of the varied parameter (indexing starts at 0).
            If ``self.par_by_name`` this should be the name (str) of the varied
            parameter.
        internal_x0 : array_like (default: None)
            Guess (*not* subject to ``self.post_processors``).
            Overrides ``x0`` when given.
        solver : str or callback
            See :meth:`solve`.
        propagate : bool (default: True)
            Use last successful solution as ``x0`` in consecutive solves.
        \\*\\*kwargs :
            Keyword arguments pass along to :meth:`solve`.

        Returns
        -------
        xout : array
            Of shape ``(varied_data.size, x0.size)``.
        info_dicts : list of dictionaries
             Dictionaries each containing keys such as containing 'success', 'nfev', 'njev' etc.

        """
        if self.x_by_name and isinstance(x0, dict):
            x0 = [x0[k] for k in self.names]
        if self.par_by_name:
            if isinstance(params, dict):
                params = [params[k] for k in self.param_names]
            if isinstance(varied_idx, str):
                varied_idx = self.param_names.index(varied_idx)

        new_params = np.atleast_1d(np.array(params, dtype=np.float64))
        xout = np.empty((len(varied_data), len(x0)))
        self.internal_xout = np.empty_like(xout)
        self.internal_params_out = np.empty((len(varied_data),
                                             len(new_params)))
        info_dicts = []
        new_x0 = np.array(x0, dtype=np.float64)  # copy
        conds = kwargs.get('initial_conditions', None)  # see ConditionalNeqSys
        for idx, value in enumerate(varied_data):
            try:
                new_params[varied_idx] = value
            except TypeError:
                new_params = value  # e.g. type(new_params) == int
            if conds is not None:
                kwargs['initial_conditions'] = conds
            x, info_dict = self.solve(new_x0, new_params, internal_x0, solver,
                                      **kwargs)
            if propagate:
                if info_dict['success']:
                    try:
                        # See ChainedNeqSys.solve
                        new_x0 = info_dict['x_vecs'][0]
                        internal_x0 = info_dict['internal_x_vecs'][0]
                        conds = info_dict['intermediate_info'][0].get(
                            'conditions', None)
                    except:
                        new_x0 = x
                        internal_x0 = None
                        conds = info_dict.get('conditions', None)
            xout[idx, :] = x
            self.internal_xout[idx, :] = self.internal_x
            self.internal_params_out[idx, :] = self.internal_params
            info_dicts.append(info_dict)
        return xout, info_dicts

Response: Solve system for a set of parameters in which one is varied.


Instruction:
Implement a function in Python 3 to
plot the results from the solve_series method.

Response:
def plot_series(self, xres, varied_data, varied_idx, **kwargs):
        """ Plots the results from :meth:`solve_series`.

        Parameters
        ----------
        xres : array
            Of shape ``(varied_data.size, self.nx)``.
        varied_data : array
            See :meth:`solve_series`.
        varied_idx : int or str
            See :meth:`solve_series`.
        \\*\\*kwargs :
            Keyword arguments passed to :func:`pyneqsys.plotting.plot_series`.

        """
        for attr in 'names latex_names'.split():
            if kwargs.get(attr, None) is None:
                kwargs[attr] = getattr(self, attr)
        ax = plot_series(xres, varied_data, **kwargs)
        if self.par_by_name and isinstance(varied_idx, str):
            varied_idx = self.param_names.index(varied_idx)
        if self.latex_param_names:
            ax.set_xlabel('$%s$' % self.latex_param_names[varied_idx])
        elif self.param_names:
            ax.set_xlabel(self.param_names[varied_idx])
        return ax


Instruction:
Here you have a function in Python 3, explain what it does
def plot_series_residuals(self, xres, varied_data, varied_idx, params, **kwargs):
        nf = len(self.f_cb(*self.pre_process(xres[0], params)))
        xerr = np.empty((xres.shape[0], nf))
        new_params = np.array(params)

        for idx, row in enumerate(xres):
            new_params[varied_idx] = varied_data[idx]
            xerr[idx, :] = self.f_cb(*self.pre_process(row, params))
        return self.plot_series(xerr, varied_data, varied_idx, **kwargs)

Response: Analogous to plot_series but will plot residuals.


Instruction:
How would you explain what the following Python 3 function does
def plot_series_residuals_internal(self, varied_data, varied_idx, **kwargs):
        nf = len(self.f_cb(*self.pre_process(
            self.internal_xout[0], self.internal_params_out[0])))
        xerr = np.empty((self.internal_xout.shape[0], nf))
        for idx, (res, params) in enumerate(zip(self.internal_xout, self.internal_params_out)):
            xerr[idx, :] = self.f_cb(res, params)
        return self.plot_series(xerr, varied_data, varied_idx, **kwargs)

Response: Analogous to plot_series but for internal residuals from last run.


Instruction:
Here you have a function in Python 3, explain what it does
def solve_and_plot_series(self, x0, params, varied_data, varied_idx, solver=None, plot_kwargs=None,
                              plot_residuals_kwargs=None, **kwargs):
        """ Solve and plot for a series of a varied parameter.

        Convenience method, see :meth:`solve_series`, :meth:`plot_series` &
        :meth:`plot_series_residuals_internal` for more information.
        """
        sol, nfo = self.solve_series(
            x0, params, varied_data, varied_idx, solver=solver, **kwargs)
        ax_sol = self.plot_series(sol, varied_data, varied_idx, info=nfo,
                                  **(plot_kwargs or {}))

        extra = dict(ax_sol=ax_sol, info=nfo)
        if plot_residuals_kwargs:
            extra['ax_resid'] = self.plot_series_residuals_internal(
                varied_data, varied_idx, info=nfo,
                **(plot_residuals_kwargs or {})
            )
        return sol, extra

Response: Solve and plot for a varied parameter.


Instruction:
How would you code a function in Python 3 to
use internally for transformation of variables.

Response:
def pre_process(self, x0, params=()):
        """ Used internally for transformation of variables. """
        # Should be used by all methods matching "solve_*"
        if self.x_by_name and isinstance(x0, dict):
            x0 = [x0[k] for k in self.names]
        if self.par_by_name and isinstance(params, dict):
            params = [params[k] for k in self.param_names]
        for pre_processor in self.pre_processors:
            x0, params = pre_processor(x0, params)
        return x0, np.atleast_1d(params)


Instruction:
Can you tell what is the following Python 3 function doing
def post_process(self, xout, params_out):
        # Should be used by all methods matching "solve_*"
        for post_processor in self.post_processors:
            xout, params_out = post_processor(xout, params_out)
        return xout, params_out

Response: Used internally for transformation of variables.


Instruction:
Explain what the following Python 3 code does
def solve(self, x0, params=(), internal_x0=None, solver=None, attached_solver=None, **kwargs):
        if not isinstance(solver, (tuple, list)):
            solver = [solver]
        if not isinstance(attached_solver, (tuple, list)):
            attached_solver = [attached_solver] + [None]*(len(solver) - 1)
        _x0, self.internal_params = self.pre_process(x0, params)
        for solv, attached_solv in zip(solver, attached_solver):
            if internal_x0 is not None:
                _x0 = internal_x0
            elif self.internal_x0_cb is not None:
                _x0 = self.internal_x0_cb(x0, params)

            nfo = self._get_solver_cb(solv, attached_solv)(_x0, **kwargs)
            _x0 = nfo['x'].copy()
        self.internal_x = _x0
        x0 = self.post_process(self.internal_x, self.internal_params)[0]
        return x0, nfo

Response: Solve with user specified solver choice.


Instruction:
Can you generate the documentation for the following Python 3 function
def _solve_scipy(self, intern_x0, tol=1e-8, method=None, **kwargs):
        from scipy.optimize import root
        if method is None:
            if self.nf > self.nx:
                method = 'lm'
            elif self.nf == self.nx:
                method = 'hybr'
            else:
                raise ValueError('Underdetermined problem')
        if 'band' in kwargs:
            raise ValueError("Set 'band' at initialization instead.")
        if 'args' in kwargs:
            raise ValueError("Set 'args' as params in initialization instead.")

        new_kwargs = kwargs.copy()
        if self.band is not None:
            warnings.warn("Band argument ignored (see SciPy docs)")
            new_kwargs['band'] = self.band
        new_kwargs['args'] = self.internal_params

        return root(self.f_cb, intern_x0, jac=self.j_cb, method=method, tol=tol, **new_kwargs)

Response: Solve the current state of a set of unsatisfied iCal objects using scipy. optimize. root.


Instruction:
Make a summary of the following Python 3 code
def solve(self, x0, params=(), internal_x0=None, solver=None,
              conditional_maxiter=20, initial_conditions=None, **kwargs):
        """ Solve the problem (systems of equations)

        Parameters
        ----------
        x0 : array
            Guess.
        params : array
            See :meth:`NeqSys.solve`.
        internal_x0 : array
            See :meth:`NeqSys.solve`.
        solver : str or callable or iterable of such.
            See :meth:`NeqSys.solve`.
        conditional_maxiter : int
            Maximum number of switches between conditions.
        initial_conditions : iterable of bools
            Corresponding conditions to ``x0``
        \\*\\*kwargs :
            Keyword arguments passed on to :meth:`NeqSys.solve`.

        """
        if initial_conditions is not None:
            conds = initial_conditions
        else:
            conds = self.get_conds(x0, params, initial_conditions)
        idx, nfev, njev = 0, 0, 0
        while idx < conditional_maxiter:
            neqsys = self.neqsys_factory(conds)
            x0, info = neqsys.solve(x0, params, internal_x0, solver, **kwargs)
            if idx == 0:
                internal_x0 = None
            nfev += info['nfev']
            njev += info.get('njev', 0)
            new_conds = self.get_conds(x0, params, conds)
            if new_conds == conds:
                break
            else:
                conds = new_conds
            idx += 1
        if idx == conditional_maxiter:
            raise Exception("Solving failed, conditional_maxiter reached")
        self.internal_x = info['x']
        self.internal_params = neqsys.internal_params
        result = {
            'x': info['x'],
            'success': info['success'],
            'conditions': conds,
            'nfev': nfev,
            'njev': njev,
        }
        if 'fun' in info:
            result['fun'] = info['fun']
        return x0, result

Response: Solve the problem for a set of conditions.


Instruction:
Write a Python 3 function for
solving a system using a user - specified solver.

Response:
def solve(guess_a, guess_b, power, solver='scipy'):
    """ Constructs a pyneqsys.symbolic.SymbolicSys instance and returns from its ``solve`` method. """
    # The problem is 2 dimensional so we need 2 symbols
    x = sp.symbols('x:2', real=True)
    # There is a user specified parameter ``p`` in this problem:
    p = sp.Symbol('p', real=True, negative=False, integer=True)
    # Our system consists of 2-non-linear equations:
    f = [x[0] + (x[0] - x[1])**p/2 - 1,
         (x[1] - x[0])**p/2 + x[1]]
    # We construct our ``SymbolicSys`` instance by passing variables, equations and parameters:
    neqsys = SymbolicSys(x, f, [p])  # (this will derive the Jacobian symbolically)

    # Finally we solve the system using user-specified ``solver`` choice:
    return neqsys.solve([guess_a, guess_b], [power], solver=solver)


Instruction:
Can you write a function in Python 3 where it
plots the values of the varied parameter vector vs the varied parameter vector.

Response:
def plot_series(xres, varied_data, indices=None, info=None,
                fail_vline=None, plot_kwargs_cb=None,
                ls=('-', '--', ':', '-.'),
                c=('k', 'r', 'g', 'b', 'c', 'm', 'y'), labels=None,
                ax=None, names=None, latex_names=None):
    """ Plot the values of the solution vector vs the varied parameter.

    Parameters
    ----------
    xres : array
        Solution vector of shape ``(varied_data.size, x0.size)``.
    varied_data : array
        Numerical values of the varied parameter.
    indices : iterable of integers, optional
        Indices of variables to be plotted. default: all
    fail_vline : bool
        Show vertical lines where the solver failed.
    plot_kwargs_cb : callable
        Takes the index as single argument, returns a dict passed to
        the plotting function
    ls : iterable of str
        Linestyles.
    c : iterable of str
        Colors.
    labels : iterable of str
    ax : matplotlib Axes instance
    names : iterable of str
    latex_names : iterable of str

    """
    import matplotlib.pyplot as plt

    if indices is None:
        indices = range(xres.shape[1])

    if fail_vline is None:
        if info is None:
            fail_vline = False
        else:
            fail_vline = True

    if ax is None:
        ax = plt.subplot(1, 1, 1)

    if labels is None:
        labels = names if latex_names is None else ['$%s$' % ln.strip('$') for ln in latex_names]

    if plot_kwargs_cb is None:
        def plot_kwargs_cb(idx, labels=None):
            kwargs = {'ls': ls[idx % len(ls)],
                      'c': c[idx % len(c)]}
            if labels:
                kwargs['label'] = labels[idx]
            return kwargs
    else:
        plot_kwargs_cb = plot_kwargs_cb or (lambda idx: {})

    for idx in indices:
        ax.plot(varied_data, xres[:, idx], **plot_kwargs_cb(idx, labels=labels))

    if fail_vline:
        for i, nfo in enumerate(info):
            if not nfo['success']:
                ax.axvline(varied_data[i], c='k', ls='--')
    return ax


Instruction:
Implement a function in Python 3 to
place a legend box outside a matplotlib Axes instance.

Response:
def mpl_outside_legend(ax, **kwargs):
    """ Places a legend box outside a matplotlib Axes instance. """
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.75, box.height])
    # Put a legend to the right of the current axis
    ax.legend(loc='upper left', bbox_to_anchor=(1, 1), **kwargs)


Instruction:
Implement a Python 3 function for
transforming a linear system to reduced row - echelon form.

Response:
def linear_rref(A, b, Matrix=None, S=None):
    """ Transform a linear system to reduced row-echelon form

    Transforms both the matrix and right-hand side of a linear
    system of equations to reduced row echelon form

    Parameters
    ----------
    A : Matrix-like
        Iterable of rows.
    b : iterable

    Returns
    -------
    A', b' - transformed versions

    """
    if Matrix is None:
        from sympy import Matrix
    if S is None:
        from sympy import S
    mat_rows = [_map2l(S, list(row) + [v]) for row, v in zip(A, b)]
    aug = Matrix(mat_rows)
    raug, pivot = aug.rref()
    nindep = len(pivot)
    return raug[:nindep, :-1], raug[:nindep, -1]


Instruction:
Can you tell what is the following Python 3 function doing
def linear_exprs(A, x, b=None, rref=False, Matrix=None):
    if b is None:
        b = [0]*len(x)
    if rref:
        rA, rb = linear_rref(A, b, Matrix)
        if Matrix is None:
            from sympy import Matrix
        return [lhs - rhs for lhs, rhs in zip(rA * Matrix(len(x), 1, x), rb)]
    else:
        return [sum([x0*x1 for x0, x1 in zip(row, x)]) - v
                for row, v in zip(A, b)]

Response: Linear expression of A.


Instruction:
Can you tell what is the following Python 3 function doing
def from_callback(cls, cb, nx=None, nparams=None, **kwargs):
        if kwargs.get('x_by_name', False):
            if 'names' not in kwargs:
                raise ValueError("Need ``names`` in kwargs.")
            if nx is None:
                nx = len(kwargs['names'])
            elif nx != len(kwargs['names']):
                raise ValueError("Inconsistency between nx and length of ``names``.")
        if kwargs.get('par_by_name', False):
            if 'param_names' not in kwargs:
                raise ValueError("Need ``param_names`` in kwargs.")
            if nparams is None:
                nparams = len(kwargs['param_names'])
            elif nparams != len(kwargs['param_names']):
                raise ValueError("Inconsistency between ``nparam`` and length of ``param_names``.")

        if nparams is None:
            nparams = 0

        if nx is None:
            raise ValueError("Need ``nx`` of ``names`` together with ``x_by_name==True``.")
        be = Backend(kwargs.pop('backend', None))
        x = be.real_symarray('x', nx)
        p = be.real_symarray('p', nparams)
        _x = dict(zip(kwargs['names'], x)) if kwargs.get('x_by_name', False) else x
        _p = dict(zip(kwargs['param_names'], p)) if kwargs.get('par_by_name', False) else p
        try:
            exprs = cb(_x, _p, be)
        except TypeError:
            exprs = _ensure_3args(cb)(_x, _p, be)
        return cls(x, exprs, p, backend=be, **kwargs)

Response: Generate a new instance of a SymbolicSys instance from a callback function.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_jac(self):
        if self._jac is True:
            if self.band is None:
                f = self.be.Matrix(self.nf, 1, self.exprs)
                _x = self.be.Matrix(self.nx, 1, self.x)
                return f.jacobian(_x)
            else:
                # Banded
                return self.be.Matrix(banded_jacobian(
                    self.exprs, self.x, *self.band))
        elif self._jac is False:
            return False
        else:
            return self._jac

Response: Returns the jacobian of the expressions


Instruction:
Can you create a Python 3 function that
generates a TransformedSys instance from a callback.

Response:
def from_callback(cls, cb, transf_cbs, nx, nparams=0, pre_adj=None,
                      **kwargs):
        """ Generate a TransformedSys instance from a callback

        Parameters
        ----------
        cb : callable
            Should have the signature ``cb(x, p, backend) -> list of exprs``.
            The callback ``cb`` should return *untransformed* expressions.
        transf_cbs : pair or iterable of pairs of callables
            Callables for forward- and backward-transformations. Each
            callable should take a single parameter (expression) and
            return a single expression.
        nx : int
            Number of unkowns.
        nparams : int
            Number of parameters.
        pre_adj : callable, optional
            To tweak expression prior to transformation. Takes a
            sinlge argument (expression) and return a single argument
            rewritten expression.
        \\*\\*kwargs :
            Keyword arguments passed on to :class:`TransformedSys`. See also
            :class:`SymbolicSys` and :class:`pyneqsys.NeqSys`.

        Examples
        --------
        >>> import sympy as sp
        >>> transformed = TransformedSys.from_callback(lambda x, p, be: [
        ...     x[0]*x[1] - p[0],
        ...     be.exp(-x[0]) + be.exp(-x[1]) - p[0]**-2
        ... ], (sp.log, sp.exp), 2, 1)
        ...

        """
        be = Backend(kwargs.pop('backend', None))
        x = be.real_symarray('x', nx)
        p = be.real_symarray('p', nparams)
        try:
            transf = [(transf_cbs[idx][0](xi),
                       transf_cbs[idx][1](xi))
                      for idx, xi in enumerate(x)]
        except TypeError:
            transf = zip(_map2(transf_cbs[0], x), _map2(transf_cbs[1], x))
        try:
            exprs = cb(x, p, be)
        except TypeError:
            exprs = _ensure_3args(cb)(x, p, be)
        return cls(x, _map2l(pre_adj, exprs), transf, p, backend=be, **kwargs)


Instruction:
Implement a function in Python 3 to
write binary header data to a file handle. This method writes exactly 512 bytes to the beginning of the given file handle. Parameters ---------- handle : file handle The given handle will be reset to 0 using `seek` and then 512 bytes will be written to describe the parameters in this Header. The handle must be writeable.

Response:
def write(self, handle):
        '''Write binary header data to a file handle.

        This method writes exactly 512 bytes to the beginning of the given file
        handle.

        Parameters
        ----------
        handle : file handle
            The given handle will be reset to 0 using `seek` and then 512 bytes
            will be written to describe the parameters in this Header. The
            handle must be writeable.
        '''
        handle.seek(0)
        handle.write(struct.pack(self.BINARY_FORMAT,
                                 self.parameter_block,
                                 0x50,
                                 self.point_count,
                                 self.analog_count,
                                 self.first_frame,
                                 self.last_frame,
                                 self.max_gap,
                                 self.scale_factor,
                                 self.data_block,
                                 self.analog_per_frame,
                                 self.frame_rate,
                                 b'',
                                 self.long_event_labels and 0x3039 or 0x0,
                                 self.label_block,
                                 b''))


Instruction:
Make a summary of the following Python 3 code
def read(self, handle):
        '''Read and parse binary header data from a file handle.

        This method reads exactly 512 bytes from the beginning of the given file
        handle.

        Parameters
        ----------
        handle : file handle
            The given handle will be reset to 0 using `seek` and then 512 bytes
            will be read to initialize the attributes in this Header. The handle
            must be readable.

        Raises
        ------
        AssertionError
            If the magic byte from the header is not 80 (the C3D magic value).
        '''
        handle.seek(0)
        (self.parameter_block,
         magic,
         self.point_count,
         self.analog_count,
         self.first_frame,
         self.last_frame,
         self.max_gap,
         self.scale_factor,
         self.data_block,
         self.analog_per_frame,
         self.frame_rate,
         _,
         self.long_event_labels,
         self.label_block,
         _) = struct.unpack(self.BINARY_FORMAT, handle.read(512))

        assert magic == 80, 'C3D magic {} != 80 !'.format(magic)

Response: Read and parse binary header data from a file handle.


Instruction:
Can you write a function in Python 3 where it
returns the number of bytes needed to store this parameter.

Response:
def binary_size(self):
        '''Return the number of bytes needed to store this parameter.'''
        return (
            1 + # group_id
            2 + # next offset marker
            1 + len(self.name.encode('utf-8')) + # size of name and name bytes
            1 + # data size
            1 + len(self.dimensions) + # size of dimensions and dimension bytes
            self.total_bytes + # data
            1 + len(self.desc.encode('utf-8')) # size of desc and desc bytes
            )