Instruction:
Create a Python 3 function to
find all the hypernyms of the synset transitively. Notes ----- Internal method. Do not call directly. Parameters ---------- hypernyms : set of Synsets An set of hypernyms met so far. Returns ------- set of Synsets Returns the input set.

Response:
def _recursive_hypernyms(self, hypernyms):
        """Finds all the hypernyms of the synset transitively.

        Notes
        -----
          Internal method. Do not call directly.

        Parameters
        ----------
          hypernyms : set of Synsets
        An set of hypernyms met so far.

        Returns
        -------
          set of Synsets
        Returns the input set.

        """
        hypernyms |= set(self.hypernyms())

        for synset in self.hypernyms():
            hypernyms |= synset._recursive_hypernyms(hypernyms)
        return hypernyms


Instruction:
Create a Python 3 function for
finding minimum path length from the root. Returns 0 if no path length is found.

Response:
def _min_depth(self):
        """Finds minimum path length from the root.

        Notes
        -----
          Internal method. Do not call directly.

        Returns
        -------
          int
        Minimum path length from the root.

        """
        if "min_depth" in self.__dict__:
            return self.__dict__["min_depth"]

        min_depth = 0
        hypernyms = self.hypernyms()
        if hypernyms:
            min_depth = 1 + min(h._min_depth() for h in hypernyms)
        self.__dict__["min_depth"] = min_depth

        return min_depth


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_related_synsets(self,relation):
        results = []

        for relation_candidate in self._raw_synset.internalLinks:
            if relation_candidate.name == relation:
                linked_synset = synset(_get_key_from_raw_synset(relation_candidate.target_concept))
                relation_candidate.target_concept = linked_synset._raw_synset
                results.append(linked_synset)
        return results

Response: Retrieves all the synsets which are related by given relation.


Instruction:
Write a Python 3 script to
find all the ancestors of the synset using the given relation.

Response:
def closure(self, relation, depth=float('inf')):
        """Finds all the ancestors of the synset using provided relation.

        Parameters
        ----------
          relation : str
        Name of the relation which is recursively used to fetch the ancestors.

        Returns
        -------
          list of Synsets
        Returns the ancestors of the synset via given relations.

        """

        ancestors = []
        unvisited_ancestors = [(synset,1) for synset in self.get_related_synsets(relation)]

        while len(unvisited_ancestors) > 0:
            ancestor_depth = unvisited_ancestors.pop()
            if ancestor_depth[1] > depth:
                continue
            unvisited_ancestors.extend([(synset,ancestor_depth[1]+1) for synset in ancestor_depth[0].get_related_synsets(relation)])
            ancestors.append(ancestor_depth[0])

        return list(set(ancestors))


Instruction:
How would you explain what the following Python 3 function does
def root_hypernyms(self):
        visited = set()
        hypernyms_next_level = set(self.hypernyms())
        current_hypernyms = set(hypernyms_next_level)

        while len(hypernyms_next_level) > 0:
            current_hypernyms = set(hypernyms_next_level)
            hypernyms_next_level = set()

            for synset in current_hypernyms:
                if synset in visited:
                    continue
                visited.add(synset)
                hypernyms_next_level |= set(synset.hypernyms())

        return list(current_hypernyms)

Response: Retrieves all the root hypernyms.


Instruction:
Can you write a function in Python 3 where it
calculates Leacock and Chodorow s similarity between two synsets.

Response:
def lch_similarity(self, synset):
        """Calculates Leacock and Chodorow's similarity between the two synsets.

        Notes
        -----
          Similarity is calculated using the formula -log( (dist(synset1,synset2)+1) / (2*maximum taxonomy depth) ).

        Parameters
        ----------
          synset : Synset
        Synset from which the similarity is calculated.

        Returns
        -------
          float
        Leacock and Chodorow's from `synset`.
        None, if synsets are not connected via hypernymy/hyponymy relations. Obvious, if part-of-speeches don't match.

        """

        if self._raw_synset.pos != synset._raw_synset.pos:
            return None

        depth = MAX_TAXONOMY_DEPTHS[self._raw_synset.pos]

        distance = self._shortest_path_distance(synset)

        if distance >= 0:
            return -math.log((distance + 1) / (2.0 * depth))
        else:
            return None


Instruction:
Make a summary of the following Python 3 code
def wup_similarity(self, target_synset):
        lchs = self.lowest_common_hypernyms(target_synset)
        lcs_depth = lchs[0]._min_depth() if lchs and len(lchs) else None
        self_depth = self._min_depth()
        other_depth = target_synset._min_depth()
        if lcs_depth is None or self_depth is None or other_depth is None:
            return None

        return (2.0 * lcs_depth) / (self_depth + other_depth)

Response: Calculates the Wu and Palmer s similarity between two synsets.


Instruction:
How would you code a function in Python 3 to
return the definition of the synset as a new - line separated concatenated string from all its variants

Response:
def definition(self):
        """Returns the definition of the synset.

        Returns
        -------
          str
        Definition of the synset as a new-line separated concatenated string from all its variants' definitions.

        """
        return '\n'.join([variant.gloss for variant in self._raw_synset.variants if variant.gloss])


Instruction:
Given the following Python 3 function, write the documentation
def examples(self):
        examples = []
        for example in [variant.examples for variant in self._raw_synset.variants if len(variant.examples)]:
            examples.extend(example)
        return examples

Response: Returns the examples of the synset.


Instruction:
Write a Python 3 function for
returning the synset s lemmas as Lemmas as strings.

Response:
def lemmas(self):
        """Returns the synset's lemmas/variants' literal represantions.

        Returns
        -------
          list of Lemmas
        List of its variations' literals as Lemma objects.

        """
        return [lemma("%s.%s"%(self.name,variant.literal)) for variant in self._raw_synset.variants]


Instruction:
Here you have a function in Python 3, explain what it does
def lowest_common_hypernyms(self,target_synset):
        self_hypernyms = self._recursive_hypernyms(set())
        other_hypernyms = target_synset._recursive_hypernyms(set())
        common_hypernyms = self_hypernyms.intersection(other_hypernyms)

        annot_common_hypernyms = [(hypernym, hypernym._min_depth()) for hypernym in common_hypernyms]

        annot_common_hypernyms.sort(key = lambda annot_hypernym: annot_hypernym[1],reverse=True)

        max_depth = annot_common_hypernyms[0][1] if len(annot_common_hypernyms) > 0 else None

        if max_depth != None:
            return [annot_common_hypernym[0] for annot_common_hypernym in annot_common_hypernyms if annot_common_hypernym[1] == max_depth]
        else:
            return None

Response: Returns the lowest common hypernyms of the synset and the target synset.


Instruction:
How would you code a function in Python 3 to
return a synset in which the given lemma belongs to.

Response:
def synset(self):
        """Returns synset into which the given lemma belongs to.

        Returns
        -------
          Synset
        Synset into which the given lemma belongs to.

        """
        return synset('%s.%s.%s.%s'%(self.synset_literal,self.synset_pos,self.synset_sense,self.literal))


Instruction:
Can you tell what is the following Python 3 function doing
def convert_vm_json_to_mrf( vabamorf_json ):
    ''' Converts from vabamorf's JSON output, given as dict, into pre-syntactic mrf
        format, given as a list of lines, as in the output of etmrf.
        The aimed format looks something like this:
        <s>
        Kolmandaks
            kolmandaks+0 //_D_  //
            kolmas+ks //_O_ sg tr //
        kihutas
            kihuta+s //_V_ s //
        end
            end+0 //_Y_ ? //
            ise+0 //_P_ sg p //
        soomlane
            soomlane+0 //_S_ sg n //
        </s>
    '''
    if not isinstance( vabamorf_json, dict ):
       raise Exception(' Expected dict as an input argument! ')
    json_sentences = []
    # 1) flatten paragraphs
    if 'paragraphs' in vabamorf_json:
       for pr in vabamorf_json['paragraphs']:
           if 'sentences' in pr:
              for sent in pr['sentences']:
                 json_sentences.append( sent )
    # 2) flatten sentences
    elif 'sentences' in vabamorf_json:
       for sent in vabamorf_json['sentences']:
           json_sentences.append( sent )
    # 3) Iterate over sentences and perform conversion
    results = []
    for sentJson in json_sentences:
        results.append('<s>')
        for wordJson in sentJson['words']:
            if wordJson['text'] == '<s>' or wordJson['text'] == '</s>':
               continue
            wordStr = wordJson['text']
            # Escape double quotation marks
            wordStr = _esc_double_quotes( wordStr )
            results.append( wordStr )
            for analysisJson in wordJson['analysis']:
               root   = analysisJson['root']
               root   = _esc_double_quotes( root )
               #   NB! ending="0" erineb ending=""-st:
               #     1) eestlane (ending="0");
               #     2) Rio (ending="") de (ending="") Jaineros;
               ending = analysisJson[ENDING]
               pos    = analysisJson['partofspeech']
               clitic = analysisJson['clitic']
               form   = analysisJson['form']
               if pos == 'Z':
                  results.append( ''.join(['    ',root,' //_Z_ //']) )
               else:
                  results.append( ''.join(['    ',root,'+',ending,clitic,' //', '_',pos,'_ ',form,' //']) )
            if 'analysis' not in wordJson:
               results.append( '    '+'####' )
        results.append('</s>')
    return results

Response: Converts from JSON output into pre - syntactic mrf format.


Instruction:
Write a Python 3 script to
convert a Text object into a list of pre - syntactic mrf format given as a list of lines.

Response:
def convert_Text_to_mrf( text ):
    ''' Converts from Text object into pre-syntactic mrf format, given as a list of
        lines, as in the output of etmrf.
        *) If the input Text has already been morphologically analysed, uses the existing
            analysis;
        *) If the input has not been analysed, performs the analysis with required settings:
            word quessing is turned on, proper-name analyses are turned off;
    '''
    from estnltk.text import Text
    if not isinstance( text, Text ):
       raise Exception(' Expected estnltk\'s Text as an input argument! ')
    if not text.is_tagged( ANALYSIS ):
       # If morphological analysis has not been performed yet, set the right arguments and
       # perform the analysis
       kwargs = text.get_kwargs()
       kwargs['vabamorf']     = True
       kwargs['guess']        = True
       kwargs['propername']   = False
       kwargs['disambiguate'] = False
       text.__kwargs = kwargs
       text = text.tag_analysis()
    # Iterate over sentences and perform conversion
    results = []
    for sentence in text.divide( layer=WORDS, by=SENTENCES ):
       results.append('<s>')
       for i in range(len(sentence)):
           wordJson = sentence[i]
           wordStr  = wordJson[TEXT]
           # Escape double quotation marks
           wordStr = _esc_double_quotes( wordStr )
           results.append( wordStr )
           for analysisJson in wordJson[ANALYSIS]:
               root   = analysisJson[ROOT]
               root   = _esc_double_quotes( root )
               #   NB! ending="0" erineb ending=""-st:
               #     1) eestlane (ending="0");
               #     2) Rio (ending="") de (ending="") Jaineros;
               ending = analysisJson[ENDING]
               pos    = analysisJson[POSTAG]
               clitic = analysisJson[CLITIC]
               form   = analysisJson[FORM]
               if pos == 'Z':
                  results.append( ''.join(['    ',root,' //_Z_ //']) )
               else:
                  results.append( ''.join(['    ',root,'+',ending,clitic,' //', '_',pos,'_ ',form,' //']) )
           if ANALYSIS not in wordJson:
               results.append( '    '+'####' )
       results.append('</s>')
    return results


Instruction:
Write a Python 3 script for
loading rules that can be used to convert from Filosoft s mrf format to syntactic analyzer s format. Returns a dict containing rules that can be used to convert from Filosoft s mrf format to syntactic analyzer s format.

Response:
def load_fs_mrf_to_syntax_mrf_translation_rules( rulesFile ):
    ''' Loads rules that can be used to convert from Filosoft's mrf format to
        syntactic analyzer's format. Returns a dict containing rules.

        Expects that each line in the input file contains a single rule, and that
        different parts of the rule separated by @ symbols, e.g.

            1@_S_ ?@Substantiiv apellatiiv@_S_ com @Noun common@Nc@NCSX@kesk-
            32@_H_ ?@Substantiiv prooprium@_S_ prop @Noun proper@Np@NPCSX@Kesk-
            313@_A_@Adjektiiv positiiv@_A_ pos@Adjective positive@A-p@ASX@salkus

        Only 2nd element and 4th element are extracted from each line; 2nd element
        will be the key of the dict entry, and 4th element will be added to the
        value of the dict entry (the value is a list of strings);

        A list is used for storing values because one Filosoft's analysis could
        be mapped to multiple syntactic analyzer's analyses;

        Lines that have ¤ in the beginning of the line will be skipped;

    '''
    rules = {}
    in_f = codecs.open(rulesFile, mode='r', encoding='utf-8')
    for line in in_f:
        line = line.rstrip()
        if line.startswith('¤'):
           continue
        parts = line.split('@')
        if len(parts) < 4:
           raise Exception(' Unexpected format of the line: ', line)
        if parts[1] not in rules:
           rules[parts[1]] = []
        rules[parts[1]].append( parts[3] )
    in_f.close()
    return rules


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _convert_punctuation( line ):
    ''' Converts given analysis line if it describes punctuation; Uses the set
        of predefined punctuation conversion rules from _punctConversions;

        _punctConversions should be a list of lists, where each outer list stands
        for a single conversion rule and inner list contains a pair of elements:
        first is the regexp pattern and the second is the replacement, used in
           re.sub( pattern, replacement, line )

        Returns the converted line (same as input, if no conversion was
        performed);
    '''
    for [pattern, replacement] in _punctConversions:
        lastline = line
        line = re.sub(pattern, replacement, line)
        if lastline != line:
            break
    return line

Response: Converts given analysis line if it describes punctuation.


Instruction:
How would you code a function in Python 3 to
convert given lines from Filosoft s mrf format to syntactic analyzer s equivalent MRF format.

Response:
def convert_mrf_to_syntax_mrf( mrf_lines, conversion_rules ):
    ''' Converts given lines from Filosoft's mrf format to syntactic analyzer's
        format, using the morph-category conversion rules from conversion_rules,
        and punctuation via method _convert_punctuation();
        As a result of conversion, the input list  mrf_lines  will be modified,
        and also returned after a successful conversion;

        Morph-category conversion rules should be loaded via method
            load_fs_mrf_to_syntax_mrf_translation_rules( rulesFile ),
        usually from a file named 'tmorftrtabel.txt';

        Note that the resulting list of lines likely has more lines than the
        original list had, because the conversion often requires that the
        original Filosoft's analysis is expanded into multiple analyses
        suitable for the syntactic analyzer;
    '''
    i = 0
    while ( i < len(mrf_lines) ):
        line = mrf_lines[i]
        if line.startswith('  '):  # only consider lines of analysis
            # 1) Convert punctuation
            if _punctOrAbbrev.search(line):
                mrf_lines[i] = _convert_punctuation( line )
                if '_Y_' not in line:
                    i += 1
                    continue
            # 2) Convert morphological analyses that have a form specified
            withFormMatch = _morfWithForm.search(line)
            if withFormMatch:
                root    = withFormMatch.group(1)
                pos     = withFormMatch.group(2)
                formStr = withFormMatch.group(3)
                forms   = formStr.split(',')
                all_new_lines = []
                for form in forms:
                    morphKey = pos+' '+form.strip()
                    if morphKey in conversion_rules:
                        newlines = [ '    '+root+' //'+_esc_que_mark(r)+' //' for r in conversion_rules[morphKey] ]
                        all_new_lines.extend( newlines )
                if all_new_lines:
                    del mrf_lines[i]
                    for newline in all_new_lines:
                        mrf_lines.insert(i, newline)
                    i += len(newlines)
                    continue
            else:
                withoutFormMatch = _morfWithoutForm.search(line)
                if withoutFormMatch:
                    # 3) Convert morphological analyses that have only POS specified
                    root = withoutFormMatch.group(1)
                    pos  = withoutFormMatch.group(2)
                    morphKey = pos
                    all_new_lines = []
                    if morphKey in conversion_rules:
                        newlines = [ '    '+root+' //'+_esc_que_mark(r)+' //' for r in conversion_rules[morphKey] ]
                        all_new_lines.extend( newlines )
                    if all_new_lines:
                        del mrf_lines[i]
                        for newline in all_new_lines:
                            mrf_lines.insert(i, newline)
                        i += len(newlines)
                        continue
        i += 1
    return mrf_lines


Instruction:
Can you create a Python 3 function that
converts pronouns from Filosoft s mrf to

Response:
def convert_pronouns( mrf_lines ):
    ''' Converts pronouns (analysis lines with '_P_') from Filosoft's mrf to
        syntactic analyzer's mrf format;
        Uses the set of predefined pronoun conversion rules from _pronConversions;

        _pronConversions should be a list of lists, where each outer list stands
        for a single conversion rule and inner list contains a pair of elements:
        first is the regexp pattern and the second is the replacement, used in
           re.sub( pattern, replacement, line )

        Returns the input mrf list, with the lines converted from one format
        to another;
    '''
    i = 0
    while ( i < len(mrf_lines) ):
        line = mrf_lines[i]
        if '_P_' in line:  # only consider lines containing pronoun analyses
           for [pattern, replacement] in _pronConversions:
               lastline = line
               line = re.sub(pattern, replacement, line)
               if lastline != line:
                  mrf_lines[i] = line
                  break
        i += 1
    return mrf_lines


Instruction:
Explain what the following Python 3 code does
def remove_duplicate_analyses( mrf_lines, allow_to_delete_all = True ):
    ''' Removes duplicate analysis lines from mrf_lines.

        Uses special logic for handling adposition analyses ('_K_ pre' && '_K_ post')
        that do not have subcategorization information:
         *) If a word has both adposition analyses, removes '_K_ pre';
         *) If a word has '_K_ post', removes it;
        Note that '_K_ pre' and '_K_ post' with subcategorization information will
        be kept.

        The parameter  allow_to_delete_all  specifies whether it is allowed to delete
        all analysis or not. If allow_to_delete_all == False, then one last analysis
        won't be deleted, regardless whether it should be deleted considering the
        adposition-deletion rules;
        The original implementation corresponds to the settings allow_to_delete_all=True
        (and this is also the default value of the parameter);

        Returns the input list where the removals have been applied;
    '''
    i = 0
    seen_analyses  = []
    analyses_count = 0
    to_delete      = []
    Kpre_index     = -1
    Kpost_index    = -1
    while ( i < len(mrf_lines) ):
        line = mrf_lines[i]
        if not line.startswith('  '):
           if Kpre_index != -1 and Kpost_index != -1:
              # If there was both _K_pre and _K_post, add _K_pre to removables;
              to_delete.append( Kpre_index )
           elif Kpost_index != -1:
              # If there was only _K_post, add _K_post to removables;
              to_delete.append( Kpost_index )
           # Delete found duplicates
           if to_delete:
              for k, j in enumerate(sorted(to_delete, reverse=True)):
                  # If we must preserve at least one analysis, and
                  # it has been found that all should be deleted, then
                  # keep the last one
                  if not allow_to_delete_all and \
                     analyses_count == len(to_delete) and \
                     k == len(to_delete) - 1:
                     continue
                  # Delete the analysis line
                  del mrf_lines[j]
                  i -= 1
           # Reset the memory for each new word/token
           seen_analyses = []
           analyses_count = 0
           to_delete     = []
           Kpre_index    = -1
           Kpost_index   = -1
        elif line.startswith('  '):   # the line of analysis
           analyses_count += 1
           if line in seen_analyses:
              # Remember line that has been already seen as a duplicate
              to_delete.append( i )
           else:
              # Remember '_K pre' and '_K_ post' indices
              if re.search('/_K_\s+pre\s+//', line):
                 Kpre_index  = i
              elif re.search('/_K_\s+post\s+//', line):
                 Kpost_index = i
              # Remember that the line has already been seen
              seen_analyses.append( line )
        i += 1
    return mrf_lines

Response: Removes duplicate analyses from the input list.


Instruction:
Can you tell what is the following Python 3 function doing
def add_hashtag_info( mrf_lines ):
    ''' Augments analysis lines with various hashtag information:
          *) marks words with capital beginning with #cap;
          *) marks finite verbs with #FinV;
          *) marks nud/tud/mine/nu/tu/v/tav/mata/ja forms;
        Hashtags are added at the end of the analysis content (just before the
        last '//');

        Returns the input list where the augmentation has been applied;
    '''
    i   = 0
    cap = False
    while ( i < len(mrf_lines) ):
        line = mrf_lines[i]
        if not line.startswith('  ') and len(line) > 0:
           cap = (line[0]).isupper()
        elif line.startswith('  '):
           if cap:
              line = re.sub('(//.+\S)\s+//', '\\1 #cap //', line)
           if _morfFinV.search( line ) and not _morfNotFinV.search( line ):
              line = re.sub('(//.+\S)\s+//', '\\1 #FinV //', line)
           for [pattern, replacement] in _mrfHashTagConversions:
               line = re.sub(pattern, replacement, line)
           mrf_lines[i] = line
        i += 1
    return mrf_lines

Response: Adds hashtag information to the input list.


Instruction:
Here you have a function in Python 3, explain what it does
def load_subcat_info( subcat_lex_file ):
    ''' Loads subcategorization rules (for verbs and adpositions) from a text
        file.

        It is expected that the rules are given as pairs, where the first item is
        the lemma (of verb/adposition), followed on the next line by the
        subcategorization rule, in the following form:
           on the left side of '>' is the condition (POS-tag requirement for the
           lemma),
         and
           on the right side is the listing of subcategorization settings (hashtag
           items, e.g. names of morphological cases of nominals);
        If there are multiple subcategorization rules to be associated with a
        single lemma, different rules are separated by '&'.

        Example, an excerpt from the rules file:
          läbi
          _V_ >#Part &_K_ post >#gen |#nom |#el &_K_ pre >#gen
          läbista
          _V_ >#NGP-P
          läbistu
          _V_ >#Intr

        Returns a dict of lemma to a-list-of-subcatrules mappings.
    '''
    rules = {}
    nonSpacePattern = re.compile('^\S+$')
    posTagPattern   = re.compile('_._')
    in_f = codecs.open(subcat_lex_file, mode='r', encoding='utf-8')
    lemma = ''
    subcatRules = ''
    for line in in_f:
        line = line.rstrip()
        if nonSpacePattern.match(line) and not posTagPattern.search(line):
           lemma = line
        elif posTagPattern.search(line):
           subcatRules = line
        if len(lemma) > 0 and len(subcatRules) > 0:
           if lemma not in rules:
              rules[lemma] = []
           parts = subcatRules.split('&')
           for part in parts:
              part = part.strip()
              rules[lemma].append( part )
           lemma = ''
           subcatRules = ''
    in_f.close()
    #print( len(rules.keys()) )   # 4484
    return rules

Response:
Loads the subcategorization rules from a text
        file.


Instruction:
Can you generate the documentation for the following Python 3 function
def tag_subcat_info( mrf_lines, subcat_rules ):
    ''' Adds subcategorization information (hashtags) to verbs and adpositions;

        Argument subcat_rules must be a dict containing subcategorization information,
        loaded via method load_subcat_info();

        Performs word lemma lookups in subcat_rules, and in case of a match, checks
        word part-of-speech conditions. If the POS conditions match, adds subcategorization
        information either to a single analysis line, or to multiple analysis lines
        (depending on the exact conditions in the rule);

        Returns the input list where verb/adposition analyses have been augmented
        with available subcategorization information;
    '''
    i = 0
    while ( i < len(mrf_lines) ):
        line = mrf_lines[i]
        if line.startswith('  '):
           lemma_match = analysisLemmaPat.match(line)
           if lemma_match:
              lemma = lemma_match.group(1)
              # Find whether there is subcategorization info associated
              # with the lemma
              if lemma in subcat_rules:
                 analysis_match = analysisPat.search(line)
                 if not analysis_match:
                    raise Exception(' Could not find analysis from the line:',line)
                 analysis = analysis_match.group(1)
                 for rule in subcat_rules[lemma]:
                     condition, addition = rule.split('>')
                     # Check the condition string; If there are multiple conditions,
                     # all must be satisfied for the rule to fire
                     condition  = condition.strip()
                     conditions = condition.split()
                     satisfied1 = [ _check_condition(c, analysis) for c in conditions ]
                     if all( satisfied1 ):
                        #
                        # There can be multiple additions:
                        #   1) additions without '|' must be added to a single analysis line;
                        #   2) additions separated by '|' must be placed on separate analysis
                        #      lines;
                        #
                        additions = addition.split('|')
                        j = i
                        # Add new line or lines
                        for a in additions:
                            line_copy = line if i == j else line[:]
                            items_to_add = a.split()
                            for item in items_to_add:
                                if not _check_condition(item, analysis):
                                   line_copy = \
                                       re.sub('(//.+\S)\s+//', '\\1 '+item+' //', line_copy)
                            if j == i:
                               # 1) replace the existing line
                               mrf_lines[i] = line_copy
                            else:
                               # 2) add a new line
                               mrf_lines.insert(i, line_copy)
                            j += 1
                        i = j - 1
                        # No need to search forward
                        break
        i += 1
    return mrf_lines

Response: Given a list of lines and a dictionary of subcat rules returns a list of subcategorization information associated with the word - of - speech analyses.


Instruction:
Explain what the following Python 3 code does
def convert_to_cg3_input( mrf_lines ):
    ''' Converts given mrf lines from syntax preprocessing format to cg3 input
        format:
          *) surrounds words/tokens with "< and >"
          *) surrounds word lemmas with " in analysis;
          *) separates word endings from lemmas in analysis, and adds prefix 'L';
          *) removes '//' and '//' from analysis;
          *) converts hashtags to tags surrounded by < and >;
          ... and provides other various fix-ups;

        Returns the input list, where elements (tokens/analyses) have been converted
        into the new format;
    '''
    i = 0
    while ( i < len(mrf_lines) ):
        line = mrf_lines[i]
        if not line.startswith('  ') and len(line) > 0:
           #
           # A line containing word/token
           #
           #  a. surround the word with "< and >"
           line = re.sub('^(\S.*)([\n\r]*)$','"<\\1>"\\2', line)
           #  b. fix the sentence begin/end tags
           line = re.sub('<<(s|/s)>>', '<\\1>', line)
           mrf_lines[i] = line
        elif line.startswith('  '):
           #
           # A line containing analysis
           #
           #  1. perform various fixes:
           line = re.sub('#cap #cap','cap', line)
           line = re.sub('#cap','cap', line)
           line = re.sub('\*\*CLB','CLB', line)
           line = re.sub('#Correct!','<Correct!>', line)
           line = re.sub('####','', line)
           line = re.sub('#(\S+)','<\\1>', line)
           line = re.sub('\$([,.;!?:<]+)','\\1', line)
           line = re.sub('_Y_\s+\? _Z_','_Z_', line)
           line = re.sub('_Y_\s+\?\s+_Z_','_Z_', line)
           line = re.sub('_Y_\s+_Z_','_Z_', line)
           line = re.sub('_Z_\s+\?','_Z_', line)
           #  2. convert analysis line \w word ending
           line = re.sub('^\s+(\S+)(.*)\+(\S+)\s*//_(\S)_ (.*)//(.*)$', \
                         '    "\\1\\2" L\\3 \\4 \\5 \\6', line)
           #  3. convert analysis line \wo word ending
           line = re.sub('^\s+(\S+)(.*)\s+//_(\S)_ (.*)//(.*)$', \
                         '    "\\1\\2" \\3 \\4 \\5', line)
           mrf_lines[i] = line
        i += 1
    return mrf_lines

Response: Converts given mrf lines from syntax preprocessing format to cg3 input format.


Instruction:
Create a Python 3 function to
execute the preprocessing pipeline on the JSON given as a dict ; returns a list of lines of analyses in the VISL CG3 input format ;

Response:
def process_vm_json( self, json_dict, **kwargs ):
        ''' Executes the preprocessing pipeline on vabamorf's JSON, given as a dict;

            Returns a list: lines of analyses in the VISL CG3 input format;
        '''
        mrf_lines = convert_vm_json_to_mrf( json_dict )
        return self.process_mrf_lines( mrf_lines, **kwargs )


Instruction:
Write a Python 3 script to
execute the preprocessing pipeline on the estnltk s Text object.

Response:
def process_Text( self, text, **kwargs ):
        ''' Executes the preprocessing pipeline on estnltk's Text object.

            Returns a list: lines of analyses in the VISL CG3 input format;
        '''
        mrf_lines = convert_Text_to_mrf( text )
        return self.process_mrf_lines( mrf_lines, **kwargs )


Instruction:
Here you have a function in Python 3, explain what it does
def process_mrf_lines( self, mrf_lines, **kwargs ):
        ''' Executes the preprocessing pipeline on mrf_lines.

            The input should be an analysis of the text in Filosoft's old mrf format;

            Returns the input list, where elements (tokens/analyses) have been converted
            into the new format;
        '''
        converted1 = convert_mrf_to_syntax_mrf( mrf_lines, self.fs_to_synt_rules )
        converted2 = convert_pronouns( converted1 )
        converted3 = remove_duplicate_analyses( converted2, allow_to_delete_all=self.allow_to_remove_all )
        converted4 = add_hashtag_info( converted3 )
        converted5 = tag_subcat_info( converted4, self.subcat_rules )
        converted6 = remove_duplicate_analyses( converted5, allow_to_delete_all=self.allow_to_remove_all )
        converted7 = convert_to_cg3_input( converted6 )
        return converted7

Response: Executes the preprocessing pipeline on mrf_lines.


Instruction:
Write a Python 3 function for
functioning to get a list of files ending with ending in src_dir.

Response:
def get_sources(src_dir='src', ending='.cpp'):
    """Function to get a list of files ending with `ending` in `src_dir`."""
    return [os.path.join(src_dir, fnm) for fnm in os.listdir(src_dir) if fnm.endswith(ending)]


Instruction:
Write a Python 3 script for
returning ANSI escape code corresponding to switching the font to given color.

Response:
def _get_ANSI_colored_font( color ):
    ''' Returns an ANSI escape code (a string) corresponding to switching the font
        to given color, or None, if the given color could not be associated with
        the available colors.

        See also:
          https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
          http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python
    '''
    color = (color.replace('-','')).lower()
    #
    #   Bright colors:
    #
    if color == 'white':
        return '\033[97m'
    elif color in ['cyan', 'aqua']:
        return '\033[96m'
    elif color in ['purple', 'magneta', 'fuchsia']:
        return '\033[95m'
    elif color == 'blue':
        return '\033[94m'
    elif color in ['yellow', 'gold']:
        return '\033[93m'
    elif color in ['green', 'lime']:
        return '\033[92m'
    elif color == 'red':
        return '\033[91m'
    #
    #   Dark colors:
    #
    elif color in ['grey', 'gray', 'silver']:
        return '\033[37m'
    elif color in ['darkcyan', 'teal']:
        return '\033[36m'
    elif color in ['darkpurple', 'darkmagneta']:
        return '\033[35m'
    elif color in ['darkblue', 'navy']:
        return '\033[34m'
    elif color in ['darkyellow', 'olive']:
        return '\033[33m'
    elif color == 'darkgreen':
        return '\033[32m'
    elif color in ['darkred', 'maroon']:
        return '\033[31m'
    return None


Instruction:
How would you code a function in Python 3 to
construct an index that stores all annotations starting from the given text layer and stores them into the given index.

Response:
def _construct_start_index(text, layer, markup_settings, spansStartingFrom=None):
    ''' Creates an index which stores all annotations of given text layer,
        indexed by the start position of the annotation (annotation[START]).
        Alternatively, if the index spansStartingFrom is already provided
        as an input argument, obtains annotation information from the given
        text layer, and stores into the index;

        The method also creates an ( ANSI-terminal compatible ) mark-up of
        the annotations (generates start and end tags), following the
        specification in markup_settings. The markup_settings should be a
        dict, setting at least one of the following visualisation options:
          * 'bracket' : True -- annotations will be surrounded with brackets;
                                This works in any terminal;
          * 'underline' : True -- annotations will be underlined; This works in
                                  an ANSI compatible terminal;
          * 'color' : ('red', 'green', 'blue' etc. ) -- annotated text will be
                                                        displayed in given color;
                                                        this works in an ANSI
                                                        compatible terminal;

        Each start position (in the index) is associated with a list of
        annotation objects (annotations starting from that position).
        An annotation object is a list containing the following information:
         *) START position (of the annotation),
         *) END position (of the annotation),
         *) layer (name),
         *) startTags -- graphic or textual formatting of the start tag,
         *) endTags -- graphic or textual formatting of the end tag,
         *) graphicFormatting -- boolean (whether graphic formatting was used?),
         *) bracketing -- boolean (whether bracketing was used?),

        Multiple annotation objects starting from the same position are sorted by
        their length: longer annotations preceding the shorter ones;

        The method returns created (or augmented) index (a dict object indexed
        by START positions);
    '''
    if not markup_settings or not isinstance(markup_settings, dict):
       raise Exception('Error: markup_settings should be a dict containing markup specification;')
    # ----------------------------
    # 1) Construct start and end tags, considering the formatting settings
    startTags = ''
    endTags   = ''
    graphicFormatting = False
    bracketing        = False
    # -- Underlining
    if ('u' in markup_settings and markup_settings['u']) or \
       ('underline' in markup_settings and markup_settings['underline']):
       startTags += '\033[4m'
       endTags   += '\033[0m'
       graphicFormatting = True
    colorName = markup_settings['c'] if 'c' in markup_settings else None
    colorName = markup_settings['color'] if 'color' in markup_settings else colorName
    # -- Coloring
    if colorName:
       color = _get_ANSI_colored_font( colorName )
       if color:
          startTags += color
          endTags   += '\033[0m'
          graphicFormatting = True
       else:
          raise Exception('Unknown color:', colorName)
    # -- Bracketing
    if ('b' in markup_settings and markup_settings['b']) or \
       ('bracket' in markup_settings and markup_settings['bracket']):
       startTags += '['
       # Add ending bracket before graphics ends (otherwise the
       # graphics have no effect on the ending bracket)
       endTags    = ']'+endTags
       bracketing = True
    # Hack: if both bracketing and graphic formatting are used, add graphic
    #       formatting before the closing bracket of the endTag (to ensure
    #       that graphic formatting of the ending bracket is not overwritten
    #       mistakenly);
    if graphicFormatting and bracketing:
       startTags2 = startTags.rstrip('[')
       endTags    = startTags2+endTags

    # ----------------------------
    # 2) Get extractor for the elements of given layer
    # >>> The following code borrows from estnltk.prettyprinter.marker :
    # decide which extractor to use
    # first just assume we need to use a multi layer text extractor
    extractor = lambda t: texts_multi(t, layer)
    # if user has specified his/her own callable, use it
    if hasattr(layer, '__call__'):
        extractor = layer
    elif text.is_simple(layer):
        # the given layer is simple, so use simple text extractor
        extractor = lambda t: texts_simple(t, layer)
    # >>>

    # ----------------------------
    # 3) Store an annotation for each span of given layer
    if not spansStartingFrom:
        spansStartingFrom = {}
    for elem in extractor(text):
        if elem[START] not in spansStartingFrom:
           spansStartingFrom[elem[START]] = []
        span1 = [elem[START], elem[END], layer, startTags, endTags, graphicFormatting, bracketing]
        # Insert the span into the index
        if not spansStartingFrom[elem[START]]:
           spansStartingFrom[elem[START]].append( span1 )
        else:
           # Make sure that spans are inserted in the order of decreasing length:
           #  longer spans preceding the shorter ones;
           inserted = False
           for i in range( len(spansStartingFrom[elem[START]]) ):
               span2 = spansStartingFrom[elem[START]][i]
               # If an existing span is shorter than the current span, insert the
               # current span before the existing span ...
               if span1[1] > span2[1]:
                  spansStartingFrom[elem[START]].insert( i, span1 )
                  inserted = True
                  break
               elif span1[1] == span2[1] and span1[2] < span2[2]:
                  # If both spans have equal length, order the spans in the alphabetical
                  # order of layer names:
                  spansStartingFrom[elem[START]].insert( i, span1 )
                  inserted = True
                  break
           if not inserted:
               spansStartingFrom[elem[START]].append(span1)
    return spansStartingFrom


Instruction:
How would you code a function in Python 3 to
construct an index that stores all the annotations ending at the given position.

Response:
def _construct_end_index( spansStartingFrom ):
   ''' Creates an index which stores all annotations (from spansStartingFrom)
       by their end position in text (annotation[END]).

       Each start position (in the index) is associated with a list of
       annotation objects (annotations ending at that position).
       An annotation object is also a list containing the following information:
        *) endTags -- graphic or textual formatting of the end tag,
        *) START position (of the annotation);
        *) layer name;

       Multiple annotation objects ending at the same position are sorted by
       their length: shorter annotations preceding the longer ones;
   '''
   endIndex = {}
   for i in spansStartingFrom:
       for span1 in spansStartingFrom[i]:
           # keep the record of endTags, start positions (for determining the length)
           # and layer names
           endSpan1 = [ span1[4], span1[0], span1[2] ]
           endLoc1 = span1[1]
           if endLoc1 not in endIndex:
              endIndex[endLoc1] = []
              endIndex[endLoc1].append( endSpan1 )
           else:
              # Make sure that spans are inserted in the order of increasing length:
              #   shorter spans preceding the longer ones;
              inserted = False
              for i in range( len(endIndex[endLoc1]) ):
                  endSpan2 = endIndex[endLoc1][i]
                  # If an existing span is longer than the current span, insert the
                  # current span before the existing span ...
                  if endSpan2[1] < endSpan1[1]:
                     endIndex[endLoc1].insert( i, endSpan1 )
                     inserted = True
                     break
                  elif endSpan2[1] == endSpan1[1] and endSpan2[2] < endSpan1[2]:
                     # If both spans have equal length, order the spans in the
                     # alphabetical order of layer names:
                     endIndex[endLoc1].insert( i, endSpan1 )
                     inserted = True
                     break
              if not inserted:
                  endIndex[endLoc1].append( endSpan1 )
   return endIndex


Instruction:
Can you generate the documentation for the following Python 3 function
def _fix_overlapping_graphics( spansStartingFrom ):
    ''' Provides a fix for overlapping annotations that are formatted graphically
        (underlined or printed in non-default color).

        If two graphically formatted annotations overlap, and if one annotation,
        say A, ends within another annotation, say B, then ending of graphics of A
        also causes graphics of B to end, and so, the end of A should restart the
        the graphics of B for a continuous visualisation;
        This method modifies ending tags in a way that annotations ending within
        other annotations will also contain restarts of the corresponding (super)-
        annotations, so that a continuous formatting is ensured.
    '''
    for startIndex in sorted( spansStartingFrom.keys() ):
        for span1 in spansStartingFrom[startIndex]:
            # If the span is not graphic, we don't have no worries - we can just skip it
            if not span1[5]:
               continue
            # Otherwise: check for other graphic spans that overlap with the given span
            span1Start = span1[0]
            span1End   = span1[1]
            for i in range( span1Start, span1End ):
                if i in spansStartingFrom:
                   for span2 in spansStartingFrom[i]:
                       span2Start = span2[0]
                       span2End   = span2[1]
                       # If the spans are not the same, and the span2 is graphic
                       if span2 != span1 and span2[5]:
                          # if the overlapping graphic span ends before the current span,
                          # we have to restart the graphic formatting of given span after
                          # the end of the overlapping span
                          if span2End <= span1End:
                             if not span1[6]:
                                # If span1 is not bracketed, just add it at the end of
                                # the overlapping span
                                span2[4] += span1[3]
                             else:
                                # If span1 is bracketed, add it at the end of the
                                # overlapping span without brackets
                                wb = span1[3].rstrip('[')
                                span2[4] += wb

Response: Provides a fix for overlapping graphic annotations that are formatted graphically with non - default color.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _preformat( text, layers, markup_settings = None ):
   ''' Formats given text, adding a special ( ANSI-terminal compatible ) markup
       to the annotations of given layers, and returns formatted text as a
       string.
       *) layers is a list containing names of the layers to be preformatted in
          the text (these layers must be present in Text);
       *) markup_settings should be a list containing annotation options for each
          layer: one dict with options per layer;
          One dict can contain the following visualization options:
          * 'bracket' : True -- annotations will be surrounded with brackets; This
                                works in any terminal;
          * 'underline' : True -- annotations will be underlined; This works in ANSI
                                  compatible terminal;
          * 'color' : ('red', 'green', 'blue' etc. ) -- annotated text will be
                      displayed in given color; this works in ANSI compatible
                      terminal;
       *) Alternatively, if markup_settings is undefined, up to 12 layers can be
          visualized following the default settings;

       Parameters
       ----------
       text: Text
          a text object. Must contain given layers;
       layer: list of str
          list of layer names to be visualised;
       markup_settings: list of dict
          list of dictionaries containing user-defined visualization options;
          (one dict per layer)

       Returns
       -------
       text: str
          preformatted text, where elements of given layers have been marked up, using
          an ANSI-terminal compatible markup;
   '''
   if markup_settings and len(layers) != len(markup_settings):
       raise Exception(' Input arguments layers and markup_settings should be equal size lists.')
   elif not markup_settings and len(layers) <= len(default_markup_settings):
       # Use default markup settings
       markup_settings = default_markup_settings[0:len(layers)]
   elif not markup_settings:
       raise Exception(' Input argument markup_settings not defined.')
   #
   # 1) Construct the index of annotations (for each layer);
   #    Annotations are indexed by their start positions;
   #    The index also contains start and end tags of each annotation;
   spansStartingFrom = {}
   for i in range( len(layers) ):
       layer    = layers[i]
       settings = markup_settings[i]
       spansStartingFrom = _construct_start_index(text, layer, settings, spansStartingFrom)
   #
   # 2) Fix overlapping graphic annotations in the index
   #    (to ensure continuous formatting of annotations)
   _fix_overlapping_graphics( spansStartingFrom )
   #
   # 3) Index the annotations by their end positions
   endTags = _construct_end_index( spansStartingFrom )

   #
   # 4) Construct the output string
   return_str = []
   for i in range( len(text[TEXT]) ):
       c = text[TEXT][i]
       emptyTags = []
       if i in endTags:
          for tag in endTags[i]:
              if tag[1] != i:
                 # Non-empty tag
                 return_str.append( tag[0] )
              else:
                 # Empty tag
                 emptyTags.append( tag )
       if i in spansStartingFrom:
          for span in spansStartingFrom[i]:
              return_str.append( span[3] )
              if span[0] == span[1]:
                 # Empty tag: Add the closing tag
                 for emptyEndTag in emptyTags:
                     if span[2] == emptyEndTag[2]:
                        return_str.append( emptyEndTag[0] )
       return_str.append( c )
   if len(text[TEXT]) in spansStartingFrom:
       for span in spansStartingFrom[len(text[TEXT])]:
           return_str.append( span[3] )
   if len(text[TEXT]) in endTags:
       for tag in endTags[len(text[TEXT])]:
           return_str.append( tag[0] )
   # Hack: fix for a potential overflow / unclosed graphics
   if return_str and '\033' in return_str[-1] and \
      not return_str[-1].endswith('\033[0m'):
      return_str.append( '\033[0m' )
   return ''.join(return_str)

Response: Formats given text with special markups.


Instruction:
Can you implement a function in Python 3 that
formats given text with special markups and prints the formatted text to the screen.

Response:
def tprint( text, layers, markup_settings = None ):
    ''' Formats given text, adding a special ( ANSI-terminal compatible ) markup
        to the annotations of given layers, and prints the formatted text to the
        screen.
        *) layers is a list containing names of the layers to be preformatted in
           the text (these layers must be present in Text);
        *) markup_settings should be a list containing annotation options for each
           layer: one dict with options per layer;
           One dict can contain the following visualization options:
           * 'bracket' : True -- annotations will be surrounded with brackets; This
                                 works in any terminal;
           * 'underline' : True -- annotations will be underlined; This works in ANSI
                                   compatible terminal;
           * 'color' : ('red', 'green', 'blue' etc. ) -- annotated text will be
                       displayed in given color; this works in ANSI compatible
                       terminal;
        *) Alternatively, if markup_settings is undefined, up to 12 layers can be
           visualized following the default settings;

        Parameters
        ----------
        text: Text
           a text object. Must contain given layers;
        layer: list of str
           list of layer names to be visualised;
        markup_settings: list of dict
           list of dictionaries containing user-defined visualization options;
           (one dict per layer)
    '''
    if markup_settings and len(layers) != len(markup_settings):
       raise Exception(' Input arguments layers and markup_settings should be equal size lists.')
    elif not markup_settings and len(layers) <= len(default_markup_settings):
       # Use a subset from default markup settings
       markup_settings = default_markup_settings[0:len(layers)]
    elif not markup_settings:
       raise Exception(' Input argument markup_settings not defined.')
    print( _preformat(text, layers, markup_settings=markup_settings) )


Instruction:
Implement a function in Python 3 to
generate CSS class for a mark tag.

Response:
def get_mark_css(aes_name, css_value):
    """Generate CSS class for <mark> tag.

    Parameters
    ----------
    aes_name: str
        The name of the class.
    css_value: str
        The value for the CSS property defined by aes_name.

    Returns
    -------
    list of str
        The CSS codeblocks
    """
    css_prop = AES_CSS_MAP[aes_name]
    if isinstance(css_value, list):
        return get_mark_css_for_rules(aes_name, css_prop, css_value)
    else:
        return get_mark_simple_css(aes_name, css_prop, css_value)


Instruction:
Can you create a Python 3 function that
loads the relations from the input file into the instance attributes.

Response:
def _loadSubcatRelations( self, inputFile ):
        ''' Laeb sisendfailist (inputFile) verb-nom/adv-vinf rektsiooniseoste mustrid.
            Iga muster peab olema failis eraldi real, kujul:
                (verbikirjeldus)\TAB(nom/adv-kirjeldus)\TAB(vinfkirjeldus)
            nt
                leid NEG	aeg;S;((sg|pl) (p)|adt)	da
                leid POS	võimalus;S;(sg|pl) (n|p|g)	da
            Salvestab laetud tulemused klassimuutujatesse nomAdvWordTemplates, verbRules
            ja verbToVinf;
        '''
        self.nomAdvWordTemplates = dict()
        self.verbRules  = dict()
        self.verbToVinf = dict()
        in_f = codecs.open(inputFile, mode='r', encoding='utf-8')
        for line in in_f:
            line = line.rstrip()
            if len(line) > 0 and not re.match("^#.+$", line):
                items = line.split('\t')
                if len(items) == 3:
                    verb    = items[0]
                    nounAdv = items[1]
                    vinf    = items[2]
                    if nounAdv not in self.nomAdvWordTemplates:
                        (root,pos,form) = nounAdv.split(';')
                        if not root.startswith('^') and not root.endswith('$'):
                            root = '^'+root+'$'
                        constraints = {ROOT:root, POSTAG:pos}
                        if form:
                            constraints[FORM] = form
                        self.nomAdvWordTemplates[nounAdv] = WordTemplate(constraints)
                    if verb not in self.verbRules:
                        self.verbRules[verb] = []
                    if verb not in self.verbToVinf:
                        self.verbToVinf[verb] = []
                    self.verbRules[verb].append( (nounAdv, 'V_'+vinf) )
                    if 'V_'+vinf not in self.verbToVinf[verb]:
                        self.verbToVinf[verb].append( 'V_'+vinf )
                else:
                    raise Exception(' Unexpected number of items in the input lexicon line: '+line)
        in_f.close()


Instruction:
Explain what the following Python 3 code does
def tokenMatchesNomAdvVinf( self, token, verb, vinf):
        ''' Teeb kindlaks, kas etteantud token v6iks olla verb'i alluv ning vinf'i ylemus (st
            paikneda nende vahel). Kui see nii on, tagastab j2rjendi vahele sobiva s6na morf
            analyysidega (meetodi _getMatchingAnalysisIDs abil), vastasel juhul tagastab tyhja
            j2rjendi;
        '''
        if verb in self.verbRules:
            for (nounAdv, vinf1) in self.verbRules[verb]:
                if vinf == vinf1 and (self.nomAdvWordTemplates[nounAdv]).matches(token):
                    return _getMatchingAnalysisIDs( token, self.nomAdvWordTemplates[nounAdv] )
        return []

Response: Returns a list of analysis IDs that match the given token and the given verb and vinf.


Instruction:
How would you implement a function in Python 3 that
extends verb chains in a sentence.

Response:
def extendChainsInSentence( self, sentence, foundChains ):
        ''' Rakendab meetodit self.extendChainsInClause() antud lause igal osalausel.
        '''
        # 1) Preprocessing
        clauses = getClausesByClauseIDs( sentence )

        # 2) Extend verb chains in each clause
        allDetectedVerbChains = []
        for clauseID in clauses:
            clause = clauses[clauseID]
            self.extendChainsInClause(clause, clauseID, foundChains)


Instruction:
Can you tell what is the following Python 3 function doing
def _isLikelyNotPhrase( self, headVerbRoot, headVerbWID, nomAdvWID, widToToken):
        ''' Kontrollib, et nom/adv ei kuuluks mingi suurema fraasi kooseisu (poleks fraasi peas6na).
            Tagastab True, kui:
               *) nom/adv j2rgneb vahetult peaverbile
               *) või nom/adv on vahetult osalause alguses
               *) või nom-ile eelneb vahetult selline s6na, mis kindlasti ei saa olla
                  eestäiendiks
               *) või nom/adv puhul on tegemist olema-verbi adv-ga;
        '''
        minWID = min(widToToken.keys())
        nomAdvToken = widToToken[nomAdvWID]
        isNom = self.wtNom.matches(nomAdvToken)
        if headVerbWID+1 == nomAdvWID:
            #   1) Kui nom/adv j2rgneb vahetult verbile, siis on ysna turvaline arvata,
            #      et need kuuluvad kokku, nt:
            #             ja seda tunnet on_0 raske_0 millegi vastu vahetada_0 .
            #             mida nad peavad_0 vajalikuks_0 läände müüa_0
            return True
        elif minWID == nomAdvWID:
            #   2) Kui nom/adv on vahetult osalause alguses, siis on samuti üsna turvaline
            #      eeldada, et see kuulub verbiga kokku, nt:
            #             Tarvis_0 on_0 muretseda_0 veel auhinnafond 250 000 dollarit .
            #             Raske_0 on_0 temaga vaielda_0 .
            return True
        elif isNom and nomAdvWID-1 in widToToken:
            prevToken = widToToken[nomAdvWID-1]
            if self.wtNotSyntAttrib.matches(prevToken):
                #
                #   3.1) Kui nom-ile eelneb vahetult adverb, mis tavaliselt allub otse
                #        verbile ning ei funktsioneeri eest2iendina (nt 'ju', 'ikka',
                #        'vist', 'veel' jms), siis on ysna turvaline eeldada, et nom
                #        ei ole mingi fraasi osa:
                #             Kaudseid näitajaid on_0 aga võimalik_0 analüüsida_0
                #             Pole_0 ju mõtet_0 hakata_0 teile ette laduma asjaolusid
                #             on_0 veel raske_0 kommenteerida_0
                #
                return True
            elif self.wtNom.matches(prevToken):
                if self.wtNomSemCase.matches(prevToken):
                    if not self.wtNomSemCase.matches(nomAdvToken):
                        #
                        #   3.2) Kui nom-ile vahetult eelnev s6na on semantilises k22ndes, aga nom
                        #        mitte, pole nad suure t6en2osusega seotud, nt:
                        #             Siis jääb_0 ootajal võimalus_0 öelda_0
                        #             vahendajate juurdehindlust on_0 riigil võimalik_0 kontrollida_0 .
                        #             Ka üürnikul on_0 selle seadusega õigus_0 maksta_0 vähem üüri
                        #
                        return True
                    else:
                        #
                        #   3.3) Kui nii nom kui vahetult eelnev s6na on m6lemad semantilises k22ndes,
                        #        aga k22nded on erinevad, ei moodusta nad t6en2oliselt yhte fraasi, nt:
                        #            pole_0 ettevõttel plaanis_0 Tartus kaugküttesooja hinda tõsta_0 .
                        #            Ginrichi teatel on_0 vabariiklastel kavas_0 luua_0 erikomisjon ,
                        #            et ühegi parkimismaja rajamist pole_0 linnal kavas_0 toetada_0 .
                        #
                        analyses1 = self.wtNomSemCase.matchingAnalyses(prevToken)
                        analyses2 = self.wtNomSemCase.matchingAnalyses(nomAdvToken)
                        forms1 = set([a[FORM] for a in analyses1])
                        forms2 = set([a[FORM] for a in analyses2])
                        if len(forms1.intersection(forms2))==0:
                            return True
        elif not isNom and headVerbRoot.startswith('ole '):
            #
            #   X) Kui tegemist on olema-ga liituva adv-ga, eeldame, et see on suurema t6en2osusega yksik,
            #      st pole mingi fraasi koosseisus:
            #            Theresel polnud_0 raskeid seasöögiämbreid tarvis_0 ubida_0 .
            #            Seepärast pole_0 meil ka häbi vaja_0 tunda_0
            #
            #      NB! Alati see siiski nii ei ole, st võib liituda tähendust intensiivistav 'väga',
            #      'pisut', 'palju' jms adverb, nt:
            #            Meil pole_0 siin palju vaja_0 pingutada_0
            #
            return True
        return False

Response: Returns True if the given head verb is not phrase.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _canBeExpanded( self, headVerbRoot, headVerbWID, suitableNomAdvExpansions, expansionVerbs, widToToken ):
        ''' Teeb kindlaks, kas kontekst on verbiahela laiendamiseks piisavalt selge/yhene:
              1) Nii 'nom/adv' kandidaate kui ka Vinf kandidaate on täpselt üks;
              2) Nom/adv ei kuulu mingi suurema fraasi kooseisu (meetodi _isLikelyNotPhrase() abil);
            Kui tingimused täidetud, tagastab lisatava verbi listist expansionVerbs, vastasel juhul
            tagastab None;
        '''
        if len(suitableNomAdvExpansions)==1 and expansionVerbs:
            # Kontrollime, kas leidub t2pselt yks laiendiks sobiv verb (kui leidub
            # rohkem, on kontekst kahtlane ja raske otsustada, kas tasub laiendada
            # v6i mitte)
            suitableExpansionVerbs = \
                [expVerb for expVerb in expansionVerbs if expVerb[2] == suitableNomAdvExpansions[0][2]]
            if len( suitableExpansionVerbs ) == 1:
                # Kontrollime, et nom/adv ei kuuluks mingi suurema fraasi kooseisu (ei oleks fraasi
                # peas6na);
                nomAdvWID = suitableNomAdvExpansions[0][0]
                if self._isLikelyNotPhrase( headVerbRoot, headVerbWID, nomAdvWID, widToToken ):
                    return suitableExpansionVerbs[0]
        return None

Response: Returns whether or not the given tag is able to be expanded.


Instruction:
Implement a Python 3 function for
extending the list of foundChains with the ones in the given clause.

Response:
def extendChainsInClause( self, clause, clauseID, foundChains ):
        '''  Proovime etteantud osalauses leiduvaid verbiahelaid täiendada 'verb-nom/adv-vinf'
            rektsiooniseostega, nt:
                andma + võimalus + Vda :  talle anti_0 võimalus_0 olukorda parandada_0
                olema + vaja + Vda    :   nüüd on_0 küll vaja_0 asi lõpetada_0
             Teeme seda kahel moel:
            1) kui mingi olemasoleva verbiahela keskelt on puudu 'nom/adv' (nt 'andma', 'jätma'
               verbide vinf rektsiooniseoste leidmisel võib tekkida selliseid lünki), siis
               lisame ahela keskele 'nom/adv' sõna.
            2) kui verbiahela lõpus on verb, mis on sageli ülemuseks 'nom/adv' sõnale, millest
               omakorda sõltub mingi Vinf verb (Vma, Vda), ning need on osalausekontekstis olemas,
               lisame need verbiahela lõppu;
        '''
        expansionPerformed = False
        # J22dvustame s6nad, mis kuuluvad juba mingi tuvastatud verbifraasi koosseisu
        annotatedWords = []
        for verbObj in foundChains:
            if clauseID == verbObj[CLAUSE_IDX] and (len(verbObj[PATTERN])==1 and \
               re.match('^(ei|ära|ega)$', verbObj[PATTERN][0])):
                # V2lja j22vad yksikuna esinevad ei/ära/ega, kuna need tõenäoliselt ei sega
                continue
            annotatedWords.extend( verbObj[PHRASE] )
        widToToken = { token[WORD_ID] : token for token in clause }
        verbDaMa   = WordTemplate({POSTAG:'V', FORM:'^(da|ma)$'})
        verbOle    = WordTemplate({ROOT:'^ole$',POSTAG:'V'})
        #
        #   1) Yritame leida, millised juba tuvastatud verbiahelatest on sellised, kust on
        #      vahelt puudu nom/adv s6na, nt:
        #            annab_0 kunstnik jälle põhjuse endast kirjutada_0
        #            see annab_0 võimaluse laua tagant tõusta_0 ja_0 minema jalutada_0
        #      Kui leiame ahelast puuduoleva s6na ja lisame selle ahelasse ...
        #
        for verbObj in foundChains:
            if clauseID == verbObj[CLAUSE_IDX]:
                headVerb      = ''
                headVerbWID   = -1
                dependentVerb = ''
                dependentVerbWIDs = []
                firstDependentVerbID = -1
                #  Leiame ahela l6pust ylemus-verbi ja sellele alluva verbi
                if len(verbObj[PATTERN]) > 3 and verbObj[PATTERN][-2] == '&':
                    headVerb = verbObj[ROOTS][-4]+" "+verbObj[POLARITY]
                    dependentVerb = verbObj[MORPH][-3]
                    headVerbWID = verbObj[PHRASE][-4]
                    dependentVerbWIDs.append( verbObj[PHRASE][-3] )
                    dependentVerbWIDs.append( verbObj[PHRASE][-1] )
                    firstDependentVerbID = len(verbObj[PHRASE])-3
                elif len(verbObj[PATTERN]) > 1 and verbObj[PATTERN][-2]=='verb':
                    headVerb = verbObj[ROOTS][-2]+" "+verbObj[POLARITY]
                    dependentVerb = verbObj[MORPH][-1]
                    headVerbWID = verbObj[PHRASE][-2]
                    dependentVerbWIDs.append(verbObj[PHRASE][-1])
                    firstDependentVerbID = len(verbObj[PHRASE])-1
                # Kontrollime, kas ylemusverb ja sellele alluv verb v6iksid olla yhendatud
                # mingi nom/adv s6na kaudu
                if headVerb in self.verbRules and headVerb in self.verbToVinf and \
                   dependentVerb in self.verbToVinf[headVerb]:
                    # Teeme kindlaks, kas s6nade vahele j22b puuduolev nom/adv
                    minInd = min(min(dependentVerbWIDs), headVerbWID-1)
                    maxInd = max(max(dependentVerbWIDs)-1, headVerbWID)
                    if minInd < maxInd:
                        for i in range(minInd, maxInd+1):
                            if i in widToToken and i not in annotatedWords:
                                token = widToToken[i]
                                matchingAnalyses = self.tokenMatchesNomAdvVinf( token, headVerb, dependentVerb )
                                if matchingAnalyses and not expansionPerformed:
                                    #   Kontrollime, kas vaheletorgatav sõna paikneb nii, et see on suure
                                    #  tõenäosusega üksiksõna, mitte fraas.
                                    if self._isLikelyNotPhrase( headVerb, headVerbWID, token[WORD_ID], widToToken ):
                                        # Torkame nimis6na/adverbi vahele
                                        verbObj[PHRASE].insert( firstDependentVerbID, token[WORD_ID] )
                                        verbObj[PATTERN].insert( firstDependentVerbID, 'nom/adv' )
                                        verbObj[ANALYSIS_IDS].insert( firstDependentVerbID, matchingAnalyses )
                                        annotatedWords.append( token[WORD_ID] )
                                        expansionPerformed = True
                                    else:
                                        # Kui me ei saa olla kindlad, et vaheletorgatav sõna pole fraas, paneme
                                        # küsimärgi, näitamaks, et verbiahelast on suure tõenäosusega midagi
                                        # puudu ...
                                        verbObj[OTHER_VERBS] = True
                                        #_debugPrint( ' '+('+'.join(verbObj[PATTERN]))+' | '+_getJsonAsTextString(clause, markTokens = [ verbObj[PHRASE] ] ))

        #
        #   2) Yritame luua uusi ahelaid, laiendades verbe olemasolevate ahelate l6pus:
        #
        #          millega antakse_0 võimalus_1 sõlmida_1 uus kokkulepe .
        #          puudub_0 võimalus_1 spetsialiste täistööajaga rakendada_1 .
        #          kui on_0 võimalus_1 rahulikult vooluga kaasa minna_1
        #
        clauseMaxWID = max( list(widToToken.keys()) )
        for verbObj in foundChains:
            if clauseID == verbObj[CLAUSE_IDX] and verbObj[OTHER_VERBS]:
                if (len(verbObj[PATTERN])==1 or (len(verbObj[PATTERN])>1 and \
                   verbObj[PATTERN][-2] != '&')):
                    headVerb = verbObj[ROOTS][-1]+" "+verbObj[POLARITY]
                    headVerbWID = verbObj[PHRASE][-1]
                    #
                    #   2.1) Esimeses l2henduses vaatame tavalisi verbe (mitte-olema);
                    #
                    if headVerb in self.verbRules and not headVerb.startswith('ole '):
                        minInd = headVerbWID-1 if verbObj[PATTERN][0]!='ega' else headVerbWID
                        suitableNomAdvExpansions = []
                        expansionVerbs = []
                        for i in range(minInd, clauseMaxWID+1):
                            if i in widToToken and i not in annotatedWords:
                                token = widToToken[i]
                                if _isFollowedByComma( i, clause ):
                                    #  Katkestame, kui satume koma otsa (kuna ei saa kindel olla,
                                    # et teisel pool koma on olevad jupid kuuluvad ikka verbi
                                    # juurde)
                                    break
                                if verbDaMa.matches( token ):
                                    analysisIDs = _getMatchingAnalysisIDs( token, verbDaMa )
                                    form = token[ANALYSIS][analysisIDs[0]][FORM]
                                    expansionVerbs.append( [i, token, "V_"+form ] )
                                else:
                                    for (nounAdv, vinf1) in self.verbRules[headVerb]:
                                        if (self.nomAdvWordTemplates[nounAdv]).matches(token):
                                            suitableNomAdvExpansions.append( [i, token, vinf1, \
                                                (self.nomAdvWordTemplates[nounAdv]), nounAdv ] )
                        # Teeme kindlaks, kas kontekst on laiendamiseks piisavalt yhene/selge ...
                        suitableExpansionVerb = \
                            self._canBeExpanded( headVerb, headVerbWID, suitableNomAdvExpansions, \
                                expansionVerbs, widToToken )
                        if suitableExpansionVerb:
                            phraseExt  = [suitableNomAdvExpansions[0][0], suitableExpansionVerb[0]]
                            expIsOle   = verbOle.matches(suitableExpansionVerb[1])
                            patternExt = ['nom/adv', 'ole' if expIsOle else 'verb']
                            analysisIDsExt = [ \
                                    _getMatchingAnalysisIDs( suitableNomAdvExpansions[0][1], \
                                    suitableNomAdvExpansions[0][3] ), \
                                    _getMatchingAnalysisIDs( suitableExpansionVerb[1], verbDaMa ) ]
                            # Lisame ahelale pikendused
                            verbObj[PHRASE].extend( phraseExt )
                            verbObj[PATTERN].extend( patternExt )
                            verbObj[ANALYSIS_IDS].extend( analysisIDsExt )
                            annotatedWords.extend( phraseExt )
                            expansionPerformed = True
                            #if headVerb.startswith('and '):
                            #    _debugPrint( ('+'.join(verbObj[PATTERN]))+' | '+getJsonAsTextString(clause, markTokens = [ verbObj[PHRASE] ] ))
                            #_debugPrint( ('+'.join(verbObj[PATTERN]))+' | '+getJsonAsTextString(clause, markTokens = [ verbObj[PHRASE] ] ))
                    elif headVerb in self.verbRules and headVerb.startswith('ole '):
                    #
                    #   2.2) Vaatame olema-verbi rektsiooniseoseid;
                    #
                        minInd = headVerbWID-1 if verbObj[PATTERN][0]!='ega' else headVerbWID
                        suitableNomAdvExpansions = []
                        expansionVerbs = []
                        for i in range(minInd, clauseMaxWID+1):
                            if i in widToToken and i not in annotatedWords:
                                token = widToToken[i]
                                if verbDaMa.matches( token ):
                                    analysisIDs = _getMatchingAnalysisIDs( token, verbDaMa )
                                    form = token[ANALYSIS][analysisIDs[0]][FORM]
                                    expansionVerbs.append( [i, token, "V_"+form ] )
                                else:
                                    for (nounAdv, vinf1) in self.verbRules[headVerb]:
                                        if (self.nomAdvWordTemplates[nounAdv]).matches(token):
                                            suitableNomAdvExpansions.append( [i, token, vinf1, \
                                                (self.nomAdvWordTemplates[nounAdv]), nounAdv] )
                                if _isFollowedByComma( i, clause ):
                                    #  Katkestame, kui satume koma otsa (kuna ei saa kindel olla,
                                    # et teisel pool koma on olevad jupid kuuluvad ikka verbi
                                    # juurde)
                                    break
                        # Teeme kindlaks, kas kontekst on laiendamiseks piisavalt yhene/selge ...
                        suitableExpansionVerb = \
                            self._canBeExpanded( headVerb, headVerbWID, suitableNomAdvExpansions, \
                                expansionVerbs, widToToken )
                        if suitableExpansionVerb:
                            phraseExt  = [suitableNomAdvExpansions[0][0], suitableExpansionVerb[0]]
                            expIsOle   = verbOle.matches(suitableExpansionVerb[1])
                            patternExt = ['nom/adv', 'ole' if expIsOle else 'verb']
                            analysisIDsExt = [ \
                                _getMatchingAnalysisIDs( suitableNomAdvExpansions[0][1], \
                                suitableNomAdvExpansions[0][3] ), \
                                _getMatchingAnalysisIDs( suitableExpansionVerb[1], verbDaMa ) ]
                            # Lisame ahelale pikendused
                            verbObj[PHRASE].extend( phraseExt )
                            verbObj[PATTERN].extend( patternExt )
                            verbObj[ANALYSIS_IDS].extend( analysisIDsExt )
                            annotatedWords.extend( phraseExt )
                            expansionPerformed = True
                            #_debugPrint( ('+'.join(verbObj[PATTERN]))+' | '+getJsonAsTextString(clause, markTokens = [ verbObj[PHRASE] ] ))
                            #if suitableNomAdvExpansions[0][4].startswith('aeg;'):
                            #    _debugPrint( ('+'.join(verbObj[PATTERN]))+' | '+_getJsonAsTextString(clause, markTokens = [ verbObj[PHRASE] ] ))
        return expansionPerformed


Instruction:
Make a summary of the following Python 3 code
def sectionsParser(text):

    textStart = 0

    #Split the text in sections. Hackish part, but seems to work fine.
    entries = re.split("\n=", text[textStart:])
    stack = [[]]
    intro = {}
    sectionTitleRegEx = re.compile(r'={1,}.+={2,}')
    section = {}
    section['text'] = entries[0]
    counts = []

    #Presumes the first section is always marked with 2 =
    #First count is always 3. (\n=)= First Section of an Article ==
    #Parens is omitted. Leaves three = marks.

    counts.append(3)
    sections = []
    sections.append(section)
    for i in entries[1:]:
        section = {}
        title = re.match(sectionTitleRegEx, i)
        if title:
            titleEnd = title.end()
            title = title.group()
            text = i[titleEnd:]
            level =  title.count('=')
            section['title']=title.strip('= ')
            section['text']=text

            sections.append(section.copy())
            counts.append(level)

    #add images, links, references, tables

    for section in sections:
        text = section['text']
        if 'wikitable' in text or '</table>' in text.lower():
            section['text'], section['tables'] = tableCollector(text)

        section = relatedArticles(section)

        if '<ref' in text:
            section = reffinder(section)

        if imageRegEx.search(text):
            section = imageParser(section)

        section['text'] = section['text'].strip()

        if ExtLinkBracketedRegex.search(text):
            section = addExternalLinks(section)

        if '[[' in text:
            section = addIntLinks(section)

        #clean uneven brackets and whatnot

        #take extlink start:end w regex.
        el = 'external_links'
        if el in section.keys():

            #section['text'] = section['text'].replace('[', '').replace(']', '')
            text = section['text']

            for link in section[el]:

                label = link['label']
                label = re.compile(re.escape(label))
                m = label.search(text)
                #if there are unbalanced brackets in the external
                #links label inside text then it fails to mark the start and end
                try:
                    link['start'] = m.start()
                    link['end'] = m.end()
                except AttributeError:
                    print('Problem with external links start:end position!')
                    print(label)
                    print(text)

    #datastructure nesting thanks to Timo!
    if counts:
        assert len(counts) == len(sections)

        n = len(sections)
        pos = 0

        levels = [counts[0]]

        while pos < n:
            count = counts[pos]
            elem = sections[pos]
            level = levels[-1]
            if count == level:
                stack[-1].append(elem)
            elif count >= level:
                stack.append([elem])
                levels.append(count)
            else:
                group = stack.pop()
                stack[-1][-1]['sections'] = group
                levels.pop()
                continue

            pos += 1

        while len(stack) > 1:
            group = stack.pop()
            stack[-1][-1]['sections'] = group

    stack = stack[0]

    return stack

Response: Parses the whole text of an wikipedia article into nested sections.


Instruction:
Write a Python 3 function for
removing all unwanted characters from text.

Response:
def clean(self, text):
        """Remove all unwanted characters from text."""
        return ''.join([c for c in text if c in self.alphabet])


Instruction:
Can you generate the documentation for the following Python 3 function
def invalid_characters(self, text):
        return ''.join(sorted(set([c for c in text if c not in self.alphabet])))

Response: Give simple list of invalid characters present in text.


Instruction:
Write a Python 3 function that can
find invalid characters in text and store information about them.

Response:
def find_invalid_chars(self, text, context_size=20):
        """Find invalid characters in text and store information about
        the findings.

        Parameters
        ----------
        context_size: int
            How many characters to return as the context.

        """
        result = defaultdict(list)
        for idx, char in enumerate(text):
            if char not in self.alphabet:
                start = max(0, idx-context_size)
                end   = min(len(text), idx+context_size)
                result[char].append(text[start:end])
        return result


Instruction:
Implement a Python 3 function for
computing statistics of invalid characters on given texts.

Response:
def compute_report(self, texts, context_size=10):
        """Compute statistics of invalid characters on given texts.

        Parameters
        ----------
        texts: list of str
            The texts to search for invalid characters.
        context_size: int
            How many characters to return as the context.

        Returns
        -------
        dict of (char -> list of tuple (index, context))
            Returns a dictionary, where keys are invalid characters.
            Values are lists containign tuples with character indices
            and context strings.
        """
        result = defaultdict(list)
        for text in texts:
            for char, examples in self.find_invalid_chars(text, context_size).items():
                result[char].extend(examples)
        return result


Instruction:
Can you implement a function in Python 3 that
computes statistics of invalid characters and print them.

Response:
def report(self, texts, n_examples=10, context_size=10, f=sys.stdout):
        """Compute statistics of invalid characters and print them.

        Parameters
        ----------
        texts: list of str
            The texts to search for invalid characters.
        n_examples: int
            How many examples to display per invalid character.
        context_size: int
            How many characters to return as the context.
        f: file
            The file to print the report (default is sys.stdout)
        """

        result = list(self.compute_report(texts, context_size).items())
        result.sort(key=lambda x: (len(x[1]), x[0]), reverse=True)
        s = 'Analyzed {0} texts.\n'.format(len(texts))
        if (len(texts)) == 0:
            f.write(s)
            return
        if len(result) > 0:
            s += 'Invalid characters and their counts:\n'
            for c, examples in result:
                s += '"{0}"\t{1}\n'.format(c, len(examples))
            s += '\n'
            for c, examples in result:
                s += 'For character "{0}", found {1} occurrences.\nExamples:\n'.format(c, len(examples))
                examples = sample(examples, min(len(examples), n_examples))
                for idx, example in enumerate(examples):
                    s += 'example {0}: {1}\n'.format(idx+1, example)
                s += '\n'
            f.write(s)
        else:
            f.write('All OK\n')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_text(self, text, **kwargs):
        # a) get the configuration:
        apply_tag_analysis = False
        augment_words      = False
        all_return_types = ["text","vislcg3","trees","dep_graphs"]
        return_type      = all_return_types[0]
        for argName, argVal in kwargs.items():
            if argName.lower() == 'return_type':
                if argVal.lower() in all_return_types:
                    return_type = argVal.lower()
                else:
                    raise Exception(' Unexpected return type: ', argVal)
            elif argName.lower() == 'augment_words':
                augment_words = bool(argVal)
            elif argName.lower() == 'apply_tag_analysis':
                apply_tag_analysis = bool(argVal)
        kwargs['split_result']  = True
        kwargs['clean_up']      = True
        kwargs['remove_clo']    = kwargs.get('remove_clo', True)
        kwargs['remove_cap']    = kwargs.get('remove_cap', True)
        kwargs['keep_old']      = kwargs.get('keep_old',  False)
        kwargs['double_quotes'] = 'unesc'

        # b) process:
        if apply_tag_analysis:
            text = text.tag_analysis()
        result_lines1 = \
            self.preprocessor.process_Text(text, **kwargs)
        result_lines2 = \
            self.vislcg3_processor.process_lines(result_lines1, **kwargs)
        alignments = \
            align_cg3_with_Text(result_lines2, text, **kwargs)
        alignments = \
            normalise_alignments( alignments, data_type=VISLCG3_DATA, **kwargs )

        # c) attach & return results
        text[LAYER_VISLCG3] = alignments
        if augment_words:
            self._augment_text_w_syntactic_info( text, text[LAYER_VISLCG3] )
        if return_type   == "vislcg3":
            return result_lines2
        elif return_type == "trees":
            return build_trees_from_text( text, layer=LAYER_VISLCG3, **kwargs )
        elif return_type == "dep_graphs":
            trees = build_trees_from_text( text, layer=LAYER_VISLCG3, **kwargs )
            graphs = [tree.as_dependencygraph() for tree in trees]
            return graphs
        else:
            return text

Response: Parses given text with VISLCG3 based syntactic analyzer and returns a new Text object containing the parsed text.


Instruction:
Can you create a Python 3 function that
filters the dict of kwargs that are in keep_list and discards all other arguments.

Response:
def _filter_kwargs(self, keep_list, **kwargs):
        ''' Filters the dict of *kwargs*, keeping only arguments
            whose keys are in *keep_list* and discarding all other
            arguments.

            Based on the filtring, constructs and returns a new
            dict.
        '''
        new_kwargs = {}
        for argName, argVal in kwargs.items():
            if argName.lower() in keep_list:
                new_kwargs[argName.lower()] = argVal
        return new_kwargs


Instruction:
Can you create a Python 3 function that
augments given Text object with the syntactic information about each token in the text_layer. More specifically adds information about SYNTAX_LABEL SYNTAX_HEAD and DEPREL to each token in the text_layer. More specifically adds information about SYNTAX_HEAD and DEPREL to each token in the Text object.

Response:
def _augment_text_w_syntactic_info( self, text, text_layer ):
        ''' Augments given Text object with the syntactic information
            from the *text_layer*. More specifically, adds information
            about SYNTAX_LABEL, SYNTAX_HEAD and DEPREL to each token
            in the Text object;

            (!) Note: this method is added to provide some initial
            consistency with MaltParser based syntactic parsing;
            If a better syntactic parsing interface is achieved in
            the future, this method will be deprecated ...
        '''
        j = 0
        for sentence in text.divide( layer=WORDS, by=SENTENCES ):
            for i in range(len(sentence)):
                estnltkToken = sentence[i]
                vislcg3Token = text_layer[j]
                parse_found = False
                if PARSER_OUT in vislcg3Token:
                    if len( vislcg3Token[PARSER_OUT] ) > 0:
                        firstParse = vislcg3Token[PARSER_OUT][0]
                        # Fetch information about the syntactic relation:
                        estnltkToken['s_label'] = str(i)
                        estnltkToken['s_head']  = str(firstParse[1])
                        # Fetch the name of the surface syntactic relation
                        deprels = '|'.join( [p[0] for p in vislcg3Token[PARSER_OUT]] )
                        estnltkToken['s_rel']       = deprels
                        parse_found = True
                if not parse_found:
                    raise Exception("(!) Unable to retrieve syntactic analysis for the ",\
                                    estnltkToken, ' from ', vislcg3Token )
                j += 1


Instruction:
How would you implement a function in Python 3 that
creates dependency links from sentence - based linking to clause - based linking.

Response:
def _create_clause_based_dep_links( orig_text, layer=LAYER_CONLL ):
    '''  Rewrites dependency links in the text from sentence-based linking to clause-
        based linking:
          *) words which have their parent outside-the-clause will become root
             nodes (will obtain link value -1), and
          *) words which have their parent inside-the-clause will have parent index
             according to word indices inside the clause;

    '''
    sent_start_index = 0
    for sent_text in orig_text.split_by( SENTENCES ):
        # 1) Create a mapping: from sentence-based dependency links to clause-based dependency links
        mapping = dict()
        cl_ind  = sent_text.clause_indices
        for wid, word in enumerate(sent_text[WORDS]):
            firstSyntaxRel = sent_text[layer][wid][PARSER_OUT][0]
            parentIndex    = firstSyntaxRel[1]
            if parentIndex != -1:
                if cl_ind[parentIndex] != cl_ind[wid]:
                    # Parent of the word is outside the current clause: make root
                    # node from the current node
                    mapping[wid] = -1
                else:
                    # Find the beginning of the clause
                    clause_start = cl_ind.index( cl_ind[wid] )
                    # Find the index of parent label in the clause
                    j = 0
                    k = 0
                    while clause_start + j < len(cl_ind):
                        if clause_start + j == parentIndex:
                            break
                        if cl_ind[clause_start + j] == cl_ind[wid]:
                            k += 1
                        j += 1
                    assert clause_start + j < len(cl_ind), '(!) Parent index not found for: '+str(parentIndex)
                    mapping[wid] = k
            else:
                mapping[wid] = -1
        # 2) Overwrite old links with new ones
        for local_wid in mapping.keys():
            global_wid = sent_start_index + local_wid
            for syntax_rel in orig_text[layer][global_wid][PARSER_OUT]:
                syntax_rel[1] = mapping[local_wid]
        # 3) Advance the index for processing the next sentence
        sent_start_index += len(cl_ind)
    return orig_text


Instruction:
Create a Python 3 function for
sorting all the analyses in the sentence.

Response:
def __sort_analyses(sentence):
    ''' Sorts analysis of all the words in the sentence.
        This is required for consistency, because by default, analyses are
        listed in arbitrary order; '''
    for word in sentence:
        if ANALYSIS not in word:
            raise Exception( '(!) Error: no analysis found from word: '+str(word) )
        else:
            word[ANALYSIS] = sorted(word[ANALYSIS], \
                key=lambda x : "_".join( [x[ROOT],x[POSTAG],x[FORM],x[CLITIC]] ))
    return sentence


Instruction:
How would you code a function in Python 3 to
convert given estnltk Text object into CONLL format and returns as a string.

Response:
def convert_text_to_CONLL( text, feature_generator ):
    ''' Converts given estnltk Text object into CONLL format and returns as a
        string.
        Uses given *feature_generator* to produce fields ID, FORM, LEMMA, CPOSTAG,
        POSTAG, FEATS for each token.
        Fields to predict (HEAD, DEPREL) will be left empty.
        This method is used in preparing parsing & testing data for MaltParser.

        Parameters
        -----------
        text : estnltk.text.Text
            Morphologically analysed text from which the CONLL file is generated;

        feature_generator : CONLLFeatGenerator
            An instance of CONLLFeatGenerator, which has method *generate_features()*
            for generating morphological features for a single token;

        The aimed format looks something like this:
        1	Öö	öö	S	S	sg|nom	_	xxx	_	_
        2	oli	ole	V	V	indic|impf|ps3|sg	_	xxx	_	_
        3	täiesti	täiesti	D	D	_	_	xxx	_	_
        4	tuuletu	tuuletu	A	A	sg|nom	_	xxx	_	_
        5	.	.	Z	Z	Fst	_	xxx	_	_
    '''
    from estnltk.text import Text
    if not isinstance( text, Text ):
        raise Exception('(!) Unexpected type of input argument! Expected EstNLTK\'s Text. ')
    try:
        granularity = feature_generator.parseScope
    except AttributeError:
        granularity = SENTENCES
    assert granularity in [SENTENCES, CLAUSES], '(!) Unsupported granularity: "'+str(granularity)+'"!'
    sentenceStrs = []
    for sentence_text in text.split_by( granularity ):
        sentence_text[WORDS] = __sort_analyses( sentence_text[WORDS] )
        for i in range(len( sentence_text[WORDS] )):
            # Generate features  ID, FORM, LEMMA, CPOSTAG, POSTAG, FEATS
            strForm = feature_generator.generate_features( sentence_text, i )
            # *** HEAD  (syntactic parent)
            strForm.append( '_' )
            strForm.append( '\t' )
            # *** DEPREL  (label of the syntactic relation)
            strForm.append( 'xxx' )
            strForm.append( '\t' )
            # *** PHEAD
            strForm.append( '_' )
            strForm.append( '\t' )
            # *** PDEPREL
            strForm.append( '_' )
            sentenceStrs.append( ''.join( strForm ) )
        sentenceStrs.append( '' )
    return '\n'.join( sentenceStrs )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def convert_text_w_syntax_to_CONLL( text, feature_generator, layer=LAYER_CONLL ):
    ''' Converts given estnltk Text object into CONLL format and returns as a
        string.
        Uses given *feature_generator* to produce fields ID, FORM, LEMMA, CPOSTAG,
        POSTAG, FEATS for each token.
        Fills fields to predict (HEAD, DEPREL) with the syntactic information from
        given *layer* (default: LAYER_CONLL).
        This method is used in preparing training data for MaltParser.

        Parameters
        -----------
        text : estnltk.text.Text
            Morphologically analysed text from which the CONLL file is generated;

        feature_generator : CONLLFeatGenerator
            An instance of CONLLFeatGenerator, which has method *generate_features()*
            for generating morphological features for a single token;

        layer : str
            Name of the *text* layer from which syntactic information is to be taken.
            Defaults to LAYER_CONLL.

        The aimed format looks something like this:
        1	Öö	öö	S	S	sg|n	2	@SUBJ	_	_
        2	oli	ole	V	V	s	0	ROOT	_	_
        3	täiesti	täiesti	D	D	_	4	@ADVL	_	_
        4	tuuletu	tuuletu	A	A	sg|n	2	@PRD	_	_
        5	.	.	Z	Z	_	4	xxx	_	_
    '''
    from estnltk.text import Text
    if not isinstance( text, Text ):
        raise Exception('(!) Unexpected type of input argument! Expected EstNLTK\'s Text. ')
    assert layer in text, ' (!) The layer "'+layer+'" is missing form the Text object.'
    try:
        granularity = feature_generator.parseScope
    except AttributeError:
        granularity = SENTENCES
    assert granularity in [SENTENCES, CLAUSES], '(!) Unsupported granularity: "'+str(granularity)+'"!'
    sentenceStrs = []
    if granularity == CLAUSES:
        _create_clause_based_dep_links( text, layer )
    for sentence_text in text.split_by( granularity ):
        sentence_text[WORDS] = __sort_analyses( sentence_text[WORDS] )
        for i in range(len( sentence_text[WORDS] )):
            # Generate features  ID, FORM, LEMMA, CPOSTAG, POSTAG, FEATS
            strForm = feature_generator.generate_features( sentence_text, i )
            # Get syntactic analysis of the token
            syntaxToken    = sentence_text[layer][i]
            firstSyntaxRel = syntaxToken[PARSER_OUT][0]
            # *** HEAD  (syntactic parent)
            parentLabel = str( firstSyntaxRel[1] + 1 )
            strForm.append( parentLabel )
            strForm.append( '\t' )
            # *** DEPREL  (label of the syntactic relation)
            if parentLabel == '0':
                strForm.append( 'ROOT' )
                strForm.append( '\t' )
            else:
                strForm.append( firstSyntaxRel[0] )
                strForm.append( '\t' )
            # *** PHEAD
            strForm.append( '_' )
            strForm.append( '\t' )
            # *** PDEPREL
            strForm.append( '_' )
            sentenceStrs.append( ''.join( strForm ) )
        sentenceStrs.append( '' )
    return '\n'.join( sentenceStrs )

Response: Converts given estnltk Text object into CONLL format.