Instruction:
Implement a function in Python 3 to
execute Maltparser on given input string and returns the result.

Response:
def _executeMaltparser( input_string, maltparser_dir, maltparser_jar, model_name ):
    ''' Executes Maltparser on given (CONLL-style) input string, and
        returns the result. The result is an array of lines from Maltparser's
        output.

        Parameters
        ----------
        input_string: string
              input text in CONLL format;
        maltparser_jar: string
              name of the Maltparser's jar file that should be executed;
        model_name: string
              name of the model that should be used;
        maltparser_dir: string
              the directory containing Maltparser's jar and the model file;

        Few of the ideas were also borrowed from NLTK's MaltParser class,
        see  http://www.nltk.org/_modules/nltk/parse/malt.html   for the reference;
    '''

    temp_input_file = \
      tempfile.NamedTemporaryFile(prefix='malt_in.', mode='w', delete=False)
    temp_input_file.close()
    # We have to open separately here for writing, because Py 2.7 does not support
    # passing parameter   encoding='utf-8'    to the NamedTemporaryFile;
    out_f = codecs.open(temp_input_file.name, mode='w', encoding='utf-8')
    out_f.write( input_string )
    out_f.close()

    temp_output_file = tempfile.NamedTemporaryFile(prefix='malt_out.', mode='w', delete=False)
    temp_output_file.close()

    current_dir = os.getcwd()
    os.chdir(maltparser_dir)
    cmd = ['java', '-jar', os.path.join(maltparser_dir, maltparser_jar), \
           '-c', model_name, \
           '-i', temp_input_file.name, \
           '-o', temp_output_file.name, \
           '-m', 'parse' ]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if p.wait() != 0:
        raise Exception(' Error on running Maltparser: ', p.stderr.read() )
    os.chdir(current_dir)

    results = []
    in_f = codecs.open(temp_output_file.name, mode='r', encoding='utf-8')
    for line in in_f:
        results.append( line.rstrip() )
    in_f.close()

    if not temp_input_file.closed:
        raise Exception('Temp input file unclosed!')
    if not temp_output_file.closed:
        raise Exception('Temp input file unclosed!')
    if not out_f.closed:
        raise Exception('Output file unclosed!')
    if not in_f.closed:
        raise Exception('Input file unclosed!')
    # TODO: For some reason, the method gives "ResourceWarning: unclosed file"
    # in Python 3.4, although, apparently, all file handles seem to be closed;
    # Nothing seems to be wrong in Python 2.7;

    os.remove(temp_input_file.name)
    os.remove(temp_output_file.name)

    return results


Instruction:
Can you tell what is the following Python 3 function doing
def loadCONLLannotations( in_file, addDepRels = False, splitIntoSentences = True ):
    ''' Loads syntactically annotated text from CONLL format input file and
        returns as an array of tokens, where each token is represented as
        an array in the format:
           [sentenceID, wordID, tokenString, morphInfo, selfID, parentID]
        If addDepRels == True, then the dependency relation label is also extracted
        and added to the end of the array:
           [sentenceID, wordID, tokenString, morphInfo, selfID, parentID, depRel]
        If splitIntoSentences == True, the array of tokens is further divided
        into subarrays representing sentences.

         Example input:
        2	Monstrumteleskoobid	Monstrum_tele_skoop	S	S	prop|pl|nom	0	ROOT	_	_
        3	(	(	Z	Z	Opr	4	xxx	_	_
        4	mosaiik-	mosaiik	A	A	pos|sg|nom	2	@<AN	_	_
        5	ja	ja	J	J	crd	6	@J	_	_
        6	mitmepeeglilised	mitme_peegli=line	A	A	pos|pl|nom	4	@<NN	_	_
        7	)	)	Z	Z	Cpr	6	xxx	_	_
        8	.	.	Z	Z	Fst	7	xxx	_	_

    '''
    sentenceCount   = 0
    wordCountInSent = 0
    tokens = []
    in_f = codecs.open(in_file, mode='r', encoding='utf-8')
    for line in in_f:
        line = line.rstrip()
        if len(line) == 0 or re.match('^\s+$', line):
            sentenceCount += 1
            wordCountInSent = 0
            continue
        features = line.split('\t')
        if len(features) != 10:
            raise Exception(' In file '+in_file+', line with unexpected format: "'+line+'" ')
        selfLabel   = features[0]
        token       = features[1]
        lemma       = features[2]
        cpos        = features[3]
        pos         = features[4]
        form        = features[5]
        parentLabel = features[6]
        tokens.append( [ str(sentenceCount), str(wordCountInSent), \
                         token, lemma+" "+pos+" "+form, selfLabel, parentLabel ] )
        if addDepRels:
            tokens[-1].append( features[7] )
        wordCountInSent += 1
    in_f.close()
    if not splitIntoSentences:
        return tokens
    else:
        sentences = []
        lastSentID = ''
        for tok in tokens:
            if tok[0] != lastSentID:
                sentences.append([])
            sentences[-1].append(tok)
            lastSentID = tok[0]
        return sentences

Response: Loads syntactically annotated text from CONLL format input file and returns as an array of tokens where each token is represented as a single word and each parent is represented as a single parent.


Instruction:
Here you have a function in Python 3, explain what it does
def convertCONLLtoText( in_file, addDepRels = False, verbose = False, **kwargs ):
    ''' Loads CONLL format data from given input file, and creates
        estnltk Text objects from the data, one Text per each
        sentence. Returns a list of Text objects.

        By default, applies estnltk's morphological analysis, clause
        detection, and verb chain detection to each input sentence.

        If addDepRels == True, in addition to SYNTAX_LABEL and SYNTAX_HEAD,
        surface syntactic function (DEPREL) is also attributed to each
        token;
    '''
    from estnltk.text import Text
    sentences = loadCONLLannotations( in_file, addDepRels = addDepRels, \
                                               splitIntoSentences = True )
    if verbose:
        print( str(len(sentences))+' sentences loaded. ')
    estnltkSentTexts = []
    for i in range(len(sentences)):
        s  = sentences[i]
        sentenceString = " ".join( [ t[2] for t in s ] )
        sentText = Text(sentenceString, **kwargs)
        sentText.tag_analysis()
        sentText.tag_clauses()
        sentText.tag_verb_chains()
        sentText = dict(sentText)
        if len(sentText[WORDS]) == len(s):
            # Add the dependency syntactic information
            for j in range(len(sentText[WORDS])):
                estnltkWord   = sentText[WORDS][j]
                depSyntaxWord = s[j]
                estnltkWord[SYNTAX_LABEL] = depSyntaxWord[4]
                estnltkWord[SYNTAX_HEAD]  = depSyntaxWord[5]
                if addDepRels:
                   estnltkWord[DEPREL] = depSyntaxWord[6]
            estnltkSentTexts.append( sentText )
            if verbose:
                print ('*', end = '')
        else:
            if verbose:
                print("The sentence segmentation of dependency syntax differs from the estnltk's sentence segmentation:", len(sentText[WORDS]), ' vs ',len(s))
    return estnltkSentTexts

Response: Load a CONLL file and create a list of estnltk Text objects.


Instruction:
Create a Python 3 function for
augmenting given Text object with the information from Maltparser s output.

Response:
def augmentTextWithCONLLstr( conll_str_array, text ):
    ''' Augments given Text object with the information from Maltparser's output.
        More specifically, adds information about SYNTAX_LABEL, SYNTAX_HEAD and
        DEPREL to each token in the Text object;
    '''
    j = 0
    for sentence in text.divide( layer=WORDS, by=SENTENCES ):
        sentence = __sort_analyses(sentence)
        for i in range(len(sentence)):
            estnltkToken    = sentence[i]
            maltparserToken = conll_str_array[j]
            if len( maltparserToken ) > 1:
                maltParserAnalysis = maltparserToken.split('\t')
                if estnltkToken[TEXT] == maltParserAnalysis[1]:
                    # Fetch information about the syntactic relation:
                    estnltkToken[SYNTAX_LABEL] = maltParserAnalysis[0]
                    estnltkToken[SYNTAX_HEAD]  = maltParserAnalysis[6]
                    # Fetch the name of the surface syntactic relation
                    estnltkToken[DEPREL]       = maltParserAnalysis[7]
                else:
                    raise Exception("A misalignment between Text and Maltparser's output: ",\
                        estnltkToken, maltparserToken )
            j += 1
        j += 1


Instruction:
Make a summary of the following Python 3 code
def align_CONLL_with_Text( lines, text, feature_generator, **kwargs ):
    ''' Aligns CONLL format syntactic analysis (a list of strings) with given EstNLTK's Text
        object.
        Basically, for each word position in the Text object, finds corresponding line(s) in
        the CONLL format output;
        Returns a list of dicts, where each dict has following attributes:
          'start'   -- start index of the word in Text;
          'end'     -- end index of the word in Text;
          'sent_id' -- index of the sentence in Text, starting from 0;
          'parser_out' -- list of analyses from the output of the syntactic parser;

        Parameters
        -----------
        lines : list of str
            The input text for the pipeline; Should be the CONLL format syntactic analysis;
        text : Text
            EstNLTK Text object containing the original text that was analysed with
            MaltParser;
        feature_generator : CONLLFeatGenerator
            The instance of CONLLFeatGenerator, which was used for generating the input of
            the MaltParser; If None, assumes a default feature-generator with the scope set
            to 'sentences';

        check_tokens : bool
            Optional argument specifying whether tokens should be checked for match
            during the alignment. In case of a mismatch, an exception is raised.
            Default:False

        add_word_ids : bool
            Optional argument specifying whether each alignment should include attributes:
            * 'text_word_id' - current word index in the whole Text, starting from 0;
            * 'sent_word_id' - index of the current word in the sentence, starting from 0;
            Default:False

    '''
    from estnltk.text import Text
    if not isinstance( text, Text ):
        raise Exception('(!) Unexpected type of input argument! Expected EstNLTK\'s Text. ')
    if not isinstance( lines, list ):
        raise Exception('(!) Unexpected type of input argument! Expected a list of strings.')
    try:
        granularity = feature_generator.parseScope
    except (AttributeError, NameError):
        granularity = SENTENCES
    assert granularity in [SENTENCES, CLAUSES], '(!) Unsupported granularity: "'+str(granularity)+'"!'
    check_tokens = False
    add_word_ids = False
    for argName, argVal in kwargs.items() :
        if argName in ['check_tokens', 'check'] and argVal in [True, False]:
           check_tokens = argVal
        if argName in ['add_word_ids', 'word_ids'] and argVal in [True, False]:
           add_word_ids = argVal
    generalWID = 0
    sentenceID = 0
    # Collect clause indices for each sentence (if required)
    clause_indices = None
    if granularity == CLAUSES:
        c = 0
        all_clause_indices = text.clause_indices
        clause_indices = []
        for sentence_words in text.divide( layer=WORDS, by=SENTENCES ):
            clause_indices.append([])
            for wid, estnltkToken in enumerate( sentence_words ):
                clause_indices[-1].append( all_clause_indices[c] )
                c += 1
    # Iterate over the sentences and perform the alignment
    results = []
    j = 0
    for sentence_words in text.divide( layer=WORDS, by=SENTENCES ):
        tokens_to_collect = len( sentence_words )
        tokens_collected  = 0
        chunks      = [[]]
        while j < len(lines):
            maltparserToken = lines[j]
            if len( maltparserToken ) > 1 and '\t' in maltparserToken:
                # extend the existing clause chunk
                token_dict = { 't':maltparserToken, \
                               'w':(maltparserToken.split('\t'))[1] }
                chunks[-1].append( token_dict )
                tokens_collected += 1
            else:
                # create a new clause chunk
                if len(chunks[-1]) != 0:
                    chunks.append( [] )
            j += 1
            if tokens_to_collect == tokens_collected:
                break
        if tokens_to_collect != tokens_collected:  # a sanity check
            raise Exception('(!) Unable to collect the following sentence from the output of MaltParser: "'+\
                                 str(sentence_words)+'"')
        # 2) Put the sentence back together
        if granularity == SENTENCES:
            # A. The easy case: sentence-wise splitting was used
            for wid, estnltkToken in enumerate( sentence_words ):
                maltparserToken = chunks[0][wid]['t']
                if check_tokens and estnltkToken[TEXT] != chunks[0][wid]['w']:
                    raise Exception("(!) A misalignment between Text and CONLL: ",\
                                    estnltkToken, maltparserToken )
                # Populate the alignment
                result_dict = { START:estnltkToken[START], END:estnltkToken[END], \
                                SENT_ID:sentenceID, PARSER_OUT: [maltparserToken] }
                if add_word_ids:
                    result_dict['text_word_id'] = generalWID # word id in the text
                    result_dict['sent_word_id'] = wid        # word id in the sentence
                results.append( result_dict )
                generalWID += 1
        elif granularity == CLAUSES:
            # B. The tricky case: clause-wise splitting was used
            results_by_wid = {}
            # B.1  Try to find the location of each chunk in the original text
            cl_ind = clause_indices[sentenceID]
            for chunk_id, chunk in enumerate(chunks):
                firstWord = chunk[0]['w']
                chunkLen  = len(chunk)
                estnltk_token_ids = []
                seen_clause_ids   = {}
                for wid, estnltkToken in enumerate( sentence_words ):
                    # Try to recollect tokens of the clause starting from location wid
                    if estnltkToken[TEXT] == firstWord and \
                       wid+chunkLen <= len(sentence_words) and cl_ind[wid] not in seen_clause_ids:
                        clause_index = cl_ind[wid]
                        i = wid
                        while i < len(sentence_words):
                            if cl_ind[i] == clause_index:
                                estnltk_token_ids.append( i )
                            i += 1
                    # Remember that we have already seen this clause
                    # (in order to avoid start collecting from the middle of the clause)
                    seen_clause_ids[cl_ind[wid]] = 1
                    if len(estnltk_token_ids) == chunkLen:
                        break
                    else:
                        estnltk_token_ids = []
                if len(estnltk_token_ids) == chunkLen:
                    # Align the CONLL clause with the clause from the original estnltk Text
                    for wid, estnltk_wid in enumerate(estnltk_token_ids):
                        estnltkToken    = sentence_words[estnltk_wid]
                        maltparserToken = chunk[wid]['t']
                        if check_tokens and estnltkToken[TEXT] != chunk[wid]['w']:
                            raise Exception("(!) A misalignment between Text and CONLL: ",\
                                                 estnltkToken, maltparserToken )
                        # Convert indices: from clause indices to sentence indices
                        tokenFields = maltparserToken.split('\t')
                        if tokenFields[6] != '0':
                            in_clause_index = int(tokenFields[6])-1
                            assert in_clause_index in range(0, len(estnltk_token_ids)), \
                                   '(!) Unexpected clause index from CONLL: '+str(in_clause_index)+\
                                   ' \ '+str(len(estnltk_token_ids))
                            in_sent_index   = estnltk_token_ids[in_clause_index]+1
                            tokenFields[6]  = str(in_sent_index)
                        tokenFields[0] = str(estnltk_wid+1)
                        maltparserToken = '\t'.join(tokenFields)
                        # Populate the alignment
                        result_dict = { START:estnltkToken[START], END:estnltkToken[END], \
                                        SENT_ID:sentenceID, PARSER_OUT: [maltparserToken] }
                        results_by_wid[estnltk_wid] = result_dict
                else:
                    raise Exception('(!) Unable to locate the clause in the original input: '+str(chunk))
            if len(results_by_wid.keys()) != len(sentence_words):
                raise Exception('(!) Error in aligning Text and CONLL - token counts not matching:'+\
                                str(len(results_by_wid.keys()))+ ' vs '+str(len(sentence_words)) )
            # B.2  Put the sentence back together
            for wid in sorted(results_by_wid.keys()):
                if add_word_ids:
                    results_by_wid[wid]['text_word_id'] = generalWID # word id in the text
                    results_by_wid[wid]['sent_word_id'] = wid        # word id in the sentence
                results.append( results_by_wid[wid] )
                generalWID += 1
        sentenceID += 1
    return results

Response: Aligns a list of lines with a given EstNLTK s Text object.


Instruction:
Create a Python 3 function to
collect clause with index clause_id from given sentence_text.

Response:
def _get_clause_words( sentence_text, clause_id ):
    ''' Collects clause with index *clause_id* from given *sentence_text*.
        Returns a pair (clause, isEmbedded), where:
         *clause* is a list of word tokens in the clause;
         *isEmbedded* is a bool indicating whether the clause is embedded;
    '''
    clause = []
    isEmbedded = False
    indices = sentence_text.clause_indices
    clause_anno = sentence_text.clause_annotations
    for wid, token in enumerate(sentence_text[WORDS]):
        if indices[wid] == clause_id:
            if not clause and clause_anno[wid] == EMBEDDED_CLAUSE_START:
                isEmbedded = True
            clause.append((wid, token))
    return clause, isEmbedded


Instruction:
Make a summary of the following Python 3 code
def _detect_quotes( sentence_text, wid, fromRight = True ):
    ''' Searches for quotation marks (both opening and closing) closest to
        given location in sentence (given as word index *wid*);

        If *fromRight == True* (default), searches from the right (all the
        words having index greater than *wid*), otherwise, searches from the
        left (all the words having index smaller than *wid*);

        Returns index of the closest quotation mark found, or -1, if none was
        found;
    '''
    i = wid
    while (i > -1 and i < len(sentence_text[WORDS])):
        token = sentence_text[WORDS][i]
        if _pat_starting_quote.match(token[TEXT]) or \
           _pat_ending_quote.match(token[TEXT]):
            return i
        i += 1 if fromRight else -1
    return -1

Response: Detects the quotation marks in a sentence.


Instruction:
Given the following Python 3 function, write the documentation
def detect_sentence_ending_saying_verbs( edt_sent_text ):
    ''' Detects cases where a saying verb (potential root of the sentence) ends the sentence.

        We use a simple heuristic: if the given sentence has multiple clauses, and the last main
        verb in the sentence is preceded by ", but is not followed by ", then the main verb is
        most likely a saying verb.

        Examples:
            " See oli ainult unes , " [vaidles] Jan .
            " Ma ei maga enam Joogaga ! " [protesteerisin] .
            " Mis mõttega te jama suust välja ajate ? " [läks] Janil nüüd juba hari punaseks .

        Note that the class of saying verbs is open, so we try not rely on a listing of verbs,
        but rather on the conventional usage patterns of reported speech, indicated by quotation
        marks.

        Returns a dict containing word indexes of saying verbs;
    '''
    from estnltk.mw_verbs.utils import WordTemplate

    if not edt_sent_text.is_tagged( VERB_CHAINS ):
        edt_sent_text.tag_verb_chains()

    saying_verbs = {}
    if len(edt_sent_text[VERB_CHAINS]) < 2:
        # Skip sentences that do not have any chains, or
        #                have only a single verb chain
        return saying_verbs

    patColon = WordTemplate({'partofspeech':'^[Z]$', 'text': '^:$'})

    for vid, vc in enumerate( edt_sent_text[VERB_CHAINS] ):
        #
        #  Look only multi-clause sentences, where the last verb chain has length 1
        #
        if len(vc['phrase']) == 1 and vid == len(edt_sent_text[VERB_CHAINS])-1:
            wid   = vc['phrase'][0]
            token = edt_sent_text[WORDS][wid]
            clause_id = vc[CLAUSE_IDX]
            # Find corresponding clause and locations of quotation marks
            clause, insideEmbeddedCl = _get_clause_words( edt_sent_text, clause_id )
            quoteLeft  = _detect_quotes( edt_sent_text, wid, fromRight = False )
            quoteRight = _detect_quotes( edt_sent_text, wid, fromRight = True )
            #
            #  Exclude cases, where there are double quotes within the same clause:
            #     ... ootab igaüks ,] [kuidas aga kähku tagasi " varrastusse " <saaks> .]
            #     ... miljonäre on ka nende seas ,] [kes oma “ papi ” mustas äris <teenivad>  .]
            #
            quotes_in_clause = []
            for (wid2, token2) in clause:
                if _pat_starting_quote.match(token2[TEXT]) or \
                    _pat_ending_quote.match(token2[TEXT]):
                    quotes_in_clause.append(wid2)
            multipleQuotes = len(quotes_in_clause) > 1 and quotes_in_clause[-1]==quoteLeft
            #
            #  If the preceding double quotes are not within the same clause, and
            #     the verb is not within an embedded clause, and a quotation mark strictly
            #     precedes, but none follows, then we have most likely a saying verb:
            #         " Ma ei tea , " [kehitan] õlga .
            #         " Miks jumal meid karistab ? " [mõtles] sir Galahad .
            #         " Kaarsild pole teatavastki elusolend , " [lõpetasin] arutelu .
            #
            if not multipleQuotes and \
               not insideEmbeddedCl and \
               (quoteLeft != -1 and quoteLeft+1 == wid and quoteRight == -1):
                saying_verbs[wid] = 'se_saying_verb'
    return saying_verbs

Response: Detects cases where a given sentence ends the given text.


Instruction:
Can you write a function in Python 3 where it
loads k - sub - category relations from a file.

Response:
def _loadKSubcatRelations( inputFile ):
    ''' Laeb sisendfailist (inputFile) kaassõnade rektsiooniseoste mustrid.
        Iga muster peab olema failis eraldi real, kujul:
        (sõnalemma);(sõnaliik);(post|pre);(nõutud_käänete_regexp)
        nt
            ees;_K_;post;g
            eest;_K_;post;g
            enne;_K_;pre;p
        Tagastab laetud andmed sõnastikuna;
    '''
    kSubCatRelations = dict()
    in_f = codecs.open(inputFile, mode='r', encoding='utf-8')
    for line in in_f:
        line = line.rstrip()
        if len(line) > 0 and not re.match("^#.+$", line):
            items = line.split(';')
            if len(items) == 4:
                root         = items[0]
                partofspeech = items[1]
                postPre      = items[2]
                morphPattern = items[3]
                fpattern = '(sg|pl)\s'+morphPattern
                if root not in kSubCatRelations:
                    kSubCatRelations[root] = []
                kSubCatRelations[root].append( [postPre, fpattern] )
                root_clean = root.replace('_', '')
                if root != root_clean:
                    if root_clean not in kSubCatRelations:
                        kSubCatRelations[root_clean] = []
                    kSubCatRelations[root_clean].append( [postPre, fpattern] )
            else:
                raise Exception(' Unexpected number of items in the input lexicon line: '+line)
    in_f.close()
    return kSubCatRelations


Instruction:
Can you generate the documentation for the following Python 3 function
def _detectKsubcatRelType( sentence, i, kSubCatRelsLexicon ):
    ''' Given the adposition appearing in the sentence at the location i,
        checks whether the adposition appears in the kSubCatRelsLexicon,
        and if so, attempts to further detect whether the adposition is a
        preposition or a postposition;
         Returns a tuple (string, int), where the first item indicates the
        type of adposition ('pre', 'post', '_'), and the second item points
        to its possible child (index of the word in sentence, or -1, if
        possible child was not detected from close range);
    '''
    curToken = sentence[i]
    root = curToken[ANALYSIS][0][ROOT]
    if root in kSubCatRelsLexicon:
        for [postPre, fpattern] in kSubCatRelsLexicon[root]:
            if postPre == 'post' and i-1 > -1:
                lastTokenAnalysis = sentence[i-1][ANALYSIS][0]
                if re.match(fpattern, lastTokenAnalysis[FORM]):
                    return ('post', i-1)
            elif postPre == 'pre' and i+1 < len(sentence):
                nextTokenAnalysis = sentence[i+1][ANALYSIS][0]
                if re.match(fpattern, nextTokenAnalysis[FORM]):
                    return ('pre', i+1)
        # If the word is not ambiguous between pre and post, but
        # the possible child was not detected, return only the
        # post/pre label:
        if len(kSubCatRelsLexicon[root]) == 1:
            return (kSubCatRelsLexicon[root][0][0], -1)
    return ('_', -1)

Response: Detects the type of the kSubcatRels in the word at the location i.


Instruction:
Implement a Python 3 function for
attempting to detect all possible K subcategorization relations from a given sentence.

Response:
def _detectPossibleKsubcatRelsFromSent( sentence, kSubCatRelsLexicon, reverseMapping = False ):
    ''' Attempts to detect all possible K subcategorization relations from
        given sentence, using the heuristic method _detectKsubcatRelType();

        Returns a dictionary of relations where the key corresponds to the
        index of its parent node (the K node) and the value corresponds to
        index of its child.

        If reverseMapping = True, the mapping is reversed: keys correspond
        to children and values correspond to parent nodes (K-s);
    '''
    relationIndex = dict()
    relationType  = dict()
    for i in range(len(sentence)):
        estnltkWord = sentence[i]
        # Pick the first analysis
        firstAnalysis = estnltkWord[ANALYSIS][0]
        if firstAnalysis[POSTAG] == 'K':
            (grammCats, kChild) = _detectKsubcatRelType( sentence, i, kSubCatRelsLexicon )
            if kChild != -1:
                if reverseMapping:
                    relationIndex[ kChild ] = i
                    relationType[ kChild ]  = grammCats
                else:
                    relationIndex[ i ] = kChild
                    relationType[ i ]  = grammCats
    return relationIndex, relationType


Instruction:
Can you write a function in Python 3 where it
finds all possible K subcategorization relations from a given sentence and returns a dictionary where the keys correspond to the indices of the tokens that are relevant to the K subcat relations.

Response:
def _findKsubcatFeatures( sentence, kSubCatRelsLexicon, addFeaturesToK = True ):
    ''' Attempts to detect all possible K subcategorization relations from
        given sentence, using the heuristic methods _detectKsubcatRelType()
        and _detectPossibleKsubcatRelsFromSent();

        Returns a dictionary where the keys correspond to token indices,
        and values are grammatical features related to K subcat relations.
        Not all tokens in the sentence are indexed, but only tokens relevant
        to K subcat relations;

        If addFeaturesToK == True, grammatical features are added to K-s,
        otherwise, grammatical features are added to K's child tokens.
    '''
    features = dict()
    # Add features to the K (adposition)
    if addFeaturesToK:
        for i in range(len(sentence)):
            estnltkWord = sentence[i]
            # Pick the first analysis
            firstAnalysis = estnltkWord[ANALYSIS][0]
            if firstAnalysis[POSTAG] == 'K':
                (grammCats, kChild) = _detectKsubcatRelType( sentence, i, kSubCatRelsLexicon )
                features[i] = grammCats
    # Add features to the noun governed by K
    else:
        relationIndex, relationType = \
            _detectPossibleKsubcatRelsFromSent( sentence, kSubCatRelsLexicon, reverseMapping = True )
        for i in relationIndex:
            features[i] = relationType[i]
    return features


Instruction:
Can you generate the documentation for the following Python 3 function
def generate_features( self, sentence_text, wid ):
        ''' Generates and returns a list of strings, containing tab-separated
            features ID, FORM, LEMMA, CPOSTAG, POSTAG, FEATS of the word
            (the word with index *wid* from the given *sentence_text*).

            Parameters
            -----------
            sentence_text : estnltk.text.Text
                Text object corresponding to a single sentence.
                Words of the sentence, along with their morphological analyses,
                should be accessible via the layer WORDS.
                And each word should be a dict, containing morphological features
                in ANALYSIS part;

            wid : int
                Index of the word/token, whose features need to be generated;

        '''
        assert WORDS in sentence_text and len(sentence_text[WORDS])>0, \
               " (!) 'words' layer missing or empty in given Text!"
        sentence = sentence_text[WORDS]
        assert -1 < wid and wid < len(sentence), ' (!) Invalid word id: '+str(wid)

        # 1) Pre-process (if required)
        if wid == 0:
            #  *** Add adposition (_K_) type
            if self.kSubCatRelsLex:
                self.kFeatures = \
                    _findKsubcatFeatures( sentence, self.kSubCatRelsLex, addFeaturesToK = True )
            #  *** Add verb chain info
            if self.addVerbcGramm or self.addNomAdvVinf:
                self.vcFeatures = generate_verb_chain_features( sentence_text, \
                                                                addGrammPred=self.addVerbcGramm, \
                                                                addNomAdvVinf=self.addNomAdvVinf )
            #  *** Add sentence ending saying verbs
            if self.addSeSayingVerbs:
                self.sayingverbs = detect_sentence_ending_saying_verbs( sentence_text )
            #  *** Add clause boundary info
            if self.addClauseBound:
                self.clbFeatures = []
                for tag in sentence_text.clause_annotations:
                    if not tag:
                        self.clbFeatures.append( [] )
                    elif tag == EMBEDDED_CLAUSE_START:
                        self.clbFeatures.append( ['emb_cl_start'] )
                    elif tag == EMBEDDED_CLAUSE_END:
                        self.clbFeatures.append( ['emb_cl_end'] )
                    elif tag == CLAUSE_BOUNDARY:
                        self.clbFeatures.append (['clb'] )

        # 2) Generate the features
        estnltkWord = sentence[wid]
        # Pick the first analysis
        firstAnalysis = estnltkWord[ANALYSIS][0]
        strForm = []
        # *** ID
        strForm.append( str(wid+1) )
        strForm.append( '\t' )
        # *** FORM
        word_text = estnltkWord[TEXT]
        word_text = word_text.replace(' ', '_')
        strForm.append( word_text )
        strForm.append( '\t' )
        # *** LEMMA
        word_root = firstAnalysis[ROOT]
        word_root = word_root.replace(' ', '_')
        if len(word_root) == 0:
            word_root = "??"
        strForm.append( word_root )
        strForm.append( '\t' )
        # *** CPOSTAG
        strForm.append( firstAnalysis[POSTAG] )
        strForm.append( '\t' )
        # *** POSTAG
        finePos = firstAnalysis[POSTAG]
        if self.addAmbiguousPos and len(estnltkWord[ANALYSIS]) > 1:
            pos_tags = sorted(list(set([ a[POSTAG] for a in estnltkWord[ANALYSIS] ])))
            finePos  = '_'.join(pos_tags)
        #if self.kFeatures and wid in self.kFeatures:
        #    finePos += '_'+self.kFeatures[wid]
        strForm.append( finePos )
        strForm.append( '\t' )
        # *** FEATS  (grammatical categories)
        grammCats = []
        if len(firstAnalysis[FORM]) != 0:
            forms = firstAnalysis[FORM].split()
            grammCats.extend( forms )
        # add features from verb chains:
        if self.vcFeatures and self.vcFeatures[wid]:
            grammCats.extend( self.vcFeatures[wid] )
        # add features from clause boundaries:
        if self.addClauseBound and self.clbFeatures[wid]:
            grammCats.extend( self.clbFeatures[wid] )
        # add adposition type ("post" or "pre")
        if self.kFeatures and wid in self.kFeatures:
            grammCats.extend( [self.kFeatures[wid]] )
        # add saying verb features
        if self.sayingverbs and wid in self.sayingverbs:
            grammCats.extend( [self.sayingverbs[wid]] )
        # wrap up
        if not grammCats:
            grammCats = '_'
        else:
            grammCats = '|'.join( grammCats )
        strForm.append( grammCats )
        strForm.append( '\t' )
        return strForm

Response: Generates and returns a list of strings containing tab - separated features ID FORM LEMMA CPOSTAG POSTAG FEATS of the word with index wid.


Instruction:
How would you explain what the following Python 3 function does
def create_rules(aes, value):
    if isinstance(value, six.string_types):
        return Rules(aes)
    else:
        rules = Rules()
        for idx, (pattern, css_value) in enumerate(value):
            rules.add_rule(pattern, '{0}_{1}'.format(aes, idx))
        return rules

Response: Create a Rules instance for a single aesthetic value.


Instruction:
How would you explain what the following Python 3 function does
def add_rule(self, pattern, css_class):
        #print('adding rule <{0}> <{1}>'.format(pattern, css_class))
        self.__patterns.append(re.compile(pattern, flags=re.U | re.M))
        self.__css_classes.append(css_class)

Response: Adds a new rule to the internal list of patterns and css classes.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_css_class(self, value):
        #print ('get_css_class for {0}'.format(value))
        for idx, pattern in enumerate(self.__patterns):
            if pattern.match(value) is not None:
                #print ('matched rule {0} and returning {1}'.format(idx, self.__css_classes[idx]))
                return self.__css_classes[idx]
        return self.__default

Response: Return the css class of first pattern that matches given value.


Instruction:
Make a summary of the following Python 3 code
def create_index(index_name, **kwargs):
    es = elasticsearch.Elasticsearch(**kwargs)
    es.indices.create(index=index_name, body=mapping)
    return connect(index_name, **kwargs)

Response: Creates an index in Elasticsearch and returns it


Instruction:
Explain what the following Python 3 code does
def _get_indexable_sentences(document):

        def unroll_lists(list_of_lists):
            for i in itertools.product(*[set(j) for j in list_of_lists]):
                yield ' '.join(i)

        sents = document.split_by_sentences()
        for order, sent in enumerate(sents):
            postags = list(unroll_lists(sent.postag_lists))
            lemmas = list(unroll_lists(sent.lemma_lists))
            text = sent.text
            words = copy.deepcopy(sent.words)
            for i in words:
                del i['start']
                del i['end']

            sentence = {
                'estnltk_text_object': json.dumps(sent),
                'meta': {
                    'order_in_parent': order
                },
                'text': text,
                'words': words,
                'postags': postags,
                'lemmas': lemmas
            }
            yield json.dumps(sentence)

Response: Yields a list of all the indexable sentences in the document.


Instruction:
Given the following Python 3 function, write the documentation
def apply_templates(toks, templates):
    for template in templates:
        name = '|'.join(['%s[%d]' % (f, o) for f, o in template])
        for t in range(len(toks)):
            values_list = []
            for field, offset in template:
                p = t + offset
                if p < 0 or p >= len(toks):
                    values_list = []
                    break
                if field in toks[p]:
                    value = toks[p][field]
                    values_list.append(value if isinstance(value, (set, list)) else [value])
            if len(template) == len(values_list):
                for values in product(*values_list):
                    toks[t]['F'].append('%s=%s' % (name, '|'.join(values)))

Response: Generates features for an item sequence by applying feature templates.


Instruction:
Can you implement a function in Python 3 that
converts an estnltk JSON - style document to an estner document.

Response:
def json_document_to_estner_document(jsondoc):
    """Convert an estnltk document to an estner document.

    Parameters
    ----------
    jsondoc: dict
        Estnltk JSON-style document.

    Returns
    -------
    estnltk.estner.ner.Document
        A ner document.
    """
    sentences = []
    for json_sent in jsondoc.split_by_sentences():
        snt = Sentence()
        zipped = list(zip(
            json_sent.word_texts,
            json_sent.lemmas,
            json_sent.root_tokens,
            json_sent.forms,
            json_sent.endings,
            json_sent.postags))
        json_toks = [{TEXT: text, LEMMA: lemma, ROOT_TOKENS: root_tokens, FORM: form, ENDING: ending, POSTAG: postag}
                     for text, lemma, root_tokens, form, ending, postag in zipped]
        # add labels, if they are present
        for tok, word in zip(json_toks, json_sent.words):
            if LABEL in word:
                tok[LABEL] = word[LABEL]
        for json_tok in json_toks:
            token = json_token_to_estner_token(json_tok)
            snt.append(token)
        if snt:
            for i in range(1, len(snt)):
                snt[i - 1].next = snt[i]
                snt[i].prew = snt[i - 1]
            sentences.append(snt)
    return Document(sentences=sentences)


Instruction:
Explain what the following Python 3 code does
def json_token_to_estner_token(json_token):
    token = Token()
    word = json_token[TEXT]
    lemma = word
    morph = ''
    label = 'O'
    ending = json_token[ENDING]
    root_toks = json_token[ROOT_TOKENS]
    if isinstance(root_toks[0], list):
        root_toks = root_toks[0]
    lemma = '_'.join(root_toks) + ('+' + ending if ending else '')
    if not lemma:
        lemma = word
    morph = '_%s_' % json_token[POSTAG]
    morph += ' ' + json_token[FORM]
    if LABEL in json_token:
        label = json_token[LABEL]
    return Token(word, lemma, morph, label)

Response: Convert a JSON - style word token to an estner token.


Instruction:
Explain what the following Python 3 code does
def copy_settings(self, settings_module):
        source = inspect.getsourcefile(settings_module)
        dest = os.path.join(self.model_dir, 'settings.py')
        shutil.copyfile(source, dest)

Response: Copy the settings module to the model_dir directory


Instruction:
How would you code a function in Python 3 to
load settings module from the model_dir directory.

Response:
def load_settings(self):
        """Load settings module from the model_dir directory."""
        mname = 'loaded_module'
        if six.PY2:
            import imp
            return imp.load_source(mname, self.settings_filename)
        else:
            import importlib.machinery
            loader = importlib.machinery.SourceFileLoader(mname, self.settings_filename)
        return loader.load_module(mname)


Instruction:
Can you tell what is the following Python 3 function doing
def train(self, jsondocs, model_dir):
        modelUtil = ModelStorageUtil(model_dir)
        modelUtil.makedir()
        modelUtil.copy_settings(self.settings)

        # Convert json documents to ner documents
        nerdocs = [json_document_to_estner_document(jsondoc)
                   for jsondoc in jsondocs]

        self.fex.prepare(nerdocs)
        self.fex.process(nerdocs)

        self.trainer.train(nerdocs, modelUtil.model_filename)

Response: Train a NER model using a list of JSON - style documents.


Instruction:
How would you explain what the following Python 3 function does
def cleanup_lines( lines, **kwargs ):
    ''' Cleans up annotation after syntactic pre-processing and processing:
        -- Removes embedded clause boundaries "<{>" and "<}>";
        -- Removes CLBC markings from analysis;
        -- Removes additional information between < and > from analysis;
        -- Removes additional information between " and " from analysis;
        -- If remove_caps==True , removes 'cap' annotations from analysis;
        -- If remove_clo==True , removes CLO CLC CLB markings from analysis;
        -- If double_quotes=='esc'   then "   will be overwritten with \\";
           and
           if double_quotes=='unesc' then \\" will be overwritten with ";
        -- If fix_sent_tags=True, then sentence tags (<s> and </s>) will be
           checked for mistakenly added analysis, and found analysis will be
           removed;

        Returns the input list, which has been cleaned from additional information;
    '''
    if not isinstance( lines, list ):
        raise Exception('(!) Unexpected type of input argument! Expected a list of strings.')
    remove_caps   = False
    remove_clo    = False
    double_quotes = None
    fix_sent_tags = False
    for argName, argVal in kwargs.items() :
        if argName in ['remove_caps', 'remove_cap']:
           remove_caps = bool(argVal)
        if argName == 'remove_clo':
           remove_clo = bool(argVal)
        if argName == 'fix_sent_tags':
           fix_sent_tags = bool(argVal)
        if argName in ['double_quotes', 'quotes'] and argVal and \
           argVal.lower() in ['esc', 'escape', 'unesc', 'unescape']:
           double_quotes = argVal.lower()
    pat_token_line     = re.compile('^"<(.+)>"\s*$')
    pat_analysis_start = re.compile('^(\s+)"(.+)"(\s[LZT].*)$')
    i = 0
    to_delete = []
    while ( i < len(lines) ):
        line = lines[i]
        isAnalysisLine = line.startswith('  ') or line.startswith('\t')
        if not isAnalysisLine:
           removeCurrentTokenAndAnalysis = False
           #  1) Remove embedded clause boundaries "<{>" and "<}>"
           if line.startswith('"<{>"'):
              if i+1 == len(lines) or (i+1 < len(lines) and not '"{"' in lines[i+1]):
                 removeCurrentTokenAndAnalysis = True
           if line.startswith('"<}>"'):
              if i+1 == len(lines) or (i+1 < len(lines) and not '"}"' in lines[i+1]):
                 removeCurrentTokenAndAnalysis = True
           if removeCurrentTokenAndAnalysis:
              # Remove the current token and all the subsequent analyses
              del lines[i]
              j=i
              while ( j < len(lines) ):
                 line2 = lines[j]
                 if line2.startswith('  ') or line2.startswith('\t'):
                    del lines[j]
                 else:
                    break
              continue
           #  2) Convert double quotes (if required)
           if double_quotes:
              #  '^"<(.+)>"\s*$'
              if pat_token_line.match( lines[i] ):
                 token_cleaned = (pat_token_line.match(lines[i])).group(1)
                 # Escape or unescape double quotes
                 if double_quotes in ['esc', 'escape']:
                    token_cleaned = token_cleaned.replace('"', '\\"')
                    lines[i] = '"<'+token_cleaned+'>"'
                 elif double_quotes in ['unesc', 'unescape']:
                    token_cleaned = token_cleaned.replace('\\"', '"')
                    lines[i] = '"<'+token_cleaned+'>"'
        else:
           #  Normalize analysis line
           lines[i] = re.sub('^\s{4,}', '\t', lines[i])
           #  Remove clause boundary markings
           lines[i] = re.sub('(.*)" ([LZT].*) CLBC (.*)', '\\1" \\2 \\3', lines[i])
           #  Remove additional information that was added during the analysis
           lines[i] = re.sub('(.*)" L([^"<]*) ["<]([^@]*) (@.*)', '\\1" L\\2 \\4', lines[i])
           #  Remove 'cap' tags
           if remove_caps:
              lines[i] = lines[i].replace(' cap ', ' ')
           #  Convert double quotes (if required)
           if double_quotes and double_quotes in ['unesc', 'unescape']:
              lines[i] = lines[i].replace('\\"', '"')
           elif double_quotes and double_quotes in ['esc', 'escape']:
              m = pat_analysis_start.match( lines[i] )
              if m:
                 # '^(\s+)"(.+)"(\s[LZT].*)$'
                 start   = m.group(1)
                 content = m.group(2)
                 end     = m.group(3)
                 content = content.replace('"', '\\"')
                 lines[i] = ''.join([start, '"', content, '"', end])
           #  Remove CLO CLC CLB markings
           if remove_clo and 'CL' in lines[i]:
              lines[i] = re.sub('\sCL[OCB]', ' ', lines[i])
              lines[i] = re.sub('\s{2,}', ' ', lines[i])
           #  Fix sentence tags that mistakenly could have analysis (in EDT corpus)
           if fix_sent_tags:
              if i-1 > -1 and ('"</s>"' in lines[i-1] or '"<s>"' in lines[i-1]):
                 lines[i] = ''
        i += 1
    return lines

Response: Removes additional information between CLBC and CLO CLC markings from analysis and returns the input list.


Instruction:
Create a Python 3 function for
aligning a list of lines with a given EstNLTK s Text object.

Response:
def align_cg3_with_Text( lines, text, **kwargs ):
    ''' Aligns VISLCG3's output (a list of strings) with given EstNLTK\'s Text object.
        Basically, for each word position in the Text object, finds corresponding VISLCG3's
        analyses;

        Returns a list of dicts, where each dict has following attributes:
          'start'   -- start index of the word in Text;
          'end'     -- end index of the word in Text;
          'sent_id' -- index of the sentence in Text, starting from 0;
          'parser_out' -- list of analyses from the output of the syntactic parser;

        Parameters
        -----------
        lines : list of str
            The input text for the pipeline; Should be in same format as the output
            of VISLCG3Pipeline;

        text : Text
            EstNLTK Text object containing the original text that was analysed via
            VISLCG3Pipeline;

        check_tokens : bool
            Optional argument specifying whether tokens should be checked for match
            during the alignment. In case of a mismatch, an exception is raised.
            Default:False

        add_word_ids : bool
            Optional argument specifying whether each alignment should include attributes:
            * 'text_word_id' - current word index in the whole Text, starting from 0;
            * 'sent_word_id' - index of the current word in the sentence, starting from 0;
            Default:False

        Example output (for text 'Jah . Öö oli täiesti tuuletu .'):
        -----------------------------------------------------------
        {'sent_id': 0, 'start': 0, 'end': 3, 'parser_out': ['\t"jah" L0 D @ADVL #1->0\r']}
        {'sent_id': 0, 'start': 4, 'end': 5, 'parser_out': ['\t"." Z Fst CLB #2->2\r']}
        {'sent_id': 1, 'start': 6, 'end': 8, 'parser_out': ['\t"öö" L0 S com sg nom @SUBJ #1->2\r']}
        {'sent_id': 1, 'start': 9, 'end': 12, 'parser_out': ['\t"ole" Li V main indic impf ps3 sg ps af @FMV #2->0\r']}
        {'sent_id': 1, 'start': 13, 'end': 20, 'parser_out': ['\t"täiesti" L0 D @ADVL #3->4\r']}
        {'sent_id': 1, 'start': 21, 'end': 28, 'parser_out': ['\t"tuuletu" L0 A pos sg nom @PRD #4->2\r']}
        {'sent_id': 1, 'start': 29, 'end': 30, 'parser_out': ['\t"." Z Fst CLB #5->5\r']}

    '''
    from estnltk.text import Text
    if not isinstance( text, Text ):
        raise Exception('(!) Unexpected type of input argument! Expected EstNLTK\'s Text. ')
    if not isinstance( lines, list ):
        raise Exception('(!) Unexpected type of input argument! Expected a list of strings.')
    check_tokens = False
    add_word_ids = False
    for argName, argVal in kwargs.items() :
        if argName in ['check_tokens', 'check'] and argVal in [True, False]:
           check_tokens = argVal
        if argName in ['add_word_ids', 'word_ids'] and argVal in [True, False]:
           add_word_ids = argVal
    pat_empty_line     = re.compile('^\s+$')
    pat_token_line     = re.compile('^"<(.+)>"$')
    pat_analysis_start = re.compile('^(\s+)"(.+)"(\s[LZTS].*)$')
    pat_sent_bound     = re.compile('^("<s>"|"</s>"|<s>|</s>)\s*$')
    generalWID  = 0
    sentWID     = 0
    sentenceID  = 0
    j = 0
    # Iterate over the sentences and perform the alignment
    results = []
    for sentence in text.divide( layer=WORDS, by=SENTENCES ):
        sentWID = 0
        for i in range(len(sentence)):
            # 1) take the next word in Text
            wordJson = sentence[i]
            wordStr  = wordJson[TEXT]
            cg3word     = None
            cg3analyses = []
            # 2) find next word in the VISLCG3's output
            while (j < len(lines)):
                # a) a sentence boundary: skip it entirely
                if pat_sent_bound.match( lines[j] ) and j+1 < len(lines) and \
                   (len(lines[j+1])==0 or pat_empty_line.match(lines[j+1])):
                    j += 2
                    continue
                # b) a word token: collect the analyses
                token_match = pat_token_line.match( lines[j].rstrip() )
                if token_match:
                    cg3word = token_match.group(1)
                    j += 1
                    while (j < len(lines)):
                        if pat_analysis_start.match(lines[j]):
                            cg3analyses.append(lines[j])
                        else:
                            break
                        j += 1
                    break
                j += 1
            # 3) Check whether two tokens match (if requested)
            if cg3word:
                if check_tokens and wordStr != cg3word:
                    raise Exception('(!) Unable to align EstNLTK\'s token nr ',generalWID,\
                                    ':',wordStr,' vs ',cg3word)
                # Populate the alignment
                result_dict = { START:wordJson[START], END:wordJson[END], \
                                SENT_ID:sentenceID, PARSER_OUT: cg3analyses }
                if add_word_ids:
                    result_dict['text_word_id'] = generalWID # word id in the text
                    result_dict['sent_word_id'] = sentWID    # word id in the sentence
                results.append( result_dict )
            else:
                if j >= len(lines):
                    print('(!) End of VISLCG3 analysis reached: '+str(j)+' '+str(len(lines)),\
                          file = sys.stderr)
                raise Exception ('(!) Unable to find matching syntactic analysis ',\
                                 'for EstNLTK\'s token nr ', generalWID, ':', wordStr)
            sentWID    += 1
            generalWID += 1
        sentenceID += 1
    return results


Instruction:
Can you write a function in Python 3 where it
converts the output of a VISL_CG3 based syntactic parsing into CONLL format.

Response:
def convert_cg3_to_conll( lines, **kwargs ):
    ''' Converts the output of VISL_CG3 based syntactic parsing into CONLL format.
        Expects that the output has been cleaned ( via method cleanup_lines() ).
        Returns a list of CONLL format lines;

        Parameters
        -----------
        lines : list of str
            The input text for the pipeline; Should be in same format as the output
            of VISLCG3Pipeline;

        fix_selfrefs : bool
            Optional argument specifying  whether  self-references  in  syntactic
            dependencies should be fixed;
            Default:True

        fix_open_punct : bool
            Optional argument specifying  whether  opening punctuation marks should
            be made dependents of the following token;
            Default:True

        unesc_quotes : bool
            Optional argument specifying  whether double quotes should be unescaped
            in the output, i.e.  converted from '\"' to '"';
            Default:True

        rep_spaces : bool
            Optional argument specifying  whether spaces in a multiword token (e.g.
            'Rio de Janeiro') should be replaced with underscores ('Rio_de_Janeiro');
            Default:False

        error_on_unexp : bool
            Optional argument specifying  whether an exception should be raised in
            case of missing or unexpected analysis line; if not, only prints warnings
            in case of such lines;
            Default:False

        Example input
        --------------
        "<s>"

        "<Öö>"
                "öö" L0 S com sg nom @SUBJ #1->2
        "<oli>"
                "ole" Li V main indic impf ps3 sg ps af @FMV #2->0
        "<täiesti>"
                "täiesti" L0 D @ADVL #3->4
        "<tuuletu>"
                "tuuletu" L0 A pos sg nom @PRD #4->2
        "<.>"
                "." Z Fst CLB #5->5
        "</s>"

        Example output
        ---------------
        1       Öö      öö      S       S       com|sg|nom      2       @SUBJ   _       _
        2       oli     ole     V       V       main|indic|impf|ps3|sg|ps|af    0       @FMV    _       _
        3       täiesti täiesti D       D       _       4       @ADVL   _       _
        4       tuuletu tuuletu A       A       pos|sg|nom      2       @PRD    _       _
        5       .       .       Z       Z       Fst|CLB 4       xxx     _       _

    '''
    if not isinstance( lines, list ):
        raise Exception('(!) Unexpected type of input argument! Expected a list of strings.')
    fix_selfrefs   = True
    fix_open_punct = True
    unesc_quotes   = True
    rep_spaces     = False
    error_on_unexp = False
    for argName, argVal in kwargs.items() :
        if argName in ['selfrefs', 'fix_selfrefs'] and argVal in [True, False]:
           fix_selfrefs = argVal
        if argName in ['fix_open_punct'] and argVal in [True, False]:
           fix_open_punct = argVal
        if argName in ['error_on_unexp'] and argVal in [True, False]:
           error_on_unexp = argVal
        if argName in ['unesc_quotes'] and argVal in [True, False]:
           unesc_quotes = argVal
        if argName in ['rep_spaces'] and argVal in [True, False]:
           rep_spaces = argVal
    pat_empty_line    = re.compile('^\s+$')
    pat_token_line    = re.compile('^"<(.+)>"$')
    pat_analysis_line = re.compile('^\s+"(.+)"\s([^"]+)$')
    # 3 types of analyses:
    pat_ending_pos_form = re.compile('^L\S+\s+\S\s+([^#@]+).+$')
    pat_pos_form        = re.compile('^\S\s+([^#@]+).+$')
    pat_ending_pos      = re.compile('^(L\S+\s+)?\S\s+[#@].+$')
    pat_opening_punct   = re.compile('.+\s(Opr|Oqu|Quo)\s')
    analyses_added = 0
    conll_lines = []
    word_id = 1
    i = 0
    while ( i < len(lines) ):
        line = lines[i]
        # Check, whether it is an analysis line or not
        if not (line.startswith('  ') or line.startswith('\t')):
            # ******  TOKEN
            if len(line)>0 and not (line.startswith('"<s>"') or \
               line.startswith('"</s>"')) and not pat_empty_line.match(line):
               # Convert double quotes back to normal form (if requested)
               if unesc_quotes:
                  line = line.replace( '\\"', '"' )
               # Broken stuff: if previous word was without analysis
               if analyses_added == 0 and word_id > 1:
                  # Missing analysis line
                  if error_on_unexp:
                      raise Exception('(!) Analysis missing at line '+str(i)+': '+\
                                      '\n'+lines[i-1])
                  else:
                      print('(!) Analysis missing at line '+str(i)+': '+\
                            '\n'+lines[i-1], file=sys.stderr)
                  # Add an empty analysis
                  conll_lines[-1] += '\t_'
                  conll_lines[-1] += '\tX'
                  conll_lines[-1] += '\tX'
                  conll_lines[-1] += '\t_'
                  conll_lines[-1] += '\t'+str(word_id-2)
                  conll_lines[-1] += '\txxx'
                  conll_lines[-1] += '\t_'
                  conll_lines[-1] += '\t_'
               # Start of a new token/word
               token_match = pat_token_line.match( line.rstrip() )
               if token_match:
                  word = token_match.group(1)
               else:
                  raise Exception('(!) Unexpected token format: ', line)
               if rep_spaces and re.search('\s', word):
                  # Replace spaces in the token with '_' symbols
                  word = re.sub('\s+', '_', word)
               conll_lines.append( str(word_id) + '\t' + word )
               analyses_added = 0
               word_id += 1
            # End of a sentence
            if line.startswith('"</s>"'):
                conll_lines.append('')
                word_id = 1
        else:
            # ******  ANALYSIS
            # If there is more than one pair of "", we have some kind of
            # inconsistency: try to remove extra quotation marks from the
            # end of the analysis line ...
            if line.count('"') > 2:
                new_line = []
                q_count = 0
                for j in range( len(line) ):
                    if line[j]=='"' and (j==0 or line[j-1]!='\\'):
                        q_count += 1
                        if q_count < 3:
                            new_line.append(line[j])
                    else:
                        new_line.append(line[j])
                line = ''.join( new_line )
            # Convert double quotes back to normal form (if requested)
            if unesc_quotes:
                line = line.replace( '\\"', '"' )
            analysis_match = pat_analysis_line.match( line )
            # Analysis line; in case of multiple analyses, pick the first one;
            if analysis_match and analyses_added==0:
                lemma = analysis_match.group(1)
                cats  = analysis_match.group(2)
                if cats.startswith('Z '):
                    postag = 'Z'
                else:
                    postag = (cats.split())[1] if len(cats.split())>1 else 'X'
                deprels = re.findall( '(@\S+)', cats )
                deprel  = deprels[0] if deprels else 'xxx'
                heads   = re.findall( '#\d+\s*->\s*(\d+)', cats )
                head    = heads[0] if heads else str(word_id-2)
                m1 = pat_ending_pos_form.match(cats)
                m2 = pat_pos_form.match(cats)
                m3 = pat_ending_pos.match(cats)
                if m1:
                    forms = (m1.group(1)).split()
                elif m2:
                    forms = (m2.group(1)).split()
                elif m3:
                    forms = ['_']  # no form (in case of adpositions and adverbs)
                else:
                    # Unexpected format of analysis line
                    if error_on_unexp:
                        raise Exception('(!) Unexpected format of analysis line: '+line)
                    else:
                        postag = 'X'
                        forms = ['_']
                        print('(!) Unexpected format of analysis line: '+line, file=sys.stderr)
                # If required, fix self-references (in punctuation):
                if fix_selfrefs and int(head) == word_id-1 and word_id-2>0:
                    head = str(word_id-2) # add link to the previous word
                # Fix opening punctuation
                if fix_open_punct and pat_opening_punct.match(line):
                    head = str(word_id)   # add link to the following word
                conll_lines[-1] += '\t'+lemma
                conll_lines[-1] += '\t'+postag
                conll_lines[-1] += '\t'+postag
                conll_lines[-1] += '\t'+('|'.join(forms))
                conll_lines[-1] += '\t'+head
                conll_lines[-1] += '\t'+deprel
                conll_lines[-1] += '\t_'
                conll_lines[-1] += '\t_'
                analyses_added += 1
        i += 1
    return conll_lines


Instruction:
Implement a function in Python 3 to
check whether given vislcg_cmd1 is in system s PATH. Returns True if there is a file named vislcg_cmd1 in the system s PATH otherwise returns False.

Response:
def check_if_vislcg_is_in_path( self, vislcg_cmd1 ):
        ''' Checks whether given vislcg_cmd1 is in system's PATH. Returns True, there is
            a file named  vislcg_cmd1  in the path, otherwise returns False;

            The idea borrows from:  http://stackoverflow.com/a/377028
        '''
        for path in os.environ["PATH"].split( os.pathsep ):
            path1 = path.strip('"')
            file1 = os.path.join(path1, vislcg_cmd1)
            if os.path.isfile(file1) or os.path.isfile(file1+'.exe'):
               return True
        return False


Instruction:
How would you implement a function in Python 3 that
executes the pipeline of subsequent VISL_CG3 commands.

Response:
def process_lines( self, input_lines, **kwargs ):
        ''' Executes the pipeline of subsequent VISL_CG3 commands. The first process
            in pipeline gets input_lines as an input, and each subsequent process gets
            the output of the previous process as an input.

            The idea of how to construct the pipeline borrows from:
              https://github.com/estnltk/estnltk/blob/1.4.0/estnltk/syntax/tagger.py

            Returns the result of the last process in the pipeline, either as a string
            or, alternatively, as a list of strings (if split_result == True);

            Parameters
            -----------
            input_lines : list of str
                 The input text for the pipeline; Should be in same format as the output
                 of SyntaxPreprocessing;

            split_result : bool
                 Optional argument specifying whether the result should be split by
                 newlines, and returned as a list of strings/lines instead;
                 Default:False

            remove_info : bool
                 Optional argument specifying whether the additional information added
                 during the preprocessing and syntactic processing should be removed
                 from the results;
                 Default:True;
                 The method  cleanup_lines()  will be used for removing additional info,
                 and all the parameters passed to this method will be also forwarded to
                 the cleanup method;

        '''
        split_result_lines = False
        remove_info = True
        for argName, argVal in kwargs.items() :
            if argName in ['split_result_lines', 'split_result'] and argVal in [True, False]:
               split_result_lines = argVal
            if argName in ['remove_info', 'info_remover', 'clean_up'] and argVal in [True, False]:
               remove_info = argVal

        # 1) Construct the input file for the first process in the pipeline
        temp_input_file = \
            tempfile.NamedTemporaryFile(prefix='vislcg3_in.', mode='w', delete=False)
        temp_input_file.close()
        # We have to open separately here for writing, because Py 2.7 does not support
        # passing parameter   encoding='utf-8'    to the NamedTemporaryFile;
        out_f = codecs.open(temp_input_file.name, mode='w', encoding='utf-8')
        for line in input_lines:
            out_f.write( line.rstrip() )
            out_f.write( '\n' )
        out_f.close()
        # TODO: tempfile is currently used to ensure that the input is in 'utf-8',
        #       but perhaps we can somehow ensure it without using tempfile ??

        # 2) Dynamically construct the pipeline and open processes
        pipeline = []
        for i in range( len(self.rules_pipeline) ):
            rule_file = self.rules_pipeline[i]
            process_cmd = [self.vislcg_cmd, '-o', '-g', os.path.join(self.rules_dir, rule_file)]
            process = None
            if i == 0:
               # The first process takes input from the file
               process_cmd.extend( ['-I', temp_input_file.name] )
               process = Popen(process_cmd, stdin=PIPE, stdout=PIPE)
            else:
               # A subsequent process takes output of the last process as an input
               process = Popen(process_cmd, stdin=pipeline[-1]['process'].stdout, stdout=PIPE)
            # Record the process
            process_dict = {'process':process, 'cmd':process_cmd}
            pipeline.append( process_dict )

        # 3) Close all stdout streams, except the last one
        for i in range( len(pipeline) ):
           if i != len(pipeline) - 1:
              pipeline[i]['process'].stdout.close()

        # 4) Communicate results form the last item in the pipeline
        result = as_unicode( pipeline[-1]['process'].communicate()[0] )
        pipeline[-1]['process'].stdout.close() # Close the last process

        # Clean-up
        # 1) remove temp file
        os.remove(temp_input_file.name)

        # 2) remove additional info, if required
        if remove_info:
              result = '\n'.join( cleanup_lines( result.split('\n'), **kwargs ))

        return result if not split_result_lines else result.split('\n')


Instruction:
Implement a function in Python 3 to
create a copy from given analysis dict.

Response:
def copy_analysis_dict( analysis ):
    ''' Creates a copy from given analysis dict. '''
    assert isinstance(analysis, dict), "(!) Input 'analysis' should be a dict!"
    new_dict = { POSTAG: analysis[POSTAG],\
                 ROOT: analysis[ROOT],\
                 FORM: analysis[FORM],\
                 CLITIC: analysis[CLITIC],\
                 ENDING: analysis[ENDING] }
    if LEMMA in analysis:
       new_dict[LEMMA] = analysis[LEMMA]
    if ROOT_TOKENS in analysis:
       new_dict[ROOT_TOKENS] = analysis[ROOT_TOKENS]
    return new_dict


Instruction:
Can you create a Python 3 function that
returns a list of clause indices for the whole text.

Response:
def get_unique_clause_indices( text ):
    ''' Returns a list of clause indices for the whole text. For each token in text,
        the list contains index of the clause the word belongs to, and the indices
        are unique over the whole text. '''
    # Add clause boundary annotation (if missing)
    if not text.is_tagged( CLAUSES ):
        text.tag_clauses()
    # Collect (unique) clause indices over the whole text
    clause_indices = []
    sent_id = 0
    for sub_text in text.split_by( SENTENCES ):
        for word, cl_index in zip( sub_text.words, sub_text.clause_indices ):
            clause_indices.append( sent_id+cl_index )
        nr_of_clauses = len(set(sub_text.clause_indices))
        sent_id += nr_of_clauses
    assert len(clause_indices) == len(text.words), '(!) Number of clause indices should match nr of words!'
    return clause_indices


Instruction:
Implement a function in Python 3 to
return a list of unique sentence indices for the whole text.

Response:
def get_unique_sentence_indices( text ):
    ''' Returns a list of sentence indices for the whole text. For each token in text,
        the list contains index of the sentence the word belongs to, and the indices
        are unique over the whole text. '''
    # Add sentence annotation (if missing)
    if not text.is_tagged( SENTENCES ):
        text.tokenize_sentences()
    # Collect (unique) sent indices over the whole text
    sent_indices = []
    sent_id = 0
    for sub_text in text.split_by( SENTENCES ):
        for word in sub_text.words:
            sent_indices.append( sent_id )
        sent_id += 1
    assert len(sent_indices) == len(text.words), '(!) Number of sent indices should match nr of words!'
    return sent_indices


Instruction:
Implement a Python 3 function for
converting nominal categories of the input analysis.

Response:
def _convert_nominal_form( analysis ):
    ''' Converts nominal categories of the input analysis.
        Performs one-to-one conversions only. '''
    assert FORM in analysis, '(!) The input analysis does not contain "'+FORM+'" key.'
    for idx, pattern_items in enumerate(_noun_conversion_rules):
        pattern_str, replacement = pattern_items
        if pattern_str in analysis[FORM]:
           analysis[FORM] = analysis[FORM].replace( pattern_str, replacement )
    return analysis


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _convert_amb_verbal_form( analysis ):
    ''' Converts ambiguous verbal categories of the input analysis.
        Performs one-to-many conversions. '''
    assert FORM in analysis, '(!) The input analysis does not contain "'+FORM+'" key.'
    results = []
    for root_pat, pos, form_pat, replacements in _amb_verb_conversion_rules:
        if analysis[POSTAG] == pos and re.match(root_pat, analysis[ROOT]) and \
           re.match(form_pat, analysis[FORM]):
           for replacement in replacements:
               new_analysis = copy_analysis_dict( analysis )
               new_form = re.sub(form_pat, replacement, analysis[FORM])
               new_analysis[FORM] = new_form
               results.append( new_analysis )
           # break after the replacement has been made
           # ( to avoid over-generation )
           break
    if not results:
       results.append( analysis )
    return results

Response: Converts ambiguous verbal form to one - to - many.


Instruction:
Can you write a function in Python 3 where it
converts ordinary verbal categories of the input analysis.

Response:
def _convert_verbal_form( analysis ):
    ''' Converts ordinary verbal categories of the input analysis.
        Performs one-to-one conversions. '''
    assert FORM in analysis, '(!) The input analysis does not contain "'+FORM+'" key.'
    for form, replacement in _verb_conversion_rules:
        # Exact match
        if analysis[FORM] == form:
           assert analysis[POSTAG] == 'V', \
               '(!) Expected analysis of verb, but got analysis of "'+str(analysis[POSTAG])+'" instead.'
           analysis[FORM] = replacement
        # Inclusion :   the case of some_prefix+' '+form ;
        elif analysis[FORM].endswith(' '+form):
           parts  = analysis[FORM].split()
           prefix = ' '.join( parts[:len(parts)-1] )
           analysis[FORM] = prefix+' '+replacement
    return analysis


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _make_postfixes_1( analysis ):
    ''' Provides some post-fixes. '''
    assert FORM in analysis, '(!) The input analysis does not contain "'+FORM+'" key.'
    if 'neg' in analysis[FORM]:
        analysis[FORM] = re.sub( '^\s*neg ([^,]*)$',  '\\1 Neg',  analysis[FORM] )
    analysis[FORM] = re.sub( ' Neg Neg$',  ' Neg',  analysis[FORM] )
    analysis[FORM] = re.sub( ' Aff Neg$',  ' Neg',  analysis[FORM] )
    analysis[FORM] = re.sub( 'neg',  'Neg',  analysis[FORM] )
    analysis[FORM] = analysis[FORM].rstrip().lstrip()
    assert 'neg' not in analysis[FORM], \
                 '(!) The label "neg" should be removed by now.'
    assert 'Neg' not in analysis[FORM] or ('Neg' in analysis[FORM] and analysis[FORM].endswith('Neg')), \
                 '(!) The label "Neg" should end the analysis line: '+str(analysis[FORM])
    return analysis

Response: Provides some post - fixes.


Instruction:
Given the following Python 3 function, write the documentation
def _keep_analyses( analyses, keep_forms, target_forms ):
    ''' Filters the given list of *analyses* by morphological forms:
        deletes analyses that are listed in *target_forms*, but not in
        *keep_forms*. '''
    to_delete = []
    for aid, analysis in enumerate(analyses):
        delete = False
        for target in target_forms:
            if (target == analysis[FORM] and not analysis[FORM] in keep_forms):
               delete = True
        if delete:
            to_delete.append( aid )
    if to_delete:
       to_delete.reverse()
       for aid in to_delete:
           del analyses[aid]

Response: Filters the given list of analyses by morphological forms and deletes analyses that are not listed in target_forms but not in keep_forms.


Instruction:
Write a Python 3 script to
disambiguate forms ambiguous between multiword negation and some other form ;

Response:
def _disambiguate_neg( words_layer ):
    ''' Disambiguates forms ambiguous between multiword negation and some
        other form;
    '''
    prev_word_lemma = ''
    for word_dict in words_layer:
        forms = [ a[FORM] for a in word_dict[ANALYSIS] ]
        if ('Pers Prs Imprt Sg2' in forms and 'Pers Prs Ind Neg' in forms):
           if (prev_word_lemma == "ei" or prev_word_lemma == "ega"):
               # ei saa, ei tee
               _keep_analyses( word_dict[ANALYSIS], ['Pers Prs Ind Neg'], ['Pers Prs Imprt Sg2', 'Pers Prs Ind Neg'] )
           else:
               # saa! tee!
               _keep_analyses( word_dict[ANALYSIS], ['Pers Prs Imprt Sg2'], ['Pers Prs Imprt Sg2', 'Pers Prs Ind Neg'] )
        if ('Pers Prt Imprt' in forms and 'Pers Prt Ind Neg' in forms and 'Pers Prt Prc' in forms):
           if (prev_word_lemma == "ei" or prev_word_lemma == "ega"):
               # ei saanud, ei teinud
               _keep_analyses( word_dict[ANALYSIS], ['Pers Prt Ind Neg'], ['Pers Prt Imprt','Pers Prt Ind Neg','Pers Prt Prc'] )
           else:
               # on, oli saanud teinud; kukkunud õun; ...
               _keep_analyses( word_dict[ANALYSIS], ['Pers Prt Prc'], ['Pers Prt Imprt','Pers Prt Ind Neg','Pers Prt Prc'] )
        if ('Impers Prt Ind Neg' in forms and 'Impers Prt Prc' in forms):
           if (prev_word_lemma == "ei" or prev_word_lemma == "ega"):
               # ei saadud, ei tehtud
               _keep_analyses( word_dict[ANALYSIS], ['Impers Prt Ind Neg'], ['Impers Prt Ind Neg','Impers Prt Prc'] )
           else:
               # on, oli saadud tehtud; saadud õun; ...
               _keep_analyses( word_dict[ANALYSIS], ['Impers Prt Prc'], ['Impers Prt Ind Neg','Impers Prt Prc'] )
        prev_word_lemma = word_dict[ANALYSIS][0][ROOT]


Instruction:
Here you have a function in Python 3, explain what it does
def _disambiguate_sid_ksid( words_layer, text, scope=CLAUSES ):
    ''' Disambiguates verb forms based on existence of 2nd person pronoun ('sina') in given scope.
        The scope could be either CLAUSES or SENTENCES.
    '''
    assert scope in [CLAUSES, SENTENCES], '(!) The scope should be either "clauses" or "sentences".'
    group_indices = get_unique_clause_indices( text ) if scope==CLAUSES else get_unique_sentence_indices( text )
    i = 0
    gr_2nd_person_pron = {}
    while i < len( words_layer ):
        gr_index = group_indices[i]
        if gr_index not in gr_2nd_person_pron:
            # 1) Find out whether the current group (clause or sentence) contains "sina"
            j = i
            gr_2nd_person_pron_found = False
            while j < len( words_layer ):
                 if group_indices[j] == gr_index:
                     forms  = [ a[FORM] for a in words_layer[j][ANALYSIS] ]
                     lemmas = [ a[ROOT] for a in words_layer[j][ANALYSIS] ]
                     if 'sina' in lemmas and 'Sg Nom' in forms:
                        gr_2nd_person_pron_found = True
                        break
                 if group_indices[j] >= gr_index+10:  # do not venture too far ...
                     break
                 j += 1
            gr_2nd_person_pron[gr_index] = gr_2nd_person_pron_found
        forms = [ a[FORM] for a in words_layer[i][ANALYSIS] ]
        # 2) Disambiguate verb forms based on existence of 'sina' in the clause
        if ('Pers Prt Ind Pl3 Aff' in forms and 'Pers Prt Ind Sg2 Aff' in forms):    # -sid
           if not gr_2nd_person_pron[ gr_index ]:
               _keep_analyses( words_layer[i][ANALYSIS], ['Pers Prt Ind Pl3 Aff'], ['Pers Prt Ind Pl3 Aff', 'Pers Prt Ind Sg2 Aff'] )
           else:
               _keep_analyses( words_layer[i][ANALYSIS], ['Pers Prt Ind Sg2 Aff'], ['Pers Prt Ind Pl3 Aff', 'Pers Prt Ind Sg2 Aff'] )
        if ('Pers Prs Cond Pl3 Aff' in forms and 'Pers Prs Cond Sg2 Aff' in forms):  # -ksid
           if not gr_2nd_person_pron[ gr_index ]:
               _keep_analyses( words_layer[i][ANALYSIS], ['Pers Prs Cond Pl3 Aff'], ['Pers Prs Cond Pl3 Aff', 'Pers Prs Cond Sg2 Aff'] )
           else:
               _keep_analyses( words_layer[i][ANALYSIS], ['Pers Prs Cond Sg2 Aff'], ['Pers Prs Cond Pl3 Aff', 'Pers Prs Cond Sg2 Aff'] )
        if ('Pers Prt Cond Pl3 Aff' in forms and 'Pers Prt Cond Sg2 Aff' in forms):  # -nuksid
           if not gr_2nd_person_pron[ gr_index ]:
               _keep_analyses( words_layer[i][ANALYSIS], ['Pers Prt Cond Pl3 Aff'], ['Pers Prt Cond Pl3 Aff', 'Pers Prt Cond Sg2 Aff'] )
           else:
               _keep_analyses( words_layer[i][ANALYSIS], ['Pers Prt Cond Sg2 Aff'], ['Pers Prt Cond Pl3 Aff', 'Pers Prt Cond Sg2 Aff'] )
        i += 1

Response: Disambiguates verb forms based on existence of 2nd person pronoun.


Instruction:
Write a Python 3 script to
provide some post - fixes after the disambiguation.

Response:
def _make_postfixes_2( words_layer ):
    ''' Provides some post-fixes after the disambiguation. '''
    for word_dict in words_layer:
        for analysis in word_dict[ANALYSIS]:
          analysis[FORM] = re.sub( '(Sg|Pl)([123])', '\\1 \\2', analysis[FORM] )
    return words_layer


Instruction:
Make a summary of the following Python 3 code
def convert_analysis( analyses ):
    ''' Converts a list of analyses (list of dict objects) from FS's vabamorf format to
        giellatekno (GT) format.
        Due to one-to-many conversion rules, the number of analyses returned by this method
        can be greater than the number of analyses in the input list.
    '''
    resulting_analyses = []
    for analysis in analyses:
        # Make a copy of the analysis
        new_analyses = [ copy_analysis_dict( analysis ) ]
        # Convert noun categories
        new_analyses[0] = _convert_nominal_form( new_analyses[0] )
        # Convert ambiguous verb categories
        new_analyses = _convert_amb_verbal_form( new_analyses[0] )
        # Convert remaining verbal categories
        new_analyses = [_convert_verbal_form( a ) for a in new_analyses]
        # Make postfixes
        new_analyses = [_make_postfixes_1( a ) for a in new_analyses]
        resulting_analyses.extend( new_analyses )
    return resulting_analyses

Response:
Converts a list of analyses from FS s vabamorf format to
        giellatekno ( GT format.


Instruction:
Make a summary of the following Python 3 code
def convert_to_gt( text, layer_name=GT_WORDS ):
    ''' Converts all words in a morphologically analysed Text from FS format to
        giellatekno (GT) format, and stores in a new layer named GT_WORDS.
        If the keyword argument *layer_name=='words'* , overwrites the old 'words'
        layer with the new layer containing GT format annotations.

        Parameters
        -----------
        text : estnltk.text.Text
            Morphologically annotated text that needs to be converted from FS format
            to GT format;

        layer_name : str
            Name of the Text's layer in which GT format morphological annotations
            are stored;
            Defaults to GT_WORDS;
    '''
    assert WORDS in text, \
        '(!) The input text should contain "'+str(WORDS)+'" layer.'
    assert len(text[WORDS])==0 or (len(text[WORDS])>0 and ANALYSIS in text[WORDS][0]), \
        '(!) Words in the input text should contain "'+str(ANALYSIS)+'" layer.'
    new_words_layer = []
    # 1) Perform the conversion
    for word in text[WORDS]:
        new_analysis = []
        new_analysis.extend( convert_analysis( word[ANALYSIS] ) )
        new_words_layer.append( {TEXT:word[TEXT], ANALYSIS:new_analysis, START:word[START], END:word[END]} )
    # 2) Perform some context-specific disambiguation
    _disambiguate_neg( new_words_layer )
    _disambiguate_sid_ksid( new_words_layer, text, scope=CLAUSES )
    _disambiguate_sid_ksid( new_words_layer, text, scope=SENTENCES )
    _make_postfixes_2( new_words_layer )
    # 3) Attach the layer
    if layer_name != WORDS:
        #  Simply attach the new layer
        text[layer_name] = new_words_layer
    else:
        #  Perform word-by-word replacements
        # (because simple attaching won't work here)
        for wid, new_word in enumerate( new_words_layer ):
            text[WORDS][wid] = new_word
    return text

Response: Converts all words in a morphologically analysed Text from FS format to GT format.


Instruction:
Write a Python 3 script for
returning the analysis dict from the input root pos and form.

Response:
def get_analysis_dict( root, pos, form ):
    '''  Takes *root*, *pos* and *form* from Filosoft's mrf input and reformats as
         EstNLTK's analysis dict:
          {
             "clitic":       string,
             "ending":       string,
             "form":         string,
             "partofspeech": string,
             "root":         string
          },
         Returns the dict;
    '''
    import sys
    result = { CLITIC:"", ENDING:"", FORM:form, POSTAG:pos, ROOT:"" }
    breakpoint = -1
    for i in range(len(root)-1, -1, -1):
        if root[i] == '+':
            breakpoint = i
            break
    if breakpoint == -1:
        result[ROOT]   = root
        result[ENDING] = "0"
        if not re.match("^\W+$", root):
            try:
                print( " No breakpoint found from: ", root, pos, form, file=sys.stderr )
            except UnicodeEncodeError:
                print( " No breakpoint found from input *root*!",  file=sys.stderr )
    else:
        result[ROOT]   = root[0:breakpoint]
        result[ENDING] = root[breakpoint+1:]
    if result[ENDING].endswith('ki') and len(result[ENDING]) > 2:
        result[CLITIC] = 'ki'
        result[ENDING] = re.sub('ki$', '', result[ENDING])
    if result[ENDING].endswith('gi') and len(result[ENDING]) > 2:
        result[CLITIC] = 'gi'
        result[ENDING] = re.sub('gi$', '', result[ENDING])
    return result


Instruction:
How would you explain what the following Python 3 function does
def read_text_from_idx_file( file_name, layer_name=WORDS, keep_init_lines=False ):
    ''' Reads IDX format morphological annotations from given file, and returns as a Text
        object.

        The Text object will be tokenized for paragraphs, sentences, words, and it will
        contain morphological annotations in the layer *layer_name* (by default: WORDS);

        Parameters
        -----------
        file_name : str
            Name of the input file; Should contain IDX format text segmentation and
            morphological annotation;

        keep_init_lines : bool
            Optional argument specifying whether the lines from the file should also be
            preserved on a special layer named 'init_lines';
            Default: False

        layer_name : str
            Name of the Text's layer in which morphological annotations from text are
            stored;
            Defaults to WORDS;

        Example: expected format of the input:
          129	1	1	"	"	"	Z
          129	2	1	Mul	mina	mina+l	P	sg ad
          129	3	1	on	olema	ole+0	V	b
          129	3	1	on	olema	ole+0	V	vad
          129	4	1	palju	palju	palju+0	D
          129	5	1	igasugust	igasugune	iga_sugune+t	P	sg p
          129	6	1	informatsiooni	informatsioon	informatsioon+0	S	sg p
          129	7	1	.	.	.	Z

    '''
    from nltk.tokenize.simple import LineTokenizer
    from nltk.tokenize.regexp import RegexpTokenizer
    from estnltk import Text
    # 1) Collect the text along with morphological analyses from the input IDX file
    init_lines = []
    words      = []
    sentence   = []
    sentences  = []
    prev_sent_id = -1
    prev_word_id = -1
    in_f = codecs.open(file_name, mode='r', encoding='utf-8')
    for line in in_f:
        fields = line.split('\t')
        assert len(fields) == 8, '(!) Unexpected number of fields in the line: '+str(len(fields))
        sent_id   = fields[0]
        word_id   = fields[1]
        clause_id = fields[2]
        token     = fields[3]
        if prev_sent_id != sent_id:
            # Record the old sentence, start a new
            if sentence:
               sentences.append( '  '.join(sentence) )
            sentence = []
        if prev_word_id != word_id:
            # Record a new token
            sentence.append( token )
            word = { TEXT:token, ANALYSIS:[] }
            words.append(word)
        # Augment the last word in the list with new analysis
        lemma  = fields[4]
        root   = fields[5]
        pos    = fields[6]
        form   = fields[7].rstrip()
        ending = ''
        clitic = ''
        analysis = get_analysis_dict( root, pos, form )
        analysis[LEMMA] = lemma
        words[-1][ANALYSIS].append( analysis )
        prev_sent_id = sent_id
        prev_word_id = word_id
        if keep_init_lines:
            init_lines.append( [sent_id+' '+word_id, line] )
    in_f.close()
    if sentence:
        # Record the last sentence
        sentences.append( '  '.join(sentence) )

    # 2) Construct the estnltk's Text
    kwargs4text = {
      # Use custom tokenization utils in order to preserve exactly the same
      # tokenization as was in the input;
      "word_tokenizer":     RegexpTokenizer("  ", gaps=True),
      "sentence_tokenizer": LineTokenizer()
    }
    from estnltk.text import Text
    text = Text( '\n'.join(sentences), **kwargs4text )
    # Tokenize up to the words layer
    text.tokenize_words()

    # 3) Create a new layer with morphological analyses, or
    #    populate the old layer with morphological analyses;
    assert len(text[WORDS]) == len(words), \
        '(!) Number of words from input does not match with the number of words in EstNLTK Text: '+\
             str(len(text[WORDS]) )+' != '+str(len(words))
    if layer_name != WORDS:
        # If necessary, create a new layer duplicating the WORDS layer
        text[layer_name] = []
        for word in text[WORDS]:
            text[layer_name].append({START:word[START], END:word[END], TEXT:word[TEXT]})
    # Copy morphological analyses to the new layer / populate the old layer
    for wid, word in enumerate( text[WORDS] ):
        text[layer_name][wid][ANALYSIS] = words[wid][ANALYSIS]
    if layer_name == WORDS:
        assert text.is_tagged(ANALYSIS), '(!) The layer of analysis should exist by now!'

    if keep_init_lines:
        # Preserve the initial lines from file in a separate layer
        text['init_lines'] = []
        i = 0
        for wid, word in enumerate( text[layer_name] ):
            words_lines = []
            # collect lines associated with the word
            while i < len(init_lines):
                [lid, line] = init_lines[i]
                if not words_lines or words_lines[-1][0]==lid:
                    words_lines.append([lid, line])
                else:
                    break
                i += 1
            # record lines
            text['init_lines'].append( \
                {START:word[START], END:word[END], 'lines':[l[1] for l in words_lines]} )
        assert len(text['init_lines']) == len(text[layer_name]), \
            '(!) The number of initial lines should match the number of words in text!'
    return text

Response: Reads the text from an IDX file and returns as a Text object.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_original_vs_converted_diff( original ,converted ):
    '''  Compares the *original* text to *converted* text, and detects changes/differences in
        morphological annotations.

        The method constructs line-by-line comparison string, where lines are separated by
        newline, and '***' at the beginning of the line indicates the difference.

        Returns a pair: results of the line-by-line comparison as a string, and boolean value
        indicating whether there were any differences.
    '''
    from estnltk.syntax.syntax_preprocessing import convert_Text_to_mrf
    old_layer_mrf = convert_Text_to_mrf( original )
    new_layer_mrf = convert_Text_to_mrf( converted )
    max_len_1 = max([len(l) for l in old_layer_mrf ])
    max_len_2 = max([len(l) for l in new_layer_mrf ])
    max_len   = max( max_len_1, max_len_2 )
    format_str = '{:<'+str(max_len+1)+'}'
    i = 0
    j = 0
    comp_lines = []
    diff_found = False
    while(i < len(old_layer_mrf) or j < len(new_layer_mrf)):
         l1 = old_layer_mrf[i]
         l2 = new_layer_mrf[j]
         # 1) Output line containing tokens
         if not l1.startswith(' ') and not l2.startswith(' '):
            diff = '*** ' if format_str.format(l1) != format_str.format(l2) else '    '
            comp_lines.append( diff+format_str.format(l1)+format_str.format(l2) )
            if diff == '*** ':
                 diff_found = True
            i += 1
            j += 1
         else:
            # 2) Output analysis line(s)
            while(i < len(old_layer_mrf) or j < len(new_layer_mrf)):
                 l1 = old_layer_mrf[i]
                 l2 = new_layer_mrf[j]
                 if l1.startswith(' ') and l2.startswith(' '):
                    diff = '*** ' if format_str.format(l1) != format_str.format(l2) else '    '
                    comp_lines.append( diff+format_str.format(l1)+format_str.format(l2) )
                    if diff == '*** ':
                       diff_found = True
                    i += 1
                    j += 1
                 elif l1.startswith(' ') and not l2.startswith(' '):
                    diff = '*** '
                    comp_lines.append( diff+format_str.format(l1)+format_str.format(' ') )
                    diff_found = True
                    i += 1
                 elif not l1.startswith(' ') and l2.startswith(' '):
                    diff = '*** '
                    comp_lines.append( diff+format_str.format(' ')+format_str.format(l2) )
                    diff_found = True
                    j += 1
                 else:
                    break
    return '\n'.join( comp_lines ), diff_found

Response: Returns a string that contains the differences between the original and converted text.


Instruction:
How would you implement a function in Python 3 that
removes redundant verb chains from the given list of found verbs.

Response:
def removeRedundantVerbChains( foundChains, removeOverlapping = True, removeSingleAraAndEi = False ):
    ''' Eemaldab yleliigsed verbiahelad: ahelad, mis katavad osaliselt v6i t2ielikult
        teisi ahelaid (removeOverlapping == True), yhes6nalised 'ei' ja 'ära' ahelad (kui
        removeSingleAraAndEi == True);

         Yldiselt on nii, et ylekattuvaid ei tohiks palju olla, kuna fraaside laiendamisel
        pyytakse alati kontrollida, et laiendus ei kattuks m6ne olemasoleva fraasiga;
        Peamiselt tekivad ylekattuvused siis, kui morf analyysi on sattunud valed
        finiitverbi analyysid (v6i analyysid on j22nud mitmesteks) ja seega tuvastatakse
        osalausest rohkem finiitverbe, kui oleks vaja.
         Heuristik: kahe ylekattuva puhul j2tame alles fraasi, mis algab eespool ning
        m2rgime sellel OTHER_VERBS v22rtuseks True, mis m2rgib, et kontekstis on mingi
        segadus teiste verbidega.
    '''
    toDelete = []
    for i in range(len(foundChains)):
        matchObj1 = foundChains[i]
        if removeOverlapping:
            for j in range(i+1, len(foundChains)):
                matchObj2 = foundChains[j]
                if matchObj1 != matchObj2 and matchObj1[CLAUSE_IDX] == matchObj2[CLAUSE_IDX]:
                    phrase1 = set(matchObj1[PHRASE])
                    phrase2 = set(matchObj2[PHRASE])
                    intersect = phrase1.intersection(phrase2)
                    if len(intersect) > 0:
                        #  Yldiselt on nii, et ylekattuvaid ei tohiks olla, kuna fraaside laiendamisel
                        # pyytakse alati kontrollida, et laiendus ei kattuks m6ne olemasoleva fraasiga;
                        #  Peamiselt tekivad ylekattuvused siis, kui morf analyysil on finiitverbi
                        # analyysidesse j22nud sisse mitmesused (v6i on sattunud valed analyysid) ja
                        # seega tuvastatakse osalausest rohkem finiitverbe, kui oleks vaja.
                        #  Heuristik: j2tame alles fraasi, mis algab eespool ning lisame selle otsa
                        # kysim2rgi (kuna pole kindel, et asjad on korras)
                        minWid1 = min(matchObj1[PHRASE])
                        minWid2 = min(matchObj2[PHRASE])
                        if minWid1 < minWid2:
                            matchObj1[OTHER_VERBS] = True
                            toDelete.append(j)
                        else:
                            matchObj2[OTHER_VERBS] = True
                            toDelete.append(i)
        if removeSingleAraAndEi:
            if ( len(matchObj1[PATTERN])==1 and re.match('^(ei|ära)$', matchObj1[PATTERN][0]) ):
                toDelete.append(i)
    if toDelete:
        if len(set(toDelete)) != len(toDelete):
            toDelete = list(set(toDelete))  # Eemaldame duplikaadid
        toDelete = [ foundChains[i] for i in toDelete ]
        for verbObj in toDelete:
            foundChains.remove(verbObj)