Instruction:
Can you write a function in Python 3 where it
computes Pgens from a file and output to another file.

Response:
def main():
    """Compute Pgens from a file and output to another file."""

    parser = OptionParser(conflict_handler="resolve")

    parser.add_option('--humanTRA', '--human_T_alpha', action='store_true', dest='humanTRA', default=False, help='use default human TRA model (T cell alpha chain)')
    parser.add_option('--humanTRB', '--human_T_beta', action='store_true', dest='humanTRB', default=False, help='use default human TRB model (T cell beta chain)')
    parser.add_option('--mouseTRB', '--mouse_T_beta', action='store_true', dest='mouseTRB', default=False, help='use default mouse TRB model (T cell beta chain)')
    parser.add_option('--humanIGH', '--human_B_heavy', action='store_true', dest='humanIGH', default=False, help='use default human IGH model (B cell heavy chain)')
    parser.add_option('--set_custom_model_VDJ', dest='vdj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VDJ generative model')
    parser.add_option('--set_custom_model_VJ', dest='vj_model_folder', metavar='PATH/TO/FOLDER/', help='specify PATH/TO/FOLDER/ for a custom VJ generative model')

    parser.add_option('-i', '--infile', dest = 'infile_name',metavar='PATH/TO/FILE', help='read in CDR3 sequences (and optionally V/J masks) from PATH/TO/FILE')
    parser.add_option('-o', '--outfile', dest = 'outfile_name', metavar='PATH/TO/FILE', help='write CDR3 sequences and pgens to PATH/TO/FILE')
    parser.add_option('--seq_in', '--seq_index', type='int', metavar='INDEX', dest='seq_in_index', default = 0, help='specifies sequences to be read in are in column INDEX. Default is index 0 (the first column).')

    parser.add_option('--v_in', '--v_mask_index', type='int', metavar='INDEX', dest='V_mask_index', help='specifies V_masks are found in column INDEX in the input file. Default is no V mask.')
    parser.add_option('--j_in', '--j_mask_index', type='int', metavar='INDEX', dest='J_mask_index', help='specifies J_masks are found in column INDEX in the input file. Default is no J mask.')

    parser.add_option('--v_mask', type='string', dest='V_mask', help='specify V usage to condition Pgen on for seqs read in as arguments.')
    parser.add_option('--j_mask', type='string', dest='J_mask', help='specify J usage to condition Pgen on for seqs read in as arguments.')

    parser.add_option('-m', '--max_number_of_seqs', type='int',metavar='N', dest='max_number_of_seqs', help='compute Pgens for at most N sequences.')
    parser.add_option('--lines_to_skip', type='int',metavar='N', dest='lines_to_skip', default = 0, help='skip the first N lines of the file. Default is 0.')
    parser.add_option('-a', '--alphabet_filename', dest='alphabet_filename', metavar='PATH/TO/FILE', help="specify PATH/TO/FILE defining a custom 'amino acid' alphabet. Default is no custom alphabet.")
    parser.add_option('--seq_type_out', type='choice',metavar='SEQ_TYPE', dest='seq_type_out',  choices=['all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'], help="if read in sequences are ntseqs, declare what type of sequence to compute pgen for. Default is all. Choices: 'all', 'ntseq', 'nucleotide', 'aaseq', 'amino_acid'")
    parser.add_option('--skip_off','--skip_empty_off', action='store_true', dest = 'skip_empty', default=True, help='stop skipping empty or blank sequences/lines (if for example you want to keep line index fidelity between the infile and outfile).')

    parser.add_option('--display_off', action='store_false', dest='display_seqs', default=True, help='turn the sequence display off (only applies in write-to-file mode). Default is on.')
    parser.add_option('--num_lines_for_display', type='int', metavar='N', default = 50, dest='num_lines_for_display', help='N lines of the output file are displayed when sequence display is on. Also used to determine the number of sequences to average over for speed and time estimates.')
    parser.add_option('--time_updates_off', action='store_false', dest='time_updates', default=True, help='turn time updates off (only applies when sequence display is disabled).')
    parser.add_option('--seqs_per_time_update', type='float', metavar='N', default = 100, dest='seqs_per_time_update', help='specify the number of sequences between time updates. Default is 1e5.')

    parser.add_option('-d', '--delimiter', type='choice', dest='delimiter',  choices=['tab', 'space', ',', ';', ':'], help="declare infile delimiter. Default is tab for .tsv input files, comma for .csv files, and any whitespace for all others. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_delimiter', type='str', dest='delimiter', help="declare infile delimiter as a raw string.")
    parser.add_option('--delimiter_out', type='choice', dest='delimiter_out',  choices=['tab', 'space', ',', ';', ':'], help="declare outfile delimiter. Default is tab for .tsv output files, comma for .csv files, and the infile delimiter for all others. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_delimiter_out', type='str', dest='delimiter_out', help="declare for the delimiter outfile as a raw string.")
    parser.add_option('--gene_mask_delimiter', type='choice', dest='gene_mask_delimiter',  choices=['tab', 'space', ',', ';', ':'], help="declare gene mask delimiter. Default comma unless infile delimiter is comma, then default is a semicolon. Choices: 'tab', 'space', ',', ';', ':'")
    parser.add_option('--raw_gene_mask_delimiter', type='str', dest='gene_mask_delimiter', help="declare delimiter of gene masks as a raw string.")
    parser.add_option('--comment_delimiter', type='str', dest='comment_delimiter', help="character or string to indicate comment or header lines to skip.")

    (options, args) = parser.parse_args()

    #Check that the model is specified properly
    main_folder = os.path.dirname(__file__)

    default_models = {}
    default_models['humanTRA'] = [os.path.join(main_folder, 'default_models', 'human_T_alpha'),  'VJ']
    default_models['humanTRB'] = [os.path.join(main_folder, 'default_models', 'human_T_beta'), 'VDJ']
    default_models['mouseTRB'] = [os.path.join(main_folder, 'default_models', 'mouse_T_beta'), 'VDJ']
    default_models['humanIGH'] = [os.path.join(main_folder, 'default_models', 'human_B_heavy'), 'VDJ']

    num_models_specified = sum([1 for x in default_models.keys() + ['vj_model_folder', 'vdj_model_folder'] if getattr(options, x)])

    if num_models_specified == 1: #exactly one model specified
        try:
            d_model = [x for x in default_models.keys() if getattr(options, x)][0]
            model_folder = default_models[d_model][0]
            recomb_type = default_models[d_model][1]
        except IndexError:
            if options.vdj_model_folder: #custom VDJ model specified
                model_folder = options.vdj_model_folder
                recomb_type = 'VDJ'
            elif options.vj_model_folder: #custom VJ model specified
                model_folder = options.vj_model_folder
                recomb_type = 'VJ'
    elif num_models_specified == 0:
        print 'Need to indicate generative model.'
        print 'Exiting...'
        return -1
    elif num_models_specified > 1:
        print 'Only specify one model'
        print 'Exiting...'
        return -1

    #Check that all model and genomic files exist in the indicated model folder
    if not os.path.isdir(model_folder):
        print 'Check pathing... cannot find the model folder: ' + model_folder
        print 'Exiting...'
        return -1

    params_file_name = os.path.join(model_folder,'model_params.txt')
    marginals_file_name = os.path.join(model_folder,'model_marginals.txt')
    V_anchor_pos_file = os.path.join(model_folder,'V_gene_CDR3_anchors.csv')
    J_anchor_pos_file = os.path.join(model_folder,'J_gene_CDR3_anchors.csv')

    for x in [params_file_name, marginals_file_name, V_anchor_pos_file, J_anchor_pos_file]:
        if not os.path.isfile(x):
            print 'Cannot find: ' + x
            print 'Please check the files (and naming conventions) in the model folder ' + model_folder
            print 'Exiting...'
            return -1

    alphabet_filename = options.alphabet_filename #used if a custom alphabet is to be specified
    if alphabet_filename is not None:
        if not os.path.isfile(alphabet_filename):
            print 'Cannot find custom alphabet file: ' + infile_name
            print 'Exiting...'
            return -1

    #Load up model based on recomb_type
    #VDJ recomb case --- used for TCRB and IGH
    if recomb_type == 'VDJ':
        genomic_data = load_model.GenomicDataVDJ()
        genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
        generative_model = load_model.GenerativeModelVDJ()
        generative_model.load_and_process_igor_model(marginals_file_name)
        pgen_model = generation_probability.GenerationProbabilityVDJ(generative_model, genomic_data, alphabet_filename)
    #VJ recomb case --- used for TCRA and light chain
    elif recomb_type == 'VJ':
        genomic_data = load_model.GenomicDataVJ()
        genomic_data.load_igor_genomic_data(params_file_name, V_anchor_pos_file, J_anchor_pos_file)
        generative_model = load_model.GenerativeModelVJ()
        generative_model.load_and_process_igor_model(marginals_file_name)
        pgen_model = generation_probability.GenerationProbabilityVJ(generative_model, genomic_data, alphabet_filename)

    aa_alphabet = ''.join(pgen_model.codons_dict.keys())

    if options.infile_name is not None:
        infile_name = options.infile_name

        if not os.path.isfile(infile_name):
            print 'Cannot find input file: ' + infile_name
            print 'Exiting...'
            return -1

    if options.outfile_name is not None:
        outfile_name = options.outfile_name
        if os.path.isfile(outfile_name):
            if not raw_input(outfile_name + ' already exists. Overwrite (y/n)? ').strip().lower() in ['y', 'yes']:
                print 'Exiting...'
                return -1

    #Parse delimiter
    delimiter = options.delimiter
    if delimiter is None: #Default case
        if options.infile_name is None:
            delimiter = '\t'
        elif infile_name.endswith('.tsv'): #parse TAB separated value file
            delimiter = '\t'
        elif infile_name.endswith('.csv'): #parse COMMA separated value file
            delimiter = ','
    else:
        try:
            delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter]
        except KeyError:
            pass #Other string passed as the delimiter.

    #Parse delimiter_out
    delimiter_out = options.delimiter_out
    if delimiter_out is None: #Default case
        if delimiter is None:
            delimiter_out = '\t'
        else:
            delimiter_out = delimiter
        if options.outfile_name is None:
            pass
        elif outfile_name.endswith('.tsv'): #output TAB separated value file
            delimiter_out = '\t'
        elif outfile_name.endswith('.csv'): #output COMMA separated value file
            delimiter_out = ','
    else:
        try:
            delimiter_out = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[delimiter_out]
        except KeyError:
            pass #Other string passed as the delimiter.

    #Parse gene_delimiter
    gene_mask_delimiter = options.gene_mask_delimiter
    if gene_mask_delimiter is None: #Default case
        gene_mask_delimiter = ','
        if delimiter == ',':
            gene_mask_delimiter = ';'
    else:
        try:
            gene_mask_delimiter = {'tab': '\t', 'space': ' ', ',': ',', ';': ';', ':': ':'}[gene_mask_delimiter]
        except KeyError:
            pass #Other string passed as the delimiter.

    #More options
    time_updates = options.time_updates
    display_seqs = options.display_seqs
    num_lines_for_display = options.num_lines_for_display
    seq_in_index = options.seq_in_index #where in the line the sequence is after line.split(delimiter)
    lines_to_skip = options.lines_to_skip #one method of skipping header
    comment_delimiter = options.comment_delimiter #another method of skipping header
    seqs_per_time_update = options.seqs_per_time_update
    max_number_of_seqs = options.max_number_of_seqs
    V_mask_index = options.V_mask_index #Default is not conditioning on V identity
    J_mask_index = options.J_mask_index #Default is not conditioning on J identity
    skip_empty = options.skip_empty

    seq_type_out = options.seq_type_out #type of pgens to be computed. Can be ntseq, aaseq, or both
    if seq_type_out is not None:
        seq_type_out = {'all': None, 'ntseq': 'ntseq', 'nucleotide': 'ntseq', 'aaseq': 'aaseq', 'amino_acid': 'aaseq'}[seq_type_out]

    if options.infile_name is None: #No infile specified -- args should be the input seqs
        print_warnings = True
        seqs = args
        seq_types = [determine_seq_type(seq, aa_alphabet) for seq in seqs]
        unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
        if len(unrecognized_seqs) > 0 and print_warnings:
            print 'The following sequences/arguments were not recognized: ' + ', '.join(unrecognized_seqs)
        seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
        seq_types = [seq_type for seq_type in seq_types if seq_type is not None]

        #Format V and J masks -- uniform for all argument input sequences
        try:
            V_mask = options.V_mask.split(',')
            unrecognized_v_genes = [v for v in V_mask if v not in pgen_model.V_mask_mapping.keys()]
            V_mask = [v for v in V_mask if v in pgen_model.V_mask_mapping.keys()]
            if len(unrecognized_v_genes) > 0:
                print 'These V genes/alleles are not recognized: ' + ', '.join(unrecognized_v_genes)
            if len(V_mask) == 0:
                print 'No recognized V genes/alleles in the provided V_mask. Continuing without conditioning on V usage.'
                V_mask = None
        except AttributeError:
            V_mask = options.V_mask #Default is None, i.e. not conditioning on V identity

        try:
            J_mask = options.J_mask.split(',')
            unrecognized_j_genes = [j for j in J_mask if j not in pgen_model.J_mask_mapping.keys()]
            J_mask = [j for j in J_mask if j in pgen_model.J_mask_mapping.keys()]
            if len(unrecognized_j_genes) > 0:
                print 'These J genes/alleles are not recognized: ' + ', '.join(unrecognized_j_genes)
            if len(J_mask) == 0:
                print 'No recognized J genes/alleles in the provided J_mask. Continuing without conditioning on J usage.'
                J_mask = None
        except AttributeError:
            J_mask = options.J_mask #Default is None, i.e. not conditioning on J identity

        print ''
        start_time = time.time()
        for seq, seq_type in zip(seqs, seq_types):
            if seq_type == 'aaseq':
                c_pgen = pgen_model.compute_aa_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
                print 'Pgen of the amino acid sequence ' + seq + ': ' + str(c_pgen)
                print ''
            elif seq_type == 'regex':
                c_pgen = pgen_model.compute_regex_CDR3_template_pgen(seq, V_mask, J_mask, print_warnings)
                print 'Pgen of the regular expression sequence ' + seq + ': ' + str(c_pgen)
                print ''
            elif seq_type == 'ntseq':
                if seq_type_out is None or seq_type_out == 'ntseq':
                    c_pgen_nt = pgen_model.compute_nt_CDR3_pgen(seq, V_mask, J_mask, print_warnings)
                    print 'Pgen of the nucleotide sequence ' + seq + ': ' + str(c_pgen_nt)
                if seq_type_out is None or seq_type_out == 'aaseq':
                    c_pgen_aa = pgen_model.compute_aa_CDR3_pgen(nt2aa(seq), V_mask, J_mask, print_warnings)
                    print 'Pgen of the amino acid sequence nt2aa(' + seq + ') = ' + nt2aa(seq) + ': ' + str(c_pgen_aa)
                print ''

        c_time = time.time() - start_time
        if c_time > 86400: #more than a day
            c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
        elif c_time > 3600: #more than an hr
            c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
        elif c_time > 60: #more than a min
            c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
        else:
            c_time_str = '%.2f seconds.'%(c_time)

        print 'Completed pgen computation in: ' + c_time_str

    else: #Read sequences in from file
        print_warnings = False #Most cases of reading in from file should have warnings disabled
        seqs = []
        seq_types = []
        V_usage_masks = []
        J_usage_masks = []

        infile = open(infile_name, 'r')

        for i, line in enumerate(infile):
            if comment_delimiter is not None: #Default case -- no comments/header delimiter
                if line.startswith(comment_delimiter): #allow comments
                    continue
            if i < lines_to_skip:
                continue

            if delimiter is None: #Default delimiter is any whitespace
                split_line = line.split()
            else:
                split_line = line.split(delimiter)

            #Find the seq
            try:
                seq = split_line[seq_in_index].strip()
                if len(seq.strip()) == 0:
                    if skip_empty:
                        continue
                    else:
                        seqs.append(seq) #keep the blank seq as a placeholder
                        seq_types.append('aaseq')
                else:
                    seqs.append(seq)
                    seq_types.append(determine_seq_type(seq, aa_alphabet))
            except IndexError: #no index match for seq
                if skip_empty and len(line.strip()) == 0:
                    continue
                print 'seq_in_index is out of range'
                print 'Exiting...'
                infile.close()
                return -1

            #Find and format V_usage_mask
            if V_mask_index is None:
                V_usage_masks.append(None) #default mask
            else:
                try:
                    V_usage_mask = split_line[V_mask_index].strip().split(gene_mask_delimiter)
                    #check that all V gene/allele names are recognized
                    if all([v in pgen_model.V_mask_mapping for v in V_usage_mask]):
                        V_usage_masks.append(V_usage_mask)
                    else:
                        print str(V_usage_mask) + " is not a usable V_usage_mask composed exclusively of recognized V gene/allele names"
                        print 'Unrecognized V gene/allele names: ' + ', '.join([v for v in V_usage_mask if not v in pgen_model.V_mask_mapping.keys()])
                        print 'Exiting...'
                        infile.close()
                        return -1
                except IndexError: #no index match for V_mask_index
                    print 'V_mask_index is out of range'
                    print 'Exiting...'
                    infile.close()
                    return -1

            #Find and format J_usage_mask
            if J_mask_index is None:
                J_usage_masks.append(None) #default mask
            else:
                try:
                    J_usage_mask = split_line[J_mask_index].strip().split(gene_mask_delimiter)
                    #check that all V gene/allele names are recognized
                    if all([j in pgen_model.J_mask_mapping for j in J_usage_mask]):
                        J_usage_masks.append(J_usage_mask)
                    else:
                        print str(J_usage_mask) + " is not a usable J_usage_mask composed exclusively of recognized J gene/allele names"
                        print 'Unrecognized J gene/allele names: ' + ', '.join([j for j in J_usage_mask if not j in pgen_model.J_mask_mapping.keys()])
                        print 'Exiting...'
                        infile.close()
                        return -1
                except IndexError: #no index match for J_mask_index
                    print 'J_mask_index is out of range'
                    print 'Exiting...'
                    infile.close()
                    return -1

            if max_number_of_seqs is not None:
                if len(seqs) >= max_number_of_seqs:
                    break

        unrecognized_seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is None]
        if len(unrecognized_seqs) > 0 and len(unrecognized_seqs) < len(seqs):
            if print_warnings or options.outfile_name is not None:
                print 'Some strings read in were not parsed as sequences -- they will be omitted.'
                print 'Examples of improperly read strings: '
                for unrecognized_seq in unrecognized_seqs[:10]:
                    print unrecognized_seq
            seqs = [seq for i, seq in enumerate(seqs) if seq_types[i] is not None]
            V_usage_masks = [V_usage_mask for i, V_usage_mask in enumerate(V_usage_masks) if seq_types[i] is not None]
            seq_types = [seq_type for seq_type in seq_types if seq_type is not None]
        elif len(unrecognized_seqs) > 0 and len(unrecognized_seqs) == len(seqs):
            print 'None of the read in strings were parsed as sequences. Check input file.'
            print 'Examples of improperly read strings:'
            for unrecognized_seq in unrecognized_seqs[:10]:
                print unrecognized_seq
            print 'Exiting...'
            return -1

        infile.close()

        if options.outfile_name is not None: #OUTFILE SPECIFIED, allow printed info/display

            print 'Successfully read in and formatted ' + str(len(seqs)) + ' sequences and any V or J usages.'
            if display_seqs:
                sys.stdout.write('\r'+'Continuing to Pgen computation in 3... ')
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r'+'Continuing to Pgen computation in 2... ')
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r'+'Continuing to Pgen computation in 1... ')
                sys.stdout.flush()
                time.sleep(0.4)
            else:
                print 'Continuing to Pgen computation.'
                print_warnings = True #Display is off, can print warnings

            if display_seqs:
                lines_for_display = []
                times_for_speed_calc = [time.time()]

            outfile = open(outfile_name, 'w')
            start_time = time.time()
            for i, seq in enumerate(seqs):
                if seq_types[i] == 'aaseq':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                if seq_types[i] == 'regex':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                elif seq_types[i] == 'ntseq':
                    ntseq = seq
                    if len(ntseq) % 3 == 0: #inframe sequence
                        aaseq = nt2aa(ntseq)
                        #Compute Pgen and print out based on recomb_type and seq_type_out
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out +  str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                    else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + '0'
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = 'out_of_frame' + delimiter_out + '0'

                outfile.write(c_pgen_line + '\n')

                #Print time update
                if display_seqs:
                    cc_time = time.time()
                    c_time = cc_time - start_time
                    times_for_speed_calc = [cc_time] + times_for_speed_calc[:num_lines_for_display]
                    c_avg_speed = (len(times_for_speed_calc)-1)/float(times_for_speed_calc[0] - times_for_speed_calc[-1])

                    #eta = ((len(seqs) - (i+1))/float(i+1))*c_time

                    eta = (len(seqs) - (i+1))/c_avg_speed

                    lines_for_display = [c_pgen_line] + lines_for_display[:num_lines_for_display]

                    c_time_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(c_time)/3600).rjust(3), repr((int(c_time)/60)%60).rjust(2), repr(int(c_time)%60).rjust(2))
                    eta_str = '%s hours, %s minutes, and %s seconds.'%(repr(int(eta)/3600).rjust(3), repr((int(eta)/60)%60).rjust(2), repr(int(eta)%60).rjust(2))
                    time_str = 'Time to compute Pgen on %s seqs: %s \nEst. time for remaining %s seqs: %s'%(repr(i+1).rjust(9), c_time_str, repr(len(seqs) - (i + 1)).rjust(9), eta_str)
                    speed_str = 'Current Pgen computation speed: %s seqs/min'%(repr(round((len(times_for_speed_calc)-1)*60/float(times_for_speed_calc[0] - times_for_speed_calc[-1]), 2)).rjust(8))
                    display_str = '\n'.join(lines_for_display[::-1]) + '\n' + '-'*80 + '\n' + time_str + '\n' + speed_str + '\n' + '-'*80
                    print '\033[2J' + display_str
                elif (i+1)%seqs_per_time_update == 0 and time_updates:
                    c_time = time.time() - start_time
                    eta = ((len(seqs) - (i+1))/float(i+1))*c_time
                    if c_time > 86400: #more than a day
                        c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
                    elif c_time > 3600: #more than an hr
                        c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
                    elif c_time > 60: #more than a min
                        c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
                    else:
                        c_time_str = '%.2f seconds.'%(c_time)

                    if eta > 86400: #more than a day
                        eta_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(eta)/86400, (int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
                    elif eta > 3600: #more than an hr
                        eta_str = '%d hours, %d minutes, and %.2f seconds.'%((int(eta)/3600)%24, (int(eta)/60)%60, eta%60)
                    elif eta > 60: #more than a min
                        eta_str = '%d minutes and %.2f seconds.'%((int(eta)/60)%60, eta%60)
                    else:
                        eta_str = '%.2f seconds.'%(eta)

                    print 'Pgen computed for %d sequences in: %s Estimated time remaining: %s'%(i+1, c_time_str, eta_str)

            c_time = time.time() - start_time
            if c_time > 86400: #more than a day
                c_time_str = '%d days, %d hours, %d minutes, and %.2f seconds.'%(int(c_time)/86400, (int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
            elif c_time > 3600: #more than an hr
                c_time_str = '%d hours, %d minutes, and %.2f seconds.'%((int(c_time)/3600)%24, (int(c_time)/60)%60, c_time%60)
            elif c_time > 60: #more than a min
                c_time_str = '%d minutes and %.2f seconds.'%((int(c_time)/60)%60, c_time%60)
            else:
                c_time_str = '%.2f seconds.'%(c_time)
            print 'Completed Pgen computation for %d sequences: in %s'%(len(seqs), c_time_str)

            outfile.close()

        else: #NO OUTFILE -- print directly to stdout
            start_time = time.time()
            for i, seq in enumerate(seqs):
                if seq_types[i] == 'aaseq':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                if seq_types[i] == 'regex':
                    #Compute Pgen and print out
                    c_pgen_line = seq + delimiter_out + str(pgen_model.compute_regex_CDR3_template_pgen(seq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                elif seq_types[i] == 'ntseq':
                    ntseq = seq
                    if len(ntseq) % 3 == 0: #inframe sequence
                        aaseq = nt2aa(ntseq)
                        #Compute Pgen and print out based on recomb_type and seq_type_out
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings)) + delimiter_out + aaseq + delimiter_out +  str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + str(pgen_model.compute_nt_CDR3_pgen(ntseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = aaseq + delimiter_out + str(pgen_model.compute_aa_CDR3_pgen(aaseq, V_usage_masks[i], J_usage_masks[i], print_warnings))
                    else: #out of frame sequence -- Pgens are 0 and use 'out_of_frame' for aaseq
                        if seq_type_out is None:
                            c_pgen_line = ntseq + delimiter_out + '0' + delimiter_out + 'out_of_frame' + delimiter_out + '0'
                        elif seq_type_out == 'ntseq':
                            c_pgen_line = ntseq + delimiter_out + '0'
                        elif seq_type_out == 'aaseq':
                            c_pgen_line = 'out_of_frame' + delimiter_out + '0'

                print c_pgen_line


Instruction:
Can you generate the documentation for the following Python 3 function
def create_plateaus(data, edges, plateau_size, plateau_vals, plateaus=None):
    '''Creates plateaus of constant value in the data.'''
    nodes = set(edges.keys())
    if plateaus is None:
        plateaus = []
        for i in range(len(plateau_vals)):
            if len(nodes) == 0:
                break
            node = np.random.choice(list(nodes))
            nodes.remove(node)
            plateau = [node]
            available = set(edges[node]) & nodes
            while len(nodes) > 0 and len(available) > 0 and len(plateau) < plateau_size:
                node = np.random.choice(list(available))
                plateau.append(node)
                available |= nodes & set(edges[node])
                available.remove(node)
            nodes -= set(plateau)
            plateaus.append(set(plateau))
    for p,v in zip(plateaus, plateau_vals):
        data[np.array(list(p), dtype=int)] = v
    return plateaus

Response: Creates plateaus of constant value in the data.


Instruction:
How would you explain what the following Python 3 function does
def matrix_str(p, decimal_places=2, print_zero=True, label_columns=False):
    '''Pretty-print the matrix.'''
    return '[{0}]'.format("\n  ".join([(str(i) if label_columns else '') + vector_str(a, decimal_places, print_zero) for i, a in enumerate(p)]))

Response: Pretty - print the matrix.


Instruction:
Can you tell what is the following Python 3 function doing
def vector_str(p, decimal_places=2, print_zero=True):
    '''Pretty-print the vector values.'''
    style = '{0:.' + str(decimal_places) + 'f}'
    return '[{0}]'.format(", ".join([' ' if not print_zero and a == 0 else style.format(a) for a in p]))

Response: Pretty - print the vector values.


Instruction:
Can you tell what is the following Python 3 function doing
def calc_plateaus(beta, edges, rel_tol=1e-4, verbose=0):
    '''Calculate the plateaus (degrees of freedom) of a graph of beta values in linear time.'''
    if not isinstance(edges, dict):
        raise Exception('Edges must be a map from each node to a list of neighbors.')
    to_check = deque(range(len(beta)))
    check_map = np.zeros(beta.shape, dtype=bool)
    check_map[np.isnan(beta)] = True
    plateaus = []

    if verbose:
        print('\tCalculating plateaus...')

    if verbose > 1:
        print('\tIndices to check {0} {1}'.format(len(to_check), check_map.shape))

    # Loop until every beta index has been checked
    while to_check:
        if verbose > 1:
            print('\t\tPlateau #{0}'.format(len(plateaus) + 1))

        # Get the next unchecked point on the grid
        idx = to_check.popleft()

        # If we already have checked this one, just pop it off
        while to_check and check_map[idx]:
            try:
                idx = to_check.popleft()
            except:
                break

        # Edge case -- If we went through all the indices without reaching an unchecked one.
        if check_map[idx]:
            break

        # Create the plateau and calculate the inclusion conditions
        cur_plateau = set([idx])
        cur_unchecked = deque([idx])
        val = beta[idx]
        min_member = val - rel_tol
        max_member = val + rel_tol

        # Check every possible boundary of the plateau
        while cur_unchecked:
            idx = cur_unchecked.popleft()

            # neighbors to check
            local_check = []

            # Generic graph case, get all neighbors of this node
            local_check.extend(edges[idx])

            # Check the index's unchecked neighbors
            for local_idx in local_check:
                if not check_map[local_idx] \
                    and beta[local_idx] >= min_member \
                    and beta[local_idx] <= max_member:
                        # Label this index as being checked so it's not re-checked unnecessarily
                        check_map[local_idx] = True

                        # Add it to the plateau and the list of local unchecked locations
                        cur_unchecked.append(local_idx)
                        cur_plateau.add(local_idx)

        # Track each plateau's indices
        plateaus.append((val, cur_plateau))

    # Returns the list of plateaus and their values
    return plateaus

Response: Calculate the plateaus of a graph of beta values in linear time.


Instruction:
Can you implement a function in Python 3 that
creates edge lists for an arbitrary hypercube.

Response:
def hypercube_edges(dims, use_map=False):
    '''Create edge lists for an arbitrary hypercube. TODO: this is probably not the fastest way.'''
    edges = []
    nodes = np.arange(np.product(dims)).reshape(dims)
    for i,d in enumerate(dims):
        for j in range(d-1):
            for n1, n2 in zip(np.take(nodes, [j], axis=i).flatten(), np.take(nodes,[j+1], axis=i).flatten()):
                edges.append((n1,n2))
    if use_map:
        return edge_map_from_edge_list(edges)
    return edges


Instruction:
Can you write a function in Python 3 where it
calculates the k - th order trend filtering matrix given the oriented edge incidence matrix and the value of k.

Response:
def get_delta(D, k):
    '''Calculate the k-th order trend filtering matrix given the oriented edge
    incidence matrix and the value of k.'''
    if k < 0:
        raise Exception('k must be at least 0th order.')
    result = D
    for i in range(k):
        result = D.T.dot(result) if i % 2 == 0 else D.dot(result)
    return result


Instruction:
Explain what the following Python 3 code does
def decompose_delta(deltak):
    '''Decomposes the k-th order trend filtering matrix into a c-compatible set
    of arrays.'''
    if not isspmatrix_coo(deltak):
        deltak = coo_matrix(deltak)
    dk_rows = deltak.shape[0]
    dk_rowbreaks = np.cumsum(deltak.getnnz(1), dtype="int32")
    dk_cols = deltak.col.astype('int32')
    dk_vals = deltak.data.astype('double')
    return dk_rows, dk_rowbreaks, dk_cols, dk_vals

Response:
Decomposes the k - th order trend filtering matrix into a c - compatible set
    of arrays.


Instruction:
Explain what the following Python 3 code does
def matrix_from_edges(edges):
    '''Returns a sparse penalty matrix (D) from a list of edge pairs. Each edge
    can have an optional weight associated with it.'''
    max_col = 0
    cols = []
    rows = []
    vals = []
    if type(edges) is defaultdict:
        edge_list = []
        for i, neighbors in edges.items():
            for j in neighbors:
                if i <= j:
                    edge_list.append((i,j))
        edges = edge_list
    for i, edge in enumerate(edges):
        s, t = edge[0], edge[1]
        weight = 1 if len(edge) == 2 else edge[2]
        cols.append(min(s,t))
        cols.append(max(s,t))
        rows.append(i)
        rows.append(i)
        vals.append(weight)
        vals.append(-weight)
        if cols[-1] > max_col:
            max_col = cols[-1]
    return coo_matrix((vals, (rows, cols)), shape=(rows[-1]+1, max_col+1))

Response: Returns a sparse penalty matrix D from a list of edge pairs. Each edge pair can have an optional weight associated with it.


Instruction:
Create a Python 3 function to
get the Kolmogorov - Smirnov distance between two densities a and b.

Response:
def ks_distance(a, b):
    '''Get the Kolmogorov-Smirnov (KS) distance between two densities a and b.'''
    if len(a.shape) == 1:
        return np.max(np.abs(a.cumsum() - b.cumsum()))
    return np.max(np.abs(a.cumsum(axis=1) - b.cumsum(axis=1)), axis=1)


Instruction:
Can you generate the documentation for the following Python 3 function
def tv_distance(a, b):
    '''Get the Total Variation (TV) distance between two densities a and b.'''
    if len(a.shape) == 1:
        return np.sum(np.abs(a - b))
    return np.sum(np.abs(a - b), axis=1)

Response: Get the Total Variation ( TV ) distance between two densities a and b.


Instruction:
Here you have a function in Python 3, explain what it does
def jdFromDate(dd, mm, yy):
    '''def jdFromDate(dd, mm, yy): Compute the (integral) Julian day number of
    day dd/mm/yyyy, i.e., the number of days between 1/1/4713 BC
    (Julian calendar) and dd/mm/yyyy.'''
    a = int((14 - mm) / 12.)
    y = yy + 4800 - a
    m = mm + 12 * a - 3
    jd = dd + int((153 * m + 2) / 5.) \
        + 365 * y + int(y / 4.) - int(y / 100.) \
        + int(y / 400.) - 32045
    if (jd < 2299161):
        jd = dd + int((153 * m + 2) / 5.) \
            + 365 * y + int(y / 4.) - 32083
    return jd

Response: Compute the Julian day number of the current date.


Instruction:
Create a Python 3 function to
convert a Julian day number to day month year.

Response:
def jdToDate(jd):
    '''def jdToDate(jd): Convert a Julian day number to day/month/year.
                       jd is an integer.'''
    if (jd > 2299160):
        # After 5/10/1582, Gregorian calendar
        a = jd + 32044
        b = int((4 * a + 3) / 146097.)
        c = a - int((b * 146097) / 4.)
    else:
        b = 0
        c = jd + 32082
    d = int((4 * c + 3) / 1461.)
    e = c - int((1461 * d) / 4.)
    m = int((5 * e + 2) / 153.)
    day = e - int((153 * m + 2) / 5.) + 1
    month = m + 3 - 12 * int(m / 10.)
    year = b * 100 + d - 4800 + int(m / 10.)
    return [day, month, year]


Instruction:
How would you explain what the following Python 3 function does
def NewMoon(k):
    '''def NewMoon(k): Compute the time of the k-th new moon after
    the new moon of 1/1/1900 13:52 UCT (measured as the number of
    days since 1/1/4713 BC noon UCT, e.g., 2451545.125 is 1/1/2000 15:00 UTC.
    Returns a floating number, e.g., 2415079.9758617813 for k=2 or
    2414961.935157746 for k=-2.'''
    # Time in Julian centuries from 1900 January 0.5
    T = k / 1236.85
    T2 = T * T
    T3 = T2 * T
    dr = math.pi / 180.
    Jd1 = 2415020.75933 + 29.53058868 * k \
        + 0.0001178 * T2 - 0.000000155 * T3
    Jd1 = Jd1 + 0.00033 * math.sin(
        (166.56 + 132.87 * T - 0.009173 * T2) * dr)
    # Mean new moon
    M = 359.2242 + 29.10535608 * k \
        - 0.0000333 * T2 - 0.00000347 * T3
    # Sun's mean anomaly
    Mpr = 306.0253 + 385.81691806 * k \
        + 0.0107306 * T2 + 0.00001236 * T3
    # Moon's mean anomaly
    F = 21.2964 + 390.67050646 * k - 0.0016528 * T2 \
        - 0.00000239 * T3
    # Moon's argument of latitude
    C1 = (0.1734 - 0.000393 * T) * math.sin(M * dr) \
        + 0.0021 * math.sin(2 * dr * M)
    C1 = C1 - 0.4068 * math.sin(Mpr * dr) \
        + 0.0161 * math.sin(dr * 2 * Mpr)
    C1 = C1 - 0.0004 * math.sin(dr * 3 * Mpr)
    C1 = C1 + 0.0104 * math.sin(dr * 2 * F) \
        - 0.0051 * math.sin(dr * (M + Mpr))
    C1 = C1 - 0.0074 * math.sin(dr * (M - Mpr)) \
        + 0.0004 * math.sin(dr * (2 * F + M))
    C1 = C1 - 0.0004 * math.sin(dr * (2 * F - M)) \
        - 0.0006 * math.sin(dr * (2 * F + Mpr))
    C1 = C1 + 0.0010 * math.sin(dr * (2 * F - Mpr)) \
        + 0.0005 * math.sin(dr * (2 * Mpr + M))
    if (T < -11):
        deltat = 0.001 + 0.000839 * T + 0.0002261 * T2 \
            - 0.00000845 * T3 - 0.000000081 * T * T3
    else:
        deltat = -0.000278 + 0.000265 * T + 0.000262 * T2
    JdNew = Jd1 + C1 - deltat
    return JdNew

Response: Return a new moon in the current date.


Instruction:
How would you explain what the following Python 3 function does
def SunLongitude(jdn):
    '''def SunLongitude(jdn): Compute the longitude of the sun at any time.
    Parameter: floating number jdn, the number of days since 1/1/4713 BC noon.
    '''
    T = (jdn - 2451545.0) / 36525.
    # Time in Julian centuries
    # from 2000-01-01 12:00:00 GMT
    T2 = T * T
    dr = math.pi / 180.  # degree to radian
    M = 357.52910 + 35999.05030 * T \
        - 0.0001559 * T2 - 0.00000048 * T * T2
    # mean anomaly, degree
    L0 = 280.46645 + 36000.76983 * T + 0.0003032 * T2
    # mean longitude, degree
    DL = (1.914600 - 0.004817 * T - 0.000014 * T2) \
        * math.sin(dr * M)
    DL += (0.019993 - 0.000101 * T) * math.sin(dr * 2 * M) \
        + 0.000290 * math.sin(dr * 3 * M)
    L = L0 + DL  # true longitude, degree
    L = L * dr
    L = L - math.pi * 2 * (float(L / (math.pi * 2)))
    # Normalize to (0, 2*math.pi)
    return L

Response: Return the longitude of the sun at any time.


Instruction:
Create a Python 3 function for
returning the day that starts the luner month11 of the given time zone.

Response:
def getLunarMonth11(yy, timeZone):
    '''def getLunarMonth11(yy, timeZone):  Find the day that starts the luner month
    11of the given year for the given time zone.'''
    # off = jdFromDate(31, 12, yy) \
    #            - 2415021.076998695
    off = jdFromDate(31, 12, yy) - 2415021.
    k = int(off / 29.530588853)
    nm = getNewMoonDay(k, timeZone)
    sunLong = getSunLongitude(nm, timeZone)
    # sun longitude at local midnight
    if (sunLong >= 9):
        nm = getNewMoonDay(k - 1, timeZone)
    return nm


Instruction:
Implement a function in Python 3 to
return the index of the leap month after the month starting on the day a11.

Response:
def getLeapMonthOffset(a11, timeZone):
    '''def getLeapMonthOffset(a11, timeZone): Find the index of the leap month
    after the month starting on the day a11.'''
    k = int((a11 - 2415021.076998695) / 29.530588853 + 0.5)
    last = 0
    i = 1  # start with month following lunar month 11
    arc = getSunLongitude(
        getNewMoonDay(k + i, timeZone), timeZone)
    while True:
        last = arc
        i += 1
        arc = getSunLongitude(
            getNewMoonDay(k + i, timeZone),
            timeZone)
        if not (arc != last and i < 14):
            break
    return i - 1


Instruction:
Can you tell what is the following Python 3 function doing
def S2L(dd, mm, yy, timeZone=7):
    '''def S2L(dd, mm, yy, timeZone = 7): Convert solar date dd/mm/yyyy to
    the corresponding lunar date.'''
    dayNumber = jdFromDate(dd, mm, yy)
    k = int((dayNumber - 2415021.076998695) / 29.530588853)
    monthStart = getNewMoonDay(k + 1, timeZone)
    if (monthStart > dayNumber):
        monthStart = getNewMoonDay(k, timeZone)
    # alert(dayNumber + " -> " + monthStart)
    a11 = getLunarMonth11(yy, timeZone)
    b11 = a11
    if (a11 >= monthStart):
        lunarYear = yy
        a11 = getLunarMonth11(yy - 1, timeZone)
    else:
        lunarYear = yy + 1
        b11 = getLunarMonth11(yy + 1, timeZone)
    lunarDay = dayNumber - monthStart + 1
    diff = int((monthStart - a11) / 29.)

    lunarLeap = 0
    lunarMonth = diff + 11

    if (b11 - a11 > 365):
        leapMonthDiff = \
            getLeapMonthOffset(a11, timeZone)
        if (diff >= leapMonthDiff):
            lunarMonth = diff + 10
            if (diff == leapMonthDiff):
                lunarLeap = 1
    if (lunarMonth > 12):
        lunarMonth = lunarMonth - 12
    if (lunarMonth >= 11 and diff < 4):
        lunarYear -= 1
    # print [lunarDay, lunarMonth, lunarYear, lunarLeap]
    return \
        [lunarDay, lunarMonth, lunarYear, lunarLeap]

Response: Convert solar date dd mm yy to a corresponding lunar date.


Instruction:
How would you implement a function in Python 3 that
converts a lunar date to the corresponding solar date.

Response:
def L2S(lunarD, lunarM, lunarY, lunarLeap, tZ=7):
    '''def L2S(lunarD, lunarM, lunarY, lunarLeap, tZ = 7): Convert a lunar date
    to the corresponding solar date.'''
    if (lunarM < 11):
        a11 = getLunarMonth11(lunarY - 1, tZ)
        b11 = getLunarMonth11(lunarY, tZ)
    else:
        a11 = getLunarMonth11(lunarY, tZ)
        b11 = getLunarMonth11(lunarY + 1, tZ)
    k = int(0.5 +
            (a11 - 2415021.076998695) / 29.530588853)
    off = lunarM - 11
    if (off < 0):
        off += 12
    if (b11 - a11 > 365):
        leapOff = getLeapMonthOffset(a11, tZ)
        leapM = leapOff - 2
        if (leapM < 0):
            leapM += 12
        if (lunarLeap != 0 and lunarM != leapM):
            return [0, 0, 0]
        elif (lunarLeap != 0 or off >= leapOff):
            off += 1
    monthStart = getNewMoonDay(k + off, tZ)
    return jdToDate(monthStart + lunarD - 1)


Instruction:
How would you implement a function in Python 3 that
checks if the given stream has the MZ signature.

Response:
def hasMZSignature(self, rd):
        """
        Check for MZ signature.

        @type rd: L{ReadData}
        @param rd: A L{ReadData} object.

        @rtype: bool
        @return: True is the given L{ReadData} stream has the MZ signature. Otherwise, False.
        """
        rd.setOffset(0)
        sign = rd.read(2)
        if sign == "MZ":
            return True
        return False


Instruction:
How would you explain what the following Python 3 function does
def hasPESignature(self, rd):
        rd.setOffset(0)
        e_lfanew_offset = unpack("<L",  rd.readAt(0x3c, 4))[0]
        sign = rd.readAt(e_lfanew_offset, 2)
        if sign == "PE":
            return True
        return False

Response: Check if the given stream has the PE signature.


Instruction:
How would you code a function in Python 3 to
validate the structure of the loaded file.

Response:
def validate(self):
        """
        Performs validations over some fields of the PE structure to determine if the loaded file has a valid PE format.

        @raise PEException: If an invalid value is found into the PE instance.
        """
        # Ange Albertini (@angie4771) can kill me for this! :)
        if self.dosHeader.e_magic.value != consts.MZ_SIGNATURE:
            raise excep.PEException("Invalid MZ signature. Found %d instead of %d." % (self.dosHeader.magic.value, consts.MZ_SIGNATURE))

        if self.dosHeader.e_lfanew.value > len(self):
            raise excep.PEException("Invalid e_lfanew value. Probably not a PE file.")

        if self.ntHeaders.signature.value != consts.PE_SIGNATURE:
            raise excep.PEException("Invalid PE signature. Found %d instead of %d." % (self.ntHeaders.optionaHeader.signature.value, consts.PE_SIGNATURE))

        if self.ntHeaders.optionalHeader.numberOfRvaAndSizes.value > 0x10:
            print excep.PEWarning("Suspicious value for NumberOfRvaAndSizes: %d." % self.ntHeaders.optionaHeader.numberOfRvaAndSizes.value)


Instruction:
Explain what the following Python 3 code does
def readFile(self, pathToFile):
        fd = open(pathToFile,  "rb")
        data = fd.read()
        fd.close()
        return data

Response: Reads the data from a file.


Instruction:
Implement a Python 3 function for
writing data from the object to a file.

Response:
def write(self, filename = ""):
        """
        Writes data from L{PE} object to a file.

        @rtype: str
        @return: The L{PE} stream data.

        @raise IOError: If the file could not be opened for write operations.
        """
        file_data = str(self)
        if filename:
            try:
                self.__write(filename, file_data)
            except IOError:
                raise IOError("File could not be opened for write operations.")
        else:
            return file_data


Instruction:
Write a Python 3 function that can
write data to a file.

Response:
def __write(self, thePath, theData):
        """
        Write data to a file.

        @type thePath: str
        @param thePath: The file path.

        @type theData: str
        @param theData: The data to write.
        """
        fd = open(thePath, "wb")
        fd.write(theData)
        fd.close()


Instruction:
How would you code a function in Python 3 to
update the data in every directory in every L{Directory object.

Response:
def _updateDirectoriesData(self, peStr):
        """
        Updates the data in every L{Directory} object.

        @type peStr: str
        @param peStr: C{str} representation of the L{PE} object.

        @rtype: str
        @return: A C{str} representation of the L{PE} object.
        """
        dataDirs = self.ntHeaders.optionalHeader.dataDirectory
        wr = utils.WriteData(data)

        for dir in dataDirs:
            dataToWrite = str(dir.info)
            if len(dataToWrite) != dir.size.value and self._verbose:
                print excep.DataLengthException("Warning: current size of %s directory does not match with dataToWrite length %d." % (dir.size.value, len(dataToWrite)))
            wr.setOffset(self.getOffsetFromRva(dir.rva.value))
            wr.write(dataToWrite)
        return str(wr)


Instruction:
Can you generate the documentation for the following Python 3 function
def _getPaddingDataToSectionOffset(self):
        start = self._getPaddingToSectionOffset()
        end = self.sectionHeaders[0].pointerToRawData.value - start
        return self._data[start:start+end]

Response: Returns the data between the last section header and the begenning of data from the first section.


Instruction:
How would you code a function in Python 3 to
return the digital signature within a digital signed PE file.

Response:
def _getSignature(self, readDataInstance, dataDirectoryInstance):
        """
        Returns the digital signature within a digital signed PE file.

        @type readDataInstance: L{ReadData}
        @param readDataInstance: A L{ReadData} instance containing a PE file data.

        @type dataDirectoryInstance: L{DataDirectory}
        @param dataDirectoryInstance: A L{DataDirectory} object containing the information about directories.

        @rtype: str
        @return: A string with the digital signature.

        @raise InstanceErrorException: If the C{readDataInstance} or the C{dataDirectoryInstance} were not specified.
        """
        signature = ""

        if readDataInstance is not None and dataDirectoryInstance is not None:
            securityDirectory = dataDirectoryInstance[consts.SECURITY_DIRECTORY]

            if(securityDirectory.rva.value and securityDirectory.size.value):
                readDataInstance.setOffset(self.getOffsetFromRva(securityDirectory.rva.value))

                signature = readDataInstance.read(securityDirectory.size.value)
        else:
            raise excep.InstanceErrorException("ReadData instance or DataDirectory instance not specified.")

        return signature


Instruction:
Given the following Python 3 function, write the documentation
def _getOverlay(self, readDataInstance, sectionHdrsInstance):
        if readDataInstance is not None and sectionHdrsInstance is not None:
            # adjust the offset in readDataInstance to the RawOffset + RawSize of the last section
            try:
                offset = sectionHdrsInstance[-1].pointerToRawData.value + sectionHdrsInstance[-1].sizeOfRawData.value
                readDataInstance.setOffset(offset)
            except excep.WrongOffsetValueException:
                if self._verbose:
                    print "It seems that the file has no overlay data."
        else:
            raise excep.InstanceErrorException("ReadData instance or SectionHeaders instance not specified.")

        return readDataInstance.data[readDataInstance.offset:]

Response: Returns the overlay data from the PE file.


Instruction:
Write a Python 3 function that can
convert an offset to an RVA.

Response:
def getOffsetFromRva(self, rva):
        """
        Converts an offset to an RVA.

        @type rva: int
        @param rva: The RVA to be converted.

        @rtype: int
        @return: An integer value representing an offset in the PE file.
        """
        offset = -1
        s = self.getSectionByRva(rva)

        if s != offset:
            offset = (rva - self.sectionHeaders[s].virtualAddress.value) + self.sectionHeaders[s].pointerToRawData.value
        else:
            offset = rva

        return offset


Instruction:
Create a Python 3 function for
converting a RVA to an offset.

Response:
def getRvaFromOffset(self, offset):
        """
        Converts a RVA to an offset.

        @type offset: int
        @param offset: The offset value to be converted to RVA.

        @rtype: int
        @return: The RVA obtained from the given offset.
        """
        rva = -1
        s = self.getSectionByOffset(offset)

        if s:
            rva = (offset - self.sectionHeaders[s].pointerToRawData.value) + self.sectionHeaders[s].virtualAddress.value

        return rva


Instruction:
How would you implement a function in Python 3 that
gives an offset in the file, tries to determine the section this offset belong to. @type offset: int @param offset: Offset value. @rtype: int @return: An index, starting at 0, that represents the section the given offset belongs to.

Response:
def getSectionByOffset(self, offset):
        """
        Given an offset in the file, tries to determine the section this offset belong to.

        @type offset: int
        @param offset: Offset value.

        @rtype: int
        @return: An index, starting at 0, that represents the section the given offset belongs to.
        """
        index = -1
        for i in range(len(self.sectionHeaders)):
            if (offset < self.sectionHeaders[i].pointerToRawData.value + self.sectionHeaders[i].sizeOfRawData.value):
                index = i
                break
        return index


Instruction:
How would you code a function in Python 3 to
try to find the section index by name.

Response:
def getSectionIndexByName(self, name):
        """
        Given a string representing a section name, tries to find the section index.

        @type name: str
        @param name: A section name.

        @rtype: int
        @return: The index, starting at 0, of the section.
        """
        index = -1

        if name:
            for i in range(len(self.sectionHeaders)):
                if self.sectionHeaders[i].name.value.find(name) >= 0:
                    index = i
                    break
        return index


Instruction:
Create a Python 3 function for
giving a RVA in the file, tries to determine the section this RVA belongs to. @type rva: int @param rva: RVA value. @rtype: int @return: An index, starting at 1, that represents the section the given RVA belongs to.

Response:
def getSectionByRva(self, rva):
        """
        Given a RVA in the file, tries to determine the section this RVA belongs to.

        @type rva: int
        @param rva: RVA value.

        @rtype: int
        @return: An index, starting at 1, that represents the section the given RVA belongs to.
        """

        index = -1
        if rva < self.sectionHeaders[0].virtualAddress.value:
            return index

        for i in range(len(self.sectionHeaders)):
            fa = self.ntHeaders.optionalHeader.fileAlignment.value
            prd = self.sectionHeaders[i].pointerToRawData.value
            srd = self.sectionHeaders[i].sizeOfRawData.value
            if len(str(self)) - self._adjustFileAlignment(prd,  fa) < srd:
                size = self.sectionHeaders[i].misc.value
            else:
                size = max(srd,  self.sectionHeaders[i].misc.value)
            if (self.sectionHeaders[i].virtualAddress.value <= rva) and rva < (self.sectionHeaders[i].virtualAddress.value + size):
                index = i
                break

        return index


Instruction:
Can you write a function in Python 3 where it
returns the offset to the last section header present in the PE file.

Response:
def _getPaddingToSectionOffset(self):
        """
        Returns the offset to last section header present in the PE file.

        @rtype: int
        @return: The offset where the end of the last section header resides in the PE file.
        """
        return len(str(self.dosHeader) + str(self.dosStub) + str(self.ntHeaders) + str(self.sectionHeaders))


Instruction:
How would you implement a function in Python 3 that
parses all the directories in the PE file.

Response:
def fullLoad(self):
        """Parse all the directories in the PE file."""
        self._parseDirectories(self.ntHeaders.optionalHeader.dataDirectory, self.PE_TYPE)


Instruction:
Given the following Python 3 function, write the documentation
def _internalParse(self, readDataInstance):
        self.dosHeader = DosHeader.parse(readDataInstance)

        self.dosStub = readDataInstance.read(self.dosHeader.e_lfanew.value - readDataInstance.offset)
        self.ntHeaders = NtHeaders.parse(readDataInstance)

        if self.ntHeaders.optionalHeader.magic.value == consts.PE32:
            self.PE_TYPE = consts.PE32
        elif self.ntHeaders.optionalHeader.magic.value == consts.PE64:
            self.PE_TYPE = consts.PE64
            readDataInstance.setOffset(readDataInstance.tell() - OptionalHeader().sizeof())
            self.ntHeaders.optionalHeader = OptionalHeader64.parse(readDataInstance)

        self.sectionHeaders = SectionHeaders.parse(readDataInstance,  self.ntHeaders.fileHeader.numberOfSections.value)

        # as padding is possible between the last section header and the beginning of the first section
        # we must adjust the offset in readDataInstance to point to the first byte of the first section.
        readDataInstance.setOffset(self.sectionHeaders[0].pointerToRawData.value)

        self.sections = Sections.parse(readDataInstance,  self.sectionHeaders)

        self.overlay = self._getOverlay(readDataInstance,  self.sectionHeaders)
        self.signature = self._getSignature(readDataInstance,  self.ntHeaders.optionalHeader.dataDirectory)

        if not self._fastLoad:
            self._parseDirectories(self.ntHeaders.optionalHeader.dataDirectory, self.PE_TYPE)

Response: Internal method to parse the contents of a file.


Instruction:
Can you write a function in Python 3 where it
adds a new section to the existing instance.

Response:
def addSection(self, data, name =".pype32\x00", flags = 0x60000000):
        """
        Adds a new section to the existing L{PE} instance.

        @type data: str
        @param data: The data to be added in the new section.

        @type name: str
        @param name: (Optional) The name for the new section.

        @type flags: int
        @param flags: (Optional) The attributes for the new section.
        """
        fa = self.ntHeaders.optionalHeader.fileAlignment.value
        sa = self.ntHeaders.optionalHeader.sectionAlignment.value

        padding = "\xcc" * (fa - len(data))
        sh = SectionHeader()

        if len(self.sectionHeaders):
            # get the va, vz, ra and rz of the last section in the array of section headers
            vaLastSection = self.sectionHeaders[-1].virtualAddress.value
            sizeLastSection = self.sectionHeaders[-1].misc.value
            pointerToRawDataLastSection = self.sectionHeaders[-1].pointerToRawData.value
            sizeOfRawDataLastSection = self.sectionHeaders[-1].sizeOfRawData.value

            sh.virtualAddress.value = self._adjustSectionAlignment(vaLastSection + sizeLastSection,  fa, sa)
            sh.pointerToRawData.value = self._adjustFileAlignment(pointerToRawDataLastSection + sizeOfRawDataLastSection,  fa)

        sh.misc.value = self._adjustSectionAlignment(len(data),  fa,  sa) or consts.DEFAULT_PAGE_SIZE
        sh.sizeOfRawData.value = self._adjustFileAlignment(len(data),  fa) or consts.DEFAULT_FILE_ALIGNMENT
        sh.characteristics.value = flags
        sh.name.value = name

        self.sectionHeaders.append(sh)
        self.sections.append(data + padding)

        self.ntHeaders.fileHeader.numberOfSections.value += 1


Instruction:
Can you tell what is the following Python 3 function doing
def extendSection(self, sectionIndex, data):

        fa = self.ntHeaders.optionalHeader.fileAlignment.value
        sa = self.ntHeaders.optionalHeader.sectionAlignment.value

        if len(self.sectionHeaders):
            if len(self.sectionHeaders) == sectionIndex:
                try:
                    # we are in the last section or self.sectionHeaders has only 1 sectionHeader instance
                    vzLastSection = self.sectionHeaders[-1].misc.value
                    rzLastSection = self.sectionHeaders[-1].sizeOfRawData.value

                    self.sectionHeaders[-1].misc.value = self._adjustSectionAlignment(vzLastSection + len(data), fa,  sa)
                    self.sectionHeaders[-1].sizeOfRawData.value = self._adjustFileAlignment(rzLastSection + len(data),  fa)

                    vz = self.sectionHeaders[-1].misc.value
                    rz = self.sectionHeaders[-1].sizeOfRawData.value

                except IndexError:
                    raise IndexError("list index out of range.")

                if vz < rz:
                    print "WARNING: VirtualSize (%x) is less than SizeOfRawData (%x)" % (vz,  rz)

                if len(data) % fa == 0:
                    self.sections[-1] += data
                else:
                    self.sections[-1] += data + "\xcc" * (fa - len(data) % fa)

            else:
                # if it is not the last section ...
                try:
                    # adjust data of the section the user wants to extend
                    counter = sectionIndex - 1

                    vzCurrentSection = self.sectionHeaders[counter].misc.value
                    rzCurrentSection = self.sectionHeaders[counter].sizeOfRawData.value

                    self.sectionHeaders[counter].misc.value = self._adjustSectionAlignment(vzCurrentSection + len(data),  fa,  sa)
                    self.sectionHeaders[counter].sizeOfRawData.value = self._adjustFileAlignment(rzCurrentSection + len(data),  fa)

                    if len(data) % fa == 0:
                        self.sections[counter] += data
                    else:
                        self.sections[counter] += data + "\xcc" * (fa - len(data) % fa)

                    counter += 1

                    while(counter != len(self.sectionHeaders)):
                        vzPreviousSection = self.sectionHeaders[counter - 1].misc.value
                        vaPreviousSection = self.sectionHeaders[counter - 1].virtualAddress.value
                        rzPreviousSection = self.sectionHeaders[counter - 1].sizeOfRawData.value
                        roPreviousSection = self.sectionHeaders[counter - 1].pointerToRawData.value

                        # adjust VA and RO of the next section
                        self.sectionHeaders[counter].virtualAddress.value = self._adjustSectionAlignment(vzPreviousSection + vaPreviousSection,  fa,  sa)
                        self.sectionHeaders[counter].pointerToRawData.value = self._adjustFileAlignment(rzPreviousSection + roPreviousSection,  fa)

                        vz = self.sectionHeaders[counter].virtualAddress.value
                        rz = self.sectionHeaders[counter].pointerToRawData.value

                        if vz < rz:
                            print "WARNING: VirtualSize (%x) is less than SizeOfRawData (%x)" % (vz,  rz)

                        counter += 1

                except IndexError:
                    raise IndexError("list index out of range.")

        else:
            raise excep.SectionHeadersException("There is no section to extend.")

Response: Extends an existing section in the instance.


Instruction:
How would you code a function in Python 3 to
fix the necessary fields in the PE file instance in order to create a valid PE32 PE32. i. e. SizeOfImage.

Response:
def _fixPe(self):
        """
        Fixes the necessary fields in the PE file instance in order to create a valid PE32. i.e. SizeOfImage.
        """
        sizeOfImage = 0
        for sh in self.sectionHeaders:
            sizeOfImage += sh.misc
        self.ntHeaders.optionaHeader.sizeoOfImage.value = self._sectionAlignment(sizeOfImage + 0x1000)


Instruction:
Can you create a Python 3 function that
adjusts the value of the file alignment parameter.

Response:
def _adjustFileAlignment(self, value, fileAlignment):
        """
        Align a value to C{FileAligment}.

        @type value: int
        @param value: The value to align.

        @type fileAlignment: int
        @param fileAlignment: The value to be used to align the C{value} parameter.

        @rtype: int
        @return: The aligned value.
        """
        if fileAlignment > consts.DEFAULT_FILE_ALIGNMENT:
            if not utils.powerOfTwo(fileAlignment):
                print "Warning: FileAlignment is greater than DEFAULT_FILE_ALIGNMENT (0x200) and is not power of two."

        if fileAlignment < consts.DEFAULT_FILE_ALIGNMENT:
            return value

        if fileAlignment and value % fileAlignment:
            return ((value / fileAlignment) + 1) * fileAlignment

        return value


Instruction:
Here you have a function in Python 3, explain what it does
def _adjustSectionAlignment(self, value, fileAlignment, sectionAlignment):
        if fileAlignment < consts.DEFAULT_FILE_ALIGNMENT:
            if fileAligment != sectionAlignment:
                print "FileAlignment does not match SectionAlignment."

        if sectionAlignment < consts.DEFAULT_PAGE_SIZE:
            sectionAlignment = fileAlignment

        if sectionAlignment and value % sectionAlignment:
            return sectionAlignment * ((value / sectionAlignment) + 1)
        return value

Response: Adjusts the value of a key to match the file alignment and section alignment.


Instruction:
Can you implement a function in Python 3 that
returns a C { DWORD } from a given RVA.

Response:
def getDwordAtRva(self, rva):
        """
        Returns a C{DWORD} from a given RVA.

        @type rva: int
        @param rva: The RVA to get the C{DWORD} from.

        @rtype: L{DWORD}
        @return: The L{DWORD} obtained at the given RVA.
        """
        return datatypes.DWORD.parse(utils.ReadData(self.getDataAtRva(rva,  4)))


Instruction:
Implement a function in Python 3 to
return a C { WORD } from a given RVA.

Response:
def getWordAtRva(self, rva):
        """
        Returns a C{WORD} from a given RVA.

        @type rva: int
        @param rva: The RVA to get the C{WORD} from.

        @rtype: L{WORD}
        @return: The L{WORD} obtained at the given RVA.
        """
        return datatypes.WORD.parse(utils.ReadData(self.getDataAtRva(rva,  2)))


Instruction:
Here you have a function in Python 3, explain what it does
def getDwordAtOffset(self, offset):
        return datatypes.DWORD.parse(utils.ReadData(self.getDataAtOffset(offset,  4)))

Response: Returns a C { DWORD } from a given offset.


Instruction:
Explain what the following Python 3 code does
def getWordAtOffset(self, offset):
        return datatypes.WORD.parse(utils.ReadData(self.getDataAtOffset(offset, 2)))

Response: Returns a C { WORD } from a given offset.


Instruction:
Here you have a function in Python 3, explain what it does
def getQwordAtRva(self, rva):
        return datatypes.QWORD.parse(utils.ReadData(self.getDataAtRva(rva,  8)))

Response: Returns a C { QWORD } from a given RVA.


Instruction:
Can you tell what is the following Python 3 function doing
def getQwordAtOffset(self, offset):
        return datatypes.QWORD.parse(utils.ReadData(self.getDataAtOffset(offset,  8)))

Response: Returns a C { QWORD } from a given offset.


Instruction:
Here you have a function in Python 3, explain what it does
def getDataAtRva(self, rva, size):
        return self.getDataAtOffset(self.getOffsetFromRva(rva),  size)

Response: Gets the binary data at a given RVA.


Instruction:
Implement a Python 3 function for
getting the binary data at a given offset.

Response:
def getDataAtOffset(self, offset, size):
        """
        Gets binary data at a given offset.

        @type offset: int
        @param offset: The offset to get the data from.

        @type size: int
        @param size: The size of the data to be obtained.

        @rtype: str
        @return: The data obtained at the given offset.
        """
        data = str(self)
        return data[offset:offset+size]


Instruction:
Implement a Python 3 function for
returning a L { String } object from a given RVA.

Response:
def readStringAtRva(self, rva):
        """
        Returns a L{String} object from a given RVA.

        @type rva: int
        @param rva: The RVA to get the string from.

        @rtype: L{String}
        @return: A new L{String} object from the given RVA.
        """
        d = self.getDataAtRva(rva,  1)
        resultStr = datatypes.String("")
        while d != "\x00":
            resultStr.value += d
            rva += 1
            d = self.getDataAtRva(rva, 1)
        return resultStr


Instruction:
Write a Python 3 function that can
determine if the current instance is an executable file.

Response:
def isExe(self):
        """
        Determines if the current L{PE} instance is an Executable file.

        @rtype: bool
        @return: C{True} if the current L{PE} instance is an Executable file. Otherwise, returns C{False}.
        """
        if not self.isDll() and not self.isDriver() and ( consts.IMAGE_FILE_EXECUTABLE_IMAGE & self.ntHeaders.fileHeader.characteristics.value) == consts.IMAGE_FILE_EXECUTABLE_IMAGE:
            return True
        return False


Instruction:
Write a Python 3 function for
determining if the current L { PE } instance is a Dynamic Link Library file.

Response:
def isDll(self):
        """
        Determines if the current L{PE} instance is a Dynamic Link Library file.

        @rtype: bool
        @return: C{True} if the current L{PE} instance is a DLL. Otherwise, returns C{False}.
        """
        if (consts.IMAGE_FILE_DLL & self.ntHeaders.fileHeader.characteristics.value) == consts.IMAGE_FILE_DLL:
            return True
        return False


Instruction:
Can you generate a brief explanation for the following Python 3 code
def isDriver(self):
        modules = []
        imports = self.ntHeaders.optionalHeader.dataDirectory[consts.IMPORT_DIRECTORY].info
        for module in imports:
            modules.append(module.metaData.moduleName.value.lower())

        if set(["ntoskrnl.exe", "hal.dll", "ndis.sys", "bootvid.dll", "kdcom.dll"]).intersection(modules):
            return True
        return False

Response: Determines if the current L { PE } instance is a driver.


Instruction:
Make a summary of the following Python 3 code
def isPe32(self):
        if self.ntHeaders.optionalHeader.magic.value == consts.PE32:
            return True
        return False

Response: Determines if the current instance is a PE32 file.


Instruction:
Can you write a function in Python 3 where it
determines if the current instance is a PE64 file.

Response:
def isPe64(self):
        """
        Determines if the current L{PE} instance is a PE64 file.

        @rtype: bool
        @return: C{True} if the current L{PE} instance is a PE64 file. Otherwise, returns C{False}.
        """
        if self.ntHeaders.optionalHeader.magic.value == consts.PE64:
            return True
        return False


Instruction:
Can you write a function in Python 3 where it
determines if the current instance is bounded i. e. has a C { BED_IMPORT_DIRECTORY }.

Response:
def isPeBounded(self):
        """
        Determines if the current L{PE} instance is bounded, i.e. has a C{BOUND_IMPORT_DIRECTORY}.

        @rtype: bool
        @return: Returns C{True} if the current L{PE} instance is bounded. Otherwise, returns C{False}.
        """
        boundImportsDir = self.ntHeaders.optionalHeader.dataDirectory[consts.BOUND_IMPORT_DIRECTORY]
        if boundImportsDir.rva.value and boundImportsDir.size.value:
            return True
        return False


Instruction:
How would you explain what the following Python 3 function does
def isNXEnabled(self):
        return self.ntHeaders.optionalHeader.dllCharacteristics.value & consts.IMAGE_DLL_CHARACTERISTICS_NX_COMPAT == consts.IMAGE_DLL_CHARACTERISTICS_NX_COMPAT

Response: Determines if the NX characteristics of the current L { PE } instance is set to NXCOMPAT.


Instruction:
Write a Python 3 script to
determine if the current L { PE } instance has the CFG flag enabled.

Response:
def isCFGEnabled(self):
        """
        Determines if the current L{PE} instance has CFG (Control Flow Guard) flag enabled.
        @see: U{http://blogs.msdn.com/b/vcblog/archive/2014/12/08/visual-studio-2015-preview-work-in-progress-security-feature.aspx}
        @see: U{https://msdn.microsoft.com/en-us/library/dn919635%%28v=vs.140%%29.aspx}

        @rtype: bool
        @return: Returns C{True} if the current L{PE} instance has the CFG flag enabled. Otherwise, return C{False}.
        """
        return self.ntHeaders.optionalHeader.dllCharacteristics.value & consts.IMAGE_DLL_CHARACTERISTICS_GUARD_CF == consts.IMAGE_DLL_CHARACTERISTICS_GUARD_CF


Instruction:
Write a Python 3 function for
determining if the ASLR characteristics of the current L{PE} instance is enabled.

Response:
def isASLREnabled(self):
        """
        Determines if the current L{PE} instance has the DYNAMICBASE (Use address space layout randomization) flag enabled.
        @see: U{http://msdn.microsoft.com/en-us/library/bb384887.aspx}

        @rtype: bool
        @return: Returns C{True} if the current L{PE} instance has the DYNAMICBASE flag enabled. Otherwise, returns C{False}.
        """
        return self.ntHeaders.optionalHeader.dllCharacteristics.value & consts.IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE == consts.IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE


Instruction:
Can you tell what is the following Python 3 function doing
def isSAFESEHEnabled(self):
        NOSEH = -1
        SAFESEH_OFF = 0
        SAFESEH_ON = 1

        if self.ntHeaders.optionalHeader.dllCharacteristics.value & consts.IMAGE_DLL_CHARACTERISTICS_NO_SEH:
            return NOSEH

        loadConfigDir = self.ntHeaders.optionalHeader.dataDirectory[consts.CONFIGURATION_DIRECTORY]
        if loadConfigDir.info:
            if loadConfigDir.info.SEHandlerTable.value:
                return SAFESEH_ON
        return SAFESEH_OFF

Response: Determines if the current L { PE } instance has the SAFESEH flag enabled. Returns C { True } if SAFESEH is off or - 1 if SAFESEH is set to NO. Returns C { False } if SAFESEH is set to OFF or - 1 if SAFESEH is set to NO. Returns C { false } if SAFESEH is set to OFF or - 1 if SAFESEH is set to NOSEH is set to NOSEH is set to NO.


Instruction:
How would you explain what the following Python 3 function does
def _parseDirectories(self, dataDirectoryInstance, magic = consts.PE32):
        directories = [(consts.EXPORT_DIRECTORY, self._parseExportDirectory),\
                         (consts.IMPORT_DIRECTORY, self._parseImportDirectory),\
                         (consts.RESOURCE_DIRECTORY, self._parseResourceDirectory),\
                         (consts.EXCEPTION_DIRECTORY, self._parseExceptionDirectory),\
                         (consts.RELOCATION_DIRECTORY, self._parseRelocsDirectory),\
                         (consts.TLS_DIRECTORY, self._parseTlsDirectory),\
                         (consts.DEBUG_DIRECTORY, self._parseDebugDirectory),\
                         (consts.BOUND_IMPORT_DIRECTORY, self._parseBoundImportDirectory),\
                         (consts.DELAY_IMPORT_DIRECTORY, self._parseDelayImportDirectory),\
                         (consts.CONFIGURATION_DIRECTORY, self._parseLoadConfigDirectory),\
                         (consts.NET_METADATA_DIRECTORY, self._parseNetDirectory)]

        for directory in directories:
            dir = dataDirectoryInstance[directory[0]]
            if dir.rva.value and dir.size.value:
                try:
                    dataDirectoryInstance[directory[0]].info = directory[1](dir.rva.value, dir.size.value, magic)
                except Exception as e:
                    print excep.PEWarning("Error parsing PE directory: %s." % directory[1].__name__.replace("_parse", ""))

Response: Parses all the directories in the instance.


Instruction:
Can you generate the documentation for the following Python 3 function
def _parseResourceDirectory(self, rva, size, magic = consts.PE32):
        return self.getDataAtRva(rva, size)

Response: Parses the C { IMAGE_RESOURCE_DIRECTORY } directory.


Instruction:
Given the following Python 3 function, write the documentation
def _parseExceptionDirectory(self, rva, size, magic = consts.PE32):
        return self.getDataAtRva(rva, size)

Response: Parses the C { IMAGE_EXCEPTION_DIRECTORY } directory.


Instruction:
How would you code a function in Python 3 to
parse the delay imports directory.

Response:
def _parseDelayImportDirectory(self, rva, size, magic = consts.PE32):
        """
        Parses the delay imports directory.

        @type rva: int
        @param rva: The RVA where the delay imports directory starts.

        @type size: int
        @param size: The size of the delay imports directory.

        @type magic: int
        @param magic: (Optional) The type of PE. This value could be L{consts.PE32} or L{consts.PE64}.

        @rtype: str
        @return: The delay imports directory data.
        """
        return self.getDataAtRva(rva, size)


Instruction:
Make a summary of the following Python 3 code
def _parseBoundImportDirectory(self, rva, size, magic = consts.PE32):
        data = self.getDataAtRva(rva, size)
        rd = utils.ReadData(data)
        boundImportDirectory = directories.ImageBoundImportDescriptor.parse(rd)

        # parse the name of every bounded import.
        for i in range(len(boundImportDirectory) - 1):
            if hasattr(boundImportDirectory[i],  "forwarderRefsList"):
                if boundImportDirectory[i].forwarderRefsList:
                    for forwarderRefEntry in boundImportDirectory[i].forwarderRefsList:
                        offset = forwarderRefEntry.offsetModuleName.value
                        forwarderRefEntry.moduleName = self.readStringAtRva(offset + rva)

            offset = boundImportDirectory[i].offsetModuleName.value
            boundImportDirectory[i].moduleName = self.readStringAtRva(offset + rva)
        return boundImportDirectory

Response: Parses the bound import directory.


Instruction:
Create a Python 3 function for
parsing IMAGE_LOAD_CONFIG_DIRECTORY and returns a new object.

Response:
def _parseLoadConfigDirectory(self, rva, size, magic = consts.PE32):
        """
        Parses IMAGE_LOAD_CONFIG_DIRECTORY.

        @type rva: int
        @param rva: The RVA where the IMAGE_LOAD_CONFIG_DIRECTORY starts.

        @type size: int
        @param size: The size of the IMAGE_LOAD_CONFIG_DIRECTORY.

        @type magic: int
        @param magic: (Optional) The type of PE. This value could be L{consts.PE32} or L{consts.PE64}.

        @rtype: L{ImageLoadConfigDirectory}
        @return: A new L{ImageLoadConfigDirectory}.
        @note: if the L{PE} instance is a PE64 file then a new L{ImageLoadConfigDirectory64} is returned.
        """
        # print "RVA: %x - SIZE: %x" % (rva, size)

        # I've found some issues when parsing the IMAGE_LOAD_CONFIG_DIRECTORY in some DLLs.
        # There is an inconsistency with the size of the struct between MSDN docs and VS.
        # sizeof(IMAGE_LOAD_CONFIG_DIRECTORY) should be 0x40, in fact, that's the size Visual Studio put
        # in the directory table, even if the DLL was compiled with SAFESEH:ON. But If that is the case, the sizeof the
        # struct should be 0x48.
        # more information here: http://www.accuvant.com/blog/old-meets-new-microsoft-windows-safeseh-incompatibility
        data = self.getDataAtRva(rva, directories.ImageLoadConfigDirectory().sizeof())
        rd = utils.ReadData(data)

        if magic == consts.PE32:
            return directories.ImageLoadConfigDirectory.parse(rd)
        elif magic == consts.PE64:
            return directories.ImageLoadConfigDirectory64.parse(rd)
        else:
            raise excep.InvalidParameterException("Wrong magic")


Instruction:
How would you explain what the following Python 3 function does
def _parseTlsDirectory(self, rva, size, magic = consts.PE32):
        data = self.getDataAtRva(rva, size)
        rd = utils.ReadData(data)

        if magic == consts.PE32:
            return directories.TLSDirectory.parse(rd)
        elif magic == consts.PE64:
            return directories.TLSDirectory64.parse(rd)
        else:
            raise excep.InvalidParameterException("Wrong magic")

Response: Parses the TLS directory.


Instruction:
Can you tell what is the following Python 3 function doing
def _parseRelocsDirectory(self, rva, size, magic = consts.PE32):
        data = self.getDataAtRva(rva,  size)
        #print "Length Relocation data: %x" % len(data)
        rd = utils.ReadData(data)

        relocsArray = directories.ImageBaseRelocation()
        while rd.offset < size:
            relocEntry = directories.ImageBaseRelocationEntry.parse(rd)
            relocsArray.append(relocEntry)
        return relocsArray

Response: Parses the relocation directory.


Instruction:
Can you generate the documentation for the following Python 3 function
def _parseExportDirectory(self, rva, size, magic = consts.PE32):
        data = self.getDataAtRva(rva,  size)
        rd = utils.ReadData(data)

        iet = directories.ImageExportTable.parse(rd)

        auxFunctionRvaArray = list()

        numberOfNames = iet.numberOfNames.value
        addressOfNames = iet.addressOfNames.value
        addressOfNameOrdinals = iet.addressOfNameOrdinals.value
        addressOfFunctions = iet.addressOfFunctions.value

        # populate the auxFunctionRvaArray
        for i in xrange(iet.numberOfFunctions.value):
            auxFunctionRvaArray.append(self.getDwordAtRva(addressOfFunctions).value)
            addressOfFunctions += datatypes.DWORD().sizeof()

        for i in xrange(numberOfNames):

            nameRva = self.getDwordAtRva(addressOfNames).value
            nameOrdinal = self.getWordAtRva(addressOfNameOrdinals).value
            exportName = self.readStringAtRva(nameRva).value

            entry = directories.ExportTableEntry()

            ordinal = nameOrdinal + iet.base.value
            #print "Ordinal value: %d" % ordinal
            entry.ordinal.value = ordinal

            entry.nameOrdinal.vaue = nameOrdinal
            entry.nameRva.value = nameRva
            entry.name.value = exportName
            entry.functionRva.value = auxFunctionRvaArray[nameOrdinal]

            iet.exportTable.append(entry)

            addressOfNames += datatypes.DWORD().sizeof()
            addressOfNameOrdinals += datatypes.WORD().sizeof()

        #print "export table length: %d" % len(iet.exportTable)

        #print "auxFunctionRvaArray: %r" % auxFunctionRvaArray
        for i in xrange(iet.numberOfFunctions.value):
            #print "auxFunctionRvaArray[%d]: %x" % (i,  auxFunctionRvaArray[i])
            if auxFunctionRvaArray[i] != iet.exportTable[i].functionRva.value:
                entry = directories.ExportTableEntry()

                entry.functionRva.value = auxFunctionRvaArray[i]
                entry.ordinal.value = iet.base.value + i

                iet.exportTable.append(entry)

        #print "export table length: %d" % len(iet.exportTable)
        sorted(iet.exportTable, key=lambda entry:entry.ordinal)
        return iet

Response: Parses the C { IMAGE_EXPORT_DIRECTORY } directory and returns a new object.


Instruction:
How would you code a function in Python 3 to
parse the C { IMAGE_DEBUG_DIRECTORY } directory.

Response:
def _parseDebugDirectory(self, rva, size, magic = consts.PE32):
        """
        Parses the C{IMAGE_DEBUG_DIRECTORY} directory.
        @see: U{http://msdn.microsoft.com/es-es/library/windows/desktop/ms680307(v=vs.85).aspx}

        @type rva: int
        @param rva: The RVA where the C{IMAGE_DEBUG_DIRECTORY} directory starts.

        @type size: int
        @param size: The size of the C{IMAGE_DEBUG_DIRECTORY} directory.

        @type magic: int
        @param magic: (Optional) The type of PE. This value could be L{consts.PE32} or L{consts.PE64}.

        @rtype: L{ImageDebugDirectory}
        @return: A new L{ImageDebugDirectory} object.
        """
        debugDirData = self.getDataAtRva(rva, size)
        numberOfEntries = size / consts.SIZEOF_IMAGE_DEBUG_ENTRY32
        rd = utils.ReadData(debugDirData)
        return directories.ImageDebugDirectories.parse(rd,  numberOfEntries)


Instruction:
Implement a Python 3 function for
parsing the C { IMAGE_IMPORT_DIRECTORY } directory and returns a new object.

Response:
def _parseImportDirectory(self, rva, size, magic = consts.PE32):
        """
        Parses the C{IMAGE_IMPORT_DIRECTORY} directory.

        @type rva: int
        @param rva: The RVA where the C{IMAGE_IMPORT_DIRECTORY} directory starts.

        @type size: int
        @param size: The size of the C{IMAGE_IMPORT_DIRECTORY} directory.

        @type magic: int
        @param magic: (Optional) The type of PE. This value could be L{consts.PE32} or L{consts.PE64}.

        @rtype: L{ImageImportDescriptor}
        @return: A new L{ImageImportDescriptor} object.

        @raise InvalidParameterException: If wrong magic was specified.
        """
        #print "RVA: %x - Size: %x" % (rva, size)
        importsDirData = self.getDataAtRva(rva,  size)
        #print "Length importsDirData: %d" % len(importsDirData)
        numberOfEntries = size / consts.SIZEOF_IMAGE_IMPORT_ENTRY32
        rd = utils.ReadData(importsDirData)

        # In .NET binaries, the size of the data directory corresponding to the import table
        # is greater than the number of bytes in the file. Thats why we check for the last group of 5 null bytes
        # that indicates the end of the IMAGE_IMPORT_DESCRIPTOR array.
        rdAux = utils.ReadData(importsDirData)
        count = 0
        entry = rdAux.read(consts.SIZEOF_IMAGE_IMPORT_ENTRY32)
        while rdAux.offset < len(rdAux.data) and not utils.allZero(entry):
            try:
                entry = rdAux.read(consts.SIZEOF_IMAGE_IMPORT_ENTRY32)
                count += 1
            except excep.DataLengthException:
                if self._verbose:
                    print "[!] Warning: DataLengthException detected!."

        if numberOfEntries - 1 > count:
            numberOfEntries = count + 1

        iid = directories.ImageImportDescriptor.parse(rd, numberOfEntries)
        iidLength = len(iid)

        peIsBounded = self.isPeBounded()

        if magic == consts.PE64:
            ORDINAL_FLAG = consts.IMAGE_ORDINAL_FLAG64
            ADDRESS_MASK = consts.ADDRESS_MASK64
        elif magic == consts.PE32:
            ORDINAL_FLAG = consts.IMAGE_ORDINAL_FLAG
            ADDRESS_MASK = consts.ADDRESS_MASK32
        else:
            raise InvalidParameterException("magic value %d is not PE64 nor PE32." % magic)

        for i in range(iidLength -1):
            if iid[i].originalFirstThunk.value != 0:
                iltRva = iid[i].originalFirstThunk.value
                iatRva = iid[i].firstThunk.value

                if magic == consts.PE64:
                    entry = self.getQwordAtRva(iltRva).value
                elif magic == consts.PE32:
                    entry = self.getDwordAtRva(iltRva).value

                while entry != 0:

                    if magic == consts.PE64:
                        iatEntry = directories.ImportAddressTableEntry64()
                    elif magic == consts.PE32:
                        iatEntry = directories.ImportAddressTableEntry()

                    iatEntry.originalFirstThunk.value = entry

                    if iatEntry.originalFirstThunk.value & ORDINAL_FLAG:
                        iatEntry.hint.value = None
                        iatEntry.name.value = iatEntry.originalFirstThunk.value & ADDRESS_MASK
                    else:
                        iatEntry.hint.value = self.getWordAtRva(iatEntry.originalFirstThunk.value).value
                        iatEntry.name.value = self.readStringAtRva(iatEntry.originalFirstThunk.value + 2).value

                    if magic == consts.PE64:
                        iatEntry.firstThunk.value = self.getQwordAtRva(iatRva).value
                        iltRva += 8
                        iatRva += 8
                        entry = self.getQwordAtRva(iltRva).value
                    elif magic == consts.PE32:
                        iatEntry.firstThunk.value = self.getDwordAtRva(iatRva).value
                        iltRva += 4
                        iatRva += 4
                        entry = self.getDwordAtRva(iltRva).value

                    iid[i].iat.append(iatEntry)

            else:
                iatRva = iid[i].firstThunk.value

                if magic == consts.PE64:
                    entry = self.getQwordAtRva(iatRva).value
                elif magic == consts.PE32:
                    entry = self.getDwordAtRva(iatRva).value

                while entry != 0:

                    if magic == consts.PE64:
                        iatEntry = directories.ImportAddressTableEntry64()
                    elif magic == consts.PE32:
                        iatEntry = directories.ImportAddressTableEntry()

                    iatEntry.firstThunk.value = entry
                    iatEntry.originalFirstThunk.value = 0

                    if not peIsBounded:
                        ft = iatEntry.firstThunk.value

                        if ft & ORDINAL_FLAG:
                            iatEntry.hint.value = None
                            iatEntry.name.value = ft & ADDRESS_MASK
                        else:
                            iatEntry.hint.value = self.getWordAtRva(ft).value
                            iatEntry.name.value = self.readStringAtRva(ft + 2).value
                    else:
                        iatEntry.hint.value = None
                        iatEntry.name.value = None

                    if magic == consts.PE64:
                        iatRva += 8
                        entry = self.getQwordAtRva(iatRva).value
                    elif magic == consts.PE32:
                        iatRva += 4
                        entry = self.getDwordAtRva(iatRva).value

                    iid[i].iat.append(iatEntry)

            iid[i].metaData.moduleName.value = self.readStringAtRva(iid[i].name.value).value
            iid[i].metaData.numberOfImports.value = len(iid[i].iat)
        return iid


Instruction:
How would you implement a function in Python 3 that
parses the. NET Directory and returns a new object containing the properties of the NET Directory and the attributes of the NET Directory.

Response:
def _parseNetDirectory(self, rva, size, magic = consts.PE32):
        """
        Parses the NET directory.
        @see: U{http://www.ntcore.com/files/dotnetformat.htm}

        @type rva: int
        @param rva: The RVA where the NET directory starts.

        @type size: int
        @param size: The size of the NET directory.

        @type magic: int
        @param magic: (Optional) The type of PE. This value could be L{consts.PE32} or L{consts.PE64}.

        @rtype: L{NETDirectory}
        @return: A new L{NETDirectory} object.
        """
        if not rva or not size:
            return None

        # create a NETDirectory class to hold the data
        netDirectoryClass = directories.NETDirectory()

        # parse the .NET Directory
        netDir = directories.NetDirectory.parse(utils.ReadData(self.getDataAtRva(rva,  size)))

        netDirectoryClass.directory = netDir

        # get the MetaData RVA and Size
        mdhRva = netDir.metaData.rva.value
        mdhSize = netDir.metaData.size.value

        # read all the MetaData
        rd = utils.ReadData(self.getDataAtRva(mdhRva, mdhSize))

        # parse the MetaData headers
        netDirectoryClass.netMetaDataHeader = directories.NetMetaDataHeader.parse(rd)

        # parse the NET metadata streams
        numberOfStreams = netDirectoryClass.netMetaDataHeader.numberOfStreams.value
        netDirectoryClass.netMetaDataStreams = directories.NetMetaDataStreams.parse(rd, numberOfStreams)

        for i in range(numberOfStreams):
            stream = netDirectoryClass.netMetaDataStreams[i]
            name = stream.name.value
            rd.setOffset(stream.offset.value)
            rd2 = utils.ReadData(rd.read(stream.size.value))
            stream.info = []
            if name == "#~" or i == 0:
                stream.info = rd2
            elif name == "#Strings" or i == 1:
                while len(rd2) > 0:
                    offset = rd2.tell()
                    stream.info.append({ offset: rd2.readDotNetString() })
            elif name == "#US" or i == 2:
                while len(rd2) > 0:
                    offset = rd2.tell()
                    stream.info.append({ offset: rd2.readDotNetUnicodeString() })
            elif name == "#GUID" or i == 3:
                while len(rd2) > 0:
                    offset = rd2.tell()
                    stream.info.append({ offset: rd2.readDotNetGuid() })
            elif name == "#Blob" or i == 4:
                while len(rd2) > 0:
                    offset = rd2.tell()
                    stream.info.append({ offset: rd2.readDotNetBlob() })

        for i in range(numberOfStreams):
            stream = netDirectoryClass.netMetaDataStreams[i]
            name = stream.name.value
            if name == "#~" or i == 0:
                stream.info = directories.NetMetaDataTables.parse(stream.info, netDirectoryClass.netMetaDataStreams)

        # parse .NET resources
        # get the Resources RVA and Size
        resRva = netDir.resources.rva.value
        resSize = netDir.resources.size.value

        # read all the MetaData
        rd = utils.ReadData(self.getDataAtRva(resRva, resSize))

        resources = []

        for i in netDirectoryClass.netMetaDataStreams[0].info.tables["ManifestResource"]:
            offset = i["offset"]
            rd.setOffset(offset)
            size = rd.readDword()
            data = rd.read(size)
            if data[:4] == "\xce\xca\xef\xbe":
                data = directories.NetResources.parse(utils.ReadData(data))
            resources.append({ "name": i["name"], "offset": offset + 4, "size": size, "data": data })

        netDirectoryClass.directory.resources.info = resources

        return netDirectoryClass


Instruction:
Can you implement a function in Python 3 that
parses the contents of a DOS header and returns a new object.

Response:
def parse(readDataInstance):
        """
        Returns a new L{DosHeader} object.

        @type readDataInstance: L{ReadData}
        @param readDataInstance: A L{ReadData} object with data to be parsed as a L{DosHeader} object.

        @rtype: L{DosHeader}
        @return: A new L{DosHeader} object.
        """
        dosHdr = DosHeader()

        dosHdr.e_magic.value  = readDataInstance.readWord()
        dosHdr.e_cblp.value  = readDataInstance.readWord()
        dosHdr.e_cp.value  = readDataInstance.readWord()
        dosHdr.e_crlc.value  = readDataInstance.readWord()
        dosHdr.e_cparhdr.value  = readDataInstance.readWord()
        dosHdr.e_minalloc.value  = readDataInstance.readWord()
        dosHdr.e_maxalloc.value  = readDataInstance.readWord()
        dosHdr.e_ss.value  = readDataInstance.readWord()
        dosHdr.e_sp.value  = readDataInstance.readWord()
        dosHdr.e_csum.value  = readDataInstance.readWord()
        dosHdr.e_ip.value  = readDataInstance.readWord()
        dosHdr.e_cs.value  = readDataInstance.readWord()
        dosHdr.e_lfarlc.value  = readDataInstance.readWord()
        dosHdr.e_ovno.value  = readDataInstance.readWord()

        dosHdr.e_res = datatypes.Array(datatypes.TYPE_WORD)
        for i in range(4):
            dosHdr.e_res.append(datatypes.WORD(readDataInstance.readWord()))

        dosHdr.e_oemid.value  = readDataInstance.readWord()
        dosHdr.e_oeminfo.value  = readDataInstance.readWord()

        dosHdr.e_res2 = datatypes.Array(datatypes.TYPE_WORD)
        for i in range (10):
            dosHdr.e_res2.append(datatypes.WORD(readDataInstance.readWord()))

        dosHdr.e_lfanew.value = readDataInstance.readDword()
        return dosHdr