Instruction:
Can you generate a brief explanation for the following Python 3 code
def init_pop(self):
        pop = Pop(self.population_size)
        seed_with_raw_features = False
        # make programs
        if self.seed_with_ml:
            # initial population is the components of the default ml model
            if (self.ml_type == 'SVC' or self.ml_type == 'SVR'):
                # this is needed because svm has a bug that throws valueerror
                #on attribute check
                seed_with_raw_features=True
            elif (hasattr(self.pipeline.named_steps['ml'],'coef_') or
                  hasattr(self.pipeline.named_steps['ml'],'feature_importances_')):
                # add model components with non-zero coefficients to initial
                # population, in order of coefficient size
                coef = (self.pipeline.named_steps['ml'].coef_ if
                        hasattr(self.pipeline.named_steps['ml'],'coef_') else
                        self.pipeline.named_steps['ml'].feature_importances_)
                # compress multiple coefficients for each feature into single
                # numbers (occurs with multiclass classification)
                if len(coef.shape)>1:
                    coef = [np.mean(abs(c)) for c in coef.transpose()]

                # remove zeros
                coef = [c for c in coef if c!=0]
                # sort feature locations based on importance/coefficient
                locs = np.arange(len(coef))
                locs = locs[np.argsort(np.abs(coef))[::-1]]
                for i,p in enumerate(pop.individuals):
                    if i < len(locs):
                        p.stack = [node('x',loc=locs[i])]
                    else:
                        # make program if pop is bigger than n_features
                        self.make_program(p.stack,self.func_set,self.term_set,
                                     self.random_state.randint(self.min_depth,
                                                       self.max_depth+1),
                                     self.otype)
                        p.stack = list(reversed(p.stack))
            else:
                seed_with_raw_features = True
            # seed with random features if no importance info available
            if seed_with_raw_features:
                for i,p in enumerate(pop.individuals):
                    if i < self.n_features:
                        p.stack = [node('x',
                                        loc=self.random_state.randint(self.n_features))]
                    else:
                        # make program if pop is bigger than n_features
                        self.make_program(p.stack,self.func_set,self.term_set,
                                     self.random_state.randint(self.min_depth,
                                                       self.max_depth+1),
                                     self.otype)
                        p.stack = list(reversed(p.stack))

            # print initial population
            if self.verbosity > 2:
                print("seeded initial population:",
                      self.stacks_2_eqns(pop.individuals))

        else: # don't seed with ML
            for I in pop.individuals:
                depth = self.random_state.randint(self.min_depth,self.max_depth_init)
                self.make_program(I.stack,self.func_set,self.term_set,depth,
                             self.otype)
                #print(I.stack)
                I.stack = list(reversed(I.stack))

        return pop

Response: initializes population of features as GP stacks.


Instruction:
Can you tell what is the following Python 3 function doing
def main():
    parser = argparse.ArgumentParser(description='A feature engineering wrapper'
                                     ' for machine learning algorithms.',
                                     add_help=False)

    parser.add_argument('INPUT_FILE', type=str,
                        help='Data file to run FEW on; ensure that the '
                        'target/label column is labeled as "label" or "class".')

    parser.add_argument('-h', '--help', action='help',
                        help='Show this help message and exit.')

    parser.add_argument('-is', action='store',dest='INPUT_SEPARATOR',
                        default=None,type=str,
                        help='Character separating columns in the input file.')

    parser.add_argument('-o', action='store', dest='OUTPUT_FILE', default='',
                        type=str, help='File to export the final model.')

    parser.add_argument('-g', action='store', dest='GENERATIONS', default=100,
                        type=positive_integer,
                        help='Number of generations to run FEW.')

    parser.add_argument('-p', action='store', dest='POPULATION_SIZE',default=50,
                         help='Number of individuals in the GP population. '
                         'Follow the number with x to set population size as a'
                         'multiple of raw feature size.')

    parser.add_argument('-mr', action='store', dest='MUTATION_RATE',default=0.5,
                        type=float_range,
                        help='GP mutation rate in the range [0.0, 1.0].')

    parser.add_argument('-xr', action='store', dest='CROSSOVER_RATE',
                        default=0.5,type=float_range,
                        help='GP crossover rate in the range [0.0, 1.0].')

    parser.add_argument('-ml', action='store', dest='MACHINE_LEARNER',
                        default=None,
                        choices = ['lasso','svr','lsvr','lr','svc','rfc','rfr',
                                   'dtc','dtr','dc','knc','knr','sgd'],
                        type=str, help='ML algorithm to pair with features. '
                        'Default: Lasso (regression), LogisticRegression '
                        '(classification)')

    parser.add_argument('-min_depth', action='store', dest='MIN_DEPTH',
                        default=1,type=positive_integer,
                        help='Minimum length of GP programs.')

    parser.add_argument('-max_depth', action='store', dest='MAX_DEPTH',
                        default=2,type=positive_integer,
                        help='Maximum number of nodes in GP programs.')

    parser.add_argument('-max_depth_init', action='store',dest='MAX_DEPTH_INIT',
                        default=2,type=positive_integer,
                        help='Maximum nodes in initial programs.')

    parser.add_argument('-op_weight', action='store',dest='OP_WEIGHT',default=1,
                        type=bool, help='Weight attributes for incuded in'
                        ' features based on ML scores. Default: off')

    parser.add_argument('-ms', action='store', dest='MAX_STALL',default=100,
                        type=positive_integer, help='If model CV does not '
                        'improve for this many generations, end optimization.')

    parser.add_argument('--weight_parents', action='store_true',
                        dest='WEIGHT_PARENTS',default=True,
                        help='Feature importance weights parent selection.')

    parser.add_argument('--lex_size', action='store_true',dest='LEX_SIZE',default=False,
                        help='Size mediated parent selection for lexicase survival.')

    parser.add_argument('-sel', action='store', dest='SEL',
                        default='epsilon_lexicase',
                        choices = ['tournament','lexicase','epsilon_lexicase',
                                   'deterministic_crowding','random'],
                        type=str, help='Selection method (Default: tournament)')

    parser.add_argument('-tourn_size', action='store', dest='TOURN_SIZE',
                        default=2, type=positive_integer,
                        help='Tournament size (Default: 2)')

    parser.add_argument('-fit', action='store', dest='FIT_CHOICE', default=None,
                        choices = ['mse','mae','r2','vaf','mse_rel','mae_rel',
                                   'r2_rel','vaf_rel','silhouette','inertia',
                                   'separation','fisher','random','relief'],
                        type=str,
                        help='Fitness metric (Default: dependent on ml used)')

    parser.add_argument('--no_seed', action='store_false', dest='SEED_WITH_ML',
                        default=True,
                        help='Turn off initial GP population seeding.')

    parser.add_argument('--elitism', action='store_true', dest='ELITISM',
                        default=False,
                        help='Force survival of best feature in GP population.')

    parser.add_argument('--erc', action='store_true', dest='ERC', default=False,
                    help='Use random constants in GP feature construction.')

    parser.add_argument('--bool', action='store_true', dest='BOOLEAN',
                        default=False,
                        help='Include boolean operators in features.')

    parser.add_argument('-otype', action='store', dest='OTYPE', default='f',
                        choices=['f','b'],
                        type=str,
                        help='Feature output type. f: float, b: boolean.')

    parser.add_argument('-ops', action='store', dest='OPS', default=None,
                        type=str,
                        help='Specify operators separated by commas')

    parser.add_argument('-dtypes', action='store', dest='DTYPES', default=None,
                        type=str,
                        help='Specify datafile types separated by a comma')

    parser.add_argument('--class', action='store_true', dest='CLASSIFICATION',
                        default=False,
                        help='Conduct classification rather than regression.')

    parser.add_argument('--mdr', action='store_true',dest='MDR',default=False,
                        help='Use MDR nodes.')

    parser.add_argument('--nonorm', action='store_false',dest='NORMALIZE',default=True,
                        help='Disable standard scaler preprocessor.')

    parser.add_argument('--diversity', action='store_true',
                        dest='TRACK_DIVERSITY', default=False,
                        help='Store diversity of feature transforms each gen.')

    parser.add_argument('--clean', action='store_true', dest='CLEAN',
                        default=False,
                        help='Clean input data of missing values.')

    parser.add_argument('--no_lib', action='store_false', dest='c',
                        default=True,
                        help='Don''t use optimized c libraries.')

    parser.add_argument('-s', action='store', dest='RANDOM_STATE',
                        default=None,
                        type=int,
                        help='Random number generator seed for reproducibility.'
                        'Note that using multi-threading may make exact results'
                        ' impossible to reproduce.')

    parser.add_argument('-v', action='store', dest='VERBOSITY', default=1,
                        choices=[0, 1, 2, 3], type=int,
                        help='How much information FEW communicates while it is'
                        ' running: 0 = none, 1 = minimal, 2 = lots, 3 = all.')

    parser.add_argument('--no-update-check', action='store_true',
                        dest='DISABLE_UPDATE_CHECK', default=False,
                        help='Don''t check the FEW version.')

    parser.add_argument('--version', action='version',
                        version='FEW {version}'.format(version=__version__),
                        help='Show FEW\'s version number and exit.')

    args = parser.parse_args()

    # if args.VERBOSITY >= 2:
    #     print('\nFEW settings:')
    #     for arg in sorted(args.__dict__):
    #         if arg == 'DISABLE_UPDATE_CHECK':
    #             continue
    #         print('{}\t=\t{}'.format(arg, args.__dict__[arg]))
    #     print('')

    # load data from csv file
    if args.INPUT_SEPARATOR is None:
        input_data = pd.read_csv(args.INPUT_FILE, sep=args.INPUT_SEPARATOR,
                                 engine='python')
    else: # use c engine for read_csv is separator is specified
        input_data = pd.read_csv(args.INPUT_FILE, sep=args.INPUT_SEPARATOR)

    # if 'Label' in input_data.columns.values:
    input_data.rename(columns={'Label': 'label','Class':'label','class':'label',
                               'target':'label'}, inplace=True)

    RANDOM_STATE = args.RANDOM_STATE

    train_i, test_i = train_test_split(input_data.index,
                                       stratify = None,
                                       #stratify=input_data['label'].values,
                                       train_size=0.75,
                                       test_size=0.25,
                                       random_state=RANDOM_STATE)

    training_features = input_data.loc[train_i].drop('label', axis=1).values
    training_labels = input_data.loc[train_i, 'label'].values

    testing_features = input_data.loc[test_i].drop('label', axis=1).values
    testing_labels = input_data.loc[test_i, 'label'].values

    learner = FEW(generations=args.GENERATIONS,
                  population_size=args.POPULATION_SIZE,
                  mutation_rate=args.MUTATION_RATE,
                  crossover_rate=args.CROSSOVER_RATE,
                  ml = ml_dict[args.MACHINE_LEARNER],
                  min_depth = args.MIN_DEPTH,max_depth = args.MAX_DEPTH,
                  sel = args.SEL, tourn_size = args.TOURN_SIZE,
                  seed_with_ml = args.SEED_WITH_ML, op_weight = args.OP_WEIGHT,
                  max_stall = args.MAX_STALL,
                  erc = args.ERC, random_state=args.RANDOM_STATE,
                  verbosity=args.VERBOSITY,
                  disable_update_check=args.DISABLE_UPDATE_CHECK,
                  fit_choice = args.FIT_CHOICE,boolean=args.BOOLEAN,
                  classification=args.CLASSIFICATION,clean = args.CLEAN,
                  track_diversity=args.TRACK_DIVERSITY,mdr=args.MDR,
                  otype=args.OTYPE,c=args.c, lex_size = args.LEX_SIZE,
                  weight_parents = args.WEIGHT_PARENTS,operators=args.OPS,
                  normalize=args.NORMALIZE, dtypes = args.DTYPES)

    learner.fit(training_features, training_labels)
    # pdb.set_trace()
    if args.VERBOSITY >= 1:
        print('\nTraining accuracy: {:1.3f}'.format(
            learner.score(training_features, training_labels)))
        print('Test accuracy: {:1.3f}'.format(
            learner.score(testing_features, testing_labels)))

    if args.OUTPUT_FILE != '':
        learner.export(args.OUTPUT_FILE)

Response: This function is called by the command line to run the FEW.


Instruction:
How would you explain what the following Python 3 function does
def fit(self, features, labels):

        # setup data
        # imputation
        if self.clean:
            features = self.impute_data(features)
        # save the number of features
        self.n_features = features.shape[1]
        self.n_samples = features.shape[0]
        # set population size
        if type(self.population_size) is str:
            if 'x' in self.population_size: # set pop size prop to features
                self.population_size = int(
                        float(self.population_size[:-1])*features.shape[1])
            else:
                self.population_size = int(self.population_size)

        if self.verbosity >0: print("population size:",self.population_size)

        # re-initialize pipeline (needs to be here rather than init for GridSearchCV)
        if self.normalize:
            self.pipeline = Pipeline([('standardScaler',StandardScaler()), ('ml', self.ml)])
        else:
            self.pipeline = Pipeline([('ml',self.ml)])

        # set variable names if they haven't been set
        if self.names is None:
            self.names = ['x_'+str(i) for i in np.arange(features.shape[1])]
        # set variable data types if they haven't been set
        if self.dtypes is None:
            self.dtypes = ['f' for i in np.arange(features.shape[1])]

        # create terminal set
        for i in np.arange(self.n_features):
            self.term_set.append(node('x',loc=i,otype=self.dtypes[i])) # features
            # add ephemeral random constants if flag
            if self.erc: # ephemeral random constants
                self.term_set.append(node('k',value=self.random_state.rand()))

        # edit function set if boolean
        if self.boolean or self.otype=='b': # include boolean functions
            self.func_set += [node('!'), node('&'), node('|'), node('=='),
                        node('>_f'), node('<_f'), node('>=_f'), node('<=_f'),
                        node('>_b'), node('<_b'), node('>=_b'), node('<=_b'),
                        node('xor_b'), node('xor_f')]

        # add mdr if specified
        if self.mdr:
            self.func_set += [node('mdr2')]

        # print few settings
        if self.verbosity > 1:
            for arg in self.get_params():
                print('{}\t=\t{}'.format(arg, self.get_params()[arg]))
            print('')

        ######################################################### initial model
        # fit to original data
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            if self.scoring_function == roc_auc_score:
                self._best_score = self.roc_auc_cv(features,labels)
            else:
                self._best_score = np.mean(
                                   [self.scoring_function(labels[test],
                                       self.pipeline.fit(features[train],labels[train]).
                                                     predict(features[test]))
                                   for train, test in KFold().split(features,
                                                                     labels)])

        initial_score = self._best_score
        if self.verbosity > 0:
            print("initial ML CV: {:1.3f}".format(self._best_score))

        ############################################# Create initial population
        # for now, force seed_with_ml to be off if otype is 'b', since data
        # types are assumed to be float
        if self.otype=='b':
            self.seed_with_ml = False
        self.pop = self.init_pop()
        # check that uuids are unique in population
        uuids = [p.id for p in self.pop.individuals]
        if len(uuids) != len(set(uuids)):
            pdb.set_trace()
        # Evaluate the entire population
        # X represents a matrix of the population outputs (number of samples x
        # population size)
        # single thread
        self.X = self.transform(features,self.pop.individuals,labels).transpose()
        # pdb.set_trace()
        # parallel:
        # X = np.asarray(Parallel(n_jobs=-1)(
        # delayed(out)(I,features,self.otype,labels) for I in self.pop.individuals),
        # order = 'F')

        # calculate fitness of individuals
        # fitnesses = list(map(lambda I: fitness(I,labels,self.pipeline),X))
        self.F = self.calc_fitness(self.X,labels,self.fit_choice,self.sel)
        #pdb.set_trace()
        #with Parallel(n_jobs=10) as parallel:
        ####################

        self.diversity=[]
        # progress bar
        pbar = tqdm(total=self.generations,disable = self.verbosity==0,
                    desc='Internal CV: {:1.3f}'.format(self._best_score))
        stall_count = 0
        ########################################################### main GP loop
        for g in np.arange(self.generations):
            if stall_count == self.max_stall:
                if self.verbosity > 0: print('max stall count reached.')
                break

            if self.track_diversity:
                self.get_diversity(self.X)

            # mid verbosity printouts
            if self.verbosity > 1:

                print("generation", str(g))

                print("median fitness pop: %0.2f" % np.median(
                        [np.mean(f) for f in self.F]))

                print("best fitness pop: %0.2f" % np.min(
                    [np.mean(f) for f in self.F]))
                if self.track_diversity:
                    print("feature diversity: %0.2f" % self.diversity[-1])

            # high verbosity printouts
            if self.verbosity > 2:
                eqns = self.stacks_2_eqns(self.pop.individuals)
                fs = [np.mean(f) for f in self.F]
                print("population:",[("%0.2f" % f, eqns[i]) for f,i in
                                     zip(np.sort(fs), np.argsort(fs))])
                #print("pop fitnesses:", ["%0.2f" % np.mean(f) for f in self.F])

            ####################################################### fit ml model
            if self.verbosity > 1: print("ml fitting...")

            tmp_score=0
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                try:
                    if self.valid_loc():
                        if self.scoring_function == roc_auc_score:
                            tmp_score =  self.roc_auc_cv(self.X[self.valid_loc(),:].transpose(),
                                                         labels)
                        else:
                            tmp_score =  np.mean(
                                [self.scoring_function(labels[test], self.pipeline.fit(
                                self.X[self.valid_loc(),:].transpose()[train], labels[train]).
                                predict(self.X[self.valid_loc(),:].transpose()[test]))
                                    for train, test in KFold().split(features,
                                                                     labels)])

                except ValueError as detail:
                    print("warning: ValueError in ml fit. X.shape:",
                          self.X[:,self.valid_loc()].transpose().shape,
                          "labels shape:",labels.shape)
                    print("First ten entries X:",
                          self.X[self.valid_loc(),:].transpose()[:10])
                    print("First ten entries labels:",labels[:10])
                    print("equations:",self.stacks_2_eqns(self.pop.individuals))
                    print("FEW parameters:",self.get_params())
                    print("---\ndetailed error message:",
                                                 detail)
                    raise(detail)

            if self.verbosity > 1:
                print("current ml validation score:",tmp_score)

            #################################################### save best model
            if self.valid_loc() and tmp_score > self._best_score:
                self._best_estimator = copy.deepcopy(self.pipeline)
                self._best_score = tmp_score
                stall_count = 0;
                self._best_inds = copy.deepcopy(self.valid())
                if self.verbosity > 1:
                    print("updated best internal CV:",self._best_score)
            else:
                stall_count = stall_count + 1

            ########################################################## variation
            if self.verbosity > 2:
                print("variation...")
            offspring,elite,elite_index = self.variation(self.pop.individuals)

            ################################################# evaluate offspring
            if self.verbosity > 2:
                print("output...")
            X_offspring = self.transform(features,offspring).transpose()

            if self.verbosity > 2:
                print("fitness...")
            F_offspring = self.calc_fitness(X_offspring,
                                            labels,self.fit_choice,self.sel)

            ########################################################### survival
            if self.verbosity > 2: print("survival..")
            survivors,survivor_index = self.survival(self.pop.individuals, offspring,
                                                     elite, elite_index,
                                                     X = self.X, X_O=X_offspring,
                                                     F=self.F, F_O=F_offspring)
            # set survivors
            self.pop.individuals[:] = survivors
            self.X = np.vstack((self.X, X_offspring))[survivor_index]
            if 'lexicase' in self.sel:
                self.F = np.asarray(
                        np.vstack((self.F, F_offspring))[survivor_index],
                        order='F')
            else:
                self.F = np.asarray(
                        np.hstack((self.F,F_offspring))[survivor_index],
                        order='F')

            if self.verbosity > 2:
                print("median fitness survivors: %0.2f" % np.median(
                        [np.mean(f) for f in self.F]))
            if self.verbosity>2:
                print("best features:",
                      self.stacks_2_eqns(self._best_inds) if self._best_inds
                      else 'original')
            pbar.set_description('Internal CV: {:1.3f}'.format(self._best_score))
            pbar.update(1)
        # end of main GP loop
            ####################
        if self.verbosity > 0: print('finished. best internal val score:'
                                     ' {:1.3f}'.format(self._best_score))
        if self.verbosity > 0: print("final model:\n",self.print_model())

        if not self._best_estimator:
            # if no better model found, just return underlying method fit to the
            # training data
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                self._best_estimator = self.pipeline.fit(features,labels)
        else:
            # fit final estimator to all the training data
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                self._best_estimator.fit(self.transform(features),labels)
        return self

Response: Fit the model to data


Instruction:
How would you code a function in Python 3 to
return a transformation of x using population outputs

Response:
def transform(self,x,inds=None,labels = None):
        """return a transformation of x using population outputs"""
        if inds:
            # return np.asarray(Parallel(n_jobs=10)(delayed(self.out)(I,x,labels,self.otype)
            #                           for I in inds)).transpose()
            return np.asarray(
                [self.out(I,x,labels,self.otype) for I in inds]).transpose()
        elif self._best_inds:
            # return np.asarray(Parallel(n_jobs=10)(delayed(self.out)(I,x,labels,self.otype)
            #                                   for I in self._best_inds)).transpose()
            return np.asarray(
                [self.out(I,x,labels,self.otype) for I in self._best_inds]).transpose()
        else:
            return x


Instruction:
Can you create a Python 3 function that
imputes data set containing Nan values

Response:
def impute_data(self,x):
        """Imputes data set containing Nan values"""
        imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
        return imp.fit_transform(x)


Instruction:
Create a Python 3 function to
remove nan and inf rows from x

Response:
def clean(self,x):
        """remove nan and inf rows from x"""
        return x[~np.any(np.isnan(x) | np.isinf(x),axis=1)]


Instruction:
Can you implement a function in Python 3 that
sets nan and inf rows from x to zero

Response:
def clean_with_zeros(self,x):
        """ set nan and inf rows from x to zero"""
        x[~np.any(np.isnan(x) | np.isinf(x),axis=1)] = 0
        return x


Instruction:
Write a Python 3 script to
predict on a holdout data set.

Response:
def predict(self, testing_features):
        """predict on a holdout data set."""
        # print("best_inds:",self._best_inds)
        # print("best estimator size:",self._best_estimator.coef_.shape)
        if self.clean:
            testing_features = self.impute_data(testing_features)

        if self._best_inds:
            X_transform = self.transform(testing_features)
            try:
                return self._best_estimator.predict(self.transform(testing_features))
            except ValueError as detail:
                # pdb.set_trace()
                print('shape of X:',testing_features.shape)
                print('shape of X_transform:',X_transform.transpose().shape)
                print('best inds:',self.stacks_2_eqns(self._best_inds))
                print('valid locs:',self.valid_loc(self._best_inds))
                raise ValueError(detail)
        else:
            return self._best_estimator.predict(testing_features)


Instruction:
Can you implement a function in Python 3 that
estimates accuracy on testing set

Response:
def score(self, testing_features, testing_labels):
        """estimates accuracy on testing set"""
        # print("test features shape:",testing_features.shape)
        # print("testing labels shape:",testing_labels.shape)
        yhat = self.predict(testing_features)
        return self.scoring_function(testing_labels,yhat)


Instruction:
Make a summary of the following Python 3 code
def export(self, output_file_name):
        if self._best_estimator is None:
            raise ValueError('A model has not been optimized. Please call fit()'
                             ' first.')

        # Write print_model() to file
        with open(output_file_name, 'w') as output_file:
            output_file.write(self.print_model())
        # if decision tree, print tree into dot file
        if 'DecisionTree' in self.ml_type:
            export_graphviz(self._best_estimator,
                            out_file=output_file_name+'.dot',
                            feature_names = self.stacks_2_eqns(self._best_inds)
                            if self._best_inds else None,
                            class_names=['True','False'],
                            filled=False,impurity = True,rotate=True)

Response: Exports engineered features of the current model to a file.


Instruction:
Given the following Python 3 function, write the documentation
def print_model(self,sep='\n'):
        model = ''
        # print('ml type:',self.ml_type)
        # print('ml:',self._best_estimator)

        if self._best_inds:

            if self.ml_type == 'GridSearchCV':
                ml = self._best_estimator.named_steps['ml'].best_estimator_
            else:
                ml = self._best_estimator.named_steps['ml']

            if self.ml_type != 'SVC' and self.ml_type != 'SVR':
            # this is need because svm has a bug that throws valueerror on
            # attribute check

                if hasattr(ml,'coef_'):
                    if len(ml.coef_.shape)==1:
                        s = np.argsort(np.abs(ml.coef_))[::-1]
                        scoef = ml.coef_[s]
                        bi = [self._best_inds[k] for k in s]
                        model = (' +' + sep).join(
                            [str(round(c,3))+'*'+self.stack_2_eqn(f)
                             for i,(f,c) in enumerate(zip(bi,scoef))
                             if round(scoef[i],3) != 0])
                    else:
                        # more than one decision function is fit. print all.
                        for j,coef in enumerate(ml.coef_):
                            s = np.argsort(np.abs(coef))[::-1]
                            scoef = coef[s]
                            bi =[self._best_inds[k] for k in s]
                            model += sep + 'class'+str(j)+' :'+' + '.join(
                                [str(round(c,3))+'*'+self.stack_2_eqn(f)
                                 for i,(f,c) in enumerate(zip(bi,coef))
                                 if coef[i] != 0])
                elif hasattr(ml,'feature_importances_'):
                    s = np.argsort(ml.feature_importances_)[::-1]
                    sfi = ml.feature_importances_[s]
                    bi = [self._best_inds[k] for k in s]
                    # model = 'importance:feature'+sep

                    model += sep.join(
                        [str(round(c,3))+':'+self.stack_2_eqn(f)
                         for i,(f,c) in enumerate(zip(bi,sfi))
                         if round(sfi[i],3) != 0])
                else:
                    return sep.join(self.stacks_2_eqns(self._best_inds))
            else:
                return sep.join(self.stacks_2_eqns(self._best_inds))
        else:
            return 'original features'

        return model

Response: prints model contained in best inds if ml has a coefficient property. otherwise prints the features generated by FEW.


Instruction:
How would you explain what the following Python 3 function does
def valid_loc(self,F=None):
        if F is not None:
            return [i for i,f in enumerate(F) if np.all(f < self.max_fit) and np.all(f >= 0)]
        else:
            return [i for i,f in enumerate(self.F) if np.all(f < self.max_fit) and np.all(f >= 0)]

Response: returns the indices of individuals with valid fitness.


Instruction:
How would you implement a function in Python 3 that
returns the sublist of individuals with valid fitness.

Response:
def valid(self,individuals=None,F=None):
        """returns the sublist of individuals with valid fitness."""
        if F:
            valid_locs = self.valid_loc(F)
        else:
            valid_locs = self.valid_loc(self.F)

        if individuals:
            return [ind for i,ind in enumerate(individuals) if i in valid_locs]
        else:
            return [ind for i,ind in enumerate(self.pop.individuals) if i in valid_locs]


Instruction:
How would you code a function in Python 3 to
compute mean diversity of individual outputs

Response:
def get_diversity(self,X):
        """compute mean diversity of individual outputs"""
        # diversity in terms of cosine distances between features
        feature_correlations = np.zeros(X.shape[0]-1)
        for i in np.arange(1,X.shape[0]-1):
            feature_correlations[i] = max(0.0,r2_score(X[0],X[i]))
        # pdb.set_trace()
        self.diversity.append(1-np.mean(feature_correlations))


Instruction:
Create a Python 3 function to
return an roc auc score depending on the underlying estimator.

Response:
def roc_auc_cv(self,features,labels):
        """returns an roc auc score depending on the underlying estimator."""
        if callable(getattr(self.ml, "decision_function", None)):
            return np.mean([self.scoring_function(labels[test],
                            self.pipeline.fit(features[train],labels[train]).
                                         decision_function(features[test]))
                            for train, test in KFold().split(features, labels)])
        elif callable(getattr(self.ml, "predict_proba", None)):
            return np.mean([self.scoring_function(labels[test],
                            self.pipeline.fit(features[train],labels[train]).
                                            predict_proba(features[test])[:,1])
                            for train, test in KFold().split(features, labels)])
        else:
            raise ValueError("ROC AUC score won't work with " + self.ml_type + ". No "
                    "decision_function or predict_proba method found for this learner.")


Instruction:
How would you explain what the following Python 3 function does
def divs(x,y):
    tmp = np.ones(x.shape)
    nonzero_y = y != 0
    tmp[nonzero_y] = x[nonzero_y]/y[nonzero_y]
    return tmp

Response: safe division of x and y


Instruction:
Implement a function in Python 3 to
return non - aggregate version of r2 score.

Response:
def r2_score_vec(y_true,y_pred):
    """ returns non-aggregate version of r2 score.

    based on r2_score() function from sklearn (http://sklearn.org)
    """

    numerator = (y_true - y_pred) ** 2
    denominator = (y_true - np.average(y_true)) ** 2

    nonzero_denominator = denominator != 0
    nonzero_numerator = numerator != 0
    valid_score = nonzero_denominator & nonzero_numerator
    output_scores = np.ones([y_true.shape[0]])
    output_scores[valid_score] = 1 - (numerator[valid_score] /
                                      denominator[valid_score])
    # arbitrary set to zero to avoid -inf scores, having a constant
    # y_true is not interesting for scoring a regression anyway
    output_scores[nonzero_numerator & ~nonzero_denominator] = 0.

    return output_scores


Instruction:
Can you create a Python 3 function that
calculates the within - class squared distance from the centroid

Response:
def inertia(X,y,samples=False):
    """ return the within-class squared distance from the centroid"""
    # pdb.set_trace()
    if samples:
        # return within-class distance for each sample
        inertia = np.zeros(y.shape)
        for label in np.unique(y):
            inertia[y==label] = (X[y==label] - np.mean(X[y==label])) ** 2

    else: # return aggregate score
        inertia = 0
        for i,label in enumerate(np.unique(y)):
            inertia += np.sum((X[y==label] - np.mean(X[y==label])) ** 2)/len(y[y==label])
        inertia = inertia/len(np.unique(y))

    return inertia


Instruction:
Write a Python 3 function that can
return the sum of the between - class squared distance

Response:
def separation(X,y,samples=False):
    """ return the sum of the between-class squared distance"""
    # pdb.set_trace()
    num_classes = len(np.unique(y))
    total_dist = (X.max()-X.min())**2
    if samples:
        # return intra-class distance for each sample
        separation = np.zeros(y.shape)
        for label in np.unique(y):
            for outsider in np.unique(y[y!=label]):
                separation[y==label] += (X[y==label] - np.mean(X[y==outsider])) ** 2

        #normalize between 0 and 1
        print('separation:',separation)
        print('num_classes:',num_classes)
        print('total_dist:',total_dist)
        separation = separation#/separation.max()

        print('separation after normalization:',separation)

    else:
        # return aggregate score
        separation = 0
        for i,label in enumerate(np.unique(y)):
            for outsider in np.unique(y[y!=label]):
                separation += np.sum((X[y==label] - np.mean(X[y==outsider])) ** 2)/len(y[y==label])
        separation = separation/len(np.unique(y))

    return separation


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fisher(yhat,y,samples=False):
    classes = np.unique(y)
    mu = np.zeros(len(classes))
    v = np.zeros(len(classes))
    # pdb.set_trace()
    for c in classes.astype(int):
        mu[c] = np.mean(yhat[y==c])
        v[c] = np.var(yhat[y==c])

    if not samples:
        fisher = 0
        for c1,c2 in pairwise(classes.astype(int)):
            fisher += np.abs(mu[c1] - mu[c2])/np.sqrt(v[c1]+v[c2])
    else:
        # lexicase version
        fisher = np.zeros(len(yhat))
        # get closests classes to each class (min mu distance)
        mu_d = pairwise_distances(mu.reshape(-1,1))
        min_mu=np.zeros(len(classes),dtype=int)
        for i in np.arange(len(min_mu)):
            min_mu[i] = np.argsort(mu_d[i])[1]
        # for c1, pairwise(classes.astype(int)):
        #     min_mu[c1] = np.argmin()
        for i,l in enumerate(yhat.astype(int)):
            fisher[i] = np.abs(l - mu[min_mu[y[i]]])/np.sqrt(v[y[i]]+v[min_mu[y[i]]])

    # pdb.set_trace()
    return fisher

Response: Fisher criterion for the given set of classes y.


Instruction:
Make a summary of the following Python 3 code
def proper(self,x):
        x[x < 0] = self.max_fit
        x[np.isnan(x)] = self.max_fit
        x[np.isinf(x)] = self.max_fit
        return x

Response: cleans fitness vector x


Instruction:
Here you have a function in Python 3, explain what it does
def safe(self,x):
        x[np.isinf(x)] = 1
        x[np.isnan(x)] = 1
        return x

Response: removes nans and infs from outputs


Instruction:
Explain what the following Python 3 code does
def evaluate(self,n, features, stack_float, stack_bool,labels=None):
        np.seterr(all='ignore')
        if len(stack_float) >= n.arity['f'] and len(stack_bool) >= n.arity['b']:
            if n.out_type == 'f':
                stack_float.append(
                    self.safe(self.eval_dict[n.name](n,features,stack_float,
                                                     stack_bool,labels)))
                if (np.isnan(stack_float[-1]).any() or
                    np.isinf(stack_float[-1]).any()):
                    print("problem operator:",n)
            else:
                stack_bool.append(self.safe(self.eval_dict[n.name](n,features,
                                                                   stack_float,
                                                                   stack_bool,
                                                                   labels)))
                if np.isnan(stack_bool[-1]).any() or np.isinf(stack_bool[-1]).any():
                    print("problem operator:",n)

Response: evaluate node in program


Instruction:
Can you generate the documentation for the following Python 3 function
def all_finite(self,X):
        # Adapted from sklearn utils: _assert_all_finite(X)
        # First try an O(n) time, O(1) space solution for the common case that
        # everything is finite; fall back to O(n) space np.isfinite to prevent
        # false positives from overflow in sum method.
        # Note: this is basically here because sklearn tree.py uses float32 internally,
        # and float64's that are finite are not finite in float32.
        if (X.dtype.char in np.typecodes['AllFloat']
            and not np.isfinite(np.asarray(X,dtype='float32').sum())
            and not np.isfinite(np.asarray(X,dtype='float32')).all()):
            return False
        return True

Response: returns true if X is finite false otherwise


Instruction:
Create a Python 3 function to
compute the output for individual I

Response:
def out(self,I,features,labels=None,otype='f'):
        """computes the output for individual I"""
        stack_float = []
        stack_bool = []
        # print("stack:",I.stack)
        # evaulate stack over rows of features,labels
        # pdb.set_trace()
        for n in I.stack:
            self.evaluate(n,features,stack_float,stack_bool,labels)
            # print("stack_float:",stack_float)
        if otype=='f':
            return (stack_float[-1] if self.all_finite(stack_float[-1])
                    else np.zeros(len(features)))
        else:
            return (stack_bool[-1].astype(float) if self.all_finite(stack_bool[-1])
                    else np.zeros(len(features)))


Instruction:
Can you tell what is the following Python 3 function doing
def calc_fitness(self,X,labels,fit_choice,sel):

        if 'lexicase' in sel:
            # return list(map(lambda yhat: self.f_vec[fit_choice](labels,yhat),X))
            return np.asarray(
                              [self.proper(self.f_vec[fit_choice](labels,
                                                        yhat)) for yhat in X],
                                                        order='F')
            # return list(Parallel(n_jobs=-1)(delayed(self.f_vec[fit_choice])(labels,yhat) for yhat in X))
        else:
            # return list(map(lambda yhat: self.f[fit_choice](labels,yhat),X))
            return np.asarray([self.f[fit_choice](labels,yhat) for yhat in X],
                            order='F').reshape(-1)

Response: computes fitness of individual output yhat.


Instruction:
How would you code a function in Python 3 to
convert the ImageData dictionary imdict to an nd image.

Response:
def imagedatadict_to_ndarray(imdict):
    """
    Converts the ImageData dictionary, imdict, to an nd image.
    """
    arr = imdict['Data']
    im = None
    if isinstance(arr, parse_dm3.array.array):
        im = numpy.asarray(arr, dtype=arr.typecode)
    elif isinstance(arr, parse_dm3.structarray):
        t = tuple(arr.typecodes)
        im = numpy.frombuffer(
            arr.raw_data,
            dtype=structarray_to_np_map[t])
    # print "Image has dmimagetype", imdict["DataType"], "numpy type is", im.dtype
    assert dm_image_dtypes[imdict["DataType"]][1] == im.dtype
    assert imdict['PixelDepth'] == im.dtype.itemsize
    im = im.reshape(imdict['Dimensions'][::-1])
    if imdict["DataType"] == 23:  # RGB
        im = im.view(numpy.uint8).reshape(im.shape + (-1, ))[..., :-1]  # strip A
        # NOTE: RGB -> BGR would be [:, :, ::-1]
    return im


Instruction:
Write a Python 3 function for
converting the numpy array nparr into a suitable ImageList entry dictionary.

Response:
def ndarray_to_imagedatadict(nparr):
    """
    Convert the numpy array nparr into a suitable ImageList entry dictionary.
    Returns a dictionary with the appropriate Data, DataType, PixelDepth
    to be inserted into a dm3 tag dictionary and written to a file.
    """
    ret = {}
    dm_type = None
    for k, v in iter(dm_image_dtypes.items()):
        if v[1] == nparr.dtype.type:
            dm_type = k
            break
    if dm_type is None and nparr.dtype == numpy.uint8 and nparr.shape[-1] in (3, 4):
        ret["DataType"] = 23
        ret["PixelDepth"] = 4
        if nparr.shape[2] == 4:
            rgb_view = nparr.view(numpy.int32).reshape(nparr.shape[:-1])  # squash the color into uint32
        else:
            assert nparr.shape[2] == 3
            rgba_image = numpy.empty(nparr.shape[:-1] + (4,), numpy.uint8)
            rgba_image[:,:,0:3] = nparr
            rgba_image[:,:,3] = 255
            rgb_view = rgba_image.view(numpy.int32).reshape(rgba_image.shape[:-1])  # squash the color into uint32
        ret["Dimensions"] = list(rgb_view.shape[::-1])
        ret["Data"] = parse_dm3.array.array(platform_independent_char(rgb_view.dtype), rgb_view.flatten())
    else:
        ret["DataType"] = dm_type
        ret["PixelDepth"] = nparr.dtype.itemsize
        ret["Dimensions"] = list(nparr.shape[::-1])
        if nparr.dtype.type in np_to_structarray_map:
            types = np_to_structarray_map[nparr.dtype.type]
            ret["Data"] = parse_dm3.structarray(types)
            ret["Data"].raw_data = bytes(numpy.array(nparr, copy=False).data)
        else:
            ret["Data"] = parse_dm3.array.array(platform_independent_char(nparr.dtype), numpy.array(nparr, copy=False).flatten())
    return ret


Instruction:
Explain what the following Python 3 code does
def load_image(file) -> DataAndMetadata.DataAndMetadata:
    if isinstance(file, str) or isinstance(file, str):
        with open(file, "rb") as f:
            return load_image(f)
    dmtag = parse_dm3.parse_dm_header(file)
    dmtag = fix_strings(dmtag)
    # display_keys(dmtag)
    img_index = -1
    image_tags = dmtag['ImageList'][img_index]
    data = imagedatadict_to_ndarray(image_tags['ImageData'])
    calibrations = []
    calibration_tags = image_tags['ImageData'].get('Calibrations', dict())
    for dimension in calibration_tags.get('Dimension', list()):
        origin, scale, units = dimension.get('Origin', 0.0), dimension.get('Scale', 1.0), dimension.get('Units', str())
        calibrations.append((-origin * scale, scale, units))
    calibrations = tuple(reversed(calibrations))
    if len(data.shape) == 3 and data.dtype != numpy.uint8:
        if image_tags['ImageTags'].get('Meta Data', dict()).get("Format", str()).lower() in ("spectrum", "spectrum image"):
            if data.shape[1] == 1:
                data = numpy.squeeze(data, 1)
                data = numpy.moveaxis(data, 0, 1)
                data_descriptor = DataAndMetadata.DataDescriptor(False, 1, 1)
                calibrations = (calibrations[2], calibrations[0])
            else:
                data = numpy.moveaxis(data, 0, 2)
                data_descriptor = DataAndMetadata.DataDescriptor(False, 2, 1)
                calibrations = tuple(calibrations[1:]) + (calibrations[0],)
        else:
            data_descriptor = DataAndMetadata.DataDescriptor(False, 1, 2)
    elif len(data.shape) == 4 and data.dtype != numpy.uint8:
        # data = numpy.moveaxis(data, 0, 2)
        data_descriptor = DataAndMetadata.DataDescriptor(False, 2, 2)
    elif data.dtype == numpy.uint8:
        data_descriptor = DataAndMetadata.DataDescriptor(False, 0, len(data.shape[:-1]))
    else:
        data_descriptor = DataAndMetadata.DataDescriptor(False, 0, len(data.shape))
    brightness = calibration_tags.get('Brightness', dict())
    origin, scale, units = brightness.get('Origin', 0.0), brightness.get('Scale', 1.0), brightness.get('Units', str())
    intensity = -origin * scale, scale, units
    timestamp = None
    timezone = None
    timezone_offset = None
    title = image_tags.get('Name')
    properties = dict()
    if 'ImageTags' in image_tags:
        voltage = image_tags['ImageTags'].get('ImageScanned', dict()).get('EHT', dict())
        if voltage:
            properties.setdefault("hardware_source", dict())["autostem"] = { "high_tension_v": float(voltage) }
        dm_metadata_signal = image_tags['ImageTags'].get('Meta Data', dict()).get('Signal')
        if dm_metadata_signal and dm_metadata_signal.lower() == "eels":
            properties.setdefault("hardware_source", dict())["signal_type"] = dm_metadata_signal
        if image_tags['ImageTags'].get('Meta Data', dict()).get("Format", str()).lower() in ("spectrum", "spectrum image"):
            data_descriptor.collection_dimension_count += data_descriptor.datum_dimension_count - 1
            data_descriptor.datum_dimension_count = 1
        if image_tags['ImageTags'].get('Meta Data', dict()).get("IsSequence", False) and data_descriptor.collection_dimension_count > 0:
            data_descriptor.is_sequence = True
            data_descriptor.collection_dimension_count -= 1
        timestamp_str = image_tags['ImageTags'].get("Timestamp")
        if timestamp_str:
            timestamp = get_datetime_from_timestamp_str(timestamp_str)
        timezone = image_tags['ImageTags'].get("Timezone")
        timezone_offset = image_tags['ImageTags'].get("TimezoneOffset")
        # to avoid having duplicate copies in Swift, get rid of these tags
        image_tags['ImageTags'].pop("Timestamp", None)
        image_tags['ImageTags'].pop("Timezone", None)
        image_tags['ImageTags'].pop("TimezoneOffset", None)
        # put the image tags into properties
        properties.update(image_tags['ImageTags'])
    dimensional_calibrations = [Calibration.Calibration(c[0], c[1], c[2]) for c in calibrations]
    while len(dimensional_calibrations) < data_descriptor.expected_dimension_count:
        dimensional_calibrations.append(Calibration.Calibration())
    intensity_calibration = Calibration.Calibration(intensity[0], intensity[1], intensity[2])
    return DataAndMetadata.new_data_and_metadata(data,
                                                 data_descriptor=data_descriptor,
                                                 dimensional_calibrations=dimensional_calibrations,
                                                 intensity_calibration=intensity_calibration,
                                                 metadata=properties,
                                                 timestamp=timestamp,
                                                 timezone=timezone,
                                                 timezone_offset=timezone_offset)

Response: Loads the image from the file - like object or string file.


Instruction:
Can you tell what is the following Python 3 function doing
def save_image(xdata: DataAndMetadata.DataAndMetadata, file):
    # we need to create a basic DM tree suitable for an image
    # we'll try the minimum: just an data list
    # doesn't work. Do we need a ImageSourceList too?
    # and a DocumentObjectList?

    data = xdata.data
    data_descriptor = xdata.data_descriptor
    dimensional_calibrations = xdata.dimensional_calibrations
    intensity_calibration = xdata.intensity_calibration
    metadata = xdata.metadata
    modified = xdata.timestamp
    timezone = xdata.timezone
    timezone_offset = xdata.timezone_offset
    needs_slice = False
    is_sequence = False

    if len(data.shape) == 3 and data.dtype != numpy.uint8 and data_descriptor.datum_dimension_count == 1:
        data = numpy.moveaxis(data, 2, 0)
        dimensional_calibrations = (dimensional_calibrations[2],) + tuple(dimensional_calibrations[0:2])
    if len(data.shape) == 2 and data.dtype != numpy.uint8 and data_descriptor.datum_dimension_count == 1:
        is_sequence = data_descriptor.is_sequence
        data = numpy.moveaxis(data, 1, 0)
        data = numpy.expand_dims(data, axis=1)
        dimensional_calibrations = (dimensional_calibrations[1], Calibration.Calibration(), dimensional_calibrations[0])
        data_descriptor = DataAndMetadata.DataDescriptor(False, 2, 1)
        needs_slice = True
    data_dict = ndarray_to_imagedatadict(data)
    ret = {}
    ret["ImageList"] = [{"ImageData": data_dict}]
    if dimensional_calibrations and len(dimensional_calibrations) == len(data.shape):
        dimension_list = data_dict.setdefault("Calibrations", dict()).setdefault("Dimension", list())
        for dimensional_calibration in reversed(dimensional_calibrations):
            dimension = dict()
            if dimensional_calibration.scale != 0.0:
                origin = -dimensional_calibration.offset / dimensional_calibration.scale
            else:
                origin = 0.0
            dimension['Origin'] = origin
            dimension['Scale'] = dimensional_calibration.scale
            dimension['Units'] = dimensional_calibration.units
            dimension_list.append(dimension)
    if intensity_calibration:
        if intensity_calibration.scale != 0.0:
            origin = -intensity_calibration.offset / intensity_calibration.scale
        else:
            origin = 0.0
        brightness = data_dict.setdefault("Calibrations", dict()).setdefault("Brightness", dict())
        brightness['Origin'] = origin
        brightness['Scale'] = intensity_calibration.scale
        brightness['Units'] = intensity_calibration.units
    if modified:
        timezone_str = None
        if timezone_str is None and timezone:
            try:
                import pytz
                tz = pytz.timezone(timezone)
                timezone_str = tz.tzname(modified)
            except:
                pass
        if timezone_str is None and timezone_offset:
            timezone_str = timezone_offset
        timezone_str = " " + timezone_str if timezone_str is not None else ""
        date_str = modified.strftime("%x")
        time_str = modified.strftime("%X") + timezone_str
        ret["DataBar"] = {"Acquisition Date": date_str, "Acquisition Time": time_str}
    # I think ImageSource list creates a mapping between ImageSourceIds and Images
    ret["ImageSourceList"] = [{"ClassName": "ImageSource:Simple", "Id": [0], "ImageRef": 0}]
    # I think this lists the sources for the DocumentObjectlist. The source number is not
    # the indxe in the imagelist but is either the index in the ImageSourceList or the Id
    # from that list. We also need to set the annotation type to identify it as an data
    ret["DocumentObjectList"] = [{"ImageSource": 0, "AnnotationType": 20}]
    # finally some display options
    ret["Image Behavior"] = {"ViewDisplayID": 8}
    dm_metadata = copy.deepcopy(metadata)
    if metadata.get("hardware_source", dict()).get("signal_type", "").lower() == "eels":
        if len(data.shape) == 1 or (len(data.shape) == 2 and data.shape[0] == 1):
            dm_metadata.setdefault("Meta Data", dict())["Format"] = "Spectrum"
            dm_metadata.setdefault("Meta Data", dict())["Signal"] = "EELS"
    elif data_descriptor.collection_dimension_count == 2 and data_descriptor.datum_dimension_count == 1:
        dm_metadata.setdefault("Meta Data", dict())["Format"] = "Spectrum image"
        dm_metadata.setdefault("Meta Data", dict())["Signal"] = "EELS"
    elif data_descriptor.datum_dimension_count == 1:
        dm_metadata.setdefault("Meta Data", dict())["Format"] = "Spectrum"
    if (1 if data_descriptor.is_sequence else 0) + data_descriptor.collection_dimension_count == 1 or needs_slice:
        if data_descriptor.is_sequence or is_sequence:
            dm_metadata.setdefault("Meta Data", dict())["IsSequence"] = True
        ret["ImageSourceList"] = [{"ClassName": "ImageSource:Summed", "Do Sum": True, "Id": [0], "ImageRef": 0, "LayerEnd": 0, "LayerStart": 0, "Summed Dimension": len(data.shape) - 1}]
        if needs_slice:
            ret["DocumentObjectList"][0]["AnnotationGroupList"] = [{"AnnotationType": 23, "Name": "SICursor", "Rectangle": (0, 0, 1, 1)}]
            ret["DocumentObjectList"][0]["ImageDisplayType"] = 1  # display as an image
    if modified:
        dm_metadata["Timestamp"] = modified.isoformat()
    if timezone:
        dm_metadata["Timezone"] = timezone
    if timezone_offset:
        dm_metadata["TimezoneOffset"] = timezone_offset
    ret["ImageList"][0]["ImageTags"] = dm_metadata
    ret["InImageMode"] = True
    parse_dm3.parse_dm_header(file, ret)

Response: Save the image data to a nparray file - like object or string.


Instruction:
Write a Python 3 function for
parsing the DM file header and return a new file containing the tag_root entry.

Response:
def parse_dm_header(f, outdata=None):
    """
    This is the start of the DM file. We check for some
    magic values and then treat the next entry as a tag_root

    If outdata is supplied, we write instead of read using the dictionary outdata as a source
    Hopefully parse_dm_header(newf, outdata=parse_dm_header(f)) copies f to newf
    """
    # filesize is sizeondisk - 16. But we have 8 bytes of zero at the end of
    # the file.
    if outdata is not None:  # this means we're WRITING to the file
        if verbose:
            print("write_dm_header start", f.tell())
        ver, file_size, endianness = 3, -1, 1
        put_into_file(f, "> l l l", ver, file_size, endianness)
        start = f.tell()
        parse_dm_tag_root(f, outdata)
        end = f.tell()
        # start is end of 3 long header. We want to write 2nd long
        f.seek(start - 8)
        # the real file size. We started counting after 12-byte version,fs,end
        # and we need to subtract 16 total:
        put_into_file(f, "> l", end - start + 4)
        f.seek(end)
        enda, endb = 0, 0
        put_into_file(f, "> l l", enda, endb)
        if verbose:
            print("write_dm_header end", f.tell())
    else:
        if verbose:
            print("read_dm_header start", f.tell())
        ver = get_from_file(f, "> l")
        assert ver in [3,4], "Version must be 3 or 4, not %s" % ver
        # argh. why a global?
        global size_type, version
        if ver == 3:
            size_type = 'L'  # may be Q?
            version = 3
        if ver == 4:
            size_type = 'Q'  # may be Q?
            version = 4
        file_size, endianness = get_from_file(f, ">%c l" % size_type)
        assert endianness == 1, "Endianness must be 1, not %s"%endianness
        start = f.tell()
        ret = parse_dm_tag_root(f, outdata)
        end = f.tell()
        # print("fs", file_size, end - start, (end-start)%8)
        # mfm 2013-07-11 the file_size value is not always
        # end-start, sometimes there seems to be an extra 4 bytes,
        # other times not. Let's just ignore it for the moment
        # assert(file_size == end - start)
        enda, endb = get_from_file(f, "> l l")
        assert(enda == endb == 0)
        if verbose:
            print("read_dm_header end", f.tell())
        return ret


Instruction:
Implement a function in Python 3 to
return the dmtype for the python typeorobject typeorobj.

Response:
def get_structdmtypes_for_python_typeorobject(typeorobj):
    """
    Return structchar, dmtype for the python (or numpy)
    type or object typeorobj.
    For more complex types we only return the dm type
    """
    # not isinstance is probably a bit more lenient than 'is'
    # ie isinstance(x,str) is nicer than type(x) is str.
    # hence we use isinstance when available
    if isinstance(typeorobj, type):
        comparer = lambda test: test is typeorobj
    else:
        comparer = lambda test: isinstance(typeorobj, test)

    if comparer(int) and not -2**31 < typeorobj < 2**31 - 1:
        return 'q', 11

    for key, name, sc, types in dm_simple_names:
        for t in types:
            if comparer(t):
                return sc, key
    if comparer(str):
        return None, get_dmtype_for_name('array')  # treat all strings as arrays!
    elif comparer(unicode_type):
        return None, get_dmtype_for_name('array')  # treat all strings as arrays!
    elif comparer(array.array):
        return None, get_dmtype_for_name('array')
    elif comparer(tuple):
        return None, get_dmtype_for_name('struct')
    elif comparer(structarray):
        return None, get_dmtype_for_name('array')
    logging.warn("No appropriate DMType found for %s, %s", typeorobj, type(typeorobj))
    return None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def imread(files, **kwargs):
    kwargs_file = parse_kwargs(kwargs, 'is_ome', 'multifile', '_useframes',
                               'name', 'offset', 'size',
                               'multifile_close', 'fastij', 'movie')  # legacy
    kwargs_seq = parse_kwargs(kwargs, 'pattern')

    if kwargs.get('pages', None) is not None:
        if kwargs.get('key', None) is not None:
            raise TypeError(
                "the 'pages' and 'key' arguments cannot be used together")
        log.warning("imread: the 'pages' argument is deprecated")
        kwargs['key'] = kwargs.pop('pages')

    if isinstance(files, basestring) and any(i in files for i in '?*'):
        files = glob.glob(files)
    if not files:
        raise ValueError('no files found')
    if not hasattr(files, 'seek') and len(files) == 1:
        files = files[0]

    if isinstance(files, basestring) or hasattr(files, 'seek'):
        with TiffFile(files, **kwargs_file) as tif:
            return tif.asarray(**kwargs)
    else:
        with TiffSequence(files, **kwargs_seq) as imseq:
            return imseq.asarray(**kwargs)

Response: Read image data from TIFF files and return as numpy array.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def imwrite(file, data=None, shape=None, dtype=None, **kwargs):
    tifargs = parse_kwargs(kwargs, 'append', 'bigtiff', 'byteorder', 'imagej')
    if data is None:
        dtype = numpy.dtype(dtype)
        size = product(shape) * dtype.itemsize
        byteorder = dtype.byteorder
    else:
        try:
            size = data.nbytes
            byteorder = data.dtype.byteorder
        except Exception:
            size = 0
            byteorder = None
    bigsize = kwargs.pop('bigsize', 2**32-2**25)
    if 'bigtiff' not in tifargs and size > bigsize and not (
            tifargs.get('imagej', False) or tifargs.get('truncate', False)):
        tifargs['bigtiff'] = True
    if 'byteorder' not in tifargs:
        tifargs['byteorder'] = byteorder

    with TiffWriter(file, **tifargs) as tif:
        return tif.save(data, shape, dtype, **kwargs)

Response: Write numpy array to TIFF file.


Instruction:
Make a summary of the following Python 3 code
def memmap(filename, shape=None, dtype=None, page=None, series=0, mode='r+',
           **kwargs):
    """Return memory-mapped numpy array stored in TIFF file.

    Memory-mapping requires data stored in native byte order, without tiling,
    compression, predictors, etc.
    If 'shape' and 'dtype' are provided, existing files will be overwritten or
    appended to depending on the 'append' parameter.
    Otherwise the image data of a specified page or series in an existing
    file will be memory-mapped. By default, the image data of the first page
    series is memory-mapped.
    Call flush() to write any changes in the array to the file.
    Raise ValueError if the image data in the file is not memory-mappable.

    Parameters
    ----------
    filename : str
        Name of the TIFF file which stores the array.
    shape : tuple
        Shape of the empty array.
    dtype : numpy.dtype
        Data-type of the empty array.
    page : int
        Index of the page which image data to memory-map.
    series : int
        Index of the page series which image data to memory-map.
    mode : {'r+', 'r', 'c'}
        The file open mode. Default is to open existing file for reading and
        writing ('r+').
    kwargs : dict
        Additional parameters passed to imwrite() or TiffFile().

    """
    if shape is not None and dtype is not None:
        # create a new, empty array
        kwargs.update(data=None, shape=shape, dtype=dtype, returnoffset=True,
                      align=TIFF.ALLOCATIONGRANULARITY)
        result = imwrite(filename, **kwargs)
        if result is None:
            # TODO: fail before creating file or writing data
            raise ValueError('image data are not memory-mappable')
        offset = result[0]
    else:
        # use existing file
        with TiffFile(filename, **kwargs) as tif:
            if page is not None:
                page = tif.pages[page]
                if not page.is_memmappable:
                    raise ValueError('image data are not memory-mappable')
                offset, _ = page.is_contiguous
                shape = page.shape
                dtype = page.dtype
            else:
                series = tif.series[series]
                if series.offset is None:
                    raise ValueError('image data are not memory-mappable')
                shape = series.shape
                dtype = series.dtype
                offset = series.offset
            dtype = tif.byteorder + dtype.char
    return numpy.memmap(filename, dtype, mode, offset, shape, 'C')

Response: Return memory - mapped numpy array stored in TIFF file.


Instruction:
Can you create a Python 3 function that
reads tags from the chain of IFDs and return as list of dicts.

Response:
def read_tags(fh, byteorder, offsetsize, tagnames, customtags=None,
              maxifds=None):
    """Read tags from chain of IFDs and return as list of dicts.

    The file handle position must be at a valid IFD header.

    """
    if offsetsize == 4:
        offsetformat = byteorder+'I'
        tagnosize = 2
        tagnoformat = byteorder+'H'
        tagsize = 12
        tagformat1 = byteorder+'HH'
        tagformat2 = byteorder+'I4s'
    elif offsetsize == 8:
        offsetformat = byteorder+'Q'
        tagnosize = 8
        tagnoformat = byteorder+'Q'
        tagsize = 20
        tagformat1 = byteorder+'HH'
        tagformat2 = byteorder+'Q8s'
    else:
        raise ValueError('invalid offset size')

    if customtags is None:
        customtags = {}
    if maxifds is None:
        maxifds = 2**32

    result = []
    unpack = struct.unpack
    offset = fh.tell()
    while len(result) < maxifds:
        # loop over IFDs
        try:
            tagno = unpack(tagnoformat, fh.read(tagnosize))[0]
            if tagno > 4096:
                raise TiffFileError('suspicious number of tags')
        except Exception:
            log.warning('read_tags: corrupted tag list at offset %i', offset)
            break

        tags = {}
        data = fh.read(tagsize * tagno)
        pos = fh.tell()
        index = 0
        for _ in range(tagno):
            code, type_ = unpack(tagformat1, data[index:index+4])
            count, value = unpack(tagformat2, data[index+4:index+tagsize])
            index += tagsize
            name = tagnames.get(code, str(code))
            try:
                dtype = TIFF.DATA_FORMATS[type_]
            except KeyError:
                raise TiffFileError('unknown tag data type %i' % type_)

            fmt = '%s%i%s' % (byteorder, count * int(dtype[0]), dtype[1])
            size = struct.calcsize(fmt)
            if size > offsetsize or code in customtags:
                offset = unpack(offsetformat, value)[0]
                if offset < 8 or offset > fh.size - size:
                    raise TiffFileError('invalid tag value offset %i' % offset)
                fh.seek(offset)
                if code in customtags:
                    readfunc = customtags[code][1]
                    value = readfunc(fh, byteorder, dtype, count, offsetsize)
                elif type_ == 7 or (count > 1 and dtype[-1] == 'B'):
                    value = read_bytes(fh, byteorder, dtype, count, offsetsize)
                elif code in tagnames or dtype[-1] == 's':
                    value = unpack(fmt, fh.read(size))
                else:
                    value = read_numpy(fh, byteorder, dtype, count, offsetsize)
            elif dtype[-1] == 'B' or type_ == 7:
                value = value[:size]
            else:
                value = unpack(fmt, value[:size])

            if code not in customtags and code not in TIFF.TAG_TUPLE:
                if len(value) == 1:
                    value = value[0]
            if type_ != 7 and dtype[-1] == 's' and isinstance(value, bytes):
                # TIFF ASCII fields can contain multiple strings,
                #   each terminated with a NUL
                try:
                    value = bytes2str(stripascii(value).strip())
                except UnicodeDecodeError:
                    log.warning(
                        'read_tags: coercing invalid ASCII to bytes (tag %i)',
                        code)

            tags[name] = value

        result.append(tags)
        # read offset to next page
        fh.seek(pos)
        offset = unpack(offsetformat, fh.read(offsetsize))[0]
        if offset == 0:
            break
        if offset >= fh.size:
            log.warning('read_tags: invalid page offset (%i)', offset)
            break
        fh.seek(offset)

    if result and maxifds == 1:
        result = result[0]
    return result


Instruction:
How would you code a function in Python 3 to
read EXIF tags from file and return as dict.

Response:
def read_exif_ifd(fh, byteorder, dtype, count, offsetsize):
    """Read EXIF tags from file and return as dict."""
    exif = read_tags(fh, byteorder, offsetsize, TIFF.EXIF_TAGS, maxifds=1)
    for name in ('ExifVersion', 'FlashpixVersion'):
        try:
            exif[name] = bytes2str(exif[name])
        except Exception:
            pass
    if 'UserComment' in exif:
        idcode = exif['UserComment'][:8]
        try:
            if idcode == b'ASCII\x00\x00\x00':
                exif['UserComment'] = bytes2str(exif['UserComment'][8:])
            elif idcode == b'UNICODE\x00':
                exif['UserComment'] = exif['UserComment'][8:].decode('utf-16')
        except Exception:
            pass
    return exif


Instruction:
Explain what the following Python 3 code does
def read_gps_ifd(fh, byteorder, dtype, count, offsetsize):
    return read_tags(fh, byteorder, offsetsize, TIFF.GPS_TAGS, maxifds=1)

Response: Read GPS tags from file and return as dict.


Instruction:
Implement a Python 3 function for
reading Interoperability tags from file and return as dict.

Response:
def read_interoperability_ifd(fh, byteorder, dtype, count, offsetsize):
    """Read Interoperability tags from file and return as dict."""
    tag_names = {1: 'InteroperabilityIndex'}
    return read_tags(fh, byteorder, offsetsize, tag_names, maxifds=1)


Instruction:
Can you implement a function in Python 3 that
reads tag data from file and return as byte string.

Response:
def read_bytes(fh, byteorder, dtype, count, offsetsize):
    """Read tag data from file and return as byte string."""
    dtype = 'B' if dtype[-1] == 's' else byteorder+dtype[-1]
    count *= numpy.dtype(dtype).itemsize
    data = fh.read(count)
    if len(data) != count:
        log.warning('read_bytes: failed to read all bytes (%i < %i)',
                    len(data), count)
    return data


Instruction:
Write a Python 3 function that can
read tag data from file and return as unicode string.

Response:
def read_utf8(fh, byteorder, dtype, count, offsetsize):
    """Read tag data from file and return as unicode string."""
    return fh.read(count).decode('utf-8')


Instruction:
Implement a function in Python 3 to
read tag data from file and return as numpy array.

Response:
def read_numpy(fh, byteorder, dtype, count, offsetsize):
    """Read tag data from file and return as numpy array."""
    dtype = 'b' if dtype[-1] == 's' else byteorder+dtype[-1]
    return fh.read_array(dtype, count)


Instruction:
Make a summary of the following Python 3 code
def read_colormap(fh, byteorder, dtype, count, offsetsize):
    cmap = fh.read_array(byteorder+dtype[-1], count)
    cmap.shape = (3, -1)
    return cmap

Response: Read ColorMap data from file and return as numpy array.


Instruction:
Can you write a function in Python 3 where it
reads JSON tag data from file and return as object.

Response:
def read_json(fh, byteorder, dtype, count, offsetsize):
    """Read JSON tag data from file and return as object."""
    data = fh.read(count)
    try:
        return json.loads(unicode(stripnull(data), 'utf-8'))
    except ValueError:
        log.warning('read_json: invalid JSON')


Instruction:
Can you implement a function in Python 3 that
reads FluoView mm_header tag from file and return as dict.

Response:
def read_mm_header(fh, byteorder, dtype, count, offsetsize):
    """Read FluoView mm_header tag from file and return as dict."""
    mmh = fh.read_record(TIFF.MM_HEADER, byteorder=byteorder)
    mmh = recarray2dict(mmh)
    mmh['Dimensions'] = [
        (bytes2str(d[0]).strip(), d[1], d[2], d[3], bytes2str(d[4]).strip())
        for d in mmh['Dimensions']]
    d = mmh['GrayChannel']
    mmh['GrayChannel'] = (
        bytes2str(d[0]).strip(), d[1], d[2], d[3], bytes2str(d[4]).strip())
    return mmh


Instruction:
Write a Python 3 script for
reading MetaMorph STK UIC1Tag from file and return as dict.

Response:
def read_uic1tag(fh, byteorder, dtype, count, offsetsize, planecount=None):
    """Read MetaMorph STK UIC1Tag from file and return as dict.

    Return empty dictionary if planecount is unknown.

    """
    assert dtype in ('2I', '1I') and byteorder == '<'
    result = {}
    if dtype == '2I':
        # pre MetaMorph 2.5 (not tested)
        values = fh.read_array('<u4', 2*count).reshape(count, 2)
        result = {'ZDistance': values[:, 0] / values[:, 1]}
    elif planecount:
        for _ in range(count):
            tagid = struct.unpack('<I', fh.read(4))[0]
            if tagid in (28, 29, 37, 40, 41):
                # silently skip unexpected tags
                fh.read(4)
                continue
            name, value = read_uic_tag(fh, tagid, planecount, offset=True)
            result[name] = value
    return result


Instruction:
Can you write a function in Python 3 where it
reads MetaMorph STK UIC2Tag from file and return as dict.

Response:
def read_uic2tag(fh, byteorder, dtype, planecount, offsetsize):
    """Read MetaMorph STK UIC2Tag from file and return as dict."""
    assert dtype == '2I' and byteorder == '<'
    values = fh.read_array('<u4', 6*planecount).reshape(planecount, 6)
    return {
        'ZDistance': values[:, 0] / values[:, 1],
        'DateCreated': values[:, 2],  # julian days
        'TimeCreated': values[:, 3],  # milliseconds
        'DateModified': values[:, 4],  # julian days
        'TimeModified': values[:, 5]}


Instruction:
Write a Python 3 function that can
read MetaMorph STK UIC4Tag from file and return as dict.

Response:
def read_uic4tag(fh, byteorder, dtype, planecount, offsetsize):
    """Read MetaMorph STK UIC4Tag from file and return as dict."""
    assert dtype == '1I' and byteorder == '<'
    result = {}
    while True:
        tagid = struct.unpack('<H', fh.read(2))[0]
        if tagid == 0:
            break
        name, value = read_uic_tag(fh, tagid, planecount, offset=False)
        result[name] = value
    return result


Instruction:
Implement a function in Python 3 to
read a single UIC tag from file and return tag name and value.

Response:
def read_uic_tag(fh, tagid, planecount, offset):
    """Read a single UIC tag value from file and return tag name and value.

    UIC1Tags use an offset.

    """
    def read_int(count=1):
        value = struct.unpack('<%iI' % count, fh.read(4*count))
        return value[0] if count == 1 else value

    try:
        name, dtype = TIFF.UIC_TAGS[tagid]
    except IndexError:
        # unknown tag
        return '_TagId%i' % tagid, read_int()

    Fraction = TIFF.UIC_TAGS[4][1]

    if offset:
        pos = fh.tell()
        if dtype not in (int, None):
            off = read_int()
            if off < 8:
                if dtype is str:
                    return name, ''
                log.warning("read_uic_tag: invalid offset for tag '%s' (%i)",
                            name, off)
                return name, off
            fh.seek(off)

    if dtype is None:
        # skip
        name = '_' + name
        value = read_int()
    elif dtype is int:
        # int
        value = read_int()
    elif dtype is Fraction:
        # fraction
        value = read_int(2)
        value = value[0] / value[1]
    elif dtype is julian_datetime:
        # datetime
        value = julian_datetime(*read_int(2))
    elif dtype is read_uic_image_property:
        # ImagePropertyEx
        value = read_uic_image_property(fh)
    elif dtype is str:
        # pascal string
        size = read_int()
        if 0 <= size < 2**10:
            value = struct.unpack('%is' % size, fh.read(size))[0][:-1]
            value = bytes2str(stripnull(value))
        elif offset:
            value = ''
            log.warning("read_uic_tag: corrupt string in tag '%s'", name)
        else:
            raise ValueError('read_uic_tag: invalid string size %i' % size)
    elif dtype == '%ip':
        # sequence of pascal strings
        value = []
        for _ in range(planecount):
            size = read_int()
            if 0 <= size < 2**10:
                string = struct.unpack('%is' % size, fh.read(size))[0][:-1]
                string = bytes2str(stripnull(string))
                value.append(string)
            elif offset:
                log.warning("read_uic_tag: corrupt string in tag '%s'", name)
            else:
                raise ValueError('read_uic_tag: invalid string size: %i' %
                                 size)
    else:
        # struct or numpy type
        dtype = '<' + dtype
        if '%i' in dtype:
            dtype = dtype % planecount
        if '(' in dtype:
            # numpy type
            value = fh.read_array(dtype, 1)[0]
            if value.shape[-1] == 2:
                # assume fractions
                value = value[..., 0] / value[..., 1]
        else:
            # struct format
            value = struct.unpack(dtype, fh.read(struct.calcsize(dtype)))
            if len(value) == 1:
                value = value[0]

    if offset:
        fh.seek(pos + 4)

    return name, value


Instruction:
Make a summary of the following Python 3 code
def read_uic_image_property(fh):
    # TODO: test this
    size = struct.unpack('B', fh.read(1))[0]
    name = struct.unpack('%is' % size, fh.read(size))[0][:-1]
    flags, prop = struct.unpack('<IB', fh.read(5))
    if prop == 1:
        value = struct.unpack('II', fh.read(8))
        value = value[0] / value[1]
    else:
        size = struct.unpack('B', fh.read(1))[0]
        value = struct.unpack('%is' % size, fh.read(size))[0]
    return dict(name=name, flags=flags, value=value)

Response: Read UIC ImagePropertyEx tag from file and return as dict.


Instruction:
Can you implement a function in Python 3 that
reads CZ_LSMINFO tag from file and return as dict.

Response:
def read_cz_lsminfo(fh, byteorder, dtype, count, offsetsize):
    """Read CZ_LSMINFO tag from file and return as dict."""
    assert byteorder == '<'
    magic_number, structure_size = struct.unpack('<II', fh.read(8))
    if magic_number not in (50350412, 67127628):
        raise ValueError('invalid CZ_LSMINFO structure')
    fh.seek(-8, 1)

    if structure_size < numpy.dtype(TIFF.CZ_LSMINFO).itemsize:
        # adjust structure according to structure_size
        lsminfo = []
        size = 0
        for name, dtype in TIFF.CZ_LSMINFO:
            size += numpy.dtype(dtype).itemsize
            if size > structure_size:
                break
            lsminfo.append((name, dtype))
    else:
        lsminfo = TIFF.CZ_LSMINFO

    lsminfo = fh.read_record(lsminfo, byteorder=byteorder)
    lsminfo = recarray2dict(lsminfo)

    # read LSM info subrecords at offsets
    for name, reader in TIFF.CZ_LSMINFO_READERS.items():
        if reader is None:
            continue
        offset = lsminfo.get('Offset' + name, 0)
        if offset < 8:
            continue
        fh.seek(offset)
        try:
            lsminfo[name] = reader(fh)
        except ValueError:
            pass
    return lsminfo


Instruction:
Create a Python 3 function to
read LSM sequence of float pairs from file and return as list.

Response:
def read_lsm_floatpairs(fh):
    """Read LSM sequence of float pairs from file and return as list."""
    size = struct.unpack('<i', fh.read(4))[0]
    return fh.read_array('<2f8', count=size)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_lsm_positions(fh):
    size = struct.unpack('<I', fh.read(4))[0]
    return fh.read_array('<2f8', count=size)

Response: Read LSM positions from file and return as list.


Instruction:
How would you explain what the following Python 3 function does
def read_lsm_timestamps(fh):
    size, count = struct.unpack('<ii', fh.read(8))
    if size != (8 + 8 * count):
        log.warning('read_lsm_timestamps: invalid LSM TimeStamps block')
        return []
    # return struct.unpack('<%dd' % count, fh.read(8*count))
    return fh.read_array('<f8', count=count)

Response: Read LSM time stamps from file and return as list.


Instruction:
How would you implement a function in Python 3 that
reads LSM events from file and return as list of ( time type text ).

Response:
def read_lsm_eventlist(fh):
    """Read LSM events from file and return as list of (time, type, text)."""
    count = struct.unpack('<II', fh.read(8))[1]
    events = []
    while count > 0:
        esize, etime, etype = struct.unpack('<IdI', fh.read(16))
        etext = bytes2str(stripnull(fh.read(esize - 16)))
        events.append((etime, etype, etext))
        count -= 1
    return events


Instruction:
Can you implement a function in Python 3 that
reads LSM ChannelColors structure from file and return as dict.

Response:
def read_lsm_channelcolors(fh):
    """Read LSM ChannelColors structure from file and return as dict."""
    result = {'Mono': False, 'Colors': [], 'ColorNames': []}
    pos = fh.tell()
    (size, ncolors, nnames,
     coffset, noffset, mono) = struct.unpack('<IIIIII', fh.read(24))
    if ncolors != nnames:
        log.warning(
            'read_lsm_channelcolors: invalid LSM ChannelColors structure')
        return result
    result['Mono'] = bool(mono)
    # Colors
    fh.seek(pos + coffset)
    colors = fh.read_array('uint8', count=ncolors*4).reshape((ncolors, 4))
    result['Colors'] = colors.tolist()
    # ColorNames
    fh.seek(pos + noffset)
    buffer = fh.read(size - noffset)
    names = []
    while len(buffer) > 4:
        size = struct.unpack('<I', buffer[:4])[0]
        names.append(bytes2str(buffer[4:3+size]))
        buffer = buffer[4+size:]
    result['ColorNames'] = names
    return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_lsm_scaninfo(fh):
    block = {}
    blocks = [block]
    unpack = struct.unpack
    if struct.unpack('<I', fh.read(4))[0] != 0x10000000:
        # not a Recording sub block
        log.warning('read_lsm_scaninfo: invalid LSM ScanInfo structure')
        return block
    fh.read(8)
    while True:
        entry, dtype, size = unpack('<III', fh.read(12))
        if dtype == 2:
            # ascii
            value = bytes2str(stripnull(fh.read(size)))
        elif dtype == 4:
            # long
            value = unpack('<i', fh.read(4))[0]
        elif dtype == 5:
            # rational
            value = unpack('<d', fh.read(8))[0]
        else:
            value = 0
        if entry in TIFF.CZ_LSMINFO_SCANINFO_ARRAYS:
            blocks.append(block)
            name = TIFF.CZ_LSMINFO_SCANINFO_ARRAYS[entry]
            newobj = []
            block[name] = newobj
            block = newobj
        elif entry in TIFF.CZ_LSMINFO_SCANINFO_STRUCTS:
            blocks.append(block)
            newobj = {}
            block.append(newobj)
            block = newobj
        elif entry in TIFF.CZ_LSMINFO_SCANINFO_ATTRIBUTES:
            name = TIFF.CZ_LSMINFO_SCANINFO_ATTRIBUTES[entry]
            block[name] = value
        elif entry == 0xffffffff:
            # end sub block
            block = blocks.pop()
        else:
            # unknown entry
            block['Entry0x%x' % entry] = value
        if not blocks:
            break
    return block

Response: Read LSM ScanInfo structure from file and return as dict.


Instruction:
Can you generate the documentation for the following Python 3 function
def read_sis(fh, byteorder, dtype, count, offsetsize):
    result = {}

    (magic, _, minute, hour, day, month, year, _, name, tagcount
     ) = struct.unpack('<4s6shhhhh6s32sh', fh.read(60))

    if magic != b'SIS0':
        raise ValueError('invalid OlympusSIS structure')

    result['name'] = bytes2str(stripnull(name))
    try:
        result['datetime'] = datetime.datetime(1900+year, month+1, day,
                                               hour, minute)
    except ValueError:
        pass

    data = fh.read(8 * tagcount)
    for i in range(0, tagcount*8, 8):
        tagtype, count, offset = struct.unpack('<hhI', data[i:i+8])
        fh.seek(offset)
        if tagtype == 1:
            # general data
            (_, lenexp, xcal, ycal, _, mag, _, camname, pictype,
             ) = struct.unpack('<10shdd8sd2s34s32s', fh.read(112))  # 220
            m = math.pow(10, lenexp)
            result['pixelsizex'] = xcal * m
            result['pixelsizey'] = ycal * m
            result['magnification'] = mag
            result['cameraname'] = bytes2str(stripnull(camname))
            result['picturetype'] = bytes2str(stripnull(pictype))
        elif tagtype == 10:
            # channel data
            continue
            # TODO: does not seem to work?
            # (length, _, exptime, emv, _, camname, _, mictype,
            #  ) = struct.unpack('<h22sId4s32s48s32s', fh.read(152))  # 720
            # result['exposuretime'] = exptime
            # result['emvoltage'] = emv
            # result['cameraname2'] = bytes2str(stripnull(camname))
            # result['microscopename'] = bytes2str(stripnull(mictype))

    return result

Response: Read OlympusSIS structure and return as dict.


Instruction:
Can you implement a function in Python 3 that
reads OlympusSIS INI string and return as dict.

Response:
def read_sis_ini(fh, byteorder, dtype, count, offsetsize):
    """Read OlympusSIS INI string and return as dict."""
    inistr = fh.read(count)
    inistr = bytes2str(stripnull(inistr))
    try:
        return olympusini_metadata(inistr)
    except Exception as exc:
        log.warning('olympusini_metadata: %s: %s', exc.__class__.__name__, exc)
        return {}


Instruction:
Can you implement a function in Python 3 that
reads TVIPS EM - Menu headers and return as dict.

Response:
def read_tvips_header(fh, byteorder, dtype, count, offsetsize):
    """Read TVIPS EM-MENU headers and return as dict."""
    result = {}
    header = fh.read_record(TIFF.TVIPS_HEADER_V1, byteorder=byteorder)
    for name, typestr in TIFF.TVIPS_HEADER_V1:
        result[name] = header[name].tolist()
    if header['Version'] == 2:
        header = fh.read_record(TIFF.TVIPS_HEADER_V2, byteorder=byteorder)
        if header['Magic'] != int(0xaaaaaaaa):
            log.warning('read_tvips_header: invalid TVIPS v2 magic number')
            return {}
        # decode utf16 strings
        for name, typestr in TIFF.TVIPS_HEADER_V2:
            if typestr.startswith('V'):
                s = header[name].tostring().decode('utf16', errors='ignore')
                result[name] = stripnull(s, null='\0')
            else:
                result[name] = header[name].tolist()
        # convert nm to m
        for axis in 'XY':
            header['PhysicalPixelSize' + axis] /= 1e9
            header['PixelSize' + axis] /= 1e9
    elif header.version != 1:
        log.warning('read_tvips_header: unknown TVIPS header version')
        return {}
    return result


Instruction:
Here you have a function in Python 3, explain what it does
def read_fei_metadata(fh, byteorder, dtype, count, offsetsize):
    result = {}
    section = {}
    data = bytes2str(stripnull(fh.read(count)))
    for line in data.splitlines():
        line = line.strip()
        if line.startswith('['):
            section = {}
            result[line[1:-1]] = section
            continue
        try:
            key, value = line.split('=')
        except ValueError:
            continue
        section[key] = astype(value)
    return result

Response: Read FEI SFEG headers and return as dict.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_cz_sem(fh, byteorder, dtype, count, offsetsize):
    result = {'': ()}
    key = None
    data = bytes2str(stripnull(fh.read(count)))
    for line in data.splitlines():
        if line.isupper():
            key = line.lower()
        elif key:
            try:
                name, value = line.split('=')
            except ValueError:
                try:
                    name, value = line.split(':', 1)
                except Exception:
                    continue
            value = value.strip()
            unit = ''
            try:
                v, u = value.split()
                number = astype(v, (int, float))
                if number != v:
                    value = number
                    unit = u
            except Exception:
                number = astype(value, (int, float))
                if number != value:
                    value = number
                if value in ('No', 'Off'):
                    value = False
                elif value in ('Yes', 'On'):
                    value = True
            result[key] = (name.strip(), value)
            if unit:
                result[key] += (unit,)
            key = None
        else:
            result[''] += (astype(line, (int, float)),)
    return result

Response: Read Zeiss SEM tag and return as dict.


Instruction:
How would you explain what the following Python 3 function does
def read_nih_image_header(fh, byteorder, dtype, count, offsetsize):
    a = fh.read_record(TIFF.NIH_IMAGE_HEADER, byteorder=byteorder)
    a = a.newbyteorder(byteorder)
    a = recarray2dict(a)
    a['XUnit'] = a['XUnit'][:a['XUnitSize']]
    a['UM'] = a['UM'][:a['UMsize']]
    return a

Response: Read NIH_IMAGE_HEADER tag from file and return as dict.


Instruction:
Write a Python 3 function for
reading ScanImage BigTIFF v3 static and ROI metadata from open file.

Response:
def read_scanimage_metadata(fh):
    """Read ScanImage BigTIFF v3 static and ROI metadata from open file.

    Return non-varying frame data as dict and ROI group data as JSON.

    The settings can be used to read image data and metadata without parsing
    the TIFF file.

    Raise ValueError if file does not contain valid ScanImage v3 metadata.

    """
    fh.seek(0)
    try:
        byteorder, version = struct.unpack('<2sH', fh.read(4))
        if byteorder != b'II' or version != 43:
            raise Exception
        fh.seek(16)
        magic, version, size0, size1 = struct.unpack('<IIII', fh.read(16))
        if magic != 117637889 or version != 3:
            raise Exception
    except Exception:
        raise ValueError('not a ScanImage BigTIFF v3 file')

    frame_data = matlabstr2py(bytes2str(fh.read(size0)[:-1]))
    roi_data = read_json(fh, '<', None, size1, None) if size1 > 1 else {}
    return frame_data, roi_data


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_micromanager_metadata(fh):
    fh.seek(0)
    try:
        byteorder = {b'II': '<', b'MM': '>'}[fh.read(2)]
    except IndexError:
        raise ValueError('not a MicroManager TIFF file')

    result = {}
    fh.seek(8)
    (index_header, index_offset, display_header, display_offset,
     comments_header, comments_offset, summary_header, summary_length
     ) = struct.unpack(byteorder + 'IIIIIIII', fh.read(32))

    if summary_header != 2355492:
        raise ValueError('invalid MicroManager summary header')
    result['Summary'] = read_json(fh, byteorder, None, summary_length, None)

    if index_header != 54773648:
        raise ValueError('invalid MicroManager index header')
    fh.seek(index_offset)
    header, count = struct.unpack(byteorder + 'II', fh.read(8))
    if header != 3453623:
        raise ValueError('invalid MicroManager index header')
    data = struct.unpack(byteorder + 'IIIII'*count, fh.read(20*count))
    result['IndexMap'] = {'Channel': data[::5],
                          'Slice': data[1::5],
                          'Frame': data[2::5],
                          'Position': data[3::5],
                          'Offset': data[4::5]}

    if display_header != 483765892:
        raise ValueError('invalid MicroManager display header')
    fh.seek(display_offset)
    header, count = struct.unpack(byteorder + 'II', fh.read(8))
    if header != 347834724:
        raise ValueError('invalid MicroManager display header')
    result['DisplaySettings'] = read_json(fh, byteorder, None, count, None)

    if comments_header != 99384722:
        raise ValueError('invalid MicroManager comments header')
    fh.seek(comments_offset)
    header, count = struct.unpack(byteorder + 'II', fh.read(8))
    if header != 84720485:
        raise ValueError('invalid MicroManager comments header')
    result['Comments'] = read_json(fh, byteorder, None, count, None)

    return result

Response: Read MicroManager non - TIFF metadata from open file and return as dict.


Instruction:
Make a summary of the following Python 3 code
def imagej_metadata_tag(metadata, byteorder):
    header = [{'>': b'IJIJ', '<': b'JIJI'}[byteorder]]
    bytecounts = [0]
    body = []

    def _string(data, byteorder):
        return data.encode('utf-16' + {'>': 'be', '<': 'le'}[byteorder])

    def _doubles(data, byteorder):
        return struct.pack(byteorder+('d' * len(data)), *data)

    def _ndarray(data, byteorder):
        return data.tobytes()

    def _bytes(data, byteorder):
        return data

    metadata_types = (
        ('Info', b'info', 1, _string),
        ('Labels', b'labl', None, _string),
        ('Ranges', b'rang', 1, _doubles),
        ('LUTs', b'luts', None, _ndarray),
        ('Plot', b'plot', 1, _bytes),
        ('ROI', b'roi ', 1, _bytes),
        ('Overlays', b'over', None, _bytes))

    for key, mtype, count, func in metadata_types:
        if key.lower() in metadata:
            key = key.lower()
        elif key not in metadata:
            continue
        if byteorder == '<':
            mtype = mtype[::-1]
        values = metadata[key]
        if count is None:
            count = len(values)
        else:
            values = [values]
        header.append(mtype + struct.pack(byteorder+'I', count))
        for value in values:
            data = func(value, byteorder)
            body.append(data)
            bytecounts.append(len(data))

    if not body:
        return ()
    body = b''.join(body)
    header = b''.join(header)
    data = header + body
    bytecounts[0] = len(header)
    bytecounts = struct.pack(byteorder+('I' * len(bytecounts)), *bytecounts)
    return ((50839, 'B', len(data), data, True),
            (50838, 'I', len(bytecounts)//4, bytecounts, True))

Response: Return IJMetadata and IJMetadataByteCounts tags from the metadata dict.


Instruction:
Can you create a Python 3 function that
returns IJMetadata tag value as dict.

Response:
def imagej_metadata(data, bytecounts, byteorder):
    """Return IJMetadata tag value as dict.

    The 'Info' string can have multiple formats, e.g. OIF or ScanImage,
    that might be parsed into dicts using the matlabstr2py or
    oiffile.SettingsFile functions.

    """
    def _string(data, byteorder):
        return data.decode('utf-16' + {'>': 'be', '<': 'le'}[byteorder])

    def _doubles(data, byteorder):
        return struct.unpack(byteorder+('d' * (len(data) // 8)), data)

    def _lut(data, byteorder):
        return numpy.frombuffer(data, 'uint8').reshape(-1, 256)

    def _bytes(data, byteorder):
        return data

    metadata_types = {  # big-endian
        b'info': ('Info', _string),
        b'labl': ('Labels', _string),
        b'rang': ('Ranges', _doubles),
        b'luts': ('LUTs', _lut),
        b'plot': ('Plots', _bytes),
        b'roi ': ('ROI', _bytes),
        b'over': ('Overlays', _bytes)}
    metadata_types.update(  # little-endian
        dict((k[::-1], v) for k, v in metadata_types.items()))

    if not bytecounts:
        raise ValueError('no ImageJ metadata')

    if not data[:4] in (b'IJIJ', b'JIJI'):
        raise ValueError('invalid ImageJ metadata')

    header_size = bytecounts[0]
    if header_size < 12 or header_size > 804:
        raise ValueError('invalid ImageJ metadata header size')

    ntypes = (header_size - 4) // 8
    header = struct.unpack(byteorder+'4sI'*ntypes, data[4:4+ntypes*8])
    pos = 4 + ntypes * 8
    counter = 0
    result = {}
    for mtype, count in zip(header[::2], header[1::2]):
        values = []
        name, func = metadata_types.get(mtype, (bytes2str(mtype), read_bytes))
        for _ in range(count):
            counter += 1
            pos1 = pos + bytecounts[counter]
            values.append(func(data[pos:pos1], byteorder))
            pos = pos1
        result[name.strip()] = values[0] if count == 1 else values
    return result


Instruction:
Can you tell what is the following Python 3 function doing
def imagej_description_metadata(description):
    def _bool(val):
        return {'true': True, 'false': False}[val.lower()]

    result = {}
    for line in description.splitlines():
        try:
            key, val = line.split('=')
        except Exception:
            continue
        key = key.strip()
        val = val.strip()
        for dtype in (int, float, _bool):
            try:
                val = dtype(val)
                break
            except Exception:
                pass
        result[key] = val

    if 'ImageJ' not in result:
        raise ValueError('not a ImageJ image description')
    return result

Response: Return metatata from ImageJ image description as dict.