Instruction:
Explain what the following Python 3 code does
def add_empty_magic_table(self, dtype, col_names=None, groups=None):
        if dtype not in self.table_names:
            print("-W- {} is not a valid MagIC table name".format(dtype))
            print("-I- Valid table names are: {}".format(", ".join(self.table_names)))
            return
        data_container = MagicDataFrame(dtype=dtype, columns=col_names, groups=groups)
        self.tables[dtype] = data_container

Response: Add a blank MagicDataFrame to the contribution.


Instruction:
Write a Python 3 function that can
add a MagIC table to the contribution from a data list

Response:
def add_magic_table_from_data(self, dtype, data):
        """
        Add a MagIC table to the contribution from a data list

        Parameters
        ----------
        dtype : str
            MagIC table type, i.e. 'specimens'
        data : list of dicts
            data list with format [{'key1': 'val1', ...}, {'key1': 'val2', ...}, ... }]
        """
        self.tables[dtype] = MagicDataFrame(dtype=dtype, data=data)
        if dtype == 'measurements':
            self.tables['measurements'].add_sequence()
        return dtype, self.tables[dtype]


Instruction:
Here you have a function in Python 3, explain what it does
def add_magic_table(self, dtype, fname=None, df=None):
        if df is None:
            # if providing a filename but no data type
            if dtype == "unknown":
                filename = os.path.join(self.directory, fname)
                if not os.path.exists(filename):
                    return False, False
                data_container = MagicDataFrame(filename, dmodel=self.data_model)
                dtype = data_container.dtype
                if dtype == 'empty':
                    return False, False
                else:
                    self.tables[dtype] = data_container
                    return dtype, data_container
            # if providing a data type, use the canonical filename
            elif dtype not in self.filenames:
                print('-W- "{}" is not a valid MagIC table type'.format(dtype))
                print("-I- Available table types are: {}".format(", ".join(self.table_names)))
                return False, False
            #filename = os.path.join(self.directory, self.filenames[dtype])
            filename = pmag.resolve_file_name(self.filenames[dtype], self.directory)
            if os.path.exists(filename):
                data_container = MagicDataFrame(filename, dtype=dtype,
                                                dmodel=self.data_model)
                if data_container.dtype != "empty":
                    self.tables[dtype] = data_container
                    return dtype, data_container
                else:
                    return False, False
            else:
                #print("-W- No such file: {}".format(filename))
                return False, False
        # df is not None
        else:
            if not dtype:
                print("-W- Must provide dtype")
                return False, False
            data_container = MagicDataFrame(dtype=dtype, df=df)
            self.tables[dtype] = data_container
        self.tables[dtype].sort_dataframe_cols()
        return dtype, self.tables[dtype]

Response: Read in a new file to add a new table to self. tables.


Instruction:
How would you explain what the following Python 3 function does
def propagate_measurement_info(self):
        meas_df = self.tables['measurements'].df
        names_list = ['specimen', 'sample', 'site', 'location']
        # add in any tables that you can
        for num, name in enumerate(names_list):
            # don't replace tables that already exist
            if (name + "s") in self.tables:
                continue
            elif name in meas_df.columns:
                items = meas_df[name].unique()
                df = pd.DataFrame(columns=[name], index=items)
                df[name] = df.index
                # add in parent name if possible
                # (i.e., sample name to specimens table)
                if num < (len(names_list) - 1):
                    parent = names_list[num+1]
                    if parent in meas_df.columns:
                        meas_df = meas_df.where(meas_df.notnull(), "")
                        df[parent] = meas_df.drop_duplicates(subset=[name])[parent].values.astype(str)
                df = df.where(df != "", np.nan)
                df = df.dropna(how='all', axis='rows')
                if len(df):
                    self.tables[name + "s"] = MagicDataFrame(dtype=name + "s", df=df)
                    self.write_table_to_file(name + "s")

Response: Take a contribution with a measurement table and create the index for the unique names in the measurement table.


Instruction:
How would you code a function in Python 3 to
propagate all items from all tables to the file.

Response:
def propagate_all_tables_info(self, write=True):
        """
        Find any items (specimens, samples, sites, or locations) from
        tables other than measurements and make sure they each have a
        row in their own table.  For example, if a site name is in
        the samples table but not in the sites table, create a row
        for it in the sites table.
        """
        for table_name in ["specimens", "samples", "sites", "locations"]:
            if not table_name in self.tables:
                continue
            df = self.tables[table_name].df
            parent_name, child_name = self.get_parent_and_child(table_name)
            if parent_name:
                if parent_name[:-1] in df.columns:
                    parents = sorted(set(df[parent_name[:-1]].dropna().values.astype(str)))
                    if parent_name in self.tables: # if there is a parent table, update it
                        parent_df = self.tables[parent_name].df
                        missing_parents = set(parents) - set(parent_df.index)
                        if missing_parents: # add any missing values
                            print("-I- Updating {} table with values from {} table".format(parent_name, table_name))
                            for item in missing_parents:
                                self.add_item(parent_name, {parent_name[:-1]: item}, label=item)
                            # save any changes to file
                            if write:
                                self.write_table_to_file(parent_name)

                    else:  # if there is no parent table, create it if necessary
                        if parents:
                            # create a parent_df with the names you got from the child
                            print("-I- Creating new {} table with data from {} table".format(parent_name, table_name))
                            # add in the grandparent if available
                            grandparent_name = self.get_parent_and_child(parent_name)[0]
                            if grandparent_name:
                                grandparent = ""
                                if grandparent_name in df.columns:
                                    grandparent = df[df[parent_name] == item][grandparent_name].values[0]
                                columns = [parent_name[:-1]]#, grandparent_name[:-1]]
                            else:
                                columns = [parent_name[:-1]]

                            parent_df = pd.DataFrame(columns=columns, index=parents)
                            parent_df[parent_name[:-1]] = parent_df.index
                            if grandparent_name:
                                if grandparent_name[:-1] in df.columns:
                                    parent_df = pd.merge(df[[parent_name[:-1], grandparent_name[:-1]]], parent_df, on=parent_name[:-1])
                            self.tables[parent_name] = MagicDataFrame(dtype=parent_name,
                                                                      df=parent_df)
                            if write:
                                # save new table to file
                                self.write_table_to_file(parent_name)
            if child_name:
                if child_name in df.columns:
                    raw_children = df[child_name].dropna().str.split(':')
                    # create dict of all children with parent info
                    parent_of_child = {}
                    for parent, children in raw_children.items():
                        for child in children:
                            # remove whitespace
                            child = child.strip()
                            old_parent = parent_of_child.get(child)
                            if old_parent and parent and (old_parent != parent):
                                print('-I- for {} {}, replacing: {} with: {}'.format(child_name[:-1], child,
                                                                                old_parent, parent))
                            parent_of_child[child] = parent
                    # old way:
                    # flatten list, ignore duplicates
                    #children = sorted(set([item.strip() for sublist in raw_children for item in sublist]))
                    if child_name in self.tables: # if there is already a child table, update it
                        child_df = self.tables[child_name].df
                        missing_children = set(parent_of_child.keys()) - set(child_df.index)
                        if missing_children: # add any missing values
                            print("-I- Updating {} table with values from {} table".format(child_name, table_name))
                            for item in missing_children:
                                data = {child_name[:-1]: item, table_name[:-1]: parent_of_child[item]}
                                self.add_item(child_name, data, label=item)
                            if write:
                                # save any changes to file
                                self.write_table_to_file(child_name)
                    else: # if there is no child table, create it if necessary
                        if children:
                            # create a child_df with the names you got from the parent
                            print("-I- Creating new {} table with data from {} table".format(child_name, table_name))
                            # old way to make new table:
                            #child_df = pd.DataFrame(columns=[table_name[:-1]], index=children)
                            # new way to make new table
                            children_list = sorted(parent_of_child.keys())
                            children_data = [[child_name, parent_of_child[c_name]] for c_name in children_list]
                            child_df = pd.DataFrame(index=children_list, columns=[child_name[:-1], table_name[:-1]], data=children_data)

                            self.tables[child_name] = MagicDataFrame(dtype=child_name, df=child_df)
                            if write:
                                # save new table to file
                                self.write_table_to_file(child_name)


Instruction:
Explain what the following Python 3 code does
def get_parent_and_child(self, table_name):
        if table_name not in self.ancestry:
            return None, None
        parent_ind = self.ancestry.index(table_name) + 1
        if parent_ind + 1 > len(self.ancestry):
            parent_name = None
        else:
            parent_name = self.ancestry[parent_ind]
        child_ind = self.ancestry.index(table_name) - 1
        if child_ind < 0:
            child_name = None
        else:
            child_name = self.ancestry[child_ind]
        return parent_name, child_name

Response: Get the name of the parent table and child table for a given MagIC table name.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_min_max_lat_lon(self):
        if 'sites' not in self.tables:
            return
        # get min/max lat/lon from sites table
        site_container = self.tables['sites']
        if not ('lat' in site_container.df.columns and 'lon' in site_container.df.columns):
            return
        # convert lat/lon columns to string type
        # (this is necessary for consistency because they MAY be string type already)
        site_container.df['lat'] = site_container.df['lat'].fillna('').astype(str)
        site_container.df['lon'] = site_container.df['lon'].fillna('').astype(str)
        # replace empty strings with np.nan
        site_container.df['lat'] = np.where(site_container.df['lat'].str.len(), site_container.df['lat'], np.nan)
        site_container.df['lon'] = np.where(site_container.df['lon'].str.len(), site_container.df['lon'], np.nan)
        # convert lat/lon values to float (they make be string from grid)
        try:
            site_container.df['lat'] = site_container.df['lat'].astype(float)
        except ValueError as ex:
            print('-W- Improperly formatted numbers in sites.lat')
            return
        try:
            site_container.df['lon'] = site_container.df['lon'].astype(float)
        except ValueError as ex:
            print('-W- Improperly formatted numbers in sites.lon')
            return
        # group lat/lon by location
        grouped_lon = site_container.df[['lon', 'location']].groupby('location')
        grouped_lat = site_container.df[['lat', 'location']].groupby('location')
        # get min/max longitude by location
        lon_w = grouped_lon.min()
        lon_e = grouped_lon.max()
        # get min/max latitude by location
        lat_s = grouped_lat.min()
        lat_n = grouped_lat.max()
        # assign lat/lon to location table
        locs = {}
        if 'locations' not in self.tables:
            return
        loc_container = self.tables['locations']
        for loc in lat_s.index:
            coords = {}
            coords['lat_s'] = lat_s.loc[loc]['lat']
            coords['lat_n'] = lat_n.loc[loc]['lat']
            coords['lon_e'] = lon_e.loc[loc]['lon']
            coords['lon_w'] = lon_w.loc[loc]['lon']
            locs[loc] = coords
        loc_container = self.tables['locations']
        for loc_name in locs:
            if loc_name in loc_container.df.index:
                coords = locs[loc_name]
                for coord in locs[loc_name]:
                    # warn user if an old value will be overwritten
                    new_value = coords[coord]
                    # if the new value is null, ignore it
                    if is_null(new_value, zero_as_null=False):
                        continue
                    # set old value to None if it wasn't in table
                    if coord not in loc_container.df.columns:
                        loc_container.df[coord] = None
                    old_value = loc_container.df.loc[loc_name, coord]
                    # use first value if multiple values returned, but don't shorten a string
                    if not (isinstance(old_value, str)):
                        try:
                            old_value = old_value.values.astype(str)[0]
                        except (TypeError,IndexError,AttributeError) as e: # if only one value, or np.nan, or NoneType
                            pass
                    if is_null(old_value, zero_as_null=False):
                        pass
                    elif isinstance(old_value, str):
                        try:
                            old_value = float(old_value)
                        except ValueError:
                            print('-W- In {}, automatically generated {} value ({}) will overwrite previous value ({})'.format(loc_name, coord, new_value, old_value))
                            old_value = None
                    elif not math.isclose(new_value, old_value):
                        print('-W- In {}, automatically generated {} value ({}) will overwrite previous value ({})'.format(loc_name, coord, new_value, old_value))
                    # set new value
                    new_value = round(float(new_value), 5)
                    loc_container.df.loc[loc_name, coord] = new_value
        self.write_table_to_file('locations')
        return locs

Response:
Find latitude and longitude information from sites table
            and return the minimum and maximum latitude.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def propagate_lithology_cols(self):
        cols = ['lithologies', 'geologic_types', 'geologic_classes']
        #for table in ['specimens', 'samples']:
            # convert "Not Specified" to blank
            #self.tables[table].df.replace("^[Nn]ot [Ss]pecified", '',
            #                              regex=True, inplace=True)
        self.propagate_cols(cols, 'samples', 'sites')
        cols = ['lithologies', 'geologic_types', 'geologic_classes']
        self.propagate_cols(cols, 'specimens', 'samples')
        # if sites table is missing any values,
        # go ahead and propagate values UP as well
        if 'sites' not in self.tables:
            return
        for col in cols:
            if col not in self.tables['sites'].df.columns:
                self.tables['sites'].df[col] = None
        if not all(self.tables['sites'].df[cols].values.ravel()):
            print('-I- Propagating values up from samples to sites...')
            self.propagate_cols_up(cols, 'sites', 'samples')

Response: Propagate any data from lithologies geologic_types or geologic_classes to the samples and specimens tables.


Instruction:
Can you implement a function in Python 3 that
renames an item in a table.

Response:
def rename_item(self, table_name, item_old_name, item_new_name):
        """
        Rename item (such as a site) everywhere that it occurs.
        This change often spans multiple tables.
        For example, a site name will occur in the sites table,
        the samples table, and possibly in the locations/ages tables.
        """
        # define some helper methods:

        def put_together_if_list(item):
            """
            String joining function
            that doesn't break with None/np.nan
            """
            try:
                res = ":".join(item)
                return ":".join(item)
            except TypeError as ex:
                #print ex
                return item

        def replace_colon_delimited_value(df, col_name, old_value, new_value):
            """
            Col must contain list
            """
            count = 1
            for index, row in df[df[col_name].notnull()].iterrows():
                names_list = row[col_name]
                names_list = [name.strip() for name in names_list]
                try:
                    ind = names_list.index(old_value)
                except ValueError as ex:
                    count += 1
                    continue
                names_list[ind] = new_value
                df.loc[count, col_name] = names_list
                count += 1

        # initialize some things
        item_type = table_name
        ###col_name = item_type[:-1] + "_name"
        col_name = item_type[:-1]
        col_name_plural = col_name + "s"
        table_df = self.tables[item_type].df

        if item_old_name == '':
            # just add a new item
            self.add_item(table_name, {col_name: item_new_name}, item_new_name)
            return

        # rename item in its own table
        table_df.rename(index={item_old_name: item_new_name}, inplace=True)
        # rename in any parent/child tables
        for table_name in self.tables:
            df = self.tables[table_name].df
            col_names = df.columns
            # change anywhere col_name (singular, i.e. site) is found
            if col_name in col_names:
                df[col_name].where(df[col_name] != item_old_name, item_new_name, inplace=True)
                # change anywhere col_name (plural, i.e. sites) is found
            if col_name_plural in col_names:
                df[col_name_plural + "_list"] = df[col_name_plural].str.split(":")
                replace_colon_delimited_value(df, col_name_plural + "_list", item_old_name, item_new_name)
                df[col_name_plural] = df[col_name_plural + "_list"].apply(put_together_if_list)
                df.drop(col_name_plural + "_list", axis=1, inplace=True)
            self.tables[table_name].df = df


Instruction:
How would you implement a function in Python 3 that
returns both the table_name and col_name for a given index in the ancestry.

Response:
def get_table_name(self, ind):
        """
        Return both the table_name (i.e., 'specimens')
        and the col_name (i.e., 'specimen')
        for a given index in self.ancestry.
        """
        if ind >= len(self.ancestry):
            return "", ""
        if ind > -1:
            table_name = self.ancestry[ind]
            ###name = table_name[:-1] + "_name"
            name = table_name[:-1]
            return table_name, name
        return "", ""


Instruction:
Can you generate the documentation for the following Python 3 function
def propagate_name_down(self, col_name, df_name, verbose=False):
        if df_name not in self.tables:
            table = self.add_magic_table(df_name)[1]
            if is_null(table):
                return
        df = self.tables[df_name].df
        if col_name in df.columns:
            if all(df[col_name].apply(not_null)):
                #print('{} already in {}'.format(col_name, df_name))
                return df

        # otherwise, do necessary merges to get col_name into df
        # get names for each level
        grandparent_table_name = col_name.split('_')[0] + "s"
        grandparent_name = grandparent_table_name[:-1]
        ind = self.ancestry.index(grandparent_table_name) - 1
        #
        parent_table_name, parent_name = self.get_table_name(ind)
        child_table_name, child_name = self.get_table_name(ind - 1)
        bottom_table_name, bottom_name = self.get_table_name(ind - 2)

        # merge in bottom level
        if child_name not in df.columns:
            # add child table if missing
            if bottom_table_name not in self.tables:
                result = self.add_magic_table(bottom_table_name)[1]
                if not isinstance(result, MagicDataFrame):
                    if verbose:
                        print("-W- Couldn't read in {} data for data propagation".format(bottom_table_name))
                    return df
            # add child_name to df
            add_df = self.tables[bottom_table_name].df
            # drop duplicate names
            add_df = add_df.drop_duplicates(subset=bottom_name)
            if child_name not in df.columns:
                if verbose:
                    print("-W- Cannot complete propagation, {} table is missing {} column".format(df_name, child_name))
            else:
                add_df = stringify_col(add_df, child_name)
                df = stringify_col(df, bottom_name)
                df = df.merge(add_df[[child_name]],
                              left_on=[bottom_name],
                              right_index=True, how="left")
                self.tables[df_name].df = df

        # merge in one level above
        if parent_name not in df.columns:
            # add parent_table if missing
            if child_table_name not in self.tables:
                result = self.add_magic_table(child_table_name)[1]
                if not isinstance(result, MagicDataFrame):
                    if verbose:
                        print("-W- Couldn't read in {} data".format(child_table_name))
                        print("-I- Make sure you've provided the correct file name")
                    return df
            # add parent_name to df
            add_df = self.tables[child_table_name].df
            # drop duplicate names
            add_df = add_df.drop_duplicates(subset=child_name)
            if parent_name not in add_df:
                if verbose:
                    print('-W- could not finish propagating names: {} table is missing {} column'.format(child_table_name, parent_name))
            elif parent_name not in df:
                if verbose:
                    print('-W- could not finish propagating names: {} table is missing {} column'.format(df_name, parent_name))
            else:
                add_df = stringify_col(add_df, parent_name)
                df = stringify_col(df, child_name)
                df = df.merge(add_df[[parent_name]],
                              left_on=[child_name],
                              right_index=True, how="left")
                self.tables[df_name].df = df

        # merge in two levels above
        if grandparent_name not in df.columns:
            # add grandparent table if it is missing
            if parent_table_name not in self.tables:
                result = self.add_magic_table(parent_table_name)[1]
                if not isinstance(result, MagicDataFrame):
                    if verbose:
                        print("-W- Couldn't read in {} data".format(parent_table_name))
                        print("-I- Make sure you've provided the correct file name")
                    return df
            # add grandparent name to df
            add_df = self.tables[parent_table_name].df
            # drop duplicate names
            add_df = add_df.drop_duplicates(subset=parent_name)
            if grandparent_name not in add_df.columns:
                if verbose:
                    print('-W- could not finish propagating names: {} table is missing {} column'.format(parent_table_name, grandparent_name))
            elif parent_name not in df.columns:
                if verbose:
                    print('-W- could not finish propagating names: {} table is missing {} column'.format(df_name, parent_name))
            else:
                add_df = stringify_col(add_df, grandparent_name)
                df = stringify_col(df, parent_name)
                df = df.merge(add_df[[grandparent_name]],
                              left_on=[parent_name],
                              right_index=True, how="left")
                df = stringify_col(df, grandparent_name)
        # update the Contribution
        self.tables[df_name].df = df
        return df

Response: Given a column name and a dataframe of names return the data for col_name.


Instruction:
Create a Python 3 function to
give a list of column names in the source dataframe and a list of columns in the target dataframe propagate them to the target dataframe.

Response:
def propagate_cols(self, col_names, target_df_name, source_df_name,
                       down=True):
        """
        Put the data for "col_name" from source_df into target_df
        Used to get "azimuth" from sample table into measurements table
        (for example).
        Note: if getting data from the sample table, don't include "sample"
        in the col_names list.  It is included automatically.
        """
        # make sure target table is read in
        if target_df_name not in self.tables:
            self.add_magic_table(target_df_name)
        if target_df_name not in self.tables:
            print("-W- Couldn't read in {} table".format(target_df_name))
            return
        # make sure source table is read in
        if source_df_name not in self.tables:
            self.add_magic_table(source_df_name)
            print("-W- Couldn't read in {} table".format(source_df_name))
            return
        # make sure col_names are all available in source table
        source_df = self.tables[source_df_name].df
        if not set(col_names).issubset(source_df.columns):
            for col in col_names[:]:
                if col not in source_df.columns:
                    print("-W- Column '{}' isn't in {} table, skipping it".format(col, source_df_name))
                    col_names.remove(col)
        if not col_names:
            print("-W- Invalid or missing column names, could not propagate columns")
            return
        #
        if down:
            add_name = source_df_name[:-1]
            if 'measurements' in self.tables.keys():
                self.propagate_location_to_measurements()
            elif 'specimens' in self.tables.keys():
                self.propagate_location_to_specimens()
            else:
                self.propagate_name_down('location', 'sites')
        else:
            add_name = target_df_name[:-1]

        # get dataframes for merge
        target_df = self.tables[target_df_name].df
        source_df = self.tables[source_df_name].df
        backup_source_df = source_df.copy()
        # finesse source_df to make sure it has all the right columns
        # and no unnecessary duplicates
        if source_df_name[:-1] not in source_df.columns:
            source_df[source_df_name[:-1]] = source_df.index
        source_df = source_df.drop_duplicates(inplace=False, subset=col_names + [source_df_name[:-1]])
        source_df = source_df.groupby(source_df.index, sort=False).fillna(method='ffill')
        source_df = source_df.groupby(source_df.index, sort=False).fillna(method='bfill')
        # if the groupby/fillna operation fails due to pandas bug, do the same by hand:
        if not len(source_df):
            new = []
            grouped = backup_source_df.groupby(backup_source_df.index)
            for label, group in grouped:
                new_group = group.fillna(method="ffill")
                new_group = new_group.fillna(method="bfill")
                new.append(new_group)
            source_df = pd.concat(new, sort=True)

        # if the groupby/fillna still doesn't work, we are out of luck
        if not len(source_df):
            return target_df
        # propagate down
        if down:
            # do merge
            target_df[add_name] = target_df[add_name].astype(str)
            target_df = target_df.merge(source_df[col_names], how='left',
                                        left_on=add_name, right_index=True,
                                        suffixes=["_target", "_source"])
        # propagate up
        else:
            # do merge
            col_names.append(add_name)
            source_df[add_name] = source_df[add_name].astype(str)
            target_df = target_df.merge(source_df[col_names],
                                        how='left', left_index=True,
                                        right_on=add_name,
                                        suffixes=['_target', '_source'])
            target_df.index = target_df[add_name]
            target_df.drop([add_name + "_source", add_name + "_target"], axis=1, inplace=True)

        # ignore any duplicate rows
        target_df.drop_duplicates(inplace=True)
        # mess with target_df to remove un-needed merge columns
        for col in col_names:
            # if there has been a previous merge, consolidate and delete data
            if col + "_target" in target_df.columns:
                # prioritize values from target df
                new_arr = np.where(target_df[col + "_target"],
                                   target_df[col + "_target"],
                                   target_df[col + "_source"])
                target_df.rename(columns={col + "_target": col}, inplace=True)
                target_df[col] = new_arr
            if col + "_source" in target_df.columns:
                # delete extra merge column
                del target_df[col + "_source"]
        #

        # drop any duplicate rows
        target_df.drop_duplicates(inplace=True)
        self.tables[target_df_name].df = target_df
        return target_df


Instruction:
Can you tell what is the following Python 3 function doing
def propagate_cols_up(self, cols, target_df_name, source_df_name):
        print("-I- Trying to propagate {} columns from {} table into {} table".format(cols,
                                                                                source_df_name,
                                                                                target_df_name))
        # make sure target table is read in
        if target_df_name not in self.tables:
            self.add_magic_table(target_df_name)
        if target_df_name not in self.tables:
            print("-W- Couldn't read in {} table".format(target_df_name))
            return
        # make sure source table is read in
        if source_df_name not in self.tables:
            self.add_magic_table(source_df_name)
            print("-W- Couldn't read in {} table".format(source_df_name))
            return
        target_df = self.tables[target_df_name]
        source_df = self.tables[source_df_name]
        target_name = target_df_name[:-1]
        # make sure source_df has relevant columns
        for col in cols:
            if col not in source_df.df.columns:
                source_df.df[col] = None
        # if target_df has info, propagate that into all rows
        target_df.front_and_backfill(cols)
        # make sure target_name is in source_df for merging
        if target_name not in source_df.df.columns:
            print("-W- You can't merge data from {} table into {} table".format(source_df_name, target_df_name))
            print("    Your {} table is missing {} column".format(source_df_name, target_name))
            self.tables[target_df_name] = target_df
            return target_df
        source_df.front_and_backfill([target_name])
        # group source df by target_name
        grouped = source_df.df.groupby(source_df.df[target_name])
        if not len(grouped):
            print("-W- Couldn't propagate from {} to {}".format(source_df_name, target_df_name))
            return target_df
        # function to generate capitalized, sorted, colon-delimited list
        # of unique, non-null values from a column
        def func(group, col_name):
            lst = group[col_name][group[col_name].notnull()].unique()
            split_lst = [col.split(':') for col in lst if col]
            sorted_lst = sorted(np.unique([item.capitalize() for sublist in split_lst for item in sublist]))
            group_col = ":".join(sorted_lst)
            return group_col
        # apply func to each column
        for col in cols:
            res = grouped.apply(func, col)
            target_df.df['new_' + col] = res
            target_df.df[col] = np.where(target_df.df[col], target_df.df[col], target_df.df['new_' + col])
            target_df.df.drop(['new_' + col], axis='columns', inplace=True)
        # set table
        self.tables[target_df_name] = target_df
        return target_df

Response: Given a list of columns from source table and target table and a list of columns apply them to the target table.


Instruction:
Explain what the following Python 3 code does
def propagate_average_up(self, cols=['lat', 'lon'],
                             target_df_name='sites', source_df_name='samples'):
        """
        Propagate average values from a lower table to a higher one.
        For example, propagate average lats/lons from samples to sites.
        Pre-existing values will not be overwritten.

        Parameters
        ----------
        cols : list-like
            list of columns to propagate
        target_df_name : str
            name of table to propagate values into
        source_df_name:
            name of table to propagate values from

        Returns
        ---------
        target_df : MagicDataFrame or None
            returns table with propagated data,
            or None if no propagation could be done
        """
        # make sure target/source table are appropriate
        target_ind = self.ancestry.index(target_df_name)
        source_ind = self.ancestry.index(source_df_name)
        if target_ind - source_ind != 1:
            print('-W- propagate_average_up only works with tables that are spaced one apart, i.e. sites and samples.')
            print('    Source table must be lower in the hierarchy than the target table.')
            print('    You have provided "{}" as the target table and "{}" as the source table.'.format(target_df_name, source_df_name))
            return None
        # make sure target table is read in
        if target_df_name not in self.tables:
            self.add_magic_table(target_df_name)
        if target_df_name not in self.tables:
            print("-W- Couldn't read in {} table".format(target_df_name))
            return
        # make sure source table is read in
        if source_df_name not in self.tables:
            self.add_magic_table(source_df_name)
        if source_df_name not in self.tables:
            print("-W- Couldn't read in {} table".format(source_df_name))
            return
        # get tables
        target_df = self.tables[target_df_name]
        source_df = self.tables[source_df_name]
        target_name = target_df_name[:-1]
        # step 1: make sure columns exist in target_df
        for col in cols:
            if col not in target_df.df.columns:
                target_df.df[col] = None
        # step 2: propagate target_df columns forward & back
        target_df.front_and_backfill(cols)
        # step 3: see if any column values are missing
        values = [not_null(val) for val in target_df.df[cols].values.ravel()]
        if all(values):
            print('-I- {} table already has {} filled column(s)'.format(target_df_name, cols))
            self.tables[target_df_name] = target_df
            return target_df
        # step 4: make sure columns are in source table, also target name
        if target_name not in source_df.df.columns:
            print("-W- can't propagate from {} to {} table".format(source_df_name, target_df_name))
            print("    Missing {} column in {} table".format(target_name, source_df_name))
            self.tables[target_df_name] = target_df
            return target_df
        for col in cols:
            if col not in target_df.df.columns:
                target_df.df[col] = None
        # step 5: if needed, average from source table and apply to target table
        for col in cols:
            if col not in source_df.df.columns:
                source_df.df[col] = np.nan
            else:
                # make sure is numeric
                source_df.df[col] = pd.to_numeric(source_df.df[col], errors='coerce')
        grouped = source_df.df[cols + [target_name]].groupby(target_name)
        grouped = grouped[cols].apply(np.mean)
        for col in cols:
            target_df.df['new_' + col] = grouped[col]
            # use custom not_null
            mask = [not_null(val) for val in target_df.df[col]]
            target_df.df[col] = np.where(mask, #target_df.df[col].notnull(),
                                         target_df.df[col],
                                         target_df.df['new_' + col])
            target_df.df.drop(['new_' + col], inplace=True, axis=1)
            # round column to 5 decimal points
            try:
                target_df.df[col] = target_df.df[col].astype(float)
                target_df.df = target_df.df.round({col: 5})
            except ValueError: # if there are sneaky strings...
                pass
        self.tables[target_df_name] = target_df
        return target_df

Response: Propagate average values from a lower table to a higher one.


Instruction:
Can you create a Python 3 function that
propagates minimum and maximum values into a target table.

Response:
def propagate_min_max_up(self, cols=['age'],
                             target_df_name='locations',
                             source_df_name='sites',
                             min_suffix='low',
                             max_suffix='high'):
        """
        Take minimum/maximum values for a set of columns in source_df,
        and apply them to the target table.
        This method won't overwrite values in the target table, it will only
        supply values where they are missing.

        Parameters
        ----------
        cols : list-like
            list of columns to propagate, default ['age']
        target_df_name : str
            name of table to propagate values into, default 'locations'
        source_df_name:
            name of table to propagate values from, default 'sites'
        min_suffix : str
            suffix for minimum value, default 'low'
        max_suffix : str
            suffix for maximum value, default 'high'

        Returns
        ---------
        target_df : MagicDataFrame
            updated MagicDataFrame with propagated values
        """
        # make sure target/source table are appropriate
        target_ind = self.ancestry.index(target_df_name)
        source_ind = self.ancestry.index(source_df_name)
        if target_ind - source_ind != 1:
            print('-W- propagate_min_max_up only works with tables that are spaced one apart, i.e. sites and samples.')
            print('    Source table must be lower in the hierarchy than the target table.')
            print('    You have provided "{}" as the target table and "{}" as the source table.'.format(target_df_name, source_df_name))
            return None
        # make sure target table is read in
        if target_df_name not in self.tables:
            self.add_magic_table(target_df_name)
        if target_df_name not in self.tables:
            print("-W- Couldn't read in {} table".format(target_df_name))
            return
        # make sure source table is read in
        if source_df_name not in self.tables:
            self.add_magic_table(source_df_name)
        if source_df_name not in self.tables:
            print("-W- Couldn't read in {} table".format(source_df_name))
            return
        # get tables
        target_df = self.tables[target_df_name]
        source_df = self.tables[source_df_name]
        target_name = target_df_name[:-1]
        # find and propagate min/max for each col in cols
        for col in cols:
            if col not in source_df.df.columns:
                print('-W- {} table is missing "{}" column, skipping'.format(source_df_name, col))
                continue
            min_col = col + "_" + min_suffix
            max_col = col + "_" + max_suffix
            # add min/max cols to target_df if missing
            if min_col not in target_df.df.columns:
                target_df.df[min_col] = None
            if max_col not in target_df.df.columns:
                target_df.df[max_col] = None
            # get min/max from source
            if target_name not in source_df.df.columns:
                print('-W- {} table missing {} column, cannot propagate age info'.format(target_name, source_df_name))
                return
            # make sure source is appropriately filled
            source = source_df.front_and_backfill([col], inplace=False)
            # add target_name back into front/backfilled source
            source[target_name] = source_df.df[target_name]
            grouped = source[[col, target_name]].groupby(target_name)
            if len(grouped):
                minimum, maximum = grouped.min(), grouped.max()
                minimum = minimum.reindex(target_df.df.index)
                maximum = maximum.reindex(target_df.df.index)
                # update target_df without overwriting existing values
                cond_min = target_df.df[min_col].apply(not_null)
                cond_max = target_df.df[max_col].apply(not_null)
                #
                target_df.df[min_col] = np.where(cond_min,
                                                 target_df.df[min_col],
                                                 minimum[col])
                target_df.df[max_col] = np.where(cond_max,
                                                 target_df.df[max_col],
                                                 maximum[col])
        # update contribution
        self.tables[target_df_name] = target_df
        return target_df


Instruction:
Can you create a Python 3 function that
propagates values from any age data into a specific age table.

Response:
def propagate_ages(self):
        """
        Mine ages table for any age data, and write it into
        specimens, samples, sites, locations tables.
        Do not overwrite existing age data.
        """
        # if there is no age table, skip
        if 'ages' not in self.tables:
            return
        # if age table has no data, skip
        if not len(self.tables['ages'].df):
            return
        # get levels in age table
        self.get_age_levels()
        # if age levels could not be determined, skip
        if not "level" in self.tables["ages"].df.columns:
            return
        if not any(self.tables["ages"].df["level"]):
            return
        # go through each level of age data
        for level in self.tables['ages'].df['level'].unique():
            table_name = level + 's'
            age_headers = self.data_model.get_group_headers(table_name, 'Age')
            # find age headers that are actually in table
            actual_age_headers = list(set(self.tables[table_name].df.columns).intersection(age_headers))
            # find site age headers that are available in ages table
            available_age_headers = list(set(self.tables['ages'].df.columns).intersection(age_headers))
            # fill in all available age info to all rows
            self.tables[table_name].front_and_backfill(actual_age_headers)
            # add any available headers to table
            add_headers = set(available_age_headers).difference(actual_age_headers)
            for header in add_headers:
                self.tables[table_name].df[header] = None
            # propagate values from ages into table
            def move_values(ser, level, available_headers):
                name = ser.name
                cond1 = self.tables['ages'].df[level] == name
                cond2 = self.tables['ages'].df['level'] == level
                mask = cond1 & cond2
                sli = self.tables['ages'].df[mask]
                if len(sli):
                    return list(sli[available_headers].values[0])
                return [None] * len(available_headers)

            res = self.tables[table_name].df.apply(move_values, axis=1,
                                                   args=[level, available_age_headers])
            # fill in table with values gleaned from ages
            new_df = pd.DataFrame(data=list(res.values), index=res.index,
                                  columns=available_age_headers)
            age_values = np.where(self.tables[table_name].df[available_age_headers],
                                  self.tables[table_name].df[available_age_headers],
                                  new_df)
            self.tables[table_name].df[available_age_headers] = age_values
        #
        # put age_high, age_low into locations table
        print("-I- Adding age_high and age_low to locations table based on minimum/maximum ages found in sites table")
        self.propagate_min_max_up(cols=['age'], target_df_name='locations',
                                  source_df_name='sites')


Instruction:
How would you implement a function in Python 3 that
removes all non - MagIC columns from all tables.

Response:
def remove_non_magic_cols(self):
        """
        Remove all non-MagIC columns from all tables.
        """
        for table_name in self.tables:
            table = self.tables[table_name]
            table.remove_non_magic_cols_from_table()


Instruction:
Make a summary of the following Python 3 code
def write_table_to_file(self, dtype, custom_name=None, append=False, dir_path=None):
        if custom_name:
            fname = custom_name
        else:
            fname = self.filenames[dtype]
        if not dir_path:
            dir_path=self.directory
        if dtype in self.tables:
            write_df = self.remove_names(dtype)
            outfile = self.tables[dtype].write_magic_file(custom_name=fname,
                                                          dir_path=dir_path,
                                                          append=append, df=write_df)
        return outfile

Response:
Writes out a MagIC table to file using custom filename
            as specified in self. filenames.


Instruction:
Create a Python 3 function for
removing unneeded name columns from the specified table.

Response:
def remove_names(self, dtype):
        """
        Remove unneeded name columns ('specimen'/'sample'/etc)
        from the specified table.

        Parameters
        ----------
        dtype : str

        Returns
        ---------
        pandas DataFrame without the unneeded columns

        Example
        ---------
        Contribution.tables['specimens'].df = Contribution.remove_names('specimens')
        # takes out 'location', 'site', and/or 'sample' columns from the
        # specimens dataframe if those columns have been added
        """
        if dtype not in self.ancestry:
            return
        if dtype in self.tables:
            # remove extra columns here
            self_ind = self.ancestry.index(dtype)
            parent_ind = self_ind + 1 if self_ind < (len(self.ancestry) -1) else self_ind
            remove = set(self.ancestry).difference([self.ancestry[self_ind], self.ancestry[parent_ind]])
            remove = [dtype[:-1] for dtype in remove]
            columns = self.tables[dtype].df.columns.difference(remove)
            return self.tables[dtype].df[columns]


Instruction:
Implement a Python 3 function for
finding any items that are referenced in a child table but are missing in their own table.

Response:
def find_missing_items(self, dtype):
        """
        Find any items that are referenced in a child table
        but are missing in their own table.
        For example, a site that is listed in the samples table,
        but has no entry in the sites table.

        Parameters
        ----------
        dtype : str
            table name, e.g. 'specimens'

        Returns
        ---------
        set of missing values
        """
        parent_dtype, child_dtype = self.get_parent_and_child(dtype)
        if not child_dtype in self.tables:
            return set()
        items = set(self.tables[dtype].df.index.unique())
        items_in_child_table = set(self.tables[child_dtype].df[dtype[:-1]].unique())
        return {i for i in (items_in_child_table - items) if not_null(i)}


Instruction:
Implement a function in Python 3 to
return contribution id if available

Response:
def get_con_id(self):
        """
        Return contribution id if available
        """
        con_id = ""
        if "contribution" in self.tables:
            if "id" in self.tables["contribution"].df.columns:
                con_id = str(self.tables["contribution"].df["id"].values[0])
        return con_id


Instruction:
Can you tell what is the following Python 3 function doing
def all_to_str(self):
        def stringify(x):
            # float --> string,
            # truncating floats like 3.0 --> 3
            if isinstance(x, float):
                if x.is_integer():
                    #print('{} --> {}'.format(x, str(x).rstrip('0').rstrip('.')))
                    return str(x).rstrip('0').rstrip('.')
                return(str(x))
            # keep strings as they are,
            # unless it is a string like "3.0",
            # in which case truncate that too
            if isinstance(x, str):
                try:
                    float(x)
                    if x.endswith('0'):
                        if x.rstrip('0').endswith('.'):
                            #print('{} --> {}'.format(x, x.rstrip('0').rstrip('.')))
                            return x.rstrip('0').rstrip('.')
                except (ValueError, TypeError):
                    pass
            # integer --> string
            if isinstance(x, int):
                return str(x)
            # if it is not int/str/float, just return as is
            return x

        def remove_extra_digits(x, prog):
            """
            Remove extra digits
            x is a string,
            prog is always the following '_sre.SRE_Pattern':
            prog = re.compile("\d*[.]\d*([0]{5,100}|[9]{5,100})\d*\Z").
            However, it is compiled outside of this sub-function
            for performance reasons.
            """
            if not isinstance(x, str):
                return x
            result = prog.match(x)
            if result:
                decimals = result.string.split('.')[1]
                result = result.string
                if decimals[-3] == '0':
                    result = x[:-2].rstrip('0')
                if decimals[-3] == '9':
                    result = x[:-2].rstrip('9')
                    try:
                        last_digit = int(result[-1])
                        result = result[:-1] + str(last_digit + 1)
                    except ValueError:
                        result = float(result[:-1]) + 1
                #if result != x:
                #    print('changing {} to {}'.format(x, result))
                return result
            return x

        for col in self.df.columns:
            self.df[col] = self.df[col].apply(stringify)

        prog = re.compile("\d*[.]\d*([0]{5,100}|[9]{5,100})\d*\Z")
        for col in self.df.columns:
            self.df[col] = self.df[col].apply(lambda x: remove_extra_digits(x, prog))

Response: Convert all the columns of the log file into a string.


Instruction:
How would you explain what the following Python 3 function does
def remove_non_magic_cols_from_table(self, ignore_cols=()):
        unrecognized_cols = self.get_non_magic_cols()
        for col in ignore_cols:
            if col in unrecognized_cols:
                unrecognized_cols.remove(col)
        if unrecognized_cols:
            print('-I- Removing non-MagIC column names from {}:'.format(self.dtype), end=' ')
            for col in unrecognized_cols:
                self.df.drop(col, axis='columns', inplace=True)
                print(col, end=' ')
            print("\n")
        return unrecognized_cols

Response: Removes all non - MagIC columns from the table.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def update_row(self, ind, row_data):
        if sorted(row_data.keys()) != sorted(self.df.columns):
            # add any new column names
            for key in row_data:
                if key not in self.df.columns:
                    self.df[key] = None
            # add missing column names into row_data
            for col_label in self.df.columns:
                if col_label not in list(row_data.keys()):
                    row_data[col_label] = None
        try:
            self.df.iloc[ind] = pd.Series(row_data)
        except IndexError:
            return False
        return self.df

Response: Update a row with data.


Instruction:
Write a Python 3 function for
adding a row with data.

Response:
def add_row(self, label, row_data, columns=""):
        """
        Add a row with data.
        If any new keys are present in row_data dictionary,
        that column will be added to the dataframe.
        This is done inplace
        """
        # use provided column order, making sure you don't lose any values
        # from self.df.columns
        if len(columns):
            if sorted(self.df.columns) == sorted(columns):
                self.df.columns = columns
            else:
                new_columns = []
                new_columns.extend(columns)
                for col in self.df.columns:
                    if col not in new_columns:
                        new_columns.append(col)
        # makes sure all columns have data or None
        if sorted(row_data.keys()) != sorted(self.df.columns):
            # add any new column names
            for key in row_data:
                if key not in self.df.columns:
                    self.df[key] = None
            # add missing column names into row_data
            for col_label in self.df.columns:
                if col_label not in list(row_data.keys()):
                    row_data[col_label] = None

        # (make sure you are working with strings)
        self.df.index = self.df.index.astype(str)
        label = str(label)

        # create a new row with suffix "new"
        # (this ensures that you get a unique, new row,
        #  instead of adding on to an existing row with the same label)
        self.df.loc[label + "new"] = pd.Series(row_data)
        # rename it to be correct
        self.df.rename(index={label + "new": label}, inplace=True)
        # use next line to sort index inplace
        #self.df.sort_index(inplace=True)
        return self.df


Instruction:
Write a Python 3 function for
adding df to a MagicDataFrame using a data list.

Response:
def add_data(self, data):  # add append option later
        """
        Add df to a MagicDataFrame using a data list.

        Parameters
        ----------
        data : list of dicts
            data list with format [{'key1': 'val1', ...}, {'key1': 'val2', ...}, ... }]
        dtype : str
            MagIC table type
        """
        df = pd.DataFrame(data)
        name, dtype = self.get_singular_and_plural_dtype(self.dtype)
        if name in df.columns:
            df.index = df[name]
        df.index.name = name + " name"
        self.df = df


Instruction:
Given the following Python 3 function, write the documentation
def add_blank_row(self, label):
        col_labels = self.df.columns
        blank_item = pd.Series({}, index=col_labels, name=label)
        # use .loc to add in place (append won't do that)
        self.df.loc[blank_item.name] = blank_item
        return self.df

Response:
Add a blank row with only an index value to self. df.
        This is done inplace.


Instruction:
Create a Python 3 function for
removing the row at ind

Response:
def delete_row(self, ind):
        """
        remove self.df row at ind
        inplace
        """
        self.df = pd.concat([self.df[:ind], self.df[ind+1:]], sort=True)
        return self.df


Instruction:
Can you implement a function in Python 3 that
deletes all rows that meet the condition and return a new dataframe with the new rows deleted

Response:
def delete_rows(self, condition, info_str=None):
        """
        delete all rows with  condition==True
        inplace

        Parameters
        ----------
        condition : pandas DataFrame indexer
            all self.df rows that meet this condition will be deleted
        info_str : str
            description of the kind of rows to be deleted,
            e.g "specimen rows with blank method codes"

        Returns
        --------
        df_data : pandas DataFrame
            updated self.df
        """
        self.df['num'] = list(range(len(self.df)))
        df_data = self.df
        # delete all records that meet condition
        if len(df_data[condition]) > 0:  #we have one or more records to delete
            inds = df_data[condition]['num'] # list of all rows where condition is TRUE
            for ind in inds[::-1]:
                df_data = self.delete_row(ind)
                if info_str:
                    print("-I- Deleting {}. ".format(info_str), end=' ')
                    print('deleting row {}'.format(str(ind)))
        # sort so that all rows for an item are together
        df_data.sort_index(inplace=True)
        # redo temporary index
        df_data['num'] = list(range(len(df_data)))
        self.df = df_data
        return df_data


Instruction:
Can you write a function in Python 3 where it
drops rows that have only null values ignoring certain columns.

Response:
def drop_stub_rows(self, ignore_cols=('specimen',
                                          'sample',
                                          'software_packages',
                                          'num')):
        """
        Drop self.df rows that have only null values,
        ignoring certain columns.

        Parameters
        ----------
        ignore_cols : list-like
            list of column names to ignore for

        Returns
        ---------
        self.df : pandas DataFrame
        """
        # ignore citations if they just say 'This study'
        if 'citations' in self.df.columns:
            if list(self.df['citations'].unique()) == ['This study']:
                ignore_cols = ignore_cols + ('citations',)
        drop_cols = self.df.columns.difference(ignore_cols)
        self.df.dropna(axis='index', subset=drop_cols, how='all', inplace=True)
        return self.df


Instruction:
Can you generate a brief explanation for the following Python 3 code
def drop_duplicate_rows(self, ignore_cols=['specimen', 'sample']):
        # keep any row with a unique index
        unique_index = self.df.index.unique()
        cond1 = ~self.df.index.duplicated(keep=False)
        # or with actual data
        ignore_cols = [col for col in ignore_cols if col in self.df.columns]
        relevant_df = self.df.drop(ignore_cols, axis=1)
        cond2 = relevant_df.notnull().any(axis=1)
        orig_len = len(self.df)
        new_df = self.df[cond1 | cond2]
        # make sure we haven't lost anything important
        if any(unique_index.difference(new_df.index.unique())):
                cond1 = ~self.df.index.duplicated(keep="first")
        self.df = self.df[cond1 | cond2]
        end_len = len(self.df)
        removed = orig_len - end_len
        if removed:
            print('-I- Removed {} redundant records from {} table'.format(removed, self.dtype))
        return self.df

Response: Drop duplicate rows from self. df.


Instruction:
How would you explain what the following Python 3 function does
def update_record(self, name, new_data, condition, update_only=False,
                      debug=False):
        """
        Find the first row in self.df with index == name
        and condition == True.
        Update that record with new_data, then delete any
        additional records where index == name and condition == True.
        Change is inplace
        """
        # add numeric index column temporarily
        self.df['num'] = list(range(len(self.df)))
        df_data = self.df
        condition2 = (df_data.index == name)
        # edit first of existing data that meets condition
        if len(df_data[condition & condition2]) > 0:  #we have one or more records to update or delete
            # list of all rows where condition is true and index == name
            inds = df_data[condition & condition2]['num']
            #inds = df_data[condition]['num'] # list of all rows where condition is true
            existing_data = dict(df_data.iloc[inds.iloc[0]]) # get first record of existing_data from dataframe
            existing_data.update(new_data) # update existing data with new interpretations
            # update row
            self.update_row(inds.iloc[0], existing_data)
            # now remove all the remaining records of same condition
            if len(inds) > 1:
                for ind in inds[1:]:
                    print("deleting redundant records for:", name)
                    df_data = self.delete_row(ind)
        else:
            if update_only:
                print("no record found for that condition, not updating ", name)
            else:
                print('no record found - creating new one for ', name)
                # add new row
                df_data = self.add_row(name, new_data)
        # sort so that all rows for an item are together
        df_data.sort_index(inplace=True)
        # redo temporary index
        df_data['num'] = list(range(len(df_data)))
        self.df = df_data
        return df_data

Response: Update a record in the table with new data.


Instruction:
Can you create a Python 3 function that
returns a MagicDataFrame with the selected column names replaced null values in selected columns with front and backfilled values if available.

Response:
def front_and_backfill(self, cols, inplace=True):
        """
        Groups dataframe by index name then replaces null values in selected
        columns with front/backfilled values if available.
        Changes self.df inplace.

        Parameters
        ----------
        self : MagicDataFrame
        cols : array-like
            list of column names

        Returns
        ---------
        self.df
        """
        cols = list(cols)
        for col in cols:
            if col not in self.df.columns: self.df[col] = np.nan
        short_df = self.df[cols]
        # horrible, bizarre hack to test for pandas malfunction
        tester = short_df.groupby(short_df.index, sort=False).fillna(method='ffill')
        if not_null(tester):
            short_df = short_df.groupby(short_df.index, sort=False).fillna(method='ffill').groupby(short_df.index, sort=False).fillna(method='bfill')
        else:
            print('-W- Was not able to front/back fill table {} with these columns: {}'.format(self.dtype, ', '.join(cols)))
        if inplace:
            self.df[cols] = short_df[cols]
            return self.df
        return short_df


Instruction:
Can you generate the documentation for the following Python 3 function
def find_filled_col(self, col_list):
        for col in col_list:
            if col in self.df.columns:
                if not all([is_null(val, False) for val in self.df[col]]):
                    return col

Response:
Returns the first col_name from the list that is both
TracingEnabled and
COOKIEID.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def convert_to_pmag_data_list(self, lst_or_dict="lst", df=None):

        """
        Take MagicDataFrame and turn it into a list of dictionaries.
        This will have the same format as reading in a 2.5 file
        with pmag.magic_read(), i.e.:
        if "lst":
          [{"sample": "samp_name", "azimuth": 12, ...}, {...}]
        if "dict":
          {"samp_name": {"azimuth": 12, ...}, "samp_name2": {...}, ...}
        NOTE: "dict" not recommended with 3.0, as one sample can have
        many rows, which means that dictionary items can be overwritten
        """
        if isinstance(df, type(None)):
            df = self.df
        # replace np.nan / None with ""
        df = df.where(df.notnull(), "")
        # string-i-fy everything
        df = df.astype(str)

        if lst_or_dict == "lst":
            return list(df.T.apply(dict))
        else:
            return {str(i[df.index.name.split(' ')[0]]): dict(i) for i in list(df.T.apply(dict))}

Response: Take MagicDataFrame and turn it into a list of dictionaries.


Instruction:
Create a Python 3 function to
return the name of the given column in a DataFrame or a list of index_names to slice self. df.

Response:
def get_name(self, col_name, df_slice="", index_names=""):
        """
        Takes in a column name, and either a DataFrame slice or
        a list of index_names to slice self.df using fancy indexing.
        Then return the value for that column in the relevant slice.
        (Assumes that all values for column will be the same in the
         chosen slice, so return the first one.)
        """
        # if slice is provided, use it
        if any(df_slice):
            df_slice = df_slice
        # if given index_names, grab a slice using fancy indexing
        elif index_names:
            df_slice = self.df.loc[index_names]
        # otherwise, use the full DataFrame
        else:
            df_slice = self.df
        # if the slice is empty, return ""
        if len(df_slice) == 0:
            return ""
        # if the column name isn't present in the slice, return ""
        if col_name not in df_slice.columns:
            return ""
        # otherwise, return the first value from that column
        first_val = list(df_slice[col_name].dropna())
        if any(first_val):
            return first_val[0]
        else:
            return ""


Instruction:
Can you tell what is the following Python 3 function doing
def get_di_block(self, df_slice=None, do_index=False,
                     item_names=None, tilt_corr='100',
                     excl=None, ignore_tilt=False):
        """
        Input either a DataFrame slice
        or
        do_index=True and a list of index_names.
        Optional arguments:
        Provide tilt_corr (default 100).
        Excl is a list of method codes to exclude.
        Output dec/inc from the slice in this format:
        [[dec1, inc1], [dec2, inc2], ...].
        Not inplace
        """
        tilt_corr = int(tilt_corr)
        if isinstance(df_slice, str):
            if df_slice.lower() == "all":
                # use entire DataFrame
                df_slice = self.df
        elif do_index:
            # use fancy indexing (but note this will give duplicates)
            df_slice = self.df.loc[item_names]
        elif not do_index:
            # otherwise use the provided slice
            df_slice = df_slice

        # once you have the slice, fix up the data
        # tilt correction must match
        if not ignore_tilt:
            if tilt_corr != 0:
                df_slice = df_slice[df_slice['dir_tilt_correction'] == tilt_corr]
            else:
                # if geographic ("0"),
                # use records with no tilt_corr and assume geographic
                cond1 = df_slice['dir_tilt_correction'] == None
                cond2 = df_slice['dir_tilt_correction'] == tilt_corr
                df_slice = df_slice[cond1 | cond2]
        # exclude data with unwanted codes
        if excl:
            for ex in excl:
                df_slice = self.get_records_for_code(ex, incl=False,
                                                     use_slice=True,
                                                     sli=df_slice)

        df_slice = df_slice[df_slice['dir_inc'].notnull() & df_slice['dir_dec'].notnull()]
        # possible add in:
        # split out di_block from this study from di_block from other studies (in citations column)
        # previously just used "This study", but it is no longer required
        #if 'citations' in df_slice.columns:
        #    df_slice = df_slice[df_slice['citations'].str.contains("This study")]

        # convert values into DIblock format
        di_block = [[float(row['dir_dec']), float(row['dir_inc'])] for ind, row in df_slice.iterrows()]
        return di_block

Response: Get the di_block from the slice or the entire DataFrame.


Instruction:
Explain what the following Python 3 code does
def get_records_for_code(self, meth_code, incl=True, use_slice=False,
                             sli=None, strict_match=True):
        """
        Use regex to see if meth_code is in the method_codes ":" delimited list.
        If incl == True, return all records WITH meth_code.
        If incl == False, return all records WITHOUT meth_code.
        If strict_match == True, return only records with the exact meth_code.
        If strict_match == False, return records that contain the meth_code partial string,
        (i.e., "DE-").
        Not inplace
        """
        # (must use fillna to replace np.nan with False for indexing)
        if use_slice:
            df = sli.copy()
        else:
            df = self.df.copy()
        # if meth_code not provided, return unchanged dataframe
        if not meth_code:
            return df
        # get regex
        if not strict_match:
            # grab any record that contains any part of meth_code
            cond = df['method_codes'].str.contains(meth_code).fillna(False)
        else:
            # grab only an exact match
            pattern = re.compile('{}(?=:|\s|\Z)'.format(meth_code))
            cond = df['method_codes'].str.contains(pattern).fillna(False)
        if incl:
            # return a copy of records with that method code:
            return df[cond]
        else:
            # return a copy of records without that method code
            return df[~cond]

Response: Get the records for a given method code.


Instruction:
How would you explain what the following Python 3 function does
def merge_dfs(self, df1):

        if self.df.empty:
            return df1
        elif df1.empty:
            return self.df

        #copy to prevent mutation
        cdf2 = self.df.copy()

        #split data into types and decide which to replace
#        if replace_dir_or_int == 'dir' and 'method_codes' in cdf2.columns:
#            cdf2 = cdf2[cdf2['method_codes'].notnull()]
#            acdf2 = cdf2[cdf2['method_codes'].str.contains('LP-PI')]
#            mcdf2 = cdf2[cdf2['method_codes'].str.contains('LP-DIR')]
#        elif replace_dir_or_int == 'int' and 'method_codes' in cdf2.columns:
#            cdf2 = cdf2[cdf2['method_codes'].notnull()]
#            mcdf2 = cdf2[cdf2['method_codes'].str.contains('LP-PI')]
#            acdf2 = cdf2[cdf2['method_codes'].str.contains('LP-DIR')]
#        else:
#            mcdf2 = cdf2
#            acdf2 = pd.DataFrame(columns=mcdf2.columns)

        #get rid of stupid duplicates
#        [mcdf2.drop(cx,inplace=True,axis=1) for cx in mcdf2.columns if cx in df1.columns]

        #join the new calculated data with the old data of same type
        if self.dtype.endswith('s'): dtype = self.dtype[:-1]
        else: dtype = self.dtype
        index_name = dtype + "_name"
        for df in [df1, cdf2]:
            df.index.name = index_name
        mdf = df1.join(cdf2, how='outer', rsuffix='_remove', on=index_name)
        def keep_non_null_vals(column):
            extra_column = column + "_remove"
            if column in mdf.columns and extra_column in mdf.columns:
                mdf[column] = np.where(mdf[column].apply(lambda x: not_null(x, False)), mdf[column], mdf[extra_column])
        # merge values in the following columns
        # e.g., combine info from specimen + specimen_remove into specimen column
        for col in ['specimen', 'sample', 'site', 'location', 'lat', 'lon']:
            keep_non_null_vals(col)
        #drop duplicate columns if they were created
        [mdf.drop(col,inplace=True,axis=1) for col in mdf.columns if col.endswith("_remove")]
        #duplicates rows for some freaking reason
        mdf.drop_duplicates(inplace=True,subset=[col for col in mdf.columns if col != 'description'])
        #merge the data of the other type with the new data
#        mdf = mdf.merge(acdf2, how='outer')
        if dtype in mdf.columns:
            #fix freaking indecies because pandas
            mdf = mdf.set_index(dtype)
            #really? I wanted the index changed not a column deleted?!?
            mdf[dtype] = mdf.index
            mdf.index.name = index_name
            mdf.sort_index(inplace=True)
        return mdf

Response: This method takes new calculated data and replaces the corresponding data in self. df with the new input data preserving most important metadata if they are not already saved.


Instruction:
Given the following Python 3 function, write the documentation
def write_magic_file(self, custom_name=None, dir_path=".",
                         append=False, multi_type=False, df=None):
        """
        Write self.df out to tab-delimited file.
        By default will use standard MagIC filenames (specimens.txt, etc.),
        or you can provide a custom_name to write to instead.
        By default will write to custom_name if custom_name is a full path,
        or will write to dir_path + custom_name if custom_name
        is not a full path.

        Parameters
        ----------
        self : MagIC DataFrame
        custom_name : str
            custom file name
        dir_path : str
            dir_path (used if custom_name is not a full path), default "."
        append : bool
            append to existing file, default False
        multi_type : bool
            for creating upload file

        Return
        --------
        fname : str
            output file name
        """
        # don't let custom name start with "./"
        if custom_name:
            if custom_name.startswith('.'):
                custom_name = os.path.split(custom_name)[1]
        # put columns in logical order (by group)
        self.sort_dataframe_cols()
        # if indexing column was put in, remove it
        if "num" in self.df.columns:
            self.df = self.df.drop("num", axis=1)
        #
        # make sure name is a string
        name = self.get_singular_and_plural_dtype(self.dtype)[0]
        if name in self.df.columns:
            self.df[name] = self.df[name].astype(str)
        #
        if df is None:
            df = self.df
        # get full file path
        dir_path = os.path.realpath(dir_path)
        if custom_name:
            fname = pmag.resolve_file_name(custom_name, dir_path) # os.path.join(dir_path, custom_name)
        elif self.magic_file:
            fname = pmag.resolve_file_name(self.magic_file, dir_path)
        else:
            fname = os.path.join(dir_path, self.dtype + ".txt")
        # see if there's any data
        if not len(df):
            print('-W- No data to write to {}'.format(fname))
            return False
        # add to existing file
        if append:
            print('-I- appending {} data to {}'.format(self.dtype, fname))
            mode = "a"
        # overwrite existing file
        elif os.path.exists(fname):
            print('-I- overwriting {}'.format(fname))
            mode = "w"
        # or create new file
        else:
            print('-I- writing {} records to {}'.format(self.dtype, fname))
            mode = "w"
        f = open(fname, mode)
        if append:
            header = False
            if multi_type:
                header = True
                f.write('tab\t{}\n'.format(self.dtype))
            f.flush()
            df.to_csv(f, sep="\t", header=header, index=False, mode='a')
        else:
            f.write('tab\t{}\n'.format(self.dtype))
            f.flush()
            df.to_csv(f, sep="\t", header=True, index=False, mode='a')
        print('-I- {} records written to {} file'.format(len(df), self.dtype))
        f.close()
        return fname

Response: Write out to a MagIC file.


Instruction:
Create a Python 3 function to
find all columns that are not real MagIC 3 columns.

Response:
def get_non_magic_cols(self):
        """
        Find all columns in self.df that are not real MagIC 3 columns.

        Returns
        --------
        unrecognized_cols : list
        """
        table_dm = self.data_model.dm[self.dtype]
        approved_cols = table_dm.index
        unrecognized_cols = (set(self.df.columns) - set(approved_cols))
        return unrecognized_cols


Instruction:
Can you create a Python 3 function that
returns the first non - null value in a given index and column.

Response:
def get_first_non_null_value(self, ind_name, col_name):
        """
        For a given index and column, find the first non-null value.

        Parameters
        ----------
        self : MagicDataFrame
        ind_name : str
            index name for indexing
        col_name : str
            column name for indexing

        Returns
        ---------
        single value of str, float, or int
        """
        short_df = self.df.loc[ind_name, col_name]
        mask = pd.notnull(short_df)
        print(short_df[mask])
        try:
            val = short_df[mask].unique()[0]
        except IndexError:
            val = None
        return val


Instruction:
Can you tell what is the following Python 3 function doing
def get_singular_and_plural_dtype(self, dtype):
        dtype = dtype.strip()
        if dtype.endswith('s'):
            return dtype[:-1], dtype
        elif dtype == 'criteria':
            return 'table_column', 'criteria'
        elif dtype == 'contribution':
            return 'doi', 'contribution'

Response: Returns singular and plural dtype for a specific MagIC table.


Instruction:
Can you implement a function in Python 3 that
names is the main function of the chi_magic. py script.

Response:
def main():
    """
    NAME
        chi_magic.py

    DESCRIPTION
        plots magnetic susceptibility as a function of frequency and temperature and AC field

    SYNTAX
        chi_magic.py [command line options]

    OPTIONS
        -h prints help message and quits
        -i allows interactive setting of FILE and temperature step
        -f FILE, specify magic_measurements format file
        -T IND, specify temperature step to plot
        -e EXP, specify experiment name to plot
        -fmt [svg,jpg,png,pdf] set figure format [default is svg]
        -sav save figure and quit

    DEFAULTS
         FILE: magic_measurements.txt
         IND: first
         SPEC: step through one by one
    """
    cont, FTinit, BTinit, k = "", 0, 0, 0
    meas_file = "magic_measurements.txt"
    spec = ""
    Tind, cont = 0, ""
    EXP = ""
    fmt = 'svg'  # default image type for saving
    plot = 0
    if '-h' in sys.argv:
        print(main.__doc__)
        sys.exit()
    if '-i' in sys.argv:
        fname = input(
            "Input magic_measurements file name? [magic_measurements.txt]  ")
        if fname != "":
            meas_file = fname
    if '-e' in sys.argv:
        ind = sys.argv.index('-e')
        EXP = sys.argv[ind+1]
    if '-f' in sys.argv:
        ind = sys.argv.index('-f')
        meas_file = sys.argv[ind+1]
    if '-T' in sys.argv:
        ind = sys.argv.index('-T')
        Tind = int(sys.argv[ind+1])
    if '-fmt' in sys.argv:
        ind = sys.argv.index('-fmt')
        fmt = sys.argv[ind+1]
    if '-sav' in sys.argv:
        plot = 1
    #
    meas_data, file_type = pmag.magic_read(meas_file)
    #
    # get list of unique experiment names
    #
    # initialize some variables (a continuation flag, plot initialization flags and the experiment counter
    experiment_names = []
    for rec in meas_data:
        if rec['magic_experiment_name'] not in experiment_names:
            experiment_names.append(rec['magic_experiment_name'])
    #
    # hunt through by experiment name
    if EXP != "":
        try:
            k = experiment_names.index(EXP)
        except:
            print("Bad experiment name")
            sys.exit()
    while k < len(experiment_names):
        e = experiment_names[k]
        if EXP == "":
            print(e, k+1, 'out of ', len(experiment_names))
    #
    #  initialize lists of data, susceptibility, temperature, frequency and field
        X, T, F, B = [], [], [], []
        for rec in meas_data:
            methcodes = rec['magic_method_codes']
            meths = methcodes.strip().split(':')
            if rec['magic_experiment_name'] == e and "LP-X" in meths:  # looking for chi measurement
                if 'measurement_temp' not in list(rec.keys()):
                    rec['measurement_temp'] = '300'  # set defaults
                if 'measurement_freq' not in list(rec.keys()):
                    rec['measurement_freq'] = '0'  # set defaults
                if 'measurement_lab_field_ac' not in list(rec.keys()):
                    rec['measurement_lab_field_ac'] = '0'  # set default
                if 'measurement_x' in rec.keys():
                    # backward compatibility
                    X.append(float(rec['measurement_x']))
                else:
                    # data model 2.5
                    X.append(float(rec['measurement_chi_volume']))
                T.append(float(rec['measurement_temp']))
                F.append(float(rec['measurement_freq']))
                B.append(float(rec['measurement_lab_field_ac']))
    #
    # get unique list of Ts,Fs, and Bs
    #
        Ts, Fs, Bs = [], [], []
        for k in range(len(X)):   # hunt through all the measurements
            if T[k] not in Ts:
                Ts.append(T[k])  # append if not in list
            if F[k] not in Fs:
                Fs.append(F[k])
            if B[k] not in Bs:
                Bs.append(B[k])
        Ts.sort()  # sort list of temperatures, frequencies and fields
        Fs.sort()
        Bs.sort()
        if '-x' in sys.argv:
            k = len(experiment_names)+1  # just plot the one
        else:
            k += 1  # increment experiment number
    #
    # plot chi versus T and F holding B constant
    #
        plotnum = 1  # initialize plot number to 1
        if len(X) > 2:  # if there are any data to plot, continue
            b = Bs[-1]  # keeping field constant and at maximum
            XTF = []  # initialize list of chi versus Temp and freq
            for f in Fs:   # step through frequencies sequentially
                XT = []  # initialize list of chi versus temp
                for kk in range(len(X)):  # hunt through all the data
                    if F[kk] == f and B[kk] == b:  # select data with given freq and field
                        XT.append([X[kk], T[kk]])  # append to list
                XTF.append(XT)  # append list to list of frequencies
            if len(XT) > 1:  # if there are any temperature dependent data
                pmagplotlib.plot_init(plotnum, 5, 5)  # initialize plot
                # call the plotting function
                pmagplotlib.plot_xtf(plotnum, XTF, Fs, e, b)
                if plot == 0:
                    pmagplotlib.draw_figs({'fig': plotnum})  # make it visible
                plotnum += 1  # increment plot number
            f = Fs[0]  # set frequency to minimum
            XTB = []  # initialize list if chi versus Temp and field
            for b in Bs:  # step through field values
                XT = []  # initial chi versus temp list for this field
                for kk in range(len(X)):  # hunt through all the data
                    if F[kk] == f and B[kk] == b:  # select data with given freq and field
                        XT.append([X[kk], T[kk]])  # append to list
                XTB.append(XT)
            if len(XT) > 1:  # if there are any temperature dependent data
                pmagplotlib.plot_init(plotnum, 5, 5)  # set up plot
                # call the plotting function
                pmagplotlib.plot_xtb(plotnum, XTB, Bs, e, f)
                if plot == 0:
                    pmagplotlib.draw_figs({'fig': plotnum})
                plotnum += 1  # increment plot number
            if '-i' in sys.argv:
                for ind in range(len(Ts)):  # print list of temperatures available
                    print(ind, int(Ts[ind]))
                cont = input(
                    "Enter index of desired temperature step, s[a]ve plots, [return] to quit ")
                if cont == 'a':
                    files = {}
                    PLTS = {}
                    for p in range(1, plotnum):
                        key = str(p)
                        files[key] = e+'_'+key+'.'+fmt
                        PLTS[key] = key
                    pmagplotlib.save_plots(PLTS, files)
                    cont = input(
                        "Enter index of desired temperature step, s[a]ve plots, [return] to quit ")
                if cont == "":
                    cont = 'q'
            while cont != "q":
                if '-i' in sys.argv:
                    Tind = int(cont)  # set temperature index
                b = Bs[-1]  # set field to max available
                XF = []  # initial chi versus frequency list
                for kk in range(len(X)):  # hunt through the data
                    if T[kk] == Ts[Tind] and B[kk] == b:  # if temperature and field match,
                        XF.append([X[kk], F[kk]])  # append the data
                if len(XF) > 1:  # if there are any data to plot
                    if FTinit == 0:  # if not already initialized, initialize plot
                        # print 'initializing ',plotnum
                        pmagplotlib.plot_init(plotnum, 5, 5)
                        FTinit = 1
                        XFplot = plotnum
                        plotnum += 1  # increment plotnum
                    pmagplotlib.plot_xft(XFplot, XF, Ts[Tind], e, b)
                    if plot == 0:
                        pmagplotlib.draw_figs({'fig': plotnum})
                else:
                    print(
                        '\n *** Skipping susceptibitily-frequency plot as a function of temperature *** \n')
                f = Fs[0]  # set frequency to minimum available
                XB = []  # initialize chi versus field list
                for kk in range(len(X)):  # hunt through the data
                    # if temperature and field match those desired
                    if T[kk] == Ts[Tind] and F[kk] == f:
                        XB.append([X[kk], B[kk]])  # append the data to list
                if len(XB) > 4:  # if there are any data
                    if BTinit == 0:  # if plot not already initialized
                        pmagplotlib.plot_init(plotnum, 5, 5)  # do it
                        BTinit = 1
                    # and call plotting function
                    pmagplotlib.plot_xbt(plotnum, XB, Ts[Tind], e, f)
                    if plot == 0:
                        pmagplotlib.draw_figs({'fig': plotnum})
                else:
                    print(
                        'Skipping susceptibitily - AC field plot as a function of temperature')
                files = {}
                PLTS = {}
                for p in range(1, plotnum):
                    key = str(p)
                    files[key] = e+'_'+key+'.'+fmt
                    PLTS[key] = p
                if '-i' in sys.argv:
                    # just in case you forgot, print out a new list of temperatures
                    for ind in range(len(Ts)):
                        print(ind, int(Ts[ind]))
                    # ask for new temp
                    cont = input(
                        "Enter index of next temperature step, s[a]ve plots,  [return] to quit ")
                    if cont == "":
                        sys.exit()
                    if cont == 'a':
                        pmagplotlib.save_plots(PLTS, files)
                        cont = input(
                            "Enter index of desired temperature step, s[a]ve plots, [return] to quit ")
                        if cont == "":
                            sys.exit()
                elif plot == 0:
                    ans = input(
                        "enter s[a]ve to save files,  [return] to quit ")
                    if ans == 'a':
                        pmagplotlib.save_plots(PLTS, files)
                        sys.exit()
                    else:
                        sys.exit()
                else:
                    pmagplotlib.save_plots(PLTS, files)
                    sys.exit()


Instruction:
Can you implement a function in Python 3 that
names hysteresis_magic.py DESCRIPTION calculates hystereis parameters and saves them in 3.0 specimen format file makes plots if option selected SYNTAX hysteresis_magic.py [command line options] OPTIONS -h prints help message and quits -f: specify input file, default is agm_measurements.txt -F: specify specimens.txt output file -WD: directory to output files to (default : current directory) Note: if using Windows, all figures will output to current directory -ID: directory to read files from (default : same as -WD) -P: do not make the plots -spc SPEC: specify specimen name to plot and quit -sav save all plots and quit -fmt [png,svg,eps,jpg]

Response:
def main():
    """
    NAME
        hysteresis_magic.py

    DESCRIPTION
        calculates hystereis parameters and saves them in 3.0 specimen format file
        makes plots if option selected

    SYNTAX
        hysteresis_magic.py [command line options]

    OPTIONS
        -h prints help message and quits
        -f: specify input file, default is agm_measurements.txt
        -F: specify specimens.txt output file
        -WD: directory to output files to (default : current directory)
             Note: if using Windows, all figures will output to current directory
        -ID: directory to read files from (default : same as -WD)
        -P: do not make the plots
        -spc SPEC: specify specimen name to plot and quit
        -sav save all plots and quit
        -fmt [png,svg,eps,jpg]
    """
    args = sys.argv
    fmt = pmag.get_named_arg('-fmt', 'svg')
    output_dir_path = pmag.get_named_arg('-WD', '.')
    input_dir_path = pmag.get_named_arg('-ID', "")
    if "-h" in args:
        print(main.__doc__)
        sys.exit()
    meas_file = pmag.get_named_arg('-f', 'measurements.txt')
    spec_file = pmag.get_named_arg('-F', 'specimens.txt')
    make_plots = True
    save_plots = False
    if '-P' in args:
        make_plots = False
    if '-sav' in args:
        save_plots = True
    pltspec = pmag.get_named_arg('-spc', 0)
    ipmag.hysteresis_magic(output_dir_path, input_dir_path, spec_file, meas_file,
                           fmt, save_plots, make_plots, pltspec)


Instruction:
Explain what the following Python 3 code does
def get_data_files_dir():
    if 'data_files' in os.listdir(sys.prefix):
        return os.path.join(sys.prefix, 'data_files')
    else:
        return os.path.join(get_pmag_dir(), 'data_files')

Response: Find the directory with data_files and return the path.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_pmag_dir():
    # this is correct for py2exe (DEPRECATED)
    #win_frozen = is_frozen()
    #if win_frozen:
    #    path = os.path.abspath(unicode(sys.executable, sys.getfilesystemencoding()))
    #    path = os.path.split(path)[0]
    #    return path
    # this is correct for py2app
    try:
        return os.environ['RESOURCEPATH']
    # this works for everything else
    except KeyError: pass
    # new way:
    # if we're in the local PmagPy directory:
    if os.path.isfile(os.path.join(os.getcwd(), 'pmagpy', 'pmag.py')):
        lib_dir = os.path.join(os.getcwd(), 'pmagpy')
    # if we're anywhere else:
    elif getattr(sys, 'frozen', False): #pyinstaller datafile directory
        return sys._MEIPASS
    else:
        # horrible, hack-y fix
        # (prevents namespace issue between
        # local github PmagPy and pip-installed PmagPy).
        # must reload because we may have
        # changed directories since importing
        temp = os.getcwd()
        os.chdir('..')
        reload(locator)
        lib_file = resource_filename('locator', 'resource.py')
        full_dir = os.path.split(lib_file)[0]
        ind = full_dir.rfind(os.sep)
        lib_dir = full_dir[:ind+1]
        lib_dir = os.path.realpath(os.path.join(lib_dir, 'pmagpy'))
        os.chdir(temp)
        # end fix
        # old way:
        #lib_dir = os.path.dirname(os.path.realpath(__file__))
    if not os.path.isfile(os.path.join(lib_dir, 'pmag.py')):
        lib_dir = os.getcwd()
    fname = os.path.join(lib_dir, 'pmag.py')
    if not os.path.isfile(fname):
        pmag_dir = os.path.split(os.path.split(__file__)[0])[0]
        if os.path.isfile(os.path.join(pmag_dir,'pmagpy','pmag.py')):
            return pmag_dir
        else:
            print('-W- Can\'t find the data model!  Make sure you have installed pmagpy using pip: "pip install pmagpy --upgrade"')
            return '.'
    # strip "/" or "\" and "pmagpy" to return proper PmagPy directory
    if lib_dir.endswith(os.sep):
        lib_dir = lib_dir[:-1]
    if lib_dir.endswith('pmagpy'):
        pmag_dir = os.path.split(lib_dir)[0]
    else:
        pmag_dir = lib_dir
    return pmag_dir

Response: Returns the directory in which PmagPy is installed


Instruction:
Can you generate a brief explanation for the following Python 3 code
def main():
    cmap = 'RdYlBu'
    date = 2016.
    if not Basemap:
        print(
            "-W- Cannot access the Basemap module, which is required to run plot_magmap.py")
        sys.exit()
    dir_path = '.'
    lincr = 1  # level increment for contours
    if '-WD' in sys.argv:
        ind = sys.argv.index('-WD')
        dir_path = sys.argv[ind+1]
    if '-h' in sys.argv:
        print(main.__doc__)
        sys.exit()
    if '-fmt' in sys.argv:
        ind = sys.argv.index('-fmt')
        fmt = sys.argv[ind+1]
        if fmt == 'jpg':
            print('jpg not a supported option')
            print(main.__doc__)
            sys.exit()
    else:
        fmt = 'png'
    if '-cm' in sys.argv:
        ind = sys.argv.index('-cm')
        cmap = sys.argv[ind+1]
    if '-el' in sys.argv:
        ind = sys.argv.index('-el')
        el = sys.argv[ind+1]
    else:
        el = 'B'
    if '-alt' in sys.argv:
        ind = sys.argv.index('-alt')
        alt = sys.argv[ind+1]
    else:
        alt = 0
    if '-lon0' in sys.argv:
        ind = sys.argv.index('-lon0')
        lon_0 = float(sys.argv[ind+1])
    else:
        lon_0 = 0
    if '-mod' in sys.argv:
        ind = sys.argv.index('-mod')
        mod = sys.argv[ind+1]
        ghfile = ''
    elif '-f' in sys.argv:
        ind = sys.argv.index('-f')
        ghfile = sys.argv[ind+1]
        mod = 'custom'
        date = ''
    else:
        mod, ghfile = 'cals10k', ''
    if '-age' in sys.argv:
        ind = sys.argv.index('-age')
        date = float(sys.argv[ind+1])
    if '-alt' in sys.argv:
        ind = sys.argv.index('-alt')
        alt = float(sys.argv[ind+1])
    else:
        alt = 0
    save = pmag.get_flag_arg_from_sys("-sav")
    if mod == 'custom':
        d = 'Custom'
    else:
        d = str(date)
    Ds, Is, Bs, Brs, lons, lats = pmag.do_mag_map(
        date, mod=mod, lon_0=lon_0, alt=alt, file=ghfile)
    if el == 'D':
        element = Ds
    elif el == 'I':
        element = Is
    elif el == 'B':
        element = Bs
    elif el == 'Br':
        element = Brs
    elif el == 'I':
        element = Is
    else:
        print(main.__doc__)
        sys.exit()
    pmagplotlib.plot_mag_map(1, element, lons, lats, el, lon_0=0, date=date)
    if not save:
        pmagplotlib.draw_figs({'map': 1})
        res = pmagplotlib.save_or_quit()
        if res == 'a':
            figname = 'igrf'+d+'.'+fmt
            print("1 saved in ", figname)
            plt.savefig('igrf'+d+'.'+fmt)
        sys.exit()
    plt.savefig('igrf'+d+'.'+fmt)
    print('Figure saved as: ', 'igrf'+d+'.'+fmt)

Response: This function is the main function for the basemap plotting. It is the main function for the basemap plotting.