Instruction:
Here you have a function in Python 3, explain what it does
def _values_exist(table):
        try:
            for var, data in table["columns"].items():
                if "values" in data:
                    return True
        except KeyError as e:
            logger_lpd_noaa.warn("values_exist: KeyError: {}".format(e))
        except Exception as e:
            logger_lpd_noaa.warn("values_exist: Excpetion: {}".format(e))
        return False

Response: Check that values exist in this table and we can write out data columns


Instruction:
How would you implement a function in Python 3 that
reorganizes the keys into their proper section order for the NOAA output file.

Response:
def __reorganize(self):
        """
        Reorganize the keys into their proper section order for the NOAA output file
        DO NOT parse data tables (paleoData or chronData). We will do those separately.
        :param str key:
        :param any value:
        :return none:
        """
        logger_lpd_noaa.info("enter reorganize")
        # NOAA files are organized in sections differently than NOAA. try to translate these sections.
        for key, value in self.lipd_data.items():
            # if this key has a noaa match, it'll be returned. otherwise, empty string for no match
            noaa_key = self.__get_noaa_key(key)
            # check if this lipd key is in the NOAA_KEYS conversion dictionary.
            # if it's not, then stash it in our ignore list.
            if key not in LIPD_NOAA_MAP_FLAT:
                self.noaa_data_sorted["Ignore"][noaa_key] = value
            # studyName is placed two times in file. Line #1, and under the 'title' section
            elif noaa_key == "Study_Name":
                # study name gets put in two locations
                self.noaa_data_sorted["Top"][noaa_key] = value
                self.noaa_data_sorted["Title"][noaa_key] = value
            # put archiveType in self, because we'll reuse it later for the 9-part-variables as well
            elif noaa_key == "Archive":
                self.lsts_tmp["archive"].append(value)
            # Dataset_DOI is a repeatable element. the key could be a single DOI, or a list of DOIs.
            elif noaa_key == "Dataset_DOI":
                self.__parse_dois(value)

            # all other keys. determine which noaa section they belong in.
            else:
                # noaa keys are sorted by section.
                for header, content in NOAA_KEYS_BY_SECTION.items():
                    try:
                        # if our key is a noaa header key, then that means it's the ONLY key in the section.
                        # set value directly
                        if noaa_key == header:
                            self.noaa_data_sorted[header] = value
                        # all other cases, the key is part of the section
                        elif noaa_key in content:
                            self.noaa_data_sorted[header][noaa_key] = value
                    except KeyError:
                        # this shouldn't ever really happen, but just in case
                        logger_lpd_noaa.warn("lpd_noaa: reorganize: KeyError: {}".format(noaa_key))
        return


Instruction:
Given the following Python 3 function, write the documentation
def __reorganize_author(self):
        try:
            for idx, pub in enumerate(self.noaa_data_sorted["Publication"]):
                if "author" in pub:
                    _str = pub["author"]
                    if " and " in _str:
                        self.noaa_data_sorted["Publication"][idx]["author"] = _str.replace(" and ", "; ")
                    if ";" in _str:
                        self.noaa_data_sorted["Publication"][idx]["author"] = _str.replace(";", "; ")

        except Exception:
            pass
        return

Response: Reorganizes author names by and and.


Instruction:
Here you have a function in Python 3, explain what it does
def __reorganize_coordinates(self):
        try:
            l = self.noaa_data_sorted["Site_Information"]['geometry']['coordinates']
            locations = ["Northernmost_Latitude", "Southernmost_Latitude", "Easternmost_Longitude",
                         "Westernmost_Longitude", "Elevation"]
            logger_lpd_noaa.info("coordinates: {} coordinates found".format(len(l)))

            # Amount of coordinates in the list
            _len_coords = len(l)

            # Odd number of coordinates. Elevation value exists
            if _len_coords % 2 == 1:
                # Store the elevation, which is always the last value in the list
                self.noaa_geo["Elevation"] = l[-1]
                # If elevation, then subtract one from the length
                _len_coords -= 1

            # Start compiling the lat lon coordinates

            # 0 coordinate values. fill in locations with empty values
            if _len_coords == 0:
                for location in locations:
                    self.noaa_geo[location] = ' '
            # 2 coordinates values. duplicate to fill 4 location slots.
            elif _len_coords == 2:
                self.noaa_geo[locations[0]] = l[1]
                self.noaa_geo[locations[1]] = l[1]
                self.noaa_geo[locations[2]] = l[0]
                self.noaa_geo[locations[3]] = l[0]

            # 4 coordinate values. put each in its correct location slot.
            elif _len_coords == 4:
                for index, location in enumerate(locations):
                    self.noaa_geo[locations[index]] = l[index]
            else:
                logger_lpd_noaa.info("coordinates: too many coordinates given")
        except KeyError:
            logger_lpd_noaa.info("lpd_noaa: coordinates: no coordinate information")
        except Exception:
            logger_lpd_noaa.error("lpd_noaa: coordinates: unknown exception")

        return

Response: Reorganize the coordinates of the current object based on how many values are available.


Instruction:
Can you implement a function in Python 3 that
reorganizes the Funding_Agency_Name and Grant keys.

Response:
def __reorganize_funding(self):
        """
        Funding gets added to noaa_data_sorted with LiPD keys. Change those keys to NOAA
        :return none:
        """
        _map = {"agency": "Funding_Agency_Name", "grant": "Grant"}
        try:
            _l = []
            for item in self.noaa_data_sorted["Funding_Agency"]:
                _tmp = {}
                for lpd_name, noaa_name in _map.items():
                    val = ""
                    if lpd_name in item:
                        val = item[lpd_name]
                    _tmp[noaa_name] = val
                _l.append(_tmp)
            self.noaa_data_sorted["Funding_Agency"] = _l
        except Exception:
            pass
        return


Instruction:
Write a Python 3 script for
reorganizing the geo data from self. noaa_data_sorted

Response:
def __reorganize_geo(self):
        """
        Concat geo value and units, and reorganize the rest
        References geo data from self.noaa_data_sorted
        Places new data into self.noaa_geo temporarily, and then back into self.noaa_data_sorted.
        :return:
        """
        logger_lpd_noaa.info("enter reorganize_geo")

        try:
            # Geo -> Properties
            for k, v in self.noaa_data_sorted["Site_Information"]['properties'].items():
                noaa_key = self.__get_noaa_key(k)
                self.noaa_geo[noaa_key] = v
        except KeyError:
            logger_lpd_noaa.info("reorganize_geo: KeyError: geo properties")
        try:
            # Geo -> Geometry
            self.__reorganize_coordinates()
        except Exception:
            logger_lpd_noaa.warning("reorganize_geo: Exception: missing required data: coordinates")

        # put the temporarily organized data into the self.noaa_data_sorted
        self.noaa_data_sorted["Site_Information"] = self.noaa_geo
        return


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __reorganize_sensor(self):
        _code = []
        _name = []

        # Check if any of the sensor data is misplaced, and create corrected lists.
        if self.lsts_tmp["genus"]:
            for name in self.lsts_tmp["genus"]:
                if len(name) == 4 and name.isupper():
                    _code.append(name)
                else:
                    _name.append(name)

        if self.lsts_tmp["species"]:
            for name in self.lsts_tmp["species"]:
                if len(name) != 4 and not name.isupper():
                    _name.append(name)
                else:
                    _code.append(name)

        # Set the strings into the noaa data sorted
        self.lsts_tmp["species"] = _name
        self.lsts_tmp["genus"] = _code

        return

Response: Reorganize the sensor data into the noaa data


Instruction:
Create a Python 3 function to
put the variableNames with the corresponding column data.

Response:
def __put_names_on_csv_cols(names, cols):
        """
        Put the variableNames with the corresponding column data.
        :param list names: variableNames
        :param list cols: List of Lists of column data
        :return dict:
        """
        _combined = {}
        for idx, name in enumerate(names):
            # Use the variableName, and the column data from the same index
            _combined[name] = cols[idx]

        return _combined


Instruction:
Can you generate the documentation for the following Python 3 function
def __put_year_col_first(d):
        if "year" in d:
            D = OrderedDict()
            # store the year column first
            D["year"] = d["year"]
            for k,v in d.items():
                if k != "year":
                    # store the other columns
                    D[k] = v
            return D
        else:
            # year is not found, return data as-is
            return d

Response: Always write year column first.


Instruction:
How would you code a function in Python 3 to
remove exact duplicate entries for abstract or citation fields. Check all publication entries.

Response:
def __rm_duplicate_citation_2(self, key):
        """
        Remove exact duplicate entries for abstract or citation fields. Check all publication entries.
        :return:
        """
        citations = []
        # Before writing, remove any duplicate "Full_Citations" that are found
        for idx, pub in enumerate(self.noaa_data_sorted["Publication"]):
            try:
                citations.append(pub[key])
            except KeyError:
                # Key was not found. Enter a blank entry in citations list
                citations.append("")

        # Create a backwards dictionary, that lists which indexes are duplicates
        d = defaultdict(list)
        for i, item in enumerate(citations):
            d[item].append(i)
        d = {k: v for k, v in d.items() if len(v) > 1}
        # Duplicates indexes are listed like so:
        # {"full citation info here": [0, 2, 5]}

        # Loop over duplicate indexes
        for citation, idxs in d.items():
            # do not process any duplicate entries that were initially blanks.
            if citation:
                # Loop for [1:], since we want to keep the first citation and remove the rest.
                for idx in idxs[1:]:
                    # Set citation to blank
                    self.noaa_data_sorted["Publication"][idx][key] = ""


Instruction:
Given the following Python 3 function, write the documentation
def __rm_names_on_csv_cols(d):
        _names = []
        _data = []
        for name, data in d.items():
            _names.append(name)
            _data.append(data)
        return _names, _data

Response: Remove the variableNames from the columns.


Instruction:
How would you implement a function in Python 3 that
takes a string of author names get the first author name then get the first authors last name only.

Response:
def __get_author_last_name(author):
        """
        Take a string of author(s), get the first author name, then get the first authors last name only.

        :param str author: Author(s)
        :return str _author: First author's last name
        """
        _author = ""
        if isinstance(author, str):
            try:
                # example:   'Ahmed, Moinuddin and Anchukaitis, Kevin J and ...'
                # If this is a list of authors, try to split it and just get the first author.
                if " and " in author:
                    _author = author.split(" and ")[0]
                # 'Ahmed, Moinuddin; Anchukaitis, Kevin J;  ...'
                elif ";" in author:
                    _author = author.split(";")[0]
            except Exception:
                _author = ""
            try:
                # example :  'Ahmed Moinuddin,  Anchukaitis Kevin J,  ...'
                _author = author.replace(" ", "")
                if "," in author:
                    _author = author.split(",")[0]
            except Exception:
                _author = ""
        elif isinstance(author, list):
            try:
                # example:  [{'name': 'Ahmed, Moinuddin'}, {'name': 'Anchukaitis, Kevin J.'}, ..]
                # just get the last name of the first author. this could get too long.
                _author = author[0]["name"].split(",")[0]
            except Exception as e:
                _author = ""

        return _author


Instruction:
Implement a Python 3 function for
creating a string that can be used to get the authors from the first publication.

Response:
def __create_author_investigator_str(self):
        """
        When investigators is empty, try to get authors from the first publication instead.
        :return str author: Author names
        """
        _author = ""
        try:
            for pub in self.noaa_data_sorted["Publication"]:
                if "author" in pub:
                    if pub["author"]:
                        _author_src = pub["author"]
                        if isinstance(_author_src, str):
                            try:
                                if " and " in _author_src:
                                    _author = _author_src.replace(" and ", "; ")
                                elif ";" in _author_src:
                                    # If there is a semi-colon, add a space after it, just in case it didn't have one
                                    _author = _author_src.replace(";", "; ")
                                break
                            except Exception as e:
                                _author = ""
                        elif isinstance(_author_src, list):
                            try:
                                for _entry in _author_src:
                                    _author += _entry["name"].split(",")[0] + ", "
                            except Exception as e:
                                _author = ""
        except Exception:
            _author = ""
        return _author


Instruction:
Write a Python 3 script to
get filename from a data table

Response:
def __get_filename(table):
        """
        Get filename from a data table
        :param dict table:
        :return str:
        """
        try:
            filename = table['filename']
        except KeyError as e:
            filename = ""
            logger_lpd_noaa.warning("get_filename: KeyError: Table missing filename, {}".format(e))
        except TypeError:
            try:
                filename = table[0]["filename"]
            except Exception as e:
                filename = ""
                logger_lpd_noaa.warning("get_filename: Generic: Unable to get filename from table, {}".format(e))
        return filename


Instruction:
Implement a function in Python 3 to
get DOI from this ONE publication entry.

Response:
def __get_doi(pub):
        """
        Get DOI from this ONE publication entry.
        :param dict pub: Single publication entry
        :return:
        """
        doi = ""
        # Doi location: d["pub"][idx]["identifier"][0]["id"]
        try:
            doi = pub["DOI"][0]["id"]
            doi = clean_doi(doi)
        except KeyError:
            logger_lpd_noaa.info("get_dois: KeyError: missing a doi key")
        except Exception:
            logger_lpd_noaa.info("get_dois: Exception: something went wrong")

        # if we received a doi that's a list, we want to concat into a single string
        if isinstance(doi, list):
            if len(doi) == 1:
                doi = doi[0]
            else:
                ", ".join(doi)
        return doi


Instruction:
How would you explain what the following Python 3 function does
def __get_noaa_key_w_context(header, lipd_key):
        try:
            noaa_key = LIPD_NOAA_MAP_BY_SECTION[header][lipd_key]
        except KeyError:
            return lipd_key
        return noaa_key

Response: Get the proper noaa key for a given lipd key.


Instruction:
Can you write a function in Python 3 where it
searches for either Age or Year to calculate the max, min, and time unit for this table/file. Preference: Look for Age first, and then Year second (if needed) :param dict table: Table data :return dict: Max, min, and time unit

Response:
def __get_max_min_time_1(self, table):
        """
        Search for either Age or Year to calculate the max, min, and time unit for this table/file.
        Preference: Look for Age first, and then Year second (if needed)

        :param dict table: Table data
        :return dict: Max, min, and time unit
        """
        try:
            # find the values and units we need to calculate
            vals, units = self.__get_max_min_time_2(table, ["age", "yearbp", "yrbp"], True)

            if not vals and not units:
                vals, units = self.__get_max_min_time_2(table, ["year", "yr"], True)

            if not vals and not units:
                vals, units = self.__get_max_min_time_2(table, ["age", "yearbp", "yrbp"], False)

            if not vals and not units:
                vals, units = self.__get_max_min_time_2(table, ["year", "yr"], False)

            # now put this data into the noaa data sorted self for writing to file.
            # year farthest in the past
            _max = max(vals)
            _min = min(vals)
            if _min or _min in [0, 0.0]:
                self.noaa_data_sorted["Data_Collection"]["Earliest_Year"] = str(_min)
            if _max or _max in [0, 0.0]:
                self.noaa_data_sorted["Data_Collection"]["Most_Recent_Year"] = str(_max)
            # AD or... yrs BP?
            self.noaa_data_sorted["Data_Collection"]["Time_Unit"] = units

        except Exception as e:
            logger_lpd_noaa.debug("get_max_min_time_2: {}".format(e))

        return


Instruction:
Explain what the following Python 3 code does
def __get_max_min_time_2(table, terms, exact):
        vals = []
        unit = ""
        try:
            for k, v in table["columns"].items():
                if exact:
                    if k.lower() in terms:
                        try:
                            vals = v["values"]
                            unit = v["units"]
                            break
                        except KeyError:
                            pass
                elif not exact:
                    for term in terms:
                        if term in k:
                            try:
                                vals = v["values"]
                                unit = v["units"]
                                break
                            except KeyError:
                                pass

        except Exception as e:
            logger_lpd_noaa.debug("get_max_min_time_3: {}".format(e))

        return vals, unit

Response: Get the max and min time for a given table and list of terms.


Instruction:
Write a Python 3 script for
looping over all publication entries and add data citation to self. data_citation

Response:
def __get_data_citation(self, l):
        """
        If originalDataURL / investigators not in root data, check for a dataCitation pub entry.
        :return:
        """
        # loop once for each publication entry
        for pub in l:
            try:
                # at the moment, these are the only keys of interest inside of dataCitation. Check each.
                for key in ["url", "investigators"]:
                    if pub["type"] == "dataCitation" and key in pub:
                        noaa_key = self.__get_noaa_key(key)
                        self.data_citation[noaa_key] = pub[key]
            except KeyError:
                # no "type" key in pub
                logger_lpd_noaa.info("lpd_noaa: get_data_citation: KeyError: pub is missing 'type'")
        return


Instruction:
Write a Python 3 function that can
set up the output filenames.

Response:
def __get_output_filenames(self):
        """
        Set up the output filenames. If more than one file is being written out, we need appended filenames.
        :return:
        """
        # if there is only one file, use the normal filename with nothing appended.
        if len(self.noaa_data_sorted["Data"]) == 1:
            self.output_filenames.append(self.filename_txt)

        # if there are multiple files that need to be written out, (multiple data table pairs), then append numbers
        elif len(self.noaa_data_sorted["Data"]) > 1:
            for i in range(0, self.output_file_ct):
                tmp_name = "{}-{}.txt".format(self.dsn, i+1)
                self.output_filenames.append(tmp_name)
        return


Instruction:
Write a Python 3 function that can
get the paleo tables and the chron tables.

Response:
def __get_table_pairs(self):
        """
        Use the tables in self.paleos and self.chrons (sorted by idx) to put in self.noaa_data_sorted.
        :return:
        """
        try:
            for _idx, _p in enumerate(self.data_paleos):
                _c = {}
                try:
                    _c = self.data_chrons[_idx]
                except IndexError:
                    pass
                # create entry in self object collection of data tables
                self.noaa_data_sorted["Data"].append({"paleo": _p, "chron": _c})
        except KeyError:
            logger_lpd_noaa.warning("lpd_noaa: get_meas_table: 0 paleo data tables")

        self.output_file_ct = len(self.noaa_data_sorted["Data"])
        return


Instruction:
Create a Python 3 function for
checking how many tables are in the json and if so check if we have to make the append_filenames flag

Response:
def __get_table_count(self):
        """
        Check how many tables are in the json
        :return int:
        """
        _count = 0
        try:
            keys = ["paleo", "paleoData", "paleoMeasurementTable"]
            # get the count for how many tables we have. so we know to make appended filenames or not.
            for pd_name, pd_data in self.lipd_data["paleoData"].items():
                for section_name, section_data in pd_data.items():
                    _count += len(section_data)

        except Exception:
            pass

        if _count > 1:
            self.append_filenames = True

        return


Instruction:
Make a summary of the following Python 3 code
def __create_file(self):
        logger_lpd_noaa.info("enter create_file")

        self.__get_output_filenames()

        for idx, filename in enumerate(self.output_filenames):
            try:
                # self.noaa_txt = open(os.path.join(self.path, filename), "w+")
                # self.noaa_file_output[filename] = ""
                # self.noaa_txt = self.noaa_file_output[filename]
                self.noaa_txt = ""
                print("writing: {}".format(filename))
                logger_lpd_noaa.info("write_file: opened output txt file")
            except Exception as e:
                logger_lpd_noaa.error("write_file: failed to open output txt file, {}".format(e))
                return

            self.__get_max_min_time_1(self.noaa_data_sorted["Data"][idx]["paleo"])
            self.__check_time_values()

            self.__write_top(filename)
            self.__write_generic('Contribution_Date')
            self.__write_generic('File_Last_Modified_Date')
            self.__write_generic('Title')
            self.__write_generic('Investigators')
            self.__write_generic('Description_Notes_and_Keywords')
            self.__write_pub()
            self.__write_funding()
            self.__write_geo()
            self.__write_generic('Data_Collection')
            self.__write_generic('Species')
            self.__write_data(idx)

            self.noaa_file_output[filename] = self.noaa_txt
            # logger_lpd_noaa.info("closed output text file")
            # reset the max min time unit to none
            self.max_min_time = {"min": "", "max": "", "time": ""}
            # shutil.copy(os.path.join(os.getcwd(), filename), self.dir_root)
            logger_lpd_noaa.info("exit create_file")
        return

Response: Create a new noaa file.


Instruction:
Write a Python 3 script for
writing the top section of the txt file.

Response:
def __write_top(self, filename_txt):
        """
        Write the top section of the txt file.
        :param int section_num: Section number
        :return none:
        """
        logger_lpd_noaa.info("writing section: {}".format("top"))
        self.__create_blanks("Top", self.noaa_data_sorted["Top"])

        # Start writing the NOAA file section by section, starting at the very top of the template.
        self.noaa_txt += "# {}".format(self.noaa_data_sorted["Top"]['Study_Name'])
        self.__write_template_top()
        # We don't know what the full online resource path will be yet, so leave the base path only
        self.__write_k_v("Online_Resource", "{}/{}".format(self.wds_url, filename_txt), top=True)
        self.__write_k_v("Online_Resource_Description", " This file.  NOAA WDS Paleo formatted metadata and data for version {} of this dataset.".format(self.version), indent=True)
        self.__write_k_v("Online_Resource", "{}".format(self.lpd_file_url), top=True)
        self.__write_k_v("Online_Resource_Description", " Linked Paleo Data (LiPD) formatted file containing the same metadata and data as this file, for version {} of this dataset.".format(self.version), indent=True)
        self.__write_k_v("Original_Source_URL", self.noaa_data_sorted["Top"]['Original_Source_URL'], top=True)
        self.noaa_txt += "\n# Description/Documentation lines begin with #\n# Data lines have no #\n#"
        self.__write_k_v("Archive", self.noaa_data_sorted["Top"]['Archive'])
        self.__write_k_v("Parameter_Keywords", self.noaa_data_sorted["Top"]['Parameter_Keywords'])

        # get the doi from the pub section of self.steps_dict[6]
        # doi = self.__get_doi()
        self.__write_k_v("Dataset_DOI",  ', '.join(self.doi), False, True, False, False)
        # self.__write_k_v("Parameter_Keywords", self.steps_dict[section_num]['parameterKeywords'])
        self.__write_divider()
        logger_lpd_noaa.info("exit write_top")
        return


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __write_generic(self, header, d=None):
        logger_lpd_noaa.info("writing section: {}".format(header))
        if not d:
            d = self.noaa_data_sorted[header]
        d = self.__create_blanks(header, d)
        self.__write_header_name(header)
        for key in NOAA_KEYS_BY_SECTION[header]:
            key = self.__get_noaa_key_w_context(header, key)
            # NOAA writes value and units on one line. Build the string here.
            # if key == 'coreLength':
            #     value, unit = self.__get_corelength(val)
            #     val = str(value) + " " + str(unit)
            # DOI  id is nested in "identifier" block. Retrieve it.
            if key == "DOI":
                val = self.__get_doi(d)
                # Don't write out an empty DOI list. Write out empty string instead.
                if not val:
                    val = ""
            elif key == "Investigators":
                # Investigators is a section by itself. "d" should be a direct list
                val = get_authors_as_str(d)
                # If we don't have investigator data, use the authors from the first publication entry
                if not val:
                    val = self.__create_author_investigator_str()
            # lipd uses a singular "author" key, while NOAA uses plural "authors" key.
            elif key in ["Author", "Authors"]:
                # "d" has all publication data, so only pass through the d["Authors"] piece
                val = get_authors_as_str(d[key])
            elif key == "Collection_Name":
                if not d[key]:
                    # If there is not a collection name, then use the dsn so _something_ is there.
                    val = self.dsn
            else:
                val = d[key]

            # Write the output line
            self.__write_k_v(str(self.__get_noaa_key(key)), val, indent=True)
        # Don't write a divider if there isn't a Chron section after species. It'll make a double.
        # if header == "Species" and not self.noaa_data_sorted["Species"]:
        #     return
        self.__write_divider()
        return

Response: Write a generic section to the. txt file.


Instruction:
How would you explain what the following Python 3 function does
def __write_pub(self):
        try:
            self.__reorganize_author()
            # Check all publications, and remove possible duplicate Full_Citations
            self.__rm_duplicate_citation_1()
            if not self.noaa_data_sorted["Publication"]:
                self.noaa_data_sorted["Publication"].append({"pubYear": ""})
            for idx, pub in enumerate(self.noaa_data_sorted["Publication"]):
                logger_lpd_noaa.info("publication: {}".format(idx))
                # Do not write out Data Citation publications. Check, and skip if necessary
                is_data_citation = self.__get_pub_type(pub)
                if not is_data_citation or is_data_citation and len(self.noaa_data_sorted["Publication"]) == 1:
                    pub = self.__convert_keys_1("Publication", pub)
                    self.__write_generic('Publication', pub)
        except KeyError:
            logger_lpd_noaa.info("write_pub: KeyError: pub section not found")
        except TypeError:
            logger_lpd_noaa.debug("write_pub: TypeError: pub not a list type")

        return

Response:
Write pub section. There may be multiple, so write a generic section for each one.
        :return none:


Instruction:
How would you implement a function in Python 3 that
writes funding section. There are likely multiple entries. :param dict d: :return none:

Response:
def __write_funding(self):
        """
        Write funding section. There are likely multiple entries.
        :param dict d:
        :return none:
        """
        self.__reorganize_funding()
        # if funding is empty, insert a blank entry so that it'll still write the empty section on the template.
        if not self.noaa_data_sorted["Funding_Agency"]:
            self.noaa_data_sorted["Funding_Agency"].append({"grant": "", "agency": ""})
        for idx, entry in enumerate(self.noaa_data_sorted["Funding_Agency"]):
            logger_lpd_noaa.info("funding: {}".format(idx))
            self.__write_generic('Funding_Agency', entry)
        return


Instruction:
Make a summary of the following Python 3 code
def __write_data(self, idx):
        pair = self.noaa_data_sorted["Data"][idx]
        # Run once for each pair (paleo+chron) of tables that was gathered earlier.
        # for idx, pair in enumerate(self.noaa_data_sorted["Data"]):
        lst_pc = ["chron", "paleo"]
        # loop once for paleo, once for chron
        for pc in lst_pc:
            # safeguard in case the table is an empty set.
            table = pair[pc]
            if pc == "paleo":
                self.__write_variables_1(table)
                self.__write_divider()
                self.__write_columns(pc, table)
            elif pc == "chron":
                # self.__write_variables_1(table)
                self.__write_columns(pc, table)
                self.__write_divider(nl=False)
        return

Response: Write out the measurement tables found in paleoData and chronData.


Instruction:
Given the following Python 3 function, write the documentation
def __write_variables_1(self, table):
        logger_lpd_noaa.info("writing section: {}".format("Variables"))

        # Write the template lines first
        self.__write_template_variable()

        try:
            self.noaa_txt += '#'

            # Special NOAA Request: Write the "year" column first always, if available
            # write year data first, when available
            for name, data in table["columns"].items():
                if name == "year":
                    # write first line in variables section here
                    self.__write_variables_2(data)
                    # leave the loop, because this is all we needed to accomplish
                    break

            # all other cases
            for name, data in table["columns"].items():
                # we already wrote out the year column, so don't duplicate.
                if name != "year":
                    self.__write_variables_2(data)

        except KeyError as e:
            logger_lpd_noaa.warn("write_variables: KeyError: {} not found".format(e))
        return

Response: Write the variables section of txt file.


Instruction:
How would you explain what the following Python 3 function does
def __write_variables_2(self, col):
        col = self.__convert_keys_1("Variables", col)

        # Write one line for each column. One line has all metadata for one column.
        for entry in NOAA_KEYS_BY_SECTION["Variables"]:
            # May need a better way of handling this in the future. Need a strict list for this section.
            try:
                # First entry: Add extra hash and tab
                if entry == 'shortname':
                    # DEPRECATED: Fixed spacing for variable names.
                    # self.noaa_txt.write('{:<20}'.format('#' + str(col[entry])))
                    # Fluid spacing for variable names. Spacing dependent on length of variable names.
                    self.noaa_txt += '{}\t'.format('#' + str(col[entry]))
                # Last entry: No space or comma
                elif entry == "additional":
                    e = " "
                    for item in ["notes", "uncertainty"]:
                        try:
                            if col[item]:
                                e += str(col[item]).replace(",", ";") + "; "
                        except KeyError:
                            pass
                    self.noaa_txt += '{} '.format(e)

                # elif entry == 'notes':
                #     self.noaa_txt.write('{} '.format(str(col[entry])))
                else:
                    # This is for any entry that is not first or last in the line ordering
                    # Account for nested entries.
                    # if entry == "uncertainty":
                    #     try:
                    #         e = str(col["calibration"][entry])
                    #     except KeyError:
                    #         e = ""
                    if entry == "seasonality":
                        try:
                            e = str(col["climateInterpretation"][entry])
                        except KeyError:
                            e = ""
                    elif entry == "archive":
                        e = self.noaa_data_sorted["Top"]["Archive"]
                    elif entry == "dataType":
                        # Lipd uses real data types (floats, ints), NOAA wants C or N (character or numeric)
                        if col[entry] == "float":
                            e = "N"
                        else:
                            e = "C"
                    else:
                        e = str(col[entry])
                    try:
                        e = e.replace(",", ";")
                    except AttributeError as ee:
                        logger_lpd_noaa.warn("write_variables_2: AttributeError: {}, {}".format(e, ee))
                    self.noaa_txt += '{}, '.format(e)
            except KeyError as e:
                self.noaa_txt += '{:<0}'.format(',')
                logger_lpd_noaa.info("write_variables: KeyError: missing {}".format(e))
        self.noaa_txt += '\n#'
        return

Response: Write the variables section.


Instruction:
Can you implement a function in Python 3 that
writes the numeric data to the bottom section of the txt file.

Response:
def __write_columns(self, pc, table):
        """
        Read numeric data from csv and write to the bottom section of the txt file.
        :param dict table: Paleodata dictionary
        :return none:
        """
        logger_lpd_noaa.info("writing section: data, csv values from file")
        # get filename for this table's csv data
        # filename = self.__get_filename(table)
        # logger_lpd_noaa.info("processing csv file: {}".format(filename))
        # # get missing value for this table
        # # mv = self.__get_mv(table)
        # # write template lines
        # # self.__write_template_paleo(mv)
        if pc == "paleo":
            self.__write_template_paleo()
        elif pc == "chron":
            self.__write_template_chron()

        # continue if csv exists
        if self._values_exist(table):
            # logger_lpd_noaa.info("_write_columns: csv data exists: {}".format(filename))

            # sort the dictionary so the year column is first
            _csv_data_by_name = self.__put_year_col_first(table["columns"])

            # now split the sorted dictionary back into two lists (easier format to write to file)
            _names, _data = self.__rm_names_on_csv_cols(_csv_data_by_name)

            # write column variableNames
            self.__write_data_col_header(_names, pc)

            # write data columns index by index
            self.__write_data_col_vals(_data, pc)

        return


Instruction:
Explain what the following Python 3 code does
def __write_k_v(self, k, v, top=False, bot=False, multi=False, indent=False):
        if top:
            self.noaa_txt += "\n#"
        if multi:
            for item in v:
                if indent:
                    self.noaa_txt += "\n#     {}: {}".format(str(k), str(item))
                else:
                    self.noaa_txt += "\n# {}: {}".format(str(k), str(item))
        else:
            if indent:
                self.noaa_txt += "\n#     {}: {}".format(str(k), str(v))
            else:
                self.noaa_txt += "\n# {}: {}".format(str(k), str(v))
        if bot:
            self.noaa_txt += "\n#"
        return

Response: Write a key - value pair to the output file.


Instruction:
How would you explain what the following Python 3 function does
def __write_divider(self, top=False, bot=False, nl=True):
        if top:
            self.noaa_txt += "\n#"
        if nl:
            self.noaa_txt += "\n"
        self.noaa_txt += "#------------------\n"
        if bot:
            self.noaa_txt += "\n#"
        return

Response: Write a divider line


Instruction:
How would you explain what the following Python 3 function does
def __write_data_col_header(self, l, pc):
        count = len(l)
        if pc == "chron":
            self.noaa_txt += "# "
        for name in l:
            # last column - spacing not important
            if count == 1:
                self.noaa_txt += "{}\t".format(name)
            # all [:-1] columns - fixed spacing to preserve alignment
            else:
                self.noaa_txt += "{:<15}".format(name)
                count -= 1

        self.noaa_txt += '\n'

Response: Write the variableNames that are the column header in the Data section.


Instruction:
Can you tell what is the following Python 3 function doing
def __write_data_col_vals(self, ll, pc):

        # all columns should have the same amount of values. grab that number
        try:
            _items_in_cols = len(ll[0]["values"])
            for idx in range(0, _items_in_cols):
                # amount of columns
                _count = len(ll)
                self.noaa_txt += "# "
                for col in ll:
                    self.noaa_txt += "{}\t".format(str(col["values"][idx]))
                    _count -= 1
                if (idx < _items_in_cols):
                    self.noaa_txt += '\n'

        except IndexError:
            logger_lpd_noaa("_write_data_col_vals: IndexError: couldn't get length of columns")

        return

Response: Loop over value arrays and write index by index to correspond to the rows of a txt file.


Instruction:
Explain what the following Python 3 code does
def copy(app, need, needs, option, need_id=None):
    if need_id is not None:
        need = needs[need_id]

    return need[option]

Response: Copies the value of one need option to another need.


Instruction:
How would you implement a function in Python 3 that
checks if all linked needs have a given value.

Response:
def check_linked_values(app, need, needs, result, search_option, search_value, filter_string=None, one_hit=False):
    """
    Returns a specific value, if for all linked needs a given option has a given value.

    The linked needs can be filtered by using the ``filter`` option.

    If ``one_hit`` is set to True, only one linked need must have a positive match for the searched value.

    **Examples**

    **Needs used as input data**

    .. code-block:: jinja

        .. req:: Input A
           :id: clv_A
           :status: in progress

        .. req:: Input B
           :id: clv_B
           :status: in progress

        .. spec:: Input C
           :id: clv_C
           :status: closed

    .. req:: Input A
       :id: clv_A
       :status: in progress
       :collapse: False

    .. req:: Input B
       :id: clv_B
       :status: in progress
       :collapse: False

    .. spec:: Input C
       :id: clv_C
       :status: closed
       :collapse: False

    **Example 1: Positive check**

    Status gets set to *progress*.

    .. code-block:: jinja

        .. spec:: result 1: Positive check
           :links: clv_A, clv_B
           :status: [[check_linked_values('progress', 'status', 'in progress' )]]

    .. spec:: result 1: Positive check
       :id: clv_1
       :links: clv_A, clv_B
       :status: [[check_linked_values('progress', 'status', 'in progress' )]]
       :collapse: False

    **Example 2: Negative check**

    Status gets not set to *progress*, because status of linked need *clv_C* does not match *"in progress"*.

    .. code-block:: jinja

        .. spec:: result 2: Negative check
           :links: clv_A, clv_B, clv_C
           :status: [[check_linked_values('progress', 'status', 'in progress' )]]

    .. spec:: result 2: Negative check
       :id: clv_2
       :links: clv_A, clv_B, clv_C
       :status: [[check_linked_values('progress', 'status', 'in progress' )]]
       :collapse: False

    **Example 3: Positive check thanks of used filter**

    status gets set to *progress*, because linked need *clv_C* is not part of the filter.

    .. code-block:: jinja

        .. spec:: result 3: Positive check thanks of used filter
           :links: clv_A, clv_B, clv_C
           :status: [[check_linked_values('progress', 'status', 'in progress', 'type == "req" ' )]]

    .. spec:: result 3: Positive check thanks of used filter
       :id: clv_3
       :links: clv_A, clv_B, clv_C
       :status: [[check_linked_values('progress', 'status', 'in progress', 'type == "req" ' )]]
       :collapse: False

    **Example 4: Positive check thanks of one_hit option**

    Even *clv_C* has not the searched status, status gets anyway set to *progress*.
    That's because ``one_hit`` is used so that only one linked need must have the searched
    value.

    .. code-block:: jinja

        .. spec:: result 4: Positive check thanks of one_hit option
           :links: clv_A, clv_B, clv_C
           :status: [[check_linked_values('progress', 'status', 'in progress', one_hit=True )]]

    .. spec:: result 4: Positive check thanks of one_hit option
       :id: clv_4
       :links: clv_A, clv_B, clv_C
       :status: [[check_linked_values('progress', 'status', 'in progress', one_hit=True )]]
       :collapse: False

    **Result 5: Two checks and a joint status**
    Two checks are performed and both are positive. So their results get joined.

    .. code-block:: jinja

        .. spec:: result 5: Two checks and a joint status
           :links: clv_A, clv_B, clv_C
           :status: [[check_linked_values('progress', 'status', 'in progress', one_hit=True )]] [[check_linked_values('closed', 'status', 'closed', one_hit=True )]]

    .. spec:: result 5: Two checks and a joint status
       :id: clv_5
       :links: clv_A, clv_B, clv_C
       :status: [[check_linked_values('progress', 'status', 'in progress', one_hit=True )]] [[check_linked_values('closed', 'status', 'closed', one_hit=True )]]
       :collapse: False

    :param result: value, which gets returned if all linked needs have parsed the checks
    :param search_option: option name, which is used n linked needs for the search
    :param search_value: value, which an option of a linked need must match
    :param filter_string: Checks are only performed on linked needs, which pass the defined filter
    :param one_hit: If True, only one linked need must have a positive check
    :return: result, if all checks are positive
    """
    links = need["links"]
    if not isinstance(search_value, list):
        search_value = [search_value]

    for link in links:
        if filter_string is not None:
            try:
                if not filter_single_need(needs[link], filter_string):
                    continue
            except Exception as e:
                logger.warning("CheckLinkedValues: Filter {0} not valid: Error: {1}".format(filter_string, e))

        if not one_hit and not needs[link][search_option] in search_value:
            return None
        elif one_hit and needs[link][search_option] in search_value:
            return result

    return result


Instruction:
Implement a function in Python 3 to
calculate the sum of the values of a given option in a given linked tree.

Response:
def calc_sum(app, need, needs, option, filter=None, links_only=False):
    """
    Sums the values of a given option in filtered needs up to single number.

    Useful e.g. for calculating the amount of needed hours for implementation of all linked
    specification needs.

    **Input data**

    .. spec:: Do this
       :id: sum_input_1
       :hours: 7
       :collapse: False

    .. spec:: Do that
       :id: sum_input_2
       :hours: 15
       :collapse: False

    .. spec:: Do too much
       :id: sum_input_3
       :hours: 110
       :collapse: False

    **Example 2**

    .. code-block:: jinja

       .. req:: Result 1
          :amount: [[calc_sum("hours")]]

    .. req:: Result 1
       :amount: [[calc_sum("hours")]]
       :collapse: False

    **Example 2**

    .. code-block:: jinja

       .. req:: Result 2
          :amount: [[calc_sum("hours", "hours.isdigit() and float(hours) > 10")]]

    .. req:: Result 2
       :amount: [[calc_sum("hours", "hours.isdigit() and float(hours) > 10")]]
       :collapse: False

    **Example 3**

    .. code-block:: jinja

       .. req:: Result 3
          :links: sum_input_1; sum_input_3
          :amount: [[calc_sum("hours", links_only="True")]]

    .. req:: Result 3
       :links: sum_input_1; sum_input_3
       :amount: [[calc_sum("hours", links_only="True")]]
       :collapse: False

    **Example 4**

    .. code-block:: jinja

       .. req:: Result 4
          :links: sum_input_1; sum_input_3
          :amount: [[calc_sum("hours", "hours.isdigit() and float(hours) > 10", "True")]]

    .. req:: Result 4
       :links: sum_input_1; sum_input_3
       :amount: [[calc_sum("hours", "hours.isdigit() and float(hours) > 10", "True")]]
       :collapse: False

    :param option: Options, from which the numbers shall be taken
    :param filter: Filter string, which all needs must passed to get their value added.
    :param links_only: If "True", only linked needs are taken into account.

    :return: A float number
    """
    if not links_only:
        check_needs = needs.values()
    else:
        check_needs = []
        for link in need["links"]:
            check_needs.append(needs[link])

    calculated_sum = 0

    for check_need in check_needs:
        if filter is not None:
            try:
                if not filter_single_need(check_need, filter):
                    continue
            except ValueError as e:
                pass
            except NeedInvalidFilter as ex:
                logger.warning('Given filter is not valid. Error: {}'.format(ex))
        try:
            calculated_sum += float(check_need[option])
        except ValueError:
            pass

    return calculated_sum


Instruction:
Can you write a function in Python 3 where it
prompts user for conversion between NOAA and LiPD file formats.

Response:
def noaa_prompt():
    """
    Convert between NOAA and LiPD file formats.
    :return:
    """
    logger_noaa.info("enter noaa")
    # Run lpd_noaa or noaa_lpd ?
    print("Which conversion?\n1. LPD to NOAA\n2. NOAA to LPD\n")
    mode = input("Option: ")
    logger_noaa.info("chose option: {}".format(mode))

    return mode


Instruction:
Make a summary of the following Python 3 code
def noaa_prompt_1():
    print("Enter the project information below. We'll use this to create the WDS URL")
    print("What is the project name?")
    _project = input(">")
    print("What is the project version?")
    _version = input(">")
    return _project, _version

Response: Prompt user for the project and version number of the WDS links.


Instruction:
Can you create a Python 3 function that
converts NOAA format to LiPD format

Response:
def noaa_to_lpd(files):
    """
    Convert NOAA format to LiPD format
    :param dict files: Files metadata
    :return None:
    """
    logger_noaa.info("enter process_noaa")
    # only continue if the user selected a mode correctly
    logger_noaa.info("Found {} NOAA txt file(s)".format(str(len(files[".txt"]))))
    print("Found {} NOAA txt file(s)".format(str(len(files[".txt"]))))
    # Process each available file of the specified .lpd or .txt type
    for file in files[".txt"]:
        # try to filter out example files and stuff without real data
        if "template" not in file["filename_ext"] and "example" not in file["filename_ext"]:
            os.chdir(file["dir"])
            print('processing: {}'.format(file["filename_ext"]))
            logger_noaa.info("processing: {}".format(file["filename_ext"]))

            # Unzip file and get tmp directory path
            dir_tmp = create_tmp_dir()
            try:
                NOAA_LPD(file["dir"], dir_tmp, file["filename_no_ext"]).main()
            except Exception as e:
                print("Error: Unable to convert file: {}, {}".format(file["filename_no_ext"], e))

            # Create the lipd archive in the original file's directory.
            zipper(root_dir=dir_tmp, name="bag", path_name_ext=os.path.join(file["dir"], file["filename_no_ext"] + ".lpd"))
            # Delete tmp folder and all contents
            os.chdir(file["dir"])
            try:
                shutil.rmtree(dir_tmp)
            except FileNotFoundError:
                # directory is already gone. keep going.
                pass

    logger_noaa.info("exit noaa_to_lpd")
    return


Instruction:
Create a Python 3 function for
converting a LiPD format to NOAA format

Response:
def lpd_to_noaa(D, wds_url, lpd_url, version, path=""):
    """
    Convert a LiPD format to NOAA format

    :param dict D: Metadata
    :return dict D: Metadata
    """
    logger_noaa.info("enter process_lpd")
    d = D
    try:
        dsn = get_dsn(D)
        # Remove all the characters that are not allowed here. Since we're making URLs, they have to be compliant.
        dsn = re.sub(r'[^A-Za-z-.0-9]', '', dsn)
        # project = re.sub(r'[^A-Za-z-.0-9]', '', project)
        version = re.sub(r'[^A-Za-z-.0-9]', '', version)
        # Create the conversion object, and start the conversion process
        _convert_obj = LPD_NOAA(D, dsn, wds_url, lpd_url, version, path)
        _convert_obj.main()
        # get our new, modified master JSON from the conversion object
        d = _convert_obj.get_master()
        noaas = _convert_obj.get_noaa_texts()
        __write_noaas(noaas, path)
        # remove any root level urls that are deprecated
        d = __rm_wdc_url(d)

    except Exception as e:
        logger_noaa.error("lpd_to_noaa: {}".format(e))
        print("Error: lpd_to_noaa: {}".format(e))

    # logger_noaa.info("exit lpd_to_noaa")
    return d


Instruction:
How would you explain what the following Python 3 function does
def __write_noaas(dat, path):
    for filename, text in dat.items():
        try:
            with open(os.path.join(path, filename), "w+") as f:
                f.write(text)
        except Exception as e:
            print("write_noaas: There was a problem writing the NOAA text file: {}: {}".format(filename, e))
    return

Response: Write the data as NOAA text files


Instruction:
Given the following Python 3 function, write the documentation
def _parse_java_version(line: str) -> tuple:
    m = VERSION_RE.search(line)
    version_str = m and m.group(0).replace('"', '') or '0.0.0'
    if '_' in version_str:
        fst, snd = version_str.split('_', maxsplit=2)
        version = parse_version(fst)
        return (version[1], version[2], int(snd))
    else:
        return parse_version(version_str)

Response: Parse the java version line.


Instruction:
Can you generate the documentation for the following Python 3 function
def find_java_home(cratedb_version: tuple) -> str:
    if MIN_VERSION_FOR_JVM11 <= cratedb_version < (4, 0):
        # Supports 8 to 11+, use whatever is set
        return os.environ.get('JAVA_HOME', '')
    if cratedb_version < MIN_VERSION_FOR_JVM11:
        return _find_matching_java_home(lambda ver: ver[0] == 8)
    else:
        return _find_matching_java_home(lambda ver: ver[0] >= 11)

Response: Find the JAVA_HOME suites for the given CrateDB version.


Instruction:
Can you tell what is the following Python 3 function doing
def zipper(root_dir="", name="", path_name_ext=""):
    logger_zips.info("re_zip: name: {}, dir_tmp: {}".format(path_name_ext, root_dir))
    # creates a zip archive in current directory. "somefile.lpd.zip"
    shutil.make_archive(path_name_ext, format='zip', root_dir=root_dir, base_dir=name)
    # drop the .zip extension. only keep .lpd
    os.rename("{}.zip".format(path_name_ext), path_name_ext)
    return

Response: Zip the entire tree of the current directory.


Instruction:
Given the following Python 3 function, write the documentation
def unzipper(filename, dir_tmp):
    logger_zips.info("enter unzip")
    # Unzip contents to the tmp directory
    try:
        with zipfile.ZipFile(filename) as f:
            f.extractall(dir_tmp)
    except FileNotFoundError as e:
        logger_zips.debug("unzip: FileNotFound: {}, {}".format(filename, e))
        shutil.rmtree(dir_tmp)
    logger_zips.info("exit unzip")
    return

Response: Unzip the contents of a single. lpd file into a tmp directory.


Instruction:
Can you implement a function in Python 3 that
calculates the percentile using the nearest rank method.

Response:
def percentile(sorted_values, p):
    """Calculate the percentile using the nearest rank method.

    >>> percentile([15, 20, 35, 40, 50], 50)
    35

    >>> percentile([15, 20, 35, 40, 50], 40)
    20

    >>> percentile([], 90)
    Traceback (most recent call last):
        ...
    ValueError: Too few data points (0) for 90th percentile
    """
    size = len(sorted_values)
    idx = (p / 100.0) * size - 0.5
    if idx < 0 or idx > size:
        raise ValueError('Too few data points ({}) for {}th percentile'.format(size, p))
    return sorted_values[int(idx)]


Instruction:
Implement a function in Python 3 to
return a sampler constructor for the given sample_mode.

Response:
def get_sampler(sample_mode: str):
    """Return a sampler constructor

    >>> get_sampler('all')
    <class 'cr8.metrics.All'>

    >>> get_sampler('reservoir')
    <class 'cr8.metrics.UniformReservoir'>

    >>> get_sampler('reservoir:100')
    functools.partial(<class 'cr8.metrics.UniformReservoir'>, size=100)
    """
    if sample_mode == 'all':
        return All
    mode = sample_mode.split(':')
    if mode[0] == 'reservoir':
        if len(mode) == 2:
            return partial(UniformReservoir, size=int(mode[1]))
        else:
            return UniformReservoir
    raise TypeError(f'Invalid sample_mode: {sample_mode}')


Instruction:
How would you explain what the following Python 3 function does
def query(self, criteria, **opts):
        time_now = datetime.datetime.now().replace(second=0, microsecond=0)
        right_now = time_now.isoformat()
        minutes_ago = (time_now - datetime.timedelta(minutes=15)).isoformat()

        formats = opts.get('formats', 'json')
        timezone = opts.get('timezone', 'UTC')
        time_from = opts.get('time_from', minutes_ago)
        time_to = opts.get('time_to', right_now)

        # setting up options
        t_options = {
            'q': criteria,
            'format': formats,
            'tz': timezone,
            'from': time_from,
            'to': time_to,
        }
        options = '&'.join(['{}={}'.format(k, v)
                            for k, v in t_options.items()])

        req = requests.get('%s?%s' %
                           (self.url, options), auth=self.auth)

        try:
            data = req.json()
        except json.decoder.JSONDecodeError:
            data = []

        return {
            'data': data,
            'response': req.status_code,
            'reason': req.reason,
        }

Response: Query the cache for the given criteria


Instruction:
How would you explain what the following Python 3 function does
def excel_main(file):

    os.chdir(file["dir"])
    name_ext = file["filename_ext"]

    # Filename without extension
    name = file["filename_no_ext"]
    # remove foreign characters to prevent wiki uploading errors
    name = normalize_name(name)
    print("processing: {}".format(name_ext))
    logger_excel.info("processing: {}".format(name_ext))

    pending_csv = []
    final = OrderedDict()
    logger_excel.info("variables initialized")

    # Create a temporary folder and set paths
    dir_tmp = create_tmp_dir()

    """
    EACH DATA TABLE WILL BE STRUCTURED LIKE THIS
    "paleo_chron": "paleo",
    "pc_idx": 1,
    "model_idx": "",
    "table_type": "measurement",
    "table_idx": 1,
    "name": sheet,
    "filename": sheet,
    "data": { column data }
    """

    # Open excel workbook with filename
    try:
        workbook = xlrd.open_workbook(name_ext)
        logger_excel.info("opened XLRD workbook")

    except Exception as e:
        # There was a problem opening a file with XLRD
        print("Failed to open Excel workbook: {}".format(name))
        workbook = None
        logger_excel.debug("excel: xlrd failed to open workbook: {}, {}".format(name, e))

    if workbook:

        # Build sheets, but don't make full filenames yet. Need to parse metadata sheet first to get datasetname.
        sheets, ct_paleo, ct_chron, metadata_str = _get_sheet_metadata(workbook, name)

        # METADATA WORKSHEETS
        # Parse Metadata sheet and add to output dictionary
        if metadata_str:
            logger_excel.info("parsing worksheet: {}".format(metadata_str))
            final = cells_dn_meta(workbook, metadata_str, 0, 0, final)

        # Now that we have the dataSetName, we can use it to build filenames
        dsn = __get_datasetname(final, name)
        filename = str(dsn) + ".lpd"
        sheets = __set_sheet_filenames(sheets, dsn)

        dir_bag = os.path.join(dir_tmp, "bag")
        dir_data = os.path.join(dir_bag, 'data')

        # Make folders in tmp
        os.mkdir(os.path.join(dir_bag))
        os.mkdir(os.path.join(dir_data))

        # PALEO AND CHRON SHEETS
        for sheet in sheets:
            logger_excel.info("parsing data worksheet: {}".format(sheet["new_name"]))
            sheet_meta, sheet_csv = _parse_sheet(workbook, sheet)
            if sheet_csv and sheet_meta:
                pending_csv.append(sheet_csv)
                sheet["data"] = sheet_meta

        # create the metadata skeleton where we will place the tables. dynamically add empty table blocks for data.
        skeleton_paleo, skeleton_chron = _create_skeleton_1(sheets)

        # Reorganize sheet metadata into LiPD structure
        d_paleo, d_chron = _place_tables_main(sheets, skeleton_paleo, skeleton_chron)

        # Add organized metadata into final dictionary
        final['paleoData'] = d_paleo
        final['chronData'] = d_chron

        # OUTPUT

        # Create new files and dump data in dir_data
        os.chdir(dir_data)

        # WRITE CSV
        _write_data_csv(pending_csv)

        # JSON-LD
        # Invoke DOI Resolver Class to update publisher data
        try:
            logger_excel.info("invoking doi resolver")
            final = DOIResolver(file["dir"], name, final).main()
        except Exception as e:
            print("Error: doi resolver failed: {}".format(name))
            logger_excel.debug("excel: doi resolver failed: {}, {}".format(name, e))

        # Dump final_dict to a json file.
        final["lipdVersion"] = 1.2
        final["createdBy"] = "excel"
        write_json_to_file(final)

        # Move files to bag root for re-bagging
        # dir : dir_data -> dir_bag
        logger_excel.info("start cleanup")
        dir_cleanup(dir_bag, dir_data)

        # Create a bag for the 3 files
        finish_bag(dir_bag)

        # dir: dir_tmp -> dir_root
        os.chdir(file["dir"])

        # Check if same lpd file exists. If so, delete so new one can be made
        if os.path.isfile(filename):
            os.remove(filename)

        # Zip dir_bag. Creates in dir_root directory
        logger_excel.info("re-zip and rename")
        zipper(root_dir=dir_tmp, name="bag", path_name_ext=os.path.join(file["dir"], filename))

    # Move back to dir_root for next loop.
    os.chdir(file["dir"])

    # Cleanup and remove tmp directory
    shutil.rmtree(dir_tmp)

    return dsn

Response: Main function for parsing data from Excel spreadsheets into LiPD files.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_sheet_metadata(workbook, name):
    ct_paleo = 1
    ct_chron = 1
    metadata_str = ""
    sheets = []
    skip_sheets = ["example", "sample", "lists", "guidelines"]

    # Check what worksheets are available, so we know how to proceed.
    for sheet in workbook.sheet_names():

        # Use this for when we are dealing with older naming styles of "data (qc), data, and chronology"
        old = "".join(sheet.lower().strip().split())

        # Don't parse example sheets. If these words are in the sheet, assume we skip them.
        if not any(word in sheet.lower() for word in skip_sheets):
            # Group the related sheets together, so it's easier to place in the metadata later.
            if 'metadata' in sheet.lower():
                metadata_str = sheet

            # Skip the 'about' and 'proxy' sheets altogether. Proceed with all other sheets.
            elif "about" not in sheet.lower() and "proxy" not in sheet.lower():
                logger_excel.info("creating sheets metadata")

                # If this is a valid sheet name, we will receive a regex object back.
                m = re.match(re_sheet, sheet.lower())

                # Valid regex object. This is a valid sheet name and we can use that to build the sheet metadata.
                if m:
                    sheets, paleo_ct, chron_ct = _sheet_meta_from_regex(m, sheets, sheet, name, ct_paleo, ct_chron)

                # Older excel template style: backwards compatibility. Hard coded for one sheet per table.
                elif old == "data" or "data(qc)" in old or "data(original)" in old:
                    sheets.append({
                        "paleo_chron": "paleo",
                        "idx_pc": ct_paleo,
                        "idx_model": None,
                        "table_type": "measurement",
                        "idx_table": 1,
                        "old_name": sheet,
                        "new_name": sheet,
                        "filename": "paleo{}measurementTable1.csv".format(ct_paleo),
                        "table_name": "paleo{}measurementTable1".format(ct_paleo),
                        "data": ""
                    })
                    ct_paleo += 1

                # Older excel template style: backwards compatibility. Hard coded for one sheet per table.
                elif old == "chronology":
                    sheets.append({
                        "paleo_chron": "chron",
                        "idx_pc": ct_chron,
                        "idx_model": None,
                        "table_type": "measurement",
                        "idx_table": 1,
                        "old_name": sheet,
                        "new_name": sheet,
                        "filename": "chron{}measurementTable1.csv".format(ct_chron),
                        "table_name": "chron{}measurementTable1".format(ct_chron),
                        "data": ""
                    })
                    ct_chron += 1
                else:
                    # Sheet name does not conform to standard. Guide user to create a standardized sheet name.
                    print("This sheet name does not conform to naming standard: {}".format(sheet))
                    sheets, paleo_ct, chron_ct = _sheet_meta_from_prompts(sheets, sheet, name, ct_paleo, ct_chron)

    return sheets, ct_paleo, ct_chron, metadata_str

Response: Get the metadata for a single worksheet.


Instruction:
Implement a Python 3 function for
creating a proper standardized sheet name from prompts.

Response:
def _sheet_meta_from_prompts(sheets, old_name, name, ct_paleo, ct_chron):
    """
    Guide the user to create a proper, standardized sheet name
    :param list sheets: Running list of sheet metadata
    :param str old_name: Original sheet name
    :param str name: Data set name
    :param int ct_paleo: Running count of paleoData tables
    :param int ct_chron: Running count of chronData tables
    :return sheets paleo_ct chron_ct: Updated sheets and counts
    """
    cont = True
    # Loop until valid sheet name is built, or user gives up
    while cont:
        try:
            pc = input("Is this a (p)aleo or (c)hronology sheet?").lower()
            if pc in ("p", "c", "paleo", "chron", "chronology"):
                tt = input("Is this a (d)istribution, (e)nsemble, (m)easurement, or (s)ummary sheet?").lower()
                if tt in EXCEL_SHEET_TYPES["distribution"] or tt in EXCEL_SHEET_TYPES["ensemble"] \
                        or tt in EXCEL_SHEET_TYPES["summary"] or tt in EXCEL_SHEET_TYPES["measurement"]:
                    # valid answer, keep going
                    if tt in EXCEL_SHEET_TYPES["distribution"]:
                        tt = "distribution"
                    elif tt in EXCEL_SHEET_TYPES["summary"]:
                        tt = "summary"
                    elif tt in EXCEL_SHEET_TYPES["ensemble"]:
                        tt = "ensemble"
                    elif tt in EXCEL_SHEET_TYPES["measurement"]:
                        tt = "measurement"

                    if pc in EXCEL_SHEET_TYPES["paleo"]:
                        if tt in ["ensemble", "summary"]:
                            sheet = "{}{}{}{}".format("paleo", ct_paleo, tt, 1)
                        else:
                            sheet = "{}{}{}".format("paleo", ct_paleo, tt)
                    elif pc in EXCEL_SHEET_TYPES["chron"]:
                        if tt in ["ensemble", "summary"]:
                            sheet = "{}{}{}{}".format("chron", ct_chron, tt, 1)
                        else:
                            sheet = "{}{}{}".format("chron", ct_chron, tt)
                    # Test the sheet that was built from the user responses.
                    # If it matches the Regex, then continue to build the sheet metadata. If not, try again or skip sheet.
                    m = re.match(re_sheet, sheet.lower())
                    if m:
                        sheets, ct_paleo, ct_chron = _sheet_meta_from_regex(m, sheets, old_name, name, ct_paleo, ct_chron)
                        print("Sheet created: {}".format(sheet))
                        cont = False
                    else:
                        resp = input("invalid sheet name. try again? (y/n): ")
                        if resp == "n":
                            print("No valid sheet name was created. Skipping sheet: {}".format(sheet))
                            cont = False
        except Exception as e:
            logger_excel.debug("excel: sheet_meta_from_prompts: error during prompts, {}".format(e))
            cont = False

    print("=====================================================")
    return sheets, ct_paleo, ct_chron


Instruction:
Implement a function in Python 3 to
build metadata for a single sheet from a regex match object.

Response:
def _sheet_meta_from_regex(m, sheets, old_name, name, ct_paleo, ct_chron):
    """
    Build metadata for a sheet. Receive valid regex match object and use that to create metadata.
    :param obj m: Regex match object
    :param list sheets: Running list of sheet metadata
    :param str old_name: Original sheet name
    :param str name: Data set name
    :param int ct_paleo: Running count of paleoData tables
    :param int ct_chron: Running count of chronData tables
    :return sheets paleo_ct chron_ct: Updated sheets and counts
    """
    try:
        idx_model = None
        idx_table = None
        pc = m.group(1)
        # Get the model idx number from string if it exists
        if m.group(3):
            idx_model = int(m.group(4))
        # check if there's an index (for distribution tables)
        if m.group(6):
            idx_table = int(m.group(6))
        # find out table type
        if pc == "paleodata" or pc == "paleo":
            pc = "paleo"
            ct_paleo += 1
        elif pc == "chrondata" or pc == "chron":
            pc = "chron"
            ct_chron += 1
        # build filename and table name strings. build table name first, then make filename from the table_name
        new_name = "{}{}".format(pc, m.group(2))
        if idx_model:
            new_name = "{}model{}".format(new_name, idx_model)
        new_name = "{}{}".format(new_name, m.group(5))
        if idx_table:
            new_name = "{}{}".format(new_name, m.group(6))
        filename = "{}.csv".format(new_name)
    except Exception as e:
        logger_excel.debug("excel: sheet_meta_from_regex: error during setup, {}".format(e))

    # Standard naming. This matches the regex and the sheet name is how we want it.
    # paleo/chron - idx - table_type - idx
    # Not sure what to do with m.group(2) yet
    # ex: m.groups() = [ paleo, 1, model, 1, ensemble, 1 ]
    try:
        sheets.append({
            "old_name": old_name,
            "new_name": new_name,
            "filename": filename,
            "paleo_chron": pc,
            "idx_pc": int(m.group(2)),
            "idx_model": idx_model,
            "idx_table": idx_table,
            "table_type": m.group(5),
            "data": ""
        })
    except Exception as e:
        print("error: build sheets")
        logger_excel.debug("excel: build_sheet: unable to build sheet, {}".format(e))

    return sheets, ct_paleo, ct_chron


Instruction:
Create a Python 3 function for
placing data into the skeleton for either a measurement or distribution or summary tables.

Response:
def _place_tables_section(skeleton_section, sheet, keys_section):
    """
    Place data into skeleton for either a paleo or chron section.
    :param dict skeleton_section: Empty or current progress of skeleton w/ data
    :param dict sheet: Sheet metadata
    :param list keys_section: Paleo or Chron specific keys
    :return dict: Skeleton section full of data
    """
    logger_excel.info("enter place_tables_section")
    try:
        logger_excel.info("excel: place_tables_section: placing table: {}".format(sheet["new_name"]))
        new_name = sheet["new_name"]
        logger_excel.info("placing_tables_section: {}".format(new_name))
        # get all the sheet metadata needed for this function
        idx_pc = sheet["idx_pc"] - 1
        idx_model = sheet["idx_model"]
        idx_table = sheet["idx_table"]
        table_type = sheet["table_type"]
        data = sheet["data"]
        # paleoMeas or chronMeas key
        key_1 = keys_section[0]
        # paleoModel or chronModel key
        key_2 = keys_section[1]
        # Is this a measurement, or distribution table?
        if idx_table:
            # Yes, a table idx exists, so decrement it.
            idx_table = sheet["idx_table"] - 1
        # Is this a ensemble, dist, or summary table?
        if idx_model:
            # Yes, a model idx exists, so decrement it.
            idx_model -= 1
    except Exception as e:
        logger_excel.debug("excel: place_tables_section: error during setup, {}".format(e))

    # If it's measurement table, it goes in first.
    try:
        if table_type == "measurement":
            skeleton_section[idx_pc][key_1][idx_table] = data
        # Other types of tables go one step below
        elif table_type in ["ensemble", "distribution", "summary"]:
            if table_type == "summary":
                skeleton_section[idx_pc][key_2][idx_model]["summaryTable"] = data
            elif table_type == "ensemble":
                skeleton_section[idx_pc][key_2][idx_model]["ensembleTable"] = data
            elif table_type == "distribution":
                skeleton_section[idx_pc][key_2][idx_model]["distributionTable"][idx_table] = data
    except Exception as e:
        logger_excel.warn("excel: place_tables_section: Unable to place table {}, {}".format(new_name, e))
    logger_excel.info("exit place_tables_section")
    return skeleton_section


Instruction:
Given the following Python 3 function, write the documentation
def _place_tables_main(sheets, skeleton_paleo, skeleton_chron):
    logger_excel.info("enter place_tables_main")

    for sheet in sheets:
        pc = sheet["paleo_chron"]
        if pc == "paleo":
            skeleton_paleo = _place_tables_section(skeleton_paleo, sheet, ["paleoMeasurementTable", "paleoModel"])
        elif pc == "chron":
            skeleton_chron = _place_tables_section(skeleton_chron, sheet, ["chronMeasurementTable", "chronModel"])

    # when returning, these should no longer be skeletons. They should be tables filled with data
    logger_excel.info("exit place_tables_main")
    return skeleton_paleo, skeleton_chron

Response: Place tables into the main LiPD structure.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_table_counts(sheet, num_section):
    tt = sheet["table_type"]
    idx_pc = sheet["idx_pc"]
    idx_table = sheet["idx_table"]
    idx_model = sheet["idx_model"]

    # Have we started counters for this idx model yet??
    if idx_pc not in num_section:
        # No, create the counters and start tracking table counts
        num_section[idx_pc] = {"ct_meas": 0, "ct_model": 0, "ct_in_model": {}}

    # Compare indices to get the highest index number for this table type
    # If we have a higher number model index, then increment out models count

    try:
        # Is this a ens, dist, or summary table?
        if idx_model:
            # Yes it is.

            # Is this model idx higher than ours?
            if idx_model > num_section[idx_pc]["ct_model"]:
                # Yes. Now, we have N number of model tables.
                num_section[idx_pc]["ct_model"] = idx_model

            # Have we started counters for this idx model yet??
            if idx_model not in num_section[idx_pc]["ct_in_model"]:
                # No, create the counters and start tracking table counts.
                num_section[idx_pc]["ct_in_model"][idx_model] = {"ct_ens": 0, "ct_sum": 0, "ct_dist": 0}
    except Exception as e:
        logger_excel.debug("excel: get_table_counts: error incrementing model counts, ".format(e))

    # Incrementer!
    # For the given table type, track the highest index number.
    # That number is how many tables we need to make of that type.  Ex. 'measurement4', we need to make 4 empty tables
    try:
        if tt == "measurement":
            # Is this meas table a higher idx?
            if idx_table > num_section[idx_pc]["ct_meas"]:
                # Yes, set this idx to the counter.
                num_section[idx_pc]["ct_meas"] = idx_table
        elif tt == "distribution":
            # Is this dist table a higher idx?
            if idx_table > num_section[idx_pc]["ct_in_model"][idx_model]["ct_dist"]:
                # Yes, set this idx to the counter.
                num_section[idx_pc]["ct_in_model"][idx_model]["ct_dist"] = idx_table
        elif tt == "summary":
            # Summary tables are not indexed. Only one per table.
            num_section[idx_pc]["ct_in_model"][idx_model]["ct_sum"] = 1
        elif tt == "ensemble":
            # Ensemble tables are not indexed. Only one per table.
            num_section[idx_pc]["ct_in_model"][idx_model]["ct_ens"] = 1
    except Exception as e:
        logger_excel.debug("excel: get_table_counts: error incrementing table count".format(e))

    return num_section

Response: Loop through sheet metadata and count how many of each table type is needed at each index.


Instruction:
How would you implement a function in Python 3 that
creates the metadata skeleton for the current mode and the current paleo.

Response:
def _create_skeleton_3(pc, l, num_section):
    """
    Bottom level: {"measurement": [], "model": [{summary, distributions, ensemble}]}
    Fill in measurement and model tables with N number of EMPTY meas, summary, ensemble, and distributions.
    :param str pc: Paleo or Chron "mode"
    :param list l:
    :param dict num_section:
    :return dict:
    """
    logger_excel.info("enter create_skeleton_inner_2")

    # Table Template: Model
    template_model = {"summaryTable": {}, "ensembleTable": {}, "distributionTable": []}

    # Build string appropriate for paleo/chron mode
    pc_meas = "{}MeasurementTable".format(pc)
    pc_mod = "{}Model".format(pc)

    # Loop for each table count
    for idx1, table in num_section.items():
        try:
            # Create N number of empty measurement lists
            l[idx1 - 1][pc_meas] = [None] * num_section[idx1]["ct_meas"]

            # Create N number of empty model table lists
            l[idx1 - 1][pc_mod] = [copy.deepcopy(template_model)] * num_section[idx1]["ct_model"]

            # Create N number of empty model tables at list index
            #
            for idx2, nums in table["ct_in_model"].items():
                dists = []
                try:
                    # Create N number of empty distributions at list index
                    [dists.append({}) for i in range(0, nums["ct_dist"])]
                except IndexError as e:
                    logger_excel.debug("excel: create_metadata_skeleton: paleo tables messed up, {}".format(e))

                # Model template complete, insert it at list index
                l[idx1 - 1][pc_mod][idx2-1] = {"summaryTable": {}, "ensembleTable": {}, "distributionTable": dists}
        except IndexError as e:
            logger_excel.warn("create_skeleton_inner_tables: IndexError: {}".format(e))
        except KeyError as e:
            logger_excel.warn("create_skeleton_inner_tables: KeyError: {}".format(e))
    return l


Instruction:
Can you tell what is the following Python 3 function doing
def _create_skeleton_2(l, pc, num_section, template):
    logger_excel.info("enter create_skeleton_inner_1")
    try:
        # Create N number of paleoData/chronData tables.
        l = [copy.deepcopy(template)] * len(num_section)
        # Create the necessary tables inside of "model"
        l = _create_skeleton_3(pc, l, num_section)
    except Exception as e:
        logger_excel.warn("excel: create_skeleton_inner_main: error duplicating template tables, {}".format(e))

    return l

Response: Create the main and main tables for the main section of the main section.


Instruction:
How would you code a function in Python 3 to
create a skeleton for the main table.

Response:
def _create_skeleton_1(sheets):
    """
    Top level: {"chronData", "paleoData"}
    Fill in paleoData/chronData tables with N number of EMPTY measurement and models.

    :return list: Blank list of N indices
    """
    logger_excel.info("enter create_skeleton_main")

    # Table template: paleoData
    template_paleo = {"paleoMeasurementTable": [], "paleoModel": []}
    template_chron = {"chronMeasurementTable": [], "chronModel": []}

    num_chron = {}
    num_paleo = {}
    paleo = []
    chron = []

    # Get table counts for all table types.
    # Table types: paleoData, chronData, model, meas, dist, summary, ensemble
    for sheet in sheets:
        pc = sheet["paleo_chron"]

        # Tree for chron types
        if pc == "chron":
            num_chron = _get_table_counts(sheet, num_chron)
        elif pc == "paleo":
            num_paleo = _get_table_counts(sheet, num_paleo)

    # Create metadata skeleton for using out table counts.
    paleo = _create_skeleton_2(paleo, "paleo", num_paleo, template_paleo)
    chron = _create_skeleton_2(chron, "chron", num_chron, template_chron)

    logger_excel.info("exit create_skeleton_main")
    return paleo, chron


Instruction:
Implement a function in Python 3 to
parse an universal spreadsheet sheet.

Response:
def _parse_sheet(workbook, sheet):
    """
    The universal spreadsheet parser. Parse chron or paleo tables of type ensemble/model/summary.
    :param str name: Filename
    :param obj workbook: Excel Workbook
    :param dict sheet: Sheet path and naming info
    :return dict dict: Table metadata and numeric data
    """
    logger_excel.info("enter parse_sheet: {}".format(sheet["old_name"]))

    # Markers to track where we are on the sheet
    ensemble_on = False
    var_header_done = False
    metadata_on = False
    metadata_done = False
    data_on = False
    notes = False

    # Open the sheet from the workbook
    temp_sheet = workbook.sheet_by_name(sheet["old_name"])
    filename = sheet["filename"]

    # Store table metadata and numeric data separately
    table_name = "{}DataTableName".format(sheet["paleo_chron"])

    # Organize our root table data
    table_metadata = OrderedDict()
    table_metadata[table_name] = sheet["new_name"]
    table_metadata['filename'] = filename
    table_metadata['missingValue'] = 'nan'
    if "ensemble" in sheet["new_name"]:
        ensemble_on = True

    # Store all CSV in here by rows
    table_data = {filename: []}

    # Master list of all column metadata
    column_metadata = []

    # Index tracks which cells are being parsed
    num_col = 0
    num_row = 0
    nrows = temp_sheet.nrows
    col_total = 0

    # Tracks which "number" each metadata column is assigned
    col_add_ct = 1

    header_keys = []
    variable_keys = []
    variable_keys_lower = []
    mv = ""

    try:
        # Loop for every row in the sheet
        for i in range(0, nrows):
            # Hold the contents of the current cell
            cell = temp_sheet.cell_value(num_row, num_col)
            row = temp_sheet.row(num_row)

            # Skip all template lines
            if isinstance(cell, str):
                # Note and missing value entries are rogue. They are not close to the other data entries.
                if cell.lower().strip() not in EXCEL_TEMPLATE:

                    if "notes" in cell.lower() and not metadata_on:
                        # Store at the root table level
                        nt = temp_sheet.cell_value(num_row, 1)
                        if nt not in EXCEL_TEMPLATE:
                            table_metadata["notes"] = nt

                    elif cell.lower().strip() in ALTS_MV:
                        # Store at the root table level and in our function
                        mv = temp_sheet.cell_value(num_row, 1)
                        # Add if not placeholder value
                        if mv not in EXCEL_TEMPLATE:
                            table_metadata["missingValue"] = mv

                    # Variable template header row
                    elif cell.lower() in EXCEL_HEADER and not metadata_on and not data_on:

                        # Grab the header line
                        row = temp_sheet.row(num_row)
                        header_keys = _get_header_keys(row)

                        # Turn on the marker
                        var_header_done = True

                    # Data section (bottom of sheet)
                    elif data_on:

                        # Parse the row, clean, and add to table_data
                        table_data = _parse_sheet_data_row(temp_sheet, num_row, col_total, table_data, filename, mv)

                    # Metadata section. (top)
                    elif metadata_on:

                        # Reached an empty cell while parsing metadata. Mark the end of the section.
                        if cell in EMPTY:
                            metadata_on = False
                            metadata_done = True

                            # Create a list of all the variable names found
                            for entry in column_metadata:
                                try:
                                    # var keys is used as the variableName entry in each column's metadata
                                    variable_keys.append(entry["variableName"].strip())
                                    # var keys lower is used for comparing and finding the data header row
                                    variable_keys_lower.append(entry["variableName"].lower().strip())
                                except KeyError:
                                    # missing a variableName key
                                    pass

                        # Not at the end of the section yet. Parse the metadata
                        else:
                            # Get the row data
                            row = temp_sheet.row(num_row)

                            # Get column metadata
                            col_tmp = _compile_column_metadata(row, header_keys, col_add_ct)

                            # Append to master list
                            column_metadata.append(col_tmp)
                            col_add_ct += 1

                    # Variable metadata, if variable header exists
                    elif var_header_done and not metadata_done:

                        # Start piecing column metadata together with their respective variable keys
                        metadata_on = True

                        # Get the row data
                        row = temp_sheet.row(num_row)

                        # Get column metadata
                        col_tmp = _compile_column_metadata(row, header_keys, col_add_ct)

                        # Append to master list
                        column_metadata.append(col_tmp)
                        col_add_ct += 1

                    # Variable metadata, if variable header does not exist
                    elif not var_header_done and not metadata_done and cell:
                        # LiPD Version 1.1 and earlier: Chronology sheets don't have variable headers
                        # We could blindly parse, but without a header row_num we wouldn't know where
                        # to save the metadata
                        # Play it safe and assume data for first column only: variable name
                        metadata_on = True

                        # Get the row data
                        row = temp_sheet.row(num_row)

                        # Get column metadata
                        col_tmp = _compile_column_metadata(row, header_keys, col_add_ct)

                        # Append to master list
                        column_metadata.append(col_tmp)
                        col_add_ct += 1

                    # Data variable header row. Column metadata exists and metadata_done marker is on.
                    # This is where we compare top section variableNames to bottom section variableNames to see if
                    # we need to start parsing the column values
                    else:
                        try:
                            # Clean up variable_keys_lower so we all variable names change from "age(yrs BP)" to "age"
                            # Units in parenthesis make it too difficult to compare variables. Remove them.
                            row = _rm_units_from_var_names_multi(row)

                            if metadata_done and any(i in row for i in variable_keys_lower):
                                data_on = True

                                # Take the difference of the two lists. If anything exists, then that's a problem
                                __compare_vars(row, variable_keys_lower, sheet["old_name"])

                                # Ensemble columns are counted differently.
                                if ensemble_on:
                                    # Get the next row, and count the data cells.
                                    col_total = len(temp_sheet.row(num_row+1))
                                    # If there's an empty row, between, then try the next row.
                                    if col_total < 2:
                                        col_total = temp_sheet.row(num_row + 2)
                                    try:
                                        ens_cols = []
                                        [ens_cols.append(i+1) for i in range(0, col_total-1)]
                                        column_metadata[1]["number"] = ens_cols
                                    except IndexError:
                                        logger_excel.debug("excel: parse_sheet: unable to add ensemble 'number' key")
                                    except KeyError:
                                        logger_excel.debug("excel: parse_sheet: unable to add ensemble 'number' list at key")

                                # All other cass, columns are the length of column_metadata
                                else:
                                    col_total = len(column_metadata)

                        except AttributeError:
                            pass
                            # cell is not a string, and lower() was not a valid call.

            # If this is a numeric cell, 99% chance it's parsing the data columns.
            elif isinstance(cell, float) or isinstance(cell, int):
                if data_on or metadata_done:

                    # Parse the row, clean, and add to table_data
                    table_data = _parse_sheet_data_row(temp_sheet, num_row, col_total, table_data, filename, mv)

            # Move on to the next row
            num_row += 1
        table_metadata["columns"] = column_metadata
    except IndexError as e:
        logger_excel.debug("parse_sheet: IndexError: sheet: {}, row_num: {}, col_num: {}, {}".format(sheet, num_row, num_col, e))

    # If there isn't any data in this sheet, and nothing was parsed, don't let this
    # move forward to final output.
    if not table_data[filename]:
        table_data = None
        table_metadata = None

    logger_excel.info("exit parse_sheet")
    return table_metadata, table_data


Instruction:
How would you explain what the following Python 3 function does
def _parse_sheet_data_row(temp_sheet, num_row, col_total, table_data, filename, mv):
    # Get row of data
    row = temp_sheet.row(num_row)

    # In case our row holds more cells than the amount of columns we have, slice the row
    # We don't want to have extra empty cells in our output.
    row = row[:col_total]

    # Replace missing values where necessary
    row = _replace_mvs(row, mv)

    # Append row to list we will use to write out csv file later.
    table_data[filename].append(row)

    return table_data

Response: Parse a row from the data section of the sheet.


Instruction:
Can you tell what is the following Python 3 function doing
def _replace_mvs(row, mv):
    for idx, v in enumerate(row):
        try:
            if v.value.lower() in EMPTY or v.value.lower() == mv:
                row[idx] = "nan"
            else:
                row[idx] = v.value
        except AttributeError:
            if v.value == mv:
                row[idx] = "nan"
            else:
                row[idx] = v.value

    return row

Response: Replace Missing Values in the data rows where applicable


Instruction:
How would you code a function in Python 3 to
get the header keys from this special row

Response:
def _get_header_keys(row):
    """
    Get the variable header keys from this special row
    :return list: Header keys
    """
    # Swap out NOAA keys for LiPD keys
    for idx, key in enumerate(row):
        key_low = key.value.lower()

        # Simple case: Nothing fancy here, just map to the LiPD key counterpart.
        if key_low in EXCEL_LIPD_MAP_FLAT:
            row[idx] = EXCEL_LIPD_MAP_FLAT[key_low]

        # Nested data case: Check if this is a calibration, interpretation, or some other data that needs to be nested.
        # elif key_low:
        #     pass

        # Unknown key case: Store the key as-is because we don't have a LiPD mapping for it.
        else:
            try:
                row[idx] = key.value
            except AttributeError as e:
                logger_excel.warn("excel_main: get_header_keys: unknown header key, unable to add: {}".format(e))

    # Since we took a whole row of cells, we have to drop off the empty cells at the end of the row.
    header_keys = _rm_cells_reverse(row)
    return header_keys


Instruction:
Can you tell what is the following Python 3 function doing
def _rm_units_from_var_name_single(var):
    # Use the regex to match the cell
    m = re.match(re_var_w_units, var)
    # Should always get a match, but be careful anyways.
    if m:
        # m.group(1): variableName
        # m.group(2): units in parenthesis (may not exist).
        try:
            var = m.group(1).strip().lower()
            # var = m.group(1).strip().lower()
        except Exception:
            # This must be a malformed cell somehow. This regex should match every variableName cell.
            # It didn't work out. Return the original var as a fallback
            pass
    return var

Response: Removes units from a single variable name in a single section of the metadata and data structures.


Instruction:
Can you generate the documentation for the following Python 3 function
def _rm_units_from_var_names_multi(row):
    l2 = []
    # Check each var in the row
    for idx, var in enumerate(row):
        l2.append(_rm_units_from_var_name_single(row[idx].value))
    return l2

Response: Wrapper around _rm_units_from_var_name_single for doing a list instead of a single cell.


Instruction:
Implement a Python 3 function for
compiling the interpretation data into a list of multiples based on the keys provided.

Response:
def _compile_interpretation(data):
    """
    Compile the interpretation data into a list of multiples, based on the keys provided.
    Disassemble the key to figure out how to place the data
    :param dict data: Interpretation data (unsorted)
    :return dict: Interpretation data (sorted)
    """
    # KEY FORMAT : "interpretation1_somekey"
    _count = 0

    # Determine how many entries we are going to need, by checking the interpretation index in the string
    for _key in data.keys():
        _key_low = _key.lower()
        # Get regex match
        m = re.match(re_interpretation, _key_low)
        # If regex match was successful..
        if m:
            # Check if this interpretation count is higher than what we have.
            _curr_count = int(m.group(1))
            if _curr_count > _count:
                # New max count, record it.
                _count = _curr_count

    # Create the empty list with X entries for the interpretation data
    _tmp = [{} for i in range(0, _count)]

    # Loop over all the interpretation keys and data
    for k, v in data.items():
        # Get the resulting regex data.
        # EXAMPLE ENTRY: "interpretation1_variable"
        # REGEX RESULT: ["1", "variable"]
        m = re.match(re_interpretation, k)
        # Get the interpretation index number
        idx = int(m.group(1))
        # Get the field variable
        key = m.group(2)
        # Place this data in the _tmp array. Remember to adjust given index number for 0-indexing
        _tmp[idx-1][key] = v

    # Return compiled interpretation data
    return _tmp