Instruction:
Given the following Python 3 function, write the documentation
def _compile_column_metadata(row, keys, number):
    # Store the variable keys by index in a dictionary
    _column = {}
    _interpretation = {}
    _calibration = {}
    _physical = {}

    # Use the header keys to place the column data in the dictionary
    if keys:
        for idx, key in enumerate(keys):
            _key_low = key.lower()

            # Special case: Calibration data
            if re.match(re_calibration, _key_low):
                m = re.match(re_calibration, _key_low)
                if m:
                    _key = m.group(1)
                    _calibration[_key] = row[idx].value

            # Special case: PhysicalSample data
            elif re.match(re_physical, _key_low):
                m = re.match(re_physical, _key_low)
                if m:
                    _key = m.group(1)
                    _physical[_key] = row[idx].value

            # Special case: Interpretation data
            elif re.match(re_interpretation, _key_low):
                # Put interpretation data in a tmp dictionary that we'll sort later.
                _interpretation[_key_low] = row[idx].value

            else:
                try:
                    val = row[idx].value
                except Exception:
                    logger_excel.info("compile_column_metadata: Couldn't get value from row cell")
                    val = "n/a"
                try:
                    if key == "variableName":
                        val = _rm_units_from_var_name_single(row[idx].value)
                except Exception:
                    # when a variableName fails to split, keep the name as-is and move on.
                    pass
                _column[key] = val
        _column["number"] = number

        if _calibration:
            _column["calibration"] = _calibration
        # Only allow physicalSample on measured variableTypes. duh.
        if _physical and _column["variableType"] == "measured":
            _column["physicalSample"] = _physical
        if _interpretation:
            _interpretation_data = _compile_interpretation(_interpretation)
            _column["interpretation"] = _interpretation_data

    # If there are not keys, that means it's a header-less metadata section.
    else:
        # Assume we only have one cell, because we have no keys to know what data is here.
        try:
            val = row[0].value.lower()
        except AttributeError:
            val = row[0].value
        except Exception:
            logger_excel.info("compile_column_metadata: Couldn't get value from row cell")
            val = "n/a"
        val = _rm_units_from_var_name_single(val)
        _column["variableName"] = val
        _column["number"] = number

    # Add this column to the overall metadata, but skip if there's no data present
    _column = {k: v for k, v in _column.items() if v}

    return _column

Response: Compile column metadata from one excel row


Instruction:
Can you create a Python 3 function that
removes the cells that are empty or template in reverse order. Stop when you hit data.

Response:
def _rm_cells_reverse(l):
    """
    Remove the cells that are empty or template in reverse order. Stop when you hit data.
    :param list l: One row from the spreadsheet
    :return list: Modified row
    """
    rm = []
    # Iter the list in reverse, and get rid of empty and template cells
    for idx, key in reversed(list(enumerate(l))):
        if key.lower() in EXCEL_TEMPLATE:
            rm.append(idx)
        elif key in EMPTY:
            rm.append(idx)
        else:
            break

    for idx in rm:
        l.pop(idx)
    return l


Instruction:
Can you create a Python 3 function that
writes CSV data to file by file.

Response:
def _write_data_csv(csv_data):
    """
    CSV data has been parsed by this point, so take it and write it file by file.
    :return:
    """
    logger_excel.info("enter write_data_csv")
    # Loop for each file and data that is stored
    for file in csv_data:
        for filename, data in file.items():
            # Make sure we're working with the right data types before trying to open and write a file
            if isinstance(filename, str) and isinstance(data, list):
                try:
                    with open(filename, 'w+') as f:
                        w = csv.writer(f)
                        for line in data:
                            w.writerow(line)
                except Exception:
                    logger_excel.debug("write_data_csv: Unable to open/write file: {}".format(filename))

    logger_excel.info("exit write_data_csv")
    return


Instruction:
Write a Python 3 script to
create a GeoJSON Linestring. Latitude and Longitude have 2 values each.

Response:
def geometry_linestring(lat, lon, elev):
    """
    GeoJSON Linestring. Latitude and Longitude have 2 values each.
    :param list lat: Latitude values
    :param list lon:  Longitude values
    :return dict:
    """
    logger_excel.info("enter geometry_linestring")
    d = OrderedDict()
    coordinates = []
    temp = ["", ""]

    # Point type, Matching pairs.
    if lat[0] == lat[1] and lon[0] == lon[1]:
        logger_excel.info("matching geo coordinate")
        lat.pop()
        lon.pop()
        d = geometry_point(lat, lon, elev)

    else:
        # Creates coordinates list
        logger_excel.info("unique geo coordinates")
        for i in lon:
            temp[0] = i
            for j in lat:
                temp[1] = j
                coordinates.append(copy.copy(temp))
        if elev:
            for i in coordinates:
                i.append(elev)
        # Create geometry block
        d['type'] = 'Linestring'
        d['coordinates'] = coordinates
    logger_excel.info("exit geometry_linestring")
    return d


Instruction:
How would you explain what the following Python 3 function does
def geometry_range(crd_range, elev, crd_type):

    d = OrderedDict()
    coordinates = [[] for i in range(len(crd_range))]

    # latitude
    if crd_type == "lat":
        for idx, i in enumerate(crd_range):
            coordinates[idx] = [crd_range[idx], "nan"]
            if elev:
                coordinates[idx].append(elev)

    # longitude
    elif crd_type == "lon":
        for idx, i in enumerate(crd_range):
            coordinates[idx] = ["nan", crd_range[idx]]
            if elev:
                coordinates[idx].append(elev)

    d["type"] = "Range"
    d["coordinates"] = coordinates

    return d

Response: Returns a dictionary of coordinates for a single object.


Instruction:
Make a summary of the following Python 3 code
def geometry_point(lat, lon, elev):
    logger_excel.info("enter geometry_point")
    coordinates = []
    point_dict = OrderedDict()
    for idx, val in enumerate(lat):
        try:
            coordinates.append(lon[idx])
            coordinates.append(lat[idx])
        except IndexError as e:
            print("Error: Invalid geo coordinates")
            logger_excel.debug("geometry_point: IndexError: lat: {}, lon: {}, {}".format(lat, lon, e))

    coordinates.append(elev)
    point_dict['type'] = 'Point'
    point_dict['coordinates'] = coordinates
    logger_excel.info("exit geometry_point")
    return point_dict

Response: Returns a GeoJSON point in the format of a series of coordinates.


Instruction:
Implement a Python 3 function for
returning a dictionary of geometry values for the given latitude and longitude.

Response:
def compile_geometry(lat, lon, elev):
    """
    Take in lists of lat and lon coordinates, and determine what geometry to create
    :param list lat: Latitude values
    :param list lon: Longitude values
    :param float elev: Elevation value
    :return dict:
    """
    logger_excel.info("enter compile_geometry")
    lat = _remove_geo_placeholders(lat)
    lon = _remove_geo_placeholders(lon)

    # 4 coordinate values
    if len(lat) == 2 and len(lon) == 2:
        logger_excel.info("found 4 coordinates")
        geo_dict = geometry_linestring(lat, lon, elev)

        # # 4 coordinate values
        # if (lat[0] != lat[1]) and (lon[0] != lon[1]):
        #     geo_dict = geometry_polygon(lat, lon)
        # # 3 unique coordinates
        # else:
        #     geo_dict = geometry_multipoint(lat, lon)
        #

    # 2 coordinate values
    elif len(lat) == 1 and len(lon) == 1:
        logger_excel.info("found 2 coordinates")
        geo_dict = geometry_point(lat, lon, elev)

    # coordinate range. one value given but not the other.
    elif (None in lon and None not in lat) or (len(lat) > 0 and len(lon) == 0):
        geo_dict = geometry_range(lat, elev, "lat")

    elif (None in lat and None not in lon) or (len(lon) > 0 and len(lat) == 0):
        geo_dict = geometry_range(lat, elev, "lon")

    # Too many points, or no points
    else:
        geo_dict = {}
        logger_excel.warn("compile_geometry: invalid coordinates: lat: {}, lon: {}".format(lat, lon))
    logger_excel.info("exit compile_geometry")
    return geo_dict


Instruction:
Can you implement a function in Python 3 that
compiles top - level Geography dictionary.

Response:
def compile_geo(d):
    """
    Compile top-level Geography dictionary.
    :param d:
    :return:
    """
    logger_excel.info("enter compile_geo")
    d2 = OrderedDict()

    # get max number of sites, or number of coordinate points given.
    num_loc = _get_num_locations(d)

    # if there's one more than one location put it in a collection
    if num_loc > 1:
        d2["type"] = "FeatureCollection"
        features = []
        for idx in range(0, num_loc):
            # Do process for one site
            site = _parse_geo_locations(d, idx)
            features.append(site)
        d2["features"] = features

    # if there's only one location
    elif num_loc == 1:
        d2 = _parse_geo_location(d)

    logger_excel.info("exit compile_geo")
    return d2


Instruction:
Write a Python 3 function that can
find out how many locations are being parsed. Compare lengths of each ArcGIS metadata and return the maximal locations.

Response:
def _get_num_locations(d):
    """
    Find out how many locations are being parsed. Compare lengths of each
    coordinate list and return the max
    :param dict d: Geo metadata
    :return int: Max number of locations
    """
    lengths = []
    for key in EXCEL_GEO:
        try:
            if key != "siteName":
                lengths.append(len(d[key]))
        except Exception:
            lengths.append(1)

    try:
        num = max(lengths)
    except ValueError:
        num = 0
    return num


Instruction:
Write a Python 3 script to
parse one geo location and return a dict of properties

Response:
def _parse_geo_location(d):
    """
    Parse one geo location
    :param d:
    :return:
    """
    d2 = OrderedDict()
    filt = {}
    d2['type'] = 'Feature'
    # If the necessary keys are missing, put in placeholders so there's no KeyErrors.
    for key in EXCEL_GEO:
        if key not in d:
            d[key] = ""

    # Compile the geometry based on the info available.
    d2['geometry'] = compile_geometry([d['latMin'], d['latMax']], [d['lonMin'], d['lonMax']], d['elevation'])
    d2['properties'] = {'siteName': d['siteName']}

    return d2


Instruction:
Create a Python 3 function to
parse one geo location in a dictionary

Response:
def _parse_geo_locations(d, idx):
    """
    Parse one geo location
    :param d:
    :return:
    """
    d2 = OrderedDict()
    filt = {}
    d2['type'] = 'Feature'
    # If the necessary keys are missing, put in placeholders so there's no KeyErrors.
    for key in EXCEL_GEO:
        if key not in d:
            d[key] = ""

    for key in EXCEL_GEO:
        try:
            if key == "siteName" and isinstance(d["siteName"], str):
                filt["siteName"] = d["siteName"]
            else:
                filt[key] = d[key][idx]
        except KeyError:
            filt[key] = None
        except TypeError:
            filt[key] = None

    # Compile the geometry based on the info available.
    d2['geometry'] = compile_geometry([filt['latMin'], filt['latMax']], [filt['lonMin'], filt['lonMax']], filt['elevation'])
    d2['properties'] = {'siteName': filt['siteName']}

    return d2


Instruction:
How would you implement a function in Python 3 that
splits the string of author names into the BibJSON format.

Response:
def compile_authors(cell):
    """
    Split the string of author names into the BibJSON format.
    :param str cell: Data from author cell
    :return: (list of dicts) Author names
    """
    logger_excel.info("enter compile_authors")
    author_lst = []
    s = cell.split(';')
    for w in s:
        author_lst.append(w.lstrip())
    logger_excel.info("exit compile_authors")
    return author_lst


Instruction:
Given the following Python 3 function, write the documentation
def compile_temp(d, key, value):
    if not value:
        d[key] = None
    elif len(value) == 1:
        d[key] = value[0]
    else:
        d[key] = value
    return d

Response: Compiles temporary dictionaries for metadata. Adds a new entry to an existing dictionary.


Instruction:
Can you create a Python 3 function that
compiles the funding block.

Response:
def compile_fund(workbook, sheet, row, col):
    """
    Compile funding entries. Iter both rows at the same time. Keep adding entries until both cells are empty.
    :param obj workbook:
    :param str sheet:
    :param int row:
    :param int col:
    :return list of dict: l
    """
    logger_excel.info("enter compile_fund")
    l = []
    temp_sheet = workbook.sheet_by_name(sheet)
    while col < temp_sheet.ncols:
        col += 1
        try:
            # Make a dictionary for this funding entry.
            _curr = {
                'agency': temp_sheet.cell_value(row, col),
                'grant': temp_sheet.cell_value(row+1, col),
                "principalInvestigator": temp_sheet.cell_value(row+2, col),
                "country": temp_sheet.cell_value(row + 3, col)
            }
            # Make a list for all
            _exist = [temp_sheet.cell_value(row, col), temp_sheet.cell_value(row+1, col),
                       temp_sheet.cell_value(row+2, col), temp_sheet.cell_value(row+3, col)]

            # Remove all empty items from the list
            _exist = [i for i in _exist if i]
            # If we have all empty entries, then don't continue. Quit funding and return what we have.
            if not _exist:
                return l

            # We have funding data. Add this funding block to the growing list.
            l.append(_curr)

        except IndexError as e:
            logger_excel.debug("compile_fund: IndexError: sheet:{} row:{} col:{}, {}".format(sheet, row, col, e))
    logger_excel.info("exit compile_fund")
    return l


Instruction:
How would you explain what the following Python 3 function does
def __compare_vars(a1, a2, name):
    try:
        a1 = [i for i in a1 if i]
        a2 = [i for i in a2 if i]
        a3 = set(a1).symmetric_difference(set(a2))
        if a3:
            print("- Error: Variables are not entered correctly in sheet: {}\n\tUnmatched variables: {}".format(name, a3))
    except Exception as e:
        logger_excel.error("compare_vars: {}".format(e))
    return

Response: Check that the metadata variable names are the same as the data header variable names in the sheet name.


Instruction:
Implement a Python 3 function for
getting the filename based on the dataset name in the metadata

Response:
def __get_datasetname(d, filename):
    """
    Get the filename based on the dataset name in the metadata
    :param str filename: Filename.lpd
    :return str: Filename
    """
    try:
        filename = d["dataSetName"]
    except KeyError:
        logger_excel.info("get_datasetname: KeyError: No dataSetName found. Reverting to: {}".format(filename))
    return filename


Instruction:
How would you code a function in Python 3 to
set the filenames in the sheets metadata

Response:
def __set_sheet_filenames(sheets, n):
    """
    Use the dataset name to build the filenames in the sheets metadata
    :param list sheets: Sheet metadata
    :param str n: Dataset Name
    :return list: Sheet metadata
    """
    try:
        for idx, sheet in enumerate(sheets):
            try:
                sheets[idx]["filename"] = "{}.{}".format(n, sheet["filename"])
            except Exception as e:
                logger_excel.error("set_sheet_filenames: inner: {}".format(e), exc_info=True)
    except Exception as q:
        logger_excel.error("set_sheet_filenames: outer: {}".format(q), exc_info=True)
    return sheets


Instruction:
How would you explain what the following Python 3 function does
def name_to_jsonld(title_in):
    title_out = ''
    try:
        title_in = title_in.lower()
        title_out = EXCEL_LIPD_MAP_FLAT[title_in]
    except (KeyError, AttributeError) as e:
        if "(" in title_in:
            title_in = title_in.split("(")[0].strip()
        # try to find an exact match first.
        try:
            v = EXCEL_LIPD_MAP_FLAT[title_in]
            return v
        except KeyError:
            pass

        # if no exact match, find whatever is a closest match
        for k, v in EXCEL_LIPD_MAP_FLAT.items():
            if k in title_in:
                return v
    if not title_out:
        logger_excel.debug("name_to_jsonld: No match found: {}".format(title_in))
    return title_out

Response: Convert formal titles to camelcase json_ld text that matches our context file


Instruction:
Implement a Python 3 function for
matching data type and return string

Response:
def instance_str(cell):
    """
    Match data type and return string
    :param any cell:
    :return str:
    """
    if isinstance(cell, str):
        return 'str'
    elif isinstance(cell, int):
        return 'int'
    elif isinstance(cell, float):
        return 'float'
    else:
        return 'unknown'


Instruction:
How would you code a function in Python 3 to
extract units from parenthesis in a string. i. e. elevation meters

Response:
def extract_units(string_in):
    """
    Extract units from parenthesis in a string. i.e. "elevation (meters)"
    :param str string_in:
    :return str:
    """
    start = '('
    stop = ')'
    return string_in[string_in.index(start) + 1:string_in.index(stop)]


Instruction:
Create a Python 3 function to
return a list of cells in a specific row with a specific publication section.

Response:
def cells_rt_meta_pub(workbook, sheet, row, col, pub_qty):
    """
    Publication section is special. It's possible there's more than one publication.
    :param obj workbook:
    :param str sheet:
    :param int row:
    :param int col:
    :param int pub_qty: Number of distinct publication sections in this file
    :return list: Cell data for a specific row
    """
    logger_excel.info("enter cells_rt_meta_pub")
    col_loop = 0
    cell_data = []
    temp_sheet = workbook.sheet_by_name(sheet)
    while col_loop < pub_qty:
        col += 1
        col_loop += 1
        cell_data.append(temp_sheet.cell_value(row, col))
    logger_excel.info("exit cells_rt_meta_pub")
    return cell_data


Instruction:
Implement a Python 3 function for
traversing all cells in a row and return a list of all the data in that row.

Response:
def cells_rt_meta(workbook, sheet, row, col):
    """
    Traverse all cells in a row. If you find new data in a cell, add it to the list.
    :param obj workbook:
    :param str sheet:
    :param int row:
    :param int col:
    :return list: Cell data for a specific row
    """
    logger_excel.info("enter cells_rt_meta")
    col_loop = 0
    cell_data = []
    temp_sheet = workbook.sheet_by_name(sheet)
    while col_loop < temp_sheet.ncols:
        col += 1
        col_loop += 1
        try:
            if temp_sheet.cell_value(row, col) != xlrd.empty_cell and temp_sheet.cell_value(row, col) != '':
                cell_data.append(temp_sheet.cell_value(row, col))
        except IndexError as e:
            logger_excel.warn("cells_rt_meta: IndexError: sheet: {}, row: {}, col: {}, {}".format(sheet, row, col, e))
    logger_excel.info("exit cells_right_meta")
    return cell_data


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cells_dn_meta(workbook, sheet, row, col, final_dict):
    logger_excel.info("enter cells_dn_meta")
    row_loop = 0
    pub_cases = ['id', 'year', 'author', 'journal', 'issue', 'volume', 'title', 'pages',
                 'reportNumber', 'abstract', 'alternateCitation']
    geo_cases = ['latMin', 'lonMin', 'lonMax', 'latMax', 'elevation', 'siteName', 'location']
    funding_cases = ["agency", "grant", "principalInvestigator", "country"]

    # Temp
    pub_qty = 0
    geo_temp = {}
    general_temp = {}
    pub_temp = []
    funding_temp = []

    temp_sheet = workbook.sheet_by_name(sheet)

    # Loop until we hit the max rows in the sheet
    while row_loop < temp_sheet.nrows:
        try:
            # Get cell value
            cell = temp_sheet.cell_value(row, col)

            # If there is content in the cell...
            if cell not in EMPTY:

                # Convert title to correct format, and grab the cell data for that row
                title_formal = temp_sheet.cell_value(row, col)
                title_json = name_to_jsonld(title_formal)

                # If we don't have a title for it, then it's not information we want to grab
                if title_json:

                    # Geo
                    if title_json in geo_cases:
                        cell_data = cells_rt_meta(workbook, sheet, row, col)
                        geo_temp = compile_temp(geo_temp, title_json, cell_data)

                    # Pub
                    # Create a list of dicts. One for each pub column.
                    elif title_json in pub_cases:

                        # Authors seem to be the only consistent field we can rely on to determine number of Pubs.
                        if title_json == 'author':
                            cell_data = cells_rt_meta(workbook, sheet, row, col)
                            pub_qty = len(cell_data)
                            for i in range(pub_qty):
                                author_lst = compile_authors(cell_data[i])
                                pub_temp.append({'author': author_lst, 'pubDataUrl': 'Manually Entered'})
                        else:
                            cell_data = cells_rt_meta_pub(workbook, sheet, row, col, pub_qty)
                            for pub in range(pub_qty):
                                if title_json == 'id':
                                    pub_temp[pub]['identifier'] = [{"type": "doi", "id": cell_data[pub]}]
                                else:
                                    pub_temp[pub][title_json] = cell_data[pub]
                    # Funding
                    elif title_json in funding_cases:
                        if title_json == "agency":
                            funding_temp = compile_fund(workbook, sheet, row, col)

                    # All other cases do not need fancy structuring
                    else:
                        cell_data = cells_rt_meta(workbook, sheet, row, col)
                        general_temp = compile_temp(general_temp, title_json, cell_data)

        except IndexError as e:
            logger_excel.debug("cells_dn_datasheets: IndexError: sheet: {}, row: {}, col: {}, {}".format(sheet, row, col, e))
        row += 1
        row_loop += 1

    # Compile the more complicated items
    geo = compile_geo(geo_temp)

    logger_excel.info("compile metadata dictionary")
    # Insert into final dictionary
    final_dict['@context'] = "context.jsonld"
    final_dict['pub'] = pub_temp
    final_dict['funding'] = funding_temp
    final_dict['geo'] = geo

    # Add remaining general items
    for k, v in general_temp.items():
        final_dict[k] = v
    logger_excel.info("exit cells_dn_meta")
    return final_dict

Response: Traverse all cells in a column moving downward and create a new one.


Instruction:
Can you tell what is the following Python 3 function doing
def count_chron_variables(temp_sheet):
    total_count = 0
    start_row = traverse_to_chron_var(temp_sheet)
    while temp_sheet.cell_value(start_row, 0) != '':
        total_count += 1
        start_row += 1
    return total_count

Response: Count the number of chron variables in a resource tree.


Instruction:
Write a Python 3 function that can
get all the vars in the chron sheet

Response:
def get_chron_var(temp_sheet, start_row):
    """
    Capture all the vars in the chron sheet (for json-ld output)
    :param obj temp_sheet:
    :param int start_row:
    :return: (list of dict) column data
    """
    col_dict = OrderedDict()
    out_list = []
    column = 1

    while (temp_sheet.cell_value(start_row, 0) != '') and (start_row < temp_sheet.nrows):
        short_cell = temp_sheet.cell_value(start_row, 0)
        units_cell = temp_sheet.cell_value(start_row, 1)
        long_cell = temp_sheet.cell_value(start_row, 2)

        # Fill the dictionary for this column
        col_dict['number'] = column
        col_dict['variableName'] = short_cell
        col_dict['description'] = long_cell
        col_dict['units'] = units_cell
        out_list.append(col_dict.copy())
        start_row += 1
        column += 1

    return out_list


Instruction:
Write a Python 3 function for
traversing to the first row that has chron data

Response:
def traverse_to_chron_data(temp_sheet):
    """
    Traverse down to the first row that has chron data
    :param obj temp_sheet:
    :return int: traverse_row
    """
    traverse_row = traverse_to_chron_var(temp_sheet)
    reference_var = temp_sheet.cell_value(traverse_row, 0)

    # Traverse past all the short_names, until you hit a blank cell (the barrier)
    while temp_sheet.cell_value(traverse_row, 0) != '':
        traverse_row += 1
    # Traverse past the empty cells until we hit the chron data area
    while temp_sheet.cell_value(traverse_row, 0) == '':
        traverse_row += 1

    # Check if there is a header row. If there is, move past it. We don't want that data
    if temp_sheet.cell_value(traverse_row, 0) == reference_var:
        traverse_row += 1
    logger_excel.info("traverse_to_chron_data: row:{}".format(traverse_row))
    return traverse_row


Instruction:
Create a Python 3 function to
traverse down to the first variable .

Response:
def traverse_to_chron_var(temp_sheet):
    """
    Traverse down to the row that has the first variable
    :param obj temp_sheet:
    :return int:
    """
    row = 0
    while row < temp_sheet.nrows - 1:
        if 'Parameter' in temp_sheet.cell_value(row, 0):
            row += 1
            break
        row += 1
    logger_excel.info("traverse_to_chron_var: row:{}".format(row))
    return row


Instruction:
Create a Python 3 function for
getting all data in for a specific chron data row

Response:
def get_chron_data(temp_sheet, row, total_vars):
    """
    Capture all data in for a specific chron data row (for csv output)
    :param obj temp_sheet:
    :param int row:
    :param int total_vars:
    :return list: data_row
    """
    data_row = []
    missing_val_list = ['none', 'na', '', '-']
    for i in range(0, total_vars):
        cell = temp_sheet.cell_value(row, i)
        if isinstance(cell, str):
            cell = cell.lower()
        if cell in missing_val_list:
            cell = 'nan'
        data_row.append(cell)
    return data_row


Instruction:
Here you have a function in Python 3, explain what it does
def _remove_geo_placeholders(l):
    vals = []
    for i in l:
        if isinstance(i, list):
            for k in i:
                if isinstance(k, float) or isinstance(k, int):
                    vals.append(k)
        elif isinstance(i, float) or isinstance(i, int):
            vals.append(i)
    vals.sort()
    return vals

Response: Remove placeholders from coordinate lists and sort


Instruction:
Can you write a function in Python 3 where it
reads the jsonld file in the LiPD archive and returns it in the Jsonld format.

Response:
def read_jsonld():
    """
    Find jsonld file in the cwd (or within a 2 levels below cwd), and load it in.
    :return dict: Jsonld data
    """
    _d = {}

    try:
        # Find a jsonld file in cwd. If none, fallback for a json file. If neither found, return empty.
        _filename = [file for file in os.listdir() if file.endswith(".jsonld")][0]
        if not _filename:
            _filename = [file for file in os.listdir() if file.endswith(".json")][0]

        if _filename:
            try:
                # Load and decode
                _d = demjson.decode_file(_filename, decode_float=float)
                logger_jsons.info("Read JSONLD successful: {}".format(_filename))
            except FileNotFoundError as fnf:
                print("Error: metadata file not found: {}".format(_filename))
                logger_jsons.error("read_jsonld: FileNotFound: {}, {}".format(_filename, fnf))
            except Exception:
                try:
                    _d = demjson.decode_file(_filename, decode_float=float, encoding="latin-1")
                    logger_jsons.info("Read JSONLD successful: {}".format(_filename))
                except Exception as e:
                    print("Error: unable to read metadata file: {}".format(e))
                    logger_jsons.error("read_jsonld: Exception: {}, {}".format(_filename, e))
        else:
            print("Error: metadata file (.jsonld) not found in LiPD archive")
    except Exception as e:
        print("Error: Unable to find jsonld file in LiPD archive. This may be a corrupt file.")
        logger_jsons.error("Error: Unable to find jsonld file in LiPD archive. This may be a corrupt file.")
    logger_jsons.info("exit read_json_from_file")
    return _d


Instruction:
Explain what the following Python 3 code does
def read_json_from_file(filename):
    logger_jsons.info("enter read_json_from_file")
    d = OrderedDict()
    try:
        # Load and decode
        d = demjson.decode_file(filename, decode_float=float)
        logger_jsons.info("successful read from json file")
    except FileNotFoundError:
        # Didn't find a jsonld file. Maybe it's a json file instead?
        try:
            d = demjson.decode_file(os.path.splitext(filename)[0] + '.json', decode_float=float)
        except FileNotFoundError as e:
            # No json or jsonld file. Exit
            print("Error: jsonld file not found: {}".format(filename))
            logger_jsons.debug("read_json_from_file: FileNotFound: {}, {}".format(filename, e))
        except Exception:
            print("Error: unable to read jsonld file")

    if d:
        d = rm_empty_fields(d)
    logger_jsons.info("exit read_json_from_file")
    return d

Response: Read the JSON data from a file.


Instruction:
Given the following Python 3 function, write the documentation
def idx_num_to_name(L):
    logger_jsons.info("enter idx_num_to_name")

    try:
        if "paleoData" in L:
            L["paleoData"] = _import_data(L["paleoData"], "paleo")
        if "chronData" in L:
            L["chronData"] = _import_data(L["chronData"], "chron")
    except Exception as e:
        logger_jsons.error("idx_num_to_name: {}".format(e))
        print("Error: idx_name_to_num: {}".format(e))

    logger_jsons.info("exit idx_num_to_name")
    return L

Response: Switch from index - by - number to index - by - name.


Instruction:
Write a Python 3 script for
importing the metadata and change it to index - by - name.

Response:
def _import_data(sections, crumbs):
    """
    Import the section metadata and change it to index-by-name.

    :param list sections: Metadata
    :param str pc: paleo or chron
    :return dict _sections: Metadata
    """
    logger_jsons.info("enter import_data: {}".format(crumbs))
    _sections = OrderedDict()
    try:
        for _idx, section in enumerate(sections):
            _tmp = OrderedDict()

            # Process the paleo measurement table
            if "measurementTable" in section:
                _tmp["measurementTable"] = _idx_table_by_name(section["measurementTable"], "{}{}{}".format(crumbs, _idx, "measurement"))

            # Process the paleo model
            if "model" in section:
                _tmp["model"] = _import_model(section["model"], "{}{}{}".format(crumbs, _idx, "model"))

            # Get the table name from the first measurement table, and use that as the index name for this table
            _table_name = "{}{}".format(crumbs, _idx)

            # If we only have generic table names, and one exists already, don't overwrite. Create dynamic name
            if _table_name in _sections:
                _table_name = "{}_{}".format(_table_name, _idx)

            # Put the final product into the output dictionary. Indexed by name
            _sections[_table_name] = _tmp

    except Exception as e:
        logger_jsons.error("import_data: Exception: {}".format(e))
        print("Error: import_data: {}".format(e))

    logger_jsons.info("exit import_data: {}".format(crumbs))
    return _sections


Instruction:
How would you code a function in Python 3 to
import nested paleoModel data into a single metadata dictionary.

Response:
def _import_model(models, crumbs):
    """
    Change the nested items of the paleoModel data. Overwrite the data in-place.

    :param list models: Metadata
    :param str crumbs: Crumbs
    :return dict _models: Metadata
    """
    logger_jsons.info("enter import_model".format(crumbs))
    _models = OrderedDict()
    try:
        for _idx, model in enumerate(models):
            # Keep the original dictionary, but replace the three main entries below

            # Do a direct replacement of chronModelTable columns. No table name, no table work needed.
            if "summaryTable" in model:
                model["summaryTable"] = _idx_table_by_name(model["summaryTable"], "{}{}{}".format(crumbs, _idx, "summary"))
            # Do a direct replacement of ensembleTable columns. No table name, no table work needed.
            if "ensembleTable" in model:
                model["ensembleTable"] = _idx_table_by_name(model["ensembleTable"], "{}{}{}".format(crumbs, _idx, "ensemble"))
            if "distributionTable" in model:
                model["distributionTable"] = _idx_table_by_name(model["distributionTable"], "{}{}{}".format(crumbs, _idx, "distribution"))

            _table_name = "{}{}".format(crumbs, _idx)
            _models[_table_name] = model
    except Exception as e:
        logger_jsons.error("import_model: {}".format(e))
        print("Error: import_model: {}".format(e))
    logger_jsons.info("exit import_model: {}".format(crumbs))
    return _models


Instruction:
How would you explain what the following Python 3 function does
def _idx_table_by_name(tables, crumbs):
    _tables = OrderedDict()
    try:
        for _idx, _table in enumerate(tables):
            # Use "name" as tableName
            _name = "{}{}".format(crumbs, _idx)
            # Call idx_table_by_name
            _tmp = _idx_col_by_name(_table)
            if _name in _tables:
                _name = "{}_{}".format(_name, _idx)
            _tmp["tableName"] = _name
            _tables[_name] = _tmp
    except Exception as e:
        logger_jsons.error("idx_table_by_name: {}".format(e))
        print("Error: idx_table_by_name: {}".format(e))

    return _tables

Response: Import summary ensemble or distribution data.


Instruction:
How would you explain what the following Python 3 function does
def get_csv_from_json(d):
    logger_jsons.info("enter get_csv_from_json")
    csv_data = OrderedDict()

    if "paleoData" in d:
        csv_data = _get_csv_from_section(d, "paleoData", csv_data)

    if "chronData" in d:
        csv_data = _get_csv_from_section(d, "chronData", csv_data)

    logger_jsons.info("exit get_csv_from_json")
    return csv_data

Response: Get CSV values when mixed into json data.


Instruction:
Write a Python 3 function for
getting csv from paleo and chron sections

Response:
def _get_csv_from_section(d, pc, csv_data):
    """
    Get csv from paleo and chron sections
    :param dict d: Metadata
    :param str pc: Paleo or chron
    :return dict: running csv data
    """
    logger_jsons.info("enter get_csv_from_section: {}".format(pc))

    for table, table_content in d[pc].items():
        # Create entry for this table/CSV file (i.e. Asia-1.measTable.PaleoData.csv)
        # Note: Each table has a respective CSV file.
        csv_data[table_content['filename']] = OrderedDict()
        for column, column_content in table_content['columns'].items():
            # Set the "values" into csv dictionary in order of column "number"
            csv_data[table_content['filename']][column_content['number']] = column_content['values']

    logger_jsons.info("exit get_csv_from_section: {}".format(pc))
    return csv_data


Instruction:
Can you generate the documentation for the following Python 3 function
def remove_csv_from_json(d):
    logger_jsons.info("enter remove_csv_from_json")

    # Check both sections
    if "paleoData" in d:
        d = _remove_csv_from_section(d, "paleoData")

    if "chronData" in d:
        d = _remove_csv_from_section(d, "chronData")

    logger_jsons.info("exit remove_csv_from_json")
    return d

Response: Remove all CSV data entries from paleoData table in the JSON structure.


Instruction:
Can you implement a function in Python 3 that
removes CSV from metadata in this section

Response:
def _remove_csv_from_section(d, pc):
    """
    Remove CSV from metadata in this section
    :param dict d: Metadata
    :param str pc: Paleo or chron
    :return dict: Modified metadata
    """
    logger_jsons.info("enter remove_csv_from_json: {}".format(pc))

    for table, table_content in d[pc].items():
        for column, column_content in table_content['columns'].items():
            try:
                # try to delete the values key entry
                del column_content['values']
            except KeyError as e:
                # if the key doesn't exist, keep going
                logger_jsons.debug("remove_csv_from_json: KeyError: {}, {}".format(pc, e))

    logger_jsons.info("exit remove_csv_from_json: {}".format(pc))
    return d


Instruction:
Can you generate a brief explanation for the following Python 3 code
def write_json_to_file(json_data, filename="metadata"):
    logger_jsons.info("enter write_json_to_file")
    json_data = rm_empty_fields(json_data)
    # Use demjson to maintain unicode characters in output
    json_bin = demjson.encode(json_data, encoding='utf-8', compactly=False)
    # Write json to file
    try:
        open("{}.jsonld".format(filename), "wb").write(json_bin)
        logger_jsons.info("wrote data to json file")
    except FileNotFoundError as e:
        print("Error: Writing json to file: {}".format(filename))
        logger_jsons.debug("write_json_to_file: FileNotFound: {}, {}".format(filename, e))
    logger_jsons.info("exit write_json_to_file")
    return

Response: Write all JSON in python dictionary to a new json file.


Instruction:
Create a Python 3 function for
switching from index - by - name to index - by - number.

Response:
def idx_name_to_num(L):
    """
    Switch from index-by-name to index-by-number.
    :param dict L: Metadata
    :return dict: Modified metadata
    """
    logger_jsons.info("enter idx_name_to_num")

    # Process the paleoData section
    if "paleoData" in L:
        L["paleoData"] = _export_section(L["paleoData"], "paleo")

    # Process the chronData section
    if "chronData" in L:
        L["chronData"] = _export_section(L["chronData"], "chron")

    logger_jsons.info("exit idx_name_to_num")
    return L


Instruction:
How would you code a function in Python 3 to
switch chron data to index - by - number

Response:
def _export_section(sections, pc):
    """
    Switch chron data to index-by-number
    :param dict sections: Metadata
    :return list _sections: Metadata
    """
    logger_jsons.info("enter export_data: {}".format(pc))
    _sections = []

    for name, section in sections.items():

        # Process chron models
        if "model" in section:
            section["model"] = _export_model(section["model"])

        # Process the chron measurement table
        if "measurementTable" in section:
            section["measurementTable"] = _idx_table_by_num(section["measurementTable"])

        # Add only the table to the output list
        _sections.append(section)

    logger_jsons.info("exit export_data: {}".format(pc))
    return _sections


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _export_model(models):
    logger_jsons.info("enter export_model")
    _models = []
    try:
        for name, model in models.items():

            if "summaryTable" in model:
                model["summaryTable"] = _idx_table_by_num(model["summaryTable"])

            # Process ensemble table (special two columns)
            if "ensembleTable" in model:
                model["ensembleTable"] = _idx_table_by_num(model["ensembleTable"])

            if "distributionTable" in model:
                model["distributionTable"] = _idx_table_by_num(model["distributionTable"])

            _models.append(model)

    except Exception as e:
        logger_jsons.error("export_model: {}".format(e))
        print("Error: export_model: {}".format(e))
    logger_jsons.info("exit export_model")
    return _models

Response: Switch model tables to index - by - number


Instruction:
Write a Python 3 script to
switch tables to index - by - number

Response:
def _idx_table_by_num(tables):
    """
    Switch tables to index-by-number

    :param dict tables: Metadata
    :return list _tables: Metadata
    """
    logger_jsons.info("enter idx_table_by_num")
    _tables = []
    for name, table in tables.items():
        try:
            # Get the modified table data
            tmp = _idx_col_by_num(table)
            # Append it to the growing calibrated age list of tables
            _tables.append(tmp)
        except Exception as e:
            logger_jsons.error("idx_table_by_num: {}".format(e))
    logger_jsons.info("exit idx_table_by_num")
    return _tables


Instruction:
Create a Python 3 function to
index columns by number instead of by name. Use number key in column to maintain order

Response:
def _idx_col_by_num(table):
    """
    Index columns by number instead of by name. Use "number" key in column to maintain order

    :param dict table: Metadata
    :return list _table: Metadata
    """
    _columns = []
    try:
        # Create an empty list that matches the length of the column dictionary
        _columns = [None for i in range(0, len(table["columns"]))]

        # Loop and start placing data in the output list based on its "number" entry
        for _name, _dat in table["columns"].items():
            try:
                # Special case for ensemble table "numbers" list
                if isinstance(_dat["number"], list):
                    _columns.append(_dat)
                # Place at list index based on its column number
                else:
                    # cast number to int, just in case it's stored as a string.
                    n = int(_dat["number"])
                    _columns[n - 1] = _dat
            except KeyError as ke:
                print("Error: idx_col_by_num: {}".format(ke))
                logger_jsons.error("idx_col_by_num: KeyError: missing number key: {}, {}".format(_name, ke))
            except Exception as e:
                print("Error: idx_col_by_num: {}".format(e))
                logger_jsons.error("idx_col_by_num: Exception: {}".format(e))

        table["columns"] = _columns
    except Exception as e:
        logger_jsons.error("idx_col_by_num: {}".format(e))
        print("Error: idx_col_by_num: {}".format(e))

    return table


Instruction:
Can you generate a brief explanation for the following Python 3 code
def make_bag(bag_dir, bag_info=None, processes=1, checksum=None):
    bag_dir = os.path.abspath(bag_dir)
    logger.info("creating bag for directory %s", bag_dir)
    # assume md5 checksum if not specified
    if not checksum:
        checksum = ['md5']

    if not os.path.isdir(bag_dir):
        logger.error("no such bag directory %s", bag_dir)
        raise RuntimeError("no such bag directory %s" % bag_dir)

    old_dir = os.path.abspath(os.path.curdir)
    os.chdir(bag_dir)

    try:
        unbaggable = _can_bag(os.curdir)
        if unbaggable:
            logger.error("no write permissions for the following directories and files: \n%s", unbaggable)
            raise BagError("Not all files/folders can be moved.")
        unreadable_dirs, unreadable_files = _can_read(os.curdir)
        if unreadable_dirs or unreadable_files:
            if unreadable_dirs:
                logger.error("The following directories do not have read permissions: \n%s", unreadable_dirs)
            if unreadable_files:
                logger.error("The following files do not have read permissions: \n%s", unreadable_files)
            raise BagError("Read permissions are required to calculate file fixities.")
        else:
            logger.info("creating data dir")

            cwd = os.getcwd()
            temp_data = tempfile.mkdtemp(dir=cwd)

            for f in os.listdir('.'):
                if os.path.abspath(f) == temp_data:
                    continue
                new_f = os.path.join(temp_data, f)
                logger.info("moving %s to %s", f, new_f)
                os.rename(f, new_f)

            logger.info("moving %s to %s", temp_data, 'data')
            os.rename(temp_data, 'data')

            # permissions for the payload directory should match those of the
            # original directory
            os.chmod('data', os.stat(cwd).st_mode)

            for c in checksum:
                logger.info("writing manifest-%s.txt", c)
                Oxum = _make_manifest('manifest-%s.txt' % c, 'data', processes, c)

            logger.info("writing bagit.txt")
            txt = """BagIt-Version: 0.97\nTag-File-Character-Encoding: UTF-8\n"""
            with open("bagit.txt", "w") as bagit_file:
                bagit_file.write(txt)

            logger.info("writing bag-info.txt")
            if bag_info is None:
                bag_info = {}

            # allow 'Bagging-Date' and 'Bag-Software-Agent' to be overidden
            if 'Bagging-Date' not in bag_info:
                bag_info['Bagging-Date'] = date.strftime(date.today(), "%Y-%m-%d")
            if 'Bag-Software-Agent' not in bag_info:
                bag_info['Bag-Software-Agent'] = 'bagit.py <http://github.com/libraryofcongress/bagit-python>'
            bag_info['Payload-Oxum'] = Oxum
            _make_tag_file('bag-info.txt', bag_info)

            for c in checksum:
                _make_tagmanifest_file(c, bag_dir)
    except Exception:
        logger.exception("An error occurred creating the bag")
        raise
    finally:
        os.chdir(old_dir)

    return Bag(bag_dir)

Response: Create a bag from a given directory.


Instruction:
How would you implement a function in Python 3 that
calculates the hash of the file at the provided path.

Response:
def _calculate_file_hashes(full_path, f_hashers):
    """
    Returns a dictionary of (algorithm, hexdigest) values for the provided
    filename
    """
    if not os.path.exists(full_path):
        raise BagValidationError("%s does not exist" % full_path)

    try:
        with open(full_path, 'rb') as f:
            while True:
                block = f.read(1048576)
                if not block:
                    break
                for i in list(f_hashers.values()):
                    i.update(block)
    except IOError as e:
        raise BagValidationError("could not read %s: %s" % (full_path, str(e)))
    except OSError as e:
        raise BagValidationError("could not read %s: %s" % (full_path, str(e)))

    return dict(
        (alg, h.hexdigest()) for alg, h in list(f_hashers.items())
    )


Instruction:
How would you implement a function in Python 3 that
parses a tag file and yields a list of tuples.

Response:
def _parse_tags(tag_file):
    """Parses a tag file, according to RFC 2822.  This
       includes line folding, permitting extra-long
       field values.

       See http://www.faqs.org/rfcs/rfc2822.html for
       more information.
    """

    tag_name = None
    tag_value = None

    # Line folding is handled by yielding values only after we encounter
    # the start of a new tag, or if we pass the EOF.
    for num, line in enumerate(tag_file):
        # If byte-order mark ignore it for now.
        if num == 0:
            if line.startswith(BOM):
                line = line.lstrip(BOM)
        # Skip over any empty or blank lines.
        if len(line) == 0 or line.isspace():
            continue
        elif line[0].isspace() and tag_value is not None:  # folded line
            tag_value += line
        else:
            # Starting a new tag; yield the last one.
            if tag_name:
                yield (tag_name, tag_value.strip())

            if ':' not in line:
                raise BagValidationError("invalid line '%s' in %s" % (line.strip(),
                                                                      os.path.basename(tag_file.name)))

            parts = line.strip().split(':', 1)
            tag_name = parts[0].strip()
            tag_value = parts[1]

    # Passed the EOF.  All done after this.
    if tag_name:
        yield (tag_name, tag_value.strip())


Instruction:
Can you write a function in Python 3 where it
returns ( unwriteable files and folders

Response:
def _can_bag(test_dir):
    """returns (unwriteable files/folders)
    """
    unwriteable = []
    for inode in os.listdir(test_dir):
        if not os.access(os.path.join(test_dir, inode), os.W_OK):
            unwriteable.append(os.path.join(os.path.abspath(test_dir), inode))
    return tuple(unwriteable)


Instruction:
Can you write a function in Python 3 where it
returns a tuple of tuples containing all directories and files that can be read from the test directory.

Response:
def _can_read(test_dir):
    """
    returns ((unreadable_dirs), (unreadable_files))
    """
    unreadable_dirs = []
    unreadable_files = []
    for dirpath, dirnames, filenames in os.walk(test_dir):
        for dn in dirnames:
            if not os.access(os.path.join(dirpath, dn), os.R_OK):
                unreadable_dirs.append(os.path.join(dirpath, dn))
        for fn in filenames:
            if not os.access(os.path.join(dirpath, fn), os.R_OK):
                unreadable_files.append(os.path.join(dirpath, fn))
    return (tuple(unreadable_dirs), tuple(unreadable_files))


Instruction:
Implement a function in Python 3 to
compare the files actually in the payload and returns a list of all the files that need to be fetched.

Response:
def compare_fetch_with_fs(self):
        """Compares the fetch entries with the files actually
           in the payload, and returns a list of all the files
           that still need to be fetched.
        """

        files_on_fs = set(self.payload_files())
        files_in_fetch = set(self.files_to_be_fetched())

        return list(files_in_fetch - files_on_fs)


Instruction:
Given the following Python 3 function, write the documentation
def save(self, processes=1, manifests=False):
        # Error checking
        if not self.path:
            raise BagError("Bag does not have a path.")

        # Change working directory to bag directory so helper functions work
        old_dir = os.path.abspath(os.path.curdir)
        os.chdir(self.path)

        # Generate new manifest files
        if manifests:
            unbaggable = _can_bag(self.path)
            if unbaggable:
                logger.error("no write permissions for the following directories and files: \n%s", unbaggable)
                raise BagError("Not all files/folders can be moved.")
            unreadable_dirs, unreadable_files = _can_read(self.path)
            if unreadable_dirs or unreadable_files:
                if unreadable_dirs:
                    logger.error("The following directories do not have read permissions: \n%s", unreadable_dirs)
                if unreadable_files:
                    logger.error("The following files do not have read permissions: \n%s", unreadable_files)
                raise BagError("Read permissions are required to calculate file fixities.")

            oxum = None
            self.algs = list(set(self.algs))  # Dedupe
            for alg in self.algs:
                logger.info('updating manifest-%s.txt', alg)
                oxum = _make_manifest('manifest-%s.txt' % alg, 'data', processes, alg)

            # Update Payload-Oxum
            logger.info('updating %s', self.tag_file_name)
            if oxum:
                self.info['Payload-Oxum'] = oxum

        _make_tag_file(self.tag_file_name, self.info)

        # Update tag-manifest for changes to manifest & bag-info files
        for alg in self.algs:
            _make_tagmanifest_file(alg, self.path)

        # Reload the manifests
        self._load_manifests()

        os.chdir(old_dir)

Response: Save the current bag to the file system.


Instruction:
Can you generate the documentation for the following Python 3 function
def missing_optional_tagfiles(self):
        for tagfilepath in list(self.tagfile_entries().keys()):
            if not os.path.isfile(os.path.join(self.path, tagfilepath)):
                yield tagfilepath

Response: Yields the optional tagfiles that are missing from the manifest.


Instruction:
Make a summary of the following Python 3 code
def validate(self, processes=1, fast=False):
        self._validate_structure()
        self._validate_bagittxt()
        self._validate_contents(processes=processes, fast=fast)
        return True

Response: Checks that the structure and contents are valid.


Instruction:
Write a Python 3 script to
return validation success or failure as boolean.

Response:
def is_valid(self, fast=False):
        """Returns validation success or failure as boolean.
        Optional fast parameter passed directly to validate().
        """
        try:
            self.validate(fast=fast)
        except BagError:
            return False
        return True


Instruction:
Explain what the following Python 3 code does
def _validate_entries(self, processes):
        errors = list()

        # First we'll make sure there's no mismatch between the filesystem
        # and the list of files in the manifest(s)
        only_in_manifests, only_on_fs = self.compare_manifests_with_fs()
        for path in only_in_manifests:
            e = FileMissing(path)
            logger.warning(str(e))
            errors.append(e)
        for path in only_on_fs:
            e = UnexpectedFile(path)
            logger.warning(str(e))
            errors.append(e)

        # To avoid the overhead of reading the file more than once or loading
        # potentially massive files into memory we'll create a dictionary of
        # hash objects so we can open a file, read a block and pass it to
        # multiple hash objects

        available_hashers = set()
        for alg in self.algs:
            try:
                hashlib.new(alg)
                available_hashers.add(alg)
            except ValueError:
                logger.warning("Unable to validate file contents using unknown %s hash algorithm", alg)

        if not available_hashers:
            raise RuntimeError("%s: Unable to validate bag contents: none of the hash algorithms in %s are supported!" % (self, self.algs))

        def _init_worker():
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        args = ((self.path, rel_path, hashes, available_hashers) for rel_path, hashes in list(self.entries.items()))

        try:
            if processes == 1:
                hash_results = list(map(_calc_hashes, args))
            else:
                try:
                    pool = multiprocessing.Pool(processes if processes else None, _init_worker)
                    hash_results = pool.map(_calc_hashes, args)
                finally:
                    try:
                        pool.terminate()
                    except:
                        # we really don't care about any exception in terminate()
                        pass
        # Any unhandled exceptions are probably fatal
        except:
            logger.exception("unable to calculate file hashes for %s", self)
            raise

        for rel_path, f_hashes, hashes in hash_results:
            for alg, computed_hash in list(f_hashes.items()):
                stored_hash = hashes[alg]
                if stored_hash.lower() != computed_hash:
                    e = ChecksumMismatch(rel_path, alg, stored_hash.lower(), computed_hash)
                    logger.warning(str(e))
                    errors.append(e)

        if errors:
            raise BagValidationError("invalid bag", errors)

Response: Validate that the actual file contents matches the recorded hashes stored in the manifest files


Instruction:
Explain what the following Python 3 code does
def _validate_bagittxt(self):
        bagit_file_path = os.path.join(self.path, "bagit.txt")
        with open(bagit_file_path, 'r') as bagit_file:
            first_line = bagit_file.readline()
            if first_line.startswith(BOM):
                raise BagValidationError("bagit.txt must not contain a byte-order mark")

Response: Verify that bagit. txt conforms to specification


Instruction:
How would you explain what the following Python 3 function does
def _fix_numeric_types(c):
    try:
        for var, data in c.items():
            for k, v in data.items():
                if k in ["hasMeanValue", "hasMaxValue", "hasMinValue", "hasMedianValue"]:
                    if math.isnan(v):
                        c[var][k] = "nan"
                    elif not isinstance(v, (int, float)):
                        try:
                            c[var][k] = float(v)
                        except Exception as e:
                            logger_inferred_data.info("fix_numeric_types: converting float: {}".format(e))
                elif k == "hasResolution":
                    for b, g in v.items():
                        if b in ["hasMeanValue", "hasMaxValue", "hasMinValue", "hasMedianValue"]:
                            if math.isnan(g):
                                c[var][k][b] = "nan"
                            elif not isinstance(g, (int, float)):
                                try:
                                    f = float(g)
                                    c[var][k][b] = f
                                except Exception as e:
                                    logger_inferred_data.info("fix_numeric_types: converting float: {}".format(e))
    except Exception as e:
        logger_inferred_data.error("fix_numeric_types: {}".format(e))
    return c

Response: Fix any numpy data types that didn t map back to python data types properly


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_age(columns):

    # Need to check multiple places for age, year, or yrbp
    # 1. Check the column variable name
    # 2. Check for an "inferredVariableType

    age = []
    try:
        # Step 1:
        # Check for age first (exact match)
        if "age" in columns:
            # Save the values
            age = columns["age"]["values"]
        # Check for year second (exact match)
        elif "year" in columns:
            # Save the values
            age = columns["year"]["values"]
        elif "yrbp" in columns:
            # Save the values
            age = columns["yrbp"]["values"]

        # Step 2 No exact matches, check for an "inferredVariableType" : "Age" or "Year"
        if not age:
            # Loop through column variableNames
            for k, v in columns.items():
                try:
                    if v["inferredVariableType"].lower() == "age":
                        age = v["values"]
                    elif v["inferredVariableType"].lower() == "year":
                        age = v["values"]
                except Exception:
                    # Not too concerned if we error here.
                    pass

        # Step 3: No year or age found, start searching for a loose match with "age" or "year" in the variableName.
        if not age:
            # Loop through column variableNames
            for k, v in columns.items():
                k_low = k.lower()
                # Check for age in the variableName (loose match)
                if "age" in k_low:
                    # Save the values
                    age = v["values"]
                # Check for year in variableName (loose match)
                elif "year" in k_low:
                    # Save the values
                    age = v["values"]
                elif "yrbp" in k_low:
                    # Save the values
                    age = v["values"]

    # If we expected a dictionary, and didn't get one
    except AttributeError as e:
        logger_inferred_data.warn("get_age: AttributeError: {}".format(e))
    # If we were looking for values, and didn't get one
    except KeyError as e:
        logger_inferred_data.warn("get_age: KeyError: {}".format(e))
    # Fail-safe for other problems
    except Exception as e:
        logger_inferred_data.warn("get_age: Exception: {}".format(e))

    return age

Response: Return the age of the candidate in the table column data.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_resolution(age, values):
    res = []
    try:
        # Get the nan index from the values and remove from age
        # age2 = age[np.where(~np.isnan(values))[0]]
        # res = np.diff(age2)

        # Make sure that age and values are numpy arrays
        # age = np.array(age, dtype=float)
        # values = np.array(values, dtype=float)
        # Get the nan index from the values and remove from age
        age2 = age[np.where(~np.isnan(values))[0]]
        res = np.diff(age2)

    except IndexError as e:
        print("get_resolution: IndexError: {}".format(e))
    except Exception as e:
        logger_inferred_data.warn("get_resolution: Exception: {}".format(e))

    return res

Response: Calculates the resolution of the current object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __get_inferred_data_res_2(v=None, calc=True):
    # Base: If something goes wrong, or if there are no values, then use "NaN" placeholders.
    d = {
        "hasMinValue": "nan", "hasMaxValue": "nan",
        "hasMeanValue": "nan", "hasMedianValue": "nan",
    }
    try:
        if calc:
            _min = np.nanmin(v)
            _max = np.nanmax(v)
            _mean = np.nanmean(v)
            _med = np.nanmedian(v)

            if np.isnan(_min):
                _min = "nan"
            else:
                _min = abs(_min)
            if np.isnan(_max):
                _max = "nan"
            else:
                _min = abs(_min)
            if np.isnan(_mean):
                _mean = "nan"
            else:
                _min = abs(_min)
            if np.isnan(_med):
                _med = "nan"
            else:
                _min = abs(_min)

            d = {
                "hasMinValue": _min,
                "hasMaxValue": _max,
                "hasMeanValue": _mean,
                "hasMedianValue": _med
            }
    except Exception as e:
        logger_inferred_data.error("get_inferred_data_res_2: {}".format(e))

    return d

Response: This function calculates the inferred data for a single element of the array v.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_inferred_data_res(column, age):
    try:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            # Get the values for this column
            values = column["values"]
            # Make sure that age and values are numpy arrays
            _values = np.array(copy.copy(values), dtype=float)
            # _values = _values[np.where(~np.isnan(_values))[0]]
            _age = np.array(age, dtype=float)
            # _age = _age[np.where(~np.isnan(_age))[0]]

            # If we have values, keep going
            if len(_values) != 0:
                # Get the resolution for this age and column values data
                res = _get_resolution(_age, _values)
                # If we have successful resolution data, keep going
                if len(res) != 0:
                    column["hasResolution"] = __get_inferred_data_res_2(res)

                # Remove the NaNs from the values list.
                _values = _values[np.where(~np.isnan(_values))[0]]
                # Calculate column non-resolution data, update the column with the results.
                column.update(__get_inferred_data_res_2(_values))

    except KeyError as e:
        logger_inferred_data.debug("get_inferred_data_column: KeyError: {}".format(e))
    except Exception as e:
        logger_inferred_data.debug("get_inferred_data_column: Exception: {}".format(e))

    return column

Response: Calculate Resolution and m/m for column values.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_inferred_data_column(column):
    try:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            # Get the values for this column
            values = column["values"]
            # Make sure that age and values are numpy arrays
            _values = np.array(copy.copy(values), dtype=float)
            # If we have values, keep going
            if len(_values) != 0:
                # Remove the NaNs from the values list.
                _values = _values[np.where(~np.isnan(_values))[0]]
                # Use the values to create new entries and data
                column.update(__get_inferred_data_res_2(_values))

            # Even though we're not calculating resolution, still add it with "NaN" placeholders.
            column["hasResolution"] = __get_inferred_data_res_2(None, calc=False)

    except KeyError as e:
        logger_inferred_data.debug("get_inferred_data_column: KeyError: {}".format(e))
    except Exception as e:
        logger_inferred_data.debug("get_inferred_data_column: Exception: {}".format(e))

    return column

Response: Calculate the m / m / m for the values of the column.


Instruction:
Write a Python 3 script to
calculate the inferred data for a specific variable in the metadata table.

Response:
def get_inferred_data_table(table, pc):
    """
    Table level: Dive down, calculate data, then return the new table with the inferred data.

    :param str pc: paleo or chron
    :param dict table: Metadata
    :return dict table: Metadata
    """
    age = None
    if pc == "paleo":
        # Get the age values data first, since it's needed to calculate the other column data.
        age = _get_age(table["columns"])

    try:
        # If age values were not found, then skip resolution.
        if age:
            # Loop for all the columns in the table
            for var, col in table["columns"].items():
                # Special cases
                # We do not calculate data for each of the keys below, and we cannot calculate any "string" data
                if "age" in var or "year" in var:
                    # Calculate m/m/m/m, but not resolution
                    table["columns"][var] = _get_inferred_data_column(col)
                elif not all(isinstance(i, str) for i in col["values"]):
                    # Calculate m/m/m/m and resolution
                    table["columns"][var] = _get_inferred_data_res(col, age)
                else:
                    # Fall through case. No calculations made.
                    logger_inferred_data.info("get_inferred_data_table: "
                                              "Not calculating inferred data for variableName: {}".format(var))

        # If there isn't an age, still calculate the m/m/m/m for the column values.
        else:
            for var, col in table["columns"].items():
                if not all(isinstance(i, str) for i in col["values"]):
                    # Calculate m/m/m/m and resolution
                    table["columns"][var] = _get_inferred_data_column(col)
                else:
                    # Fall through case. No calculations made.
                    logger_inferred_data.info("get_inferred_data_table: "
                                              "Not calculating inferred data for variableName: {}".format(var))

    except AttributeError as e:
        logger_inferred_data.warn("get_inferred_data_table: AttributeError: {}".format(e))
    except Exception as e:
        logger_inferred_data.warn("get_inferred_data_table: Exception: {}".format(e))

    table["columns"] = _fix_numeric_types(table["columns"])
    return table


Instruction:
Create a Python 3 function for
registering a new sphinx - needs function for the given sphinx environment.

Response:
def register_func(env, need_function):
    """
    Registers a new sphinx-needs function for the given sphinx environment.
    :param env: Sphinx environment
    :param need_function: Python method
    :return: None
    """

    if not hasattr(env, 'needs_functions'):
        env.needs_functions = {}

    func_name = need_function.__name__

    if func_name in env.needs_functions.keys():
        raise SphinxError('sphinx-needs: Function name {} already registered.'.format(func_name))

    env.needs_functions[func_name] = {
        'name': func_name,
        'function': need_function
    }


Instruction:
Write a Python 3 function for
executing a given function string.

Response:
def execute_func(env, need, func_string):
    """
    Executes a given function string.
    :param env: Sphinx environment
    :param need: Actual need, which contains the found function string
    :param func_string: string of the found function. Without [[ ]]
    :return: return value of executed function
    """
    func_name, func_args, func_kwargs = _analyze_func_string(func_string)

    if func_name not in env.needs_functions.keys():
        raise SphinxError('Unknown dynamic sphinx-needs function: {}. Found in need: {}'.format(func_name, need['id']))

    func = env.needs_functions[func_name]['function']
    func_return = func(env, need, env.needs_all_needs, *func_args, **func_kwargs)

    if not isinstance(func_return, (str, int, float, list, unicode)) and func_return is not None:
        raise SphinxError('Return value of function {} is of type {}. Allowed are str, int, float'.format(
            func_name, type(func_return)))

    if isinstance(func_return, list):
        for element in func_return:
            if not isinstance(element, (str, int, float, unicode)):
                raise SphinxError('Element of return list of function {} is of type {}. '
                                  'Allowed are str, int, float'.format(func_name, type(func_return)))
    return func_return


Instruction:
Write a Python 3 function for
finding a node and its children and replace it with the content of the node.

Response:
def find_and_replace_node_content(node, env, need):
    """
    Search inside a given node and its children for nodes of type Text,
    if found check if it contains a function string and run/replace it.

    :param node: Node to analyse
    :return: None
    """
    new_children = []
    if not node.children:
        if isinstance(node, nodes.Text):
            func_match = func_pattern.findall(node)
            new_text = node
            for func_string in func_match:
                if not is_python3:
                    func_string = func_string.encode('utf-8')
                # sphinx is replacing ' and " with language specific quotation marks (up and down), which makes
                # it impossible for the later used AST render engine to detect a python function call in the given
                # string. Therefor a replacement is needed for the execution of the found string.
                func_string_org = func_string[:]
                func_string = func_string.replace('„', '"')
                func_string = func_string.replace('“', '"')
                func_string = func_string.replace('”', '"')
                func_string = func_string.replace('”', '"')

                func_string = func_string.replace('‘', '"')
                func_string = func_string.replace('’', '"')
                func_return = execute_func(env, need, func_string)
                if not is_python3:
                    new_text = new_text.replace(u'[[{}]]'.format(func_string_org.decode('utf-8')), func_return)
                else:
                    new_text = new_text.replace(u'[[{}]]'.format(func_string_org), func_return)
            node = nodes.Text(new_text, new_text)
        return node
    else:
        for child in node.children:
            new_child = find_and_replace_node_content(child, env, need)
            new_children.append(new_child)
        node.children = new_children
    return node


Instruction:
Write a Python 3 function for
resolving dynamic values inside need data.

Response:
def resolve_dynamic_values(env):
    """
    Resolve dynamic values inside need data.

    Rough workflow:

    #. Parse all needs and their data for a string like [[ my_func(a,b,c) ]]
    #. Extract function name and call parameters
    #. Execute registered function name with extracted call parameters
    #. Replace original string with return value

    :param env: Sphinx environment
    :return: return value of given function
    """
    # Only perform calculation if not already done yet
    if env.needs_workflow['dynamic_values_resolved']:
        return

    needs = env.needs_all_needs
    for key, need in needs.items():
        for need_option in need:
            if need_option in ['docname', 'lineno', 'target_node', 'content']:
                # dynamic values in this data are not allowed.
                continue
            if not isinstance(need[need_option], (list, set)):
                func_call = True
                while func_call:
                    try:
                        func_call, func_return = _detect_and_execute(need[need_option], need, env)
                    except FunctionParsingException:
                        raise SphinxError("Function definition of {option} in file {file}:{line} has "
                                          "unsupported parameters. "
                                          "supported are str, int/float, list".format(option=need_option,
                                                                                file=need['docname'],
                                                                                line=need['lineno']))

                    if func_call is None:
                        continue
                    # Replace original function string with return value of function call
                    if func_return is None:
                        need[need_option] = need[need_option].replace('[[{}]]'.format(func_call), '')
                    else:
                        need[need_option] = need[need_option].replace('[[{}]]'.format(func_call), str(func_return))

                    if need[need_option] == '':
                        need[need_option] = None
            else:
                new_values = []
                for element in need[need_option]:
                    try:
                        func_call, func_return = _detect_and_execute(element, need, env)
                    except FunctionParsingException:
                        raise SphinxError("Function definition of {option} in file {file}:{line} has "
                                          "unsupported parameters. "
                                          "supported are str, int/float, list".format(option=need_option,
                                                                                file=need['docname'],
                                                                                line=need['lineno']))
                    if func_call is None:
                        new_values.append(element)
                    else:
                        # Replace original function string with return value of function call
                        if isinstance(need[need_option], (str, int, float)):
                            new_values.append(element.replace('[[{}]]'.format(func_call), str(func_return)))
                        else:
                            if isinstance(need[need_option], (list, set)):
                                new_values += func_return

                need[need_option] = new_values

    # Finally set a flag so that this function gets not executed several times
    env.needs_workflow['dynamic_values_resolved'] = True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _analyze_func_string(func_string):
    func = ast.parse(func_string)
    try:
        func_call = func.body[0].value
        func_name = func_call.func.id
    except AttributeError:
        raise SphinxError("Given dynamic function string is not a valid python call. Got: {}".format(func_string))

    func_args = []
    for arg in func_call.args:
        if isinstance(arg, ast.Num):
            func_args.append(arg.n)
        elif isinstance(arg, ast.Str):
            func_args.append(arg.s)
        elif isinstance(arg, ast.BoolOp):
            func_args.append(arg.s)
        elif isinstance(arg, ast.List):
            arg_list = []
            for element in arg.elts:
                if isinstance(element, ast.Num):
                    arg_list.append(element.n)
                elif isinstance(element, ast.Str):
                    arg_list.append(element.s)
            func_args.append(arg_list)
        else:
            raise FunctionParsingException()
    func_kargs = {}
    for keyword in func_call.keywords:
        kvalue = keyword.value
        kkey = keyword.arg
        if isinstance(kvalue, ast.Num):
            func_kargs[kkey] = kvalue.n
        elif isinstance(kvalue, ast.Str):
            func_kargs[kkey] = kvalue.s
        elif isinstance(kvalue, ast_boolean):  # Check if Boolean
            if is_python3:
                func_kargs[kkey] = kvalue.value
            else:
                func_kargs[kkey] = kvalue.id
        elif isinstance(kvalue, ast.List):
            arg_list = []
            for element in kvalue.elts:
                if isinstance(element, ast.Num):
                    arg_list.append(element.n)
                elif isinstance(element, ast.Str):
                    arg_list.append(element.s)
            func_kargs[kkey] = arg_list
        else:
            raise FunctionParsingException()

    return func_name, func_args, func_kargs

Response: Analyze given string of the function and extract the function name arguments keyword arguments and return the function name arguments keyword arguments


Instruction:
Explain what the following Python 3 code does
def run():
    # GLOBALS
    global cwd, files, logger_start, logger_benchmark, settings, _timeseries_data
    _timeseries_data = {}
    # files = {".lpd": [ {"full_path", "filename_ext", "filename_no_ext", "dir"} ], ".xls": [...], ".txt": [...]}
    settings = {"note_update": True, "note_validate": True, "verbose": True}
    cwd = os.getcwd()
    # logger created in whatever directory lipd is called from
    logger_start = create_logger("start")
    logger_benchmark = create_benchmark("benchmarks", "benchmark.log")
    files = {".txt": [], ".lpd": [], ".xls": []}

    return

Response: Initialize and start objects. This is called automatically when importing the package.


Instruction:
How would you explain what the following Python 3 function does
def readLipd(usr_path=""):
    global cwd, settings, files
    if settings["verbose"]:
        __disclaimer(opt="update")
    start = clock()
    files[".lpd"] = []
    __read(usr_path, ".lpd")
    _d = __read_lipd_contents()
    end = clock()
    logger_benchmark.info(log_benchmark("readLipd", start, end))
    return _d

Response: Read a. LiPD file and return a dict of metadata.


Instruction:
Given the following Python 3 function, write the documentation
def readExcel(usr_path=""):
    global cwd, files
    start = clock()
    files[".xls"] = []
    __read(usr_path, ".xls")
    end = clock()
    logger_benchmark.info(log_benchmark("readExcel", start, end))
    return cwd

Response: Read Excel file and return current working directory


Instruction:
Explain what the following Python 3 code does
def excel():
    global files, cwd, settings
    _d = {}
    # Turn off verbose. We don't want to clutter the console with extra reading/writing output statements
    settings["verbose"] = False
    # Find excel files
    print("Found " + str(len(files[".xls"])) + " Excel files")
    logger_start.info("found excel files: {}".format(len(files[".xls"])))
    # Start the clock
    start = clock()
    # Loop for each excel file
    for file in files[".xls"]:
        # Convert excel file to LiPD
        dsn = excel_main(file)
        try:
            # Read the new LiPD file back in, to get fixes, inferred calculations, updates, etc.
            _d[dsn] = readLipd(os.path.join(file["dir"], dsn + ".lpd"))
            # Write the modified LiPD file back out again.
            writeLipd(_d[dsn], cwd)
        except Exception as e:
            logger_start.error("excel: Unable to read new LiPD file, {}".format(e))
            print("Error: Unable to read new LiPD file: {}, {}".format(dsn, e))
    # Time!
    end = clock()
    logger_benchmark.info(log_benchmark("excel", start, end))
    # Start printing stuff again.
    settings["verbose"] = True
    return _d

Response: Convert Excel files to LiPD files.


Instruction:
Make a summary of the following Python 3 code
def noaa(D="", path="", wds_url="", lpd_url="", version=""):
    global files, cwd
    # When going from NOAA to LPD, use the global "files" variable.
    # When going from LPD to NOAA, use the data from the LiPD Library.

    # Choose the mode
    _mode = noaa_prompt()
    start = clock()
    # LiPD mode: Convert LiPD files to NOAA files
    if _mode == "1":
        # _project, _version = noaa_prompt_1()
        if not version or not lpd_url:
            print("Missing parameters: Please try again and provide all parameters.")
            return
        if not D:
            print("Error: LiPD data must be provided for LiPD -> NOAA conversions")
        else:
            if "paleoData" in D:
                _d = copy.deepcopy(D)
                D = lpd_to_noaa(_d, wds_url, lpd_url, version, path)
            else:
                # For each LiPD file in the LiPD Library
                for dsn, dat in D.items():
                    _d = copy.deepcopy(dat)
                    # Process this data through the converter
                    _d = lpd_to_noaa(_d, wds_url, lpd_url, version, path)
                    # Overwrite the data in the LiPD object with our new data.
                    D[dsn] = _d
            # If no wds url is provided, then remove instances from jsonld metadata
            if not wds_url:
                D = rm_wds_url(D)
            # Write out the new LiPD files, since they now contain the new NOAA URL data
            if(path):
                writeLipd(D, path)
            else:
                print("Path not provided. Writing to CWD...")
                writeLipd(D, cwd)

    # NOAA mode: Convert NOAA files to LiPD files
    elif _mode == "2":
        # Pass through the global files list. Use NOAA files directly on disk.
        noaa_to_lpd(files)

    else:
        print("Invalid input. Try again.")
    end = clock()
    logger_benchmark.info(log_benchmark("noaa", start, end))
    return

Response: Convert between NOAA and LiPD files


Instruction:
Can you generate a brief explanation for the following Python 3 code
def validate(D, detailed=True):
    start = clock()
    print("\n")
    # Fetch new results by calling lipd.net/api/validator (costly, may take a while)
    print("Fetching results from validator at lipd.net/validator... this may take a few moments.\n")
    try:
        results = []
        # Get the validator-formatted data for each dataset.
        if "paleoData" in D:
            _api_data = get_validator_format(D)
            # A list of lists of LiPD-content metadata
            results.append(call_validator_api(D["dataSetName"], _api_data))
        else:
            for dsn, dat in D.items():
                _api_data = get_validator_format(dat)
                # A list of lists of LiPD-content metadata
                results.append(call_validator_api(dsn, _api_data))
        display_results(results, detailed)
    except Exception as e:
        print("Error: validate: {}".format(e))

    end = clock()
    logger_benchmark.info(log_benchmark("validate", start, end))
    return

Response: Validate a single or multiple LiPD files.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def tsToDf(tso):
    dfs = {}
    try:
        dfs = ts_to_df(tso)
    except Exception as e:
        print("Error: Unable to create data frame")
        logger_start.warn("ts_to_df: tso malformed: {}".format(e))
    return dfs

Response: Create Pandas DataFrame from TimeSeries object.


Instruction:
Can you implement a function in Python 3 that
extracts a single time series from a LiPD data file.

Response:
def extractTs(d, whichtables="meas", mode="paleo"):
    """
    Create a time series using LiPD data (uses paleoData by default)

    | Example : (default) paleoData and meas tables
    | 1. D = lipd.readLipd()
    | 2. ts = lipd.extractTs(D)

    | Example : chronData and all tables
    | 1. D = lipd.readLipd()
    | 2. ts = lipd.extractTs(D, "all", "chron")

    :param dict d: Metadata
    :param str whichtables: "all", "summ", "meas", "ens" - The tables that you would like in the timeseries
    :param str mode: "paleo" or "chron" mode
    :return list l: Time series
    """
    # instead of storing each raw dataset per tso, store it once in the global scope. saves memory
    global _timeseries_data
    _l = []
    start = clock()
    try:
        if not d:
            print("Error: LiPD data not provided. Pass LiPD data into the function.")
        else:
            print(mode_ts("extract", mode))
            if "paleoData" in d:
                # One dataset: Process directly on file, don't loop
                try:
                    _dsn = get_dsn(d)
                    _timeseries_data[start] = {}
                    _timeseries_data[start][_dsn] = d
                    # Use the LiPD data given to start time series extract
                    print("extracting: {}".format(_dsn))
                    # Copy, so we don't affect the original data
                    _v = copy.deepcopy(d)
                    # Start extract...
                    _l = (extract(_v, whichtables, mode, start))
                except Exception as e:
                    print("Error: Unable to extractTs for dataset: {}: {}".format(_dsn, e))
                    logger_start.debug("extractTs: Exception: {}, {}".format(_dsn, e))

            else:
                _timeseries_data[start] = d
                # Multiple datasets: Loop and append for each file
                for k, v in d.items():
                    try:
                        # Use the LiPD data given to start time series extract
                        print("extracting: {}".format(k))
                        # Copy, so we don't affect the original data
                        _v = copy.deepcopy(v)
                        # Start extract...
                        _l += (extract(_v, whichtables, mode, start))
                    except Exception as e:
                        print("Error: Unable to extractTs for dataset: {}: {}".format(k, e))
                        logger_start.debug("extractTs: Exception: {}".format(e))
            print("Created time series: {} entries".format(len(_l)))
    except Exception as e:
        print("Error: Unable to extractTs: {}".format(e))
        logger_start.error("extractTs: Exception: {}".format(e))
    end = clock()
    logger_benchmark.info(log_benchmark("extractTs", start, end))
    return _l


Instruction:
Create a Python 3 function for
collapsing a time series into LiPD record form.

Response:
def collapseTs(ts=None):
    """
    Collapse a time series back into LiPD record form.

    | Example
    | 1. D = lipd.readLipd()
    | 2. ts = lipd.extractTs(D)
    | 3. New_D = lipd.collapseTs(ts)

    _timeseries_data is sorted by time_id, and then by dataSetName
    _timeseries_data[10103341]["ODP1098B"] = {data}

    :param list ts: Time series
    :return dict: Metadata
    """
    # Retrieve the associated raw data according to the "time_id" found in each object. Match it in _timeseries_data
    global _timeseries_data
    _d = {}
    if not ts:
        print("Error: Time series data not provided. Pass time series into the function.")
    else:
        # Send time series list through to be collapsed.
        try:
            _raw = _timeseries_data[ts[0]["time_id"]]
            print(mode_ts("collapse", mode="", ts=ts))
            _d = collapse(ts, _raw)
            _d = rm_empty_fields(_d)
        except Exception as e:
            print("Error: Unable to collapse the time series: {}".format(e))
            logger_start.error("collapseTs: unable to collapse the time series: {}".format(e))
    return _d


Instruction:
Can you implement a function in Python 3 that
filters the given time series by the given list of expressions.

Response:
def filterTs(ts, expressions):
    """
    Create a new time series that only contains entries that match the given expression.

    | Example:
    | D = lipd.loadLipd()
    | ts = lipd.extractTs(D)
    | new_ts = filterTs(ts, "archiveType == marine sediment")
    | new_ts = filterTs(ts, ["paleoData_variableName == sst", "archiveType == marine sediment"])
    | Expressions should use underscores to denote data nesting.
    | Ex: paleoData_hasResolution_hasMedian or

    :param list OR str expressions: Expressions
    :param list ts:                 Time series
    :return list new_ts:            Filtered time series that matches the expression
    """
    # Make a copy of the ts. We're going to work directly on it.
    new_ts = ts[:]

    # User provided a single query string
    if isinstance(expressions, str):
        # Use some magic to turn the given string expression into a machine-usable comparative expression.
        expr_lst = translate_expression(expressions)
        # Only proceed if the translation resulted in a usable expression.
        if expr_lst:
            # Return the new filtered time series. This will use the same time series
            # that filters down each loop.
            new_ts, _idx = get_matches(expr_lst, new_ts)

    # User provided a list of multiple queries
    elif isinstance(expressions, list):
        # Loop for each query
        for expr in expressions:
            # Use some magic to turn the given string expression into a machine-usable comparative expression.
            expr_lst = translate_expression(expr)
            # Only proceed if the translation resulted in a usable expression.
            if expr_lst:
                # Return the new filtered time series. This will use the same time series
                # that filters down each loop.
                new_ts, _idx = get_matches(expr_lst, new_ts)

    return new_ts


Instruction:
Here you have a function in Python 3, explain what it does
def queryTs(ts, expression):
    # Make a copy of the ts. We're going to work directly on it.
    _idx = []

    # User provided a single query string
    if isinstance(expressions, str):
        # Use some magic to turn the given string expression into a machine-usable comparative expression.
        expr_lst = translate_expression(expressions)
        # Only proceed if the translation resulted in a usable expression.
        if expr_lst:
            # Return the new filtered time series. This will use the same time series
            # that filters down each loop.
            new_ts, _idx = get_matches(expr_lst, new_ts)

    # User provided a list of multiple queries
    elif isinstance(expressions, list):
        # Loop for each query
        for expr in expressions:
            # Use some magic to turn the given string expression into a machine-usable comparative expression.
            expr_lst = translate_expression(expr)
            # Only proceed if the translation resulted in a usable expression.
            if expr_lst:
                # Return the new filtered time series. This will use the same time series
                # that filters down each loop.
                new_ts, _idx = get_matches(expr_lst, new_ts)
    return _idx

Response: Query the time series for the given expression.


Instruction:
Write a Python 3 function for
viewing the contents of one time series entry in a nicely formatted way

Response:
def viewTs(ts):
    """
    View the contents of one time series entry in a nicely formatted way

    | Example
    | 1. D = lipd.readLipd()
    | 2. ts = lipd.extractTs(D)
    | 3. viewTs(ts[0])

    :param dict ts: One time series entry
    :return none:
    """
    _ts = ts
    if isinstance(ts, list):
        _ts = ts[0]
        print("It looks like you input a full time series. It's best to view one entry at a time.\n"
              "I'll show you the first entry...")
    _tmp_sort = OrderedDict()
    _tmp_sort["ROOT"] = {}
    _tmp_sort["PUBLICATION"] = {}
    _tmp_sort["GEO"] = {}
    _tmp_sort["OTHERS"] = {}
    _tmp_sort["DATA"] = {}

    # Organize the data by section
    for k,v in _ts.items():
        if not any(i == k for i in ["paleoData", "chronData", "mode", "@context"]):
            if k in ["archiveType", "dataSetName", "googleSpreadSheetKey", "metadataMD5", "tagMD5", "googleMetadataWorksheet", "lipdVersion"]:
                _tmp_sort["ROOT"][k] = v
            elif "pub" in k:
                _tmp_sort["PUBLICATION"][k] = v
            elif "geo" in k:
                _tmp_sort["GEO"][k] = v
            elif "paleoData_" in k or "chronData_" in k:
                if isinstance(v, list) and len(v) > 2:
                    _tmp_sort["DATA"][k] = "[{}, {}, {}, ...]".format(v[0], v[1], v[2])
                else:
                    _tmp_sort["DATA"][k] = v
            else:
                if isinstance(v, list) and len(v) > 2:
                    _tmp_sort["OTHERS"][k] = "[{}, {}, {}, ...]".format(v[0], v[1], v[2])
                else:
                    _tmp_sort["OTHERS"][k] = v

    # Start printing the data to console
    for k1, v1 in _tmp_sort.items():
        print("\n{}\n===============".format(k1))
        for k2, v2 in v1.items():
            print("{} : {}".format(k2, v2))

    return


Instruction:
Can you generate a brief explanation for the following Python 3 code
def showLipds(D=None):

    if not D:
        print("Error: LiPD data not provided. Pass LiPD data into the function.")
    else:
        print(json.dumps(D.keys(), indent=2))

    return

Response: Display the dataset names of a given LiPD data


Instruction:
Write a Python 3 script to
display the metadata specified LiPD in pretty print

Response:
def showMetadata(dat):
    """
    Display the metadata specified LiPD in pretty print

    | Example
    | showMetadata(D["Africa-ColdAirCave.Sundqvist.2013"])

    :param dict dat: Metadata
    :return none:
    """
    _tmp = rm_values_fields(copy.deepcopy(dat))
    print(json.dumps(_tmp, indent=2))
    return