Instruction:
Given the following Python 3 function, write the documentation
def showDfs(d):
    if "metadata" in d:
        print("metadata")
    if "paleoData" in d:
        try:
            for k, v in d["paleoData"].items():
                print(k)
        except KeyError:
            pass
        except AttributeError:
            pass
    if "chronData" in d:
        try:
            for k, v in d["chronData"].items():
                print(k)
        except KeyError:
            pass
        except AttributeError:
            pass
    # print("Process Complete")
    return

Response: Displays the available data frame names in a given data frame collection


Instruction:
Write a Python 3 script for
getting a list of all LiPD names in the library

Response:
def getLipdNames(D=None):
    """
    Get a list of all LiPD names in the library

    | Example
    | names = lipd.getLipdNames(D)

    :return list f_list: File list
    """
    _names = []
    try:
        if not D:
            print("Error: LiPD data not provided. Pass LiPD data into the function.")
        else:
            _names = D.keys()
    except Exception:
        pass
    return _names


Instruction:
Explain what the following Python 3 code does
def getMetadata(L):
    _l = {}
    try:
        # Create a copy. Do not affect the original data.
        _l = copy.deepcopy(L)
        # Remove values fields
        _l = rm_values_fields(_l)
    except Exception as e:
        # Input likely not formatted correctly, though other problems can occur.
        print("Error: Unable to get data. Please check that input is LiPD data: {}".format(e))
    return _l

Response: Get metadata from a LiPD record in memory


Instruction:
Can you write a function in Python 3 where it
gets CSV from LiPD metadata and data set

Response:
def getCsv(L=None):
    """
    Get CSV from LiPD metadata

    | Example
    | c = lipd.getCsv(D["Africa-ColdAirCave.Sundqvist.2013"])

    :param dict L: One LiPD record
    :return dict d: CSV data
    """
    _c = {}
    try:
        if not L:
            print("Error: LiPD data not provided. Pass LiPD data into the function.")
        else:
            _j, _c = get_csv_from_metadata(L["dataSetName"], L)
    except KeyError as ke:
        print("Error: Unable to get data. Please check that input is one LiPD dataset: {}".format(ke))
    except Exception as e:
        print("Error: Unable to get data. Something went wrong: {}".format(e))
        logger_start.warn("getCsv: Exception: Unable to process lipd data: {}".format(e))
    return _c


Instruction:
Implement a function in Python 3 to
write LiPD data to file

Response:
def writeLipd(dat, path=""):
    """
    Write LiPD data to file(s)

    :param dict dat: Metadata
    :param str path: Destination (optional)
    :return none:
    """
    global settings
    start = clock()
    __write_lipd(dat, path)
    end = clock()
    logger_benchmark.info(log_benchmark("writeLipd", start, end))
    return


Instruction:
Can you implement a function in Python 3 that
uses a file path to create file metadata and load a file in the appropriate way, according to the provided file type. :param str file_path: Path to file :param str file_type: One of approved file types: xls, xlsx, txt, lpd :return none:

Response:
def __universal_read(file_path, file_type):
    """
    Use a file path to create file metadata and load a file in the appropriate way, according to the provided file type.

    :param str file_path: Path to file
    :param str file_type: One of approved file types: xls, xlsx, txt, lpd
    :return none:
    """
    global files, cwd, settings

    # check that we are using the correct function to load this file type. (i.e. readNoaa for a .txt file)
    correct_ext = load_fn_matches_ext(file_path, file_type)

    # Check that this path references a file
    valid_path = path_type(file_path, "file")

    # is the path a file?
    if valid_path and correct_ext:

        # get file metadata for one file
        file_meta = collect_metadata_file(file_path)

        # append to global files, then load in D
        if file_type == ".lpd":
            # add meta to global file meta
            files[".lpd"].append(file_meta)
        # append to global files
        elif file_type in [".xls", ".xlsx"]:
            print("reading: {}".format(print_filename(file_meta["full_path"])))
            files[".xls"].append(file_meta)
        # append to global files
        elif file_type == ".txt":
            print("reading: {}".format(print_filename(file_meta["full_path"])))
            files[".txt"].append(file_meta)

        # we want to move around with the files we load
        # change dir into the dir of the target file
        cwd = file_meta["dir"]
        if cwd:
            os.chdir(cwd)

    return


Instruction:
How would you explain what the following Python 3 function does
def __read(usr_path, file_type):
    # is there a file path specified ?
    if usr_path:
        # Is this a URL? Download the file and return the local path
        is_url = re.match(re_url, usr_path)
        if is_url:
            # The usr_path will now be a local path to a single file. It will trigger the "elif" statement below
            usr_path = download_from_url(usr_path, get_download_path())

        # Directory path
        if os.path.isdir(usr_path):
            __read_directory(usr_path, file_type)
        # File path
        elif os.path.isfile(usr_path):
            __read_file(usr_path, file_type)
        # Invalid path given
        else:
            print("Error: Path given is invalid")

    # no path specified. ask if they want to load dir or file
    else:
        choice = ""
        count = 3
        while not choice:
            try:
                print("Choose a read option:\n1. One file\n2. Multi-file select\n3. Directory")
                choice = input("Option: ")
                print("\n")
                # now use the given file type and prompt answer to call _read_file or _read_dir
                if choice in ["1", "2", "3"]:
                    # open directory picker
                    if choice == "3":
                        __read_directory(usr_path, file_type)
                    else:
                        # open a file picker
                        __read_file(usr_path, file_type)
                    break
                else:
                    count -= 1
                if count == 0:
                    print("Error: Too many failed attempts")
                    break
            except Exception as e:
                print("Error: Invalid input: {}".format(e))
    return

Response: Read in a single file in a tree tree.


Instruction:
Implement a function in Python 3 to
read in the LiPD file contents as a dataset library.

Response:
def __read_lipd_contents():
    """
    Use the file metadata to read in the LiPD file contents as a dataset library

    :return dict: Metadata
    """
    global files, settings
    _d = {}
    try:
        if len(files[".lpd"]) == 1:
            _d = lipd_read(files[".lpd"][0]["full_path"])
            if settings["verbose"]:
                print("Finished read: 1 record")
        else:
            for file in files[".lpd"]:
                _d[file["filename_no_ext"]] = lipd_read(file["full_path"])
            if settings["verbose"]:
                print("Finished read: {} records".format(len(_d)))
    except Exception as e:
        print("Error: read_lipd_contents: {}".format(e))
    return _d


Instruction:
Make a summary of the following Python 3 code
def __read_file(usr_path, file_type):
    global files

    # no path provided. start gui browse
    if not usr_path:
        # src files could be a list of one, or a list of many. depending how many files the user selects
        src_dir, src_files = get_src_or_dst("read", "file")
        # check if src_files is a list of multiple files
        if src_files:
            for file_path in src_files:
                __universal_read(file_path, file_type)
        else:
            print("No file(s) chosen")
    else:
        __universal_read(usr_path, file_type)

    return

Response: This function will read a file from the Universal read file system.


Instruction:
Can you tell what is the following Python 3 function doing
def __write_lipd(dat, usr_path):
    global settings
    # no path provided. start gui browse
    if not usr_path:
        # got dir path
        usr_path, _ignore = get_src_or_dst("write", "directory")
    # Check if this is a valid directory path
    valid_path = path_type(usr_path, "directory")
    # If dir path is valid
    if valid_path:
        # Filename is given, write out one file
        if "paleoData" in dat:
            try:
                if settings["verbose"]:
                    print("writing: {}".format(dat["dataSetName"]))
                lipd_write(dat, usr_path)
            except KeyError as ke:
                print("Error: Unable to write file: unknown, {}".format(ke))
            except Exception as e:
                print("Error: Unable to write file: {}, {}".format(dat["dataSetName"], e))
        # Filename is not given, write out whole library
        else:
            if dat:
                for name, lipd_dat in dat.items():
                    try:
                        if settings["verbose"]:
                            print("writing: {}".format(name))
                        lipd_write(lipd_dat, usr_path)
                    except Exception as e:
                        print("Error: Unable to write file: {}, {}".format(name, e))

    return

Response: Write LiPD data to file.


Instruction:
How would you explain what the following Python 3 function does
def __disclaimer(opt=""):
    global settings
    if opt is "update":
        print("Disclaimer: LiPD files may be updated and modified to adhere to standards\n")
        settings["note_update"] = False
    if opt is "validate":
        print("Note: Use lipd.validate() or www.LiPD.net/create "
              "to ensure that your new LiPD file(s) are valid")
        settings["note_validate"] = False
    return

Response: Print the disclaimers once.


Instruction:
How would you implement a function in Python 3 that
adds a file to the internal list of resources.

Response:
def safe_add_file(filename, app):
    """
    Adds files to builder resources only, if the given filename was not already registered.
    Needed mainly for tests to avoid multiple registration of the same file and therefore also multiple execution
    of e.g. a javascript file during page load.

    :param filename: filename to remove
    :param app: app object
    :return: None
    """
    data_file = filename
    static_data_file = os.path.join("_static", data_file)

    if data_file.split(".")[-1] == "js":
        if hasattr(app.builder, "script_files") and static_data_file not in app.builder.script_files:
            app.add_javascript(data_file)
    elif data_file.split(".")[-1] == "css":
        if hasattr(app.builder, "css_files") and static_data_file not in app.builder.css_files:
            app.add_stylesheet(data_file)
    else:
        raise NotImplementedError("File type {} not support by save_add_file".format(data_file.split(".")[-1]))


Instruction:
Implement a Python 3 function for
removing a given file from the resource files list.

Response:
def safe_remove_file(filename, app):
    """
    Removes a given resource file from builder resources.
    Needed mostly during test, if multiple sphinx-build are started.
    During these tests js/cass-files are not cleaned, so a css_file from run A is still registered in run B.

    :param filename: filename to remove
    :param app: app object
    :return: None
    """
    data_file = filename
    static_data_file = os.path.join("_static", data_file)

    if data_file.split(".")[-1] == "js":
        if hasattr(app.builder, "script_files") and static_data_file in app.builder.script_files:
            app.builder.script_files.remove(static_data_file)
    elif data_file.split(".")[-1] == "css":
        if hasattr(app.builder, "css_files") and static_data_file in app.builder.css_files:
            app.builder.css_files.remove(static_data_file)


Instruction:
Here you have a function in Python 3, explain what it does
def get_shreds(self, feature_extractors, sheet_name):
        if self._shreds is None:
            shreds = []
            _, contours, _ = cv2.findContours(self._foreground_mask,
                                           cv2.RETR_EXTERNAL,
                                           cv2.CHAIN_APPROX_SIMPLE)
            for i, contour in enumerate(contours):
                shred = self._make_shred(contour, i, feature_extractors,
                                         sheet_name)
                if shred is not None:
                    shreds.append(shred)
            self._shreds = shreds
        return self._shreds

Response: Detects shreds in the current sheet and constructs Shred instances.


Instruction:
Can you generate the documentation for the following Python 3 function
def _make_shred(self, c, name, feature_extractors, sheet_name):
        height, width, channels = self.orig_img.shape

        # bounding rect of currrent contour
        r_x, r_y, r_w, r_h = cv2.boundingRect(c)

        # Generating simplified contour to use it in html
        epsilon = 0.01 * cv2.arcLength(c, True)
        simplified_contour = cv2.approxPolyDP(c, epsilon, True)

        # filter out too small fragments
        if self.px_to_mm(r_w) <= 3 or self.px_to_mm(r_h) <= 3:
            print("Skipping piece #%s as too small (%spx x %s px)" % (
                name, r_w, r_h))
            return None

        if self.px_to_mm(r_w) >= 100 and self.px_to_mm(r_h) >= 100:
            print("Skipping piece #%s as too big (%spx x %s px)" % (
                name, r_w, r_h))
            return None

        # position of rect of min area.
        # this will provide us angle to straighten image
        box_center, bbox, angle = cv2.minAreaRect(c)

        # We want our pieces to be "vertical"
        if bbox[0] > bbox[1]:
            angle += 90
            bbox = (bbox[1], bbox[0])

        if bbox[1] / float(bbox[0]) > 70:
            print("Skipping piece #%s as too too long and narrow" % name)
            return None

        # Coords of region of interest using which we should crop piece after
        # rotation
        y1 = math.floor(box_center[1] - bbox[1] / 2)
        x1 = math.floor(box_center[0] - bbox[0] / 2)
        bbox = tuple(map(int, map(math.ceil, bbox)))

        # A mask we use to show only piece we are currently working on
        piece_mask = np.zeros([height, width, 1], dtype=np.uint8)
        cv2.drawContours(piece_mask, [c], -1, 255, cv2.FILLED)

        # apply mask to original image
        img_crp = self.orig_img[r_y:r_y + r_h, r_x:r_x + r_w]
        piece_in_context = self.save_image(
            "pieces/%s_ctx" % name,
            self.orig_img[max(r_y - 10, 0):r_y + r_h + 10,
                          max(r_x - 10, 0):r_x + r_w + 10])

        mask = piece_mask[r_y:r_y + r_h, r_x:r_x + r_w]
        img_roi = cv2.bitwise_and(img_crp, img_crp, mask=mask)

        # Add alpha layer and set it to the mask
        img_roi = cv2.cvtColor(img_roi, cv2.COLOR_BGR2BGRA)
        img_roi[:, :, 3] = mask[:, :, 0]

        # Straighten it
        # Because we crop original image before rotation we save us some memory
        # and a lot of time but we need to adjust coords of the center of
        # new min area rect
        M = cv2.getRotationMatrix2D((box_center[0] - r_x,
                                     box_center[1] - r_y), angle, 1)

        # And translate an image a bit to make it fit to the bbox again.
        # This is done with direct editing of the transform matrix.
        # (Wooohoo, I know matrix-fu)
        M[0][2] += r_x - x1
        M[1][2] += r_y - y1

        # Apply rotation/transform/crop
        img_roi = cv2.warpAffine(img_roi, M, bbox)
        piece_fname = self.save_image("pieces/%s" % name, img_roi, "png")

        # FEATURES MAGIC BELOW
        #
        # Get our mask/contour back after the trasnform
        _, _, _, mask = cv2.split(img_roi)

        _, contours, _ = cv2.findContours(mask.copy(), cv2.RETR_TREE,
                                          cv2.CHAIN_APPROX_SIMPLE)

        if len(contours) != 1:
            print("Piece #%s has strange contours after transform" % name)

        cnt = contours[0]

        features_fname = self.save_image("pieces/%s_mask" % name, mask, "png")

        base_features = {
            # On_sheet_* features describe the min counding box on the sheet.
            "on_sheet_x": r_x,
            "on_sheet_y": r_y,
            "on_sheet_width": r_w,
            "on_sheet_height": r_h,
            "on_sheet_angle": angle,
            "width": img_roi.shape[1],
            "height": img_roi.shape[0],
        }

        tags_suggestions = []
        for feat in feature_extractors:
            fts, tags = feat.get_info(img_roi, cnt, name)
            base_features.update(fts)
            tags_suggestions += tags

        if tags_suggestions:
            print(name, tags_suggestions)

        return Shred(
            contour=c,
            features=base_features,
            features_fname=features_fname,
            img_roi=img_roi,
            name=name,
            piece_fname=piece_fname,
            piece_in_context_fname=piece_in_context,
            sheet=sheet_name,
            simplified_contour=simplified_contour,
            tags_suggestions=tags_suggestions,
        )

Response: Creates a Shred instance from a given contour.


Instruction:
Write a Python 3 function for
returning a WKT string for a POLYGON with given amount of sides and center and distance.

Response:
def geo_shape(self, sides=5, center=None, distance=None):
        """
        Return a WKT string for a POLYGON with given amount of sides.
        The polygon is defined by its center (random point if not provided) and
        the distance (random distance if not provided; in km) of the points to
        its center.
        """
        assert isinstance(sides, int)

        if distance is None:
            distance = self.random_int(100, 1000)
        else:
            # 6371 => earth radius in km
            # assert that shape radius is maximum half of earth's circumference
            assert isinstance(distance, int)
            assert distance <= EARTH_RADIUS * math.pi, \
                'distance must not be greater than half of earth\'s circumference'

        if center is None:
            # required minimal spherical distance from north/southpole
            dp = distance * 180.0 / EARTH_RADIUS / math.pi
            center = self.geo_point(lat_min=-90.0 + dp, lat_max=90.0 - dp)
        else:
            assert -180.0 <= center[0] <= 180.0, 'Longitude out of bounds'
            assert -90.0 <= center[1] <= 90.0, 'Latitude out of bounds'

        angles = list(self.random_sample(range(360), sides))
        angles.sort()
        points = [_dest_point(center, distance, bearing, EARTH_RADIUS) for bearing in angles]
        # close polygon
        points.append(points[0])

        path = ', '.join([' '.join(p) for p in ([str(lon), str(lat)] for lon, lat in points)])
        return f'POLYGON (( {path} ))'


Instruction:
Can you tell what is the following Python 3 function doing
def run_track(track,
              result_hosts=None,
              crate_root=None,
              output_fmt=None,
              logfile_info=None,
              logfile_result=None,
              failfast=False,
              sample_mode='reservoir'):
    """Execute a track file"""
    with Logger(output_fmt=output_fmt,
                logfile_info=logfile_info,
                logfile_result=logfile_result) as log:
        executor = Executor(
            track_dir=os.path.dirname(track),
            log=log,
            result_hosts=result_hosts,
            crate_root=crate_root,
            fail_fast=failfast,
            sample_mode=sample_mode
        )
        error = executor.execute(toml.load(track))
        if error:
            sys.exit(1)

Response: Execute a Toml file and return a list of result_hosts and logfile_result.


Instruction:
Write a Python 3 function that can
convert a NOAA text file into a lipds file.

Response:
def main(self):
        """
        Convert a NOAA text file into a lipds file.
        CSV files will be created if chronology or data sections are available.
        :return dict: Metadata Dictionary
        """
        logger_noaa_lpd.info("enter main")
        # Run the file through the parser
        # Sets self.metadata, Creates CSVs in dir_tmp
        os.chdir(self.dir_tmp)
        os.mkdir("bag")
        os.chdir(self.dir_root)
        self.__parse()
        os.chdir(self.dir_bag)
        # Dump Metadata JSON to dir_tmp
        write_json_to_file(self.metadata)
        # Create a bagit bag
        finish_bag(self.dir_bag)
        logger_noaa_lpd.info("exit main")
        return


Instruction:
Explain what the following Python 3 code does
def __parse(self):
        logger_noaa_lpd.info("enter parse")

        # Strings
        missing_str = ''
        data_filename = ''

        # Counters
        grant_id = 0
        funding_id = 0
        data_col_ct = 1
        line_num = 0

        # Boolean markers
        description_on = False
        publication_on = False
        abstract_on = False
        site_info_on = False
        chronology_on = False
        chron_vals_on = False
        variables_on = False
        data_vals_on = False
        data_on = False

        # Lists
        lat = []
        lon = []
        elev = []
        pub = []
        funding = []
        temp_abstract = []
        temp_description = []
        data_var_names = []
        data_col_list = []
        data_tables = []

        # All dictionaries needed to create JSON structure
        temp_funding = OrderedDict()
        temp_pub = OrderedDict()
        core_len = OrderedDict()
        geo_properties = OrderedDict()
        chron_dict = OrderedDict()
        data_dict_upper = OrderedDict()
        final_dict = OrderedDict()

        try:
            # Open the text file in read mode. We'll read one line at a time until EOF
            with open(self.filename_txt, 'r') as f:
                logger_noaa_lpd.info("opened noaa file: {}".format(self.filename_txt))
                for line in iter(f):
                    line_num += 1

                    # PUBLICATION
                    # There can be multiple publications. Create a dictionary for each one.
                    if publication_on:
                        # End of the section. Add the dictionary for this one publication to the overall list
                        if '-----' in line:
                            temp_pub = self.__reorganize_doi(temp_pub)
                            pub.append(temp_pub.copy())
                            temp_abstract.clear()
                            temp_pub.clear()
                            publication_on = False
                            logger_noaa_lpd.info("end section: Publication")
                        elif abstract_on:
                            # End of abstract: possibly more variables after.
                            if "#" in line:
                                abstract_on = False
                                temp_pub['abstract'] = ''.join(temp_abstract)
                                logger_noaa_lpd.info("end section: Abstract")
                                line = self.__str_cleanup(line)
                                key, value = self.__slice_key_val(line)
                                temp_pub[self.__camel_case(key)] = value
                            else:
                                temp_abstract.append(self.__str_cleanup(line))

                        # Add all info into the current publication dictionary
                        else:
                            line = self.__str_cleanup(line)
                            key, value = self.__slice_key_val(line)
                            if key in ("Author", "Authors"):
                                temp_pub["author"] = self.__reorganize_authors(value)
                            else:
                                temp_pub[self.__camel_case(key)] = value
                                if key == 'Abstract':
                                    logger_noaa_lpd.info("reading section: Abstract")
                                    abstract_on = True
                                    temp_abstract.append(value)

                    # DESCRIPTION AND NOTES
                    # Descriptions are often long paragraphs spanning multiple lines, but don't follow the key/value format
                    elif description_on:

                        # End of the section. Turn marker off and combine all the lines in the section
                        if '-------' in line:
                            description_on = False
                            value = ''.join(temp_description)
                            final_dict['description'] = value
                            logger_noaa_lpd.info("end section: Description_and_Notes")

                        # The first line in the section. Split into key, value
                        elif 'Description:' in line:
                            key, val = self.__slice_key_val(line)
                            temp_description.append(val)

                        # Keep a running list of all lines in the section
                        else:
                            line = self.__str_cleanup(line)
                            temp_description.append(line)

                    # SITE INFORMATION (Geo)
                    elif site_info_on:

                        if '-------' in line:
                            site_info_on = False
                            logger_noaa_lpd.info("end section: Site_Information")

                        else:
                            line = self.__str_cleanup(line)
                            key, value = self.__slice_key_val(line)
                            if key.lower() in ["northernmost_latitude", "southernmost_latitude"]:
                                lat.append(self.__convert_num(value))

                            elif key.lower() in ["easternmost_longitude", "westernmost_longitude"]:
                                lon.append(self.__convert_num(value))

                            elif key.lower() in ["site_name", "location", "country", "elevation"]:
                                if key.lower() == 'elevation':
                                    val, unit = self.__split_name_unit(value)
                                    elev.append(val)
                                else:
                                    geo_properties[self.__camel_case(key)] = value

                    # CHRONOLOGY
                    elif chronology_on:
                        """
                        HOW IT WORKS:
                        Chronology will be started at "Chronology:" section header
                        Every line starting with a "#" will be ignored
                        The first line without a "#" is considered the variable header line. Variable names are parsed.
                        Each following line will be considered column data and sorted accordingly.
                        Once the "-----" barrier is reached, we exit the chronology section.
                        """
                        # When reaching the end of the chron section, set the marker to off and close the CSV file
                        if '-------' in line:
                            # Turn off markers to exit section
                            chronology_on = False
                            chron_vals_on = False
                            try:
                                # If nothing between the chronology start and the end barrier, then there won't be a CSV
                                if chron_start_line != line_num - 1:
                                    try:
                                        chron_csv.close()
                                        logger_noaa_lpd.info("parse: chronology: no data found in chronology section")
                                    except NameError:
                                        logger_noaa_lpd.debug(
                                            "parse: chronology_on: NameError: chron_csv ref before assignment, {}".format(
                                                self.filename_txt))
                                        print(
                                            "Chronology section is incorrectly formatted. "
                                            "Section data will not be converted")

                                logger_noaa_lpd.info("end section: Chronology")
                            except NameError:
                                logger_noaa_lpd.debug(
                                    "parse: chronology_on: NameError: chron_start_line ref before assignment, {}".format(
                                        self.filename_txt))
                                print("Chronology section is incorrectly formatted. Section data will not be converted")

                        # Data values line. Split, then write to CSV file
                        elif chron_vals_on:
                            values = line.split()
                            try:
                                cw.writerow(values)
                            except NameError:
                                logger_noaa_lpd.debug(
                                    "parse: chronology_on: NameError: csv writer ref before assignment, {}".format(
                                        self.filename_txt))
                                print("Chronology section is incorrectly formatted. Section data will not be converted")

                        else:
                            try:
                                # Chron variable headers line
                                if line and line[0] != "#":
                                    chron_filename = self.dsn + '.chron1.measurementTable1.csv'
                                    # Organize the var header into a dictionary
                                    variables = self.__reorganize_chron_header(line)

                                    # Create a dictionary of info for each column
                                    chron_col_list = self.__create_chron_cols(variables)
                                    chron_dict['filename'] = chron_filename
                                    chron_dict['chronTableName'] = 'Chronology'
                                    chron_dict['columns'] = chron_col_list

                                    # Open CSV for writing
                                    csv_path = os.path.join(self.dir_bag, chron_filename)
                                    chron_csv = open(csv_path, 'w+', newline='')
                                    logger_noaa_lpd.info("opened csv file: {}".format(chron_filename))
                                    cw = csv.writer(chron_csv)
                                    # Turn the marker on to start processing the values columns
                                    chron_vals_on = True
                            except IndexError:
                                logger_noaa_lpd.debug("parse: chronology: IndexError when attempting chron var header")

                    # VARIABLES
                    elif variables_on:
                        """
                        HOW IT WORKS:
                        Variable lines are the only lines that have a "##" in front of them.
                        Ignore all lines that don't match the "##" regex.
                        Once there's a match, start parsing the variable lines, and create a column entry for each line.
                        """
                        process_line = False
                        # End of the section. Turn marker off
                        if "------" in line:
                            variables_on = False
                            logger_noaa_lpd.info("end section: Variables")
                        for item in NOAA_VAR_LINES:
                            if item.lower() in line.lower():
                                process_line = False
                        for item in NOAA_EMPTY:
                            if item == line:
                                process_line = False
                        m = re.match(re_var, line)
                        if m:
                            process_line = True

                        # If the line isn't in the ignore list, then it's a variable line
                        if process_line:

                            # Split the line items, and cleanup
                            cleaned_line = self.__separate_data_vars(line)

                            # Add the items into a column dictionary
                            data_col_dict = self.__create_paleo_col(cleaned_line, data_col_ct)

                            # Keep a list of all variable names
                            try:
                                # Use this list later to cross check with the variable line in the Data section
                                data_var_names.append(data_col_dict['variableName'])
                            except KeyError:
                                data_var_names.append('')
                                logger_noaa_lpd.warn("parse: variables: "
                                                     "KeyError: {} not found in {}".format("variableName", "data_col_dict"))
                            # Add the column dictionary into a final dictionary
                            data_col_list.append(data_col_dict)
                            data_col_ct += 1

                    # DATA
                    # Missing Value, Create data columns, and output Data CSV
                    elif data_on:
                        """
                        HOW IT WORKS:
                        Capture the "Missing Value" entry, if it exists.
                        Data lines should not have a "#" in front of them.
                        The first line without a "#" should be the variable header line
                        All lines that follow should have column data.
                        """
                        # Do not process blank or template lines
                        process_line = True
                        for item in NOAA_DATA_LINES:
                            if item in line:
                                process_line = False
                        for item in NOAA_EMPTY:
                            if item == line:
                                process_line = False
                        for item in ALTS_MV:
                            # Missing value found. Store entry
                            if item in line.lower():
                                process_line = False
                                line = self.__str_cleanup(line)
                                key, missing_str = self.__slice_key_val(line)

                        if process_line:
                            # Split the line at each space (There SHOULD one space between each variable. Not always true)
                            values = line.split()
                            # Write all data values to CSV
                            if data_vals_on:
                                try:
                                    dw.writerow(values)
                                except NameError:
                                    logger_noaa_lpd.debug(
                                        "parse: data_on: NameError: csv writer ref before assignment, {}".format(
                                            self.filename_txt))

                            # Check for the line of variables
                            else:
                                var = self.__str_cleanup(values[0].lstrip())
                                # Check if a variable name is in the current line
                                if var.lower() in line.lower():
                                    data_vals_on = True
                                    logger_noaa_lpd.info("start section: Data_Values")
                                    # Open CSV for writing
                                    data_filename = "{}.paleoData1.measurementTable1.csv".format(self.dsn)
                                    csv_path = os.path.join(self.dir_bag, data_filename)
                                    data_csv = open(csv_path, 'w+', newline='')
                                    logger_noaa_lpd.info("opened csv file: {}".format(data_filename))
                                    dw = csv.writer(data_csv)

                    # METADATA
                    else:
                        # Line Continuation: Sometimes there are items that span a few lines.
                        # If this happens, we want to combine them all properly into one entry.
                        if '#' not in line and line not in NOAA_EMPTY and old_val:
                            if old_key in ('funding', 'agency'):
                                try:
                                    temp_funding[old_key] = old_val + line
                                except KeyError as e:
                                    logger_noaa_lpd.debug(
                                        "parse: metadata: line continuation: {} not found in {}, {}".format(old_key,
                                                                                "temp_funding",
                                                                                e))
                            else:
                                try:
                                    final_dict[old_key] = old_val + line
                                except KeyError as e:
                                    logger_noaa_lpd.debug(
                                        "parse: metadata: line continuation: {} not found in {}, {}".format(old_key,
                                                                                "temp_funding",
                                                                                e))

                        # No Line Continuation: This is the start or a new entry
                        else:
                            line = self.__str_cleanup(line)

                            # Grab the key and value from the current line
                            try:
                                # Split the line into key, value pieces
                                key, value = self.__slice_key_val(line)
                                l_key = key.lower()
                                cc_key= self.__camel_case(key)

                                # If there is no value, then we are at a section header.
                                # Data often has a blank value, so that is a special check.
                                if not value or l_key == 'data':
                                    # Turn on markers if we run into section headers
                                    if l_key == 'description_and_notes':
                                        description_on = True
                                        logger_noaa_lpd.info("reading section: Description_and_Notes")
                                    elif l_key == 'publication':
                                        publication_on = True
                                        logger_noaa_lpd.info("reading section: Publication")
                                    elif l_key == 'site_information':
                                        site_info_on = True
                                        logger_noaa_lpd.info("reading section: Site_Information")
                                    elif l_key == 'chronology':
                                        chronology_on = True
                                        logger_noaa_lpd.info("reading section: Chronology")
                                        chron_start_line = line_num
                                    elif l_key == 'variables':
                                        variables_on = True
                                        logger_noaa_lpd.info("reading section: Variables")
                                    elif l_key == 'data':
                                        data_on = True
                                        logger_noaa_lpd.info("reading section: Data")
                                # For all
                                else:
                                    # Ignore any entries that are specified in the skip list
                                    _ignore = [item.lower() for item in NOAA_KEYS_BY_SECTION["Ignore"]]
                                    if l_key not in _ignore:
                                        # There can be multiple funding agencies and grants. Keep a list of dict entries
                                        _funding = [item.lower() for item in NOAA_KEYS_BY_SECTION["Funding_Agency"]]
                                        if l_key in _funding:
                                            if l_key == 'funding_agency_name':
                                                funding_id += 1
                                                key = 'agency'
                                            elif l_key == 'grant':
                                                grant_id += 1
                                                key = 'grant'
                                            temp_funding[key] = value
                                            # If both counters are matching, we are ready to add content to the funding list
                                            if grant_id == funding_id:
                                                funding.append(temp_funding.copy())
                                                temp_funding.clear()
                                        else:
                                            # There's likely two "Online_Resource"s, and we need both, so check and concat
                                            if cc_key == "onlineResource":
                                                # If it exists, append. If not, add entry as a list
                                                if cc_key in final_dict:
                                                    final_dict[cc_key].append(value)
                                                else:
                                                    final_dict[cc_key] = [value]
                                            else:
                                                final_dict[cc_key] = value
                                        # Keep track of old key in case we have a line continuation
                                        old_key = key
                                        old_val = value.strip()
                            except TypeError as e:
                                logger_noaa_lpd.warn(
                                    "parse: TypeError: none type received from slice_key_val, {}".format(e))

            # Wait to close the data CSV until we reached the end of the text file
            try:
                data_csv.close()
                logger_noaa_lpd.info("end section: Data_Values")
                logger_noaa_lpd.info("end section: Data")
            except NameError as e:
                print("Error: NOAA text file is contains format errors. Unable to process.")
                logger_noaa_lpd.debug(
                    "parse: NameError: failed to close csv, invalid formatting in NOAA txt file, {}".format(e))

            # Piece together measurements block
            logger_noaa_lpd.info("compiling final paleoData")
            data_dict_upper['filename'] = data_filename
            data_dict_upper['paleoDataTableName'] = 'Data'
            data_dict_upper['missingValue'] = missing_str
            data_dict_upper['columns'] = data_col_list
            data_tables.append(data_dict_upper)

            # Piece together geo block
            logger_noaa_lpd.info("compiling final geo")
            geo = self.__create_coordinates(lat, lon, elev)
            geo['properties'] = geo_properties

            # Piece together final dictionary
            logger_noaa_lpd.info("compiling final master")
            final_dict['pub'] = pub
            final_dict['funding'] = funding
            final_dict['geo'] = geo
            final_dict['coreLength'] = core_len
            final_dict['chronData'] = [{"chronMeasurementTable": chron_dict}]
            final_dict['paleoData'] = data_tables
            self.metadata = final_dict
            logger_noaa_lpd.info("final dictionary compiled")

            # Start cleaning up the metadata
            logger_noaa_lpd.info("removing empty fields")
            self.metadata = rm_empty_fields(self.metadata)
            logger_noaa_lpd.info("removing empty doi")
            self.metadata = rm_empty_doi(self.metadata)
            logger_noaa_lpd.info("removing irrelevant keys")
            self.__remove_irr_fields()

        except Exception as e:
            logger_noaa_lpd.debug("parse: {}".format(e))

        logger_noaa_lpd.info("exit parse")
        return

Response: Parse the noaa file and return a compiled dictionary


Instruction:
Make a summary of the following Python 3 code
def __create_paleo_col(l, col_count):
        # Format: what, material, error, units, seasonality, archive, detail, method,
        # C or N for Character or Numeric data, direction of relation to climate (positive or negative)
        d = OrderedDict()
        d['number'] = col_count
        for idx, var_name in enumerate(NOAA_KEYS_BY_SECTION["Variables"]):
            try:
                value = l[idx]
                # These two cases are nested in the column, so treat them special
                if var_name == "seasonality":
                    d["climateInterpretation"] = {var_name: value}
                elif var_name == "uncertainty":
                    d["calibration"] = {var_name: value}
                # All other cases are root items in the column, so add normally
                else:
                    d[var_name] = value
            except IndexError as e:
                logger_noaa_lpd.debug("create_var_col: IndexError: var: {}, {}".format(var_name, e))
        return d

Response: Create a column of paleo data.


Instruction:
Can you write a function in Python 3 where it
converts a number to a float.

Response:
def __convert_num(number):
        """
        All path items are automatically strings. If you think it's an int or float, this attempts to convert it.
        :param str number:
        :return float or str:
        """
        try:
            return float(number)
        except ValueError as e:
            logger_noaa_lpd.warn("convert_num: ValueError: {}".format(e))
            return number


Instruction:
Given the following Python 3 function, write the documentation
def __camel_case(word):
        word = word.lower()
        if '_' in word:
            split_word = word.split('_')
        else:
            split_word = word.split()
        if len(split_word) > 0:
            for i, word in enumerate(split_word):
                if i > 0:
                    split_word[i] = word.title()
        strings = ''.join(split_word)
        return strings

Response: Convert underscore naming into camel case naming


Instruction:
Here you have a function in Python 3, explain what it does
def __name_unit_regex(word):
        value = ""
        unit = ""
        r = re.findall(re_name_unit, word)
        try:
            value = r[0][0]
        except IndexError as e:
            logger_noaa_lpd.warn("name_unit_regex: IndexError: value: {}, {}, {}".format(word, r, e))
        try:
            unit = r[0][1]
            # Replace unit with correct synonym.
            if unit.lower() in UNITS:
                unit = UNITS[unit]
        except IndexError as e:
            logger_noaa_lpd.warn("name_unit_regex: IndexError: unit: {}, {}, {}".format(word, r, e))
        if value:
            try:
                value = float(value)
            except ValueError as e:
                logger_noaa_lpd.warn("name_unit_regex: ValueError: val: {}, {}".format(value, e))
        return value, unit

Response: Split a name and unit that are bunched together ( i. e. 250m


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __split_name_unit(self, line):
        vals = []
        unit = ''
        if line != '' or line != ' ':
            # If there are parenthesis, remove them
            line = line.replace('(', '').replace(')', '')
            # When value and units are a range (i.e. '100 m - 200 m').
            if re.match(re_name_unit_range, line):
                m = re.findall(re_name_unit_range, line)
                if m:
                    for group in m:
                        for item in group:
                            try:
                                val = float(item)
                                vals.append(val)
                            except ValueError:
                                if item:
                                    unit = item
                # Piece the number range back together.
                if len(vals) == 1:
                    value = vals[0]
                else:
                    value = str(vals[0]) + ' to ' + str(vals[1])
            else:
                value, unit = self.__name_unit_regex(line)
            return value, unit

Response: Split a string that has value and unit as one.


Instruction:
Can you write a function in Python 3 where it
removes unnecessary characters in the line that we don t want

Response:
def __str_cleanup(line):
        """
        Remove the unnecessary characters in the line that we don't want
        :param str line:
        :return str:
        """
        if '#' in line:
            line = line.replace("#", "")
            line = line.strip()
        if '-----------' in line:
            line = ''
        return line


Instruction:
Implement a Python 3 function for
slicing the key and value items from a line.

Response:
def __slice_key_val(line):
        """
        Get the key and value items from a line by looking for and lines that have a ":"
        :param str line:
        :return str str: Key, Value
        """
        position = line.find(":")
        # If value is -1, that means the item was not found in the string.
        if position != -1:
            key = line[:position]
            value = line[position + 1:]
            value = value.lstrip()
            return key, value
        else:
            key = line
            value = None
            return key, value


Instruction:
Explain what the following Python 3 code does
def __create_coordinates(self, lat, lon, elev):
        # Sort lat an lon in numerical order
        lat.sort()
        lon.sort()
        geo_dict = {}
        # 4 coordinate values
        if len(lat) == 2 and len(lon) == 2:
            # Matching coordinate pairs. Not 4 unique values.
            if lat[0] == lat[1] and lon[0] == lon[1]:
                logger_noaa_lpd.info("coordinates found: {}".format("2"))
                lat.pop()
                lon.pop()
                geo_dict = self.__geo_point(lat, lon, elev)
            # 4 unique coordinates
            else:
                logger_noaa_lpd.info("coordinates found: {}".format("4"))
                geo_dict = self.__geo_multipoint(lat, lon, elev)
        # 2 coordinate values
        elif len(lat) == 1 and len(lon) == 1:
            logger_noaa_lpd.info("coordinates found: {}".format("2"))
            geo_dict = self.__geo_point(lat, lon, elev)
        # 0 coordinate values
        elif not lat and not lon:
            logger_noaa_lpd.info("coordinates found: {}".format("0"))
        else:
            geo_dict = {}
            logger_noaa_lpd.info("coordinates found: {}".format("too many"))
        return geo_dict

Response: Create the coordinates for the current object.


Instruction:
Explain what the following Python 3 code does
def __geo_multipoint(lat, lon, elev):
        logger_noaa_lpd.info("enter geo_multipoint")
        geo_dict = OrderedDict()
        geometry_dict = OrderedDict()
        coordinates = []
        # bbox = []
        temp = [None, None]
        # 4 unique values
        # # Creates bounding box
        # for index, point in enumerate(lat):
        #     bbox.append(lat[index])
        #     bbox.append(lon[index])

        # Creates coordinates list
        for i in lat:
            temp[0] = i
            for j in lon:
                temp[1] = j
                coordinates.append(copy.copy(temp))
        if elev:
            coordinates = coordinates + elev
        # Create geometry block
        geometry_dict['type'] = 'Polygon'
        geometry_dict['coordinates'] = coordinates
        # Create geo block
        geo_dict['type'] = 'Feature'
        # geo_dict['bbox'] = bbox
        geo_dict['geometry'] = geometry_dict

        return geo_dict

Response: Create a geoJson MultiPoint - type dictionary


Instruction:
Explain what the following Python 3 code does
def __geo_point(lat, lon, elev):
        logger_noaa_lpd.info("enter geo_point")
        coordinates = []
        geo_dict = OrderedDict()
        geometry_dict = OrderedDict()
        for index, point in enumerate(lat):
            coordinates.append(lat[index])
            coordinates.append(lon[index])
        if elev:
            coordinates = coordinates + elev
        geometry_dict['type'] = 'Point'
        geometry_dict['coordinates'] = coordinates
        geo_dict['type'] = 'Feature'
        geo_dict['geometry'] = geometry_dict
        return geo_dict

Response: Create a geoJson Point - type dictionary


Instruction:
Given the following Python 3 function, write the documentation
def __reorganize_doi(temp_pub):
        doi_out = ""
        doi_list = []
        rm = ["doiId", "doi", "doiUrl"]
        # Check if both entries exist
        if "doi" and "doiId" in temp_pub:
            if temp_pub["doi"] == temp_pub["doiId"]:
                # If the entries are the same, then pick one to use
                doi_out = temp_pub["doiId"]
            else:
                # If entries are not the same, check if it matches the regex pattern.
                if re_doi.findall(temp_pub["doiId"]):
                    doi_out = temp_pub["doiId"]
                # If it doesnt match the regex, just use the "doi" entry as-is
                else:
                    doi_out = temp_pub["doi"]
        # Check if "doiId" entry exists. Most common entry.
        elif "doiId" in temp_pub:
            doi_out = temp_pub["doiId"]
        # Check if "doi" entry exists. Fallback.
        elif "doi" in temp_pub:
            doi_out = temp_pub["doi"]
        # It's unlikely that ONLY the doi_url exists, but check if no doi found so far
        elif "doiUrl" in temp_pub:
            doi_out = temp_pub["doiUrl"]

        # Log if the DOI is invalid or not.
        if not re_doi.match(doi_out):
            logger_noaa_lpd.info("reorganize_doi: invalid doi input from NOAA file")

        # Get a list of all DOIs found in the string
        matches = re.findall(re_doi, doi_out)
        logger_noaa_lpd.info("reorganize_dois: found {} dois: {}".format(len(matches), doi_out))

        # Add identifier block to publication dictionary
        for doi in matches:
            doi_list.append({"type": "doi", "id": doi})

        # Remove the other DOI entries
        for k in rm:
            try:
                del temp_pub[k]
            except KeyError:
                # If there's a KeyError, don't worry about it. It's likely that only one of these keys will be present.
                pass

        temp_pub["identifier"] = doi_list
        return temp_pub

Response: Reorganize the DOI from a dictionary.


Instruction:
Given the following Python 3 function, write the documentation
def __create_chron_cols(metadata):
        chron_col_list = []
        chron_col_ct = 1
        for variableName, unit in metadata.items():
            temp_dict = OrderedDict()
            temp_dict['column'] = chron_col_ct
            temp_dict['variableName'] = variableName
            temp_dict['unit'] = unit
            chron_col_list.append(copy.deepcopy(temp_dict))
            chron_col_ct += 1

        return chron_col_list

Response: Create the chron columns from the chron metadata.


Instruction:
Can you generate the documentation for the following Python 3 function
def __reorganize_chron_header(line):
        d = {}
        # Header variables should be tab-delimited. Use regex to split by tabs
        m = re.split(re_tab_split, line)
        # If there was an output match from the line, then keep going
        if m:
            # Loop once for each variable in the line
            for s in m:
                # Match the variable to the 'variable (units)' regex to look for units
                m2 = re.match(re_var_w_units, s)
                # If there was a match
                if m2:
                    # If no units were found, set to blank
                    if m2.group(2) is None:
                        d[m2.group(1)] = ""
                    # Units were found
                    else:
                        # Set both values
                        d[m2.group(1)] = m2.group(2)
        return d

Response: Reorganize the list of variables given in the chron header.


Instruction:
Explain what the following Python 3 code does
def __reorganize_authors(authors):
        # String SHOULD be semi-colon separated names.
        l = []
        s = authors.split(";")
        for author in s:
            try:
                l.append({"name": author.strip()})
            except AttributeError:
                logger_noaa_lpd.warning("reorganize_authors: AttributeError: authors incorrectly formatted")
        return l

Response: Separate the string of authors and put it into a BibJSON compliant list


Instruction:
Explain what the following Python 3 code does
def get_lipd_version(L):
    version = 1.0
    _keys = ["LipdVersion", "LiPDVersion", "lipdVersion", "liPDVersion"]
    for _key in _keys:
        if _key in L:
            version = L[_key]
            # Cast the version number to a float
            try:
                version = float(version)
            except AttributeError:
                # If the casting failed, then something is wrong with the key so assume version is 1.0
                version = 1.0
            L.pop(_key)
    return L, version

Response: Get the LiPD version from the metadata dictionary L.


Instruction:
Implement a function in Python 3 to
update the metadata of a single lipd version.

Response:
def update_lipd_version(L):
    """
    Metadata is indexed by number at this step.

    Use the current version number to determine where to start updating from. Use "chain versioning" to make it
    modular. If a file is a few versions behind, convert to EACH version until reaching current. If a file is one
    version behind, it will only convert once to the newest.
    :param dict L: Metadata
    :return dict d: Metadata
    """
    # Get the lipd version number.
    L, version = get_lipd_version(L)

    # Update from (N/A or 1.0) to 1.1
    if version in [1.0, "1.0"]:
        L = update_lipd_v1_1(L)
        version = 1.1

    # Update from 1.1 to 1.2
    if version in [1.1, "1.1"]:
        L = update_lipd_v1_2(L)
        version = 1.2
    if version in [1.2, "1.2"]:
        L = update_lipd_v1_3(L)
        version = 1.3

    L = fix_doi(L)
    L["lipdVersion"] = 1.3
    return L


Instruction:
Can you create a Python 3 function that
updates LiPD v1. 1 to v1. 0

Response:
def update_lipd_v1_1(d):
    """
    Update LiPD v1.0 to v1.1
    - chronData entry is a list that allows multiple tables
    - paleoData entry is a list that allows multiple tables
    - chronData now allows measurement, model, summary, modelTable, ensemble, calibratedAges tables
    - Added 'lipdVersion' key

    :param dict d: Metadata v1.0
    :return dict d: Metadata v1.1
    """
    logger_versions.info("enter update_lipd_v1_1")
    tmp_all = []

    try:
        # ChronData is the only structure update
        if "chronData" in d:
            # As of v1.1, ChronData should have an extra level of abstraction.
            # No longer shares the same structure of paleoData

            # If no measurement table, then make a measurement table list with the table as the entry
            for table in d["chronData"]:
                if "chronMeasurementTable" not in table:
                    tmp_all.append({"chronMeasurementTable": [table]})

                # If the table exists, but it is a dictionary, then turn it into a list with one entry
                elif "chronMeasurementTable" in table:
                    if isinstance(table["chronMeasurementTable"], dict):
                        tmp_all.append({"chronMeasurementTable": [table["chronMeasurementTable"]]})
            if tmp_all:
                d["chronData"] = tmp_all

        # Log that this is now a v1.1 structured file
        d["lipdVersion"] = 1.1
    except Exception as e:
        logger_versions.error("update_lipd_v1_1: Exception: {}".format(e))
    logger_versions.info("exit update_lipd_v1_1")
    return d


Instruction:
Can you tell what is the following Python 3 function doing
def update_lipd_v1_3(d):
    # sub routine (recursive): changes all the key names and merges interpretation
    d = update_lipd_v1_3_names(d)
    # sub routine: changes ensemble and summary table structure
    d = update_lipd_v1_3_structure(d)
    d["lipdVersion"] = 1.3
    if "LiPDVersion" in d:
        del d["LiPDVersion"]
    return d

Response: Update LiPD v1. 2 to v1. 3.


Instruction:
Explain what the following Python 3 code does
def update_lipd_v1_3_names(d):
    try:
        if isinstance(d, dict):
            for k, v in d.items():
                # dive down for dictionaries
                d[k] = update_lipd_v1_3_names(v)

                # see if the key is in the remove list
                if k in VER_1_3["swap"]:
                    # replace the key in the dictionary
                    _key_swap = VER_1_3["swap"][k]
                    d[_key_swap] = d.pop(k)
                elif k in VER_1_3["tables"]:
                    d[k] = ""
            for _key in ["climateInterpretation", "isotopeInterpretation"]:
                if _key in d:
                    d = _merge_interpretations(d)

        elif isinstance(d, list):
            # dive down for lists
            for idx, i in enumerate(d):
                d[idx] = update_lipd_v1_3_names(i)
    except Exception as e:
        print("Error: Unable to update file to LiPD v1.3: {}".format(e))
        logger_versions.error("update_lipd_v1_3_names: Exception: {}".format(e))
    return d

Response: Update the key names and merge interpretations in a list of LiPD v1. 3.


Instruction:
Can you tell what is the following Python 3 function doing
def update_lipd_v1_3_structure(d):
    for key in ["paleoData", "chronData"]:
        if key in d:
            for entry1 in d[key]:
                if "model" in entry1:
                    for entry2 in entry1["model"]:
                        for key_table in ["summaryTable", "ensembleTable"]:
                            if key_table in entry2:
                                if isinstance(entry2[key_table], dict):
                                    try:
                                        _tmp = entry2[key_table]
                                        entry2[key_table] = []
                                        entry2[key_table].append(_tmp)
                                    except Exception as e:
                                        logger_versions.error("update_lipd_v1_3_structure: Exception: {}".format(e))
    return d

Response: Update the structure for summary and ensemble tables


Instruction:
Implement a Python 3 function for
merging CSV files into a metadata dictionary.

Response:
def merge_csv_metadata(d, csvs):
    """
    Using the given metadata dictionary, retrieve CSV data from CSV files, and insert the CSV
    values into their respective metadata columns. Checks for both paleoData and chronData tables.

    :param dict d: Metadata
    :return dict: Modified metadata dictionary
    """
    logger_csvs.info("enter merge_csv_metadata")

    # Add CSV to paleoData
    if "paleoData" in d:
        d["paleoData"] = _merge_csv_section(d["paleoData"], "paleo", csvs)

    # Add CSV to chronData
    if "chronData" in d:
        d["chronData"] = _merge_csv_section(d["chronData"], "chron", csvs)

    logger_csvs.info("exit merge_csv_metadata")
    return d


Instruction:
Explain what the following Python 3 code does
def _merge_csv_section(sections, pc, csvs):
    logger_csvs.info("enter merge_csv_section")

    try:
        # Loop through each table_data in paleoData
        for _name, _section in sections.items():

            if "measurementTable" in _section:
                sections[_name]["measurementTable"] = _merge_csv_table(_section["measurementTable"], pc, csvs)

            if "model" in _section:
                sections[_name]["model"] = _merge_csv_model(_section["model"], pc, csvs)

    except Exception as e:
        print("Error: There was an error merging CSV data into the metadata ")
        logger_csvs.error("merge_csv_section: {}".format(e))

    logger_csvs.info("exit merge_csv_section")
    return sections

Response: Add csv data to all paleo data tables and model data into the metadata tables


Instruction:
Write a Python 3 function for
adding csv data to each column in chron model

Response:
def _merge_csv_model(models, pc, csvs):
    """
    Add csv data to each column in chron model

    :param dict models: Metadata
    :return dict models: Metadata
    """
    logger_csvs.info("enter merge_csv_model")

    try:
        for _name, _model in models.items():

            if "summaryTable" in _model:
                models[_name]["summaryTable"] = _merge_csv_table(_model["summaryTable"], pc, csvs)

            if "ensembleTable" in _model:
                models[_name]["ensembleTable"] = _merge_csv_table(_model["ensembleTable"], pc, csvs)

            if "distributionTable" in _model:
                models[_name]["distributionTable"] = _merge_csv_table(_model["distributionTable"], pc, csvs)

    except Exception as e:
        logger_csvs.error("merge_csv_model: {}",format(e))

    logger_csvs.info("exit merge_csv_model")
    return models


Instruction:
Here you have a function in Python 3, explain what it does
def _merge_csv_column(table, csvs):

    # Start putting CSV data into corresponding column "values" key
    try:
        ensemble = is_ensemble(table["columns"])
        if ensemble:
            # realization columns
            if len(table["columns"]) == 1:
                for _name, _column in table["columns"].items():
                    _column["values"] = csvs
            # depth column + realization columns
            elif len(table["columns"]) == 2:
                _multi_column = False
                for _name, _column in table["columns"].items():
                    if isinstance(_column["number"], (int, float)):
                        col_num = cast_int(_column["number"])
                        _column['values'] = csvs[col_num - 1]
                    elif isinstance(_column["number"], list):
                        if _multi_column:
                            raise Exception("Error: merge_csv_column: This jsonld metadata looks wrong!\n"
                                  "\tAn ensemble table depth should not reference multiple columns of CSV data.\n"
                                  "\tPlease manually fix the ensemble columns in 'metadata.jsonld' inside of your LiPD file.")
                        else:
                            _multi_column = True
                            _column["values"] = csvs[2:]
        else:
            for _name, _column in table['columns'].items():
                col_num = cast_int(_column["number"])
                _column['values'] = csvs[col_num - 1]
    except IndexError:
        logger_csvs.warning("merge_csv_column: IndexError: index out of range of csv_data list")
    except KeyError:
        logger_csvs.error("merge_csv_column: KeyError: missing columns key")
    except Exception as e:
        logger_csvs.error("merge_csv_column: Unknown Error:  {}".format(e))
        print("Quitting...")
        exit(1)

    # We want to keep one missing value ONLY at the table level. Remove MVs if they're still in column-level
    return table, ensemble

Response: Add csv data to each column in a list of columns.


Instruction:
Can you create a Python 3 function that
reads the target CSV file and creates a dictionary with one list for each CSV column.

Response:
def read_csv_from_file(filename):
    """
    Opens the target CSV file and creates a dictionary with one list for each CSV column.

    :param str filename:
    :return list of lists: column values
    """
    logger_csvs.info("enter read_csv_from_file")
    d = {}
    l = []
    try:
        logger_csvs.info("open file: {}".format(filename))
        with open(filename, 'r') as f:
            r = csv.reader(f, delimiter=',')

            # Create a dict with X lists corresponding to X columns
            for idx, col in enumerate(next(r)):
                d[idx] = []
                d = cast_values_csvs(d, idx, col)

            # Start iter through CSV data
            for row in r:
                for idx, col in enumerate(row):
                    # Append the cell to the correct column list
                    d = cast_values_csvs(d, idx, col)

        # Make a list of lists out of the dictionary instead
        for idx, col in d.items():
            l.append(col)
    except FileNotFoundError as e:
        print('CSV FileNotFound: ' + filename)
        logger_csvs.warn("read_csv_to_columns: FileNotFound: {}, {}".format(filename, e))
    logger_csvs.info("exit read_csv_from_file")
    return l


Instruction:
Implement a function in Python 3 to
write columns of data to a target CSV file.

Response:
def write_csv_to_file(d):
    """
    Writes columns of data to a target CSV file.

    :param dict d: A dictionary containing one list for every data column. Keys: int, Values: list
    :return None:
    """
    logger_csvs.info("enter write_csv_to_file")

    try:
        for filename, data in d.items():
            try:
                l_columns = _reorder_csv(data, filename)
                rows = zip(*l_columns)
                with open(filename, 'w+') as f:
                    w = csv.writer(f)
                    for row in rows:
                        row2 = decimal_precision(row)
                        w.writerow(row2)
            except TypeError as e:
                print("Error: Unable to write values to CSV file, {}:\n"
                      "(1) The data table may have 2 or more identical variables. Please correct the LiPD file manually\n"
                      "(2) There may have been an error trying to prep the values for file write. The 'number' field in the data columns may be a 'string' instead of an 'integer' data type".format(filename))
                print(e)
            except Exception as e:
                print("Error: CSV file not written, {}, {}:\n"
                      "The data table may have 2 or more identical variables. Please correct the LiPD file manually".format(filename, e))
    except AttributeError as e:
        logger_csvs.error("write_csv_to_file: Unable to write CSV File: {}".format(e, exc_info=True))
    logger_csvs.info("exit write_csv_to_file")
    return


Instruction:
Explain what the following Python 3 code does
def get_csv_from_metadata(dsn, d):
    logger_csvs.info("enter get_csv_from_metadata")
    _csvs = OrderedDict()
    _d = copy.deepcopy(d)

    try:
        if "paleoData" in _d:
            # Process paleoData section
            _d["paleoData"], _csvs = _get_csv_from_section(_d["paleoData"], "{}.paleo".format(dsn), _csvs)

        if "chronData" in _d:
            _d["chronData"], _csvs = _get_csv_from_section(_d["chronData"], "{}.chron".format(dsn), _csvs)

    except Exception as e:
        print("Error: get_csv_from_metadata: {}, {}".format(dsn, e))
        logger_csvs.error("get_csv_from_metadata: {}, {}".format(dsn, e))

    logger_csvs.info("exit get_csv_from_metadata")
    return _d, _csvs

Response: Get all csv from metadata and return new metadata with generated filenames to match files.


Instruction:
How would you explain what the following Python 3 function does
def _get_csv_from_section(sections, crumbs, csvs):
    logger_csvs.info("enter get_csv_from_section: {}".format(crumbs))
    _idx = 0
    try:
        # Process the tables in section
        for _name, _section in sections.items():

            # Process each entry sub-table below if they exist
            if "measurementTable" in _section:
                sections[_name]["measurementTable"], csvs = _get_csv_from_table(_section["measurementTable"],"{}{}{}".format(crumbs, _idx, "measurement") , csvs)
            if "model" in _section:
                sections[_name]["model"], csvs = _get_csv_from_model(_section["model"], "{}{}{}".format(crumbs, _idx, "model") , csvs)
            _idx += 1

    except Exception as e:
        logger_csvs.error("get_csv_from_section: {}, {}".format(crumbs, e))
        print("Error: get_csv_from_section: {}, {}".format(crumbs, e))

    logger_csvs.info("exit get_csv_from_section: {}".format(crumbs))
    return sections, csvs

Response: Get the CSVs from the metadata and the measurements and models in the main section.


Instruction:
Can you write a function in Python 3 where it
gets csv from model data

Response:
def _get_csv_from_model(models, crumbs, csvs):
    """
    Get csv from model data

    :param dict models: Metadata
    :param str crumbs: Crumbs
    :param dict csvs: Csv
    :return dict models: Metadata
    :return dict csvs: Csv
    """
    logger_csvs.info("enter get_csv_from_model: {}".format(crumbs))
    _idx = 0
    try:
        for _name, _model in models.items():
            if "distributionTable" in _model:
                models[_name]["distributionTable"], csvs = _get_csv_from_table(_model["distributionTable"], "{}{}{}".format(crumbs, _idx, "distribution"), csvs)

            if "summaryTable" in _model:
                models[_name]["summaryTable"], csvs = _get_csv_from_table(_model["summaryTable"], "{}{}{}".format(crumbs, _idx, "summary"), csvs)

            if "ensembleTable" in _model:
                models[_name]["ensembleTable"], csvs = _get_csv_from_table(_model["ensembleTable"], "{}{}{}".format(crumbs, _idx, "ensemble"), csvs)
            _idx += 1
    except Exception as e:
        print("Error: get_csv_from_model: {}, {}".format(crumbs, e))
        logger_csvs.error("Error: get_csv_from_model: {}, {}".format(crumbs, e))
    return models, csvs


Instruction:
Explain what the following Python 3 code does
def _get_csv_from_columns(table, filename, csvs):
    csvs[filename] = OrderedDict()
    try:
        if "columns" in table:
            try:
                for _name, _column in table["columns"].items():
                    csvs[filename][_name] = {"number": _column["number"], "values": _column["values"]}
            except KeyError as ke:
                print("Error: get_csv_from_columns: {}, {}".format(filename, ke))
            except Exception as e:
                print("Error: get_csv_from_columns: inner: {}, {}".format(filename, e))
                logger_csvs.error("get_csv_from_columns: inner: {}, {}".format(filename, e))
    except Exception as e:
        print("Error: get_csv_from_columns: {}, {}".format(filename, e))
        logger_csvs.error("get_csv_from_columns: {}, {}".format(filename, e))

    return csvs

Response: Search a data tables for column values. Return a dict of column values. ref by var name


Instruction:
Write a Python 3 script for
getting the filename from a data table.

Response:
def _get_filename(table):
    """
    Get the filename from a data table. If it doesn't exist, create a new one based on table hierarchy in metadata file.
    format: <dataSetName>.<section><idx><table><idx>.csv
    example: ODP1098B.Chron1.ChronMeasurementTable.csv

    :param dict table: Table data
    :param str crumbs: Crumbs
    :return str filename: Filename
    """
    try:
        filename = table["filename"]
    except KeyError:
        logger_csvs.info("get_filename: KeyError: missing filename for a table")
        print("Error: Missing filename for a table")
        filename = ""
    except Exception as e:
        logger_csvs.error("get_filename: {}".format(e))
        filename = ""
    return filename


Instruction:
Can you generate the documentation for the following Python 3 function
def _reorder_csv(d, filename=""):
    _ensemble = is_ensemble(d)
    _d2 = []
    try:
        if _ensemble:
            # 1 column ensemble: realizations
            if len(d) == 1:
                for var, data in d.items():
                    if "values" in data:
                        _d2 = data["values"]
            # 2 column ensemble: depth and realizations
            else:
                _count = 0
                # count up how many columns total, and how many placeholders to make in our list
                for var, data in d.items():
                    if isinstance(data["number"], list):
                        _curr_count = len(data["number"])
                        _count += _curr_count
                    elif isinstance(data["number"], (int, float, str)):
                        _count += 1
                # make a list with X number of placeholders
                _d2 = [None for i in range(0, _count)]
                # Loop again and start combining all columns into one list of lists
                for var, data in d.items():
                    # realizations: insert at (hopefully) index 1,2...1001
                    if isinstance(data["number"], list):
                        for idx, number in enumerate(data["number"]):
                            # we can't trust the number entries. sometimes they start at "number 1",
                            # which isn't true, because DEPTH is number 1. Use enumerate index instead.
                            _insert_at = int(idx) + 1
                            # Insert at one above the index. Grab values at exact index
                            _d2[_insert_at] = data["values"][idx-1]

                    # depth column: insert at (hopefully) index 0
                    else:
                        # we can trust to use the number entry as an index placement
                        _insert_at = int(data["number"]) - 1
                        # insert at one below number, to compensate for 0-index
                        _d2[_insert_at] = data["values"]
        else:
            _count = len(d)
            _d2 = [None for i in range(0, _count)]
            for key, data in d.items():
                _insert_at = int(data["number"]) - 1
                _d2[_insert_at] = data["values"]

    except Exception as e:
        print("Error: Unable to write CSV: There was an error trying to prep the values for file write: {}".format(e))
        logger_csvs.error("reorder_csvs: Unable to write CSV file: {}, {}".format(filename, e))
    return _d2

Response: Reorder the CSV file d in JSONLD alphabetical order.


Instruction:
How would you code a function in Python 3 to
check if the data contains numeric lists.

Response:
def _is_numeric_data(ll):
    """
    List of lists of csv values data
    :param list ll:
    :return bool: True, all lists are numeric lists, False, data contains at least one numeric list.
    """
    for l in ll:
        try:
            if any(math.isnan(float(i)) or isinstance(i, str) for i in l):
                return False
            # if not all(isinstance(i, (int, float)) or math.isnan(float(i)) for i in l):
            #     # There is an entry that is a non-numeric entry in this list
            #     return False
        except ValueError:
            # Trying to case a str as a float didnt work, and we got an error
            return False
    # All arrays are 100% numeric or "nan" entries.
    return True


Instruction:
Can you generate the documentation for the following Python 3 function
def get_validator_format(L):
    _api_data = []

    _j, _csvs = get_csv_from_metadata(L["dataSetName"], L)
    _j = rm_values_fields(copy.deepcopy(L))
    _j = idx_name_to_num(_j)

    # All the filenames being processed
    _filenames = ["metadata.jsonld", "bagit.txt", "bag-info.txt", "manifest-md5.txt", "tagmanifest-md5.txt"]\
                 + [k for k,v in _csvs.items()]

    # Loop for each filename
    for filename in _filenames:
        # Create a blank template
        _file = {"type": "", "filenameFull": "", "filenameShort": "", "data": "", "pretty": ""}
        # filename, no path prefix
        # _short = os.path.basename(filename)
        _short = filename
        # Bagit files
        if filename.endswith(".txt"):
            _file = {"type": "bagit", "filenameFull": filename, "filenameShort": _short}
        # JSONLD files
        elif filename.endswith(".jsonld"):
            _file = {"type": "json", "filenameFull": filename, "filenameShort": _short, "data": _j}

        # CSV files
        elif filename.endswith(".csv"):
            _cols_rows = {"cols": 0, "rows": 0}
            ensemble = is_ensemble(_csvs[_short])
            # special case for calculating ensemble rows and columns
            if ensemble:
                _cols_rows = get_ensemble_counts(_csvs[_short])

            # all other non-ensemble csv files.
            else:
                _cols_rows["cols"] = len(_csvs[_short])
                for k, v in _csvs[_short].items():
                    _cols_rows["rows"] = len(v["values"])
                    break

            # take what we've gathered for this file, and add it to the list.
            _file = {"type": "csv", "filenameFull": filename, "filenameShort": _short,
                     "data": _cols_rows}

        _api_data.append(_file)

    return _api_data

Response: Returns the format of the LIPD data in the layout that the Lipd. net validator accepts.


Instruction:
Here you have a function in Python 3, explain what it does
def create_detailed_results(result):
    string = ""
    # Validation Response output
    # string += "VALIDATION RESPONSE\n"
    string += "STATUS: {}\n".format(result["status"])
    if result["feedback"]:
        string += "WARNINGS: {}\n".format(len(result["feedback"]["wrnMsgs"]))
        # Loop through and output the Warnings
        for msg in result["feedback"]["wrnMsgs"]:
            string += "- {}\n".format(msg)
        string += "ERRORS: {}\n".format(len(result["feedback"]["errMsgs"]))
        # Loop through and output the Errors
        for msg in result["feedback"]["errMsgs"]:
            string += "- {}\n".format(msg)
    return string

Response: Create a detailed string output for the file containing the results from the API call.


Instruction:
Can you create a Python 3 function that
displays the results from the validator in a brief or off state

Response:
def display_results(results, detailed=False):
    """
    Display the results from the validator in a brief or detailed output

    :param list results: API results, sorted by dataset name (multiple)
    :param bool detailed: Detailed results on or off
    :return none:
    """
    # print("\nVALIDATOR RESULTS")
    # print("======================\n")

    if not detailed:
        print('FILENAME......................................... STATUS..........')

    for entry in results:
        try:
            if detailed:
                print("\n{}".format(entry["filename"]))
                print(create_detailed_results(entry))
            else:
                print("{:<50}{}".format(entry["filename"], entry["status"]))
        except Exception as e:
            logger_validator_api.debug("display_results: Exception: {}".format(e))
            print("Error: display_results: {}".format(e))

    return


Instruction:
Implement a Python 3 function for
calling the LiPD. net validator API to get a list of files from one dataset.

Response:
def call_validator_api(dsn, api_data):
    """
    Single call to the lipd.net validator API

    'api_data' format:
     [
        {"type": "csv", "filenameFull": /path/to/filename.csv, "data": "", ...},
        {"type": "json", "filenameFull": /path/to/metadata.jsonld, "data": "", ...},
        ...
    ]
    Result format:
        {"dat": <dict>, "feedback": <dict>, "filename": "", "status": ""}

    :param str dsn: Dataset name
    :param list api_data: Prepared payload for one LiPD dataset. All the sorted files (txt, jsonld, csv), API formatted
    :return list result: Validator result for one file
    """
    _filename = dsn + ".lpd"

    try:
        # Contact server and send LiPD metadata as the payload
        # print("Sending request to LiPD.net validator...\n")
        api_data = json.dumps(api_data)

        # The payload that is going to be sent with the JSON request
        payload = {'json_payload': api_data, 'apikey': 'lipd_linked'}

        # Development Link
        # response = requests.post('http://localhost:3000/api/validator', data=payload)

        # Production Link
        response = requests.post('http://www.lipd.net/api/validator', data=payload)

        if response.status_code == 413:
            result = {"dat": {}, "feedback": {}, "filename": _filename,
                      "status": "HTTP 413: Request Entity Too Large"}
        elif response.status_code == 404:
            result = {"dat": {}, "feedback": {}, "filename": _filename,
                      "status": "HTTP 404: Not Found"}
        elif response.status_code == 400:
            result = {"dat": {}, "feedback": {}, "filename": _filename,
                      "status": response.text}

        # For an example of the JSON Response, reference the "sample_data_response" below

        # Convert JSON string into a Python dictionary
        # print("Converting response to json...\n")
        else:
            result = json.loads(response.text)

    except TypeError as e:
        logger_validator_api.warning("get_validator_results: TypeError: {}".format(e))
        result = {"dat": {}, "feedback": {}, "filename": _filename, "status": "JSON DECODE ERROR"}
    except requests.exceptions.ConnectionError as e:
        logger_validator_api.warning("get_validator_results: ConnectionError: {}".format(e))
        result = {"dat": {}, "feedback": {}, "filename": _filename, "status": "UNABLE TO REACH SERVER"}
    except Exception as e:
        logger_validator_api.debug("get_validator_results: Exception: {}".format(e))
        result = {"dat": {}, "feedback": {}, "filename": _filename, "status": "ERROR BEFORE VALIDATION, {}".format(e)}
    if not result:
        result = {"dat": {}, "feedback": {}, "filename": _filename, "status": "EMPTY RESPONSE"}

    result["filename"] = _filename
    return result


Instruction:
Can you implement a function in Python 3 that
filters all needs with given configuration and returns all needs which pass the filters

Response:
def procces_filters(all_needs, current_needlist):
    """
    Filters all needs with given configuration

    :param current_needlist: needlist object, which stores all filters
    :param all_needs: List of all needs inside document

    :return: list of needs, which passed the filters
    """

    if current_needlist["sort_by"] is not None:
        if current_needlist["sort_by"] == "id":
            all_needs = sorted(all_needs, key=lambda node: node["id"])
        elif current_needlist["sort_by"] == "status":
            all_needs = sorted(all_needs, key=status_sorter)

    found_needs_by_options = []

    # Add all need_parts of given needs to the search list
    all_needs_incl_parts = prepare_need_list(all_needs)

    for need_info in all_needs_incl_parts:
        status_filter_passed = False
        if current_needlist["status"] is None or len(current_needlist["status"]) == 0:
            # Filtering for status was not requested
            status_filter_passed = True
        elif need_info["status"] is not None and need_info["status"] in current_needlist["status"]:
            # Match was found
            status_filter_passed = True

        tags_filter_passed = False
        if len(set(need_info["tags"]) & set(current_needlist["tags"])) > 0 or len(current_needlist["tags"]) == 0:
            tags_filter_passed = True

        type_filter_passed = False
        if need_info["type"] in current_needlist["types"] \
                or need_info["type_name"] in current_needlist["types"] \
                or len(current_needlist["types"]) == 0:
            type_filter_passed = True

        if status_filter_passed and tags_filter_passed and type_filter_passed:
            found_needs_by_options.append(need_info)

    found_needs_by_string = filter_needs(all_needs_incl_parts, current_needlist["filter"])

    # found_needs = [x for x in found_needs_by_string if x in found_needs_by_options]

    found_needs = check_need_list(found_needs_by_options, found_needs_by_string)

    return found_needs


Instruction:
Can you tell what is the following Python 3 function doing
def filter_needs(needs, filter_string="", filter_parts=True, merge_part_with_parent=True):

    if filter_string is None or filter_string == "":
        return needs

    found_needs = []

    for filter_need in needs:
        try:
            if filter_single_need(filter_need, filter_string):
                found_needs.append(filter_need)
        except Exception as e:
            logger.warning("Filter {0} not valid: Error: {1}".format(filter_string, e))

    return found_needs

Response: Filters given needs based on a given filter string.


Instruction:
Can you create a Python 3 function that
checks if a single need passes a filter_string

Response:
def filter_single_need(need, filter_string=""):
    """
    Checks if a single need/need_part passes a filter_string

    :param need: need or need_part
    :param filter_string: string, which is used as input for eval()
    :return: True, if need as passed the filter_string, else False
    """
    filter_context = need.copy()
    filter_context["search"] = re.search
    result = False
    try:
        result = bool(eval(filter_string, None, filter_context))
    except Exception as e:
        raise NeedInvalidFilter("Filter {0} not valid: Error: {1}".format(filter_string, e))
    return result


Instruction:
Here you have a function in Python 3, explain what it does
def get_logging_level(debug):
    level = logging.INFO
    if debug:
        level = logging.DEBUG
    return level

Response: Returns logging level based on boolean


Instruction:
How would you implement a function in Python 3 that
runs a spec file and return a list of Crate node entries.

Response:
def run_spec(spec,
             benchmark_hosts,
             result_hosts=None,
             output_fmt=None,
             logfile_info=None,
             logfile_result=None,
             action=None,
             fail_if=None,
             sample_mode='reservoir'):
    """Run a spec file, executing the statements on the benchmark_hosts.

    Short example of a spec file:

        [setup]
        statement_files = ["sql/create_table.sql"]

            [[setup.data_files]]
            target = "t"
            source = "data/t.json"

        [[queries]]
        statement = "select count(*) from t"
        iterations = 2000
        concurrency = 10

        [teardown]
        statements = ["drop table t"]

    See https://github.com/mfussenegger/cr8/tree/master/specs
    for more examples.

    Args:
        spec: path to a spec file
        benchmark_hosts: hostname[:port] pairs of Crate nodes
        result_hosts: optional hostname[:port] Crate node pairs into which the
            runtime statistics should be inserted.
        output_fmt: output format
        action: Optional action to execute.
            Default is to execute all actions - setup, queries and teardown.
            If present only the specified action will be executed.
            The argument can be provided multiple times to execute more than
            one action.
        fail-if: An expression that causes cr8 to exit with a failure if it
            evaluates to true.
            The expression can contain formatting expressions for:
                - runtime_stats
                - statement
                - meta
                - concurrency
                - bulk_size
            For example:
                --fail-if "{runtime_stats.mean} > 1.34"
    """
    with Logger(output_fmt=output_fmt,
                logfile_info=logfile_info,
                logfile_result=logfile_result) as log:
        do_run_spec(
            spec=spec,
            benchmark_hosts=benchmark_hosts,
            log=log,
            result_hosts=result_hosts,
            action=action,
            fail_if=fail_if,
            sample_mode=sample_mode
        )


Instruction:
Can you generate a brief explanation for the following Python 3 code
def download_from_url(src_url, dst_dir):
    dst_path = ""
    if "MD982181" not in src_url:
        dsn = input("Please enter the dataset name for this file (Name.Location.Year) : ")
        dst_path = os.path.join(dst_dir, dsn + ".lpd")
        try:
            print("downloading file from url...")
            urllib.request.urlretrieve(src_url, dst_path)
        except Exception as e:
            print("Error: unable to download from url: {}".format(e))
    else:
        print("Error: That file cannot be download due to server restrictions")
    return dst_path

Response: Download a file from a given URL and save it to a local path.


Instruction:
Create a Python 3 function for
processing a single LPD file.

Response:
def process_lpd(name, dir_tmp):
    """
    Opens up json file, invokes doi_resolver, closes file, updates changelog, cleans directory, and makes new bag.
    :param str name: Name of current .lpd file
    :param str dir_tmp: Path to tmp directory
    :return none:
    """
    logger_doi_main.info("enter process_lpd")
    dir_root = os.getcwd()
    dir_bag = os.path.join(dir_tmp, name)
    dir_data = os.path.join(dir_bag, 'data')

    # Navigate down to jLD file
    # dir : dir_root -> dir_data
    os.chdir(dir_data)

    # Open jld file and read in the contents. Execute DOI Resolver.
    jld_data = read_json_from_file(os.path.join(dir_data, name + '.jsonld'))

    # Overwrite data with new data
    jld_data = DOIResolver(dir_root, name, jld_data).main()
    # Open the jld file and overwrite the contents with the new data.
    write_json_to_file(jld_data)

    # Open changelog. timestamp it. Prompt user for short description of changes. Close and save
    # update_changelog()

    # Delete old bag files, and move files to bag root for re-bagging
    # dir : dir_data -> dir_bag
    dir_cleanup(dir_bag, dir_data)
    finish_bag(dir_bag)

    logger_doi_main.info("exit process_lpd")
    return


Instruction:
Can you write a function in Python 3 where it
asks the user if they want to force update files that were previously resolved.

Response:
def prompt_force():
    """
    Ask the user if they want to force update files that were previously resolved
    :return bool: response
    """
    logger_doi_main.info("enter prompt_force")
    count = 0
    print("Do you want to force updates for previously resolved files? (y/n)")
    while True:
        force = input("> ")
        try:
            if count == 2:
                return True
            elif force.lower() in ('y', 'yes'):
                return True
            elif force.lower() in ('n', 'no'):
                return False
            else:
                print("invalid response")
        except AttributeError as e:
            print("invalid response")
            logger_doi_main.warn("invalid response: {}, {}".format(force, e))
        count += 1
    logger_doi_main.info("force update: {}".format(force))
    logger_doi_main.info("exit prompt_force")
    return True