Instruction:
Implement a Python 3 function for
dumping a file into the repository.

Response:
def dump_file(self, value, relativePath,
                        description=None,
                        dump=None, pull=None,
                        replace=False, raiseError=True, ntrials=3):
        """
        Dump a file using its value to the system and creates its
        attribute in the Repository with utc timestamp.

        :Parameters:
            #. value (object): The value of a file to dump and add to the
               repository. It is any python object or file.
            #. relativePath (str): The relative to the repository path to where
               to dump the file.
            #. description (None, string): Any description about the file.
            #. dump (None, string): The dumping method.
               If None it will be set automatically to pickle and therefore the
               object must be pickleable. If a string is given, it can be a
               keyword ('json','pickle','dill') or a string compileable code to
               dump the data. The string code must include all the necessary
               imports and a '$FILE_PATH' that replaces the absolute file path
               when the dumping will be performed.\n
               e.g. "import numpy as np; np.savetxt(fname='$FILE_PATH', X=value, fmt='%.6e')"
            #. pull (None, string): The pulling method. If None it will be set
               automatically to pickle and therefore the object must be
               pickleable. If a string is given, it can be a keyword
               ('json','pickle','dill') or a string compileable code to pull
               the data. The string code must include all the necessary imports,
               a '$FILE_PATH' that replaces the absolute file path when the
               dumping will be performed and finally a PULLED_DATA variable.\n
               e.g "import numpy as np; PULLED_DATA=np.loadtxt(fname='$FILE_PATH')"
            #. replace (boolean): Whether to replace any existing file.
            #. raiseError (boolean): Whether to raise encountered error instead
               of returning failure.
            #. ntrials (int): After aquiring all locks, ntrials is the maximum
               number of trials allowed before failing.
               In rare cases, when multiple processes
               are accessing the same repository components, different processes
               can alter repository components between successive lock releases
               of some other process. Bigger number of trials lowers the
               likelyhood of failure due to multiple processes same time
               alteration.

        :Returns:
            #. success (boolean): Whether renaming the directory was successful.
            #. message (None, string): Some explanatory message or error reason
               why directory was not dumped.
        """
        # check arguments
        assert isinstance(raiseError, bool), "raiseError must be boolean"
        assert isinstance(replace, bool), "replace must be boolean"
        assert isinstance(ntrials, int), "ntrials must be integer"
        assert ntrials>0, "ntrials must be >0"
        if description is None:
            description = ''
        assert isinstance(description, basestring), "description must be None or a string"
        # convert dump and pull methods to strings
        if pull is None and dump is not None:
            if dump.startswith('pickle') or dump.startswith('dill') or dump.startswith('numpy') or dump =='json':
                pull = dump
        dump = get_dump_method(dump, protocol=self._DEFAULT_PICKLE_PROTOCOL)
        pull = get_pull_method(pull)
        # check name and path
        relativePath = self.to_repo_relative_path(path=relativePath, split=False)
        savePath     = os.path.join(self.__path,relativePath)
        fPath, fName = os.path.split(savePath)
        # check if name is allowed
        success, reason = self.is_name_allowed(savePath)
        if not success:
            assert not raiseError, reason
            return False, reason
        # ensure directory added
        try:
            success, reason = self.add_directory(fPath, raiseError=False, ntrials=ntrials)
        except Exception as err:
            reason  = "Unable to add directory (%s)"%(str(err))
            success = False
        if not success:
            assert not raiseError, reason
            return False, reason
        # lock repository
        LR =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(self.__path, self.__repoLock))
        acquired, code = LR.acquire_lock()
        if not acquired:
            m = "code %s. Unable to aquire the repository lock. You may try again!"%(code,)
            assert raiseError, Exception(m)
            return False,m
        # lock file
        LF =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))
        acquired, code = LF.acquire_lock()
        if not acquired:
            LR.release_lock()
            error = "Code %s. Unable to aquire the lock when adding '%s'"%(code,relativePath)
            assert not raiseError, error
            return False, error
        # load repository info
        for _trial in range(ntrials):
            try:
                repo = self.__load_repository_pickle_file(os.path.join(self.__path, self.__repoFile))
                self.__repo['walk_repo'] = repo['walk_repo']
            except Exception as err:
                error = str(err)
                if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], str(error)))
            else:
                error = None
                break
        if error is not None:
            LR.release_lock()
            LF.release_lock()
            assert not raiseError, Exception(error)
            return False, error
        # dump file
        for _trial in range(ntrials):
            error = None
            try:
                isRepoFile, fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)
                if isRepoFile:
                    assert replace, "file is a registered repository file. set replace to True to replace"
                fileInfoPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileInfo%fName)
                if isRepoFile and fileOnDisk:
                    with open(fileInfoPath, 'rb') as fd:
                        info = pickle.load(fd)
                    assert info['repository_unique_name'] == self.__repo['repository_unique_name'], "it seems that file was created by another repository"
                    info['last_update_utctime'] = time.time()
                else:
                    info = {'repository_unique_name':self.__repo['repository_unique_name']}
                    info['create_utctime'] = info['last_update_utctime'] = time.time()
                info['dump'] = dump
                info['pull'] = pull
                info['description'] = description
                # get parent directory list if file is new and not being replaced
                if not isRepoFile:
                    dirList = self.__get_repository_directory(fPath)
                # dump file
                #exec( dump.replace("$FILE_PATH", str(savePath)) )
                my_exec( dump.replace("$FILE_PATH", str(savePath)), locals=locals(), globals=globals(), description='dump'  )
                # update info
                with open(fileInfoPath, 'wb') as fd:
                    pickle.dump( info,fd, protocol=self._DEFAULT_PICKLE_PROTOCOL)
                    fd.flush()
                    os.fsync(fd.fileno())
                # update class file
                fileClassPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileClass%fName)
                with open(fileClassPath, 'wb') as fd:
                    if value is None:
                        klass = None
                    else:
                        klass = value.__class__
                    pickle.dump(klass , fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )
                    fd.flush()
                    os.fsync(fd.fileno())
                # add to repo if file is new and not being replaced
                if not isRepoFile:
                    dirList.append(fName)
            except Exception as err:
                error = "unable to dump the file (%s)"%(str(err),)
                try:
                    if 'pickle.dump(' in dump:
                        mi = get_pickling_errors(value)
                        if mi is not None:
                            error += '\nmore info: %s'%str(mi)
                except:
                    pass
                if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], str(error)))
            else:
                error = None
                break
        # save repository
        if error is None:
            _, error = self.__save_repository_pickle_file(lockFirst=False, raiseError=False)
        # release locks
        LR.release_lock()
        LF.release_lock()
        assert not raiseError or error is None, "unable to dump file '%s' after %i trials (%s)"%(relativePath, ntrials, error,)
        return success, error


Instruction:
Can you write a function in Python 3 where it
updates the value of a file in the repository.

Response:
def update_file(self, value, relativePath, description=False,
                          dump=False, pull=False, raiseError=True, ntrials=3):
        """
        Update the value of a file that is already in the Repository.\n
        If file is not registered in repository, and error will be thrown.\n
        If file is missing in the system, it will be regenerated as dump method
        is called.
        Unlike dump_file, update_file won't block the whole repository but only
        the file being updated.

        :Parameters:
            #. value (object): The value of a file to update.
            #. relativePath (str): The relative to the repository path of the
               file to be updated.
            #. description (False, string): Any random description about the file.
               If False is given, the description info won't be updated,
               otherwise it will be update to what description argument value is.
            #. dump (False, string): The new dump method. If False is given,
               the old one will be used.
            #. pull (False, string): The new pull method. If False is given,
               the old one will be used.
            #. raiseError (boolean): Whether to raise encountered error instead
               of returning failure.
            #. ntrials (int): After aquiring all locks, ntrials is the maximum
               number of trials allowed before failing.
               In rare cases, when multiple processes
               are accessing the same repository components, different processes
               can alter repository components between successive lock releases
               of some other process. Bigger number of trials lowers the
               likelyhood of failure due to multiple processes same time
               alteration.

       :Returns:
           #. success (boolean): Whether renaming the directory was successful.
           #. message (None, string): Some explanatory message or error reason
              why directory was not updated.
        """
        # check arguments
        assert isinstance(raiseError, bool), "raiseError must be boolean"
        assert description is False or description is None or isinstance(description, basestring), "description must be False, None or a string"
        assert dump is False or dump is None or isinstance(dump, basestring), "dump must be False, None or a string"
        assert pull is False or pull is None or isinstance(pull, basestring), "pull must be False, None or a string"
        assert isinstance(ntrials, int), "ntrials must be integer"
        assert ntrials>0, "ntrials must be >0"
        # get name and path
        relativePath = self.to_repo_relative_path(path=relativePath, split=False)
        savePath     = os.path.join(self.__path,relativePath)
        fPath, fName = os.path.split(savePath)
        # get locker
        LF =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))
        acquired, code = LF.acquire_lock()
        if not acquired:
            error = "Code %s. Unable to aquire the lock to update '%s'"%(code,relativePath)
            assert not raiseError, error
            return False, error
        # update file
        for _trial in range(ntrials):
            message = []
            updated = False
            try:
                # check file in repository
                isRepoFile, fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)
                assert isRepoFile, "file '%s' is not registered in repository, no update can be performed."%(relativePath,)
                # get file info
                if not fileOnDisk:
                    assert description is not False,  "file '%s' is found on disk, description must be provided"%(relativePath,)
                    assert dump is not False,  "file '%s' is found on disk, dump must be provided"%(relativePath,)
                    assert pull is not False,  "file '%s' is found on disk, pull must be provided"%(relativePath,)
                    info = {}
                    info['repository_unique_name'] = self.__repo['repository_unique_name']
                    info['create_utctime'] = info['last_update_utctime'] = time.time()
                else:
                    with open(os.path.join(fPath,self.__fileInfo%fName), 'rb') as fd:
                        info = pickle.load(fd)
                        info['last_update_utctime'] = time.time()
                if not fileOnDisk:
                    message.append("file %s is registered in repository but it was found on disk prior to updating"%relativePath)
                if not infoOnDisk:
                    message.append("%s is not found on disk prior to updating"%self.__fileInfo%fName)
                if not classOnDisk:
                    message.append("%s is not found on disk prior to updating"%self.__fileClass%fName)
                # get dump and pull
                if (description is False) or (dump is False) or (pull is False):
                    if description is False:
                        description = info['description']
                    elif description is None:
                        description = ''
                    if dump is False:
                        dump = info['dump']
                    elif dump is None:
                        dump = get_dump_method(dump, protocol=self._DEFAULT_PICKLE_PROTOCOL)
                    if pull is False:
                        pull = info['pull']
                    elif pull is None:
                        pull = get_pull_method(pull)
                # update dump, pull and description
                info['dump'] = dump
                info['pull'] = pull
                info['description'] = description
                # dump file
                my_exec( dump.replace("$FILE_PATH", str(savePath)), locals=locals(), globals=globals(), description='update'  )
                #exec( dump.replace("$FILE_PATH", str(savePath)) )
                # remove file if exists
                _path = os.path.join(fPath,self.__fileInfo%fName)
                # update info
                with open(_path, 'wb') as fd:
                    pickle.dump( info,fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )
                    fd.flush()
                    os.fsync(fd.fileno())
                # update class file
                fileClassPath = os.path.join(self.__path,os.path.dirname(relativePath),self.__fileClass%fName)
                with open(fileClassPath, 'wb') as fd:
                    if value is None:
                        klass = None
                    else:
                        klass = value.__class__
                    pickle.dump(klass , fd, protocol=self._DEFAULT_PICKLE_PROTOCOL )
                    fd.flush()
                    os.fsync(fd.fileno())
            except Exception as err:
                message.append(str(err))
                updated = False
                try:
                    if 'pickle.dump(' in dump:
                        mi = get_pickling_errors(value)
                        if mi is not None:
                            message.append('more info: %s'%str(mi))
                except:
                    pass
                if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], '\n'.join(message)))
            else:
                updated = True
                break
        # release lock
        LF.release_lock()
        # check and return
        assert updated or not raiseError, "Unable to update file '%s' (%s)"%(relativePath, '\n'.join(message),)
        return updated, '\n'.join(message)


Instruction:
Write a Python 3 function that can
pull a file from the repository.

Response:
def pull_file(self, relativePath, pull=None, update=True, ntrials=3):
        """
        Pull a file's data from the Repository.

        :Parameters:
            #. relativePath (string): The relative to the repository path from
               where to pull the file.
            #. pull (None, string): The pulling method.
               If None, the pull method saved in the file info will be used.
               If a string is given, the string should include all the necessary
               imports, a '$FILE_PATH' that replaces the absolute file path when
               the dumping will be performed and finally a PULLED_DATA variable.
               e.g "import numpy as np; PULLED_DATA=np.loadtxt(fname='$FILE_PATH')"
            #. update (boolean): If pull is not None, Whether to update the pull
               method stored in the file info by the given pull method.
            #. ntrials (int): After aquiring all locks, ntrials is the maximum
               number of trials allowed before failing.
               In rare cases, when multiple processes
               are accessing the same repository components, different processes
               can alter repository components between successive lock releases
               of some other process. Bigger number of trials lowers the
               likelyhood of failure due to multiple processes same time
               alteration.

        :Returns:
            #. data (object): The pulled data from the file.
        """
        assert isinstance(ntrials, int), "ntrials must be integer"
        assert ntrials>0, "ntrials must be >0"
        # check name and path
        relativePath = self.to_repo_relative_path(path=relativePath, split=False)
        realPath     = os.path.join(self.__path,relativePath)
        fPath, fName = os.path.split(realPath)
        # check whether it's a repository file
        isRepoFile,fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)
        if not isRepoFile:
            fileOnDisk  = ["",". File itself is found on disk"][fileOnDisk]
            infoOnDisk  = ["",". %s is found on disk"%self.__fileInfo%fName][infoOnDisk]
            classOnDisk = ["",". %s is found on disk"%self.__fileClass%fName][classOnDisk]
            assert False, "File '%s' is not a repository file%s%s%s"%(relativePath,fileOnDisk,infoOnDisk,classOnDisk)
        assert fileOnDisk, "File '%s' is registered in repository but the file itself was not found on disk"%(relativePath,)
        if not infoOnDisk:
            if pull is not None:
                warnings.warn("'%s' was not found on disk but pull method is given"%(self.__fileInfo%fName))
            else:
                raise Exception("File '%s' is registered in repository but the '%s' was not found on disk and pull method is not specified"%(relativePath,(self.__fileInfo%fName)))
        # lock repository
        LF =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))
        acquired, code = LF.acquire_lock()
        if not acquired:
            error = "Code %s. Unable to aquire the lock when adding '%s'"%(code,relativePath)
            return False, error
        # pull file
        for _trial in range(ntrials):
            error = None
            try:
                # get pull method
                if pull is not None:
                    pull = get_pull_method(pull)
                else:
                    with open(os.path.join(fPath,self.__fileInfo%fName), 'rb') as fd:
                        info = pickle.load(fd)
                    pull = info['pull']
                # try to pull file
                #namespace = {}
                #namespace.update( globals() )
                #exec( pull.replace("$FILE_PATH", str(realPath) ), namespace )
                my_exec( pull.replace("$FILE_PATH", str(realPath) ), locals=locals(), globals=globals(), description='pull' )
            except Exception as err:
                LF.release_lock()
                m = str(pull).replace("$FILE_PATH", str(realPath) )
                error = "Unable to pull data using '%s' from file (%s)"%(m,err)
                if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], str(error)))
            else:
                break
        LF.release_lock()
        assert error is None, "After %i trials, %s"%(ntrials, error)
        # return data
        return locals()['PULLED_DATA']


Instruction:
Explain what the following Python 3 code does
def rename_file(self, relativePath, newRelativePath,
                          force=False, raiseError=True, ntrials=3):
        """
        Rename a file in the repository. It insures renaming the file in the system.

        :Parameters:
            #. relativePath (string): The relative to the repository path of
               the file that needst to be renamed.
            #. newRelativePath (string): The new relative to the repository path
               of where to move and rename the file.
            #. force (boolean): Whether to force renaming even when another
               repository file exists. In this case old repository file
               will be removed from the repository and the system as well.
            #. raiseError (boolean): Whether to raise encountered error instead
               of returning failure.
            #. ntrials (int): After aquiring all locks, ntrials is the maximum
               number of trials allowed before failing.
               In rare cases, when multiple processes
               are accessing the same repository components, different processes
               can alter repository components between successive lock releases
               of some other process. Bigger number of trials lowers the
               likelyhood of failure due to multiple processes same time
               alteration.

        :Returns:
            #. success (boolean): Whether renaming the file was successful.
            #. message (None, string): Some explanatory message or error reason
               why directory was not updated.
        """
        assert isinstance(raiseError, bool), "raiseError must be boolean"
        assert isinstance(force, bool), "force must be boolean"
        assert isinstance(ntrials, int), "ntrials must be integer"
        assert ntrials>0, "ntrials must be >0"
        # check old name and path
        relativePath = self.to_repo_relative_path(path=relativePath, split=False)
        realPath     = os.path.join(self.__path,relativePath)
        fPath, fName = os.path.split(realPath)
        # check new name and path
        newRelativePath = self.to_repo_relative_path(path=newRelativePath, split=False)
        newRealPath     = os.path.join(self.__path,newRelativePath)
        nfPath, nfName  = os.path.split(newRealPath)
        # lock old file
        LO =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))
        acquired, code = LO.acquire_lock()
        if not acquired:
            error = "Code %s. Unable to aquire the lock for old file '%s'"%(code,relativePath)
            assert not raiseError, error
            return False, error
        # add directory
        try:
            success, reason = self.add_directory(nfPath, raiseError=False, ntrials=ntrials)
        except Exception as err:
            reason  = "Unable to add directory (%s)"%(str(err))
            success = False
        if not success:
            LO.release_lock()
            assert not raiseError, reason
            return False, reason
        # create new file lock
        LN =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(nfPath,self.__fileLock%nfName))
        acquired, code = LN.acquire_lock()
        if not acquired:
            LO.release_lock()
            error = "Code %s. Unable to aquire the lock for new file path '%s'"%(code,newRelativePath)
            assert not raiseError, error
            return False, error
        # rename file
        for _trial in range(ntrials):
            renamed = False
            error   = None
            try:
                # check whether it's a repository file
                isRepoFile,fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)
                assert isRepoFile,  "file '%s' is not a repository file"%(relativePath,)
                assert fileOnDisk,  "file '%s' is found on disk"%(relativePath,)
                assert infoOnDisk,  "%s is found on disk"%self.__fileInfo%fName
                assert classOnDisk, "%s is found on disk"%self.__fileClass%fName
                # get new file path
                nisRepoFile,nfileOnDisk,ninfoOnDisk,nclassOnDisk = self.is_repository_file(newRelativePath)
                assert not nisRepoFile or force, "New file path is a registered repository file, set force to True to proceed regardless"
                # get parent directories list
                oDirList = self.__get_repository_directory(fPath)
                nDirList = self.__get_repository_directory(nfPath)
                # remove new file and all repository files from disk
                if os.path.isfile(newRealPath):
                    os.remove(newRealPath)
                if os.path.isfile(os.path.join(nfPath,self.__fileInfo%nfName)):
                    os.remove(os.path.join(nfPath,self.__fileInfo%nfName))
                if os.path.isfile(os.path.join(nfPath,self.__fileClass%nfName)):
                    os.remove(os.path.join(nfPath,self.__fileClass%nfName))
                # move old file to new path
                os.rename(realPath, newRealPath)
                os.rename(os.path.join(fPath,self.__fileInfo%fName), os.path.join(nfPath,self.__fileInfo%nfName))
                os.rename(os.path.join(fPath,self.__fileClass%fName), os.path.join(nfPath,self.__fileClass%nfName))
                # update list
                findex = oDirList.index(fName)
                oDirList.pop(findex)
                # update new list
                if nfName not in nDirList:
                    nDirList.append(nfName)
            except Exception as err:
                renamed = False
                error = str(err)
                if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], str(error)))
            else:
                renamed = True
                break
        # release locks
        LO.release_lock()
        LN.release_lock()
        # always clean old file lock
        try:
            if os.path.isfile(os.path.join(fPath,self.__fileLock%fName)):
                os.remove(os.path.join(fPath,self.__fileLock%fName))
        except:
            pass
        # return
        assert renamed or not raiseError, "Unable to rename file '%s' to '%s' after %i trials (%s)"%(relativePath, newRelativePath, ntrials, error,)
        #assert renamed or not raiseError, '\n'.join(message)
        return renamed, error

Response: This function rename a file in the repository.


Instruction:
Write a Python 3 script to
remove a file from the repository.

Response:
def remove_file(self, relativePath, removeFromSystem=False,
                          raiseError=True, ntrials=3):
        """
        Remove file from repository.

        :Parameters:
            #. relativePath (string): The relative to the repository path of the
               file to remove.
            #. removeFromSystem (boolean): Whether to remove file from disk as
               well.
            #. raiseError (boolean): Whether to raise encountered error instead
               of returning failure.
            #. ntrials (int): After aquiring all locks, ntrials is the maximum
               number of trials allowed before failing.
               In rare cases, when multiple processes
               are accessing the same repository components, different processes
               can alter repository components between successive lock releases
               of some other process. Bigger number of trials lowers the
               likelyhood of failure due to multiple processes same time
               alteration.
        """
        assert isinstance(raiseError, bool), "removeFromSystem must be boolean"
        assert isinstance(removeFromSystem, bool), "removeFromSystem must be boolean"
        assert isinstance(ntrials, int), "ntrials must be integer"
        assert ntrials>0, "ntrials must be >0"
        # check name and path
        relativePath = self.to_repo_relative_path(path=relativePath, split=False)
        realPath     = os.path.join(self.__path,relativePath)
        fPath, fName = os.path.split(realPath)
        # lock repository
        LF =  Locker(filePath=None, lockPass=str(uuid.uuid1()), lockPath=os.path.join(fPath,self.__fileLock%fName))
        acquired, code = LF.acquire_lock()
        if not acquired:
            error = "Code %s. Unable to aquire the lock when adding '%s'"%(code,relativePath)
            assert not raiseError, error
            return False, error
        # remove file
        for _trial in range(ntrials):
            removed = False
            message = []
            try:
                # check whether it's a repository file
                isRepoFile,fileOnDisk, infoOnDisk, classOnDisk = self.is_repository_file(relativePath)
                if not isRepoFile:
                    message("File '%s' is not a repository file"%(relativePath,))
                    if fileOnDisk:
                        message.append("File itself is found on disk")
                    if infoOnDisk:
                        message.append("%s is found on disk"%self.__fileInfo%fName)
                    if classOnDisk:
                        message.append("%s is found on disk"%self.__fileClass%fName)
                else:
                    dirList = self.__get_repository_directory(fPath)
                    findex  = dirList.index(fName)
                    dirList.pop(findex)
                    if os.path.isfile(realPath):
                        os.remove(realPath)
                    if os.path.isfile(os.path.join(fPath,self.__fileInfo%fName)):
                        os.remove(os.path.join(fPath,self.__fileInfo%fName))
                    if os.path.isfile(os.path.join(fPath,self.__fileClass%fName)):
                        os.remove(os.path.join(fPath,self.__fileClass%fName))
            except Exception as err:
                removed = False
                message.append(str(err))
                if self.DEBUG_PRINT_FAILED_TRIALS: print("Trial %i failed in Repository.%s (%s). Set Repository.DEBUG_PRINT_FAILED_TRIALS to False to mute"%(_trial, inspect.stack()[1][3], '\n'.join(message)))
            else:
                removed = True
                break
        # release lock
        LF.release_lock()
        # always clean
        try:
            if os.path.isfile(os.path.join(fPath,self.__fileLock%fName)):
                os.remove(os.path.join(fPath,self.__fileLock%fName))
        except:
            pass
        # check and return
        assert removed or not raiseError, "Unable to remove file '%s' after %i trials (%s)"%(relativePath, ntrials, '\n'.join(message),)
        return removed, '\n'.join(message)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def rename(self, key: Any, new_key: Any):
        if new_key == key:
            return

        required_locks = [self._key_locks[key], self._key_locks[new_key]]
        ordered_required_locks = sorted(required_locks, key=lambda x: id(x))
        for lock in ordered_required_locks:
            lock.acquire()

        try:
            if key not in self._data:
                raise KeyError("Attribute to rename \"%s\" does not exist" % key)
            self._data[new_key] = self[key]
            del self._data[key]
        finally:
            for lock in required_locks:
                lock.release()

Response: Renames an item in this collection as a transaction.


Instruction:
How would you explain what the following Python 3 function does
def get_text_fingerprint(text, hash_meth, encoding="utf-8"):  # pragma: no cover
    m = hash_meth()
    m.update(text.encode(encoding))
    return m.hexdigest()

Response: Get a fingerprint of a piece of text.


Instruction:
Write a Python 3 function that can
return md5 hash value of a file

Response:
def md5file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    """
    Return md5 hash value of a piece of a file

    Estimate processing time on:

    :param abspath: the absolute path to the file
    :param nbytes: only has first N bytes of the file. if 0 or None,
      hash all file

    CPU = i7-4600U 2.10GHz - 2.70GHz, RAM = 8.00 GB
    1 second can process 0.25GB data

    - 0.59G - 2.43 sec
    - 1.3G - 5.68 sec
    - 1.9G - 7.72 sec
    - 2.5G - 10.32 sec
    - 3.9G - 16.0 sec
    """
    return get_file_fingerprint(abspath, hashlib.md5, nbytes=nbytes, chunk_size=chunk_size)


Instruction:
Can you generate the documentation for the following Python 3 function
def sha256file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    return get_file_fingerprint(abspath, hashlib.sha256, nbytes=nbytes, chunk_size=chunk_size)

Response: Return the sha256 hash value of a file


Instruction:
Here you have a function in Python 3, explain what it does
def sha512file(abspath, nbytes=0, chunk_size=DEFAULT_CHUNK_SIZE):
    return get_file_fingerprint(abspath, hashlib.sha512, nbytes=nbytes, chunk_size=chunk_size)

Response: Return the sha512 hash value of a file


Instruction:
How would you explain what the following Python 3 function does
def register(registerable: Any):
    listenable = registration_event_listenable_map[type(registerable)]
    event = RegistrationEvent(registerable, RegistrationEvent.Type.REGISTERED)
    listenable.notify_listeners(event)

Response: Registers an object notifying any listeners that may be interested in it.


Instruction:
Here you have a function in Python 3, explain what it does
def unregister(registerable: Any):
    listenable = registration_event_listenable_map[type(registerable)]
    event = RegistrationEvent(registerable, RegistrationEvent.Type.UNREGISTERED)
    listenable.notify_listeners(event)

Response: Unregisters an object from the sequence of registered objects.


Instruction:
Make a summary of the following Python 3 code
def _load_module(path: str):
        spec = spec_from_file_location(os.path.basename(path), path)
        module = module_from_spec(spec)
        spec.loader.exec_module(module)

Response: Dynamically loads the python module at the given path.


Instruction:
How would you code a function in Python 3 to
check if the object is empty.

Response:
def is_empty(self, strict=True):
        """
        - If it's a file, check if it is a empty file. (0 bytes content)
        - If it's a directory, check if there's no file and dir in it.
            But if ``strict = False``, then only check if there's no file in it.

        :param strict: only useful when it is a directory. if True, only
            return True if this dir has no dir and file. if False, return True
            if it doesn't have any file.
        """
        if self.exists():
            if self.is_file():
                return self.size == 0
            elif self.is_dir():
                if strict:
                    return len(list(self.select(recursive=True))) == 0
                else:  # pragma: no cover
                    return len(list(self.select_file(recursive=True))) == 0
            else:  # pragma: no cover
                msg = "'%s' is not either file or directory! (maybe simlink)" % self
                raise EnvironmentError(msg)
        else:
            raise EnvironmentError("'%s' not exists!" % self)


Instruction:
Here you have a function in Python 3, explain what it does
def auto_complete_choices(self, case_sensitive=False):
        self_basename = self.basename
        self_basename_lower = self.basename.lower()
        if case_sensitive:  # pragma: no cover
            def match(basename):
                return basename.startswith(self_basename)
        else:
            def match(basename):
                return basename.lower().startswith(self_basename_lower)

        choices = list()
        if self.is_dir():
            choices.append(self)
            for p in self.sort_by_abspath(self.select(recursive=False)):
                choices.append(p)
        else:
            p_parent = self.parent
            if p_parent.is_dir():
                for p in self.sort_by_abspath(p_parent.select(recursive=False)):
                    if match(p.basename):
                        choices.append(p)
            else:  # pragma: no cover
                raise ValueError("'%s' directory does not exist!" % p_parent)
        return choices

Response:
A command line auto complete similar behavior. Find all items with same
        prefix of this one.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def print_big_dir(self, top_n=5):
        self.assert_is_dir_and_exists()

        size_table = sorted(
            [(p, p.dirsize) for p in self.select_dir(recursive=False)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p, size in size_table[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size), p.abspath))

Response: Print top_n big dir in this dir.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def print_big_file(self, top_n=5):
        self.assert_is_dir_and_exists()

        size_table = sorted(
            [(p, p.size) for p in self.select_file(recursive=True)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p, size in size_table[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size), p.abspath))

Response: Print the top n big files in this dir.


Instruction:
Here you have a function in Python 3, explain what it does
def print_big_dir_and_big_file(self, top_n=5):
        self.assert_is_dir_and_exists()

        size_table1 = sorted(
            [(p, p.dirsize) for p in self.select_dir(recursive=False)],
            key=lambda x: x[1],
            reverse=True,
        )
        for p1, size1 in size_table1[:top_n]:
            print("{:<9}    {:<9}".format(repr_data_size(size1), p1.abspath))
            size_table2 = sorted(
                [(p, p.size) for p in p1.select_file(recursive=True)],
                key=lambda x: x[1],
                reverse=True,
            )
            for p2, size2 in size_table2[:top_n]:
                print("    {:<9}    {:<9}".format(
                    repr_data_size(size2), p2.abspath))

Response: Print top_n big dir and big file in each dir.


Instruction:
Write a Python 3 function that can
return a dict of stat information for all files directories and total size.

Response:
def file_stat_for_all(self, filters=all_true):  # pragma: no cover
        """
        Find out how many files, directories and total size (Include file in
        it's sub-folder) it has for each folder and sub-folder.

        :returns: stat, a dict like ``{"directory path": {
          "file": number of files, "dir": number of directories,
          "size": total size in bytes}}``

        **中文文档**

        返回一个目录中的每个子目录的, 文件, 文件夹, 大小的统计数据。
        """
        self.assert_is_dir_and_exists()

        from collections import OrderedDict

        stat = OrderedDict()
        stat[self.abspath] = {"file": 0, "dir": 0, "size": 0}

        for p in self.select(filters=filters, recursive=True):
            if p.is_file():
                size = p.size
                while 1:
                    parent = p.parent

                    stat[parent.abspath]["file"] += 1
                    stat[parent.abspath]["size"] += size

                    if parent.abspath == self.abspath:
                        break

                    p = parent

            elif p.is_dir():
                stat[p.abspath] = {"file": 0, "dir": 0, "size": 0}

                while 1:
                    parent = p.parent
                    stat[parent.abspath]["dir"] += 1

                    if parent.abspath == self.abspath:
                        break

                    p = parent

        return stat


Instruction:
Write a Python 3 function for
finding out how many files directorys and total size.

Response:
def file_stat(self, filters=all_true):
        """Find out how many files, directorys and total size (Include file in
        it's sub-folder).

        :returns: stat, a dict like ``{"file": number of files,
          "dir": number of directorys, "size": total size in bytes}``

        **中文文档**

        返回一个目录中的文件, 文件夹, 大小的统计数据。
        """
        self.assert_is_dir_and_exists()

        stat = {"file": 0, "dir": 0, "size": 0}

        for p in self.select(filters=filters, recursive=True):
            if p.is_file():
                stat["file"] += 1
                stat["size"] += p.size
            elif p.is_dir():
                stat["dir"] += 1

        return stat


Instruction:
Here you have a function in Python 3, explain what it does
def mirror_to(self, dst):  # pragma: no cover
        self.assert_is_dir_and_exists()

        src = self.abspath
        dst = os.path.abspath(dst)
        if os.path.exists(dst):  # pragma: no cover
            raise Exception("distination already exist!")

        folder_to_create = list()
        file_to_create = list()

        for current_folder, _, file_list in os.walk(self.abspath):
            current_folder = current_folder.replace(src, dst)
            try:
                os.mkdir(current_folder)
            except:  # pragma: no cover
                pass
            for basename in file_list:
                abspath = os.path.join(current_folder, basename)
                with open(abspath, "wb") as _:
                    pass

Response: Mirrors the contents of this directory to another directory.


Instruction:
Write a Python 3 function for
executing every. py file as main script.

Response:
def execute_pyfile(self, py_exe=None):  # pragma: no cover
        """
        Execute every ``.py`` file as main script.

        :param py_exe: str, python command or python executable path.

        **中文文档**

        将目录下的所有Python文件作为主脚本用当前解释器运行。
        """
        import subprocess

        self.assert_is_dir_and_exists()

        if py_exe is None:
            if six.PY2:
                py_exe = "python2"
            elif six.PY3:
                py_exe = "python3"

        for p in self.select_by_ext(".py"):
            subprocess.Popen('%s "%s"' % (py_exe, p.abspath))


Instruction:
Implement a Python 3 function for
trailing white space at end of each line for every. py file.

Response:
def trail_space(self, filters=lambda p: p.ext == ".py"):  # pragma: no cover
        """
        Trail white space at end of each line for every ``.py`` file.

        **中文文档**

        将目录下的所有被选择的文件中行末的空格删除。
        """
        self.assert_is_dir_and_exists()

        for p in self.select_file(filters):
            try:
                with open(p.abspath, "rb") as f:
                    lines = list()
                    for line in f:
                        lines.append(line.decode("utf-8").rstrip())

                with open(p.abspath, "wb") as f:
                    f.write("\n".join(lines).encode("utf-8"))

            except Exception as e:  # pragma: no cover
                raise e


Instruction:
Can you generate a brief explanation for the following Python 3 code
def autopep8(self, **kwargs):  # pragma: no cover
        self.assert_is_dir_and_exists()

        for p in self.select_by_ext(".py"):
            with open(p.abspath, "rb") as f:
                code = f.read().decode("utf-8")

            formatted_code = autopep8.fix_code(code, **kwargs)

            with open(p.abspath, "wb") as f:
                f.write(formatted_code.encode("utf-8"))

Response: Auto convert your python code in a directory to pep8 styled code.


Instruction:
Write a Python 3 function for
notifying all event listeners that the data has been passed to the event listeners.

Response:
def notify_listeners(self, data: Optional[_ListenableDataType]=_NO_DATA_MARKER):
        """
        Notify event listeners, passing them the given data (if any).
        :param data: the data to pass to the event listeners
        """
        for listener in self._listeners:
            if data is not Listenable._NO_DATA_MARKER:
                listener(data)
            else:
                listener()


Instruction:
Can you implement a function in Python 3 that
returns the size of the file in bytes.

Response:
def size(self):
        """
        File size in bytes.
        """
        try:
            return self._stat.st_size
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.size


Instruction:
Create a Python 3 function to
get most recent modify time in timestamp.

Response:
def mtime(self):
        """
        Get most recent modify time in timestamp.
        """
        try:
            return self._stat.st_mtime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.mtime


Instruction:
Write a Python 3 script for
getting most recent access time in timestamp.

Response:
def atime(self):
        """
        Get most recent access time in timestamp.
        """
        try:
            return self._stat.st_atime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.atime


Instruction:
Here you have a function in Python 3, explain what it does
def ctime(self):
        try:
            return self._stat.st_ctime
        except:  # pragma: no cover
            self._stat = self.stat()
            return self.ctime

Response: Get most recent create time in timestamp.


Instruction:
Write a Python 3 script for
returning current terminal size or a default if detect fails.

Response:
def get_terminal_size(default_cols=80, default_rows=25):
    """Return current terminal size (cols, rows) or a default if detect fails.

    This snippet comes from color ls by Chuck Blake:
    http://pdos.csail.mit.edu/~cblake/cls/cls.py

    """
    def ioctl_GWINSZ(fd):
        """Get (cols, rows) from a putative fd to a tty."""
        try:
            rows_cols = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
            return tuple(reversed(rows_cols))
        except:
            return None
    # Try std in/out/err...
    cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
    # ...or ctty...
    if not cr:
        try:
            fd = os.open(os.ctermid(), os.O_RDONLY)
            cr = ioctl_GWINSZ(fd)
            os.close(fd)
        except:
            # ...or fall back to defaults
            cr = (int(os.environ.get('COLUMNS', default_cols)),
                  int(os.environ.get('LINES', default_rows)))
    return cr


Instruction:
How would you explain what the following Python 3 function does
def get_metainfo(scriptfile,
                 keywords=['author', 'contact', 'copyright', 'download', 'git', 'subversion', 'version', 'website'],
                 special={},
                 first_line_pattern=r'^(?P<progname>.+)(\s+v(?P<version>\S+))?',
                 keyword_pattern_template=r'^\s*%(pretty)s:\s*(?P<%(keyword)s>\S.+?)\s*$',
                 prettify = lambda kw: kw.capitalize().replace('_', ' ')):
    """Dumb helper for pulling metainfo from a script __doc__ string.

    Returns a metainfo dict with command, description, progname and the given
    keywords (if present).

    This function will only make minimal efforts to succeed. If you need
    anything else: roll your own.

    The docstring needs to be multiline and the closing quotes need to be first
    on a line, optionally preceeded by whitespace.

    The first non-whitespace line is re.search'ed using first_line_pattern,
    default e.g (version optional, contains no whitespace): PROGNAME [vVERSION]

    The next non-whitespace, non-keyword line is expected to be the program
    description.

    The following lines are re.search'ed against a keyword:pattern dict which
    is constructed using
    keyword_pattern % dict(pretty=prettify(keyword), keyword=keyword)
    Default prettify is keyword.capitalize().replace('_', ' '). Example,
    for the keyword "licence" will match the following line:
    License: The MIT license.
    and set the license metainfo to "The MIT license.".

    Any keyword:pattern pairs that need special treatment can be supplied with
    special.
    """
    patterns = dict((kw, re.compile(keyword_pattern_template % dict(pretty=prettify(kw), keyword=kw))) for kw in keywords)
    patterns.update(special)
    metainfo = dict()

    if scriptfile[-4:] in ['.pyc', '.pyo']:
        scriptfile = scriptfile[:-1]
    script = open(scriptfile)
    closer = ''
    for line in script:
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        if line[:3] in ('"""', "'''"):
            closer = line[:3]
            break
        raise ValueError('file contains no docstring')
    if not line:
        for line in script:
            line = line.strip()
            if line:
                break
    g = re.search(first_line_pattern, line[3:]).groupdict()
    metainfo['progname'] = g['progname']
    if g['version']:
        metainfo['version'] = g['version']
    for line in script:
        if line.strip().startswith(closer):
            break
        for keyword, pattern in patterns.items():
            m = pattern.search(line)
            if m:
                metainfo[keyword] = m.group(keyword)
                break
        if line.strip() and not 'description' in metainfo:
            metainfo['description'] = line.strip()
    return metainfo

Response: This function takes a script file and returns a dictionary with command description progname and keywords.


Instruction:
Write a Python 3 function for
listing options that have not been used to format other values in their sections.

Response:
def unusedoptions(self, sections):
        """Lists options that have not been used to format other values in
        their sections.

        Good for finding out if the user has misspelled any of the options.
        """
        unused = set([])
        for section in _list(sections):
            if not self.has_section(section):
                continue
            options = self.options(section)
            raw_values = [self.get(section, option, raw=True) for option in options]
            for option in options:
                formatter = "%(" + option + ")s"
                for raw_value in raw_values:
                    if formatter in raw_value:
                        break
                else:
                    unused.add(option)
            return list(unused)


Instruction:
Implement a function in Python 3 to
consume and process arguments and store the result in self. value.

Response:
def parse(self, argv, usedname, location):
        """Consume and process arguments and store the result.
        ARGS:
        argv <list str>:
            The argument list to parse.
        usedname <str>:
            The string used by the user to invoke the option.
        location <str>:
            A user friendly sring describing where the parser got this
            data from.

        """
        try:
            value = self.format.parse(argv)
        except formats.BadNumberOfArguments, e:
            raise BadNumberOfArguments(usedname, e.required, e.supplied)
        except formats.BadArgument, e:
            raise BadArgument(usedname, e.argument, e.message)
        if self.recurring:
            self.value.append(value)
        else:
            self.value = value
        self.location = location


Instruction:
Create a Python 3 function for
parsing a string lexically and stores the result in the internal object.

Response:
def parsestr(self, argsstr, usedname, location):
        """Parse a string lexically and store the result.
        ARGS:
        argsstr <str>:
            The string to parse.
        usedname <str>:
            The string used by the user to invoke the option.
        location <str>:
            A user friendly sring describing where the parser got this
            data from.

        """
        try:
            value = self.format.parsestr(argsstr)
        except formats.BadNumberOfArguments, e:
            raise BadNumberOfArguments(usedname, e.required, e.supplied)
        except formats.BadArgument, e:
            raise BadArgument(usedname, e.argument, e.message)
        if self.recurring:
            self.value.append(value)
        else:
            self.value = value
        self.location = location


Instruction:
Given the following Python 3 function, write the documentation
def parse(self, argv):
        if not argv and self.optional:
            self.value = [] if self.recurring else None
            return
        try:
            value = self.format.parse(argv)
            if not self.recurring:
                self.value = value
                return
            self.value = [value]
            while argv:
                self.value.append(self.format.parse(argv))
        except formats.BadNumberOfArguments, e:
            raise BadNumberOfArguments(self.displayname, e.required, e.given)
        except formats.BadArgument, e:
            raise BadArgument(self.displayname, e.argument, e.details)

Response: Consume and process arguments and store the result in self. value.


Instruction:
Explain what the following Python 3 code does
def keys(self):
        return self.options.keys() + [p.name for p in self.positional_args]

Response: List names of options and positional arguments.


Instruction:
How would you explain what the following Python 3 function does
def values(self):
        return self.options.values() + [p.value for p in self.positional_args]

Response: List values of options and positional arguments.


Instruction:
Can you tell what is the following Python 3 function doing
def items(self):
        return [(p.name, p.value) for p in self.options.values() + self.positional_args]

Response: List values of options and positional arguments.


Instruction:
Here you have a function in Python 3, explain what it does
def getparam(self, key):
        try:
            return self.options[key]
        except:
            pass
        for posarg in self.positional_args:
            if posarg.name == key:
                return posarg
        try:
            return self.abbreviations[key[1:]]
        except:
            raise KeyError('no such option or positional argument')

Response: Get option or positional argument by name index or abbreviation.


Instruction:
Implement a function in Python 3 to
add an Option object to the user interface.

Response:
def _add_option(self, option):
        """Add an Option object to the user interface."""
        if option.name in self.options:
            raise ValueError('name already in use')
        if option.abbreviation in self.abbreviations:
            raise ValueError('abbreviation already in use')
        if option.name in [arg.name for arg in self.positional_args]:
            raise ValueError('name already in use by a positional argument')
        self.options[option.name] = option
        if option.abbreviation:
            self.abbreviations[option.abbreviation] = option
        self.option_order.append(option.name)


Instruction:
Given the following Python 3 function, write the documentation
def _add_positional_argument(self, posarg):
        if self.positional_args:
            if self.positional_args[-1].recurring:
                raise ValueError("recurring positional arguments must be last")
            if self.positional_args[-1].optional and not posarg.optional:
                raise ValueError("required positional arguments must precede optional ones")
        self.positional_args.append(posarg)

Response: Append a positional argument to the user interface.


Instruction:
Here you have a function in Python 3, explain what it does
def read_docs(self, docsfiles):
        updates = DocParser()
        for docsfile in _list(docsfiles):
            if os.path.isfile(docsfile):
                updates.parse(docsfile)
        self.docs.update((k, _docs(updates[k], self.docvars)) for k in self.docs if updates.blocks[k])
        for name, text in updates['parameters'].items():
            if name in self:
                self.getparam(name).docs = text[0] % self.docvars
            elif name not in self.ignore:
                raise ValueError("parameter %r does not exist" % name)

Response: Read program documentation from a list of paths to potential docsfiles.


Instruction:
Write a Python 3 function for
parsing configfiles. INDRA API.

Response:
def parse_files(self, files=None, sections=None):
        """Parse configfiles.
        files <list str>, <str> or None:
            What files to parse. None means use self.configfiles. New
            values override old ones. A string value will be interpreted
            as a list of one item.
        sections <list str>, <str> or None:
            Which sections to parse from the files. None means use
            self.sections. A string value will be interpreted as a list
            of one item. The [DEFAULT]
            section is always read, and it is read as if its contents were
            copied to the beginning of all other sections.

        """
        files = _list(files, self.configfiles)
        sections = _list(sections, self.sections)
        for file in files:
            parser = StrictConfigParser()
            parser.read(file)
            for section in sections:
                if not parser.has_section(section):
                    continue
                for unused in parser.unusedoptions(section):
                    if unused not in self.options and unused not in self.ignore:
                        templ = "The option %r in section [%s] of file %s does not exist."
                        raise InvalidOption(unused, message=templ % (unused, section, file))
                for name in parser.options(section):
                    if name in self.options:
                        if self.options[name].reserved:
                            templ = "The option %s in section [%s] of file %s is reserved for command line use."
                            raise ReservedOptionError(message=templ % (unused, section, file))
                        value = parser.get(section, name)
                        self.options[name].parsestr(value, name, '%s [%s]' % (file, section))


Instruction:
Given the following Python 3 function, write the documentation
def _parse_options(self, argv, location):
        observed = []
        while argv:
            if argv[0].startswith('--'):
                name = argv.pop(0)[2:]
                # '--' means end of options.
                if not name:
                    break
                if name not in self.options:
                    raise InvalidOption(name)
                option = self.options[name]
                if not option.recurring:
                    if option in observed:
                        raise OptionRecurrenceError(name)
                    observed.append(option)
                option.parse(argv, name, location)
            elif argv[0].startswith('-'):
                # A single - is not an abbreviation block, but the first positional arg.
                if argv[0] == '-':
                    break
                block = argv.pop(0)[1:]
                # Abbrevs for options that take values go last in the block.
                for abbreviation in block[:-1]:
                    if self.abbreviations[abbreviation].nargs != 0:
                        raise BadAbbreviationBlock(abbreviation, block, "options that require value arguments must be last in abbreviation blocks")
                # Parse individual options.
                for abbreviation in block:
                    option = self.abbreviations[abbreviation]
                    if not option.recurring:
                        if option in observed:
                            raise OptionRecurrenceError(option.name)
                        observed.append(option)
                    option.parse(argv, '-' + abbreviation, location)
            # only arguments that start with -- or - can be Options.
            else:
                break

Response: Parse the options part of an argument list.


Instruction:
Make a summary of the following Python 3 code
def _parse_positional_arguments(self, argv):
        for posarg in self.positional_args:
            posarg.parse(argv)
        if argv:
            if None in [p.nargs for p in self.positional_args]:
                msg = '%s too many argument%s given'
                plural_s = len(argv) > 1 and 's' or ''
                raise BadNumberOfArguments(message=msg % (len(argv), plural_s))
            msg = 'This program accepts exactly %s positional arguments (%s given).'
            required = len([p.nargs for p in self.positional_args])
            raise BadNumberOfArguments(message=msg % (required, required + len(argv)))

Response: Parse the positional arguments part of an argument list. Will be altered.


Instruction:
Write a Python 3 function for
parsing command line arguments.

Response:
def parse_argv(self, argv=None, location='Command line.'):
        """Parse command line arguments.

        args <list str> or None:
            The argument list to parse. None means use a copy of sys.argv. argv[0] is
            ignored.
        location = '' <str>:
            A user friendly string describing where the parser got this
            data from. '' means use "Command line." if args == None, and
            "Builtin default." otherwise.

        """
        if argv is None:
            argv = list(sys.argv)
        argv.pop(0)
        self._parse_options(argv, location)
        self._parse_positional_arguments(argv)


Instruction:
Create a Python 3 function for
returning user friendly help on program options.

Response:
def optionhelp(self, indent=0, maxindent=25, width=79):
        """Return user friendly help on program options."""
        def makelabels(option):
            labels = '%*s--%s' % (indent, ' ', option.name)
            if option.abbreviation:
                labels += ', -' + option.abbreviation
            return labels + ': '
        docs = []
        helpindent = _autoindent([makelabels(o) for o in self.options.values()], indent, maxindent)
        for name in self.option_order:
            option = self.options[name]
            labels = makelabels(option)
            helpstring = "%s(%s). %s" % (option.formatname, option.strvalue, option.docs)
            wrapped = self._wrap_labelled(labels, helpstring, helpindent, width)
            docs.extend(wrapped)
        return '\n'.join(docs)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def posarghelp(self, indent=0, maxindent=25, width=79):
        docs = []
        makelabel = lambda posarg: ' ' * indent + posarg.displayname + ': '
        helpindent = _autoindent([makelabel(p) for p in self.positional_args], indent, maxindent)
        for posarg in self.positional_args:
            label = makelabel(posarg)
            text = posarg.formatname + '. ' + posarg.docs
            wrapped = self._wrap_labelled(label, text, helpindent, width)
            docs.extend(wrapped)
        return '\n'.join(docs)

Response: Return user friendly help on positional arguments in the program.


Instruction:
Here you have a function in Python 3, explain what it does
def format_usage(self, usage=None):
        if usage is None:
            usage = self.docs['usage']
        if usage is not None:
            return usage[0] % self.docvars
        usage = self.docvars['command']
        if self.basic_option_names.get('help'):
            usage += ' [--%s]' % self.basic_option_names.get('help')
        if self.options:
            usage += ' <OPTIONS>'
        optional = 0
        for posarg in self.positional_args:
            usage += ' '
            if posarg.optional:
                usage += "["
                optional += 1
            usage += posarg.displayname
            if posarg.recurring:
                usage += ' [%s2 [...]]' % posarg.displayname
        usage += ']' * optional
        return usage

Response: Return a formatted usage string.


Instruction:
Here you have a function in Python 3, explain what it does
def _wraptext(self, text, indent=0, width=0):
        return '\n'.join(self._wrap(par, indent, width) for par in text)

Response: Wrap text with the given indent and width.


Instruction:
Create a Python 3 function for
returning brief help containing Title and usage instructions.

Response:
def shorthelp(self, width=0):
        """Return brief help containing Title and usage instructions.
        ARGS:
        width = 0 <int>:
            Maximum allowed page width. 0 means use default from
            self.iMaxHelpWidth.

        """
        out = []
        out.append(self._wrap(self.docs['title'], width=width))
        if self.docs['description']:
            out.append(self._wrap(self.docs['description'], indent=2, width=width))
        out.append('')
        out.append(self._wrapusage(width=width))
        out.append('')
        return '\n'.join(out)


Instruction:
Can you generate the documentation for the following Python 3 function
def strsettings(self, indent=0, maxindent=25, width=0):
        out = []
        makelabel = lambda name: ' ' * indent + name + ': '
        settingsindent = _autoindent([makelabel(s) for s in self.options], indent, maxindent)
        for name in self.option_order:
            option = self.options[name]
            label = makelabel(name)
            settingshelp = "%s(%s): %s" % (option.formatname, option.strvalue, option.location)
            wrapped = self._wrap_labelled(label, settingshelp, settingsindent, width)
            out.extend(wrapped)
        return '\n'.join(out)

Response: Return a string representation of the options for the current object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def settingshelp(self, width=0):
        out = []
        out.append(self._wrap(self.docs['title'], width=width))
        if self.docs['description']:
            out.append(self._wrap(self.docs['description'], indent=2, width=width))
        out.append('')
        out.append('SETTINGS:')
        out.append(self.strsettings(indent=2, width=width))
        out.append('')
        return '\n'.join(out)

Response: Return a summary of program options their values and origins.


Instruction:
How would you explain what the following Python 3 function does
def launch(self,
               argv=None,
               showusageonnoargs=False,
               width=0,
               helphint="Use with --help for more information.\n",
               debug_parser=False):
        """Do the usual stuff to initiallize the program.

        Read config files and parse arguments, and if the user has used any
        of the help/version/settings options, display help and exit.

        If debug_parser is false, don't catch ParseErrors and exit with user
        friendly help. Crash with traceback instead.

        configfiles is a list of config files. None means use self.configfiles.

        sections is a list of configfile section names to read. None means use
        self.sections.

        argv is a list of arguments to parse. Will be modified. None means use
        copy of sys.argv. argv[0] is ignored.

        If showusageonnoargs is true, show usage and exit if the user didn't
        give any args. Should be False if there are no required PositionalArgs.

        width is the maximum allowed page width. 0 means use self.width.

        helphint is a string that hints on how to get more help which is
        displayed at the end of usage help messages.
        """
        if showusageonnoargs and len(argv) == 1:
            print self.shorthelp(width=width)
            if helphint:
                print self._wrap(helphint, indent=2, width=width)
            sys.exit(0)
        parsing_error = None
        try:
            self.parse_files()
            self.parse_argv(argv)
        except ParseError, parsing_error:
            if debug_parser:
                raise
        for optiontype in ['help', 'longhelp', 'settings', 'version']:
            name = self.basic_option_names.get(optiontype)
            if name and self[name]:
                methodname = optiontype.rstrip('help') + 'help'
                print getattr(self, methodname)(width)
                sys.exit()
        if parsing_error:
            self.graceful_exit(parsing_error, width)

Response:
Do the usual stuff to initiallize the program.

        Read config files and parse arguments, and if the user has used any
        of the help/version/settings options, display help and exit.

        If debug_parser is false, don't catch ParseErrors and exit with user
        friendly help. Crash with traceback instead.

        configfiles is a list of config files. None means use self.configfiles.

        sections is a list of configfile section names to read. None means use
        self.sections.

        argv is a list of arguments to parse. Will be modified. None means use
        copy of sys.argv. argv[0] is ignored.

        If showusageonnoargs is true, show usage and exit if the user didn't
        give any args. Should be False if there are no required PositionalArgs.

        width is the maximum allowed page width. 0 means use self.width.

        helphint is a string that hints on how to get more help which is
        displayed at the end of usage help messages.


Instruction:
Create a Python 3 function for
parsing text blocks from a file.

Response:
def parse(self, file):
        """Parse text blocks from a file."""
        if isinstance(file, basestring):
            file = open(file)
        line_number = 0
        label = None
        block = self.untagged
        for line in file:
            line_number += 1
            line = line.rstrip('\n')
            if self.tabsize > 0:
                line = line.replace('\t', ' ' * self.tabsize)
            if self.decommenter:
                line = self.decommenter.decomment(line)
                if line is None:
                    continue
            tag = line.split(':', 1)[0].strip()
            # Still in the same block?
            if tag not in self.names:
                if block is None:
                    if line and not line.isspace():
                        raise ParseError(file.name, line, "garbage before first block: %r" % line)
                    continue
                block.addline(line)
                continue
            # Open a new block.
            name = self.names[tag]
            label = line.split(':',1)[1].strip()
            if name in self.labelled_classes:
                if not label:
                    raise ParseError(file.name, line, "missing label for %r block" % name)
                block = self.blocks[name].setdefault(label, self.labelled_classes[name]())
            else:
                if label:
                    msg = "label %r present for unlabelled block %r" % (label, name)
                    raise ParseError(file.name, line_number, msg)
                block = self.blocks[name]
            block.startblock()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_format(format):
    if isinstance(format, BaseFormat):
        return format
    if isinstance(format, basestring):
        for name, formatclass in globals().items():
            if name.lower() == format.lower():
                if not issubclass(formatclass, BaseFormat):
                    raise ValueError('%s is not the name of a format class' % format)
                return formatclass()
    try:
        return format()
    except:
        raise ValueError('no such format')

Response: Get a format object.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parsestr(self, argstr):
        argv = shlex.split(argstr, comments=True)
        if len(argv) != self.nargs:
            raise BadNumberOfArguments(self.nargs, len(argv))
        return self.parse(argv)

Response: Parse arguments found in settings files.


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_argument(self, arg):
        lookup = self.casesensitive and arg or arg.lower()
        if lookup in self.special:
            return self.special[lookup]
        try:
            return self.to_python(arg, *self.args, **self.kw)
        except Exception, e:
            raise BadArgument(arg, str(e))

Response: Parse a single argument.


Instruction:
Write a Python 3 script for
popping parse and return the first self. nargs items from args. Raise BadNumberOfArguments or BadArgument on errors.

Response:
def parse(self, argv):
        """Pop, parse and return the first self.nargs items from args.

        if self.nargs > 1 a list of parsed values will be returned.

        Raise BadNumberOfArguments or BadArgument on errors.

        NOTE: argv may be modified in place by this method.
        """
        if len(argv) < self.nargs:
            raise BadNumberOfArguments(self.nargs, len(argv))
        if self.nargs == 1:
            return self.parse_argument(argv.pop(0))
        return [self.parse_argument(argv.pop(0)) for tmp in range(self.nargs)]


Instruction:
Can you write a function in Python 3 where it
returns a user - friendly representation of a value.

Response:
def present(self, value):
        """Return a user-friendly representation of a value.

        Lookup value in self.specials, or call .to_literal() if absent.
        """
        for k, v in self.special.items():
            if v == value:
                return k
        return self.to_literal(value, *self.args, **self.kw)


Instruction:
Create a Python 3 function to
parse the arguments found in settings files.

Response:
def parsestr(self, argstr):
        """Parse arguments found in settings files.

        Use the values in self.true for True in settings files, or those in
        self.false for False, case insensitive.
        """
        argv = shlex.split(argstr, comments=True)
        if len(argv) != 1:
            raise BadNumberOfArguments(1, len(argv))
        arg = argv[0]
        lower = arg.lower()
        if lower in self.true:
            return True
        if lower in self.false:
            return False
        raise BadArgument(arg, "Allowed values are " + self.allowed + '.')


Instruction:
Can you create a Python 3 function that
pops parse and return the first arg from argv.

Response:
def parse(self, argv):
        """Pop, parse and return the first arg from argv.

        The arg will be .split() based on self.separator and the (optionally
        stripped) items will be parsed by self.format and returned as a list.

        Raise BadNumberOfArguments or BadArgument on errors.

        NOTE: args will be modified.
        """
        if not argv:
            raise BadNumberOfArguments(1, 0)
        argument = argv.pop(0)
        lookup = self.casesensitive and argument or argument.lower()
        if lookup in self.special:
            return self.special[lookup]
        argv = [(self.strip and s.strip() or s) for s in argument.split(self.separator)]
        values = []
        while argv:
            values.append(self.format.parse(argv))
        return values


Instruction:
Implement a function in Python 3 to
return a user - friendly representation of a value.

Response:
def present(self, value):
        """Return a user-friendly representation of a value.

        Lookup value in self.specials, or call .to_literal() if absent.
        """
        for k, v in self.special.items():
            if v == value:
                return k
        return self.separator.join(self.format.present(v) for v in value)


Instruction:
Create a Python 3 function for
returning the separator that preceding format i or an empty string for i == 0.

Response:
def get_separator(self, i):
        """Return the separator that preceding format i, or '' for i == 0."""
        return i and self.separator[min(i - 1, len(self.separator) - 1)] or ''


Instruction:
Can you generate the documentation for the following Python 3 function
def parse(self, argv):
        if not argv:
            raise BadNumberOfArguments(1, 0)
        remainder = argv.pop(0)
        lookup = self.casesensitive and remainder or remainder.lower()
        if lookup in self.special:
            return self.special[lookup]
        values = []
        for i, format in enumerate(self.format[:-1]):
            print i, self.get_separator(i)
            try:
                arg, remainder = remainder.split(self.get_separator(i + 1), 1)
            except:
                raise BadArgument(remainder, 'does not contain required separator ' + repr(self.get_separator(i + 1)))
            if self.strip:
                arg = arg.strip()
            values.append(format.parse([arg]))
        if self.strip:
            remainder = remainder.strip()
        values.append(format.parse([remainder]))
        return values

Response: Pop parse and return the first arg from argv.


Instruction:
Given the following Python 3 function, write the documentation
def present(self, value):
        for k, v in self.special.items():
            if v == value:
                return k
        return ''.join(self.get_separator(i) + self.format[i].present(v) for i, v in enumerate(value))

Response: Return a user - friendly representation of a value.


Instruction:
Explain what the following Python 3 code does
def authorize_url(self):
        auth_url = OAUTH_ROOT + '/authorize'
        params = {
            'client_id': self.client_id,
            'redirect_uri': self.redirect_uri,
        }
        return "{}?{}".format(auth_url, urlencode(params))

Response: Return a URL to redirect the user to for OAuth authentication.


Instruction:
How would you code a function in Python 3 to
exchange the authorization code for an access token.

Response:
def exchange_token(self, code):
        """
        Exchange the authorization code for an access token.
        """
        access_token_url = OAUTH_ROOT + '/access_token'
        params = {
            'client_id': self.client_id,
            'client_secret': self.client_secret,
            'redirect_uri': self.redirect_uri,
            'code': code,
        }
        resp = requests.get(access_token_url, params=params)
        if not resp.ok:
            raise MixcloudOauthError("Could not get access token.")
        return resp.json()['access_token']


Instruction:
Can you write a function in Python 3 where it
gets a PORT that will be available on the machine.

Response:
def get_open_port() -> int:
    """
    Gets a PORT that will (probably) be available on the machine.
    It is possible that in-between the time in which the open PORT of found and when it is used, another process may
    bind to it instead.
    :return: the (probably) available PORT
    """
    free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    free_socket.bind(("", 0))
    free_socket.listen(1)
    port = free_socket.getsockname()[1]
    free_socket.close()
    return port


Instruction:
Explain what the following Python 3 code does
def release(self):
        self._lock.release()

        with self._stat_lock:
            self._locked = False
            self._last_released = datetime.now()

Response: Removes the lock and resets the last acquired time.


Instruction:
Implement a function in Python 3 to
handle a dict that might contain a wrapped state for a custom type.

Response:
def default_decoder(self, obj):
        """Handle a dict that might contain a wrapped state for a custom type."""
        typename, marshalled_state = self.unwrap_callback(obj)
        if typename is None:
            return obj

        try:
            cls, unmarshaller = self.serializer.unmarshallers[typename]
        except KeyError:
            raise LookupError('no unmarshaller found for type "{}"'.format(typename)) from None

        if cls is not None:
            instance = cls.__new__(cls)
            unmarshaller(instance, marshalled_state)
            return instance
        else:
            return unmarshaller(marshalled_state)


Instruction:
Can you write a function in Python 3 where it
wraps the marshalled state in a dictionary.

Response:
def wrap_state_dict(self, typename: str, state) -> Dict[str, Any]:
        """
        Wrap the marshalled state in a dictionary.

        The returned dictionary has two keys, corresponding to the ``type_key`` and ``state_key``
        options. The former holds the type name and the latter holds the marshalled state.

        :param typename: registered name of the custom type
        :param state: the marshalled state of the object
        :return: an object serializable by the serializer

        """
        return {self.type_key: typename, self.state_key: state}


Instruction:
Make a summary of the following Python 3 code
def unwrap_state_dict(self, obj: Dict[str, Any]) -> Union[Tuple[str, Any], Tuple[None, None]]:
        if len(obj) == 2:
            typename = obj.get(self.type_key)
            state = obj.get(self.state_key)
            if typename is not None:
                return typename, state

        return None, None

Response: Unwraps a marshalled state dictionary into a tuple of the type and state.


Instruction:
Write a Python 3 function for
enabling HTTP access to a dataset.

Response:
def publish(quiet, dataset_uri):
    """Enable HTTP access to a dataset.

    This only works on datasets in some systems. For example, datasets stored
    in AWS S3 object storage and Microsoft Azure Storage can be published as
    datasets accessible over HTTP. A published dataset is world readable.
    """
    access_uri = http_publish(dataset_uri)
    if not quiet:
        click.secho("Dataset accessible at ", nl=False, fg="green")
    click.secho(access_uri)


Instruction:
Given the following Python 3 function, write the documentation
def register_custom_type(
            self, cls: type, marshaller: Optional[Callable[[Any], Any]] = default_marshaller,
            unmarshaller: Union[Callable[[Any, Any], None],
                                Callable[[Any], Any], None] = default_unmarshaller, *,
            typename: str = None, wrap_state: bool = True) -> None:
        """
        Register a marshaller and/or unmarshaller for the given class.

        The state object returned by the marshaller and passed to the unmarshaller can be any
        serializable type. Usually a dictionary mapping of attribute names to values is used.

        .. warning:: Registering marshallers/unmarshallers for any custom type will override any
            serializer specific encoding/decoding hooks (respectively) already in place!

        :param cls: the class to register
        :param marshaller: a callable that takes the object to be marshalled as the argument and
              returns a state object
        :param unmarshaller: a callable that either:

            * takes an uninitialized instance of ``cls`` and its state object as arguments and
              restores the state of the object
            * takes a state object and returns a new instance of ``cls``
        :param typename: a unique identifier for the type (defaults to the ``module:varname``
            reference to the class)
        :param wrap_state: ``True`` to wrap the marshalled state before serialization so that it
            can be recognized later for unmarshalling, ``False`` to serialize it as is

        """
        assert check_argument_types()
        typename = typename or qualified_name(cls)

        if marshaller:
            self.marshallers[cls] = typename, marshaller, wrap_state
            self.custom_type_codec.register_object_encoder_hook(self)

        if unmarshaller and self.custom_type_codec is not None:
            target_cls = cls  # type: Optional[type]
            if len(signature(unmarshaller).parameters) == 1:
                target_cls = None

            self.unmarshallers[typename] = target_cls, unmarshaller
            self.custom_type_codec.register_object_decoder_hook(self)

Response: Registers a custom type for the given class.


Instruction:
Can you implement a function in Python 3 that
updates the descriptive metadata interactively.

Response:
def _prompt_for_values(d):
    """Update the descriptive metadata interactively.

    Uses values entered by the user. Note that the function keeps recursing
    whenever a value is another ``CommentedMap`` or a ``list``. The
    function works as passing dictionaries and lists into a function edits
    the values in place.
    """
    for key, value in d.items():
        if isinstance(value, CommentedMap):
            _prompt_for_values(value)
        elif isinstance(value, list):
            for item in value:
                _prompt_for_values(item)
        else:
            typ = type(value)

            if isinstance(value, ScalarFloat):  # Deal with ruamel.yaml floats.
                typ = float

            new_value = click.prompt(key, type=typ, default=value)
            d[key] = new_value
    return d


Instruction:
Make a summary of the following Python 3 code
def create(quiet, name, base_uri, symlink_path):
    _validate_name(name)

    admin_metadata = dtoolcore.generate_admin_metadata(name)
    parsed_base_uri = dtoolcore.utils.generous_parse_uri(base_uri)

    if parsed_base_uri.scheme == "symlink":
        if symlink_path is None:
            raise click.UsageError("Need to specify symlink path using the -s/--symlink-path option")  # NOQA

    if symlink_path:
        base_uri = dtoolcore.utils.sanitise_uri(
            "symlink:" + parsed_base_uri.path
        )
        parsed_base_uri = dtoolcore.utils.generous_parse_uri(base_uri)

    # Create the dataset.
    proto_dataset = dtoolcore.generate_proto_dataset(
        admin_metadata=admin_metadata,
        base_uri=dtoolcore.utils.urlunparse(parsed_base_uri),
        config_path=CONFIG_PATH)

    # If we are creating a symlink dataset we need to set the symlink_path
    # attribute on the storage broker.
    if symlink_path:
        symlink_abspath = os.path.abspath(symlink_path)
        proto_dataset._storage_broker.symlink_path = symlink_abspath
    try:
        proto_dataset.create()
    except dtoolcore.storagebroker.StorageBrokerOSError as err:
        raise click.UsageError(str(err))

    proto_dataset.put_readme("")

    if quiet:
        click.secho(proto_dataset.uri)
    else:
        # Give the user some feedback and hints on what to do next.
        click.secho("Created proto dataset ", nl=False, fg="green")
        click.secho(proto_dataset.uri)
        click.secho("Next steps: ")

        step = 1

        if parsed_base_uri.scheme != "symlink":
            click.secho("{}. Add raw data, eg:".format(step))
            click.secho(
                "   dtool add item my_file.txt {}".format(proto_dataset.uri),
                fg="cyan")

            if parsed_base_uri.scheme == "file":
                # Find the abspath of the data directory for user feedback.
                data_path = proto_dataset._storage_broker._data_abspath
                click.secho("   Or use your system commands, e.g: ")
                click.secho(
                    "   mv my_data_directory {}/".format(data_path),
                    fg="cyan"
                )
            step = step + 1

        click.secho("{}. Add descriptive metadata, e.g: ".format(step))
        click.secho(
            "   dtool readme interactive {}".format(proto_dataset.uri),
            fg="cyan")
        step = step + 1

        click.secho(
            "{}. Convert the proto dataset into a dataset: ".format(step)
        )
        click.secho("   dtool freeze {}".format(proto_dataset.uri), fg="cyan")

Response: Create a new proto dataset.


Instruction:
Here you have a function in Python 3, explain what it does
def name(dataset_uri, new_name):
    if new_name != "":
        _validate_name(new_name)

        try:
            dataset = dtoolcore.ProtoDataSet.from_uri(
                uri=dataset_uri,
                config_path=CONFIG_PATH
            )
        except dtoolcore.DtoolCoreTypeError:
            dataset = dtoolcore.DataSet.from_uri(
                uri=dataset_uri,
                config_path=CONFIG_PATH
            )

        dataset.update_name(new_name)

    admin_metadata = dtoolcore._admin_metadata_from_uri(
        uri=dataset_uri,
        config_path=CONFIG_PATH
    )
    click.secho(admin_metadata["name"])

Response: Update the name of the dataset.


Instruction:
Can you implement a function in Python 3 that
prompts to populate the readme.

Response:
def interactive(proto_dataset_uri):
    """Interactive prompting to populate the readme."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH)

    # Create an CommentedMap representation of the yaml readme template.
    readme_template = _get_readme_template()
    yaml = YAML()
    yaml.explicit_start = True
    yaml.indent(mapping=2, sequence=4, offset=2)
    descriptive_metadata = yaml.load(readme_template)

    descriptive_metadata = _prompt_for_values(descriptive_metadata)

    # Write out the descriptive metadata to the readme file.
    stream = StringIO()

    yaml.dump(descriptive_metadata, stream)

    proto_dataset.put_readme(stream.getvalue())

    click.secho("Updated readme ", fg="green")
    click.secho("To edit the readme using your default editor:")
    click.secho(
        "dtool readme edit {}".format(proto_dataset_uri),
        fg="cyan")


Instruction:
Here you have a function in Python 3, explain what it does
def edit(dataset_uri):
    try:
        dataset = dtoolcore.ProtoDataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        dataset = dtoolcore.DataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    readme_content = dataset.get_readme_content()

    try:
        # Python2 compatibility.
        readme_content = unicode(readme_content, "utf-8")
    except NameError:
        pass

    edited_content = click.edit(readme_content)
    if edited_content is not None:
        _validate_and_put_readme(dataset, edited_content)
        click.secho("Updated readme ", nl=False, fg="green")
    else:
        click.secho("Did not update readme ", nl=False, fg="red")
    click.secho(dataset_uri)

Response: Default editor updating of readme content.


Instruction:
Can you tell what is the following Python 3 function doing
def show(dataset_uri):
    try:
        dataset = dtoolcore.ProtoDataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    except dtoolcore.DtoolCoreTypeError:
        dataset = dtoolcore.DataSet.from_uri(
            uri=dataset_uri,
            config_path=CONFIG_PATH
        )
    readme_content = dataset.get_readme_content()
    click.secho(readme_content)

Response: Show the descriptive metadata in the readme.


Instruction:
Can you tell what is the following Python 3 function doing
def write(proto_dataset_uri, input):
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri
    )
    _validate_and_put_readme(proto_dataset, input.read())

Response: Write the content of a file or stdin to the README.


Instruction:
Can you write a function in Python 3 where it
adds a file to the proto dataset.

Response:
def item(proto_dataset_uri, input_file, relpath_in_dataset):
    """Add a file to the proto dataset."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        proto_dataset_uri,
        config_path=CONFIG_PATH)
    if relpath_in_dataset == "":
        relpath_in_dataset = os.path.basename(input_file)
    proto_dataset.put_item(input_file, relpath_in_dataset)


Instruction:
Write a Python 3 script to
add metadata to a file in the proto dataset.

Response:
def metadata(proto_dataset_uri, relpath_in_dataset, key, value):
    """Add metadata to a file in the proto dataset."""
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH)
    proto_dataset.add_item_metadata(
        handle=relpath_in_dataset,
        key=key,
        value=value)


Instruction:
Given the following Python 3 function, write the documentation
def freeze(proto_dataset_uri):
    proto_dataset = dtoolcore.ProtoDataSet.from_uri(
        uri=proto_dataset_uri,
        config_path=CONFIG_PATH
    )

    num_items = len(list(proto_dataset._identifiers()))
    max_files_limit = int(dtoolcore.utils.get_config_value(
        "DTOOL_MAX_FILES_LIMIT",
        CONFIG_PATH,
        10000
    ))
    assert isinstance(max_files_limit, int)
    if num_items > max_files_limit:
        click.secho(
            "Too many items ({} > {}) in proto dataset".format(
                num_items,
                max_files_limit
            ),
            fg="red"
        )
        click.secho("1. Consider splitting the dataset into smaller datasets")
        click.secho("2. Consider packaging small files using tar")
        click.secho("3. Increase the limit using the DTOOL_MAX_FILES_LIMIT")
        click.secho("   environment variable")
        sys.exit(2)

    handles = [h for h in proto_dataset._storage_broker.iter_item_handles()]
    for h in handles:
        if not valid_handle(h):
            click.secho(
                "Invalid item name: {}".format(h),
                fg="red"
            )
            click.secho("1. Consider renaming the item")
            click.secho("2. Consider removing the item")
            sys.exit(3)

    with click.progressbar(length=len(list(proto_dataset._identifiers())),
                           label="Generating manifest") as progressbar:
        try:
            proto_dataset.freeze(progressbar=progressbar)
        except dtoolcore.storagebroker.DiskStorageBrokerValidationWarning as e:
            click.secho("")
            click.secho(str(e), fg="red", nl=False)
            sys.exit(4)

    click.secho("Dataset frozen ", nl=False, fg="green")
    click.secho(proto_dataset_uri)

Response: Convert a proto dataset into a dataset.


Instruction:
Can you tell what is the following Python 3 function doing
def copy(resume, quiet, dataset_uri, dest_base_uri):
    click.secho(
        "The ``dtool copy`` command is deprecated",
        fg="red",
        err=True
    )
    click.secho(
        "Use ``dtool cp`` instead",
        fg="red",
        err=True
    )
    _copy(resume, quiet, dataset_uri, dest_base_uri)

Response: DEPRECATED - Copy a dataset to a different location.


Instruction:
Explain what the following Python 3 code does
def cp(resume, quiet, dataset_uri, dest_base_uri):
    _copy(resume, quiet, dataset_uri, dest_base_uri)

Response: Copy a dataset to a different location.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def compress(obj, level=6, return_type="bytes"):
    if isinstance(obj, binary_type):
        b = zlib.compress(obj, level)
    elif isinstance(obj, string_types):
        b = zlib.compress(obj.encode("utf-8"), level)
    else:
        b = zlib.compress(pickle.dumps(obj, protocol=2), level)

    if return_type == "bytes":
        return b
    elif return_type == "str":
        return base64.b64encode(b).decode("utf-8")
    else:
        raise ValueError("'return_type' has to be one of 'bytes', 'str'!")

Response: Compress a single object to bytes or base64.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _parsems(value):
    if "." not in value:
        return int(value), 0
    else:
        i, f = value.split(".")
        return int(i), int(f.ljust(6, "0")[:6])

Response: Parse a I [. F ] seconds value into ( seconds microseconds ).


Instruction:
Can you tell what is the following Python 3 function doing
def find_probable_year_index(self, tokens):
        for index, token in enumerate(self):
            potential_year_tokens = _ymd.find_potential_year_tokens(
                token, tokens)
            if len(potential_year_tokens) == 1 and len(potential_year_tokens[0]) > 2:
                return index

Response: Find the index of the first occurrence of a token in the list that is not longer than 100 year.