Instruction:
How would you explain what the following Python 3 function does
def _lexsorted_specs(self, order):
        specs = self.specs[:]
        if not all(el[0] in ['+', '-'] for el in order):
            raise Exception("Please specify the keys for sorting, use"
                            "'+' prefix for ascending,"
                            "'-' for descending.)")

        sort_cycles = [(el[1:], True if el[0]=='+' else False)
                       for el in reversed(order)
                       if el[1:] in self.varying_keys]

        for (key, ascending) in sort_cycles:
            specs = sorted(specs, key=lambda s: s.get(key, None),
                           reverse=(not ascending))
        return specs

Response: Return a list of the keys that are sorted in the given order.


Instruction:
Given the following Python 3 function, write the documentation
def extract_log(log_path, dict_type=dict):
        log_path = (log_path if os.path.isfile(log_path)
                    else os.path.join(os.getcwd(), log_path))
        with open(log_path,'r') as log:
            splits = (line.split() for line in log)
            uzipped = ((int(split[0]), json.loads(" ".join(split[1:]))) for split in splits)
            szipped = [(i, dict((str(k),v) for (k,v) in d.items())) for (i,d) in uzipped]
        return dict_type(szipped)

Response: Extracts the log file generated by a launcher and returns the dictionary with tid keys and specification values.


Instruction:
Can you tell what is the following Python 3 function doing
def write_log(log_path, data, allow_append=True):
        append = os.path.isfile(log_path)
        islist = isinstance(data, list)

        if append and not allow_append:
            raise Exception('Appending has been disabled'
                            ' and file %s exists' % log_path)

        if not (islist or isinstance(data, Args)):
            raise Exception('Can only write Args objects or dictionary'
                            ' lists to log file.')

        specs = data if islist else data.specs
        if not all(isinstance(el,dict) for el in specs):
            raise Exception('List elements must be dictionaries.')

        log_file = open(log_path, 'r+') if append else open(log_path, 'w')
        start = int(log_file.readlines()[-1].split()[0])+1 if append else 0
        ascending_indices = range(start, start+len(data))

        log_str = '\n'.join(['%d %s' % (tid, json.dumps(el))
                             for (tid, el) in zip(ascending_indices,specs)])
        log_file.write("\n"+log_str if append else log_str)
        log_file.close()

Response: Writes the supplied data to the log file.


Instruction:
Write a Python 3 function for
loading all the files in a given directory selecting only files with the given extension if specified.

Response:
def directory(cls, directory, root=None, extension=None, **kwargs):
        """
        Load all the files in a given directory selecting only files
        with the given extension if specified. The given kwargs are
        passed through to the normal constructor.
        """
        root = os.getcwd() if root is None else root
        suffix = '' if extension is None else '.' + extension.rsplit('.')[-1]
        pattern = directory + os.sep + '*' + suffix
        key = os.path.join(root, directory,'*').rsplit(os.sep)[-2]
        format_parse = list(string.Formatter().parse(key))
        if not all([el is None for el in zip(*format_parse)[1]]):
            raise Exception('Directory cannot contain format field specifications')
        return cls(key, pattern, root, **kwargs)


Instruction:
Make a summary of the following Python 3 code
def fields(self):
        parse = list(string.Formatter().parse(self.pattern))
        return [f for f in zip(*parse)[1] if f is not None]

Response:
Return the fields specified in the pattern using Python s
        formatting mini - language.


Instruction:
Create a Python 3 function to
load the files that match the given pattern.

Response:
def _load_expansion(self, key, root, pattern):
        """
        Loads the files that match the given pattern.
        """
        path_pattern = os.path.join(root, pattern)
        expanded_paths = self._expand_pattern(path_pattern)

        specs=[]
        for (path, tags) in expanded_paths:
            filelist = [os.path.join(path,f) for f in os.listdir(path)] if os.path.isdir(path) else [path]
            for filepath in filelist:
                specs.append(dict(tags,**{key:os.path.abspath(filepath)}))

        return sorted(specs, key=lambda s: s[key])


Instruction:
Here you have a function in Python 3, explain what it does
def _expand_pattern(self, pattern):
        (globpattern, regexp, fields, types) = self._decompose_pattern(pattern)
        filelist = glob.glob(globpattern)
        expansion = []

        for fname in filelist:
            if fields == []:
                expansion.append((fname, {}))
                continue
            match = re.match(regexp, fname)
            if match is None: continue
            match_items = match.groupdict().items()
            tags = dict((k,types.get(k, str)(v)) for (k,v) in match_items)
            expansion.append((fname, tags))

        return expansion

Response: Given a pattern expands the pattern into a list of absolute paths and returns a list of dicts where the keys are the absolute paths of the files and the values are the tags of the files.


Instruction:
How would you implement a function in Python 3 that
gives a path pattern with format declaration generates a four - tuple

Response:
def _decompose_pattern(self, pattern):
        """
        Given a path pattern with format declaration, generates a
        four-tuple (glob_pattern, regexp pattern, fields, type map)
        """
        sep = '~lancet~sep~'
        float_codes = ['e','E','f', 'F','g', 'G', 'n']
        typecodes = dict([(k,float) for k in float_codes]
                         + [('b',bin), ('d',int), ('o',oct), ('x',hex)])
        parse = list(string.Formatter().parse(pattern))
        text, fields, codes, _ = zip(*parse)

        # Finding the field types from format string
        types = []
        for (field, code) in zip(fields, codes):
            if code in ['', None]: continue
            constructor =  typecodes.get(code[-1], None)
            if constructor: types += [(field, constructor)]

        stars =  ['' if not f else '*' for f in fields]
        globpat = ''.join(text+star for (text,star) in zip(text,stars))

        refields = ['' if not f else sep+('(?P<%s>.*?)'% f)+sep for f in fields]
        parts = ''.join(text+group for (text,group) in zip(text, refields)).split(sep)
        for i in range(0, len(parts), 2): parts[i] = re.escape(parts[i])

        regexp_pattern = ''.join(parts).replace('\\*','.*')
        fields = list(f for f in fields if f)
        return globpat, regexp_pattern , fields, dict(types)


Instruction:
Make a summary of the following Python 3 code
def from_pattern(cls, pattern, filetype=None, key='filename', root=None, ignore=[]):
        filepattern = FilePattern(key, pattern, root=root)
        if FileInfo.filetype and filetype is None:
            filetype = FileInfo.filetype
        elif filetype is None:
            raise Exception("The filetype argument must be supplied unless "
                            "an appropriate default has been specified as "
                            "FileInfo.filetype")
        return FileInfo(filepattern, key, filetype, ignore=ignore)

Response: Convenience method to chain a pattern into a FileInfo instance.


Instruction:
Can you generate the documentation for the following Python 3 function
def load(self, val, **kwargs):
        if Table and isinstance(val, Table):
            return self.load_table(val, **kwargs)
        elif DataFrame and isinstance(val, DataFrame):
            return self.load_dframe(val, **kwargs)
        else:
            raise Exception("Type %s not a DataFrame or Table." % type(val))

Response: Load the contents into the supplied pandas dataframe or HoloViews Table.


Instruction:
How would you explain what the following Python 3 function does
def load_table(self, table):
        items,  data_keys = [], None
        for key, filename in table.items():
            data_dict = self.filetype.data(filename[0])
            current_keys = tuple(sorted(data_dict.keys()))
            values = [data_dict[k] for k in current_keys]
            if data_keys is None:
                data_keys = current_keys
            elif data_keys != current_keys:
                raise Exception("Data keys are inconsistent")
            items.append((key, values))

        return Table(items, kdims=table.kdims, vdims=data_keys)

Response: Load the contents of the file into the supplied Table using the specified key and filetype.


Instruction:
Here you have a function in Python 3, explain what it does
def load_dframe(self, dframe):
        filename_series = dframe[self.key]
        loaded_data = filename_series.map(self.filetype.data)
        keys = [list(el.keys()) for el in loaded_data.values]
        for key in set().union(*keys):
            key_exists = key in dframe.columns
            if key_exists:
                self.warning("Appending '_data' suffix to data key %r to avoid"
                             "overwriting existing metadata with the same name." % key)
            suffix = '_data' if key_exists else ''
            dframe[key+suffix] = loaded_data.map(lambda x: x.get(key, np.nan))
        return dframe

Response: Load the file contents into the supplied dataframe using the specified key and filetype.


Instruction:
Make a summary of the following Python 3 code
def _info(self, source, key, filetype, ignore):
        specs, mdata = [], {}
        mdata_clashes  = set()

        for spec in source.specs:
            if key not in spec:
                raise Exception("Key %r not available in 'source'." % key)

            mdata = dict((k,v) for (k,v) in filetype.metadata(spec[key]).items()
                         if k not in ignore)
            mdata_spec = {}
            mdata_spec.update(spec)
            mdata_spec.update(mdata)
            specs.append(mdata_spec)
            mdata_clashes = mdata_clashes | (set(spec.keys()) & set(mdata.keys()))
        # Metadata clashes can be avoided by using the ignore list.
        if mdata_clashes:
            self.warning("Loaded metadata keys overriding source keys.")
        return specs

Response: Generates the union of the source. specs and the metadata dicts loaded by the filetype object.


Instruction:
Can you implement a function in Python 3 that
pushes new data into the buffer. Resume looping if paused.

Response:
async def _push(self, *args, **kwargs):
        """Push new data into the buffer. Resume looping if paused."""
        self._data.append((args, kwargs))
        if self._future is not None:

            future, self._future = self._future, None
            future.set_result(True)


Instruction:
Here you have a function in Python 3, explain what it does
def newVersion():
    version=None
    fname='../swhlab/version.py'
    with open(fname) as f:
        raw=f.read().split("\n")
        for i,line in enumerate(raw):
            if line.startswith("__counter__"):
                if version is None:
                    version = int(line.split("=")[1])
                raw[i]="__counter__=%d"%(version+1)
    with open(fname,'w') as f:
        f.write("\n".join(raw))
    print("upgraded from version %03d to %03d"%(version,version+1))

Response: increments version counter in swhlab version. py


Instruction:
Implement a Python 3 function for
creating a plot of one area of interest of a single sweep.

Response:
def figureStimulus(abf,sweeps=[0]):
    """
    Create a plot of one area of interest of a single sweep.
    """

    stimuli=[2.31250, 2.35270]
    for sweep in sweeps:
        abf.setsweep(sweep)
        for stimulus in stimuli:
            S1=int(abf.pointsPerSec*stimulus)
            S2=int(abf.pointsPerSec*(stimulus+0.001)) # 1ms of blanking
            abf.sweepY[S1:S2]=np.nan # blank out the stimulus area
        I1=int(abf.pointsPerSec*2.2) # time point (sec) to start
        I2=int(abf.pointsPerSec*2.6) # time point (sec) to end
        baseline=np.average(abf.sweepY[int(abf.pointsPerSec*2.0):int(abf.pointsPerSec*2.2)])
        Ys=lowPassFilter(abf.sweepY[I1:I2])-baseline
        Xs=abf.sweepX2[I1:I1+len(Ys)].flatten()
        plt.plot(Xs,Ys,alpha=.5,lw=2)
    return


Instruction:
Make a summary of the following Python 3 code
def analyzeSingle(abfFname):
    assert os.path.exists(abfFname) and abfFname.endswith(".abf")
    ABFfolder,ABFfname=os.path.split(abfFname)
    abfID=os.path.splitext(ABFfname)[0]
    IN=INDEX(ABFfolder)
    IN.analyzeABF(abfID)
    IN.scan()
    IN.html_single_basic([abfID],overwrite=True)
    IN.html_single_plot([abfID],overwrite=True)
    IN.scan()
    IN.html_index()

    return

Response: Reanalyze data for a single ABF. Also remakes child and parent html.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def scan(self):
        t1=cm.timeit()
        self.files1=cm.list_to_lowercase(sorted(os.listdir(self.folder1)))
        self.files2=cm.list_to_lowercase(sorted(os.listdir(self.folder2)))
        self.files1abf=[x for x in self.files1 if x.endswith(".abf")]
        self.files1abf=cm.list_to_lowercase(cm.abfSort(self.files1abf))
        self.IDs=[x[:-4] for x in self.files1abf]
        self.log.debug("folder1 has %d files",len(self.files1))
        self.log.debug("folder1 has %d abfs",len(self.files1abf))
        self.log.debug("folder2 has %d files",len(self.files2))
        self.log.debug("scanning folders took %s",cm.timeit(t1))

Response: Scan the folder1 and folder2 into files1 and files2.


Instruction:
Write a Python 3 function that can
run this to turn all files in folder1 TIFs and JPGs into folder2 data.

Response:
def convertImages(self):
        """
        run this to turn all folder1 TIFs and JPGs into folder2 data.
        TIFs will be treated as micrographs and converted to JPG with enhanced
        contrast. JPGs will simply be copied over.
        """

        # copy over JPGs (and such)
        exts=['.jpg','.png']
        for fname in [x for x in self.files1 if cm.ext(x) in exts]:
            ID="UNKNOWN"
            if len(fname)>8 and fname[:8] in self.IDs:
                ID=fname[:8]
            fname2=ID+"_jpg_"+fname
            if not fname2 in self.files2:
                self.log.info("copying over [%s]"%fname2)
                shutil.copy(os.path.join(self.folder1,fname),os.path.join(self.folder2,fname2))
            if not fname[:8]+".abf" in self.files1:
                self.log.error("orphan image: %s",fname)

        # convert TIFs (and such) to JPGs
        exts=['.tif','.tiff']
        for fname in [x for x in self.files1 if cm.ext(x) in exts]:
            ID="UNKNOWN"
            if len(fname)>8 and fname[:8] in self.IDs:
                ID=fname[:8]
            fname2=ID+"_tif_"+fname+".jpg"
            if not fname2 in self.files2:
                self.log.info("converting micrograph [%s]"%fname2)
                imaging.TIF_to_jpg(os.path.join(self.folder1,fname),saveAs=os.path.join(self.folder2,fname2))
            if not fname[:8]+".abf" in self.files1:
                self.log.error("orphan image: %s",fname)


Instruction:
Write a Python 3 function that can
analyze every unanalyzed ABF in the folder

Response:
def analyzeAll(self):
        """analyze every unanalyzed ABF in the folder."""
        searchableData=str(self.files2)
        self.log.debug("considering analysis for %d ABFs",len(self.IDs))
        for ID in self.IDs:
            if not ID+"_" in searchableData:
                self.log.debug("%s needs analysis",ID)
                try:
                    self.analyzeABF(ID)
                except:
                    print("EXCEPTION! "*100)
            else:
                self.log.debug("%s has existing analysis, not overwriting",ID)
        self.log.debug("verified analysis of %d ABFs",len(self.IDs))


Instruction:
Here you have a function in Python 3, explain what it does
def analyzeABF(self,ID):
        for fname in self.files2:
            if fname.startswith(ID+"_data_"):
                self.log.debug("deleting [%s]",fname)
                os.remove(os.path.join(self.folder2,fname))
        self.log.info("analyzing (with overwrite) [%s]",ID)
        protocols.analyze(os.path.join(self.folder1,ID+".abf"))

Response: Analye a single ABF and index it.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def htmlFor(self,fname):
        if os.path.splitext(fname)[1].lower() in ['.jpg','.png']:
            html=''%(fname,fname)
            if "_tif_" in fname:
                html=html.replace('<img ','<img class="datapic micrograph"')
            if "_plot_" in fname:
                html=html.replace('<img ','<img class="datapic intrinsic" ')
            if "_experiment_" in fname:
                html=html.replace('<img ','<img class="datapic experiment" ')
        elif os.path.splitext(fname)[1].lower() in ['.html','.htm']:
            html='LINK: %s'%fname
        else:
            html='
Not sure how to show: [%s]
'%fname
        return html

Response: return appropriate HTML determined by file extension.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def html_single_basic(self,abfID,launch=False,overwrite=False):
        if type(abfID) is str:
            abfID=[abfID]
        for thisABFid in cm.abfSort(abfID):
            parentID=cm.parent(self.groups,thisABFid)
            saveAs=os.path.abspath("%s/%s_basic.html"%(self.folder2,parentID))
            if overwrite is False and os.path.basename(saveAs) in self.files2:
                continue
            filesByType=cm.filesByType(self.groupFiles[parentID])
            html=""
            html+='
'
            html+='summary of data from: %s
'%parentID
            html+='%s'%os.path.abspath(self.folder1+"/"+parentID+".abf")
            html+='
'
            catOrder=["experiment","plot","tif","other"]
            categories=cm.list_order_by(filesByType.keys(),catOrder)
            for category in [x for x in categories if len(filesByType[x])]:
                if category=='experiment':
                    html+="
Experimental Data:
"
                elif category=='plot':
                    html+="
Intrinsic Properties:
"
                elif category=='tif':
                    html+="
Micrographs:
"
                elif category=='other':
                    html+="
Additional Files:
"
                else:
                    html+="
????:
"
                #html+="
"
                #html+='
'*3
                for fname in filesByType[category]:
                    html+=self.htmlFor(fname)
                html+='
'*3
            print("creating",saveAs,'...')
            style.save(html,saveAs,launch=launch)

Response: Generate a generic flat file html for a single ABF.


Instruction:
Can you create a Python 3 function that
creates HTML of just intrinsic properties for a single file.

Response:
def html_single_plot(self,abfID,launch=False,overwrite=False):
        """create ID_plot.html of just intrinsic properties."""
        if type(abfID) is str:
            abfID=[abfID]
        for thisABFid in cm.abfSort(abfID):
            parentID=cm.parent(self.groups,thisABFid)
            saveAs=os.path.abspath("%s/%s_plot.html"%(self.folder2,parentID))
            if overwrite is False and os.path.basename(saveAs) in self.files2:
                continue
            filesByType=cm.filesByType(self.groupFiles[parentID])
            html=""
            html+='
'
            html+='intrinsic properties for: %s
'%parentID
            html+='%s'%os.path.abspath(self.folder1+"/"+parentID+".abf")
            html+='
'
            for fname in filesByType['plot']:
                html+=self.htmlFor(fname)
            print("creating",saveAs,'...')
            style.save(html,saveAs,launch=launch)


Instruction:
Explain what the following Python 3 code does
def lowpass(data,filterSize=None):
    if filterSize is None:
        filterSize=len(data)/10
    kernel=kernel_gaussian(size=filterSize)
    data=convolve(data,kernel) # do the convolution with padded edges
    return data

Response: minimal complexity low - pass filtering.


Instruction:
Here you have a function in Python 3, explain what it does
def convolve(signal,kernel):
    pad=np.ones(len(kernel)/2)
    signal=np.concatenate((pad*signal[0],signal,pad*signal[-1]))
    signal=np.convolve(signal,kernel,mode='same')
    signal=signal[len(pad):-len(pad)]
    return signal

Response: This function applies a kernel to a signal and returns the result.


Instruction:
Can you tell what is the following Python 3 function doing
def timeit(timer=None):
    if timer is None:
        return time.time()
    else:
        took=time.time()-timer
        if took<1:
            return "%.02f ms"%(took*1000.0)
        elif took<60:
            return "%.02f s"%(took)
        else:
            return "%.02f min"%(took/60.0)

Response: simple timer. returns a time object or a string.


Instruction:
Create a Python 3 function to
move the value to the front and return it

Response:
def list_move_to_front(l,value='other'):
    """if the value is in the list, move it to the front and return it."""
    l=list(l)
    if value in l:
        l.remove(value)
        l.insert(0,value)
    return l


Instruction:
Here you have a function in Python 3, explain what it does
def list_move_to_back(l,value='other'):
    l=list(l)
    if value in l:
        l.remove(value)
        l.append(value)
    return l

Response: move the value to the back and return it


Instruction:
How would you implement a function in Python 3 that
gives a list and a list of items to be first return the list in the same order except that it begins with each of the first items.

Response:
def list_order_by(l,firstItems):
    """given a list and a list of items to be first, return the list in the
    same order except that it begins with each of the first items."""
    l=list(l)
    for item in firstItems[::-1]: #backwards
        if item in l:
            l.remove(item)
            l.insert(0,item)
    return l


Instruction:
Create a Python 3 function to
give a list of goofy ABF names return it sorted intelligently.

Response:
def abfSort(IDs):
    """
    given a list of goofy ABF names, return it sorted intelligently.
    This places things like 16o01001 after 16901001.
    """
    IDs=list(IDs)
    monO=[]
    monN=[]
    monD=[]
    good=[]
    for ID in IDs:
        if ID is None:
            continue
        if 'o' in ID:
            monO.append(ID)
        elif 'n' in ID:
            monN.append(ID)
        elif 'd' in ID:
            monD.append(ID)
        else:
            good.append(ID)
    return sorted(good)+sorted(monO)+sorted(monN)+sorted(monD)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def abfGroups(abfFolder):

    # prepare the list of files, filenames, and IDs
    files=False
    if type(abfFolder) is str and os.path.isdir(abfFolder):
        files=abfSort(os.listdir(abfFolder))
    elif type(abfFolder) is list:
        files=abfSort(abfFolder)
    assert type(files) is list
    files=list_to_lowercase(files)

    # group every filename in a different list, and determine parents
    abfs, IDs, others, parents, days = [],[],[],[],[]
    for fname in files:
        if fname.endswith(".abf"):
            abfs.append(fname)
            IDs.append(fname[:-4])
            days.append(fname[:5])
        else:
            others.append(fname)
    for ID in IDs:
        for fname in others:
            if fname.startswith(ID):
                parents.append(ID)
    parents=abfSort(set(parents)) # allow only one copy each
    days=abfSort(set(days)) # allow only one copy each

    # match up children with parents, respecting daily orphans.
    groups={}
    for day in days:
        parent=None
        for fname in [x for x in abfs if x.startswith(day)]:
            ID=fname[:-4]
            if ID in parents:
                parent=ID
            if not parent in groups.keys():
                groups[parent]=[]
            groups[parent].extend([ID])
    return groups

Response: Given a folder path or list of files return a dict of parent IDs and children IDs.


Instruction:
Given the following Python 3 function, write the documentation
def abfGroupFiles(groups,folder):
    assert os.path.exists(folder)
    files=os.listdir(folder)
    group2={}
    for parent in groups.keys():
        if not parent in group2.keys():
            group2[parent]=[]
        for ID in groups[parent]:
            for fname in [x.lower() for x in files if ID in x.lower()]:
                group2[parent].extend([fname])
    return group2

Response: This function takes a dictionary of groups and a folder and returns a dictionary of all files in that folder.


Instruction:
Explain what the following Python 3 code does
def parent(groups,ID):
    if ID in groups.keys():
        return ID # already a parent
    if not ID in groups.keys():
        for actualParent in groups.keys():
            if ID in groups[actualParent]:
                return actualParent # found the actual parent
    return None

Response: given a groups dictionary and an ID return its actual parent ID.


Instruction:
Here you have a function in Python 3, explain what it does
def filesByType(fileList):
    features=["plot","tif","data","other","experiment"]
    files={}
    for feature in features:
        files[feature]=[]
    for fname in fileList:
        other=True
        for feature in features:
            if "_"+feature+"_" in fname:
                files[feature].extend([fname])
                other=False
        if other:
            files['other'].extend([fname])
    return files

Response: return a list of files sorted by type


Instruction:
How would you explain what the following Python 3 function does
def userFolder():
    #path=os.path.abspath(tempfile.gettempdir()+"/swhlab/")
    #don't use tempdir! it will get deleted easily.
    path=os.path.expanduser("~")+"/.swhlab/" # works on windows or linux
    # for me, path=r"C:\Users\swharden\.swhlab"
    if not os.path.exists(path):
        print("creating",path)
        os.mkdir(path)
    return os.path.abspath(path)

Response: return the semi - temporary user folder


Instruction:
Here you have a function in Python 3, explain what it does
def abfFname_Load():
    fname=userFolder()+"/abfFname.ini"
    if os.path.exists(fname):
        abfFname=open(fname).read().strip()
        if os.path.exists(abfFname) or abfFname.endswith("_._"):
            return abfFname
    return os.path.abspath(os.sep)

Response: return the path of the last loaded ABF


Instruction:
Write a Python 3 function for
returning the path of the last loaded ABF.

Response:
def abfFname_Save(abfFname):
    """return the path of the last loaded ABF."""
    fname=userFolder()+"/abfFname.ini"
    with open(fname,'w') as f:
        f.write(os.path.abspath(abfFname))
    return


Instruction:
Can you tell what is the following Python 3 function doing
def gui_getFile():
    import tkinter as tk
    from tkinter import filedialog
    root = tk.Tk() # this is natively supported by python
    root.withdraw() # hide main window
    root.wm_attributes('-topmost', 1) # always on top
    fname = filedialog.askopenfilename(title = "select ABF file",
                                       filetypes=[('ABF Files', '.abf')],
                                       initialdir=os.path.dirname(abfFname_Load()))
    if fname.endswith(".abf"):
        abfFname_Save(fname)
        return fname
    else:
        print("didn't select an ABF!")
        return None

Response: Launch an ABF file selection file dialog.


Instruction:
Can you generate the documentation for the following Python 3 function
def gui_getFolder():
    import tkinter as tk
    from tkinter import filedialog
    root = tk.Tk() # this is natively supported by python
    root.withdraw() # hide main window
    root.wm_attributes('-topmost', 1) # always on top
    fname = filedialog.askdirectory(title = "select folder of ABFs",
                                       initialdir=os.path.dirname(abfFname_Load()))
    if len(fname)>3:
        abfFname_Save(fname+"/_._")
        return fname
    else:
        print("didn't select an ABF!")
        return None

Response: Launch a folder selection dialog.


Instruction:
Can you generate the documentation for the following Python 3 function
async def _try_catch_coro(emitter, event, listener, coro):
    try:

        await coro

    except Exception as exc:

        if event == emitter.LISTENER_ERROR_EVENT:

            raise

        emitter.emit(emitter.LISTENER_ERROR_EVENT, event, listener, exc)

Response: Coroutine wrapper to catch errors after async scheduling.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _check_limit(self, event):
        if self.count(event) > self.max_listeners:

            warnings.warn(
                'Too many listeners for event {}'.format(event),
                ResourceWarning,
            )

Response: Check if the listener limit is hit and warn if needed.


Instruction:
Implement a Python 3 function for
binding a listener to an event.

Response:
def add_listener(self, event, listener):
        """Bind a listener to a particular event.

        Args:
            event (str): The name of the event to listen for. This may be any
                string value.
            listener (def or async def): The callback to execute when the event
                fires. This may be a sync or async function.
        """
        self.emit('new_listener', event, listener)
        self._listeners[event].append(listener)
        self._check_limit(event)
        return self


Instruction:
Write a Python 3 script for
adding a listener that is only called once.

Response:
def once(self, event, listener):
        """Add a listener that is only called once."""
        self.emit('new_listener', event, listener)
        self._once[event].append(listener)
        self._check_limit(event)
        return self


Instruction:
Create a Python 3 function to
remove a listener from the emitter.

Response:
def remove_listener(self, event, listener):
        """Remove a listener from the emitter.

        Args:
            event (str): The event name on which the listener is bound.
            listener: A reference to the same object given to add_listener.

        Returns:
            bool: True if a listener was removed else False.

        This method only removes one listener at a time. If a listener is
        attached multiple times then this method must be called repeatedly.
        Additionally, this method removes listeners first from the those
        registered with 'on' or 'add_listener'. If none are found it continue
        to remove afterwards from those added with 'once'.
        """
        with contextlib.suppress(ValueError):

            self._listeners[event].remove(listener)
            return True

        with contextlib.suppress(ValueError):

            self._once[event].remove(listener)
            return True

        return False


Instruction:
Can you write a function in Python 3 where it
removes all listeners or those of the specified event.

Response:
def remove_all_listeners(self, event=None):
        """Remove all listeners, or those of the specified *event*.

        It's not a good idea to remove listeners that were added elsewhere in
        the code, especially when it's on an emitter that you didn't create
        (e.g. sockets or file streams).
        """
        if event is None:
            self._listeners = collections.defaultdict(list)
            self._once = collections.defaultdict(list)
        else:
            del self._listeners[event]
            del self._once[event]


Instruction:
How would you explain what the following Python 3 function does
def _dispatch_coroutine(self, event, listener, *args, **kwargs):
        try:

            coro = listener(*args, **kwargs)

        except Exception as exc:

            if event == self.LISTENER_ERROR_EVENT:

                raise

            return self.emit(self.LISTENER_ERROR_EVENT, event, listener, exc)

        asyncio.ensure_future(
            _try_catch_coro(self, event, listener, coro),
            loop=self._loop,
        )

Response: Dispatches a coroutine for execution.


Instruction:
Implement a function in Python 3 to
dispatch a sync function.

Response:
def _dispatch_function(self, event, listener, *args, **kwargs):
        """Execute a sync function.

        Args:
            event (str): The name of the event that triggered this call.
            listener (def): The def that needs to be executed.
            *args: Any number of positional arguments.
            **kwargs: Any number of keyword arguments.

        The values of *args and **kwargs are passed, unaltered, to the def
        when exceuting. If there is an exception executing the def, such as the
        wrong number of arguments, the emitter's error event is triggered. If
        the triggering event _is_ the emitter's error event then the exception
        is reraised. The reraised exception may show in debug mode for the
        event loop but is otherwise silently dropped.
        """
        try:

            return listener(*args, **kwargs)

        except Exception as exc:

            if event == self.LISTENER_ERROR_EVENT:

                raise

            return self.emit(self.LISTENER_ERROR_EVENT, event, listener, exc)


Instruction:
Here you have a function in Python 3, explain what it does
def _dispatch(self, event, listener, *args, **kwargs):
        if (
            asyncio.iscoroutinefunction(listener) or
            isinstance(listener, functools.partial) and
            asyncio.iscoroutinefunction(listener.func)
        ):

            return self._dispatch_coroutine(event, listener, *args, **kwargs)

        return self._dispatch_function(event, listener, *args, **kwargs)

Response: Dispatch an event to a listener.


Instruction:
Can you generate the documentation for the following Python 3 function
def emit(self, event, *args, **kwargs):
        listeners = self._listeners[event]
        listeners = itertools.chain(listeners, self._once[event])
        self._once[event] = []
        for listener in listeners:

            self._loop.call_soon(
                functools.partial(
                    self._dispatch,
                    event,
                    listener,
                    *args,
                    **kwargs,
                )
            )

        return self

Response: Calls each listener for the given event with the given arguments.


Instruction:
Can you implement a function in Python 3 that
returns the number of listeners for the event.

Response:
def count(self, event):
        """Get the number of listeners for the event.

        Args:
            event (str): The event for which to count all listeners.

        The resulting count is a combination of listeners added using
        'on'/'add_listener' and 'once'.
        """
        return len(self._listeners[event]) + len(self._once[event])


Instruction:
Write a Python 3 script to
convert each TIF to PNG. Return filenames of new PNGs.

Response:
def genPNGs(folder,files=None):
    """Convert each TIF to PNG. Return filenames of new PNGs."""
    if files is None:
        files=glob.glob(folder+"/*.*")
    new=[]
    for fname in files:
        ext=os.path.basename(fname).split(".")[-1].lower()
        if ext in ['tif','tiff']:
            if not os.path.exists(fname+".png"):
                print(" -- converting %s to PNG..."%os.path.basename(fname))
                cm.image_convert(fname)
                new.append(fname) #fancy burn-in of image data
            else:
                pass
                #print(" -- already converted %s to PNG..."%os.path.basename(fname))
    return new


Instruction:
Implement a Python 3 function for
generating text to go inside HTML page for single ABF page.

Response:
def htmlABFcontent(ID,group,d):
    """generate text to go inside <body> for single ABF page."""
    html=""
    files=[]
    for abfID in group:
        files.extend(d[abfID])
    files=sorted(files)

    #start with root images
    html+="<hr>"
    for fname in files:
        if ".png" in fname.lower() and not "swhlab4" in fname:
            fname="../"+os.path.basename(fname)
            html+='<a href="%s"><img src="%s" width="348"></a> '%(fname,fname)

    #progress to /swhlab4/ images
    html+="<hr>"
    #ABFinfo
    lastID=''
    for fname in sorted(files):
        if not "swhlab4" in fname:
            continue
        ID=os.path.basename(fname).split("_")[0]
        if not ID==lastID:
            lastID=ID
            html+="<h3>%s</h3>"%os.path.basename(fname).split("_")[0]
        if ".png" in fname.lower():
            fname=os.path.basename(fname)
            html+='<a href="%s"><img src="%s" height="300"></a> '%(fname,fname)
            continue

    html+="<hr>"
    for fname in files:
        if not "swhlab4" in fname:
            continue
        if ".pkl" in fname:
            callit=os.path.basename(fname)
            thing=cm.getPkl(fname)
            if "_APs.pkl" in fname:
                callit+=" (first AP)"
                thing=cm.dictFlat(thing)
                if len(thing):
                    thing=thing[0]
            elif "_MTs.pkl" in fname:
                if type(thing) == dict:
                    callit+=" (from AVG of all sweeps)"
                else:
                    callit+=" (first sweep)"
                    thing=thing[0]
            elif "_SAP.pkl" in fname:
                continue #don't plot those, too complicated
            elif "_info.pkl" in fname or "_iv.pkl" in fname:
                pass #no trouble, go for it
            else:
                print(" ?? not sure how to index [%s]"%os.path.basename(fname))
                continue
            if type(thing) is dict:
                thing=cm.msgDict(thing)
            if type(thing) is list:
                out=''
                for item in thing:
                    out+=str(item)+"\n"
                thing=out
            thing=str(thing) #lol stringthing
            thing="### %s ###\n"%os.path.basename(fname)+thing

            # putting it in a textbox is obnoxious. put it in the source instead.
            #html+='<br><br><textarea rows="%d" cols="70">%s</textarea>'%(str(thing).count("\n")+5,thing)

            html+="(view source for %s) <!--\n\n%s\n\n-->"%(os.path.basename(fname),thing)

    return html


Instruction:
Write a Python 3 script to
generate a static html for that abf

Response:
def htmlABF(ID,group,d,folder,overwrite=False):
    """given an ID and the dict of files, generate a static html for that abf."""
    fname=folder+"/swhlab4/%s_index.html"%ID
    if overwrite is False and os.path.exists(fname):
        return
    html=TEMPLATES['abf']
    html=html.replace("~ID~",ID)
    html=html.replace("~CONTENT~",htmlABFcontent(ID,group,d))
    print(" <- writing [%s]"%os.path.basename(fname))
    with open(fname,'w') as f:
        f.write(html)
    return


Instruction:
Write a Python 3 script to
read experiment. txt and return a dict with [ firstOfNewExp color star comments

Response:
def expMenu(groups,folder):
    """read experiment.txt and return a dict with [firstOfNewExp, color, star, comments]."""
    ### GENERATE THE MENU DATA BASED ON EXPERIMENT FILE
    orphans = sorted(list(groups.keys()))
    menu=[]
    if os.path.exists(folder+'/experiment.txt'):
        with open(folder+'/experiment.txt') as f:
            raw=f.read()
    else:
        raw=""
    for line in raw.split("\n"):
        item={}
        if len(line)==0:
            continue
        if line.startswith("~"):
            line=line[1:].split(" ",2)
            item["ID"]=line[0]
            item["symbol"]=''
            if len(line)>1:
                item["color"]=line[1]
            else:
                item["color"]="white"
            if len(line)>2 and len(line[2]):
                item["comment"]=line[2]
                if item["comment"][0]=="*":
                    item["symbol"]='*'
            else:
                item["comment"]=''
            if item["ID"] in orphans:
                orphans.remove(item["ID"])
        elif line.startswith("###"):
            line=line[3:].strip().split(" ",1)
            item["title"]=line[0]
            item["comment"]=''
            if len(line)>1:
                if line[1].startswith("- "):
                    line[1]=line[1][2:]
                item["comment"]=line[1]
        else:
            item["unknown"]=line
        menu.append(item)
    menu.append({"title":"orphans","comment":""})
    for ophan in orphans:
        menu.append({"orphan":ophan,"ID":ophan,"color":'',"symbol":'',"comment":''})
    return menu


Instruction:
Can you tell what is the following Python 3 function doing
def genIndex(folder,forceIDs=[]):
    if not os.path.exists(folder+"/swhlab4/"):
        print(" !! cannot index if no /swhlab4/")
        return
    timestart=cm.timethis()
    files=glob.glob(folder+"/*.*") #ABF folder
    files.extend(glob.glob(folder+"/swhlab4/*.*"))
    print(" -- indexing glob took %.02f ms"%(cm.timethis(timestart)*1000))
    files.extend(genPNGs(folder,files))
    files=sorted(files)
    timestart=cm.timethis()
    d=cm.getIDfileDict(files) #TODO: this is really slow
    print(" -- filedict length:",len(d))
    print(" -- generating ID dict took %.02f ms"%(cm.timethis(timestart)*1000))
    groups=cm.getABFgroups(files)
    print(" -- groups length:",len(groups))
    for ID in sorted(list(groups.keys())):
        overwrite=False
        for abfID in groups[ID]:
            if abfID in forceIDs:
                overwrite=True
        try:
            htmlABF(ID,groups[ID],d,folder,overwrite)
        except:
            print("~~ HTML GENERATION FAILED!!!")
    menu=expMenu(groups,folder)
    makeSplash(menu,folder)
    makeMenu(menu,folder)
    htmlFrames(d,folder)
    makeMenu(menu,folder)
    makeSplash(menu,folder)

Response: generates an index of the given folder


Instruction:
Write a Python 3 function for
drawing the phase plot of the current sweep.

Response:
def drawPhasePlot(abf,m1=0,m2=None):
    """
    Given an ABF object (SWHLab), draw its phase plot of the current sweep.
    m1 and m2 are optional marks (in seconds) for plotting only a range of data.
    Assume a matplotlib figure is already open and just draw on top if it.
    """

    if not m2:
        m2 = abf.sweepLength

    cm = plt.get_cmap('CMRmap')
    #cm = plt.get_cmap('CMRmap_r')
    #cm = plt.get_cmap('spectral')
    #cm = plt.get_cmap('winter')

    # prepare Xs, Ys, and dYs
    Y = abf.sweepY
    Y = Y[int(abf.pointsPerSec*m1):int(abf.pointsPerSec*m2)]
    dY = (Y[1:]-Y[:-1])*abf.rate/1000.0 # mV/ms
    dY = np.append(dY,dY[-1])
    Xs = np.arange(len(dY))/abf.pointsPerSec
    Xs = Xs + Xs[-1]*abf.sweep

    # plot the voltage
    plt.subplot(131)
    plt.grid(alpha=.5)
    plt.plot(Xs,Y,lw=.5,color=cm(abf.sweep/abf.sweeps))
    plt.title("membrane voltage")
    plt.ylabel("V (mV)")
    plt.xlabel("time (sec)")
    plt.margins(0,.1)

    # plot the first derivative of the voltage
    plt.subplot(132)
    plt.grid(alpha=.5)
    plt.plot(Xs,dY,lw=.5,color=cm(abf.sweep/abf.sweeps))
    plt.title("voltage velocity")
    plt.ylabel("dV (mV/ms)")
    plt.xlabel("time (sec)")
    plt.margins(0,.1)

    # make the phase plot
    plt.subplot(133)
    plt.grid(alpha=.5)
    plt.plot(Y,dY,alpha=.5,lw=.5,color=cm(abf.sweep/abf.sweeps))
    plt.title("phase plot")
    plt.ylabel("dV (mV/ms)")
    plt.xlabel("V (mV)")
    plt.margins(.1,.1)

    # tighten up the figure
    plt.tight_layout()


Instruction:
Can you tell what is the following Python 3 function doing
def TIF_to_jpg(fnameTiff, overwrite=False, saveAs=""):

    if saveAs == "":
        saveAs=fnameTiff+".jpg"

    if overwrite is False and os.path.exists(saveAs):
        print("file exists, not overwriting...")
        return

    # load the image
    img=pylab.imread(fnameTiff)
    img=img/np.max(img) # now the data is from 0 to 1

    # determine the old histogram
    hist1,bins1=np.histogram(img.ravel(),bins=256, range=(0,1))
    #pylab.plot(bins[:-1],hist)

    # detect darkfield by average:
    if np.average(img)<.2:
        vmin=None
        vmax=None
        msg=" | FLU"
        while np.average(img)<.5:
            img=np.sqrt(img)
            msg+="^(.5)"
    else:
        msg=" | DIC"
        percentile=.005
        vmin=np.percentile(img.ravel(),percentile)
        vmax=np.percentile(img.ravel(),100-percentile)

    # determine the new histogram
    hist2,bins2=np.histogram(img.ravel(),bins=256, range=(0,1))

    # plot it with resizing magic
    fig=pylab.figure(facecolor='r')
    fig.gca().imshow(img,cmap=pylab.gray(),vmin=vmin,vmax=vmax)
    pylab.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0)
    pylab.gca().xaxis.set_major_locator(pylab.NullLocator())
    pylab.gca().yaxis.set_major_locator(pylab.NullLocator())
    pylab.axis('off')

    # resize it to the original size
    fig.set_size_inches(img.shape[1]/100, img.shape[0]/100)

    # add text
    msg="%s | %s"%(os.path.basename(fnameTiff),
         datetime.datetime.fromtimestamp(os.path.getmtime(fnameTiff)))+msg
    center=10
    pylab.text(center,center,"%s"%(msg),va="top",color='w',size='small',
               family='monospace',weight='bold',
               bbox=dict(facecolor='k', alpha=.5))

    # add scale bar
    scaleWidthPx=False
    if "f10" in fnameTiff:
        scaleWidthPx,scaleBarText=39,"25 um"
    if "f20" in fnameTiff:
        scaleWidthPx,scaleBarText=31,"10 um"
    if scaleWidthPx:
        scaleBarPadding=10
        x2,y2=img.shape[1]-scaleBarPadding,img.shape[0]-scaleBarPadding
        x1,y1=x2-scaleWidthPx,y2
        for offset,color,alpha in [[2,'k',.5],[0,'w',1]]:
            pylab.plot([x1+offset,x2+offset],[y1+offset,y2+offset],'-',
                       color=color,lw=4,alpha=alpha)
            pylab.text((x1+x2)/2+offset,y1-5+offset,scaleBarText,color=color,
                       ha="center",weight="bold",alpha=alpha,
                       size="small",va="bottom",family="monospace")

    # add histogram
    #pylab.plot(img.shape[1]-bins1[:-1][::-1]*200,-hist1/max(hist1)*100+110,color='g')
    #pylab.plot(img.shape[1]-bins2[:-1][::-1]*200,-hist2/max(hist2)*100+110,color='b')
    #pylab.show()

    # save it
    pylab.savefig(saveAs,dpi=100)

    # clean up
    pylab.close()

Response: This function takes a TIF file and creates a pretty labeled JPG.


Instruction:
How would you explain what the following Python 3 function does
def TIF_to_jpg_all(path):
    for fname in sorted(glob.glob(path+"/*.tif")):
        print(fname)
        TIF_to_jpg(fname)

Response: run TIF_to_jpg on every TIF of a folder


Instruction:
Write a Python 3 function for
analyzing the sweep and return the EPSCs and IPSCs.

Response:
def analyzeSweep(abf,sweep,m1=None,m2=None,plotToo=False):
    """
    m1 and m2, if given, are in seconds.
    returns [# EPSCs, # IPSCs]
    """
    abf.setsweep(sweep)
    if m1 is None: m1=0
    else: m1=m1*abf.pointsPerSec
    if m2 is None: m2=-1
    else: m2=m2*abf.pointsPerSec

    # obtain X and Y
    Yorig=abf.sweepY[int(m1):int(m2)]
    X=np.arange(len(Yorig))/abf.pointsPerSec

    Ylpf=linear_gaussian(Yorig,sigmaSize=abf.pointsPerMs*300,forwardOnly=False)
    Yflat=Yorig-Ylpf

    EPSCs,IPSCs=[],[]

    if plotToo:
        plt.figure(figsize=(15,6))
        ax1=plt.subplot(211)
        plt.title("%s sweep %d"%(abf.ID,sweep))
        plt.grid()
        plt.plot(X,Yorig,alpha=.5)
        plt.plot(X,Ylpf,'k',alpha=.5,lw=2)
        plt.margins(0,.2)

        plt.subplot(212,sharex=ax1)
        plt.title("gaussian baseline subtraction")
        plt.grid()
        plt.plot(X,Yflat,alpha=.5)
        plt.axhline(0,color='k',lw=2,alpha=.5)

        plt.tight_layout()
        plt.show()

    # TEST GAUSS
    hist, bin_edges = np.histogram(Yflat, density=True, bins=200)
    peakPa=bin_edges[np.where(hist==max(hist))[0][0]+1]

    if plotToo:
        plt.figure()
        plt.grid()
        plt.plot(bin_edges[1:],hist,alpha=.5)
        plt.axvline(0,color='k')
        plt.axvline(peakPa,color='r',ls='--',lw=2,alpha=.5)
        plt.semilogy()
        plt.title("sweep data distribution")
        plt.ylabel("power")
        plt.xlabel("pA deviation")
        plt.show()

    return peakPa


Instruction:
Make a summary of the following Python 3 code
def convert(fname,saveAs=True,showToo=False):

    # load the image
    #im = Image.open(fname) #PIL can't handle 12-bit TIFs well
    im=ndimage.imread(fname) #scipy does better with it
    im=np.array(im,dtype=float) # now it's a numpy array

    # do all image enhancement here
    cutoffLow=np.percentile(im,.01)
    cutoffHigh=np.percentile(im,99.99)
    im[np.where(im<cutoffLow)]=cutoffLow
    im[np.where(im>cutoffHigh)]=cutoffHigh

    # IMAGE FORMATTING
    im-=np.min(im) #auto contrast
    im/=np.max(im) #normalize
    im*=255 #stretch contrast (8-bit)
    im = Image.fromarray(im)

    # IMAGE DRAWING
    msg="%s\n"%os.path.basename(fname)
    msg+="%s\n"%cm.epochToString(os.path.getmtime(fname))
    d = ImageDraw.Draw(im)
    fnt = ImageFont.truetype("arial.ttf", 20)
    d.text((6,6),msg,font=fnt,fill=0)
    d.text((4,4),msg,font=fnt,fill=255)

    if showToo:
        im.show()
    if saveAs is False:
        return
    if saveAs is True:
        saveAs=fname+".png"
    im.convert('RGB').save(saveAs)
    return saveAs

Response: Convert weird TIF files into web - friendly versions.


Instruction:
How would you implement a function in Python 3 that
plots X and Y data then shade its background by variance.

Response:
def plot_shaded_data(X,Y,variances,varianceX):
    """plot X and Y data, then shade its background by variance."""
    plt.plot(X,Y,color='k',lw=2)
    nChunks=int(len(Y)/CHUNK_POINTS)
    for i in range(0,100,PERCENT_STEP):
        varLimitLow=np.percentile(variances,i)
        varLimitHigh=np.percentile(variances,i+PERCENT_STEP)
        varianceIsAboveMin=np.where(variances>=varLimitLow)[0]
        varianceIsBelowMax=np.where(variances<=varLimitHigh)[0]
        varianceIsRange=[chunkNumber for chunkNumber in range(nChunks) \
                         if chunkNumber in varianceIsAboveMin \
                         and chunkNumber in varianceIsBelowMax]
        for chunkNumber in varianceIsRange:
            t1=chunkNumber*CHUNK_POINTS/POINTS_PER_SEC
            t2=t1+CHUNK_POINTS/POINTS_PER_SEC
            plt.axvspan(t1,t2,alpha=.3,color=COLORMAP(i/100),lw=0)


Instruction:
Create a Python 3 function for
creating some fancy graphs to show color - coded variances.

Response:
def show_variances(Y,variances,varianceX,logScale=False):
    """create some fancy graphs to show color-coded variances."""

    plt.figure(1,figsize=(10,7))
    plt.figure(2,figsize=(10,7))
    varSorted=sorted(variances)

    plt.figure(1)
    plt.subplot(211)
    plt.grid()
    plt.title("chronological variance")
    plt.ylabel("original data")
    plot_shaded_data(X,Y,variances,varianceX)
    plt.margins(0,.1)
    plt.subplot(212)
    plt.ylabel("variance (pA) (log%s)"%str(logScale))
    plt.xlabel("time in sweep (sec)")
    plt.plot(varianceX,variances,'k-',lw=2)

    plt.figure(2)
    plt.ylabel("variance (pA) (log%s)"%str(logScale))
    plt.xlabel("chunk number")
    plt.title("sorted variance")
    plt.plot(varSorted,'k-',lw=2)

    for i in range(0,100,PERCENT_STEP):
        varLimitLow=np.percentile(variances,i)
        varLimitHigh=np.percentile(variances,i+PERCENT_STEP)
        label="%2d-%d percentile"%(i,i++PERCENT_STEP)
        color=COLORMAP(i/100)
        print("%s: variance = %.02f - %.02f"%(label,varLimitLow,varLimitHigh))
        plt.figure(1)
        plt.axhspan(varLimitLow,varLimitHigh,alpha=.5,lw=0,color=color,label=label)
        plt.figure(2)
        chunkLow=np.where(varSorted>=varLimitLow)[0][0]
        chunkHigh=np.where(varSorted>=varLimitHigh)[0][0]
        plt.axvspan(chunkLow,chunkHigh,alpha=.5,lw=0,color=color,label=label)

    for fignum in [1,2]:
        plt.figure(fignum)
        if logScale:
            plt.semilogy()
        plt.margins(0,0)
        plt.grid()
        if fignum is 2:
            plt.legend(fontsize=10,loc='upper left',shadow=True)
        plt.tight_layout()
        plt.savefig('2016-12-15-variance-%d-log%s.png'%(fignum,str(logScale)))
    plt.show()


Instruction:
Create a Python 3 function to
check if event detection occured. Checks if APs flag is set and runs detection on all sweeps.

Response:
def ensureDetection(self):
        """
        run this before analysis. Checks if event detection occured.
        If not, runs AP detection on all sweeps.
        """
        if self.APs==False:
            self.log.debug("analysis attempted before event detection...")
            self.detect()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def detect(self):
        self.log.info("initializing AP detection on all sweeps...")
        t1=cm.timeit()
        for sweep in range(self.abf.sweeps):
            self.detectSweep(sweep)
        self.log.info("AP analysis of %d sweeps found %d APs (completed in %s)",
                      self.abf.sweeps,len(self.APs),cm.timeit(t1))

Response: runs AP detection on every sweep.


Instruction:
Can you implement a function in Python 3 that
performs AP detection on current sweep.

Response:
def detectSweep(self,sweep=0):
        """perform AP detection on current sweep."""

        if self.APs is False: # indicates detection never happened
            self.APs=[] # now indicates detection occured

        # delete every AP from this sweep from the existing array
        for i,ap in enumerate(self.APs):
            if ap["sweep"]==sweep:
                self.APs[i]=None
        if self.APs.count(None):
            self.log.debug("deleting %d existing APs from memory",self.APs.count(None))
            while None in self.APs:
                self.APs.remove(None)
        self.log.debug("initiating AP detection (%d already in memory)",len(self.APs))

        self.abf.derivative=True
        self.abf.setsweep(sweep)

        # detect potential AP (Is) by a dV/dT threshold crossing
        Is = cm.where_cross(self.abf.sweepD,self.detect_over)
        self.log.debug("initial AP detection: %d APs"%len(Is))

        # eliminate APs where dV/dT doesn't cross below -10 V/S within 2 ms
        for i,I in enumerate(Is):
            if np.min(self.abf.sweepD[I:I+2*self.abf.pointsPerMs])>-10:
                Is[i]=0
        Is=Is[np.nonzero(Is)]
        self.log.debug("after lower threshold checking: %d APs"%len(Is))

        # walk 1ms backwards and find point of +10 V/S threshold crossing
        for i,I in enumerate(Is):
            stepBack=0
            while(self.abf.sweepD[I-stepBack])>10 and stepBack/self.abf.pointsPerMs<1: #2ms max
                stepBack+=1
            Is[i]-=stepBack

        # analyze each AP
        sweepAPs=[]
        for i,I in enumerate(Is):
            try:
                timeInSweep=I/self.abf.pointsPerSec
                if timeInSweep<self.detect_time1 or timeInSweep>self.detect_time2:
                    continue # skip because it's not within the marks
                ap={} # create the AP entry
                ap["sweep"]=sweep # number of the sweep containing this AP
                ap["I"]=I # index sweep point of start of AP (10 mV/ms threshold crossing)
                ap["Tsweep"]=I/self.abf.pointsPerSec # time in the sweep of index crossing (sec)
                ap["T"]=ap["Tsweep"]+self.abf.sweepInterval*sweep # time in the experiment
                ap["Vthreshold"]=self.abf.sweepY[I] # threshold at rate of -10mV/ms

                # determine how many points from the start dV/dt goes below -10 (from a 5ms chunk)
                chunk=self.abf.sweepD[I:I+5*self.abf.pointsPerMs] # give it 5ms to cross once
                I_toNegTen=np.where(chunk<-10)[0][0]
                chunk=self.abf.sweepD[I+I_toNegTen:I+I_toNegTen+10*self.abf.pointsPerMs] # give it 30ms to cross back
                if not max(chunk)>-10:
                    self.log.debug("skipping unreal AP at T=%f"%ap["T"])
                    self.log.error("^^^ can you confirm this is legit?")
                    continue # probably a pre-AP "bump" to be ignored
                I_recover=np.where(chunk>-10)[0][0]+I_toNegTen+I # point where trace returns to above -10 V/S
                ap["dVfastIs"]=[I,I_recover] # span of the fast component of the dV/dt trace
                ap["dVfastMS"]=(I_recover-I)/self.abf.pointsPerMs # time (in ms) of this fast AP component

                # determine derivative min/max from a 2ms chunk which we expect to capture the fast AP
                chunk=self.abf.sweepD[ap["dVfastIs"][0]:ap["dVfastIs"][1]]
                ap["dVmax"]=np.max(chunk)
                ap["dVmaxI"]=np.where(chunk==ap["dVmax"])[0][0]+I
                ap["dVmin"]=np.min(chunk)
                ap["dVminI"]=np.where(chunk==ap["dVmin"])[0][0]+I
                if ap["dVmax"]<10 or ap["dVmin"]>-10:
                    self.log.debug("throwing out AP with low dV/dt to be an AP")
                    self.log.error("^^^ can you confirm this is legit?")
                    continue

                # before determining AP shape stats, see where trace recovers to threshold
                chunkSize=self.abf.pointsPerMs*10 #AP shape may be 10ms
                if len(Is)-1>i and Is[i+1]<(I+chunkSize): # if slow AP runs into next AP
                    chunkSize=Is[i+1]-I # chop it down
                if chunkSize<(self.abf.pointsPerMs*2):
                    continue # next AP is so soon, it's >500 Hz. Can't be real.
                ap["VslowIs"]=[I,I+chunkSize] # time range of slow AP dynamics
                chunk=self.abf.sweepY[I:I+chunkSize]

                # determine AP peak and minimum
                ap["Vmax"]=np.max(chunk)
                ap["VmaxI"]=np.where(chunk==ap["Vmax"])[0][0]+I
                chunkForMin=np.copy(chunk) # so we can destroy it
                chunkForMin[:ap["VmaxI"]-I]=np.inf # minimum won't be before peak now
                ap["Vmin"]=np.min(chunkForMin) # supposedly the minimum is the AHP
                ap["VminI"]=np.where(chunkForMin==ap["Vmin"])[0][0]+I
                if ap["VminI"]<ap["VmaxI"]:
                    self.log.error("-------------------------------")
                    self.log.error("how is the AHP before the peak?") #TODO: start chunk at the peak
                    self.log.error("-------------------------------")
                #print((I+len(chunk))-ap["VminI"],len(chunk))
                if (len(chunk))-((I+len(chunk))-ap["VminI"])<10:
                    self.log.error("-------------------------------")
                    self.log.error("HP too close for comfort!")
                    self.log.error("-------------------------------")

                ap["msRiseTime"]=(ap["VmaxI"]-I)/self.abf.pointsPerMs # time from threshold to peak
                ap["msFallTime"]=(ap["VminI"]-ap["VmaxI"])/self.abf.pointsPerMs # time from peak to nadir

                # determine halfwidth
                ap["Vhalf"]=np.average([ap["Vmax"],ap["Vthreshold"]]) # half way from threshold to peak
                ap["VhalfI1"]=cm.where_cross(chunk,ap["Vhalf"])[0]+I # time it's first crossed
                ap["VhalfI2"]=cm.where_cross(-chunk,-ap["Vhalf"])[1]+I # time it's second crossed
                ap["msHalfwidth"]=(ap["VhalfI2"]-ap["VhalfI1"])/self.abf.pointsPerMs # time between crossings

                # AP error checking goes here
                # TODO:

                # if we got this far, add the AP to the list
                sweepAPs.extend([ap])
            except Exception as e:
                self.log.error("crashed analyzing AP %d of %d",i,len(Is))
                self.log.error(cm.exceptionToString(e))
                #cm.pause()
                #cm.waitFor(30)
                #self.log.error("EXCEPTION!:\n%s"%str(sys.exc_info()))

        self.log.debug("finished analyzing sweep. Found %d APs",len(sweepAPs))
        self.APs.extend(sweepAPs)
        self.abf.derivative=False


Instruction:
How would you code a function in Python 3 to
return an array of times in sec of all APs

Response:
def get_times(self):
        """return an array of times (in sec) of all APs."""
        self.ensureDetection()
        times=[]
        for ap in self.APs:
            times.append(ap["T"])
        return np.array(sorted(times))


Instruction:
Make a summary of the following Python 3 code
def get_bySweep(self,feature="freqs"):
        self.ensureDetection()
        bySweepTimes=[[]]*self.abf.sweeps

        # determine AP spike times by sweep
        for sweep in range(self.abf.sweeps):
            sweepTimes=[]
            for ap in self.APs:
                if ap["sweep"]==sweep:
                    sweepTimes.append(ap["Tsweep"])
            bySweepTimes[sweep]=sweepTimes

        # determine instantaneous frequencies by sweep
        bySweepFreqs=[[]]*self.abf.sweeps
        for i,times in enumerate(bySweepTimes):
            if len(times)<2:
                continue
            diffs=np.array(times[1:])-np.array(times[:-1])
            bySweepFreqs[i]=np.array(1/diffs).tolist()

        # give the user what they want
        if feature == "freqs":
            return bySweepFreqs

        elif feature == "firsts":
            result=np.zeros(self.abf.sweeps) # initialize to this
            for i,freqs in enumerate(bySweepFreqs):
                if len(freqs):
                    result[i]=freqs[0]
            return result

        elif feature == "times":
            return bySweepTimes

        elif feature == "count":
            result=np.zeros(self.abf.sweeps) # initialize to this
            for i,times in enumerate(bySweepTimes):
                result[i]=len(bySweepTimes[i])
            return result

        elif feature == "average":
            result=np.zeros(self.abf.sweeps) # initialize to this
            for i,freqs in enumerate(bySweepFreqs):
                if len(freqs):
                    result[i]=np.nanmean(freqs)
            return result

        elif feature == "median":
            result=np.zeros(self.abf.sweeps) # initialize to this
            for i,freqs in enumerate(bySweepFreqs):
                if len(freqs):
                    result[i]=np.nanmedian(freqs)
            return result

        else:
            self.log.error("get_bySweep() can't handle [%s]",feature)
            return None

Response: returns the information of the AP in the sweeps in the local file.


Instruction:
Write a Python 3 function for
getting package author and version as listed in init. py.

Response:
def get_author_and_version(package):
    """
    Return package author and version as listed in `init.py`.
    """
    init_py = open(os.path.join(package, '__init__.py')).read()
    author = re.search("__author__ = ['\"]([^'\"]+)['\"]", init_py).group(1)
    version = re.search("__version__ = ['\"]([^'\"]+)['\"]", init_py).group(1)
    return author, version


Instruction:
How would you implement a function in Python 3 that
creates a new subclass of the base class with fewer methods than its base class.

Response:
def api_subclass_factory(name, docstring, remove_methods, base=SlackApi):
    """Create an API subclass with fewer methods than its base class.

    Arguments:
      name (:py:class:`str`): The name of the new class.
      docstring (:py:class:`str`): The docstring for the new class.
      remove_methods (:py:class:`dict`): The methods to remove from
        the base class's :py:attr:`API_METHODS` for the subclass. The
        key is the name of the root method (e.g. ``'auth'`` for
        ``'auth.test'``, the value is either a tuple of child method
        names (e.g. ``('test',)``) or, if all children should be
        removed, the special value :py:const:`ALL`.
      base (:py:class:`type`, optional): The base class (defaults to
        :py:class:`SlackApi`).

    Returns:
      :py:class:`type`: The new subclass.

    Raises:
      :py:class:`KeyError`: If the method wasn't in the superclass.

    """
    methods = deepcopy(base.API_METHODS)
    for parent, to_remove in remove_methods.items():
        if to_remove is ALL:
            del methods[parent]
        else:
            for method in to_remove:
                del methods[parent][method]
    return type(name, (base,), dict(API_METHODS=methods, __doc__=docstring))


Instruction:
Create a Python 3 function for
executing a specified Slack Web API method.

Response:
async def execute_method(self, method, **params):
        """Execute a specified Slack Web API method.

        Arguments:
          method (:py:class:`str`): The name of the method.
          **params (:py:class:`dict`): Any additional parameters
            required.

        Returns:
          :py:class:`dict`: The JSON data from the response.

        Raises:
          :py:class:`aiohttp.web_exceptions.HTTPException`: If the HTTP
            request returns a code other than 200 (OK).
          SlackApiError: If the Slack API is reached but the response
           contains an error message.

        """
        url = self.url_builder(method, url_params=params)
        logger.info('Executing method %r', method)
        response = await aiohttp.get(url)
        logger.info('Status: %r', response.status)
        if response.status == 200:
            json = await response.json()
            logger.debug('...with JSON %r', json)
            if json.get('ok'):
                return json
            raise SlackApiError(json['error'])
        else:
            raise_for_status(response)


Instruction:
Here you have a function in Python 3, explain what it does
def method_exists(cls, method):
        methods = cls.API_METHODS
        for key in method.split('.'):
            methods = methods.get(key)
            if methods is None:
                break
        if isinstance(methods, str):
            logger.debug('%r: %r', method, methods)
            return True
        return False

Response: Determines if a given method exists in the known API.


Instruction:
Can you write a function in Python 3 where it
extends XPath evaluation with Parsley extensions

Response:
def _add_parsley_ns(cls, namespace_dict):
        """
        Extend XPath evaluation with Parsley extensions' namespace
        """

        namespace_dict.update({
            'parslepy' : cls.LOCAL_NAMESPACE,
            'parsley' : cls.LOCAL_NAMESPACE,
        })
        return namespace_dict


Instruction:
Can you generate a brief explanation for the following Python 3 code
def make(self, selection):

        cached = self._selector_cache.get(selection)
        if cached:
            return cached

        try:
            selector = lxml.etree.XPath(selection,
                namespaces = self.namespaces,
                extensions = self.extensions,
                smart_strings=(self.SMART_STRINGS
                            or self._test_smart_strings_needed(selection)),
                )

        except lxml.etree.XPathSyntaxError as syntax_error:
            syntax_error.msg += ": %s" % selection
            raise syntax_error

        except Exception as e:
            if self.DEBUG:
                print(repr(e), selection)
            raise

        # wrap it/cache it
        self._selector_cache[selection] = Selector(selector)
        return self._selector_cache[selection]

Response: Creates a new selector for the given XPath expression.


Instruction:
Create a Python 3 function for
extracting the elements from the document that match the selector.

Response:
def extract(self, document, selector, debug_offset=''):
        """
        Try and convert matching Elements to unicode strings.

        If this fails, the selector evaluation probably already
        returned some string(s) of some sort, or boolean value,
        or int/float, so return that instead.
        """
        selected = self.select(document, selector)
        if selected is not None:

            if isinstance(selected, (list, tuple)):

                # FIXME: return None or return empty list?
                if not len(selected):
                    return

                return [self._extract_single(m) for m in selected]

            else:
                return self._extract_single(selected)

        # selector did not match anything
        else:
            if self.DEBUG:
                print(debug_offset, "selector did not match anything; return None")
            return None


Instruction:
Make a summary of the following Python 3 code
def make(self, selection):
        cached = self._selector_cache.get(selection)
        if cached:
            return cached

        namespaces = self.EXSLT_NAMESPACES
        self._add_parsley_ns(namespaces)
        try:
            # CSS with attribute? (non-standard but convenient)
            # CSS selector cannot select attributes
            # this "<css selector> @<attr>" syntax is a Parsley extension
            # construct CSS selector and append attribute to XPath expression
            m = self.REGEX_ENDING_ATTRIBUTE.match(selection)
            if m:
                # the selector should be a regular CSS selector
                cssxpath = css_to_xpath(m.group("expr"))

                # if "|" is used for namespace prefix reference,
                #   convert it to XPath prefix syntax
                attribute = m.group("attr").replace('|', ':')

                cssxpath = "%s/%s" % (cssxpath, attribute)
            else:
                cssxpath = css_to_xpath(selection)

            selector = lxml.etree.XPath(
                cssxpath,
                namespaces = self.namespaces,
                extensions = self.extensions,
                smart_strings=(self.SMART_STRINGS
                            or self._test_smart_strings_needed(selection)),
                )

        except tuple(self.CSSSELECT_SYNTAXERROR_EXCEPTIONS) as syntax_error:
            if self.DEBUG:
                print(repr(syntax_error), selection)
                print("Try interpreting as XPath selector")
            try:
                selector = lxml.etree.XPath(selection,
                    namespaces = self.namespaces,
                    extensions = self.extensions,
                    smart_strings=(self.SMART_STRINGS
                                or self._test_smart_strings_needed(selection)),
                    )

            except lxml.etree.XPathSyntaxError as syntax_error:
                syntax_error.msg += ": %s" % selection
                raise syntax_error

            except Exception as e:
                if self.DEBUG:
                    print(repr(e), selection)
                raise

        # for exception when trying to convert <cssselector> @<attribute> syntax
        except lxml.etree.XPathSyntaxError as syntax_error:
            syntax_error.msg += ": %s" % selection
            raise syntax_error

        except Exception as e:
            if self.DEBUG:
                print(repr(e), selection)
            raise

        # wrap it/cache it
        self._selector_cache[selection] = Selector(selector)
        return self._selector_cache[selection]

Response: Creates a new CSS selector and XPath expression from a CSS selector.


Instruction:
Write a Python 3 function for
joining the real - time messaging service.

Response:
async def join_rtm(self, filters=None):
        """Join the real-time messaging service.

        Arguments:
          filters (:py:class:`dict`, optional): Dictionary mapping
            message filters to the functions they should dispatch to.
            Use a :py:class:`collections.OrderedDict` if precedence is
            important; only one filter, the first match, will be
            applied to each message.

        """
        if filters is None:
            filters = [cls(self) for cls in self.MESSAGE_FILTERS]
        url = await self._get_socket_url()
        logger.debug('Connecting to %r', url)
        async with ws_connect(url) as socket:
            first_msg = await socket.receive()
            self._validate_first_message(first_msg)
            self.socket = socket
            async for message in socket:
                if message.tp == MsgType.text:
                    await self.handle_message(message, filters)
                elif message.tp in (MsgType.closed, MsgType.error):
                    if not socket.closed:
                        await socket.close()
                    self.socket = None
                    break
        logger.info('Left real-time messaging.')


Instruction:
Here you have a function in Python 3, explain what it does
async def handle_message(self, message, filters):
        data = self._unpack_message(message)
        logger.debug(data)
        if data.get('type') == 'error':
            raise SlackApiError(
                data.get('error', {}).get('msg', str(data))
            )
        elif self.message_is_to_me(data):
            text = data['text'][len(self.address_as):].strip()
            if text == 'help':
                return self._respond(
                    channel=data['channel'],
                    text=self._instruction_list(filters),
                )
            elif text == 'version':
                return self._respond(
                    channel=data['channel'],
                    text=self.VERSION,
                )
        for _filter in filters:
            if _filter.matches(data):
                logger.debug('Response triggered')
                async for response in _filter:
                    self._respond(channel=data['channel'], text=response)

Response:
Handle an incoming
            message appropriately.


Instruction:
Write a Python 3 function for
creating a new Bot instance from the API token.

Response:
async def from_api_token(cls, token=None, api_cls=SlackBotApi):
        """Create a new instance from the API token.

        Arguments:
          token (:py:class:`str`, optional): The bot's API token
            (defaults to ``None``, which means looking in the
            environment).
          api_cls (:py:class:`type`, optional): The class to create
            as the ``api`` argument for API access (defaults to
            :py:class:`aslack.slack_api.SlackBotApi`).

        Returns:
          :py:class:`SlackBot`: The new instance.

        """
        api = api_cls.from_env() if token is None else api_cls(api_token=token)
        data = await api.execute_method(cls.API_AUTH_ENDPOINT)
        return cls(data['user_id'], data['user'], api)


Instruction:
Create a Python 3 function to
format an outgoing message for transmission.

Response:
def _format_message(self, channel, text):
        """Format an outgoing message for transmission.

        Note:
          Adds the message type (``'message'``) and incremental ID.

        Arguments:
          channel (:py:class:`str`): The channel to send to.
          text (:py:class:`str`): The message text to send.

        Returns:
          :py:class:`str`: The JSON string of the message.

        """
        payload = {'type': 'message', 'id': next(self._msg_ids)}
        payload.update(channel=channel, text=text)
        return json.dumps(payload)


Instruction:
How would you explain what the following Python 3 function does
async def _get_socket_url(self):
        data = await self.api.execute_method(
            self.RTM_START_ENDPOINT,
            simple_latest=True,
            no_unreads=True,
        )
        return data['url']

Response: Get the WebSocket URL for the RTM session.


Instruction:
Can you generate the documentation for the following Python 3 function
def _instruction_list(self, filters):
        return '\n\n'.join([
            self.INSTRUCTIONS.strip(),
            '*Supported methods:*',
            'If you send "@{}: help" to me I reply with these '
            'instructions.'.format(self.user),
            'If you send "@{}: version" to me I reply with my current '
            'version.'.format(self.user),
        ] + [filter.description() for filter in filters])

Response: Generates the list of instructions for a specific class and filters.


Instruction:
Write a Python 3 script for
responding to a message on the current socket.

Response:
def _respond(self, channel, text):
        """Respond to a message on the current socket.

        Args:
          channel (:py:class:`str`): The channel to send to.
          text (:py:class:`str`): The message text to send.

        """
        result = self._format_message(channel, text)
        if result is not None:
            logger.info(
                'Sending message: %r',
                truncate(result, max_len=50),
            )
        self.socket.send_str(result)


Instruction:
Implement a Python 3 function for
validating the first message.

Response:
def _validate_first_message(cls, msg):
        """Check the first message matches the expected handshake.

        Note:
          The handshake is provided as :py:attr:`RTM_HANDSHAKE`.

        Arguments:
          msg (:py:class:`aiohttp.Message`): The message to validate.

        Raises:
          :py:class:`SlackApiError`: If the data doesn't match the
            expected handshake.

        """
        data = cls._unpack_message(msg)
        logger.debug(data)
        if data != cls.RTM_HANDSHAKE:
            raise SlackApiError('Unexpected response: {!r}'.format(data))
        logger.info('Joined real-time messaging.')


Instruction:
Create a Python 3 function for
finding first existing executable file in a list of options

Response:
def find_first_existing_executable(exe_list):
    """
    Accepts list of [('executable_file_path', 'options')],
    Returns first working executable_file_path
    """
    for filepath, opts in exe_list:
        try:
            proc = subprocess.Popen([filepath, opts],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            proc.communicate()
        except OSError:
            pass
        else:
            return filepath


Instruction:
Explain what the following Python 3 code does
def get_app_locations():
    return [os.path.dirname(os.path.normpath(import_module(app_name).__file__))
            for app_name in PROJECT_APPS]

Response: Returns list of paths to tested apps


Instruction:
How would you explain what the following Python 3 function does
def get_tasks():
    task_classes = []
    for task_path in TASKS:
        try:
            module, classname = task_path.rsplit('.', 1)
        except ValueError:
            raise ImproperlyConfigured('%s isn\'t a task module' % task_path)
        try:
            mod = import_module(module)
        except ImportError as e:
            raise ImproperlyConfigured('Error importing task %s: "%s"'
                                       % (module, e))
        try:
            task_class = getattr(mod, classname)
        except AttributeError:
            raise ImproperlyConfigured('Task module "%s" does not define a '
                                       '"%s" class' % (module, classname))
        task_classes.append(task_class)
    return task_classes

Response: Get the imported task classes for each task that will be run


Instruction:
Explain what the following Python 3 code does
def get_task_options():
    options = ()

    task_classes = get_tasks()
    for cls in task_classes:
        options += cls.option_list

    return options

Response: Get the options for each task that will be run


Instruction:
Can you create a Python 3 function that
writes the data from the db to a CLDF dataset according to the metadata in self. dataset.

Response:
def to_cldf(self, dest, mdname='cldf-metadata.json'):
        """
        Write the data from the db to a CLDF dataset according to the metadata in `self.dataset`.

        :param dest:
        :param mdname:
        :return: path of the metadata file
        """
        dest = Path(dest)
        if not dest.exists():
            dest.mkdir()

        data = self.read()

        if data[self.source_table_name]:
            sources = Sources()
            for src in data[self.source_table_name]:
                sources.add(Source(
                    src['genre'],
                    src['id'],
                    **{k: v for k, v in src.items() if k not in ['id', 'genre']}))
            sources.write(dest / self.dataset.properties.get('dc:source', 'sources.bib'))

        for table_type, items in data.items():
            try:
                table = self.dataset[table_type]
                table.common_props['dc:extent'] = table.write(
                    [self.retranslate(table, item) for item in items],
                    base=dest)
            except KeyError:
                assert table_type == self.source_table_name, table_type
        return self.dataset.write_metadata(dest / mdname)


Instruction:
Can you create a Python 3 function that
prints basic stats for CLDF dataset

Response:
def stats(args):
    """
    cldf stats <DATASET>

    Print basic stats for CLDF dataset <DATASET>, where <DATASET> may be the path to
    - a CLDF metadata file
    - a CLDF core data file
    """
    ds = _get_dataset(args)
    print(ds)
    md = Table('key', 'value')
    md.extend(ds.properties.items())
    print(md.render(condensed=False, tablefmt=None))
    print()
    t = Table('Path', 'Type', 'Rows')
    for p, type_, r in ds.stats():
        t.append([p, type_, r])
    print(t.render(condensed=False, tablefmt=None))


Instruction:
Can you generate the documentation for the following Python 3 function
def createdb(args):
    if len(args.args) < 2:
        raise ParserError('not enough arguments')
    ds = _get_dataset(args)
    db = Database(ds, fname=args.args[1])
    db.write_from_tg()
    args.log.info('{0} loaded in {1}'.format(ds, db.fname))

Response:
cldf createdb <DATASET> <SQLITE_DB_PATH>

    Load CLDF dataset <DATASET> into a SQLite DB, where <DATASET> may be the path to
    - a CLDF metadata file
    - a CLDF core data file


Instruction:
Make a summary of the following Python 3 code
def from_jsonfile(cls, fp, selector_handler=None, strict=False, debug=False):

        return cls._from_jsonlines(fp,
            selector_handler=selector_handler, strict=strict, debug=debug)

Response: Create a Parselet instance from a file containing a JSON object containing the Parsley script as a JSON object containing the Parsley script s ID and the optional selector handler.


Instruction:
How would you code a function in Python 3 to
create a Parselet instance from a file containing a YAML object containing the Parsley script as a YAML object.

Response:
def from_yamlfile(cls, fp, selector_handler=None, strict=False, debug=False):
        """
        Create a Parselet instance from a file containing
        the Parsley script as a YAML object

        >>> import parslepy
        >>> with open('parselet.yml') as fp:
        ...     parslepy.Parselet.from_yamlfile(fp)
        ...
        <parslepy.base.Parselet object at 0x2014e50>

        :param file fp: an open file-like pointer containing the Parsley script
        :rtype: :class:`.Parselet`

        Other arguments: same as for :class:`.Parselet` contructor
        """

        return cls.from_yamlstring(fp.read(), selector_handler=selector_handler, strict=strict, debug=debug)


Instruction:
Explain what the following Python 3 code does
def from_yamlstring(cls, s, selector_handler=None, strict=False, debug=False):

        import yaml
        return cls(yaml.load(s), selector_handler=selector_handler, strict=strict, debug=debug)

Response: Create a Parselet instance from a YAML string containing a Parsley script as YAML string containing a Parsley language.


Instruction:
How would you implement a function in Python 3 that
creates a Parselet instance from a JSON string containing a Parsley script as a JSON string.

Response:
def from_jsonstring(cls, s, selector_handler=None, strict=False, debug=False):
        """
        Create a Parselet instance from s (str) containing
        the Parsley script as JSON

        >>> import parslepy
        >>> parsley_string = '{ "title": "h1", "link": "a @href"}'
        >>> p = parslepy.Parselet.from_jsonstring(parsley_string)
        >>> type(p)
        <class 'parslepy.base.Parselet'>
        >>>

        :param string s: a Parsley script as a JSON string
        :rtype: :class:`.Parselet`

        Other arguments: same as for :class:`.Parselet` contructor
        """

        return cls._from_jsonlines(s.split("\n"),
            selector_handler=selector_handler, strict=strict, debug=debug)


Instruction:
How would you explain what the following Python 3 function does
def _from_jsonlines(cls, lines, selector_handler=None, strict=False, debug=False):

        return cls(json.loads(
                "\n".join([l for l in lines if not cls.REGEX_COMMENT_LINE.match(l)])
            ), selector_handler=selector_handler, strict=strict, debug=debug)

Response: Create a new object from a list of lines.


Instruction:
Here you have a function in Python 3, explain what it does
def parse(self, fp, parser=None, context=None):

        if parser is None:
            parser = lxml.etree.HTMLParser()
        doc = lxml.etree.parse(fp, parser=parser).getroot()
        return self.extract(doc, context=context)

Response: Parse an HTML or XML document and return the extracted object.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_fromstring(self, s, parser=None, context=None):
        if parser is None:
            parser = lxml.etree.HTMLParser()
        doc = lxml.etree.fromstring(s, parser=parser)
        return self.extract(doc, context=context)

Response: Parse an HTML or XML document and return the extacted object.


Instruction:
Can you write a function in Python 3 where it
compiles the Parsley tree starting from the root node

Response:
def compile(self):
        """
        Build the abstract Parsley tree starting from the root node
        (recursive)
        """
        if not isinstance(self.parselet, dict):
            raise ValueError("Parselet must be a dict of some sort. Or use .from_jsonstring(), " \
                ".from_jsonfile(), .from_yamlstring(), or .from_yamlfile()")
        self.parselet_tree = self._compile(self.parselet)


Instruction:
Can you create a Python 3 function that
builds the abstract Parsley extraction tree.

Response:
def _compile(self, parselet_node, level=0):
        """
        Build part of the abstract Parsley extraction tree

        Arguments:
        parselet_node (dict) -- part of the Parsley tree to compile
                                (can be the root dict/node)
        level (int)          -- current recursion depth (used for debug)
        """

        if self.DEBUG:
            debug_offset = "".join(["    " for x in range(level)])

        if self.DEBUG:
            print(debug_offset, "%s::compile(%s)" % (
                self.__class__.__name__, parselet_node))

        if isinstance(parselet_node, dict):
            parselet_tree = ParsleyNode()
            for k, v in list(parselet_node.items()):

                # we parse the key raw elements but without much
                # interpretation (which is done by the SelectorHandler)
                try:
                    m = self.REGEX_PARSELET_KEY.match(k)
                    if not m:
                        if self.DEBUG:
                            print(debug_offset, "could not parse key", k)
                        raise InvalidKeySyntax(k)
                except:
                    raise InvalidKeySyntax("Key %s is not valid" % k)

                key = m.group('key')
                # by default, fields are required
                key_required = True
                operator = m.group('operator')
                if operator == '?':
                    key_required = False
                # FIXME: "!" operator not supported (complete array)
                scope = m.group('scope')

                # example: get list of H3 tags
                # { "titles": ["h3"] }
                # FIXME: should we support multiple selectors in list?
                #        e.g. { "titles": ["h1", "h2", "h3", "h4"] }
                if isinstance(v, (list, tuple)):
                    v = v[0]
                    iterate = True
                else:
                    iterate = False

                # keys in the abstract Parsley trees are of type `ParsleyContext`
                try:
                    parsley_context = ParsleyContext(
                        key,
                        operator=operator,
                        required=key_required,
                        scope=self.selector_handler.make(scope) if scope else None,
                        iterate=iterate)
                except SyntaxError:
                    if self.DEBUG:
                        print("Invalid scope:", k, scope)
                    raise

                if self.DEBUG:
                    print(debug_offset, "current context:", parsley_context)

                # go deeper in the Parsley tree...
                try:
                    child_tree = self._compile(v, level=level+1)
                except SyntaxError:
                    if self.DEBUG:
                        print("Invalid value: ", v)
                    raise
                except:
                    raise

                if self.DEBUG:
                    print(debug_offset, "child tree:", child_tree)

                parselet_tree[parsley_context] = child_tree

            return parselet_tree

        # a string leaf should match some kind of selector,
        # let the selector handler deal with it
        elif isstr(parselet_node):
            return self.selector_handler.make(parselet_node)
        else:
            raise ValueError(
                    "Unsupported type(%s) for Parselet node <%s>" % (
                        type(parselet_node), parselet_node))


Instruction:
Make a summary of the following Python 3 code
def extract(self, document, context=None):
        if context:
            self.selector_handler.context = context
        return self._extract(self.parselet_tree, document)

Response: Extract the values from a lxml - parsed document into a Python dict.


Instruction:
Can you generate the documentation for the following Python 3 function
def _extract(self, parselet_node, document, level=0):

        if self.DEBUG:
            debug_offset = "".join(["    " for x in range(level)])

        # we must go deeper in the Parsley tree
        if isinstance(parselet_node, ParsleyNode):

            # default output
            output = {}

            # process all children
            for ctx, v in list(parselet_node.items()):
                if self.DEBUG:
                    print(debug_offset, "context:", ctx, v)
                extracted=None
                try:
                    # scoped-extraction:
                    # extraction should be done deeper in the document tree
                    if ctx.scope:
                        extracted = []
                        selected = self.selector_handler.select(document, ctx.scope)
                        if selected:
                            for i, elem in enumerate(selected, start=1):
                                parse_result = self._extract(v, elem, level=level+1)

                                if isinstance(parse_result, (list, tuple)):
                                    extracted.extend(parse_result)
                                else:
                                    extracted.append(parse_result)

                                # if we're not in an array,
                                # we only care about the first iteration
                                if not ctx.iterate:
                                    break

                            if self.DEBUG:
                                print(debug_offset,
                                    "parsed %d elements in scope (%s)" % (i, ctx.scope))

                    # local extraction
                    else:
                        extracted = self._extract(v, document, level=level+1)

                except NonMatchingNonOptionalKey as e:
                    if self.DEBUG:
                        print(debug_offset, str(e))
                    if not ctx.required or not self.STRICT_MODE:
                        output[ctx.key] = {}
                    else:
                        raise
                except Exception as e:
                    if self.DEBUG:
                        print(str(e))
                    raise

                # replace empty-list result when not looping by empty dict
                if (    isinstance(extracted, list)
                    and not extracted
                    and not ctx.iterate):
                        extracted = {}

                # keep only the first element if we're not in an array
                if self.KEEP_ONLY_FIRST_ELEMENT_IF_LIST:
                    try:
                        if (    isinstance(extracted, list)
                            and extracted
                            and not ctx.iterate):

                            if self.DEBUG:
                                print(debug_offset, "keep only 1st element")
                            extracted =  extracted[0]

                    except Exception as e:
                        if self.DEBUG:
                            print(str(e))
                            print(debug_offset, "error getting first element")

                # extraction for a required key gave nothing
                if (    self.STRICT_MODE
                    and ctx.required
                    and extracted is None):
                    raise NonMatchingNonOptionalKey(
                        'key "%s" is required but yield nothing\nCurrent path: %s/(%s)\n' % (
                            ctx.key,
                            document.getroottree().getpath(document),v
                            )
                        )

                # special key to extract a selector-defined level deeper
                # but still output at same level
                # this can be useful for breaking up long selectors
                # or when you need to mix XPath and CSS selectors
                # e.g.
                # {
                #   "something(#content div.main)": {
                #       "--(.//div[re:test(@class, 'style\d{3,6}')])": {
                #           "title": "h1",
                #           "subtitle": "h2"
                #       }
                #   }
                # }
                #
                if ctx.key == self.SPECIAL_LEVEL_KEY:
                    if isinstance(extracted, dict):
                        output.update(extracted)
                    elif isinstance(extracted, list):
                        if extracted:
                            raise RuntimeError(
                                "could not merge non-empty list at higher level")
                        else:
                            #empty list, dont bother?
                            pass
                else:
                    # required keys are handled above
                    if extracted is not None:
                        output[ctx.key] = extracted
                    else:
                        # do not add this optional key/value pair in the output
                        pass

            return output

        # a leaf/Selector node
        elif isinstance(parselet_node, Selector):
            return self.selector_handler.extract(document, parselet_node)

        else:
            # FIXME: can this happen?
            #        if selector handler returned None at compile time,
            #        probably yes
            pass

Response: Extract values at this document node level using the parselet_node instructions.


Instruction:
How would you explain what the following Python 3 function does
def auto_constraints(self, component=None):
        if not component:
            for table in self.tables:
                self.auto_constraints(table)
            return

        if not component.tableSchema.primaryKey:
            idcol = component.get_column(term_uri('id'))
            if idcol:
                component.tableSchema.primaryKey = [idcol.name]

        self._auto_foreign_keys(component)

        try:
            table_type = self.get_tabletype(component)
        except ValueError:
            # New component is not a known CLDF term, so cannot add components
            # automatically. TODO: We might me able to infer some based on
            # `xxxReference` column properties?
            return

        # auto-add foreign keys targetting the new component:
        for table in self.tables:
            self._auto_foreign_keys(table, component=component, table_type=table_type)

Response: Auto - adds foreign key constraints to the specified component.


Instruction:
How would you explain what the following Python 3 function does
def url_builder(self, endpoint, *, root=None, params=None, url_params=None):
        if root is None:
            root = self.ROOT
        scheme, netloc, path, _, _ = urlsplit(root)
        return urlunsplit((
            scheme,
            netloc,
            urljoin(path, endpoint),
            urlencode(url_params or {}),
            '',
        )).format(**params or {})

Response: Create a URL for the specified endpoint.


Instruction:
Implement a Python 3 function for
raising an appropriate error for a given response.

Response:
def raise_for_status(response):
    """Raise an appropriate error for a given response.

    Arguments:
      response (:py:class:`aiohttp.ClientResponse`): The API response.

    Raises:
      :py:class:`aiohttp.web_exceptions.HTTPException`: The appropriate
        error for the response's status.

    """
    for err_name in web_exceptions.__all__:
        err = getattr(web_exceptions, err_name)
        if err.status_code == response.status:
            payload = dict(
                headers=response.headers,
                reason=response.reason,
            )
            if issubclass(err, web_exceptions._HTTPMove):  # pylint: disable=protected-access
                raise err(response.headers['Location'], **payload)
            raise err(**payload)


Instruction:
Explain what the following Python 3 code does
def truncate(text, max_len=350, end='...'):
    if len(text) <= max_len:
        return text
    return text[:max_len].rsplit(' ', maxsplit=1)[0] + end

Response: Truncate the supplied text for display.


Instruction:
Implement a Python 3 function for
adding a source or a list of source entries to the list.

Response:
def add(self, *entries):
        """
        Add a source, either specified by glottolog reference id, or as bibtex record.
        """
        for entry in entries:
            if isinstance(entry, string_types):
                self._add_entries(database.parse_string(entry, bib_format='bibtex'))
            else:
                self._add_entries(entry)