Instruction:
Write a Python 3 script for
plotting histograms of all items in dat.

Response:
def histograms(dat, keys=None, bins=25, logy=False, cmap=None, ncol=4):
    """
    Plot histograms of all items in dat.

    Parameters
    ----------
    dat : dict
        Data in {key: array} pairs.
    keys : arra-like
        The keys in dat that you want to plot. If None,
        all are plotted.
    bins : int
        The number of bins in each histogram (default = 25)
    logy : bool
        If true, y axis is a log scale.
    cmap : dict
        The colours that the different items should be. If None,
        all are grey.

    Returns
    -------
    fig, axes
    """
    if keys is None:
        keys = dat.keys()

    ncol = int(ncol)
    nrow = calc_nrow(len(keys), ncol)

    fig, axs = plt.subplots(nrow, 4, figsize=[ncol * 2, nrow * 2])

    pn = 0
    for k, ax in zip(keys, axs.flat):
        tmp = nominal_values(dat[k])
        x = tmp[~np.isnan(tmp)]

        if cmap is not None:
            c = cmap[k]
        else:
            c = (0, 0, 0, 0.5)
        ax.hist(x, bins=bins, color=c)

        if logy:
            ax.set_yscale('log')
            ylab = '$log_{10}(n)$'
        else:
            ylab = 'n'

        ax.set_ylim(1, ax.get_ylim()[1])

        if ax.is_first_col():
            ax.set_ylabel(ylab)

        ax.set_yticklabels([])

        ax.text(.95, .95, k, ha='right', va='top', transform=ax.transAxes)

        pn += 1

    for ax in axs.flat[pn:]:
        ax.set_visible(False)
    fig.tight_layout()

    return fig, axs


Instruction:
Can you generate a brief explanation for the following Python 3 code
def autorange_plot(t, sig, gwin=7, swin=None, win=30,
                   on_mult=(1.5, 1.), off_mult=(1., 1.5),
                   nbin=10, thresh=None):
    """
    Function for visualising the autorange mechanism.

    Parameters
    ----------
    t : array-like
        Independent variable (usually time).
    sig : array-like
        Dependent signal, with distinctive 'on' and 'off' regions.
    gwin : int
        The window used for calculating first derivative.
        Defaults to 7.
    swin : int
        The window ised for signal smoothing. If None, gwin // 2.
    win : int
        The width (c +/- win) of the transition data subsets.
        Defaults to 20.
    on_mult and off_mult : tuple, len=2
        Control the width of the excluded transition regions, which is defined
        relative to the peak full-width-half-maximum (FWHM) of the transition
        gradient. The region n * FHWM below the transition, and m * FWHM above
        the tranision will be excluded, where (n, m) are specified in `on_mult`
        and `off_mult`.
        `on_mult` and `off_mult` apply to the off-on and on-off transitions,
        respectively.
        Defaults to (1.5, 1) and (1, 1.5).
    nbin : ind
        Used to calculate the number of bins in the data histogram.
        bins = len(sig) // nbin

    Returns
    -------
    fig, axes
    """
    if swin is None:
        swin = gwin // 2

    sigs = fastsmooth(sig, swin)

    # perform autorange calculations

    # bins = 50
    bins = sig.size // nbin
    kde_x = np.linspace(sig.min(), sig.max(), bins)

    kde = gaussian_kde(sigs)
    yd = kde.pdf(kde_x)
    mins = findmins(kde_x, yd)  # find minima in kde

    if thresh is not None:
        mins = [thresh]
    if len(mins) > 0:
        bkg = sigs < (mins[0])  # set background as lowest distribution
    else:
        bkg = np.ones(sig.size, dtype=bool)
    # bkg[0] = True  # the first value must always be background

    # assign rough background and signal regions based on kde minima
    fbkg = bkg
    fsig = ~bkg

    g = abs(fastgrad(sigs, gwin))  # calculate gradient of signal
    # 2. determine the approximate index of each transition
    zeros = bool_2_indices(fsig)

    if zeros is not None:
        zeros = zeros.flatten()
        lohi = []
        pgs = []
        excl = []
        tps = []
        failed = []

        for z in zeros:  # for each approximate transition
            # isolate the data around the transition
            if z - win < 0:
                lo = gwin // 2
                hi = int(z + win)
            elif z + win > (len(sig) - gwin // 2):
                lo = int(z - win)
                hi = len(sig) - gwin // 2
            else:
                lo = int(z - win)
                hi = int(z + win)

            xs = t[lo:hi]
            ys = g[lo:hi]

            lohi.append([lo, hi])

            # determine type of transition (on/off)
            mid = (hi + lo) // 2
            tp = sigs[mid + 3] > sigs[mid - 3]  # True if 'on' transition.
            tps.append(tp)

            c = t[z]  # center of transition
            width = (t[1] - t[0]) * 2  # initial width guess
            try:
                pg, _ = curve_fit(gauss, xs, ys,
                                  p0=(np.nanmax(ys),
                                      c,
                                      width),
                                  sigma=(xs - c)**2 + .01)
                pgs.append(pg)
                fwhm = abs(2 * pg[-1] * np.sqrt(2 * np.log(2)))
                # apply on_mult or off_mult, as appropriate.
                if tp:
                    lim = np.array([-fwhm, fwhm]) * on_mult + pg[1]
                else:
                    lim = np.array([-fwhm, fwhm]) * off_mult + pg[1]
                excl.append(lim)

                fbkg[(t > lim[0]) & (t < lim[1])] = False
                fsig[(t > lim[0]) & (t < lim[1])] = False
                failed.append(False)
            except RuntimeError:
                failed.append(True)
                lohi.append([np.nan, np.nan])
                pgs.append([np.nan, np.nan, np.nan])
                excl.append([np.nan, np.nan])
                tps.append(tp)
                pass
    else:
        zeros = []

    # make plot
    nrows = 2 + len(zeros) // 2 + len(zeros) % 2

    fig, axs = plt.subplots(nrows, 2, figsize=(6, 4 + 1.5 * nrows))

    # Trace
    ax1, ax2, ax3, ax4 = axs.flat[:4]
    ax4.set_visible(False)

    # widen ax1 & 3
    for ax in [ax1, ax3]:
        p = ax.axes.get_position()
        p2 = [p.x0, p.y0, p.width * 1.75, p.height]
        ax.axes.set_position(p2)

    # move ax3 up
    p = ax3.axes.get_position()
    p2 = [p.x0, p.y0 + 0.15 * p.height, p.width, p.height]
    ax3.axes.set_position(p2)

    # truncate ax2
    p = ax2.axes.get_position()
    p2 = [p.x0 + p.width * 0.6, p.y0, p.width * 0.4, p.height]
    ax2.axes.set_position(p2)

    # plot traces and gradient
    ax1.plot(t, sig, color='k', lw=1)
    ax1.set_xticklabels([])
    ax1.set_ylabel('Signal')
    ax3.plot(t, g, color='k', lw=1)
    ax3.set_xlabel('Time (s)')
    ax3.set_ylabel('Gradient')

    # plot kde
    ax2.fill_betweenx(kde_x, yd, color=(0, 0, 0, 0.2))
    ax2.plot(yd, kde_x, color='k')
    ax2.set_ylim(ax1.get_ylim())
    ax2.set_yticklabels([])
    ax2.set_xlabel('Data\nDensity')

    # limit
    for ax in [ax1, ax2]:
        ax.axhline(mins[0], color='k', ls='dashed', alpha=0.4)

    if len(zeros) > 0:
        # zeros
        for z in zeros:
            ax1.axvline(t[z], color='r', alpha=0.5)
            ax3.axvline(t[z], color='r', alpha=0.5)

        # plot individual transitions
        n = 1
        for (lo, hi), lim, tp, pg, fail, ax in zip(lohi, excl, tps, pgs, failed, axs.flat[4:]):
            # plot region on gradient axis
            ax3.axvspan(t[lo], t[hi], color='r', alpha=0.1, zorder=-2)

            # plot individual transitions
            x = t[lo:hi]
            y = g[lo:hi]
            ys = sig[lo:hi]
            ax.scatter(x, y, color='k', marker='x', zorder=-1, s=10)
            ax.set_yticklabels([])
            ax.set_ylim(rangecalc(y))

            tax = ax.twinx()
            tax.plot(x, ys, color='k', alpha=0.3, zorder=-5)
            tax.set_yticklabels([])
            tax.set_ylim(rangecalc(ys))

            # plot fitted gaussian
            xn = np.linspace(x.min(), x.max(), 100)
            ax.plot(xn, gauss(xn, *pg), color='r', alpha=0.5)

            # plot center and excluded region
            ax.axvline(pg[1], color='b', alpha=0.5)
            ax.axvspan(*lim, color='b', alpha=0.1, zorder=-2)

            ax1.axvspan(*lim, color='b', alpha=0.1, zorder=-2)
            if tp:
                ax.text(.05, .95, '{} (on)'.format(n), ha='left',
                        va='top', transform=ax.transAxes)
            else:
                ax.text(.95, .95, '{} (off)'.format(n), ha='right',
                        va='top', transform=ax.transAxes)

            if ax.is_last_row():
                ax.set_xlabel('Time (s)')
            if ax.is_first_col():
                ax.set_ylabel('Gradient (x)')
            if ax.is_last_col():
                tax.set_ylabel('Signal (line)')

            if fail:
                ax.axes.set_facecolor((1, 0, 0, 0.2))
                ax.text(.5, .5, 'FAIL', ha='center', va='center',
                        fontsize=16, color=(1, 0, 0, 0.5), transform=ax.transAxes)

            n += 1

        # should never be, but just in case...
        if len(zeros) % 2 == 1:
            axs.flat[-1].set_visible = False

    return fig, axs

Response: Function for visualising the autorange mechanism.


Instruction:
Implement a Python 3 function for
plotting the calibration lines between measured and known SRM values.

Response:
def calibration_plot(self, analytes=None, datarange=True, loglog=False, ncol=3, srm_group=None, save=True):
    """
    Plot the calibration lines between measured and known SRM values.

    Parameters
    ----------
    analytes : optional, array_like or str
        The analyte(s) to plot. Defaults to all analytes.
    datarange : boolean
        Whether or not to show the distribution of the measured data
        alongside the calibration curve.
    loglog : boolean
        Whether or not to plot the data on a log - log scale. This is
        useful if you have two low standards very close together,
        and want to check whether your data are between them, or
        below them.

    Returns
    -------
    (fig, axes)
    """

    if isinstance(analytes, str):
        analytes = [analytes]

    if analytes is None:
        analytes = [a for a in self.analytes if self.internal_standard not in a]

    if srm_group is not None:
        srm_groups = {int(g): t for g, t in self.stdtab.loc[:, ['group', 'gTime']].values}
        try:
            gTime = srm_groups[srm_group]
        except KeyError:
            text = ('Invalid SRM group selection. Valid options are:\n' +
                    ' Key:  Time Centre\n' +
                    '\n'.join(['   {:}:  {:.1f}s'.format(k, v) for k, v in srm_groups.items()]))
            print(text)
    else:
        gTime = None

    ncol = int(ncol)
    n = len(analytes)
    nrow = calc_nrow(n + 1, ncol)

    axes = []

    if not datarange:
        fig = plt.figure(figsize=[4.1 * ncol, 3 * nrow])
    else:
        fig = plt.figure(figsize=[4.7 * ncol, 3 * nrow])
        self.get_focus()

    gs = mpl.gridspec.GridSpec(nrows=int(nrow), ncols=int(ncol),
                            hspace=0.35, wspace=0.3)

    mdict = self.srm_mdict

    for g, a in zip(gs, analytes):
        if not datarange:
            ax = fig.add_axes(g.get_position(fig))
            axes.append((ax,))
        else:
            f = 0.8
            p0 = g.get_position(fig)
            p1 = [p0.x0, p0.y0, p0.width * f, p0.height]
            p2 = [p0.x0 + p0.width * f, p0.y0, p0.width * (1 - f), p0.height]
            ax = fig.add_axes(p1)
            axh = fig.add_axes(p2)
            axes.append((ax, axh))

        if gTime is None:
            sub = idx[a]
        else:
            sub = idx[a, :, :, gTime]
        x = self.srmtabs.loc[sub, 'meas_mean'].values
        xe = self.srmtabs.loc[sub, 'meas_err'].values
        y = self.srmtabs.loc[sub, 'srm_mean'].values
        ye = self.srmtabs.loc[sub, 'srm_err'].values
        srm = self.srmtabs.loc[sub].index.get_level_values('SRM')

        # plot calibration data
        for s, m in mdict.items():
            ind = srm == s
            ax.errorbar(x[ind], y[ind], xerr=xe[ind], yerr=ye[ind],
                        color=self.cmaps[a], alpha=0.6,
                        lw=0, elinewidth=1, marker=m, #'o',
                        capsize=0, markersize=5, label='_')

        # work out axis scaling
        if not loglog:
            xmax = np.nanmax(x + xe)
            ymax = np.nanmax(y + ye)
            if any(x - xe < 0):
                xmin = np.nanmin(x - xe)
                xpad = (xmax - xmin) * 0.05
                xlim = [xmin - xpad, xmax + xpad]
            else:
                xlim = [0, xmax * 1.05]

            if any(y - ye < 0):
                ymin = np.nanmin(y - ye)
                ypad = (ymax - ymin) * 0.05
                ylim = [ymin - ypad, ymax + ypad]
            else:
                ylim = [0, ymax * 1.05]

        else:
            xd = self.srmtabs.loc[a, 'meas_mean'][self.srmtabs.loc[a, 'meas_mean'] > 0].values
            yd = self.srmtabs.loc[a, 'srm_mean'][self.srmtabs.loc[a, 'srm_mean'] > 0].values

            xlim = [10**np.floor(np.log10(np.nanmin(xd))),
                    10**np.ceil(np.log10(np.nanmax(xd)))]
            ylim = [10**np.floor(np.log10(np.nanmin(yd))),
                    10**np.ceil(np.log10(np.nanmax(yd)))]

            # scale sanity checks
            if xlim[0] == xlim[1]:
                xlim[0] = ylim[0]
            if ylim[0] == ylim[1]:
                ylim[0] = xlim[0]

            ax.set_xscale('log')
            ax.set_yscale('log')

        ax.set_xlim(xlim)
        ax.set_ylim(ylim)

        # visual warning if any values < 0
        if xlim[0] < 0:
            ax.axvspan(xlim[0], 0, color=(1,0.8,0.8), zorder=-1)
        if ylim[0] < 0:
            ax.axhspan(ylim[0], 0, color=(1,0.8,0.8), zorder=-1)
        if any(x < 0) or any(y < 0):
            ax.text(.5, .5, 'WARNING: Values below zero.', color='r', weight='bold',
                    ha='center', va='center', rotation=40, transform=ax.transAxes, alpha=0.6)

        # calculate line and R2
        if loglog:
            x = np.logspace(*np.log10(xlim), 100)
        else:
            x = np.array(xlim)

        if gTime is None:
            coefs = self.calib_params.loc[:, a]
        else:
            coefs = self.calib_params.loc[gTime, a]

        m = np.nanmean(coefs['m'])
        m_nom = nominal_values(m)
        # calculate case-specific paramers
        if 'c' in coefs:
            c = np.nanmean(coefs['c'])
            c_nom = nominal_values(c)
            # calculate R2
            ym = self.srmtabs.loc[a, 'meas_mean'] * m_nom + c_nom
            R2 = R2calc(self.srmtabs.loc[a, 'srm_mean'], ym, force_zero=False)
            # generate line and label
            line = x * m_nom + c_nom
            label = 'y = {:.2e} x'.format(m)
            if c > 0:
                label += '\n+ {:.2e}'.format(c)
            else:
                label += '\n {:.2e}'.format(c)
        else:
            # calculate R2
            ym = self.srmtabs.loc[a, 'meas_mean'] * m_nom
            R2 = R2calc(self.srmtabs.loc[a, 'srm_mean'], ym, force_zero=True)
            # generate line and label
            line = x * m_nom
            label = 'y = {:.2e} x'.format(m)

        # plot line of best fit
        ax.plot(x, line, color=(0, 0, 0, 0.5), ls='dashed')

        # add R2 to label
        if round(R2, 3) == 1:
            label = '$R^2$: >0.999\n' + label
        else:
            label = '$R^2$: {:.3f}\n'.format(R2) + label

        ax.text(.05, .95, pretty_element(a), transform=ax.transAxes,
                weight='bold', va='top', ha='left', size=12)
        ax.set_xlabel('counts/counts ' + self.internal_standard)
        ax.set_ylabel('mol/mol ' + self.internal_standard)
        # write calibration equation on graph happens after data distribution

        # plot data distribution historgram alongside calibration plot
        if datarange:
            # isolate data
            meas = nominal_values(self.focus[a])
            meas = meas[~np.isnan(meas)]

            # check and set y scale
            if np.nanmin(meas) < ylim[0]:
                if loglog:
                    mmeas = meas[meas > 0]
                    ylim[0] = 10**np.floor(np.log10(np.nanmin(mmeas)))
                else:
                    ylim[0] = 0
                ax.set_ylim(ylim)

            m95 = np.percentile(meas[~np.isnan(meas)], 95) * 1.05
            if m95 > ylim[1]:
                if loglog:
                    ylim[1] = 10**np.ceil(np.log10(m95))
                else:
                    ylim[1] = m95

            # hist
            if loglog:
                bins = np.logspace(*np.log10(ylim), 30)
            else:
                bins = np.linspace(*ylim, 30)

            axh.hist(meas, bins=bins, orientation='horizontal',
                        color=self.cmaps[a], lw=0.5, alpha=0.5)

            if loglog:
                axh.set_yscale('log')
            axh.set_ylim(ylim)  # ylim of histogram axis
            ax.set_ylim(ylim)  # ylim of calibration axis
            axh.set_xticks([])
            axh.set_yticklabels([])

        # write calibration equation on graph
        cmax = np.nanmax(y)
        if cmax / ylim[1] > 0.5:
            ax.text(0.98, 0.04, label, transform=ax.transAxes,
                    va='bottom', ha='right')
        else:
            ax.text(0.02, 0.75, label, transform=ax.transAxes,
                    va='top', ha='left')

    if srm_group is None:
        title = 'All SRMs'
    else:
        title = 'SRM Group {:} (centre at {:.1f}s)'.format(srm_group, gTime)
    axes[0][0].set_title(title, loc='left', weight='bold', fontsize=12)

    # SRM legend
    ax = fig.add_axes(gs[-1].get_position(fig))
    for lab, m in mdict.items():
        ax.scatter([],[],marker=m, label=lab, color=(0,0,0,0.6))
    ax.legend()
    ax.axis('off')

    if save:
        fig.savefig(self.report_dir + '/calibration.pdf')

    return fig, axes


Instruction:
How would you explain what the following Python 3 function does
def filter_report(Data, filt=None, analytes=None, savedir=None, nbin=5):
    if filt is None or filt == 'all':
        sets = Data.filt.sets
    else:
        sets = {k: v for k, v in Data.filt.sets.items() if any(filt in f for f in v)}

    regex = re.compile('^([0-9]+)_([A-Za-z0-9-]+)_'
                    '([A-Za-z0-9-]+)[_$]?'
                    '([a-z0-9]+)?')

    cm = plt.cm.get_cmap('Spectral')
    ngrps = len(sets)

    if analytes is None:
        analytes = Data.analytes
    elif isinstance(analytes, str):
        analytes = [analytes]

    axes = []
    for analyte in analytes:
        if analyte != Data.internal_standard:
            fig = plt.figure()

            for i in sorted(sets.keys()):
                filts = sets[i]
                nfilts = np.array([re.match(regex, f).groups() for f in filts])
                fgnames = np.array(['_'.join(a) for a in nfilts[:, 1:3]])
                fgrp = np.unique(fgnames)[0]

                fig.set_size_inches(10, 3.5 * ngrps)
                h = .8 / ngrps

                y = nominal_values(Data.focus[analyte])
                yh = y[~np.isnan(y)]

                m, u = unitpicker(np.nanmax(y),
                                denominator=Data.internal_standard,
                                focus_stage=Data.focus_stage)

                axs = tax, hax = (fig.add_axes([.1, .9 - (i + 1) * h, .6, h * .98]),
                                fig.add_axes([.7, .9 - (i + 1) * h, .2, h * .98]))
                axes.append(axs)

                # get variables
                fg = sets[i]
                cs = cm(np.linspace(0, 1, len(fg)))
                fn = ['_'.join(x) for x in nfilts[:, (0, 3)]]
                an = nfilts[:, 0]
                bins = np.linspace(np.nanmin(y), np.nanmax(y), len(yh) // nbin) * m

                if 'DBSCAN' in fgrp:
                    # determine data filters
                    core_ind = Data.filt.components[[f for f in fg
                                                    if 'core' in f][0]]
                    other = np.array([('noise' not in f) & ('core' not in f)
                                    for f in fg])
                    tfg = fg[other]
                    tfn = fn[other]
                    tcs = cm(np.linspace(0, 1, len(tfg)))

                    # plot all data
                    hax.hist(m * yh, bins, alpha=0.2, orientation='horizontal',
                            color='k', lw=0)
                    # legend markers for core/member
                    tax.scatter([], [], s=20, label='core', color='w', lw=0.5, edgecolor='k')
                    tax.scatter([], [], s=7.5, label='member', color='w', lw=0.5, edgecolor='k')
                    # plot noise
                    try:
                        noise_ind = Data.filt.components[[f for f in fg
                                                        if 'noise' in f][0]]
                        tax.scatter(Data.Time[noise_ind], m * y[noise_ind],
                                    lw=1, color='k', s=10, marker='x',
                                    label='noise', alpha=0.6)
                    except:
                        pass

                    # plot filtered data
                    for f, c, lab in zip(tfg, tcs, tfn):
                        ind = Data.filt.components[f]
                        tax.scatter(Data.Time[~core_ind & ind],
                                    m * y[~core_ind & ind], lw=.5, color=c, s=5, edgecolor='k')
                        tax.scatter(Data.Time[core_ind & ind],
                                    m * y[core_ind & ind], lw=.5, color=c, s=15, edgecolor='k',
                                    label=lab)
                        hax.hist(m * y[ind][~np.isnan(y[ind])], bins, color=c, lw=0.1,
                                orientation='horizontal', alpha=0.6)

                else:
                    # plot all data
                    tax.scatter(Data.Time, m * y, color='k', alpha=0.2, lw=0.1,
                                s=20, label='excl')
                    hax.hist(m * yh, bins, alpha=0.2, orientation='horizontal',
                             color='k', lw=0)

                    # plot filtered data
                    for f, c, lab in zip(fg, cs, fn):
                        ind = Data.filt.components[f]
                        tax.scatter(Data.Time[ind], m * y[ind],
                                    edgecolor=(0,0,0,0), color=c, s=15, label=lab)
                        hax.hist(m * y[ind][~np.isnan(y[ind])], bins, color=c, lw=0.1,
                                orientation='horizontal', alpha=0.6)

                if 'thresh' in fgrp and analyte in fgrp:
                    tax.axhline(Data.filt.params[fg[0]]['threshold'] * m,
                                ls='dashed', zorder=-2, alpha=0.5, color='k')
                    hax.axhline(Data.filt.params[fg[0]]['threshold'] * m,
                                ls='dashed', zorder=-2, alpha=0.5, color='k')

                # formatting
                for ax in axs:
                    mn = np.nanmin(y) * m
                    mx = np.nanmax(y) * m
                    rn = mx - mn
                    ax.set_ylim(mn - .05 * rn, mx + 0.05 * rn)

                # legend
                hn, la = tax.get_legend_handles_labels()
                hax.legend(hn, la, loc='upper right', scatterpoints=1)

                tax.text(.02, .98, Data.sample + ': ' + fgrp, size=12,
                        weight='bold', ha='left', va='top',
                        transform=tax.transAxes)
                tax.set_ylabel(pretty_element(analyte) + ' (' + u + ')')
                tax.set_xticks(tax.get_xticks()[:-1])
                hax.set_yticklabels([])

                if i < ngrps - 1:
                    tax.set_xticklabels([])
                    hax.set_xticklabels([])
                else:
                    tax.set_xlabel('Time (s)')
                    hax.set_xlabel('n')

        if isinstance(savedir, str):
            fig.savefig(savedir + '/' + Data.sample + '_' +
                        analyte + '.pdf')
            plt.close(fig)

    return fig, axes

Response: Visualise effect of data filters.


Instruction:
Implement a function in Python 3 to
calculate the reproducibility of LA - ICPMS based on unique pairs of repeat analyses.

Response:
def pairwise_reproducibility(df, plot=False):
    """
    Calculate the reproducibility of LA-ICPMS based on unique pairs of repeat analyses.

    Pairwise differences are fit with a half-Cauchy distribution, and the median and
    95% confidence limits are returned for each analyte.

    Parameters
    ----------
    df : pandas.DataFrame
        A dataset

    plot : bool
        Whether or not to plot the resulting error distributions.

    Returns
    -------
    pdiffs : pandas.DataFrame
        Unique pairwise differences for all analytes.
    rep_dists : dict of scipy.stats.halfcauchy
        Half-Cauchy distribution objects fitted to the
        differences.
    rep_stats : dict of tuples
        The 50% and 95% quantiles of the half-cauchy
        distribution.
    (fig, axs) : matplotlib objects
        The figure. If not made, returnes (None, None) placeholder

    """

    ans = df.columns.values
    pdifs = []

    # calculate differences between unique pairs
    for ind, d in df.groupby(level=0):
        d.index = d.index.droplevel(0)

        difs = []
        for i, r in d.iterrows():
            t = d.loc[i+1:, :]
            difs.append(t[ans] - r[ans])

        pdifs.append(pd.concat(difs))
    pdifs = pd.concat(pdifs).abs()

    # calculate stats
    rep_stats = {}
    rep_dists = {}
    errfn = stats.halfcauchy

    for a in ans:
        d = pdifs.loc[:, a].dropna().values
        hdist = errfn.fit(d, floc=0)
        rep_dists[a] = errfn(*hdist)
        rep_stats[a] = rep_dists[a].ppf((0.5, 0.95))

    # make plot
    if not plot:
        return pdifs, rep_dists, rep_stats, (None, None)

    fig, axs = plt.subplots(1, len(ans), figsize=[len(ans) * 2, 2])
    for a, ax in zip(ans, axs):
        d = pdifs.loc[:, a].dropna().values
        hist, edges, _ = ax.hist(d, 30)
        ax.plot(edges, rep_dists[a].pdf(edges) * (sum(hist) * np.mean(np.diff(edges))))
        ax.set_title(a, loc='left')

    return pdifs, rep_dists, rep_stats, (fig, axs)


Instruction:
Write a Python 3 function for
computing comparison stats for a set of elements in a set of objects.

Response:
def comparison_stats(df, els=['Mg', 'Sr', 'Ba', 'Al', 'Mn']):
    """
    Compute comparison stats for test and LAtools data.

    Population-level similarity assessed by a Kolmogorov-Smirnov test.

    Individual similarity assessed by a pairwise Wilcoxon signed rank test.

    Trends in residuals assessed by regression analysis, where significance of
    the slope and intercept is determined by t-tests (both relative to zero).

    Parameters
    ----------
    df : pandas.DataFrame
        A dataframe containing reference ('X/Ca_r'), test user
        ('X/Ca_t') and LAtools ('X123') data.
    els : list
        list of elements (names only) to plot.

    Returns
    -------
    pandas.DataFrame

    """

    # get corresponding analyte and ratio names
    As = []
    Rs = []
    analytes = [c for c in df.columns if ('_r' not in c) and ('_t' not in c)]
    ratios = [c for c in df.columns if ('_r' in c)]

    for e in els:
        if e == 'Sr':
            As.append('Sr88')
        elif e == 'Mg':
            As.append('Mg24')
        else:
            As.append([a for a in analytes if e in a][0])
        Rs.append([r for r in ratios if e in r][0][:-2])

    yt_stats = []
    yl_stats = []

    for i, (e, a) in enumerate(zip(Rs, As)):
        if a == 'Ba138':
            m = 1e3
            u = '$\mu$mol/mol'
        else:
            m = 1
            u = 'mmol/mol'

        x = df.loc[:, e + '_r'].values * m
        yt = df.loc[:, e + '_t'].values * m
        yl = df.loc[:, a].values * m

        yt_stats.append(summary_stats(x, yt, e))
        yl_stats.append(summary_stats(x, yl, e))

    yt_stats = pd.concat(yt_stats).T
    yl_stats = pd.concat(yl_stats).T

    return pd.concat([yt_stats, yl_stats], keys=['Test User', 'LAtools']).T


Instruction:
How would you implement a function in Python 3 that
computes summary statistics for paired x y data.

Response:
def summary_stats(x, y, nm=None):
    """
    Compute summary statistics for paired x, y data.

    Tests
    -----

    Parameters
    ----------
    x, y : array-like
        Data to compare
    nm : str (optional)
        Index value of created dataframe.

    Returns
    -------
    pandas dataframe of statistics.
    """
    # create datafrane for results
    if isinstance(nm, str):
        nm = [nm]
    # cols = pd.MultiIndex.from_arrays([['', 'Pairwise', 'Pairwise', cat, cat, cat, cat],
    #                                   ['N', 'W', 'p', 'Median', 'IQR', 'W', 'p']])
#     cols = ['Median', 'IQR', 'CI95', 'L95', 'LQ', 'UQ', 'U95', 'N',
#             'Wilcoxon_stat', 'Wilcoxon_p',
#             'KS_stat', 'KS_p',
#             'LR_slope', 'LR_intercept', 'LR_slope_tvalue', 'LR_intercept_tvalue', 'LR_slope_p', 'LR_intercept_p', 'LR_R2adj']
#     out = pd.DataFrame(index=nm, columns=cols)

    cols = pd.MultiIndex.from_tuples([('Residual Summary', 'N'),
                                      ('Residual Summary', 'Median'),
                                      ('Residual Summary', 'LQ'),
                                      ('Residual Summary', 'IQR'),
                                      ('Residual Summary', 'UQ'),
                                      ('Residual Regression', 'Slope'),
                                      ('Residual Regression', 'Slope t'),
                                      ('Residual Regression', 'Slope p'),
                                      ('Residual Regression', 'Intercept'),
                                      ('Residual Regression', 'Intercept t'),
                                      ('Residual Regression', 'Intercept p'),
                                      ('Residual Regression', 'R2'),
                                      ('Kolmogorov-Smirnov', 'KS'),
                                      ('Kolmogorov-Smirnov', 'p')])

    out = pd.DataFrame(index=nm, columns=cols)

    # remove nan values
    ind = ~(np.isnan(x) | np.isnan(y))
    x = x[ind]
    y = y[ind]

    # calculate residuals
    r = y - x

    # summary statistics
    cat = 'Residual Summary'
    out.loc[:, (cat, 'N')] = len(x)
    out.loc[:, (cat, 'Median')] = np.median(r)
    out.loc[:, [(cat, 'LQ'), (cat, 'UQ')]] = np.percentile(r, [25, 75])
    out.loc[:, (cat, 'IQR')] = out.loc[:, (cat, 'UQ')] - out.loc[:, (cat, 'LQ')]

    # non-paired test for same distribution
    cat = 'Kolmogorov-Smirnov'
    ks = stats.ks_2samp(x, y)
    out.loc[:, (cat, 'KS')] = ks.statistic
    out.loc[:, (cat, 'p')] = ks.pvalue

    # regression analysis of residuals - slope should be 0, intercept should be 0
    cat = 'Residual Regression'
    X = sm.add_constant(x)
    reg = sm.OLS(r, X, missing='drop')
    fit = reg.fit()

    out.loc[:, [(cat, 'Intercept'), (cat, 'Slope')]] = fit.params
    out.loc[:, [(cat, 'Intercept t'), (cat, 'Slope t')]] = fit.tvalues
    out.loc[:, (cat, 'R2')] = fit.rsquared
    out.loc[:, [(cat, 'Intercept p'), (cat, 'Slope p')]] = fit.pvalues

    return out


Instruction:
Implement a Python 3 function for
downloading the data from the LAtools reference repository.

Response:
def load_reference_data(name=None):
    """
    Fetch LAtools reference data from online repository.

    Parameters
    ----------
    name : str<
        Which data to download. Can be one of 'culture_reference',
        'culture_test', 'downcore_reference', 'downcore_test', 'iolite_reference'
        or 'zircon_reference'.
        If None, all are downloaded and returned as a dict.

    Returns
    -------
    pandas.DataFrame or dict.
    """
    base_url = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vQJfCeuqrtFFMAeSpA9rguzLAo9OVuw50AHhAULuqjMJzbd3h46PK1KjF69YiJAeNAAjjMDkJK7wMpG/pub?gid={:}&single=true&output=csv'
    gids = {'culture_reference': '0',
            'culture_test': '1170065442',
            'downcore_reference': '190752797',
            'downcore_test': '721359794',
            'iolite_reference': '483581945',
            'zircon_reference': '1355554964'}

    if name is None:
        out = {}
        for nm, gid in gids.items():
            url = base_url.format(gid)
            tmp = pd.read_csv(url, header=[0], index_col=[0, 1])
            tmp.index.names = ['sample', 'rep']
            tmp.columns.names = ['analyte']
            tmp.sort_index(1, inplace=True)
            out[nm] = tmp
    else:
        gid = gids[name]
        url = base_url.format(gid)
        out = pd.read_csv(url, index_col=[0, 1])
        out.columns.names = ['analyte']
        out.sort_index(1, inplace=True)
    return out


Instruction:
Explain what the following Python 3 code does
def lookup(self, TC: type, G: type) -> Optional[TypeClass]:
        ''' Find an instance of the type class `TC` for type `G`.
        Iterates `G`'s parent classes, looking up instances for each,
        checking whether the instance is a subclass of the target type
        class `TC`.
        '''
        if isinstance(G, str):
            raise ImplicitNotFound(TC, G, f'{G} is a string annotation')
        if not isinstance(G, (type, TypeVar, _GenericAlias)):
            raise ImplicitNotFound(TC, G, f'{G} is neither type, _GenericAlias nor TypeVar: {type(G)}')
        match = lambda a: self._lookup_type(TC, a)
        def attach_type(tc: TypeClass) -> TypeClass:
            setattr(tc, 'tpe', G)
            return tc
        scrutinee = (
            (
                unbounded_typevar(TC, G)
                if G.__bound__ is None else
                G.__bound__
            )
            if isinstance(G, TypeVar) else
            G
        )
        safe_mro = lambda t: getattr(t, '__mro__', (t,))
        target = scrutinee.__origin__ if isinstance(scrutinee, _GenericAlias) else scrutinee
        mro = safe_mro(target)
        return next((attach_type(a) for a in map(match, mro) if a is not None), None)

Response: Find an instance of the type class TC for type G.


Instruction:
Write a Python 3 function for
calculating the rangecalc of a single series.

Response:
def rangecalc(x, y=None, pad=0.05):
    """
    Calculate padded range limits for axes.
    """
    mn = np.nanmin([np.nanmin(x), np.nanmin(y)])
    mx = np.nanmax([np.nanmax(x), np.nanmax(y)])
    rn = mx - mn

    return (mn - pad * rn, mx + pad * rn)


Instruction:
Implement a Python 3 function for
calculating the rangecalcx limits for axes.

Response:
def rangecalcx(x, pad=0.05):
    """
    Calculate padded range limits for axes.
    """
    mn = np.nanmin(x)
    mx = np.nanmax(x)
    rn = mx - mn

    return (mn - pad * rn, mx + pad * rn)


Instruction:
Can you generate the documentation for the following Python 3 function
def bland_altman(x, y, interval=None, indep_conf=None, ax=None, c=None, **kwargs):
    ret = False

    if ax is None:
        fig, ax = plt.subplots(1, 1)
        ret = True

    # NaN screening
    ind = ~(np.isnan(x) | np.isnan(y))
    x = x[ind]
    y = y[ind]

    xy_mean = (x + y) / 2
    xy_resid = (y - x)

    ax.scatter(xy_mean, xy_resid, lw=0.5, edgecolor='k', alpha=0.6, c=c, s=15, **kwargs)

    # markup
    ax.axhline(0, ls='dashed', c='k', alpha=0.6, zorder=-1)

    ax.axhline(np.median(xy_resid), ls='dashed', c=c, alpha=0.8)

    if interval is not None:
        perc = 100 - interval * 100
        ints = [perc / 2, 100 - perc / 2]
        lims = np.percentile(xy_resid, ints)
        ax.axhspan(*lims, color=c, alpha=0.1, zorder=-3)

    if indep_conf is not None:
        ax.axhspan(-indep_conf, indep_conf, color=(0,0,0,0.1), zorder=-2)

    # labels
    ax.set_ylabel('y - x')
    ax.set_xlabel('mean (x, y)')

    if ret:
        return fig, ax

Response: Plots a Bland - Altman plot of x and y data.


Instruction:
Create a Python 3 function for
returning the components of the autorange algorithm.

Response:
def autorange_components(t, sig, transform='log', gwin=7, swin=None,
                         win=30, on_mult=(1.5, 1.), off_mult=(1., 1.5),
                         thresh=None):
    """
    Returns the components underlying the autorange algorithm.

    Returns
    -------
    t : array-like
        Time axis (independent variable)
    sig : array-like
        Raw signal (dependent variable)
    sigs : array-like
        Smoothed signal (swin)
    tsig : array-like
        Transformed raw signal (transform)
    tsigs : array-like
        Transformed smoothed signal (transform, swin)
    kde_x : array-like
        kernel density estimate of smoothed signal.
    yd : array-like
        bins of kernel density estimator.
    g : array-like
        gradient of smoothed signal (swin, gwin)
    trans : dict
        per-transition data.
    thresh : float
        threshold identified from kernel density plot
    """
    failed = []
    # smooth signal
    if swin is not None:
        sigs = fastsmooth(sig, swin)
    else:
        sigs = sig

    # transform signal
    if transform == 'log':
        tsigs = np.log10(sigs)
        tsig = np.log10(sig)
    else:
        tsigs = sigs
        tsig = sig

    if thresh is None:
        bins = 50
        kde_x = np.linspace(tsigs.min(), tsigs.max(), bins)

        kde = gaussian_kde(tsigs)
        yd = kde.pdf(kde_x)
        mins = findmins(kde_x, yd)  # find minima in kde

        if len(mins) > 0:
            bkg = tsigs < (mins[0])  # set background as lowest distribution
            thresh = mins[0]
        else:
            bkg = np.ones(tsigs.size, dtype=bool)
    else:
        bkg = tsigs < thresh

    # assign rough background and signal regions based on kde minima
    fbkg = bkg
    fsig = ~bkg

    # remove transitions by fitting a gaussian to the gradients of
    # each transition

    # 1. determine the approximate index of each transition
    zeros = bool_2_indices(fsig)

    # 2. calculate the absolute gradient of the target trace.
    g = abs(fastgrad(sigs, gwin))  # gradient of untransformed data.

    if zeros is not None:
        zeros = zeros.flatten()
        trans = dict(zeros=zeros.flatten(),
                     lohi=[],
                     pgs=[],
                     excl=[],
                     tps=[],
                     failed=[],
                     xs=[],
                     ys=[])

        for z in zeros:  # for each approximate transition
            # isolate the data around the transition
            if z - win < 0:
                lo = gwin // 2
                hi = int(z + win)
            elif z + win > (len(sig) - gwin // 2):
                lo = int(z - win)
                hi = len(sig) - gwin // 2
            else:
                lo = int(z - win)
                hi = int(z + win)

            xs = t[lo:hi]
            ys = g[lo:hi]

            trans['xs'].append(xs)
            trans['ys'].append(ys)

            trans['lohi'].append([lo, hi])

            # determine type of transition (on/off)
            mid = (hi + lo) // 2
            tp = sigs[mid + 3] > sigs[mid - 3]  # True if 'on' transition.
            trans['tps'].append(tp)

            c = t[z]  # center of transition
            width = (t[1] - t[0]) * 2  # initial width guess
            try:
                pg, _ = curve_fit(gauss, xs, ys,
                                  p0=(np.nanmax(ys),
                                      c,
                                      width),
                                  sigma=(xs - c)**2 + .01)
                trans['pgs'].append(pg)
                fwhm = abs(2 * pg[-1] * np.sqrt(2 * np.log(2)))
                # apply on_mult or off_mult, as appropriate.
                if tp:
                    lim = np.array([-fwhm, fwhm]) * on_mult + pg[1]
                else:
                    lim = np.array([-fwhm, fwhm]) * off_mult + pg[1]
                trans['excl'].append(lim)

                fbkg[(t > lim[0]) & (t < lim[1])] = False
                fsig[(t > lim[0]) & (t < lim[1])] = False
                failed.append(False)
            except RuntimeError:
                failed.append(True)
                trans['lohi'].append([np.nan, np.nan])
                trans['pgs'].append([np.nan, np.nan, np.nan])
                trans['excl'].append([np.nan, np.nan])
                trans['tps'].append(tp)
                pass
    else:
        zeros = []

    return t, sig, sigs, tsig, tsigs, kde_x, yd, g, trans, thresh


Instruction:
Can you generate the documentation for the following Python 3 function
def elements(all_isotopes=True):
    el = pd.read_pickle(pkgrs.resource_filename('latools', 'resources/elements.pkl'))
    if all_isotopes:
        return el.set_index('element')
    else:
        def wmean(g):
            return (g.atomic_weight * g.percent).sum() / 100
        iel = el.groupby('element').apply(wmean)
        iel.name = 'atomic_weight'
        return iel

Response: Load a DataFrame of all elements and isotopes.


Instruction:
Write a Python 3 script for
calculating the molecular weight of a molecule.

Response:
def calc_M(molecule):
    """
    Returns molecular weight of molecule.

    Where molecule is in standard chemical notation,
    e.g. 'CO2', 'HCO3' or B(OH)4

    Returns
    -------
    molecular_weight : float
    """

    # load periodic table
    els = elements()

    # define regexs
    parens = re.compile('\(([A-z0-9]+)\)([0-9]+)?')
    stoich = re.compile('([A-Z][a-z]?)([0-9]+)?')

    ps = parens.findall(molecule)  # find subgroups in parentheses
    rem = parens.sub('', molecule)  # get remainder

    m = 0
    # deal with sub-groups
    if len(ps) > 0:
        for sub, ns in ps:
            ms = 0
            for e, n in stoich.findall(sub):
                me = (els.loc[e, 'atomic_weight'] *
                      els.loc[e, 'percent'] / 100).sum()
                if n == '':
                    n = 1
                else:
                    n = int(n)
                ms += me * n
            if ns == '':
                ns = 1
            else:
                ns = int(ns)
            m += ms * ns
    # deal with remainder
    for e, n in stoich.findall(rem):
        me = (els.loc[e, 'atomic_weight'] *
              els.loc[e, 'percent'] / 100).sum()
        if n == '':
            n = 1
        else:
            n = int(n)
        m += me * n
    return m


Instruction:
Implement a Python 3 function for
generating single escape sequence mapping.

Response:
def gen_keywords(*args: Union[ANSIColors, ANSIStyles], **kwargs: Union[ANSIColors, ANSIStyles]) -> tuple:
    '''generate single escape sequence mapping.'''
    fields: tuple = tuple()
    values: tuple = tuple()
    for tpl in args:
        fields += tpl._fields
        values += tpl
    for prefix, tpl in kwargs.items():
        fields += tuple(map(lambda x: '_'.join([prefix, x]), tpl._fields))
        values += tpl
    return namedtuple('ANSISequences', fields)(*values)


Instruction:
Write a Python 3 script to
handle Resets in input stack. Breaks the input stack if a Reset operator is encountered.

Response:
def zero_break(stack: tuple) -> tuple:
    '''Handle Resets in input stack.
    Breaks the input stack if a Reset operator (zero) is encountered.
    '''
    reducer = lambda x, y: tuple() if y == 0 else x + (y,)
    return reduce(reducer, stack, tuple())


Instruction:
Can you implement a function in Python 3 that
squashes and reduce the input stack.

Response:
def annihilate(predicate: tuple, stack: tuple) -> tuple:
    '''Squash and reduce the input stack.
    Removes the elements of input that match predicate and only keeps the last
    match at the end of the stack.
    '''
    extra = tuple(filter(lambda x: x not in predicate, stack))
    head = reduce(lambda x, y: y if y in predicate else x, stack, None)
    return extra + (head,) if head else extra


Instruction:
Create a Python 3 function to
remove duplicates from the stack in first - seen order.

Response:
def dedup(stack: tuple) -> tuple:
    '''Remove duplicates from the stack in first-seen order.'''
    # Initializes with an accumulator and then reduces the stack with first match
    # deduplication.
    reducer = lambda x, y: x if y in x else x + (y,)
    return reduce(reducer, stack, tuple())


Instruction:
Explain what the following Python 3 code does
def gauss_weighted_stats(x, yarray, x_new, fwhm):
    sigma = fwhm / (2 * np.sqrt(2 * np.log(2)))

    # create empty mask array
    mask = np.zeros((x.size, yarray.shape[1], x_new.size))
    # fill mask
    for i, xni in enumerate(x_new):
        mask[:, :, i] = gauss(x[:, np.newaxis], 1, xni, sigma)
    # normalise mask
    nmask = mask / mask.sum(0)  # sum of each gaussian = 1

    # calculate moving average
    av = (nmask * yarray[:, :, np.newaxis]).sum(0)  # apply mask to data
    # sum along xn axis to get means

    # calculate moving sd
    diff = np.power(av - yarray[:, :, np.newaxis], 2)
    std = np.sqrt((diff * nmask).sum(0))
    # sqrt of weighted average of data-mean

    # calculate moving se
    se = std / np.sqrt(mask.sum(0))
    # max amplitude of weights is 1, so sum of weights scales
    # a fn of how many points are nearby. Use this as 'n' in
    # SE calculation.

    return av, std, se

Response: Calculate the weighted average SD and SE of a gaussian weigted moving mean.


Instruction:
Can you create a Python 3 function that
returns the gaussian function.

Response:
def gauss(x, *p):
    """ Gaussian function.

    Parameters
    ----------
    x : array_like
        Independent variable.
    *p : parameters unpacked to A, mu, sigma
        A = amplitude, mu = centre, sigma = width

    Return
    ------
    array_like
        gaussian descriped by *p.
    """
    A, mu, sigma = p
    return A * np.exp(-0.5 * (-mu + x)**2 / sigma**2)


Instruction:
How would you implement a function in Python 3 that
calculates the standard error of a sequence.

Response:
def stderr(a):
    """
    Calculate the standard error of a.
    """
    return np.nanstd(a) / np.sqrt(sum(np.isfinite(a)))


Instruction:
Create a Python 3 function to
calculate the Huber ( H15 ) Robust mean of x.

Response:
def H15_mean(x):
    """
    Calculate the Huber (H15) Robust mean of x.

    For details, see:
        http://www.cscjp.co.jp/fera/document/ANALYSTVol114Decpgs1693-97_1989.pdf
        http://www.rsc.org/images/robust-statistics-technical-brief-6_tcm18-214850.pdf
    """
    mu = np.nanmean(x)
    sd = np.nanstd(x) * 1.134
    sig = 1.5

    hi = x > mu + sig * sd
    lo = x < mu - sig * sd

    if any(hi | lo):
        x[hi] = mu + sig * sd
        x[lo] = mu - sig * sd
        return H15_mean(x)
    else:
        return mu


Instruction:
Can you generate a brief explanation for the following Python 3 code
def H15_se(x):
    sd = H15_std(x)
    return sd / np.sqrt(sum(np.isfinite(x)))

Response: Calculates the Huber ( H15 ) Robust standard deviation of x


Instruction:
How would you explain what the following Python 3 function does
def reproduce(past_analysis, plotting=False, data_folder=None,
              srm_table=None, custom_stat_functions=None):
    """
    Reproduce a previous analysis exported with :func:`latools.analyse.minimal_export`

    For normal use, supplying `log_file` and specifying a plotting option should be
    enough to reproduce an analysis. All requisites (raw data, SRM table and any
    custom stat functions) will then be imported from the minimal_export folder.

    You may also specify your own raw_data, srm_table and custom_stat_functions,
    if you wish.

    Parameters
    ----------
    log_file : str
        The path to the log file produced by :func:`~latools.analyse.minimal_export`.
    plotting : bool
        Whether or not to output plots.
    data_folder : str
        Optional. Specify a different data folder. Data folder
        should normally be in the same folder as the log file.
    srm_table : str
        Optional. Specify a different SRM table. SRM table
        should normally be in the same folder as the log file.
    custom_stat_functions : str
        Optional. Specify a python file containing custom
        stat functions for use by reproduce. Any custom
        stat functions should normally be included in the
        same folder as the log file.
    """
    if '.zip' in past_analysis:
        dirpath = utils.extract_zipdir(past_analysis)
        logpath = os.path.join(dirpath, 'analysis.lalog')
    elif os.path.isdir(past_analysis):
        if os.path.exists(os.path.join(past_analysis, 'analysis.lalog')):
            logpath = os.path.join(past_analysis, 'analysis.lalog')
    elif 'analysis.lalog' in past_analysis:
        logpath = past_analysis
    else:
        raise ValueError(('\n\n{} is not a valid input.\n\n' +
                          'Must be one of:\n' +
                          '  - A .zip file exported by latools\n' +
                          '  - An analysis.lalog file\n' +
                          '  - A directory containing an analysis.lalog files\n'))

    runargs, paths = logging.read_logfile(logpath)

    # parse custom stat functions
    csfs = Bunch()
    if custom_stat_functions is None and 'custom_stat_functions' in paths.keys():
        # load custom functions as a dict
        with open(paths['custom_stat_functions'], 'r') as f:
            csf = f.read()

        fname = re.compile('def (.*)\(.*')

        for c in csf.split('\n\n\n\n'):
            if fname.match(c):
                csfs[fname.match(c).groups()[0]] = c

    # create analysis object
    rep = analyse(*runargs[0][-1]['args'], **runargs[0][-1]['kwargs'])

    # rest of commands
    for fname, arg in runargs:
        if fname != '__init__':
            if 'plot' in fname.lower() and plotting:
                getattr(rep, fname)(*arg['args'], **arg['kwargs'])
            elif 'sample_stats' in fname.lower():
                rep.sample_stats(*arg['args'], csf_dict=csfs, **arg['kwargs'])
            else:
                getattr(rep, fname)(*arg['args'], **arg['kwargs'])

    return rep

Response: Reproduces a previous analysis.


Instruction:
How would you code a function in Python 3 to
determine exponential decay coefficient for despike filter. Fits an exponential decay function to the washout phase of standards to determine the washout time of your laser cell. The exponential coefficient reported is `nsd_below` standard deviations below the fitted exponent, to ensure that no real data is removed. Total counts are used in fitting, rather than a specific analyte. Parameters ---------- nsd_below : float The number of standard deviations to subtract from the fitted coefficient when calculating the filter exponent. plot : bool or str If True, creates a plot of the fit, if str the plot is to the location specified in str. trimlim : float A threshold limit used in determining the start of the exponential decay region of the washout. Defaults to half the increase in signal over background. If the data in the plot don't fall on an exponential decay line, change this number. Normally you'll need to increase it. Returns ------- None

Response:
def find_expcoef(self, nsd_below=0., plot=False,
                     trimlim=None, autorange_kwargs={}):
        """
        Determines exponential decay coefficient for despike filter.

        Fits an exponential decay function to the washout phase of standards
        to determine the washout time of your laser cell. The exponential
        coefficient reported is `nsd_below` standard deviations below the
        fitted exponent, to ensure that no real data is removed.

        Total counts are used in fitting, rather than a specific analyte.

        Parameters
        ----------
        nsd_below : float
            The number of standard deviations to subtract from the fitted
            coefficient when calculating the filter exponent.
        plot : bool or str
            If True, creates a plot of the fit, if str the plot is to the
            location specified in str.
        trimlim : float
            A threshold limit used in determining the start of the
            exponential decay region of the washout. Defaults to half
            the increase in signal over background. If the data in
            the plot don't fall on an exponential decay line, change
            this number. Normally you'll need to increase it.

        Returns
        -------
        None
        """
        print('Calculating exponential decay coefficient\nfrom SRM washouts...')

        def findtrim(tr, lim=None):
            trr = np.roll(tr, -1)
            trr[-1] = 0
            if lim is None:
                lim = 0.5 * np.nanmax(tr - trr)
            ind = (tr - trr) >= lim
            return np.arange(len(ind))[ind ^ np.roll(ind, -1)][0]

        if not hasattr(self.stds[0], 'trnrng'):
            for s in self.stds:
                s.autorange(**autorange_kwargs, ploterrs=False)

        trans = []
        times = []
        for v in self.stds:
            for trnrng in v.trnrng[-1::-2]:
                tr = minmax_scale(v.data['total_counts'][(v.Time > trnrng[0]) & (v.Time < trnrng[1])])
                sm = np.apply_along_axis(np.nanmean, 1,
                                         rolling_window(tr, 3, pad=0))
                sm[0] = sm[1]
                trim = findtrim(sm, trimlim) + 2
                trans.append(minmax_scale(tr[trim:]))
                times.append(np.arange(tr[trim:].size) *
                             np.diff(v.Time[1:3]))

        times = np.concatenate(times)
        times = np.round(times, 2)
        trans = np.concatenate(trans)

        ti = []
        tr = []
        for t in np.unique(times):
            ti.append(t)
            tr.append(np.nanmin(trans[times == t]))

        def expfit(x, e):
            """
            Exponential decay function.
            """
            return np.exp(e * x)

        ep, ecov = curve_fit(expfit, ti, tr, p0=(-1.))

        eeR2 = R2calc(trans, expfit(times, ep))

        if plot:
            fig, ax = plt.subplots(1, 1, figsize=[6, 4])

            ax.scatter(times, trans, alpha=0.2, color='k', marker='x', zorder=-2)
            ax.scatter(ti, tr, alpha=1, color='k', marker='o')
            fitx = np.linspace(0, max(ti))
            ax.plot(fitx, expfit(fitx, ep), color='r', label='Fit')
            ax.plot(fitx, expfit(fitx, ep - nsd_below * np.diag(ecov)**.5, ),
                    color='b', label='Used')
            ax.text(0.95, 0.75,
                    ('y = $e^{%.2f \pm %.2f * x}$\n$R^2$= %.2f \nCoefficient: '
                     '%.2f') % (ep,
                                np.diag(ecov)**.5,
                                eeR2,
                                ep - nsd_below * np.diag(ecov)**.5),
                    transform=ax.transAxes, ha='right', va='top', size=12)
            ax.set_xlim(0, ax.get_xlim()[-1])
            ax.set_xlabel('Time (s)')
            ax.set_ylim(-0.05, 1.05)
            ax.set_ylabel('Proportion of Signal')
            plt.legend()
            if isinstance(plot, str):
                fig.savefig(plot)

        self.expdecay_coef = ep - nsd_below * np.diag(ecov)**.5

        print('  {:0.2f}'.format(self.expdecay_coef[0]))

        return


Instruction:
Can you generate the documentation for the following Python 3 function
def get_background(self, n_min=10, n_max=None, focus_stage='despiked', bkg_filter=False, f_win=5, f_n_lim=3):
        allbkgs = {'uTime': [],
                   'ns': []}

        if focus_stage == 'despiked':
            if 'despiked' not in self.stages_complete:
                focus_stage = 'rawdata'

        for a in self.analytes:
            allbkgs[a] = []

        n0 = 0
        for s in self.data.values():
            if sum(s.bkg) > 0:
                allbkgs['uTime'].append(s.uTime[s.bkg])
                allbkgs['ns'].append(enumerate_bool(s.bkg, n0)[s.bkg])
                n0 = allbkgs['ns'][-1][-1]
                for a in self.analytes:
                    allbkgs[a].append(s.data[focus_stage][a][s.bkg])

        allbkgs.update((k, np.concatenate(v)) for k, v in allbkgs.items())
        bkgs = pd.DataFrame(allbkgs)  # using pandas here because it's much more efficient than loops.

        self.bkg = Bunch()
        # extract background data from whole dataset
        if n_max is None:
            self.bkg['raw'] = bkgs.groupby('ns').filter(lambda x: len(x) > n_min)
        else:
            self.bkg['raw'] = bkgs.groupby('ns').filter(lambda x: (len(x) > n_min) & (len(x) < n_max))
        # calculate per - background region stats
        self.bkg['summary'] = self.bkg['raw'].groupby('ns').aggregate([np.mean, np.std, stderr])
        # sort summary by uTime
        self.bkg['summary'].sort_values(('uTime', 'mean'), inplace=True)
        # self.bkg['summary'].index = np.arange(self.bkg['summary'].shape[0])
        # self.bkg['summary'].index.name = 'ns'

        if bkg_filter:
            # calculate rolling mean and std from summary
            t = self.bkg['summary'].loc[:, idx[:, 'mean']]
            r = t.rolling(f_win).aggregate([np.nanmean, np.nanstd])
            # calculate upper threshold
            upper = r.loc[:, idx[:, :, 'nanmean']] + f_n_lim * r.loc[:, idx[:, :, 'nanstd']].values
            # calculate which are over upper threshold
            over = r.loc[:, idx[:, :, 'nanmean']] > np.roll(upper.values, 1, 0)
            # identify them
            ns_drop = over.loc[over.apply(any, 1), :].index.values
            # drop them from summary
            self.bkg['summary'].drop(ns_drop, inplace=True)
            # remove them from raw
            ind = np.ones(self.bkg['raw'].shape[0], dtype=bool)
            for ns in ns_drop:
                ind = ind & (self.bkg['raw'].loc[:, 'ns'] != ns)
            self.bkg['raw'] = self.bkg['raw'].loc[ind, :]
        return

Response: Extract all background data from all samples on universal time scale.


Instruction:
Can you generate the documentation for the following Python 3 function
def bkg_calc_weightedmean(self, analytes=None, weight_fwhm=None,
                              n_min=20, n_max=None, cstep=None,
                              bkg_filter=False, f_win=7, f_n_lim=3, focus_stage='despiked'):
        """
        Background calculation using a gaussian weighted mean.

        Parameters
        ----------
        analytes : str or iterable
            Which analyte or analytes to calculate.
        weight_fwhm : float
            The full-width-at-half-maximum of the gaussian used
            to calculate the weighted average.
        n_min : int
            Background regions with fewer than n_min points
            will not be included in the fit.
        cstep : float or None
            The interval between calculated background points.
        filter : bool
            If true, apply a rolling filter to the isolated background regions
            to exclude regions with anomalously high values. If True, two parameters
            alter the filter's behaviour:
        f_win : int
            The size of the rolling window
        f_n_lim : float
            The number of standard deviations above the rolling mean
            to set the threshold.
        focus_stage : str
            Which stage of analysis to apply processing to.
            Defaults to 'despiked' if present, or 'rawdata' if not.
            Can be one of:
            * 'rawdata': raw data, loaded from csv file.
            * 'despiked': despiked data.
            * 'signal'/'background': isolated signal and background data.
              Created by self.separate, after signal and background
              regions have been identified by self.autorange.
            * 'bkgsub': background subtracted data, created by
              self.bkg_correct
            * 'ratios': element ratio data, created by self.ratio.
            * 'calibrated': ratio data calibrated to standards, created by self.calibrate.
        """
        if analytes is None:
            analytes = self.analytes
            self.bkg = Bunch()
        elif isinstance(analytes, str):
            analytes = [analytes]

        if weight_fwhm is None:
            weight_fwhm = 600  # 10 minute default window

        self.get_background(n_min=n_min, n_max=n_max,
                            bkg_filter=bkg_filter,
                            f_win=f_win, f_n_lim=f_n_lim, focus_stage=focus_stage)

        # Gaussian - weighted average
        if 'calc' not in self.bkg.keys():
            # create time points to calculate background
            if cstep is None:
                cstep = weight_fwhm / 20
            elif cstep > weight_fwhm:
                warnings.warn("\ncstep should be less than weight_fwhm. Your backgrounds\n" +
                              "might not behave as expected.\n")
            bkg_t = np.linspace(0,
                                self.max_time,
                                self.max_time // cstep)
            self.bkg['calc'] = Bunch()
            self.bkg['calc']['uTime'] = bkg_t

        # TODO : calculation then dict assignment is clumsy...
        mean, std, stderr = gauss_weighted_stats(self.bkg['raw'].uTime,
                                                 self.bkg['raw'].loc[:, analytes].values,
                                                 self.bkg['calc']['uTime'],
                                                 fwhm=weight_fwhm)

        for i, a in enumerate(analytes):
            self.bkg['calc'][a] = {'mean': mean[i],
                                    'std': std[i],
                                    'stderr': stderr[i]}

Response: This function calculates the weighted mean of the background regions.


Instruction:
Can you write a function in Python 3 where it
backgrounds calculation using a 1D interpolation. scipy.interpolate.interp1D is used for interpolation. Parameters ---------- analytes : str or iterable Which analyte or analytes to calculate. kind : str or int Integer specifying the order of the spline interpolation used, or string specifying a type of interpolation. Passed to `scipy.interpolate.interp1D` n_min : int Background regions with fewer than n_min points will not be included in the fit. cstep : float or None The interval between calculated background points. filter : bool If true, apply a rolling filter to the isolated background regions to exclude regions with anomalously high values. If True, two parameters alter the filter's behaviour: f_win : int The size of the rolling window f_n_lim : float The number of standard deviations above the rolling mean to set the threshold. focus_stage : str Which stage of analysis to apply processing to. Defaults to 'despiked' if present, or 'rawdata' if not. Can be one of: * 'rawdata': raw data, loaded from csv file. * 'despiked': despiked data. * 'signal'/'background': isolated signal and background data. Created by self.separate, after signal and background regions have been identified by self.autorange. * 'bkgsub': background subtracted data, created by self.bkg_correct * 'ratios': element ratio data, created by self.ratio. * 'calibrated': ratio data calibrated to standards, created by self.calibrate.

Response:
def bkg_calc_interp1d(self, analytes=None, kind=1, n_min=10, n_max=None, cstep=None,
                          bkg_filter=False, f_win=7, f_n_lim=3, focus_stage='despiked'):
        """
        Background calculation using a 1D interpolation.

        scipy.interpolate.interp1D is used for interpolation.

        Parameters
        ----------
        analytes : str or iterable
            Which analyte or analytes to calculate.
        kind : str or int
            Integer specifying the order of the spline interpolation
            used, or string specifying a type of interpolation.
            Passed to `scipy.interpolate.interp1D`
        n_min : int
            Background regions with fewer than n_min points
            will not be included in the fit.
        cstep : float or None
            The interval between calculated background points.
        filter : bool
            If true, apply a rolling filter to the isolated background regions
            to exclude regions with anomalously high values. If True, two parameters
            alter the filter's behaviour:
        f_win : int
            The size of the rolling window
        f_n_lim : float
            The number of standard deviations above the rolling mean
            to set the threshold.
        focus_stage : str
            Which stage of analysis to apply processing to.
            Defaults to 'despiked' if present, or 'rawdata' if not.
            Can be one of:
            * 'rawdata': raw data, loaded from csv file.
            * 'despiked': despiked data.
            * 'signal'/'background': isolated signal and background data.
              Created by self.separate, after signal and background
              regions have been identified by self.autorange.
            * 'bkgsub': background subtracted data, created by
              self.bkg_correct
            * 'ratios': element ratio data, created by self.ratio.
            * 'calibrated': ratio data calibrated to standards, created by self.calibrate.
        """
        if analytes is None:
            analytes = self.analytes
            self.bkg = Bunch()
        elif isinstance(analytes, str):
            analytes = [analytes]

        self.get_background(n_min=n_min, n_max=n_max,
                            bkg_filter=bkg_filter,
                            f_win=f_win, f_n_lim=f_n_lim, focus_stage=focus_stage)

        def pad(a, lo=None, hi=None):
            if lo is None:
                lo = [a[0]]
            if hi is None:
                hi = [a[-1]]
            return np.concatenate((lo, a, hi))

        if 'calc' not in self.bkg.keys():
            # create time points to calculate background
            # if cstep is None:
                # cstep = self.bkg['raw']['uTime'].ptp() / 100
            # bkg_t = np.arange(self.bkg['summary']['uTime']['mean'].min(),
            #                   self.bkg['summary']['uTime']['mean'].max(),
            #                   cstep)

            bkg_t = pad(self.bkg['summary'].loc[:, ('uTime', 'mean')], [0], [self.max_time])

            self.bkg['calc'] = Bunch()
            self.bkg['calc']['uTime'] = bkg_t

        d = self.bkg['summary']
        with self.pbar.set(total=len(analytes), desc='Calculating Analyte Backgrounds') as prog:
            for a in analytes:
                self.bkg['calc'][a] = {'mean': pad(d.loc[:, (a, 'mean')].values),
                                    'std': pad(d.loc[:, (a, 'std')].values),
                                    'stderr': pad(d.loc[:, (a, 'stderr')].values)}
                prog.update()

        self.bkg['calc']

        return


Instruction:
Explain what the following Python 3 code does
def bkg_subtract(self, analytes=None, errtype='stderr', focus_stage='despiked'):
        if analytes is None:
            analytes = self.analytes
        elif isinstance(analytes, str):
            analytes = [analytes]

        if focus_stage == 'despiked':
            if 'despiked' not in self.stages_complete:
                focus_stage = 'rawdata'

        # make uncertainty-aware background interpolators
        bkg_interps = {}
        for a in analytes:
            bkg_interps[a] = un_interp1d(x=self.bkg['calc']['uTime'],
                                         y=un.uarray(self.bkg['calc'][a]['mean'],
                                                     self.bkg['calc'][a][errtype]))
        self.bkg_interps = bkg_interps

        # apply background corrections
        with self.pbar.set(total=len(self.data), desc='Background Subtraction') as prog:
            for d in self.data.values():
                # [d.bkg_subtract(a, bkg_interps[a].new(d.uTime), None, focus_stage=focus_stage) for a in analytes]
                [d.bkg_subtract(a, bkg_interps[a].new(d.uTime), ~d.sig, focus_stage=focus_stage) for a in analytes]
                d.setfocus('bkgsub')

                prog.update()

        self.stages_complete.update(['bkgsub'])
        self.focus_stage = 'bkgsub'
        return

Response: Subtracts background from data.


Instruction:
Implement a function in Python 3 to
correct the spectral interference of two sets of peaks.

Response:
def correct_spectral_interference(self, target_analyte, source_analyte, f):
        """
        Correct spectral interference.

        Subtract interference counts from target_analyte, based on the
        intensity of a source_analayte and a known fractional contribution (f).

        Correction takes the form:
        target_analyte -= source_analyte * f

        Only operates on background-corrected data ('bkgsub'). To undo a correction,
        rerun `self.bkg_subtract()`.

        Example
        -------
        To correct 44Ca+ for an 88Sr++ interference, where both 43.5 and 44 Da
        peaks are known:
        f = abundance(88Sr) / (abundance(87Sr)

        counts(44Ca) = counts(44 Da) - counts(43.5 Da) * f

        Parameters
        ----------
        target_analyte : str
            The name of the analyte to modify.
        source_analyte : str
            The name of the analyte to base the correction on.
        f : float
            The fraction of the intensity of the source_analyte to
            subtract from the target_analyte. Correction is:
            target_analyte - source_analyte * f

        Returns
        -------
        None
        """

        if target_analyte not in self.analytes:
            raise ValueError('target_analyte: {:} not in available analytes ({:})'.format(target_analyte, ', '.join(self.analytes)))

        if source_analyte not in self.analytes:
            raise ValueError('source_analyte: {:} not in available analytes ({:})'.format(source_analyte, ', '.join(self.analytes)))

        with self.pbar.set(total=len(self.data), desc='Interference Correction') as prog:
            for d in self.data.values():
                d.correct_spectral_interference(target_analyte, source_analyte, f)

                prog.update()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def bkg_plot(self, analytes=None, figsize=None, yscale='log',
                 ylim=None, err='stderr', save=True):
        """
        Plot the calculated background.

        Parameters
        ----------
        analytes : str or iterable
            Which analyte(s) to plot.
        figsize : tuple
            The (width, height) of the figure, in inches.
            If None, calculated based on number of samples.
        yscale : str
            'log' (default) or 'linear'.
        ylim : tuple
            Manually specify the y scale.
        err : str
            What type of error to plot. Default is stderr.
        save : bool
            If True, figure is saved.

        Returns
        -------
        fig, ax : matplotlib.figure, matplotlib.axes
        """
        if not hasattr(self, 'bkg'):
            raise ValueError("\nPlease calculate a background before attempting to\n" +
                             "plot it... either:\n" +
                             "   bkg_calc_interp1d\n" +
                             "   bkg_calc_weightedmean\n")

        if analytes is None:
            analytes = self.analytes
        elif isinstance(analytes, str):
            analytes = [analytes]

        if figsize is None:
            if len(self.samples) > 50:
                figsize = (len(self.samples) * 0.15, 5)
            else:
                figsize = (7.5, 5)

        fig = plt.figure(figsize=figsize)

        ax = fig.add_axes([.07, .1, .84, .8])

        with self.pbar.set(total=len(analytes), desc='Plotting backgrounds') as prog:
            for a in analytes:
                # draw data points
                ax.scatter(self.bkg['raw'].uTime, self.bkg['raw'].loc[:, a],
                        alpha=0.5, s=3, c=self.cmaps[a],
                        lw=0.5)

                # draw STD boxes
                for i, r in self.bkg['summary'].iterrows():
                    x = (r.loc['uTime', 'mean'] - r.loc['uTime', 'std'] * 2,
                        r.loc['uTime', 'mean'] + r.loc['uTime', 'std'] * 2)
                    yl = [r.loc[a, 'mean'] - r.loc[a, err]] * 2
                    yu = [r.loc[a, 'mean'] + r.loc[a, err]] * 2

                    ax.fill_between(x, yl, yu, alpha=0.8, lw=0.5, color=self.cmaps[a], zorder=1)
                prog.update()

        if yscale == 'log':
            ax.set_yscale('log')
        if ylim is not None:
            ax.set_ylim(ylim)
        else:
            ax.set_ylim(ax.get_ylim() * np.array([1, 10]))  # x10 to make sample names readable.

        for a in analytes:
            # draw confidence intervals of calculated
            x = self.bkg['calc']['uTime']
            y = self.bkg['calc'][a]['mean']
            yl = self.bkg['calc'][a]['mean'] - self.bkg['calc'][a][err]
            yu = self.bkg['calc'][a]['mean'] + self.bkg['calc'][a][err]

            # trim values below zero if log scale=
            if yscale == 'log':
                yl[yl < ax.get_ylim()[0]] = ax.get_ylim()[0]

            ax.plot(x, y,
                    c=self.cmaps[a], zorder=2, label=a)
            ax.fill_between(x, yl, yu,
                            color=self.cmaps[a], alpha=0.3, zorder=-1)

        ax.set_xlabel('Time (s)')
        ax.set_ylabel('Background Counts')

        ax.set_title('Points = raw data; Bars = {:s}; Lines = Calculated Background; Envelope = Background {:s}'.format(err, err),
                     fontsize=10)

        ha, la = ax.get_legend_handles_labels()

        ax.legend(labels=la[:len(analytes)], handles=ha[:len(analytes)], bbox_to_anchor=(1, 1))

        # scale x axis to range ± 2.5%
        xlim = rangecalc(self.bkg['raw']['uTime'], 0.025)
        ax.set_xlim(xlim)

        # add sample labels
        for s, d in self.data.items():
            ax.axvline(d.uTime[0], alpha=0.2, color='k', zorder=-1)
            ax.text(d.uTime[0], ax.get_ylim()[1], s, rotation=90,
                    va='top', ha='left', zorder=-1, fontsize=7)

        if save:
            fig.savefig(self.report_dir + '/background.png', dpi=200)

        return fig, ax

Response: Plots the calculated background for the specified analytes.


Instruction:
Make a summary of the following Python 3 code
def ratio(self, internal_standard=None):
        if 'bkgsub' not in self.stages_complete:
            raise RuntimeError('Cannot calculate ratios before background subtraction.')

        if internal_standard is not None:
            self.internal_standard = internal_standard
            self.minimal_analytes.update([internal_standard])

        with self.pbar.set(total=len(self.data), desc='Ratio Calculation') as prog:
            for s in self.data.values():
                s.ratio(internal_standard=self.internal_standard)
                prog.update()

        self.stages_complete.update(['ratios'])
        self.focus_stage = 'ratios'
        return

Response: Calculates the ratio of all analytes to a single analyte.


Instruction:
Can you tell what is the following Python 3 function doing
def srm_id_auto(self, srms_used=['NIST610', 'NIST612', 'NIST614'], n_min=10, reload_srm_database=False):
        if isinstance(srms_used, str):
            srms_used = [srms_used]

        # get mean and standard deviations of measured standards
        self.srm_compile_measured(n_min)
        stdtab = self.stdtab.copy()

        # load corresponding SRM database
        self.srm_load_database(srms_used, reload_srm_database)

        # create blank srm table
        srm_tab = self.srmdat.loc[:, ['mol_ratio', 'element']].reset_index().pivot(index='SRM', columns='element', values='mol_ratio')

        # Auto - ID STDs
        # 1. identify elements in measured SRMS with biggest range of values
        meas_tab = stdtab.loc[:, (slice(None), 'mean')]  # isolate means of standards
        meas_tab.columns = meas_tab.columns.droplevel(1)  # drop 'mean' column names
        meas_tab.columns = [re.findall('[A-Za-z]+', a)[0] for a in meas_tab.columns]  # rename to element names
        meas_tab = meas_tab.T.groupby(level=0).first().T  # remove duplicate columns

        ranges = nominal_values(meas_tab.apply(lambda a: np.ptp(a) / np.nanmean(a), 0))  # calculate relative ranges of all elements
        # (used as weights later)

        # 2. Work out which standard is which
        # normalise all elements between 0-1
        def normalise(a):
            a = nominal_values(a)
            if np.nanmin(a) < np.nanmax(a):
                return (a - np.nanmin(a)) / np.nanmax(a - np.nanmin(a))
            else:
                return np.ones(a.shape)

        nmeas = meas_tab.apply(normalise, 0)
        nmeas.dropna(1, inplace=True)  # remove elements with NaN values
        # nmeas.replace(np.nan, 1, inplace=True)
        nsrm_tab = srm_tab.apply(normalise, 0)
        nsrm_tab.dropna(1, inplace=True)
        # nsrm_tab.replace(np.nan, 1, inplace=True)

        for uT, r in nmeas.iterrows():  # for each standard...
            idx = np.nansum(((nsrm_tab - r) * ranges)**2, 1)
            idx = abs((nsrm_tab - r) * ranges).sum(1)
            # calculate the absolute difference between the normalised elemental
            # values for each measured SRM and the SRM table. Each element is
            # multiplied by the relative range seen in that element (i.e. range / mean
            # measuerd value), so that elements with a large difference are given
            # more importance in identifying the SRM.
            # This produces a table, where wach row contains the difference between
            # a known vs. measured SRM. The measured SRM is identified as the SRM that
            # has the smallest weighted sum value.
            stdtab.loc[uT, 'SRM'] = srm_tab.index[idx == min(idx)].values[0]

        # calculate mean time for each SRM
        # reset index and sort
        stdtab.reset_index(inplace=True)
        stdtab.sort_index(1, inplace=True)
        # isolate STD and uTime
        uT = stdtab.loc[:, ['gTime', 'STD']].set_index('STD')
        uT.sort_index(inplace=True)
        uTm = uT.groupby(level=0).mean()  # mean uTime for each SRM
        # replace uTime values with means
        stdtab.set_index(['STD'], inplace=True)
        stdtab.loc[:, 'gTime'] = uTm
        # reset index
        stdtab.reset_index(inplace=True)
        stdtab.set_index(['STD', 'SRM', 'gTime'], inplace=True)

        # combine to make SRM reference tables
        srmtabs = Bunch()
        for a in self.analytes:
            el = re.findall('[A-Za-z]+', a)[0]

            sub = stdtab.loc[:, a]

            srmsub = self.srmdat.loc[self.srmdat.element == el, ['mol_ratio', 'mol_ratio_err']]

            srmtab = sub.join(srmsub)
            srmtab.columns = ['meas_err', 'meas_mean', 'srm_mean', 'srm_err']

            srmtabs[a] = srmtab

        self.srmtabs = pd.concat(srmtabs).apply(nominal_values).sort_index()
        self.srmtabs.dropna(subset=['srm_mean'], inplace=True)
        # replace any nan error values with zeros - nans cause problems later.
        self.srmtabs.loc[:, ['meas_err', 'srm_err']] = self.srmtabs.loc[:, ['meas_err', 'srm_err']].replace(np.nan, 0)

        # remove internal standard from calibration elements
        self.srmtabs.drop(self.internal_standard, inplace=True)

        self.srms_ided = True
        return

Response: Function for automarically identifying the SRMs in the current SRM table.


Instruction:
Here you have a function in Python 3, explain what it does
def calibrate(self, analytes=None, drift_correct=True,
                  srms_used=['NIST610', 'NIST612', 'NIST614'],
                  zero_intercept=True, n_min=10, reload_srm_database=False):
        """
        Calibrates the data to measured SRM values.

        Assumes that y intercept is zero.

        Parameters
        ----------
        analytes : str or iterable
            Which analytes you'd like to calibrate. Defaults to all.
        drift_correct : bool
            Whether to pool all SRM measurements into a single calibration,
            or vary the calibration through the run, interpolating
            coefficients between measured SRMs.
        srms_used : str or iterable
            Which SRMs have been measured. Must match names given in
            SRM data file *exactly*.
        n_min : int
            The minimum number of data points an SRM measurement
            must have to be included.

        Returns
        -------
        None
        """
        if analytes is None:
            analytes = self.analytes[self.analytes != self.internal_standard]
        elif isinstance(analytes, str):
            analytes = [analytes]

        if not hasattr(self, 'srmtabs'):
            self.srm_id_auto(srms_used=srms_used, n_min=n_min, reload_srm_database=reload_srm_database)

        # make container for calibration params
        if not hasattr(self, 'calib_params'):
            gTime = self.stdtab.gTime.unique()
            self.calib_params = pd.DataFrame(columns=pd.MultiIndex.from_product([analytes, ['m']]),
                                            index=gTime)

        calib_analytes = self.srmtabs.index.get_level_values(0).unique()

        if zero_intercept:
            fn  = lambda x, m: x * m
        else:
            fn = lambda x, m, c: x * m + c

        for a in calib_analytes:
            if zero_intercept:
                if (a, 'c') in self.calib_params:
                    self.calib_params.drop((a, 'c'), 1, inplace=True)
            if drift_correct:
                for g in self.stdtab.gTime.unique():
                    ind = idx[a, :, :, g]
                    if self.srmtabs.loc[ind].size == 0:
                        continue
                    # try:
                    meas = self.srmtabs.loc[ind, 'meas_mean']
                    srm = self.srmtabs.loc[ind, 'srm_mean']
                    # TODO: replace curve_fit with Sambridge's 2D likelihood function for better uncertainty incorporation.
                    merr = self.srmtabs.loc[ind, 'meas_err']
                    serr = self.srmtabs.loc[ind, 'srm_err']
                    sigma = np.sqrt(merr**2 + serr**2)

                    if len(meas) > 1:
                        # multiple SRMs - do a regression
                        p, cov = curve_fit(fn, meas, srm, sigma=sigma)
                        pe = unc.correlated_values(p, cov)
                        self.calib_params.loc[g, (a, 'm')] = pe[0]
                        if not zero_intercept:
                            self.calib_params.loc[g, (a, 'c')] = pe[1]
                    else:
                        # deal with case where there's only one datum
                        self.calib_params.loc[g, (a, 'm')] = (un.uarray(srm, serr) /
                                                              un.uarray(meas, merr))[0]
                        if not zero_intercept:
                            self.calib_params.loc[g, (a, 'c')] = 0

                    # This should be obsolete, because no-longer sourcing locator from calib_params index.
                    # except KeyError:
                    #     # If the calibration is being recalculated, calib_params
                    #     # will have t=0 and t=max(uTime) values that are outside
                    #     # the srmtabs index.
                    #     # If this happens, drop them, and re-fill them at the end.
                    #     self.calib_params.drop(g, inplace=True)
            else:
                ind = idx[a, :, :, :]
                meas = self.srmtabs.loc[ind, 'meas_mean']
                srm = self.srmtabs.loc[ind, 'srm_mean']
                merr = self.srmtabs.loc[ind, 'meas_err']
                serr = self.srmtabs.loc[ind, 'srm_err']
                sigma = np.sqrt(merr**2 + serr**2)

                if len(meas) > 1:
                    p, cov = curve_fit(fn, meas, srm, sigma=sigma)
                    pe = unc.correlated_values(p, cov)
                    self.calib_params.loc[:, (a, 'm')] = pe[0]
                    if not zero_intercept:
                        self.calib_params.loc[:, (a, 'c')] = pe[1]
                else:
                    self.calib_params.loc[:, (a, 'm')] = (un.uarray(srm, serr) /
                                                          un.uarray(meas, merr))[0]
                    if not zero_intercept:
                        self.calib_params.loc[:, (a, 'c')] = 0

        # if fill:
        # fill in uTime=0 and uTime = max cases for interpolation
        if self.calib_params.index.min() == 0:
            self.calib_params.drop(0, inplace=True)
            self.calib_params.drop(self.calib_params.index.max(), inplace=True)
        self.calib_params.loc[0, :] = self.calib_params.loc[self.calib_params.index.min(), :]
        maxuT = np.max([d.uTime.max() for d in self.data.values()])  # calculate max uTime
        self.calib_params.loc[maxuT, :] = self.calib_params.loc[self.calib_params.index.max(), :]
        # sort indices for slice access
        self.calib_params.sort_index(1, inplace=True)
        self.calib_params.sort_index(0, inplace=True)

        # calculcate interpolators for applying calibrations
        self.calib_ps = Bunch()
        for a in analytes:
            # TODO: revisit un_interp1d to see whether it plays well with correlated values.
            # Possible re-write to deal with covariance matrices?
            self.calib_ps[a] = {'m': un_interp1d(self.calib_params.index.values,
                                                self.calib_params.loc[:, (a, 'm')].values)}
            if not zero_intercept:
                self.calib_ps[a]['c'] = un_interp1d(self.calib_params.index.values,
                                                    self.calib_params.loc[:, (a, 'c')].values)

        with self.pbar.set(total=len(self.data), desc='Applying Calibrations') as prog:
            for d in self.data.values():
                d.calibrate(self.calib_ps, analytes)
                prog.update()

        # record SRMs used for plotting
        markers = 'osDsv<>PX'  # for future implementation of SRM-specific markers.
        if not hasattr(self, 'srms_used'):
            self.srms_used = set(srms_used)
        else:
            self.srms_used.update(srms_used)
        self.srm_mdict = {k: markers[i] for i, k in enumerate(self.srms_used)}

        self.stages_complete.update(['calibrated'])
        self.focus_stage = 'calibrated'

        return

Response: Calibrates the data to measured SRM values.


Instruction:
Make a summary of the following Python 3 code
def make_subset(self, samples=None, name=None):
        # Check if a subset containing the same samples already exists.
        for k, v in self.subsets.items():
            if set(v) == set(samples) and k != 'not_in_set':
                return k

        if isinstance(samples, str):
            samples = [samples]

        not_exists = [s for s in samples if s not in self.subsets['All_Analyses']]
        if len(not_exists) > 0:
            raise ValueError(', '.join(not_exists) + ' not in the list of sample names.\nPlease check your sample names.\nNote: Sample names are stored in the .samples attribute of your analysis.')

        if name is None:
            name = max([-1] + [x for x in self.subsets.keys() if isinstance(x, int)]) + 1

        self._subset_names.append(name)

        if samples is not None:
            self.subsets[name] = samples
            for s in samples:
                try:
                    self.subsets['not_in_set'].remove(s)
                except ValueError:
                    pass

        self._has_subsets = True

        # for subset in np.unique(list(self.subsets.values())):
        #     self.subsets[subset] = sorted([k for k, v in self.subsets.items() if str(v) == subset])

        return name

Response: Creates a subset of samples which can be treated independently.


Instruction:
Make a summary of the following Python 3 code
def zeroscreen(self, focus_stage=None):
        if focus_stage is None:
            focus_stage = self.focus_stage

        for s in self.data.values():
            ind = np.ones(len(s.Time), dtype=bool)
            for v in s.data[focus_stage].values():
                ind = ind & (nominal_values(v) > 0)

            for k in s.data[focus_stage].keys():
                s.data[focus_stage][k][~ind] = unc.ufloat(np.nan, np.nan)

        self.set_focus(focus_stage)

        return

Response: Remove all points containing data below zero which are impossible!


Instruction:
Write a Python 3 script to
apply a threshold filter to the data.

Response:
def filter_threshold(self, analyte, threshold,
                         samples=None, subset=None):
        """
        Applies a threshold filter to the data.

        Generates two filters above and below the threshold value for a
        given analyte.

        Parameters
        ----------
        analyte : str
            The analyte that the filter applies to.
        threshold : float
            The threshold value.
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.
        samples : array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.
        subset : str or number
            The subset of samples (defined by make_subset) you want to apply
            the filter to.

        Returns
        -------
        None
        """
        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        self.minimal_analytes.update([analyte])

        with self.pbar.set(total=len(samples), desc='Threshold Filter') as prog:
            for s in samples:
                self.data[s].filter_threshold(analyte, threshold)
                prog.update()


Instruction:
Can you create a Python 3 function that
applies a threshold filter to the data.

Response:
def filter_threshold_percentile(self, analyte, percentiles, level='population', filt=False,
                                    samples=None, subset=None):
        """
        Applies a threshold filter to the data.

        Generates two filters above and below the threshold value for a
        given analyte.

        Parameters
        ----------
        analyte : str
            The analyte that the filter applies to.
        percentiles : float or iterable of len=2
            The percentile values.
        level : str
            Whether to calculate percentiles from the entire dataset
            ('population') or for each individual sample ('individual')
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.
        samples : array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.
        subset : str or number
            The subset of samples (defined by make_subset) you want to apply
            the filter to.

        Returns
        -------
        None
        """
        params = locals()
        del(params['self'])

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        self.minimal_analytes.update([analyte])

        if isinstance(percentiles, (int, float)):
            percentiles = [percentiles]

        if level == 'population':
            # Get all samples
            self.get_focus(filt=filt, subset=subset, nominal=True)
            dat = self.focus[analyte][~np.isnan(self.focus[analyte])]

            # calculate filter limits
            lims = np.percentile(dat, percentiles)

        # Calculate filter for individual samples
        with self.pbar.set(total=len(samples), desc='Percentile theshold filter') as prog:
            for s in samples:
                d = self.data[s]
                setn = d.filt.maxset + 1
                g = d.focus[analyte]

                if level == 'individual':
                    gt = nominal_values(g)
                    lims = np.percentile(gt[~np.isnan(gt)], percentiles)

                if len(lims) == 1:
                    above = g >= lims[0]
                    below = g < lims[0]

                    d.filt.add(analyte + '_{:.1f}-pcnt_below'.format(percentiles[0]),
                            below,
                            'Values below {:.1f}th {:} percentile ({:.2e})'.format(percentiles[0], analyte, lims[0]),
                            params, setn=setn)
                    d.filt.add(analyte + '_{:.1f}-pcnt_above'.format(percentiles[0]),
                            above,
                            'Values above {:.1f}th {:} percentile ({:.2e})'.format(percentiles[0], analyte, lims[0]),
                            params, setn=setn)

                elif len(lims) == 2:
                    inside = (g >= min(lims)) & (g <= max(lims))
                    outside = (g < min(lims)) | (g > max(lims))

                    lpc = '-'.join(['{:.1f}'.format(p) for p in percentiles])
                    d.filt.add(analyte + '_' + lpc + '-pcnt_inside',
                            inside,
                            'Values between ' + lpc + ' ' + analyte + 'percentiles',
                            params, setn=setn)
                    d.filt.add(analyte + '_' + lpc + '-pcnt_outside',
                            outside,
                            'Values outside ' + lpc + ' ' + analyte + 'percentiles',
                            params, setn=setn)
                prog.update()
        return


Instruction:
Write a Python 3 script for
calculating a gradient threshold filter for the specified analyte and percentile values.

Response:
def filter_gradient_threshold_percentile(self, analyte, percentiles, level='population', win=15, filt=False,
                                             samples=None, subset=None):
        """
        Calculate a gradient threshold filter to the data.

        Generates two filters above and below the threshold value for a
        given analyte.

        Parameters
        ----------
        analyte : str
            The analyte that the filter applies to.
        win : int
            The window over which to calculate the moving gradient
        percentiles : float or iterable of len=2
            The percentile values.
        filt : bool
            Whether or not to apply existing filters to the data before
            calculating this filter.
        samples : array_like or None
            Which samples to apply this filter to. If None, applies to all
            samples.
        subset : str or number
            The subset of samples (defined by make_subset) you want to apply
            the filter to.

        Returns
        -------
        None
        """
        params = locals()
        del(params['self'])

        if samples is not None:
            subset = self.make_subset(samples)

        samples = self._get_samples(subset)

        self.minimal_analytes.update([analyte])

        # Calculate gradients of all samples
        self.get_gradients(analytes=[analyte], win=win, filt=filt, subset=subset)
        grad = self.gradients[analyte][~np.isnan(self.gradients[analyte])]

        if isinstance(percentiles, (int, float)):
            percentiles = [percentiles]

        if level == 'population':
            # calculate filter limits
            lims = np.percentile(grad, percentiles)

        # Calculate filter for individual samples
        with self.pbar.set(total=len(samples), desc='Percentile Threshold Filter') as prog:
            for s in samples:
                d = self.data[s]
                setn = d.filt.maxset + 1
                g = calc_grads(d.Time, d.focus, [analyte], win)[analyte]

                if level == 'individual':
                    gt = nominal_values(g)
                    lims = np.percentile(gt[~np.isnan(gt)], percentiles)

                if len(lims) == 1:
                    above = g >= lims[0]
                    below = g < lims[0]

                    d.filt.add(analyte + '_{:.1f}-grd-pcnt_below'.format(percentiles[0]),
                            below,
                            'Gradients below {:.1f}th {:} percentile ({:.2e})'.format(percentiles[0], analyte, lims[0]),
                            params, setn=setn)
                    d.filt.add(analyte + '_{:.1f}-grd-pcnt_above'.format(percentiles[0]),
                            above,
                            'Gradients above {:.1f}th {:} percentile ({:.2e})'.format(percentiles[0], analyte, lims[0]),
                            params, setn=setn)

                elif len(lims) == 2:
                    inside = (g >= min(lims)) & (g <= max(lims))
                    outside = (g < min(lims)) | (g > max(lims))

                    lpc = '-'.join(['{:.1f}'.format(p) for p in percentiles])
                    d.filt.add(analyte + '_' + lpc + '-grd-pcnt_inside',
                            inside,
                            'Gradients between ' + lpc + ' ' + analyte + 'percentiles',
                            params, setn=setn)
                    d.filt.add(analyte + '_' + lpc + '-grd-pcnt_outside',
                            outside,
                            'Gradients outside ' + lpc + ' ' + analyte + 'percentiles',
                            params, setn=setn)
                prog.update()
        return