Instruction:
Given the following Python 3 function, write the documentation
def generative_model(s,x,y,z,r, factor=1.1):
    pl.close('all')

    slicez = int(round(z.mean()))
    slicex = s.image.shape[2]//2
    slicer1 = np.s_[slicez,s.pad:-s.pad,s.pad:-s.pad]
    slicer2 = np.s_[s.pad:-s.pad,s.pad:-s.pad,slicex]
    center = (slicez, s.image.shape[1]//2, slicex)

    fig = pl.figure(figsize=(factor*13,factor*10))

    #=========================================================================
    #=========================================================================
    gs1 = ImageGrid(fig, rect=[0.0, 0.6, 1.0, 0.35], nrows_ncols=(1,3),
            axes_pad=0.1)
    ax_real = gs1[0]
    ax_fake = gs1[1]
    ax_diff = gs1[2]

    diff = s.get_model_image() - s.image
    ax_real.imshow(s.image[slicer1], cmap=pl.cm.bone_r)
    ax_real.set_xticks([])
    ax_real.set_yticks([])
    ax_real.set_title("Confocal image", fontsize=24)
    ax_fake.imshow(s.get_model_image()[slicer1], cmap=pl.cm.bone_r)
    ax_fake.set_xticks([])
    ax_fake.set_yticks([])
    ax_fake.set_title("Model image", fontsize=24)
    ax_diff.imshow(diff[slicer1], cmap=pl.cm.RdBu, vmin=-0.1, vmax=0.1)
    ax_diff.set_xticks([])
    ax_diff.set_yticks([])
    ax_diff.set_title("Difference", fontsize=24)

    #=========================================================================
    #=========================================================================
    gs2 = ImageGrid(fig, rect=[0.1, 0.0, 0.4, 0.55], nrows_ncols=(3,2),
            axes_pad=0.1)
    ax_plt1 = fig.add_subplot(gs2[0])
    ax_plt2 = fig.add_subplot(gs2[1])
    ax_ilm1 = fig.add_subplot(gs2[2])
    ax_ilm2 = fig.add_subplot(gs2[3])
    ax_psf1 = fig.add_subplot(gs2[4])
    ax_psf2 = fig.add_subplot(gs2[5])

    c = int(z.mean()), int(y.mean())+s.pad, int(x.mean())+s.pad
    if s.image.shape[0] > 2*s.image.shape[1]//3:
        w = s.image.shape[2] - 2*s.pad
        h = 2*w//3
    else:
        h = s.image.shape[0] - 2*s.pad
        w = 3*h//2

    w,h = w//2, h//2
    xyslice = np.s_[slicez, c[1]-h:c[1]+h, c[2]-w:c[2]+w]
    yzslice = np.s_[c[0]-h:c[0]+h, c[1]-w:c[1]+w, slicex]

    #h = s.image.shape[2]/2 - s.image.shape[0]/2
    #slicer2 = np.s_[s.pad:-s.pad, s.pad:-s.pad, slicex]
    #slicer3 = np.s_[slicez, s.pad+h:-s.pad-h, s.pad:-s.pad]

    ax_plt1.imshow(1-s.obj.get_field()[xyslice], cmap=pl.cm.bone_r, vmin=0, vmax=1)
    ax_plt1.set_xticks([])
    ax_plt1.set_yticks([])
    ax_plt1.set_ylabel("Platonic", fontsize=22)
    ax_plt1.set_title("x-y", fontsize=24)
    ax_plt2.imshow(1-s._platonic_image()[yzslice], cmap=pl.cm.bone_r, vmin=0, vmax=1)
    ax_plt2.set_xticks([])
    ax_plt2.set_yticks([])
    ax_plt2.set_title("y-z", fontsize=24)

    ax_ilm1.imshow(s.ilm.get_field()[xyslice], cmap=pl.cm.bone_r)
    ax_ilm1.set_xticks([])
    ax_ilm1.set_yticks([])
    ax_ilm1.set_ylabel("ILM", fontsize=22)
    ax_ilm2.imshow(s.ilm.get_field()[yzslice], cmap=pl.cm.bone_r)
    ax_ilm2.set_xticks([])
    ax_ilm2.set_yticks([])

    t = s.ilm.get_field().copy()
    t *= 0
    t[c] = 1
    s.psf.set_tile(util.Tile(t.shape))
    psf = (s.psf.execute(t)+5e-5)**0.1

    ax_psf1.imshow(psf[xyslice], cmap=pl.cm.bone)
    ax_psf1.set_xticks([])
    ax_psf1.set_yticks([])
    ax_psf1.set_ylabel("PSF", fontsize=22)
    ax_psf2.imshow(psf[yzslice], cmap=pl.cm.bone)
    ax_psf2.set_xticks([])
    ax_psf2.set_yticks([])

    #=========================================================================
    #=========================================================================
    ax_zoom = fig.add_axes([0.48, 0.018, 0.45, 0.52])

    #s.model_to_true_image()
    im = s.image[slicer1]
    sh = np.array(im.shape)
    cx = x.mean()
    cy = y.mean()

    extent = [0,sh[0],0,sh[1]]
    ax_zoom.set_xticks([])
    ax_zoom.set_yticks([])
    ax_zoom.imshow(im, extent=extent, cmap=pl.cm.bone_r)
    ax_zoom.set_xlim(cx-12, cx+12)
    ax_zoom.set_ylim(cy-12, cy+12)
    ax_zoom.set_title("Sampled positions", fontsize=24)
    ax_zoom.hexbin(x,y, gridsize=32, mincnt=0, cmap=pl.cm.hot)

    zoom1 = zoomed_inset_axes(ax_zoom, 30, loc=3)
    zoom1.imshow(im, extent=extent, cmap=pl.cm.bone_r)
    zoom1.set_xlim(cx-1.0/6, cx+1.0/6)
    zoom1.set_ylim(cy-1.0/6, cy+1.0/6)
    zoom1.hexbin(x,y,gridsize=32, mincnt=5, cmap=pl.cm.hot)
    zoom1.set_xticks([])
    zoom1.set_yticks([])
    zoom1.hlines(cy-1.0/6 + 1.0/32, cx-1.0/6+5e-2, cx-1.0/6+5e-2+1e-1, lw=3)
    zoom1.text(cx-1.0/6 + 1.0/24, cy-1.0/6+5e-2, '0.1px')
    mark_inset(ax_zoom, zoom1, loc1=2, loc2=4, fc="none", ec="0.0")

Response: Generate a model for the state s.


Instruction:
Explain what the following Python 3 code does
def examine_unexplained_noise(state, bins=1000, xlim=(-10,10)):
    r = state.residuals
    q = np.fft.fftn(r)
    #Get the expected values of `sigma`:
    calc_sig = lambda x: np.sqrt(np.dot(x,x) / x.size)
    rh, xr = np.histogram(r.ravel() / calc_sig(r.ravel()), bins=bins,
            density=True)
    bigq = np.append(q.real.ravel(), q.imag.ravel())
    qh, xq = np.histogram(bigq / calc_sig(q.real.ravel()), bins=bins,
            density=True)
    xr = 0.5*(xr[1:] + xr[:-1])
    xq = 0.5*(xq[1:] + xq[:-1])

    gauss = lambda t : np.exp(-t*t*0.5) / np.sqrt(2*np.pi)

    plt.figure(figsize=[16,8])
    axes = []
    for a, (x, r, lbl) in enumerate([[xr, rh, 'Real'], [xq, qh, 'Fourier']]):
        ax = plt.subplot(1,2,a+1)
        ax.semilogy(x, r, label='Data')
        ax.plot(x, gauss(x), label='Gauss Fit', scalex=False, scaley=False)
        ax.set_xlabel('Residuals value $r/\sigma$')
        ax.set_ylabel('Probability $P(r/\sigma)$')
        ax.legend(loc='upper right')
        ax.set_title('{}-Space'.format(lbl))
        ax.set_xlim(xlim)
        axes.append(ax)
    return axes

Response: Examines a state s residuals in real and Fourier space with a Gaussian.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def compare_data_model_residuals(s, tile, data_vmin='calc', data_vmax='calc',
         res_vmin=-0.1, res_vmax=0.1, edgepts='calc', do_imshow=True,
         data_cmap=plt.cm.bone, res_cmap=plt.cm.RdBu):
    """
    Compare the data, model, and residuals of a state.

    Makes an image of any 2D slice of a state that compares the data,
    model, and residuals. The upper left portion of the image is the raw
    data, the central portion the model, and the lower right portion the
    image. Either plots the image using plt.imshow() or returns a
    np.ndarray of the image pixels for later use.

    Parameters
    ----------
        st : peri.ImageState object
            The state to plot.
        tile : peri.util.Tile object
            The slice of the image to plot. Can be any xy, xz, or yz
            projection, but it must return a valid 2D slice (the slice is
            squeezed internally).

        data_vmin : {Float, `calc`}, optional
            vmin for the imshow for the data and generative model (shared).
            Default is 'calc' = 0.5(data.min() + model.min())
        data_vmax : {Float, `calc`}, optional
            vmax for the imshow for the data and generative model (shared).
            Default is 'calc' = 0.5(data.max() + model.max())
        res_vmin : Float, optional
            vmin for the imshow for the residuals. Default is -0.1
            Default is 'calc' = 0.5(data.min() + model.min())
        res_vmax : Float, optional
            vmax for the imshow for the residuals. Default is +0.1
        edgepts : {Nested list-like, Float, 'calc'}, optional.
            The vertices of the triangles which determine the splitting of
            the image. The vertices are at (image corner, (edge, y), and
            (x,edge), where edge is the appropriate edge of the image.
                edgepts[0] : (x,y) points for the upper edge
                edgepts[1] : (x,y) points for the lower edge
            where `x` is the coordinate along the image's 0th axis and `y`
            along the images 1st axis. Default is 'calc,' which calculates
            edge points by splitting the image into 3 regions of equal
            area. If edgepts is a float scalar, calculates the edge points
            based on a constant fraction of distance from the edge.
        do_imshow : Bool
            If True, imshow's and returns the returned handle.
            If False, returns the array as a [M,N,4] array.
        data_cmap : matplotlib colormap instance
            The colormap to use for the data and model.
        res_cmap : matplotlib colormap instance
            The colormap to use for the residuals.

    Returns
    -------
        image : {matplotlib.pyplot.AxesImage, numpy.ndarray}
            If `do_imshow` == True, the returned handle from imshow.
            If `do_imshow` == False, an [M,N,4] np.ndarray of the image
            pixels.
    """
    # This could be modified to alpha the borderline... or to embiggen
    # the image and slice it more finely
    residuals = s.residuals[tile.slicer].squeeze()
    data = s.data[tile.slicer].squeeze()
    model = s.model[tile.slicer].squeeze()
    if data.ndim != 2:
        raise ValueError('tile does not give a 2D slice')

    im = np.zeros([data.shape[0], data.shape[1], 4])
    if data_vmin == 'calc':
        data_vmin = 0.5*(data.min() + model.min())
    if data_vmax == 'calc':
        data_vmax = 0.5*(data.max() + model.max())

    #1. Get masks:
    upper_mask, center_mask, lower_mask = trisect_image(im.shape, edgepts)

    #2. Get colorbar'd images
    gm = data_cmap(center_data(model, data_vmin, data_vmax))
    dt = data_cmap(center_data(data, data_vmin, data_vmax))
    rs = res_cmap(center_data(residuals, res_vmin, res_vmax))

    for a in range(4):
        im[:,:,a][upper_mask] = rs[:,:,a][upper_mask]
        im[:,:,a][center_mask] = gm[:,:,a][center_mask]
        im[:,:,a][lower_mask] = dt[:,:,a][lower_mask]
    if do_imshow:
        return plt.imshow(im)
    else:
        return im

Response: Compare the data model and residuals of a state.


Instruction:
Can you implement a function in Python 3 that
trisects an image into 3 triangular portions.

Response:
def trisect_image(imshape, edgepts='calc'):
    """
    Returns 3 masks that trisect an image into 3 triangular portions.

    Parameters
    ----------
        imshape : 2-element list-like of ints
            The shape of the image. Elements after the first 2 are ignored.

        edgepts : Nested list-like, float, or `calc`, optional.
            The vertices of the triangles which determine the splitting of
            the image. The vertices are at (image corner, (edge, y), and
            (x,edge), where edge is the appropriate edge of the image.
                edgepts[0] : (x,y) points for the upper edge
                edgepts[1] : (x,y) points for the lower edge
            where `x` is the coordinate along the image's 0th axis and `y`
            along the images 1st axis. Default is 'calc,' which calculates
            edge points by splitting the image into 3 regions of equal
            area. If edgepts is a float scalar, calculates the edge points
            based on a constant fraction of distance from the edge.

    Returns
    -------
        upper_mask : numpy.ndarray
            Boolean array; True in the image's upper  region.
        center_mask : numpy.ndarray
            Boolean array; True in the image's center region.
        lower_mask : numpy.ndarray
            Boolean array; True in the image's lower  region.
    """
    im_x, im_y = np.meshgrid(np.arange(imshape[0]), np.arange(imshape[1]),
            indexing='ij')
    if np.size(edgepts) == 1:
        #Gets equal-area sections, at sqrt(2/3) of the sides
        f = np.sqrt(2./3.) if edgepts == 'calc' else edgepts
        # f = np.sqrt(2./3.)
        lower_edge = (imshape[0] * (1-f),  imshape[1] * f)
        upper_edge = (imshape[0] * f,      imshape[1] * (1-f))
    else:
        upper_edge, lower_edge = edgepts

    #1. Get masks
    lower_slope = lower_edge[1] / max(float(imshape[0] - lower_edge[0]), 1e-9)
    upper_slope = (imshape[1] - upper_edge[1]) / float(upper_edge[0])
    #and the edge points are the x or y intercepts
    lower_intercept = -lower_slope * lower_edge[0]
    upper_intercept = upper_edge[1]
    lower_mask = im_y < (im_x * lower_slope + lower_intercept)
    upper_mask = im_y > (im_x * upper_slope + upper_intercept)

    center_mask= -(lower_mask | upper_mask)
    return upper_mask, center_mask, lower_mask


Instruction:
Explain what the following Python 3 code does
def center_data(data, vmin, vmax):
    ans = data - vmin
    ans /= (vmax - vmin)
    return np.clip(ans, 0, 1)

Response: Clips data on vmin vmax then rescales to 1


Instruction:
Explain what the following Python 3 code does
def crb_compare(state0, samples0, state1, samples1, crb0=None, crb1=None,
        zlayer=None, xlayer=None):
    """
    To run, do:

    s,h = pickle...
    s1,h1 = pickle...
        i.e. /media/scratch/bamf/vacancy/vacancy_zoom-1.tif_t002.tif-featured-v2.pkl
        i.e. /media/scratch/bamf/frozen-particles/0.tif-featured-full.pkl
    crb0 = diag_crb_particles(s); crb1 = diag_crb_particles(s1)
    crb_compare(s,h[-25:],s1,h1[-25:], crb0, crb1)
    """
    s0 = state0
    s1 = state1
    h0 = np.array(samples0)
    h1 = np.array(samples1)

    slicez = zlayer or s0.image.shape[0]//2
    slicex = xlayer or s0.image.shape[2]//2
    slicer1 = np.s_[slicez,s0.pad:-s0.pad,s0.pad:-s0.pad]
    slicer2 = np.s_[s0.pad:-s0.pad,s0.pad:-s0.pad,slicex]
    center = (slicez, s0.image.shape[1]//2, slicex)

    mu0 = h0.mean(axis=0)
    mu1 = h1.mean(axis=0)

    std0 = h0.std(axis=0)
    std1 = h1.std(axis=0)

    mask0 = (s0.state[s0.b_typ]==1.) & (
        analyze.trim_box(s0, mu0[s0.b_pos].reshape(-1,3)))
    mask1 = (s1.state[s1.b_typ]==1.) & (
        analyze.trim_box(s1, mu1[s1.b_pos].reshape(-1,3)))
    active0 = np.arange(s0.N)[mask0]#s0.state[s0.b_typ]==1.]
    active1 = np.arange(s1.N)[mask1]#s1.state[s1.b_typ]==1.]

    pos0 = mu0[s0.b_pos].reshape(-1,3)[active0]
    pos1 = mu1[s1.b_pos].reshape(-1,3)[active1]
    rad0 = mu0[s0.b_rad][active0]
    rad1 = mu1[s1.b_rad][active1]

    link = analyze.nearest(pos0, pos1)
    dpos = pos0 - pos1[link]
    drad = rad0 - rad1[link]

    drift = dpos.mean(axis=0)
    log.info('drift {}'.format(drift))

    dpos -= drift

    fig = pl.figure(figsize=(24,10))

    #=========================================================================
    #=========================================================================
    gs0 = ImageGrid(fig, rect=[0.02, 0.4, 0.4, 0.60], nrows_ncols=(2,3), axes_pad=0.1)

    lbl(gs0[0], 'A')
    for i,slicer in enumerate([slicer1, slicer2]):
        ax_real = gs0[3*i+0]
        ax_fake = gs0[3*i+1]
        ax_diff = gs0[3*i+2]

        diff0 = s0.get_model_image() - s0.image
        diff1 = s1.get_model_image() - s1.image
        a = (s0.image - s1.image)
        b = (s0.get_model_image() - s1.get_model_image())
        c = (diff0 - diff1)

        ptp = 0.7*max([np.abs(a).max(), np.abs(b).max(), np.abs(c).max()])
        cmap = pl.cm.RdBu_r
        ax_real.imshow(a[slicer], cmap=cmap, vmin=-ptp, vmax=ptp)
        ax_real.set_xticks([])
        ax_real.set_yticks([])
        ax_fake.imshow(b[slicer], cmap=cmap, vmin=-ptp, vmax=ptp)
        ax_fake.set_xticks([])
        ax_fake.set_yticks([])
        ax_diff.imshow(c[slicer], cmap=cmap, vmin=-ptp, vmax=ptp)#cmap=pl.cm.RdBu, vmin=-1.0, vmax=1.0)
        ax_diff.set_xticks([])
        ax_diff.set_yticks([])

        if i == 0:
            ax_real.set_title(r"$\Delta$ Confocal image", fontsize=24)
            ax_fake.set_title(r"$\Delta$ Model image", fontsize=24)
            ax_diff.set_title(r"$\Delta$ Difference", fontsize=24)
            ax_real.set_ylabel('x-y')
        else:
            ax_real.set_ylabel('x-z')

    #=========================================================================
    #=========================================================================
    gs1 = GridSpec(1,3, left=0.05, bottom=0.125, right=0.42, top=0.37,
                wspace=0.15, hspace=0.05)

    spos0 = std0[s0.b_pos].reshape(-1,3)[active0]
    spos1 = std1[s1.b_pos].reshape(-1,3)[active1]
    srad0 = std0[s0.b_rad][active0]
    srad1 = std1[s1.b_rad][active1]

    def hist(ax, vals, bins, *args, **kwargs):
        y,x = np.histogram(vals, bins=bins)
        x = (x[1:] + x[:-1])/2
        y /= len(vals)
        ax.plot(x,y, *args, **kwargs)

    def pp(ind, tarr, tsim, tcrb, var='x'):
        bins = 10**np.linspace(-3, 0.0, 30)
        bin2 = 10**np.linspace(-3, 0.0, 100)
        bins = np.linspace(0.0, 0.2, 30)
        bin2 = np.linspace(0.0, 0.2, 100)
        xlim = (0.0, 0.12)
        #xlim = (1e-3, 1e0)
        ylim = (1e-2, 30)

        ticks = ticker.FuncFormatter(lambda x, pos: '{:0.0f}'.format(np.log10(x)))
        scaler = lambda x: x #np.log10(x)

        ax_crb = pl.subplot(gs1[0,ind])
        ax_crb.hist(scaler(np.abs(tarr)), bins=bins,
                normed=True, alpha=0.7, histtype='stepfilled', lw=1)
        ax_crb.hist(scaler(np.abs(tcrb)).ravel(), bins=bin2,
                normed=True, alpha=1.0, histtype='step', ls='solid', lw=1.5, color='k')
        ax_crb.hist(scaler(np.abs(tsim).ravel()), bins=bin2,
                normed=True, alpha=1.0, histtype='step', lw=3)
        ax_crb.set_xlabel(r"$\Delta = |%s(t_1) - %s(t_0)|$" % (var,var), fontsize=24)
        #ax_crb.semilogx()
        ax_crb.set_xlim(xlim)
        #ax_crb.semilogy()
        #ax_crb.set_ylim(ylim)
        #ax_crb.xaxis.set_major_formatter(ticks)
        ax_crb.grid(b=False, which='both', axis='both')

        if ind == 0:
            lbl(ax_crb, 'B')
            ax_crb.set_ylabel(r"$P(\Delta)$")
        else:
            ax_crb.set_yticks([])

        ax_crb.locator_params(axis='x', nbins=3)

    f,g = 1.5, 1.95
    sim = f*sim_crb_diff(spos0[:,1], spos1[:,1][link])
    crb = g*sim_crb_diff(crb0[0][:,1][active0], crb1[0][:,1][active1][link])
    pp(0, dpos[:,1], sim, crb, 'x')

    sim = f*sim_crb_diff(spos0[:,0], spos1[:,0][link])
    crb = g*sim_crb_diff(crb0[0][:,0][active0], crb1[0][:,0][active1][link])
    pp(1, dpos[:,0], sim, crb, 'z')

    sim = f*sim_crb_diff(srad0, srad1[link])
    crb = g*sim_crb_diff(crb0[1][active0], crb1[1][active1][link])
    pp(2, drad, sim, crb, 'a')

    #ax_crb_r.locator_params(axis='both', nbins=3)
    #gs1.tight_layout(fig)

    #=========================================================================
    #=========================================================================
    gs2 = GridSpec(2,2, left=0.48, bottom=0.12, right=0.99, top=0.95,
                wspace=0.35, hspace=0.35)

    ax_hist = pl.subplot(gs2[0,0])
    ax_hist.hist(std0[s0.b_pos], bins=np.logspace(-3.0, 0, 50), alpha=0.7, label='POS', histtype='stepfilled')
    ax_hist.hist(std0[s0.b_rad], bins=np.logspace(-3.0, 0, 50), alpha=0.7, label='RAD', histtype='stepfilled')
    ax_hist.set_xlim((10**-3.0, 1))
    ax_hist.semilogx()
    ax_hist.set_xlabel(r"$\bar{\sigma}$")
    ax_hist.set_ylabel(r"$P(\bar{\sigma})$")
    ax_hist.legend(loc='upper right')
    lbl(ax_hist, 'C')

    imdiff = ((s0.get_model_image() - s0.image)/s0._sigma_field)[s0.image_mask==1.].ravel()
    mu = imdiff.mean()
    #sig = imdiff.std()
    #print mu, sig
    x = np.linspace(-5,5,10000)

    ax_diff = pl.subplot(gs2[0,1])
    ax_diff.plot(x, 1.0/np.sqrt(2*np.pi) * np.exp(-(x-mu)**2 / 2), '-', alpha=0.7, color='k', lw=2)
    ax_diff.hist(imdiff, bins=1000, histtype='step', alpha=0.7, normed=True)
    ax_diff.semilogy()
    ax_diff.set_ylabel(r"$P(\delta)$")
    ax_diff.set_xlabel(r"$\delta = (M_i - d_i)/\sigma_i$")
    ax_diff.locator_params(axis='x', nbins=5)
    ax_diff.grid(b=False, which='minor', axis='y')
    ax_diff.set_xlim(-5, 5)
    ax_diff.set_ylim(1e-4, 1e0)
    lbl(ax_diff, 'D')

    pos = mu0[s0.b_pos].reshape(-1,3)
    rad = mu0[s0.b_rad]
    mask = analyze.trim_box(s0, pos)
    pos = pos[mask]
    rad = rad[mask]

    gx, gy = analyze.gofr(pos, rad, mu0[s0.b_zscale][0], resolution=5e-2,mask_start=0.5)
    mask = gx < 5
    gx = gx[mask]
    gy = gy[mask]
    ax_gofr = pl.subplot(gs2[1,0])
    ax_gofr.plot(gx, gy, '-', lw=1)
    ax_gofr.set_xlabel(r"$r/a$")
    ax_gofr.set_ylabel(r"$g(r/a)$")
    ax_gofr.locator_params(axis='both', nbins=5)
    #ax_gofr.semilogy()
    lbl(ax_gofr, 'E')

    gx, gy = analyze.gofr(pos, rad, mu0[s0.b_zscale][0], method='surface')
    mask = gx < 5
    gx = gx[mask]
    gy = gy[mask]
    gy[gy <= 0.] = gy[gy>0].min()
    ax_gofrs = pl.subplot(gs2[1,1])
    ax_gofrs.plot(gx, gy, '-', lw=1)
    ax_gofrs.set_xlabel(r"$r/a$")
    ax_gofrs.set_ylabel(r"$g_{\rm{surface}}(r/a)$")
    ax_gofrs.locator_params(axis='both', nbins=5)
    ax_gofrs.grid(b=False, which='minor', axis='y')
    #ax_gofrs.semilogy()
    lbl(ax_gofrs, 'F')

    ylim = ax_gofrs.get_ylim()
    ax_gofrs.set_ylim(gy.min(), ylim[1])

Response: Compare two states and returns the crb of the two states.


Instruction:
Can you implement a function in Python 3 that
plots two parts of the ortho view the two sections of the ortho view.

Response:
def twoslice(field, center=None, size=6.0, cmap='bone_r', vmin=0, vmax=1,
        orientation='vertical', figpad=1.09, off=0.01):
    """
    Plot two parts of the ortho view, the two sections given by ``orientation``.
    """
    center = center or [i//2 for i in field.shape]
    slices = []
    for i,c in enumerate(center):
        blank = [np.s_[:]]*len(center)
        blank[i] = c
        slices.append(tuple(blank))

    z,y,x = [float(i) for i in field.shape]
    w = float(x + z)
    h = float(y + z)

    def show(field, ax, slicer, transpose=False):
        tmp = field[slicer] if not transpose else field[slicer].T
        ax.imshow(
            tmp, cmap=cmap, interpolation='nearest',
            vmin=vmin, vmax=vmax
        )
        ax.set_xticks([])
        ax.set_yticks([])
        ax.grid('off')

    if orientation.startswith('v'):
        # rect = l,b,w,h
        log.info('{} {} {} {} {} {}'.format(x, y, z, w, h, x/h))
        r = x/h
        q = y/h
        f = 1 / (1 + 3*off)
        fig = pl.figure(figsize=(size*r, size*f))
        ax1 = fig.add_axes((off, f*(1-q)+2*off, f, f*q))
        ax2 = fig.add_axes((off, off,           f, f*(1-q)))

        show(field, ax1, slices[0])
        show(field, ax2, slices[1])
    else:
        # rect = l,b,w,h
        r = y/w
        q = x/w
        f = 1 / (1 + 3*off)
        fig = pl.figure(figsize=(size*f, size*r))
        ax1 = fig.add_axes((off,    off,   f*q, f))
        ax2 = fig.add_axes((2*off+f*q, off, f*(1-q), f))

        show(field, ax1, slices[0])
        show(field, ax2, slices[2], transpose=True)

    return fig, ax1, ax2


Instruction:
Can you tell what is the following Python 3 function doing
def circles(st, layer, axis, ax=None, talpha=1.0, cedge='white', cface='white'):
    pos = st.obj_get_positions()
    rad = st.obj_get_radii()
    shape = st.ishape.shape.tolist()
    shape.pop(axis) #shape is now the shape of the image
    if ax is None:
        fig = plt.figure()
        axisbg = 'white' if cface == 'black' else 'black'
        sx, sy = ((1,shape[1]/float(shape[0])) if shape[0] > shape[1] else
                (shape[0]/float(shape[1]), 1))
        ax = fig.add_axes((0,0, sx, sy), axisbg=axisbg)
    # get the index of the particles we want to include
    particles = np.arange(len(pos))[np.abs(pos[:,axis] - layer) < rad]

    # for each of these particles display the effective radius
    # in the proper place
    scale = 1.0 #np.max(shape).astype('float')
    for i in particles:
        p = pos[i].copy()
        r = 2*np.sqrt(rad[i]**2 - (p[axis] - layer)**2)
        #CIRCLE IS IN FIGURE COORDINATES!!!
        if axis==0:
            ix = 1; iy = 2
        elif axis == 1:
            ix = 0; iy = 2
        elif axis==2:
            ix = 0; iy = 1
        c = Circle((p[ix]/scale, p[iy]/scale), radius=r/2/scale, fc=cface,
                ec=cedge, alpha=talpha)
        ax.add_patch(c)
    # plt.axis([0,1,0,1])
    plt.axis('equal') #circles not ellipses
    return ax

Response:
Returns a set of circles corresponding to a slice through the platonic
    structure.


Instruction:
Write a Python 3 script to
create a two particle state and compare it to featuring using a single particle guess

Response:
def missing_particle(separation=0.0, radius=RADIUS, SNR=20):
    """ create a two particle state and compare it to featuring using a single particle guess """
    # create a base image of one particle
    s = init.create_two_particle_state(imsize=6*radius+4, axis='x', sigma=1.0/SNR,
            delta=separation, radius=radius, stateargs={'varyn': True}, psfargs={'error': 1e-6})
    s.obj.typ[1] = 0.
    s.reset()

    return s, s.obj.pos.copy()


Instruction:
Create a Python 3 function for
calculating a random approximation to J by returning J only at the given indices.

Response:
def get_rand_Japprox(s, params, num_inds=1000, include_cost=False, **kwargs):
    """
    Calculates a random approximation to J by returning J only at a
    set of random pixel/voxel locations.

    Parameters
    ----------
        s : :class:`peri.states.State`
            The state to calculate J for.
        params : List
            The list of parameter names to calculate the gradient of.
        num_inds : Int, optional.
            The number of pix/voxels at which to calculate the random
            approximation to J. Default is 1000.
        include_cost : Bool, optional
            Set to True to append a finite-difference measure of the full
            cost gradient onto the returned J.

    Other Parameters
    ----------------
        All kwargs parameters get passed to s.gradmodel only.

    Returns
    -------
        J : numpy.ndarray
            [d, num_inds] array of J, at the given indices.

        return_inds : numpy.ndarray or slice
            [num_inds] element array or slice(0, None) of the model
            indices at which J was evaluated.
    """
    start_time = time.time()
    tot_pix = s.residuals.size
    if num_inds < tot_pix:
        inds = np.random.choice(tot_pix, size=num_inds, replace=False)
        slicer = None
        return_inds = np.sort(inds)
    else:
        inds = None
        return_inds = slice(0, None)
        slicer = [slice(0, None)]*len(s.residuals.shape)
    if include_cost:
        Jact, ge = s.gradmodel_e(params=params, inds=inds, slicer=slicer,flat=False,
                **kwargs)
        Jact *= -1
        J = [Jact, ge]
    else:
        J = -s.gradmodel(params=params, inds=inds, slicer=slicer, flat=False,
                **kwargs)
    CLOG.debug('J:\t%f' % (time.time()-start_time))
    return J, return_inds


Instruction:
Write a Python 3 function that can
return a list of the global parameter names of the state s.

Response:
def name_globals(s, remove_params=None):
    """
    Returns a list of the global parameter names.

    Parameters
    ----------
        s : :class:`peri.states.ImageState`
            The state to name the globals of.
        remove_params : Set or None
            A set of unique additional parameters to remove from the globals
            list.

    Returns
    -------
        all_params : list
            The list of the global parameter names, with each of
            remove_params removed.
    """
    all_params = s.params
    for p in s.param_particle(np.arange(s.obj_get_positions().shape[0])):
        all_params.remove(p)
    if remove_params is not None:
        for p in set(remove_params):
            all_params.remove(p)
    return all_params


Instruction:
How would you code a function in Python 3 to
calculate the number of pixels to use for a given state and parameters.

Response:
def get_num_px_jtj(s, nparams, decimate=1, max_mem=1e9, min_redundant=20):
    """
    Calculates the number of pixels to use for J at a given memory usage.

    Tries to pick a number of pixels as (size of image / `decimate`).
    However, clips this to a maximum size and minimum size to ensure that
    (1) too much memory isn't used and (2) J has enough elements so that
    the inverse of JTJ will be well-conditioned.

    Parameters
    ----------
        s : :class:`peri.states.State`
            The state on which to calculate J.
        nparams : Int
            The number of parameters that will be included in J.
        decimate : Int, optional
            The amount to decimate the number of pixels in the image by,
            i.e. tries to pick num_px = size of image / decimate.
            Default is 1
        max_mem : Numeric, optional
            The maximum allowed memory, in bytes, for J to occupy at
            double-precision. Default is 1e9.
        min_redundant : Int, optional
            The number of pixels must be at least `min_redundant` *
            `nparams`. If not, an error is raised. Default is 20

    Returns
    -------
        num_px : Int
            The number of pixels at which to calcualte J.
    """
    #1. Max for a given max_mem:
    px_mem = int(max_mem // 8 // nparams) #1 float = 8 bytes
    #2. num_pix for a given redundancy
    px_red = min_redundant*nparams
    #3. And # desired for decimation
    px_dec = s.residuals.size//decimate

    if px_red > px_mem:
        raise RuntimeError('Insufficient max_mem for desired redundancy.')
    num_px = np.clip(px_dec, px_red, px_mem).astype('int')
    return num_px


Instruction:
Here you have a function in Python 3, explain what it does
def vectorize_damping(params, damping=1.0, increase_list=[['psf-', 1e4]]):
    damp_vec = np.ones(len(params)) * damping
    for nm, fctr in increase_list:
        for a in range(damp_vec.size):
            if nm in params[a]:
                damp_vec[a] *= fctr
    return damp_vec

Response: Vectorizes the damping of a set of parameters.


Instruction:
Explain what the following Python 3 code does
def low_mem_sq(m, step=100000):
    if not m.flags.c_contiguous:
        raise ValueError('m must be C ordered for this to work with less mem.')
    # -- can make this even faster with pre-allocating arrays, but not worth it
    # right now
    # mmt = np.zeros([m.shape[0], m.shape[0]])  #6us
    # mt_tmp = np.zeros([step, m.shape[0]])
    # for a in range(0, m.shape[1], step):
        # mx = min(a+step, m.shape[1])
        # mt_tmp[:mx-a,:] = m.T[a:mx]
        # # np.dot(m_tmp, m.T, out=mmt[a:mx])
        # # np.dot(m, m[a:mx].T, out=mmt[:, a:mx])
        # np.dot(m[:,a:mx], mt_tmp[:mx], out=mmt)
    # return mmt
    mmt = np.zeros([m.shape[0], m.shape[0]])  #6us
    # m_tmp = np.zeros([step, m.shape[1]])
    for a in range(0, m.shape[0], step):
        mx = min(a+step, m.shape[1])
        # m_tmp[:] = m[a:mx]
        # np.dot(m_tmp, m.T, out=mmt[a:mx])
        mmt[:, a:mx] = np.dot(m, m[a:mx].T)
    return mmt

Response: Compute the low mem vector for a given set of tables.


Instruction:
Write a Python 3 script for
finding the particles in a tile as numpy. ndarray of ints.

Response:
def find_particles_in_tile(positions, tile):
    """
    Finds the particles in a tile, as numpy.ndarray of ints.

    Parameters
    ----------
        positions : `numpy.ndarray`
            [N,3] array of the particle positions to check in the tile
        tile : :class:`peri.util.Tile` instance
            Tile of the region inside which to check for particles.

    Returns
    -------
        numpy.ndarray, int
            The indices of the particles in the tile.
    """
    bools = tile.contains(positions)
    return np.arange(bools.size)[bools]


Instruction:
Make a summary of the following Python 3 code
def separate_particles_into_groups(s, region_size=40, bounds=None,
        doshift=False):
    """
    Separates particles into convenient groups for optimization.

    Given a state, returns a list of groups of particles. Each group of
    particles are located near each other in the image. Every particle
    located in the desired region is contained in exactly 1 group.

    Parameters
    ----------
    s : :class:`peri.states.ImageState`
        The peri state to find particles in.
    region_size : Int or 3-element list-like of ints, optional
        The size of the box. Groups particles into boxes of shape
        (region_size[0], region_size[1], region_size[2]). If region_size
        is a scalar, the box is a cube of length region_size.
        Default is 40.
    bounds : 2-element list-like of 3-element lists, optional
        The sub-region of the image over which to look for particles.
            bounds[0]: The lower-left  corner of the image region.
            bounds[1]: The upper-right corner of the image region.
        Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
        image size, i.e. the default places every particle in the image
        somewhere in the groups.
    doshift : {True, False, `'rand'`}, optional
        Whether or not to shift the tile boxes by half a region size, to
        prevent the same particles to be chosen every time. If `'rand'`,
        randomly chooses either True or False. Default is False

    Returns
    -------
    particle_groups : List
        Each element of particle_groups is an int numpy.ndarray of the
        group of nearby particles. Only contains groups with a nonzero
        number of particles, so the elements don't necessarily correspond
        to a given image region.
    """
    imtile = s.oshape.translate(-s.pad)
    bounding_tile = (imtile if bounds is None else Tile(bounds[0], bounds[1]))
    rs = (np.ones(bounding_tile.dim, dtype='int')*region_size if
            np.size(region_size) == 1 else np.array(region_size))

    n_translate = np.ceil(bounding_tile.shape.astype('float')/rs).astype('int')
    particle_groups = []
    tile = Tile(left=bounding_tile.l, right=bounding_tile.l + rs)
    if doshift == 'rand':
        doshift = np.random.choice([True, False])
    if doshift:
        shift = rs // 2
        n_translate += 1
    else:
        shift = 0
    deltas = np.meshgrid(*[np.arange(i) for i in n_translate])
    positions = s.obj_get_positions()
    if bounds is None:
        # FIXME this (deliberately) masks a problem where optimization
        # places particles outside the image. However, it ensures that
        # all particles are in at least one group when `bounds is None`,
        # which is the use case within opt. The 1e-3 is to ensure that
        # they are inside the box and not on the edge.
        positions = np.clip(positions, imtile.l+1e-3, imtile.r-1e-3)
    groups = list(map(lambda *args: find_particles_in_tile(positions,
            tile.translate( np.array(args) * rs - shift)), *[d.ravel()
            for d in deltas]))

    for i in range(len(groups)-1, -1, -1):
        if groups[i].size == 0:
            groups.pop(i)
    assert _check_groups(s, groups)
    return groups

Response: Separates particles into convenient groups for optimization.


Instruction:
Given the following Python 3 function, write the documentation
def _check_groups(s, groups):
    ans = []
    for g in groups:
        ans.extend(g)
    if np.unique(ans).size != np.size(ans):
        return False
    elif np.unique(ans).size != s.obj_get_positions().shape[0]:
        return False
    else:
        return (np.arange(s.obj_get_radii().size) == np.sort(ans)).all()

Response: Ensures that all particles in exactly 1 group are included in exactly 1 group


Instruction:
Create a Python 3 function for
calculating the size of the region size of the particle group.

Response:
def calc_particle_group_region_size(s, region_size=40, max_mem=1e9, **kwargs):
    """
    Finds the biggest region size for LM particle optimization with a
    given memory constraint.

    Input Parameters
    ----------------
        s : :class:`peri.states.ImageState`
            The state with the particles
        region_size : Int or 3-element list-like of ints, optional.
            The initial guess for the region size. Default is 40
        max_mem : Numeric, optional
            The maximum memory for the optimizer to take. Default is 1e9

    Other Parameters
    ----------------
        bounds: 2-element list-like of 3-element lists.
            The sub-region of the image over which to look for particles.
                bounds[0]: The lower-left  corner of the image region.
                bounds[1]: The upper-right corner of the image region.
            Default (None -> ([0,0,0], s.oshape.shape)) is a box of the entire
            image size, i.e. the default places every particle in the image
            somewhere in the groups.
    Returns
    -------
        region_size : numpy.ndarray of ints of the region size.
    """
    region_size = np.array(region_size).astype('int')

    def calc_mem_usage(region_size):
        rs = np.array(region_size)
        particle_groups = separate_particles_into_groups(s, region_size=
                rs.tolist(), **kwargs)
        # The actual mem usage is the max of the memory usage of all the
        # particle groups. However this is too slow. So instead we use the
        # max of the memory of the biggest 5 particle groups:
        numpart = [np.size(g) for g in particle_groups]
        biggroups = [particle_groups[i] for i in np.argsort(numpart)[-5:]]
        def get_tile_jsize(group):
            nms = s.param_particle(group)
            tile = s.get_update_io_tiles(nms, s.get_values(nms))[2]
            return tile.shape.prod() * len(nms)
        mems = [8*get_tile_jsize(g) for g in biggroups]  # 8 for bytes/float64
        return np.max(mems)

    im_shape = s.oshape.shape
    if calc_mem_usage(region_size) > max_mem:
        while ((calc_mem_usage(region_size) > max_mem) and
                np.any(region_size > 2)):
            region_size = np.clip(region_size-1, 2, im_shape)
    else:
        while ((calc_mem_usage(region_size) < max_mem) and
                np.any(region_size < im_shape)):
            region_size = np.clip(region_size+1, 2, im_shape)
        region_size -= 1 #need to be < memory, so we undo 1 iteration

    return region_size


Instruction:
Write a Python 3 script for
returning a new state with the same state and a tile that is in the unpadded image.

Response:
def get_residuals_update_tile(st, padded_tile):
    """
    Translates a tile in the padded image to the unpadded image.

    Given a state and a tile that corresponds to the padded image, returns
    a tile that corresponds to the the corresponding pixels of the difference
    image

    Parameters
    ----------
        st : :class:`peri.states.State`
            The state
        padded_tile : :class:`peri.util.Tile`
            The tile in the padded image.

    Returns
    -------
        :class:`peri.util.Tile`
            The tile corresponding to padded_tile in the unpadded image.
    """
    inner_tile = st.ishape.intersection([st.ishape, padded_tile])
    return inner_tile.translate(-st.pad)


Instruction:
Can you generate the documentation for the following Python 3 function
def find_best_step(err_vals):
    if np.all(np.isnan(err_vals)):
        raise ValueError('All err_vals are nans!')
    return np.nanargmin(err_vals)

Response: Returns the index of the lowest of the passed values. Catches nans etc.


Instruction:
Create a Python 3 function to
run Levenberg-Marquardt optimization on a state. Convenience wrapper for LMGlobals. Same keyword args, but the defaults have been set to useful values for optimizing globals. See LMGlobals and LMEngine for documentation. See Also -------- do_levmarq_particles : Levenberg-Marquardt optimization of a specified set of particles. do_levmarq_all_particle_groups : Levenberg-Marquardt optimization of all the particles in the state. LMGlobals : Optimizer object; the workhorse of do_levmarq. LMEngine : Engine superclass for all the optimizers.

Response:
def do_levmarq(s, param_names, damping=0.1, decrease_damp_factor=10.,
        run_length=6, eig_update=True, collect_stats=False, rz_order=0,
        run_type=2, **kwargs):
    """
    Runs Levenberg-Marquardt optimization on a state.

    Convenience wrapper for LMGlobals. Same keyword args, but the defaults
    have been set to useful values for optimizing globals.
    See LMGlobals and LMEngine for documentation.

    See Also
    --------
        do_levmarq_particles : Levenberg-Marquardt optimization of a
            specified set of particles.

        do_levmarq_all_particle_groups : Levenberg-Marquardt optimization
            of all the particles in the state.

        LMGlobals : Optimizer object; the workhorse of do_levmarq.

        LMEngine : Engine superclass for all the optimizers.
    """
    if rz_order > 0:
        aug = AugmentedState(s, param_names, rz_order=rz_order)
        lm = LMAugmentedState(aug, damping=damping, run_length=run_length,
                decrease_damp_factor=decrease_damp_factor, eig_update=
                eig_update, **kwargs)
    else:
        lm = LMGlobals(s, param_names, damping=damping, run_length=run_length,
                decrease_damp_factor=decrease_damp_factor, eig_update=
                eig_update, **kwargs)
    if run_type == 2:
        lm.do_run_2()
    elif run_type == 1:
        lm.do_run_1()
    else:
        raise ValueError('run_type=1,2 only')
    if collect_stats:
        return lm.get_termination_stats()


Instruction:
Here you have a function in Python 3, explain what it does
def do_levmarq_particles(s, particles, damping=1.0, decrease_damp_factor=10.,
        run_length=4, collect_stats=False, max_iter=2, **kwargs):
    """
    Levenberg-Marquardt optimization on a set of particles.

    Convenience wrapper for LMParticles. Same keyword args, but the
    defaults have been set to useful values for optimizing particles.
    See LMParticles and LMEngine for documentation.

    See Also
    --------
        do_levmarq_all_particle_groups : Levenberg-Marquardt optimization
            of all the particles in the state.

        do_levmarq : Levenberg-Marquardt optimization of the entire state;
            useful for optimizing global parameters.

        LMParticles : Optimizer object; the workhorse of do_levmarq_particles.

        LMEngine : Engine superclass for all the optimizers.
    """
    lp = LMParticles(s, particles, damping=damping, run_length=run_length,
            decrease_damp_factor=decrease_damp_factor, max_iter=max_iter,
            **kwargs)
    lp.do_run_2()
    if collect_stats:
        return lp.get_termination_stats()

Response: Wrapper for Levenberg - Marquardt optimization on a set of particles.


Instruction:
How would you explain what the following Python 3 function does
def do_levmarq_all_particle_groups(s, region_size=40, max_iter=2, damping=1.0,
        decrease_damp_factor=10., run_length=4, collect_stats=False, **kwargs):
    """
    Levenberg-Marquardt optimization for every particle in the state.

    Convenience wrapper for LMParticleGroupCollection. Same keyword args,
    but I've set the defaults to what I've found to be useful values for
    optimizing particles. See LMParticleGroupCollection for documentation.

    See Also
    --------
        do_levmarq_particles : Levenberg-Marquardt optimization of a
            specified set of particles.

        do_levmarq : Levenberg-Marquardt optimization of the entire state;
            useful for optimizing global parameters.

        LMParticleGroupCollection : The workhorse of do_levmarq.

        LMEngine : Engine superclass for all the optimizers.
    """
    lp = LMParticleGroupCollection(s, region_size=region_size, damping=damping,
            run_length=run_length, decrease_damp_factor=decrease_damp_factor,
            get_cos=collect_stats, max_iter=max_iter, **kwargs)
    lp.do_run_2()
    if collect_stats:
        return lp.stats

Response: This function is used to run all the particle groups of a state.


Instruction:
Can you generate the documentation for the following Python 3 function
def burn(s, n_loop=6, collect_stats=False, desc='', rz_order=0, fractol=1e-4,
        errtol=1e-2, mode='burn', max_mem=1e9, include_rad=True,
        do_line_min='default', partial_log=False, dowarn=True):
    """
    Optimizes all the parameters of a state.

    Burns a state through calling LMParticleGroupCollection and LMGlobals/
    LMAugmentedState.

    Parameters
    ----------
        s : :class:`peri.states.ImageState`
            The state to optimize
        n_loop : Int, optional
            The number of times to loop over in the optimizer. Default is 6.
        collect_stats : Bool, optional
            Whether or not to collect information on the optimizer's
            performance. Default is False.
        desc : string, optional
            Description to append to the states.save() call every loop.
            Set to None to avoid saving. Default is '', which selects
            one of 'burning', 'polishing', 'doing_positions'
        rz_order: Int, optional
            Set to an int > 0 to optimize with an augmented state (R(z) as
            a global parameter) vs. with the normal global parameters;
            rz_order is the order of the polynomial approximate for R(z).
            Default is 0 (no augmented state).
        fractol : Float, optional
            Fractional change in error at which to terminate. Default 1e-4
        errtol : Float, optional
            Absolute change in error at which to terminate. Default 1e-2
        mode : {'burn', 'do-particles', or 'polish'}, optional
            What mode to optimize with.
            * 'burn'          : Your state is far from the minimum.
            * 'do-particles'  : Positions far from minimum, globals well-fit.
            * 'polish'        : The state is close to the minimum.
            'burn' is the default. Only `polish` will get to the global
            minimum.
        max_mem : Numeric, optional
            The maximum amount of memory allowed for the optimizers' J's,
            for both particles & globals. Default is 1e9, i.e. 1GB per
            optimizer.
        do_line_min : Bool or 'default', optional
            Set to True to do an additional, third optimization per loop
            which optimizes along the subspace spanned by the last 3 steps
            of the burn()'s trajectory. In principle this should signifi-
            cantly speed up the convergence; in practice it sometimes does,
            sometimes doesn't. Default is 'default', which picks by mode:
            * 'burn'          : False
            * 'do-particles'  : False
            * 'polish'        : True
        dowarn : Bool, optional
            Whether to log a warning if termination results from finishing
            loops rather than from convergence. Default is True.

    Returns
    -------
        dictionary
            Dictionary of convergence information. Contains whether the
            optimization has converged (key ``'converged'``), the values of the
            state after each loop (key ``'all_loop_values'``).
            The values of the state's parameters after each part of the
            loop: globals, particles, linemin. If ``collect_stats`` is set,
            then also contains lists of termination dicts from globals,
            particles, and line minimization (keys ``'global_stats'``,
            ``'particle_stats'``, and ``'line_stats``').

    Notes
    -----
    Proceeds by alternating between one Levenberg-Marquardt step
    optimizing the globals, one optimizing the particles, and repeating
    until termination.

    In addition, if `do_line_min` is True, at the end of each loop
    step an additional optimization is tried along the subspaced spanned
    by the steps taken during the last 3 loops. Ideally, this changes the
    convergence from linear to quadratic, but it doesn't always do much.

    Each of the 3 options proceed by optimizing as follows:
    * burn            : lm.do_run_2(), lp.do_run_2(). No psf, 2 loops on lm.
    * do-particles    : lp.do_run_2(), scales for ilm, bkg's
    * polish          : lm.do_run_2(), lp.do_run_2(). Everything, 1 loop each.
    where lm is a globals LMGlobals instance, and lp a
    LMParticleGroupCollection instance.
    """
    # It would be nice if some of these magic #'s (region size,
    # num_eig_dirs, etc) were calculated in a good way. FIXME
    mode = mode.lower()
    if mode not in {'burn', 'do-particles', 'polish'}:
        raise ValueError('mode must be one of burn, do-particles, polish')

    #1. Setting Defaults
    if desc is '':
        desc = mode + 'ing' if mode != 'do-particles' else 'doing-particles'

    eig_update = (mode != 'do-particles')
    glbl_run_length = 3 if mode == 'do-particles' else 6
    glbl_mx_itr = 2 if mode == 'burn' else 1
    use_accel = (mode == 'burn')
    rz_order = int(rz_order)
    if do_line_min == 'default':
        # do_line_min = (mode == 'polish')
        # temporary fix until we solve the particles-leaving-image issue:
        do_line_min = False

    if mode == 'do-particles':
        # FIXME explicit params
        # We pick some parameters for an overall illumination scale:
        glbl_nms = ['ilm-scale', 'ilm-xy-0-0', 'bkg-xy-0-0', 'offset']
        # And now, since we have explicit parameters, we check that they
        # are actually in the state:
        glbl_nms = [nm for nm in glbl_nms if nm in s.params]
    else:
        if mode == 'polish':
            remove_params = None
        else:
            # FIXME explicit params
            remove_params = s.get('psf').params
            if ('zscale' not in remove_params) and ('zscale' in s.params):
                remove_params.append('zscale')
        glbl_nms = name_globals(s, remove_params=remove_params)

    all_lp_stats = []
    all_lm_stats = []
    all_line_stats = []
    all_loop_values = []

    _delta_vals = []  # storing the directions we've moved along for line min
    #2. Optimize
    CLOG.info('Start of loop %d:\t%f' % (0, s.error))
    for a in range(n_loop):
        start_err = s.error
        start_params = np.copy(s.state[s.params])
        #2a. Globals
        # glbl_dmp = 0.3 if a == 0 else 3e-2
        ####FIXME we damp degenerate but convenient spaces in the ilm, bkg
        ####manually, but we should do it more betterer.
        BAD_DAMP = 1e7
        BAD_LIST = [['ilm-scale', BAD_DAMP], ['ilm-off', BAD_DAMP], ['ilm-z-0',
                BAD_DAMP], ['bkg-z-0', BAD_DAMP]]
        ####
        glbl_dmp = vectorize_damping(glbl_nms + ['rz']*rz_order, damping=1.0,
                increase_list=[['psf-', 3e1]] + BAD_LIST)
        if a != 0 or mode != 'do-particles':
            if partial_log:
                log.set_level('debug')
            gstats = do_levmarq(s, glbl_nms, max_iter=glbl_mx_itr, run_length=
                    glbl_run_length, eig_update=eig_update, num_eig_dirs=10,
                    eig_update_frequency=3, rz_order=rz_order, damping=
                    glbl_dmp, decrease_damp_factor=10., use_accel=use_accel,
                    collect_stats=collect_stats, fractol=0.1*fractol,
                    max_mem=max_mem)
            if partial_log:
                log.set_level('info')
            all_lm_stats.append(gstats)
        if desc is not None:
            states.save(s, desc=desc)
        CLOG.info('Globals,   loop {}:\t{}'.format(a, s.error))
        all_loop_values.append(s.values)

        #2b. Particles
        prtl_dmp = 1.0 if a==0 else 1e-2
        #For now, I'm calculating the region size. This might be a bad idea
        #because 1 bad particle can spoil the whole group.
        pstats = do_levmarq_all_particle_groups(s, region_size=40, max_iter=1,
                do_calc_size=True, run_length=4, eig_update=False,
                damping=prtl_dmp, fractol=0.1*fractol, collect_stats=
                collect_stats, max_mem=max_mem, include_rad=include_rad)
        all_lp_stats.append(pstats)
        if desc is not None:
            states.save(s, desc=desc)
        CLOG.info('Particles, loop {}:\t{}'.format(a, s.error))
        gc.collect()
        all_loop_values.append(s.values)

        #2c. Line min?
        end_params = np.copy(s.state[s.params])
        _delta_vals.append(start_params - end_params)
        if do_line_min:
            all_line_stats.append(do_levmarq_n_directions(s, _delta_vals[-3:],
                    collect_stats=collect_stats))
            if desc is not None:
                states.save(s, desc=desc)
            CLOG.info('Line min., loop {}:\t{}'.format(a, s.error))
            all_loop_values.append(s.values)

        #2d. terminate?
        new_err = s.error
        derr = start_err - new_err
        dobreak = (derr/new_err < fractol) or (derr < errtol)
        if dobreak:
            break

    if dowarn and (not dobreak):
        CLOG.warn('burn() did not converge; consider re-running')

    d = {'converged':dobreak, 'all_loop_values':all_loop_values}
    if collect_stats:
        d.update({'global_stats':all_lm_stats, 'particle_stats':all_lp_stats,
                'line_stats':all_line_stats})
    return d

Response: Burns a state through the LMParticleGroupCollection and LMAugmentedState LMGlobalGlobal and LMGlobalGlobalGlobalGlobalGlobalGlobal and LMParticleGroupCollection and LMAugmentedState and LMAugmentedState LMAugmentedState. LMState. LMA


Instruction:
How would you implement a function in Python 3 that
crawls slowly to the minimum-cost state. Blocks the global parameters into small enough sections such that each can be optimized separately while including all the pixels (i.e. no decimation). Optimizes the globals, then the psf separately if desired, then particles, then a line minimization along the step direction to speed up convergence. Parameters ---------- s : :class:`peri.states.ImageState` The state to optimize desc : string, optional Description to append to the states.save() call every loop. Set to `None` to avoid saving. Default is `'finish'`. n_loop : Int, optional The number of times to loop over in the optimizer. Default is 4. max_mem : Numeric, optional The maximum amount of memory allowed for the optimizers' J's, for both particles & globals. Default is 1e9. separate_psf : Bool, optional If True, does the psf optimization separately from the rest of the globals, since the psf has a more tortuous fit landscape. Default is True. fractol : Float, optional Fractional change in error at which to terminate. Default 1e-4 errtol : Float, optional Absolute change in error at which to terminate. Default 1e-2 dowarn : Bool, optional Whether to log a warning if termination results from finishing loops rather than from convergence. Default is True. Returns ------- dictionary Information about the optimization. Has two keys: ``'converged'``, a Bool which of whether optimization stopped due to convergence (True) or due to max number of iterations (False), and ``'loop_values'``, a [n_loop+1, N] ``numpy.ndarray`` of the state's values, at the start of optimization and at the end of each loop, before the line minimization.

Response:
def finish(s, desc='finish', n_loop=4, max_mem=1e9, separate_psf=True,
        fractol=1e-7, errtol=1e-3, dowarn=True):
    """
    Crawls slowly to the minimum-cost state.

    Blocks the global parameters into small enough sections such that each
    can be optimized separately while including all the pixels (i.e. no
    decimation). Optimizes the globals, then the psf separately if desired,
    then particles, then a line minimization along the step direction to
    speed up convergence.

    Parameters
    ----------
        s : :class:`peri.states.ImageState`
            The state to optimize
        desc : string, optional
            Description to append to the states.save() call every loop.
            Set to `None` to avoid saving. Default is `'finish'`.
        n_loop : Int, optional
            The number of times to loop over in the optimizer. Default is 4.
        max_mem : Numeric, optional
            The maximum amount of memory allowed for the optimizers' J's,
            for both particles & globals. Default is 1e9.
        separate_psf : Bool, optional
            If True, does the psf optimization separately from the rest of
            the globals, since the psf has a more tortuous fit landscape.
            Default is True.
        fractol : Float, optional
            Fractional change in error at which to terminate. Default 1e-4
        errtol : Float, optional
            Absolute change in error at which to terminate. Default 1e-2
        dowarn : Bool, optional
            Whether to log a warning if termination results from finishing
            loops rather than from convergence. Default is True.

    Returns
    -------
        dictionary
            Information about the optimization. Has two keys: ``'converged'``,
            a Bool which of whether optimization stopped due to convergence
            (True) or due to max number of iterations (False), and
            ``'loop_values'``, a [n_loop+1, N] ``numpy.ndarray`` of the
            state's values, at the start of optimization and at the end of
            each loop, before the line minimization.
    """
    values = [np.copy(s.state[s.params])]
    remove_params = s.get('psf').params if separate_psf else None
    # FIXME explicit params
    global_params = name_globals(s, remove_params=remove_params)
    #FIXME this could be done much better, since much of the globals such
    #as the ilm are local. Could be done with sparse matrices and/or taking
    #nearby globals in a group and using the update tile only as the slicer,
    #rather than the full residuals.
    gs = np.floor(max_mem / s.residuals.nbytes).astype('int')
    groups = [global_params[a:a+gs] for a in range(0, len(global_params), gs)]
    CLOG.info('Start  ``finish``:\t{}'.format(s.error))
    for a in range(n_loop):
        start_err = s.error
        #1. Min globals:
        for g in groups:
            do_levmarq(s, g, damping=0.1, decrease_damp_factor=20.,
                    max_iter=1, max_mem=max_mem, eig_update=False)
        if separate_psf:
            do_levmarq(s, remove_params, max_mem=max_mem, max_iter=4,
                    eig_update=False)
        CLOG.info('Globals,   loop {}:\t{}'.format(a, s.error))
        if desc is not None:
            states.save(s, desc=desc)
        #2. Min particles
        do_levmarq_all_particle_groups(s, max_iter=1, max_mem=max_mem)
        CLOG.info('Particles, loop {}:\t{}'.format(a, s.error))
        if desc is not None:
            states.save(s, desc=desc)
        #3. Append vals, line min:
        values.append(np.copy(s.state[s.params]))
        # dv = (np.array(values[1:]) - np.array(values[0]))[-3:]
        # do_levmarq_n_directions(s, dv, damping=1e-2, max_iter=2, errtol=3e-4)
        # CLOG.info('Line min., loop {}:\t{}'.format(a, s.error))
        # if desc is not None:
            # states.save(s, desc=desc)
        #4. terminate?
        new_err = s.error
        derr = start_err - new_err
        dobreak = (derr/new_err < fractol) or (derr < errtol)
        if dobreak:
            break

    if dowarn and (not dobreak):
        CLOG.warn('finish() did not converge; consider re-running')
    return {'converged':dobreak, 'loop_values':np.array(values)}


Instruction:
Create a Python 3 function to
fit a new component to an old component and returns a new object.

Response:
def fit_comp(new_comp, old_comp, **kwargs):
    """
    Fits a new component to an old component

    Calls do_levmarq to match the .get() fields of the two objects. The
    parameters of new_comp are modified in place.

    Parameters
    ----------
    new_comp : :class:`peri.comps.comp`
        The new object, whose parameters to update to fit the field of
        `old_comp`. Must have a .get() attribute which returns an ndarray
    old_comp : peri.comp
        The old ilm to match to.

    Other Parameters
    ----------------
        Any keyword arguments to be passed to the optimizer LMGlobals
        through do_levmarq.

    See Also
    --------
    do_levmarq : Levenberg-Marquardt minimization using a random subset
        of the image pixels.
    """
    #resetting the category to ilm:
    new_cat = new_comp.category
    new_comp.category = 'ilm'
    fake_s = states.ImageState(Image(old_comp.get().copy()), [new_comp], pad=0,
            mdl=mdl.SmoothFieldModel())
    do_levmarq(fake_s, new_comp.params, **kwargs)
    new_comp.category = new_cat


Instruction:
Implement a Python 3 function for
resetting all the internal state of the object to its initial state.

Response:
def reset(self, new_damping=None):
        """
        Keeps all user supplied options the same, but resets counters etc.
        """
        self._num_iter = 0
        self._inner_run_counter = 0
        self._J_update_counter = self.update_J_frequency
        self._fresh_JTJ = False
        self._has_run = False
        if new_damping is not None:
            self.damping = np.array(new_damping).astype('float')
        self._set_err_paramvals()


Instruction:
Can you generate the documentation for the following Python 3 function
def do_run_1(self):
        while not self.check_terminate():
            self._has_run = True
            self._run1()
            self._num_iter += 1; self._inner_run_counter += 1

Response: Run the LM run until the run is complete.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def do_run_2(self):
        while not self.check_terminate():
            self._has_run = True
            self._run2()
            self._num_iter += 1

Response: Runs 2 steps until the process terminate.


Instruction:
Make a summary of the following Python 3 code
def find_LM_updates(self, grad, do_correct_damping=True, subblock=None):
        if subblock is not None:
            if (subblock.sum() == 0) or (subblock.size == 0):
                CLOG.fatal('Empty subblock in find_LM_updates')
                raise ValueError('Empty sub-block')
            j = self.J[subblock]
            JTJ = np.dot(j, j.T)
            damped_JTJ = self._calc_damped_jtj(JTJ, subblock=subblock)
            grad = grad[subblock]  #select the subblock of the grad
        else:
            damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)

        delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)

        if self.use_accel:
            accel_correction = self.calc_accel_correction(damped_JTJ, delta)
            nrm_d0 = np.sqrt(np.sum(delta**2))
            nrm_corr = np.sqrt(np.sum(accel_correction**2))
            CLOG.debug('|correction| / |LM step|\t%e' % (nrm_corr/nrm_d0))
            if nrm_corr/nrm_d0 < self.max_accel_correction:
                delta += accel_correction
            elif do_correct_damping:
                CLOG.debug('Untrustworthy step! Increasing damping...')
                self.increase_damping()
                damped_JTJ = self._calc_damped_jtj(self.JTJ, subblock=subblock)
                delta = self._calc_lm_step(damped_JTJ, grad, subblock=subblock)

        if np.any(np.isnan(delta)):
            CLOG.fatal('Calculated steps have nans!?')
            raise FloatingPointError('Calculated steps have nans!?')
        return delta

Response: Calculates the LM updates for the current state of the object.


Instruction:
How would you explain what the following Python 3 function does
def _calc_lm_step(self, damped_JTJ, grad, subblock=None):
        delta0, res, rank, s = np.linalg.lstsq(damped_JTJ, -0.5*grad,
                rcond=self.min_eigval)
        if self._fresh_JTJ:
            CLOG.debug('%d degenerate of %d total directions' % (
                    delta0.size-rank, delta0.size))
        if subblock is not None:
            delta = np.zeros(self.J.shape[0])
            delta[subblock] = delta0
        else:
            delta = delta0.copy()
        return delta

Response: Calculates a Levenberg - Marquard step w / o acceleration


Instruction:
Explain what the following Python 3 code does
def update_param_vals(self, new_vals, incremental=False):
        self._last_vals = self.param_vals.copy()
        if incremental:
            self.param_vals += new_vals
        else:
            self.param_vals = new_vals.copy()
        #And we've updated, so JTJ is no longer valid:
        self._fresh_JTJ = False

Response: Updates the current set of parameter values and previous values.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def find_expected_error(self, delta_params='calc'):
        grad = self.calc_grad()
        if list(delta_params) in [list('calc'), list('perfect')]:
            jtj = (self.JTJ if delta_params == 'perfect' else
                    self._calc_damped_jtj(self.JTJ))
            delta_params = self._calc_lm_step(jtj, self.calc_grad())
        #If the model were linear, then the cost would be quadratic,
        #with Hessian 2*`self.JTJ` and gradient `grad`
        expected_error = (self.error + np.dot(grad, delta_params) +
                np.dot(np.dot(self.JTJ, delta_params), delta_params))
        return expected_error

Response: Calculates the expected error after an update.


Instruction:
Create a Python 3 function to
calculate the cosine of the residuals with the model.

Response:
def calc_model_cosine(self, decimate=None, mode='err'):
        """
        Calculates the cosine of the residuals with the model.

        Parameters
        ----------
            decimate : Int or None, optional
                Decimate the residuals by `decimate` pixels. If None, no
                decimation is used. Valid only with mode='svd'. Default
                is None
            mode : {'svd', 'err'}
                Which mode to use; see Notes section. Default is 'err'.

        Returns
        -------
            abs_cos : numpy.float64
                The absolute value of the model cosine.

        Notes
        -----
        The model cosine is defined in terms of the geometric view of
        curve-fitting, as a model manifold embedded in a high-dimensional
        space. The model cosine is the cosine of the residuals vector
        with its projection on the tangent space: :math:`cos(phi) = |P^T r|/|r|`
        where :math:`P^T` is the projection operator onto the model manifold
        and :math:`r` the residuals. This can be calculated two ways: By
        calculating the projection operator P directly with SVD (mode=`svd`),
        or by using the expected error if the model were linear to calculate
        a model sine first (mode=`err`). Since the SVD of a large matrix is
        slow, mode=`err` is faster.

        `decimate` allows for every nth pixel only to be counted in the
        SVD matrix of J for speed. While this is n x faster, it is
        considerably less accurate, so the default is no decimation.
        """
        if mode == 'svd':
            slicer = slice(0, None, decimate)

            #1. Calculate projection term
            u, sig, v = np.linalg.svd(self.J[:,slicer], full_matrices=False) #slow part
            # p = np.dot(v.T, v) - memory error, so term-by-term
            r = self.calc_residuals()[slicer]
            abs_r = np.sqrt((r*r).sum())

            v_r = np.dot(v,r/abs_r)
            projected = np.dot(v.T, v_r)

            abs_cos = np.sqrt((projected*projected).sum())
        elif mode == 'err':
            expected_error = self.find_expected_error(delta_params='perfect')
            model_sine_2 = expected_error / self.error  #error = distance^2
            abs_cos = np.sqrt(1 - model_sine_2)
        else:
            raise ValueError('mode must be one of `svd`, `err`')
        return abs_cos


Instruction:
Write a Python 3 script to
return a dict of termination statistics for the current J.

Response:
def get_termination_stats(self, get_cos=True):
        """
        Returns a dict of termination statistics

        Parameters
        ----------
            get_cos : Bool, optional
                Whether or not to calcualte the cosine of the residuals
                with the tangent plane of the model using the current J.
                The calculation may take some time. Default is True

        Returns
        -------
            dict
                Has keys
                    delta_vals  : The last change in parameter values.
                    delta_err   : The last change in the error.
                    exp_err     : The expected (last) change in the error.
                    frac_err    : The fractional change in the error.
                    num_iter    : The number of iterations completed.
                    error       : The current error.
        """
        delta_vals = self._last_vals - self.param_vals
        delta_err = self._last_error - self.error
        frac_err = delta_err / self.error
        to_return = {'delta_vals':delta_vals, 'delta_err':delta_err,
                'num_iter':1*self._num_iter, 'frac_err':frac_err,
                'error':self.error, 'exp_err':self._exp_err}
        if get_cos:
            model_cosine = self.calc_model_cosine()
            to_return.update({'model_cosine':model_cosine})
        return to_return


Instruction:
Can you implement a function in Python 3 that
checks if the algorithm has found a satisfactory minimum

Response:
def check_completion(self):
        """
        Returns a Bool of whether the algorithm has found a satisfactory minimum
        """
        terminate = False
        term_dict = self.get_termination_stats(get_cos=self.costol is not None)
        terminate |= np.all(np.abs(term_dict['delta_vals']) < self.paramtol)
        terminate |= (term_dict['delta_err'] < self.errtol)
        terminate |= (term_dict['exp_err'] < self.exptol)
        terminate |= (term_dict['frac_err'] < self.fractol)
        if self.costol is not None:
            terminate |= (curcos < term_dict['model_cosine'])

        return terminate


Instruction:
Here you have a function in Python 3, explain what it does
def check_terminate(self):
        if not self._has_run:
            return False
        else:
            #1-3. errtol, paramtol, model cosine low enough?
            terminate = self.check_completion()

            #4. too many iterations??
            terminate |= (self._num_iter >= self.max_iter)
            return terminate

Response: Checks whether a satisfactory minimum has been found and whether to terminate.


Instruction:
Given the following Python 3 function, write the documentation
def check_update_J(self):
        self._J_update_counter += 1
        update = self._J_update_counter >= self.update_J_frequency
        return update & (not self._fresh_JTJ)

Response: Checks if the full J should be updated.


Instruction:
Given the following Python 3 function, write the documentation
def update_J(self):
        self.calc_J()
        # np.dot(j, j.T) is slightly faster but 2x as much mem
        step = np.ceil(1e-2 * self.J.shape[1]).astype('int')  # 1% more mem...
        self.JTJ = low_mem_sq(self.J, step=step)
        #copies still, since J is not C -ordered but a slice of j_e...
        #doing self.J.copy() works but takes 2x as much ram..
        self._fresh_JTJ = True
        self._J_update_counter = 0
        if np.any(np.isnan(self.JTJ)):
            raise FloatingPointError('J, JTJ have nans.')
        #Update self._exp_err
        self._exp_err = self.error - self.find_expected_error(delta_params='perfect')

Response: Updates J TJ and internal counters.


Instruction:
Explain what the following Python 3 code does
def calc_grad(self):
        residuals = self.calc_residuals()
        return 2*np.dot(self.J, residuals)

Response: Calculates the gradient of the cost w. r. t. the parameters.


Instruction:
Write a Python 3 function for
updating the internal J matrix with the new values.

Response:
def _rank_1_J_update(self, direction, values):
        """
        Does J += np.outer(direction, new_values - old_values) without
        using lots of memory
        """
        vals_to_sub = np.dot(direction, self.J)
        delta_vals = values - vals_to_sub
        for a in range(direction.size):
            self.J[a] += direction[a] * delta_vals


Instruction:
How would you explain what the following Python 3 function does
def update_Broyden_J(self):
        CLOG.debug('Broyden update.')
        delta_vals = self.param_vals - self._last_vals
        delta_residuals = self.calc_residuals() - self._last_residuals
        nrm = np.sqrt(np.dot(delta_vals, delta_vals))
        direction = delta_vals / nrm
        vals = delta_residuals / nrm
        self._rank_1_J_update(direction, vals)
        self.JTJ = np.dot(self.J, self.J.T)

Response: Execute a Broyden update of J


Instruction:
Write a Python 3 script to
execute an eigen update of J

Response:
def update_eig_J(self):
        """Execute an eigen update of J"""
        CLOG.debug('Eigen update.')
        vls, vcs = np.linalg.eigh(self.JTJ)
        res0 = self.calc_residuals()
        for a in range(min([self.num_eig_dirs, vls.size])):
            #1. Finding stiff directions
            stif_dir = vcs[-(a+1)] #already normalized

            #2. Evaluating derivative along that direction, we'll use dl=5e-4:
            dl = self.eig_dl #1e-5
            _ = self.update_function(self.param_vals + dl*stif_dir)
            res1 = self.calc_residuals()

            #3. Updating
            grad_stif = (res1-res0)/dl
            self._rank_1_J_update(stif_dir, grad_stif)

        self.JTJ = np.dot(self.J, self.J.T)
        #Putting the parameters back:
        _ = self.update_function(self.param_vals)


Instruction:
Explain what the following Python 3 code does
def calc_accel_correction(self, damped_JTJ, delta0):
        #Get the derivative:
        _ = self.update_function(self.param_vals)
        rm0 = self.calc_residuals().copy()
        _ = self.update_function(self.param_vals + delta0)
        rm1 = self.calc_residuals().copy()
        _ = self.update_function(self.param_vals - delta0)
        rm2 = self.calc_residuals().copy()
        der2 = (rm2 + rm1 - 2*rm0)

        corr, res, rank, s = np.linalg.lstsq(damped_JTJ, np.dot(self.J, der2),
                rcond=self.min_eigval)
        corr *= -0.5
        return corr

Response: Calculates the acceleration correction for a given set of geodesic acceleration in the LM step.


Instruction:
Create a Python 3 function for
updating the J matrix with the values of the parameters in the boolean taxonomy.

Response:
def update_select_J(self, blk):
        """
        Updates J only for certain parameters, described by the boolean
        mask `blk`.
        """
        p0 = self.param_vals.copy()
        self.update_function(p0)  #in case things are not put back...
        r0 = self.calc_residuals().copy()
        dl = np.zeros(p0.size, dtype='float')
        blk_J = []
        for i in np.nonzero(blk)[0]:
            dl *= 0; dl[i] = self.eig_dl
            self.update_function(p0 + dl)
            r1 = self.calc_residuals().copy()
            blk_J.append((r1-r0)/self.eig_dl)
        self.J[blk] = np.array(blk_J)
        self.update_function(p0)
        #Then we also need to update JTJ:
        self.JTJ = np.dot(self.J, self.J.T)
        if np.any(np.isnan(self.J)) or np.any(np.isnan(self.JTJ)):
            raise FloatingPointError('J, JTJ have nans.')


Instruction:
Implement a Python 3 function for
setting error and param_vals for the exception.

Response:
def _set_err_paramvals(self):
        """
        Must update:
            self.error, self._last_error, self.param_vals, self._last_vals
        """
        # self.param_vals = p0 #sloppy...
        self._last_vals = self.param_vals.copy()
        self.error = self.update_function(self.param_vals)
        self._last_error = (1 + 2*self.fractol) * self.error


Instruction:
Can you generate a brief explanation for the following Python 3 code
def calc_J(self):
        del self.J
        self.J = np.zeros([self.param_vals.size, self.data.size])
        dp = np.zeros_like(self.param_vals)
        f0 = self.model.copy()
        for a in range(self.param_vals.size):
            dp *= 0
            dp[a] = self.dl[a]
            f1 = self.func(self.param_vals + dp, *self.func_args, **self.func_kwargs)
            grad_func = (f1 - f0) / dp[a]
            #J = grad(residuals) = -grad(model)
            self.J[a] = -grad_func

Response: Updates self. J and returns nothing


Instruction:
Write a Python 3 script for
taking an array param_vals updates function returns the new error

Response:
def update_function(self, param_vals):
        """Takes an array param_vals, updates function, returns the new error"""
        self.model = self.func(param_vals, *self.func_args, **self.func_kwargs)
        d = self.calc_residuals()
        return np.dot(d.flat, d.flat)


Instruction:
Here you have a function in Python 3, explain what it does
def update_function(self, param_vals):
        self.opt_obj.update_function(param_vals)
        return self.opt_obj.get_error()

Response: Updates the opt_obj returns new error.


Instruction:
Can you tell what is the following Python 3 function doing
def update_function(self, param_vals):
        dp = np.zeros(self.p0.size)
        for a in range(param_vals.size):
            dp += param_vals[a] * self.directions[a]
        self.state.update(self.state.params, self.p0 + dp)
        self.param_vals[:] = param_vals
        return None

Response: Updates with param_vals = distance from self. p0 along self. direction.


Instruction:
How would you explain what the following Python 3 function does
def calc_J(self):
        r0 = self.state.residuals.copy().ravel()
        dl = np.zeros(self.param_vals.size)
        p0 = self.param_vals.copy()
        J = []
        for a in range(self.param_vals.size):
            dl *= 0
            dl[a] += self.dl
            self.update_function(p0 + dl)
            r1 = self.state.residuals.copy().ravel()
            J.append( (r1-r0)/self.dl)
        self.update_function(p0)
        return np.array(J)

Response: Calculates the J along the direction.


Instruction:
How would you explain what the following Python 3 function does
def update_select_J(self, blk):
        self.update_function(self.param_vals)
        params = np.array(self.param_names)[blk].tolist()
        blk_J = -self.state.gradmodel(params=params, inds=self._inds, flat=False)
        self.J[blk] = blk_J
        #Then we also need to update JTJ:
        self.JTJ = np.dot(self.J, self.J.T)
        if np.any(np.isnan(self.J)) or np.any(np.isnan(self.JTJ)):
            raise FloatingPointError('J, JTJ have nans.')

Response:
Updates J only for certain parameters described by the boolean
        mask blk.


Instruction:
Write a Python 3 script for
finding the expected error after an update.

Response:
def find_expected_error(self, delta_params='calc', adjust=True):
        """
        Returns the error expected after an update if the model were linear.

        Parameters
        ----------
            delta_params : {numpy.ndarray, 'calc', or 'perfect'}, optional
                The relative change in parameters. If 'calc', uses update
                calculated from the current damping, J, etc; if 'perfect',
                uses the update calculated with zero damping.

        Returns
        -------
            numpy.float64
                The expected error after the update with `delta_params`
        """
        expected_error = super(LMGlobals, self).find_expected_error(
                delta_params=delta_params)
        if adjust:
            #adjust for num_pix
            derr = (expected_error - self.error) * (self.state.residuals.size /
                    float(self.num_pix))
            expected_error = self.error + derr
        return expected_error


Instruction:
Here you have a function in Python 3, explain what it does
def calc_model_cosine(self, decimate=None, mode='err'):
        #we calculate the model cosine only in the data space of the
        #sampled indices
        if mode == 'err':
            expected_error = self.find_expected_error(delta_params='perfect',
                    adjust=False)
            derr = self.error - expected_error
            residuals_err = lambda r: np.dot(r,r).sum()
            current_partial_error = residuals_err(self.calc_residuals())
            expected_partial_error = current_partial_error - derr
            model_sine_2 = expected_partial_error / current_partial_error
            abs_cos = np.sqrt(1 - model_sine_2)
        else:
            #superclass is fine
            abs_cos = super(self.__class__, self).calc_model_cosine(decimate=
                    decimate, mode=mode)
        return abs_cos

Response: Calculates the model cosine of the residuals with the model.


Instruction:
Create a Python 3 function to
calculate the gradient of the cost w. r. t. the parameters.

Response:
def calc_grad(self):
        """The gradient of the cost w.r.t. the parameters."""
        if self._fresh_JTJ:
            return self._graderr
        else:
            residuals = self.calc_residuals()
            return 2*np.dot(self.J, residuals)


Instruction:
Create a Python 3 function to
reset the particle groups and optionally the region size and damping of the particle groups.

Response:
def reset(self, new_region_size=None, do_calc_size=True, new_damping=None,
            new_max_mem=None):
        """
        Resets the particle groups and optionally the region size and damping.

        Parameters
        ----------
            new_region_size : : Int or 3-element list-like of ints, optional
                The region size for sub-blocking particles. Default is 40
            do_calc_size : Bool, optional
                If True, calculates the region size internally based on
                the maximum allowed memory. Default is True
            new_damping : Float or None, optional
                The new damping of the optimizer. Set to None to leave
                as the default for LMParticles. Default is None.
            new_max_mem : Numeric, optional
                The maximum allowed memory for J to occupy. Default is 1e9
        """
        if new_region_size is not None:
            self.region_size = new_region_size
        if new_max_mem != None:
            self.max_mem = new_max_mem
        if do_calc_size:
            self.region_size = calc_particle_group_region_size(self.state,
                    region_size=self.region_size, max_mem=self.max_mem)
        self.stats = []
        self.particle_groups = separate_particles_into_groups(self.state,
                self.region_size, doshift='rand')
        if new_damping is not None:
            self._kwargs.update({'damping':new_damping})
        if self.save_J:
            if len(self.particle_groups) > 90:
                CLOG.warn('Attempting to create many open files. Consider increasing max_mem and/or region_size to avoid crashes.')
            self._tempfiles = []
            self._has_saved_J = []
            for a in range(len(self.particle_groups)):
                #TemporaryFile is automatically deleted
                for _ in ['j','tile']:
                    self._tempfiles.append(tempfile.TemporaryFile(dir=os.getcwd()))
                self._has_saved_J.append(False)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _do_run(self, mode='1'):
        for a in range(len(self.particle_groups)):
            group = self.particle_groups[a]
            lp = LMParticles(self.state, group, **self._kwargs)
            if mode == 'internal':
                lp.J, lp.JTJ, lp._dif_tile = self._load_j_diftile(a)

            if mode == '1':
                lp.do_run_1()
            if mode == '2':
                lp.do_run_2()
            if mode == 'internal':
                lp.do_internal_run()

            self.stats.append(lp.get_termination_stats(get_cos=self.get_cos))
            if self.save_J and (mode != 'internal'):
                self._dump_j_diftile(a, lp.J, lp._dif_tile)
                self._has_saved_J[a] = True

Response: workhorse for the self. do_run_xx methods.


Instruction:
Create a Python 3 function for
calling self. _do_internal_run for each group of particles.

Response:
def do_internal_run(self):
        """Calls LMParticles.do_internal_run for each group of particles."""
        if not self.save_J:
            raise RuntimeError('self.save_J=True required for do_internal_run()')
        if not np.all(self._has_saved_J):
            raise RuntimeError('J, JTJ have not been pre-computed. Call do_run_1 or do_run_2')
        self._do_run(mode='internal')


Instruction:
Can you tell what is the following Python 3 function doing
def reset(self):
        inds = list(range(self.state.obj_get_positions().shape[0]))
        self._rad_nms = self.state.param_particle_rad(inds)
        self._pos_nms = self.state.param_particle_pos(inds)
        self._initial_rad = np.copy(self.state.state[self._rad_nms])
        self._initial_pos = np.copy(self.state.state[self._pos_nms]).reshape((-1,3))
        self.param_vals[self.rscale_mask] = 0

Response: Resets the initial radii used for updating the particles.


Instruction:
Implement a Python 3 function for
righting now legval z

Response:
def _poly(self, z):
        """Right now legval(z)"""
        shp = self.state.oshape.shape
        zmax = float(shp[0])
        zmin = 0.0
        zmid = zmax * 0.5

        coeffs = self.param_vals[self.rscale_mask].copy()
        if coeffs.size == 0:
            ans = 0*z
        else:
            ans = np.polynomial.legendre.legval((z-zmid)/zmid,
                    self.param_vals[self.rscale_mask])
        return ans


Instruction:
Given the following Python 3 function, write the documentation
def update(self, param_vals):
        self.update_rscl_x_params(param_vals[self.rscale_mask])
        self.state.update(self.param_names, param_vals[self.globals_mask])
        self.param_vals[:] = param_vals.copy()
        if np.any(np.isnan(self.state.residuals)):
            raise FloatingPointError('state update caused nans in residuals')

Response: Updates all the parameters of the state + rscale x


Instruction:
Can you tell what is the following Python 3 function doing
def reset(self, **kwargs):
        self.aug_state.reset()
        super(LMAugmentedState, self).reset(**kwargs)

Response: Resets the aug_state and the LMEngine


Instruction:
Write a Python 3 function that can
return a list of objects with a numbers of shares a link has had using Buffer.

Response:
def get_shares(self):
    '''
      Returns an object with a the numbers of shares a link has had using
      Buffer.

      www will be stripped, but other subdomains will not.
    '''

    self.shares = self.api.get(url=PATHS['GET_SHARES'] % self.url)['shares']

    return self.shares


Instruction:
Write a Python 3 script to
take a sample from a field given flat indices or a shaped slice

Response:
def sample(field, inds=None, slicer=None, flat=True):
    """
    Take a sample from a field given flat indices or a shaped slice

    Parameters
    -----------
    inds : list of indices
        One dimensional (raveled) indices to return from the field

    slicer : slice object
        A shaped (3D) slicer that returns a section of image

    flat : boolean
        Whether to flatten the sampled item before returning
    """
    if inds is not None:
        out = field.ravel()[inds]
    elif slicer is not None:
        out = field[slicer].ravel()
    else:
        out = field

    if flat:
        return out.ravel()
    return out


Instruction:
Can you generate the documentation for the following Python 3 function
def save(state, filename=None, desc='', extra=None):
    if isinstance(state.image, util.RawImage):
        desc = desc or 'save'
        filename = filename or state.image.filename + '-peri-' + desc + '.pkl'
    else:
        if not filename:
            raise AttributeError("Must provide filename since RawImage is not used")

    if extra is None:
        save = state
    else:
        save = [state] + extra

    if os.path.exists(filename):
        ff = "{}-tmp-for-copy".format(filename)

        if os.path.exists(ff):
            os.remove(ff)

        os.rename(filename, ff)

    pickle.dump(save, open(filename, 'wb'), protocol=2)

Response: Save the current state of the state object to a file.


Instruction:
Can you generate the documentation for the following Python 3 function
def load(filename):
    path, name = os.path.split(filename)
    path = path or '.'

    with util.indir(path):
        return pickle.load(open(name, 'rb'))

Response: Load the state from a file.


Instruction:
How would you explain what the following Python 3 function does
def error(self):
        r = self.residuals.ravel()
        return np.dot(r,r)

Response: Sum of the squared errors


Instruction:
Make a summary of the following Python 3 code
def loglikelihood(self):
        sig = self.hyper_parameters.get_values('sigma')
        err = self.error
        N = np.size(self.data)
        return -0.5*err/sig**2 - np.log(np.sqrt(2*np.pi)*sig)*N

Response: Loglikelihood calculated by the model error


Instruction:
Write a Python 3 script to
update a single parameter or group of parameters params with values.

Response:
def update(self, params, values):
        """
        Update a single parameter or group of parameters ``params``
        with ``values``.

        Parameters
        ----------
        params : string or list of strings
            Parameter names which to update

        value : number or list of numbers
            Values of those parameters which to update
        """
        return super(State, self).update(params, values)


Instruction:
Write a Python 3 function for
performing a parameter update and keep track of the change on the state.

Response:
def push_update(self, params, values):
        """
        Perform a parameter update and keep track of the change on the state.
        Same call structure as :func:`peri.states.States.update`
        """
        curr = self.get_values(params)
        self.stack.append((params, curr))
        self.update(params, values)


Instruction:
Write a Python 3 function for
popping the last update from the stack push by the PeriState. update method.

Response:
def pop_update(self):
        """
        Pop the last update from the stack push by
        :func:`peri.states.States.push_update` by undoing the chnage last
        performed.
        """
        params, values = self.stack.pop()
        self.update(params, values)


Instruction:
Explain what the following Python 3 code does
def _hess_two_param(self, funct, p0, p1, dl=2e-5, rts=False, **kwargs):
        vals0 = self.get_values(p0)
        vals1 = self.get_values(p1)

        f00 = funct(**kwargs)

        self.update(p0, vals0+dl)
        f10 = funct(**kwargs)

        self.update(p1, vals1+dl)
        f11 = funct(**kwargs)

        self.update(p0, vals0)
        f01 = funct(**kwargs)

        if rts:
            self.update(p0, vals0)
            self.update(p1, vals1)
        return (f11 - f10 - f01 + f00) / (dl**2)

Response: Hessian of funct wrt two parameters p0 and p1.


Instruction:
Explain what the following Python 3 code does
def _jtj(self, funct, params=None, dl=2e-5, rts=False, **kwargs):
        grad = self._grad(funct=funct, params=params, dl=dl, rts=rts, **kwargs)
        return np.dot(grad, grad.T)

Response: get the JTJ of a function wrt to parmaeters params


Instruction:
Can you generate the documentation for the following Python 3 function
def build_funcs(self):
        # create essentially lambda functions, but with a nice signature
        def m(inds=None, slicer=None, flat=True):
            return sample(self.model, inds=inds, slicer=slicer, flat=flat).copy()

        def r(inds=None, slicer=None, flat=True):
            return sample(self.residuals, inds=inds, slicer=slicer, flat=flat).copy()

        def l():
            return self.loglikelihood

        def r_e(**kwargs):
            """sliced etc residuals, with state.error appended on"""
            return r(**kwargs), np.copy(self.error)

        def m_e(**kwargs):
            """sliced etc residuals, with state.error appended on"""
            return m(**kwargs), np.copy(self.error)

        # set the member functions using partial
        self.fisherinformation = partial(self._jtj, funct=m)
        self.gradloglikelihood = partial(self._grad, funct=l)
        self.hessloglikelihood = partial(self._hess, funct=l)
        self.gradmodel = partial(self._grad, funct=m)
        self.hessmodel = partial(self._hess, funct=m)
        self.JTJ = partial(self._jtj, funct=r)
        self.J = partial(self._grad, funct=r)
        self.J_e = partial(self._grad, funct=r_e, nout=2)
        self.gradmodel_e = partial(self._grad, funct=m_e, nout=2)

        # add the appropriate documentation to the following functions
        self.fisherinformation.__doc__ = _graddoc + _sampledoc
        self.gradloglikelihood.__doc__ = _graddoc
        self.hessloglikelihood.__doc__ = _graddoc
        self.gradmodel.__doc__ = _graddoc + _sampledoc
        self.hessmodel.__doc__ = _graddoc + _sampledoc
        self.JTJ.__doc__ = _graddoc + _sampledoc
        self.J.__doc__ = _graddoc + _sampledoc

        # add documentation to the private functions as well. this is done
        # slightly differently, hence the function call
        self._dograddoc(self._grad_one_param)
        self._dograddoc(self._hess_two_param)
        self._dograddoc(self._grad)
        self._dograddoc(self._hess)

        # the state object is a workaround so that other interfaces still
        # work. this should probably be removed in the long run
        class _Statewrap(object):
            def __init__(self, obj):
                self.obj = obj
            def __getitem__(self, d=None):
                if d is None:
                    d = self.obj.params
                return util.delistify(self.obj.get_values(d), d)

        self.state = _Statewrap(self)

Response: Builds the gradients and hessian functions based on the properties of the state.


Instruction:
Write a Python 3 script for
calculating the diagonal elements of the minimum covariance of the model containing the model with respect to parameters params.

Response:
def crb(self, params=None, *args, **kwargs):
        """
        Calculate the diagonal elements of the minimum covariance of the model
        with respect to parameters params. ``*args`` and ``**kwargs`` go to
        ``fisherinformation``.
        """
        fish = self.fisherinformation(params=params, *args, **kwargs)
        return np.sqrt(np.diag(np.linalg.inv(fish))) * self.sigma


Instruction:
Write a Python 3 script to
set the model of the image.

Response:
def set_model(self, mdl):
        """
        Setup the image model formation equation and corresponding objects into
        their various objects. `mdl` is a `peri.models.Model` object
        """
        self.mdl = mdl
        self.mdl.check_inputs(self.comps)

        for c in self.comps:
            setattr(self, '_comp_'+c.category, c)


Instruction:
Can you create a Python 3 function that
updates the current comparison ( real ) image

Response:
def set_image(self, image):
        """
        Update the current comparison (real) image
        """
        if isinstance(image, np.ndarray):
            image = util.Image(image)

        if isinstance(image, util.NullImage):
            self.model_as_data = True
        else:
            self.model_as_data = False

        self.image = image
        self._data = self.image.get_padded_image(self.pad)

        # set up various slicers and Tiles associated with the image and pad
        self.oshape = util.Tile(self._data.shape)
        self.ishape = self.oshape.pad(-self.pad)
        self.inner = self.ishape.slicer

        for c in self.comps:
            c.set_shape(self.oshape, self.ishape)

        self._model = np.zeros(self._data.shape, dtype=np.float64)
        self._residuals = np.zeros(self._data.shape, dtype=np.float64)
        self.calculate_model()


Instruction:
Write a Python 3 script for
switching out the data for the model s recreation of the data.

Response:
def model_to_data(self, sigma=0.0):
        """ Switch out the data for the model's recreation of the data. """
        im = self.model.copy()
        im += sigma*np.random.randn(*im.shape)
        self.set_image(util.NullImage(image=im))


Instruction:
Can you generate the documentation for the following Python 3 function
def get_update_io_tiles(self, params, values):
        # get the affected area of the model image
        otile = self.get_update_tile(params, values)
        if otile is None:
            return [None]*3
        ptile = self.get_padding_size(otile) or util.Tile(0, dim=otile.dim)

        otile = util.Tile.intersection(otile, self.oshape)

        if (otile.shape <= 0).any():
            raise UpdateError("update triggered invalid tile size")

        if (ptile.shape < 0).any() or (ptile.shape > self.oshape.shape).any():
            raise UpdateError("update triggered invalid padding tile size")

        # now remove the part of the tile that is outside the image and pad the
        # interior part with that overhang. reflect the necessary padding back
        # into the image itself for the outer slice which we will call outer
        outer = otile.pad((ptile.shape+1)//2)
        inner, outer = outer.reflect_overhang(self.oshape)
        iotile = inner.translate(-outer.l)

        outer = util.Tile.intersection(outer, self.oshape)
        inner = util.Tile.intersection(inner, self.oshape)
        return outer, inner, iotile

Response: Get the tile that needs to be updated. Inputs are the parameters and values. Returns the tile that needs to be updated.