Instruction:
Can you generate a brief explanation for the following Python 3 code
def poisson_equation(image, gradient=1, max_iter=100, convergence=.01, percentile = 90.0):
    '''Estimate the solution to the Poisson Equation

    The Poisson Equation is the solution to gradient(x) = h^2/4 and, in this
    context, we use a boundary condition where x is zero for background
    pixels. Also, we set h^2/4 = 1 to indicate that each pixel is a distance
    of 1 from its neighbors.

    The estimation exits after max_iter iterations or if the given percentile
    of foreground pixels differ by less than the convergence fraction
    from one pass to the next.

    Some ideas taken from Gorelick, "Shape representation and classification
    using the Poisson Equation", IEEE Transactions on Pattern Analysis and
    Machine Intelligence V28, # 12, 2006

    image - binary image with foreground as True
    gradient - the target gradient between 4-adjacent pixels
    max_iter - maximum # of iterations at a given level
    convergence - target fractional difference between values from previous
                  and next pass
    percentile - measure convergence at this percentile
    '''
    # Evaluate the poisson equation with zero-padded boundaries
    pe = np.zeros((image.shape[0]+2, image.shape[1]+2))
    if image.shape[0] > 64 and image.shape[1] > 64:
        #
        # Sub-sample to get seed values
        #
        sub_image = image[::2, ::2]
        sub_pe = poisson_equation(sub_image,
                                  gradient=gradient*2,
                                  max_iter=max_iter,
                                  convergence=convergence)
        coordinates = np.mgrid[0:(sub_pe.shape[0]*2),
                               0:(sub_pe.shape[1]*2)].astype(float) / 2
        pe[1:(sub_image.shape[0]*2+1), 1:(sub_image.shape[1]*2+1)] = \
            scind.map_coordinates(sub_pe, coordinates, order=1)
        pe[:image.shape[0], :image.shape[1]][~image] = 0
    else:
        pe[1:-1,1:-1] = image
    #
    # evaluate only at i and j within the foreground
    #
    i, j = np.mgrid[0:pe.shape[0], 0:pe.shape[1]]
    mask = (i>0) & (i<pe.shape[0]-1) & (j>0) & (j<pe.shape[1]-1)
    mask[mask] = image[i[mask]-1, j[mask]-1]
    i = i[mask]
    j = j[mask]
    if len(i) == 0:
        return pe[1:-1, 1:-1]
    if len(i) == 1:
        # Just in case "percentile" can't work when unable to interpolate
        # between a single value... Isolated pixels have value = 1
        #
        pe[mask] = 1
        return pe[1:-1, 1:-1]

    for itr in range(max_iter):
        next_pe = (pe[i+1, j] + pe[i-1, j] + pe[i, j+1] + pe[i, j-1]) / 4 + 1
        difference = np.abs((pe[mask] - next_pe) / next_pe)
        pe[mask] = next_pe
        if np.percentile(difference, percentile) <= convergence:
            break
    return pe[1:-1, 1:-1]

Response: Estimate the solution to the Poisson Equation using the Poisson Equation V28.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def predicted_state_vec(self):
        '''The predicted state vector for the next time point

        From Welch eqn 1.9
        '''
        if not self.has_cached_predicted_state_vec:
            self.p_state_vec = dot_n(
                self.translation_matrix,
                self.state_vec[:, :, np.newaxis])[:,:,0]
        return self.p_state_vec

Response:
The predicted state vector for the next time point
            From Welch eqn 1. 9


Instruction:
How would you explain what the following Python 3 function does
def predicted_obs_vec(self):
        '''The predicted observation vector

        The observation vector for the next step in the filter.
        '''
        if not self.has_cached_obs_vec:
            self.obs_vec = dot_n(
                self.observation_matrix,
                self.predicted_state_vec[:,:,np.newaxis])[:,:,0]
        return self.obs_vec

Response: The predicted observation vector for the next step in the filter.


Instruction:
Here you have a function in Python 3, explain what it does
def map_frames(self, old_indices):
        '''Rewrite the feature indexes based on the next frame's identities

        old_indices - for each feature in the new frame, the index of the
                      old feature
        '''
        nfeatures = len(old_indices)
        noldfeatures = len(self.state_vec)
        if nfeatures > 0:
            self.state_vec = self.state_vec[old_indices]
            self.state_cov = self.state_cov[old_indices]
            self.noise_var = self.noise_var[old_indices]
            if self.has_cached_obs_vec:
                self.obs_vec = self.obs_vec[old_indices]
            if self.has_cached_predicted_state_vec:
                self.p_state_vec = self.p_state_vec[old_indices]
            if len(self.state_noise_idx) > 0:
                #
                # We have to renumber the new_state_noise indices and get rid
                # of those that don't map to numbers. Typical index trick here:
                # * create an array for each legal old element: -1 = no match
                # * give each old element in the array the new number
                # * Filter out the "no match" elements.
                #
                reverse_indices = -np.ones(noldfeatures, int)
                reverse_indices[old_indices] = np.arange(nfeatures)
                self.state_noise_idx = reverse_indices[self.state_noise_idx]
                self.state_noise = self.state_noise[self.state_noise_idx != -1,:]
                self.state_noise_idx = self.state_noise_idx[self.state_noise_idx != -1]

Response:
Map the feature indexes based on the next frame s identities
COOKIE.


Instruction:
How would you implement a function in Python 3 that
adds new features to the state object.

Response:
def add_features(self, kept_indices, new_indices,
                     new_state_vec, new_state_cov, new_noise_var):
        '''Add new features to the state

        kept_indices - the mapping from all indices in the state to new
                       indices in the new version

        new_indices - the indices of the new features in the new version

        new_state_vec - the state vectors for the new indices

        new_state_cov - the covariance matrices for the new indices

        new_noise_var - the noise variances for the new indices
        '''
        assert len(kept_indices) == len(self.state_vec)
        assert len(new_indices) == len(new_state_vec)
        assert len(new_indices) == len(new_state_cov)
        assert len(new_indices) == len(new_noise_var)

        if self.has_cached_obs_vec:
            del self.obs_vec
        if self.has_cached_predicted_state_vec:
            del self.predicted_obs_vec

        nfeatures = len(kept_indices) + len(new_indices)
        next_state_vec = np.zeros((nfeatures, self.state_len))
        next_state_cov = np.zeros((nfeatures, self.state_len, self.state_len))
        next_noise_var = np.zeros((nfeatures, self.state_len))

        if len(kept_indices) > 0:
            next_state_vec[kept_indices] = self.state_vec
            next_state_cov[kept_indices] = self.state_cov
            next_noise_var[kept_indices] = self.noise_var
            if len(self.state_noise_idx) > 0:
                self.state_noise_idx = kept_indices[self.state_noise_idx]
        if len(new_indices) > 0:
            next_state_vec[new_indices] = new_state_vec
            next_state_cov[new_indices] = new_state_cov
            next_noise_var[new_indices] = new_noise_var
        self.state_vec = next_state_vec
        self.state_cov = next_state_cov
        self.noise_var = next_noise_var


Instruction:
Write a Python 3 script to
return a deep copy of the state

Response:
def deep_copy(self):
        '''Return a deep copy of the state'''
        c = KalmanState(self.observation_matrix, self.translation_matrix)
        c.state_vec = self.state_vec.copy()
        c.state_cov = self.state_cov.copy()
        c.noise_var = self.noise_var.copy()
        c.state_noise = self.state_noise.copy()
        c.state_noise_idx = self.state_noise_idx.copy()
        return c


Instruction:
Here you have a function in Python 3, explain what it does
def prcntiles(x,percents):
    '''Equivalent to matlab prctile(x,p), uses linear interpolation.'''
    x=np.array(x).flatten()
    listx = np.sort(x)
    xpcts=[]
    lenlistx=len(listx)
    refs=[]
    for i in range(0,lenlistx):
        r=100*((.5+i)/lenlistx) #refs[i] is percentile of listx[i] in matrix x
        refs.append(r)

    rpcts=[]
    for p in percents:
        if p<refs[0]:
            rpcts.append(listx[0])
        elif p>refs[-1]:
            rpcts.append(listx[-1])
        else:
            for j in range(0,lenlistx): #lenlistx=len(refs)
                if refs[j]<=p and refs[j+1]>=p:
                    my=listx[j+1]-listx[j]
                    mx=refs[j+1]-refs[j]
                    m=my/mx #slope of line between points
                    rpcts.append((m*(p-refs[j]))+listx[j])
                    break
    xpcts.append(rpcts)
    return np.array(xpcts).transpose()

Response: Equivalent to matlab prctile ( x p uses linear interpolation.


Instruction:
Write a Python 3 script to
return spline factors for a single sequence

Response:
def spline_factors(u):
    '''u is np.array'''

    X = np.array([(1.-u)**3 , 4-(6.*(u**2))+(3.*(u**3)) , 1.+(3.*u)+(3.*(u**2))-(3.*(u**3)) , u**3]) * (1./6)

    return X


Instruction:
Implement a function in Python 3 to
return the number of entries in a picklist that is larger than val.

Response:
def pick(picklist,val):
    '''Index to first value in picklist that is larger than val.
If none is larger, index=len(picklist).'''

    assert np.all(np.sort(picklist) == picklist), "pick list is not ordered correctly"
    val = np.array(val)
    i_pick, i_val = np.mgrid[0:len(picklist),0:len(val)]
    #
    # Mark a picklist entry as 1 if the value is before or at,
    # mark it as zero if it is afterward
    #
    is_not_larger = picklist[i_pick] <= val[i_val]
    #
    # The index is the number of entries that are 1
    #
    p = np.sum(is_not_larger, 0)

    return p


Instruction:
How would you code a function in Python 3 to
confine x to [ low high ). Values outside are set to low / high. See also restrict.

Response:
def confine(x,low,high):
    '''Confine x to [low,high]. Values outside are set to low/high.
See also restrict.'''

    y=x.copy()
    y[y < low] = low
    y[y > high] = high
    return y


Instruction:
Can you generate a brief explanation for the following Python 3 code
def gauss(x,m_y,sigma):
    '''returns the gaussian with mean m_y and std. dev. sigma,
calculated at the points of x.'''

    e_y = [np.exp((1.0/(2*float(sigma)**2)*-(n-m_y)**2)) for n in np.array(x)]
    y = [1.0/(float(sigma) * np.sqrt(2 * np.pi)) * e for e in e_y]

    return np.array(y)

Response: returns the gaussian with mean m_y and std. dev. sigma at the points of x.


Instruction:
Write a Python 3 script for
returning the second derivative of the gaussian with mean m_y and standard deviation sigma calculated at the points of x.

Response:
def d2gauss(x,m_y,sigma):
    '''returns the second derivative of the gaussian with mean m_y,
and standard deviation sigma, calculated at the points of x.'''

    return gauss(x,m_y,sigma)*[-1/sigma**2 + (n-m_y)**2/sigma**4 for n in x]


Instruction:
Can you tell what is the following Python 3 function doing
def spline_matrix2d(x,y,px,py,mask=None):
    '''For boundary constraints, the first two and last two spline pieces are constrained
to be part of the same cubic curve.'''
    V = np.kron(spline_matrix(x,px),spline_matrix(y,py))

    lenV = len(V)

    if mask is not None:
        indices = np.nonzero(mask.T.flatten())
        if len(indices)>1:
            indices = np.nonzero(mask.T.flatten())[1][0]
        newV=V.T[indices]
        V=newV.T

        V=V.reshape((V.shape[0],V.shape[1]))

    return V

Response:
For boundary constraints the first two and last two spline pieces are constrained
to be part of the same cubic curve.


Instruction:
Implement a Python 3 function for
making a least squares fit of the spline to the surface x y z.

Response:
def splinefit2d(x, y, z, px, py, mask=None):
    '''Make a least squares fit of the spline (px,py,pz) to the surface (x,y,z).
If mask is given, only masked points are used for the regression.'''

    if mask is None:
        V = np.array(spline_matrix2d(x, y, px, py))
        a = np.array(z.T.flatten())
        pz = np.linalg.lstsq(V.T, a.T)[0].T
    else:
        V = np.array(spline_matrix2d(x,y,px,py,mask))
        indices = np.nonzero(np.array(mask).T.flatten())
        if len(indices[0])==0:
            pz = np.zeros((len(py),len(px)))
        #indices is empty when mask changes to all zeros
        else:
            a = np.array((z.T.flatten()[indices[0]]))
            pz = np.linalg.lstsq(V.T, a.T)[0].T

    pz=pz.reshape((len(py),len(px)))
    return pz.transpose()


Instruction:
How would you explain what the following Python 3 function does
def bg_compensate(img, sigma, splinepoints, scale):
    '''Reads file, subtracts background. Returns [compensated image, background].'''

    from PIL import Image
    import pylab
    from matplotlib.image import pil_to_array
    from centrosome.filter import canny
    import matplotlib

    img = Image.open(img)
    if img.mode=='I;16':
        # 16-bit image
        # deal with the endianness explicitly... I'm not sure
        # why PIL doesn't get this right.
        imgdata = np.fromstring(img.tostring(),np.uint8)
        imgdata.shape=(int(imgdata.shape[0]/2),2)
        imgdata = imgdata.astype(np.uint16)
        hi,lo = (0,1) if img.tag.prefix == 'MM' else (1,0)
        imgdata = imgdata[:,hi]*256 + imgdata[:,lo]
        img_size = list(img.size)
        img_size.reverse()
        new_img = imgdata.reshape(img_size)
        # The magic # for maximum sample value is 281
        if 281 in img.tag:
            img = new_img.astype(np.float32) / img.tag[281][0]
        elif np.max(new_img) < 4096:
            img = new_img.astype(np.float32) / 4095.
        else:
            img = new_img.astype(np.float32) / 65535.
    else:
        img = pil_to_array(img)

    pylab.subplot(1,3,1).imshow(img, cmap=matplotlib.cm.Greys_r)
    pylab.show()

    if len(img.shape)>2:
        raise ValueError('Image must be grayscale')

## Create mask that will fix problem when image has black areas outside of well
    edges = canny(img, np.ones(img.shape, bool), 2, .1, .3)
    ci = np.cumsum(edges, 0)
    cj = np.cumsum(edges, 1)
    i,j = np.mgrid[0:img.shape[0], 0:img.shape[1]]
    mask = ci > 0
    mask = mask & (cj > 0)
    mask[1:,:] &= (ci[0:-1,:] < ci[-1,j[0:-1,:]])
    mask[:,1:] &= (cj[:,0:-1] < cj[i[:,0:-1],-1])

    import time
    t0 = time.clock()
    bg = backgr(img, mask, MODE_AUTO, sigma, splinepoints=splinepoints, scale=scale)
    print("Executed in %f sec" % (time.clock() - t0))
    bg[~mask] = img[~mask]

    pylab.subplot(1,3,2).imshow(img - bg, cmap=matplotlib.cm.Greys_r)
    pylab.subplot(1,3,3).imshow(bg, cmap=matplotlib.cm.Greys_r)
    pylab.show()

Response: Reads file subtracts background. Returns [ compensated image background.


Instruction:
Can you generate the documentation for the following Python 3 function
def mode(a):
    '''Compute the mode of an array

    a: an array

    returns a vector of values which are the most frequent (more than one
    if there is a tie).
    '''
    a = np.asanyarray(a)
    if a.size == 0:
        return np.zeros(0, a.dtype)
    aa = a.flatten()
    aa.sort()
    indices = np.hstack([[0], np.where(aa[:-1] != aa[1:])[0]+1, [aa.size]])
    counts = indices[1:] - indices[:-1]
    best_indices = indices[:-1][counts == np.max(counts)]
    return aa[best_indices]

Response:
Compute the mode of an array

    returns a vector of values which are the most frequent.


Instruction:
Given the following Python 3 function, write the documentation
def otsu(data, min_threshold=None, max_threshold=None,bins=256):
    assert min_threshold is None or max_threshold is None or min_threshold < max_threshold
    def constrain(threshold):
        if not min_threshold is None and threshold < min_threshold:
            threshold = min_threshold
        if not max_threshold is None and threshold > max_threshold:
            threshold = max_threshold
        return threshold

    data = np.atleast_1d(data)
    data = data[~ np.isnan(data)]
    if len(data) == 0:
        return (min_threshold if not min_threshold is None
                else max_threshold if not max_threshold is None
                else 0)
    elif len(data) == 1:
        return constrain(data[0])
    if bins > len(data):
        bins = len(data)
    data.sort()
    var = running_variance(data)
    rvar = np.flipud(running_variance(np.flipud(data)))
    thresholds = data[1:len(data):len(data)//bins]
    score_low = (var[0:len(data)-1:len(data)//bins] *
                 np.arange(0,len(data)-1,len(data)//bins))
    score_high = (rvar[1:len(data):len(data)//bins] *
                  (len(data) - np.arange(1,len(data),len(data)//bins)))
    scores = score_low + score_high
    if len(scores) == 0:
        return constrain(thresholds[0])
    index = np.argwhere(scores == scores.min()).flatten()
    if len(index)==0:
        return constrain(thresholds[0])
    #
    # Take the average of the thresholds to either side of
    # the chosen value to get an intermediate in cases where there is
    # a steep step between the background and foreground
    index = index[0]
    if index == 0:
        index_low = 0
    else:
        index_low = index-1
    if index == len(thresholds)-1:
        index_high = len(thresholds)-1
    else:
        index_high = index+1
    return constrain((thresholds[index_low]+thresholds[index_high]) / 2)

Response: Compute a threshold using Otsu s method


Instruction:
Here you have a function in Python 3, explain what it does
def entropy(data, bins=256):

    data = np.atleast_1d(data)
    data = data[~ np.isnan(data)]
    if len(data) == 0:
        return 0
    elif len(data) == 1:
        return data[0]

    if bins > len(data):
        bins = len(data)
    data.sort()
    var = running_variance(data)+1.0/512.0
    rvar = np.flipud(running_variance(np.flipud(data)))+1.0/512.0
    thresholds = data[1:len(data):len(data)//bins]
    w = np.arange(0,len(data)-1,len(data)//bins)
    score_low = w * np.log(var[0:len(data)-1:len(data)//bins] *
                           w * np.sqrt(2*np.pi*np.exp(1)))
    score_low[np.isnan(score_low)]=0

    w = len(data) - np.arange(1,len(data),len(data)//bins)
    score_high = w * np.log(rvar[1:len(data):len(data)//bins] * w *
                            np.sqrt(2*np.pi*np.exp(1)))
    score_high[np.isnan(score_high)]=0
    scores = score_low + score_high
    index = np.argwhere(scores == scores.min()).flatten()
    if len(index)==0:
        return thresholds[0]
    #
    # Take the average of the thresholds to either side of
    # the chosen value to get an intermediate in cases where there is
    # a steep step between the background and foreground
    index = index[0]
    if index == 0:
        index_low = 0
    else:
        index_low = index-1
    if index == len(thresholds)-1:
        index_high = len(thresholds)-1
    else:
        index_high = index+1
    return (thresholds[index_low]+thresholds[index_high]) / 2

Response:
Compute a threshold using Ray s entropy measurement
    data           - an array of intensity values between zero and one
    bins - an array of bins that are used to bin the data into this many equally - spaced bins


Instruction:
How would you explain what the following Python 3 function does
def otsu3(data, min_threshold=None, max_threshold=None,bins=128):
    assert min_threshold is None or max_threshold is None or min_threshold < max_threshold

    #
    # Compute the running variance and reverse running variance.
    #
    data = np.atleast_1d(data)
    data = data[~ np.isnan(data)]
    data.sort()
    if len(data) == 0:
        return 0
    var = running_variance(data)
    rvar = np.flipud(running_variance(np.flipud(data)))
    if bins > len(data):
        bins = len(data)
    bin_len = int(len(data)//bins)
    thresholds = data[0:len(data):bin_len]
    score_low = (var[0:len(data):bin_len] *
                 np.arange(0,len(data),bin_len))
    score_high = (rvar[0:len(data):bin_len] *
                  (len(data) - np.arange(0,len(data),bin_len)))
    #
    # Compute the middles
    #
    cs = data.cumsum()
    cs2 = (data**2).cumsum()
    i,j = np.mgrid[0:score_low.shape[0],0:score_high.shape[0]]*bin_len
    diff = (j-i).astype(float)
    w = diff
    mean = (cs[j] - cs[i]) / diff
    mean2 = (cs2[j] - cs2[i]) / diff
    score_middle = w * (mean2 - mean**2)
    score_middle[i >= j] = np.Inf
    score = score_low[i*bins//len(data)] + score_middle + score_high[j*bins//len(data)]
    best_score = np.min(score)
    best_i_j = np.argwhere(score==best_score)
    return (thresholds[best_i_j[0,0]],thresholds[best_i_j[0,1]])

Response: Compute a threshold using a 3 - category Otsu - like method.


Instruction:
Implement a function in Python 3 to
compute the entropy score given a variance and # of bins

Response:
def entropy_score(var,bins, w=None, decimate=True):
    '''Compute entropy scores, given a variance and # of bins

    '''
    if w is None:
        n = len(var)
        w = np.arange(0,n,n//bins) / float(n)
    if decimate:
        n = len(var)
        var = var[0:n:n//bins]
    score = w * np.log(var * w * np.sqrt(2*np.pi*np.exp(1)))
    score[np.isnan(score)]=np.Inf
    return score


Instruction:
Create a Python 3 function for
giving a vector x compute the variance for x

Response:
def running_variance(x):
    '''Given a vector x, compute the variance for x[0:i]

    Thank you http://www.johndcook.com/standard_deviation.html
    S[i] = S[i-1]+(x[i]-mean[i-1])*(x[i]-mean[i])
    var(i) = S[i] / (i-1)
    '''
    n = len(x)
    # The mean of x[0:i]
    m = x.cumsum() / np.arange(1,n+1)
    # x[i]-mean[i-1] for i=1...
    x_minus_mprev = x[1:]-m[:-1]
    # x[i]-mean[i] for i=1...
    x_minus_m = x[1:]-m[1:]
    # s for i=1...
    s = (x_minus_mprev*x_minus_m).cumsum()
    var = s / np.arange(1,n)
    # Prepend Inf so we have a variance for x[0]
    return np.hstack(([0],var))


Instruction:
Can you tell what is the following Python 3 function doing
def outline(labels):

    output = numpy.zeros(labels.shape, labels.dtype)
    lr_different = labels[1:,:]!=labels[:-1,:]
    ud_different = labels[:,1:]!=labels[:,:-1]
    d1_different = labels[1:,1:]!=labels[:-1,:-1]
    d2_different = labels[1:,:-1]!=labels[:-1,1:]
    different = numpy.zeros(labels.shape, bool)
    different[1:,:][lr_different]  = True
    different[:-1,:][lr_different] = True
    different[:,1:][ud_different]  = True
    different[:,:-1][ud_different] = True
    different[1:,1:][d1_different] = True
    different[:-1,:-1][d1_different] = True
    different[1:,:-1][d2_different] = True
    different[:-1,1:][d2_different] = True
    #
    # Labels on edges need outlines
    #
    different[0,:] = True
    different[:,0] = True
    different[-1,:] = True
    different[:,-1] = True

    output[different] = labels[different]
    return output

Response: Given a label matrix return a matrix of the outlines of the labeled objects.


Instruction:
Can you implement a function in Python 3 that
computes the Euclidean distance between two points.

Response:
def euclidean_dist(point1, point2):
    """Compute the Euclidean distance between two points.

    Parameters
    ----------
    point1, point2 : 2-tuples of float
        The input points.

    Returns
    -------
    d : float
        The distance between the input points.

    Examples
    --------
    >>> point1 = (1.0, 2.0)
    >>> point2 = (4.0, 6.0)  # (3., 4.) away, simplest Pythagorean triangle
    >>> euclidean_dist(point1, point2)
    5.0
    """
    (x1, y1) = point1
    (x2, y2) = point2
    return math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2)


Instruction:
Explain what the following Python 3 code does
def from_labels(labels):
        labels = labels.astype(int)

        areas = scipy.ndimage.measurements.sum(labels != 0, labels, list(range(1, numpy.max(labels) + 1)))

        existing_labels = [i for (i, a) in enumerate(areas, 1) if a > 0]

        existing_areas = [a for a in areas if a > 0]

        existing_centers = scipy.ndimage.measurements.center_of_mass(labels != 0, labels, existing_labels)

        zipped = zip(existing_labels, existing_centers, existing_areas)

        features = [CellFeatures(c, a, i, labels.shape)
                    for i, c, a in zipped if a != 0]

        return features

Response: Creates a list of cell features based on label image


Instruction:
Can you implement a function in Python 3 that
creates a list of traces from given assignment and cell data.

Response:
def from_detections_assignment(detections_1, detections_2, assignments):
        """
        Creates traces out of given assignment and cell data.
        """
        traces = []

        for d1n, d2n in six.iteritems(assignments):
            # check if the match is between existing cells
            if d1n < len(detections_1) and d2n < len(detections_2):
                traces.append(Trace(detections_1[d1n], detections_2[d2n]))

        return traces


Instruction:
Write a Python 3 script for
running the tracking algorithm.

Response:
def run_tracking(self, label_image_1, label_image_2):
        """
        Tracks cells between input label images.

        @returns: injective function from old objects to new objects (pairs of [old, new]). Number are compatible with labels.
        """
        self.scale = self.parameters_tracking["avgCellDiameter"] / 35.0

        detections_1 = self.derive_detections(label_image_1)

        detections_2 = self.derive_detections(label_image_2)

        # Calculate tracking based on cell features and position.
        traces = self.find_initials_traces(detections_1, detections_2)

        # Use neighbourhoods to improve tracking.
        for _ in range(int(self.parameters_tracking["iterations"])):
            traces = self.improve_traces(detections_1, detections_2, traces)

        # Filter traces.
        return [(trace.previous_cell.number, trace.current_cell.number) for trace in traces]


Instruction:
Make a summary of the following Python 3 code
def is_cell_big(self, cell_detection):
        return cell_detection.area > self.parameters_tracking["big_size"] * self.scale * self.scale

Response: Check if the cell is considered big.


Instruction:
Implement a function in Python 3 to
find k closest neighbours of the given cell.

Response:
def find_closest_neighbours(cell, all_cells, k, max_dist):
        """
        Find k closest neighbours of the given cell.
        :param CellFeatures cell: cell of interest
        :param all_cells: cell to consider as neighbours
        :param int k: number of neighbours to be returned
        :param int max_dist: maximal distance in pixels to consider neighbours
        :return: k closest neighbours
        """
        all_cells = [c for c in all_cells if c != cell]

        sorted_cells = sorted([(cell.distance(c), c) for c in all_cells])

        return [sc[1] for sc in sorted_cells[:k] if sc[0] <= max_dist]


Instruction:
Make a summary of the following Python 3 code
def calculate_basic_cost(self, d1, d2):
        distance = euclidean_dist(d1.center, d2.center) / self.scale

        area_change = 1 - min(d1.area, d2.area) / max(d1.area, d2.area)

        return distance + self.parameters_cost_initial["area_weight"] * area_change

Response: Calculates the basic cost between two cells.


Instruction:
Write a Python 3 script for
calculating the cost between two cells taking into account the movement of cells neighbours.

Response:
def calculate_localised_cost(self, d1, d2, neighbours, motions):
        """
        Calculates assignment cost between two cells taking into account the movement of cells neighbours.

        :param CellFeatures d1: detection in first frame
        :param CellFeatures d2: detection in second frame
        """
        my_nbrs_with_motion = [n for n in neighbours[d1] if n in motions]

        my_motion = (d1.center[0] - d2.center[0], d1.center[1] - d2.center[1])

        if my_nbrs_with_motion == []:
            distance = euclidean_dist(d1.center, d2.center) / self.scale
        else:
            # it is not in motions if there is no trace (cell is considered to vanish)
            distance = min([euclidean_dist(my_motion, motions[n]) for n in my_nbrs_with_motion]) / self.scale

        area_change = 1 - min(d1.area, d2.area) / max(d1.area, d2.area)

        return distance + self.parameters_cost_iteration["area_weight"] * area_change


Instruction:
How would you implement a function in Python 3 that
calculates assignment costs between two detections and empty spaces. The smaller cost the better.

Response:
def calculate_costs(self, detections_1, detections_2, calculate_match_cost, params):
        """
        Calculates assignment costs between detections and 'empty' spaces. The smaller cost the better.

        @param detections_1: cell list of size n in previous frame
        @param detections_2: cell list of size m in current frame

        @return: cost matrix (n+m)x(n+m) extended by cost of matching cells with emptiness
        """
        global invalid_match

        size_sum = len(detections_1) + len(detections_2)

        # Cost matrix extended by matching cells with nothing
        # (for detection 1 it means losing cells, for detection 2 it means new cells).
        cost_matrix = numpy.zeros((size_sum, size_sum))

        # lost cells cost
        cost_matrix[0:len(detections_1), len(detections_2):size_sum] = params["default_empty_cost"] + (1 - numpy.eye(
            len(detections_1), len(detections_1))) * invalid_match

        # new cells cost
        cost_matrix[len(detections_1):size_sum, 0:len(detections_2)] = params["default_empty_cost"] + (1 - numpy.eye(
            len(detections_2), len(detections_2))) * invalid_match

        # increase costs for reliable detections
        for row in [i for i in range(0, len(detections_1)) if detections_1[i].is_reliable() and (
            not params["check_if_big"] or self.is_cell_big(detections_1[i]))]:
            cost_matrix[row, len(detections_2):size_sum] *= params["default_empty_reliable_cost_mult"]

        for col in [i for i in range(0, len(detections_2)) if detections_2[i].is_reliable() and (
            not params["check_if_big"] or self.is_cell_big(detections_2[i]))]:
            cost_matrix[len(detections_1):size_sum, col] *= params["default_empty_reliable_cost_mult"]

        # calculate cost of matching cells
        def cost_if_not_too_far(detection_1, detection_2):
            if detection_1.distance(detection_2) <= self.parameters_tracking["max_distance"]:
                return calculate_match_cost(detection_1, detection_2)
            else:
                return invalid_match

        cost_matrix[0:len(detections_1), 0:len(detections_2)] = [[cost_if_not_too_far(d1, d2) for d2 in detections_2] for d1 in detections_1]

        return cost_matrix


Instruction:
Given the following Python 3 function, write the documentation
def solve_assignement(self, costs):
        if costs is None or len(costs) == 0:
            return dict()

        n = costs.shape[0]

        pairs = [(i, j) for i in range(0, n) for j in range(0, n) if costs[i, j] < invalid_match]

        costs_list = [costs[i, j] for (i, j) in pairs]

        assignment = lapjv.lapjv(list(zip(*pairs))[0], list(zip(*pairs))[1], costs_list)

        indexes = enumerate(list(assignment[0]))

        return dict([(row, col) for row, col in indexes])

Response: Solves assignment problem using Hungarian implementation by Brian M. Clapper.


Instruction:
Write a Python 3 script for
smoothing the image with a per - pixel random multiplier

Response:
def smooth_with_noise(image, bits):
    """Smooth the image with a per-pixel random multiplier

    image - the image to perturb
    bits - the noise is this many bits below the pixel value

    The noise is random with normal distribution, so the individual pixels
    get either multiplied or divided by a normally distributed # of bits
    """

    rr = np.random.RandomState()
    rr.seed(0)
    r = rr.normal(size=image.shape)
    delta = pow(2.0,-bits)
    image_copy = np.clip(image, delta, 1)
    result = np.exp2(np.log2(image_copy + delta) * r +
                     (1-r) * np.log2(image_copy))
    result[result>1] = 1
    result[result<0] = 0
    return result


Instruction:
Can you generate the documentation for the following Python 3 function
def smooth_with_function_and_mask(image, function, mask):
    not_mask               = np.logical_not(mask)
    bleed_over             = function(mask.astype(float))
    masked_image           = np.zeros(image.shape, image.dtype)
    masked_image[mask]     = image[mask]
    smoothed_image         = function(masked_image)
    output_image           = smoothed_image / (bleed_over + np.finfo(float).eps)
    return output_image

Response: Smooth an image with a linear function ignoring the contribution of masked pixels and mask the image


Instruction:
Can you implement a function in Python 3 that
creates a 2 - d Gaussian convolution kernel

Response:
def circular_gaussian_kernel(sd,radius):
    """Create a 2-d Gaussian convolution kernel

    sd     - standard deviation of the gaussian in pixels
    radius - build a circular kernel that convolves all points in the circle
             bounded by this radius
    """
    i,j = np.mgrid[-radius:radius+1,-radius:radius+1].astype(float) / radius
    mask = i**2 + j**2 <= 1
    i = i * radius / sd
    j = j * radius / sd

    kernel = np.zeros((2*radius+1,2*radius+1))
    kernel[mask] = np.e ** (-(i[mask]**2+j[mask]**2) /
                            (2 * sd **2))
    #
    # Normalize the kernel so that there is no net effect on a uniform image
    #
    kernel = kernel / np.sum(kernel)
    return kernel


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fit_polynomial(pixel_data, mask, clip=True):
    '''Return an "image" which is a polynomial fit to the pixel data

    Fit the image to the polynomial Ax**2+By**2+Cxy+Dx+Ey+F

    pixel_data - a two-dimensional numpy array to be fitted

    mask - a mask of pixels whose intensities should be considered in the
           least squares fit

    clip - if True, clip the output array so that pixels less than zero
           in the fitted image are zero and pixels that are greater than
           one are one.
    '''
    mask = np.logical_and(mask,pixel_data > 0)
    if not np.any(mask):
        return pixel_data
    x,y = np.mgrid[0:pixel_data.shape[0],0:pixel_data.shape[1]]
    x2 = x*x
    y2 = y*y
    xy = x*y
    o  = np.ones(pixel_data.shape)
    a = np.array([x[mask],y[mask],x2[mask],y2[mask],xy[mask],o[mask]])
    coeffs = scipy.linalg.lstsq(a.transpose(),pixel_data[mask])[0]
    output_pixels = np.sum([coeff * index for coeff, index in
                            zip(coeffs, [x,y,x2,y2,xy,o])],0)
    if clip:
        output_pixels[output_pixels > 1] = 1
        output_pixels[output_pixels < 0] = 0
    return output_pixels

Response: Fit the image to the polynomial Ax^2 + By ** 2 + Cxy + Dx + Ey + F


Instruction:
Can you generate the documentation for the following Python 3 function
def get_threshold(threshold_method, threshold_modifier, image,
                  mask=None, labels = None,
                  threshold_range_min = None, threshold_range_max = None,
                  threshold_correction_factor = 1.0,
                  adaptive_window_size = 10, **kwargs):
    """Compute a threshold for an image

    threshold_method - one of the TM_ methods above
    threshold_modifier - TM_GLOBAL to calculate one threshold over entire image
                         TM_ADAPTIVE to calculate a per-pixel threshold
                         TM_PER_OBJECT to calculate a different threshold for
                         each object
    image - a NxM numpy array of the image data

    Returns a tuple of local_threshold and global_threshold where:
    * global_threshold is the single number calculated using the threshold
    method over the whole image
    * local_threshold is the global_threshold for global methods. For adaptive
    and per-object thresholding, local_threshold is a matrix of threshold
    values representing the threshold to be applied at each pixel of the
    image.

    Different methods have optional and required parameters:
    Required:
    TM_PER_OBJECT:
    labels - a labels matrix that defines the extents of the individual objects
             to be thresholded separately.

    Optional:
    All:
    mask - a mask of the significant pixels in the image
    threshold_range_min, threshold_range_max - constrain the threshold
        values to be examined to values between these limits
    threshold_correction_factor - the calculated threshold is multiplied
        by this number to get the final threshold
    TM_MOG (mixture of Gaussians):
    object_fraction - fraction of image expected to be occupied by objects
        (pixels that are above the threshold)
    TM_OTSU - We have algorithms derived from Otsu. There is a three-class
              version of Otsu in addition to the two class. There is also
              an entropy measure in addition to the weighted variance.
              two_class_otsu - assume that the distribution represents
                               two intensity classes if true, three if false.
              use_weighted_variance - use Otsu's weighted variance if true,
                                      an entropy measure if false
              assign_middle_to_foreground - assign pixels in the middle class
                               in a three-class Otsu to the foreground if true
                               or the background if false.
    """
    global_threshold = get_global_threshold(
        threshold_method, image, mask, **kwargs)
    global_threshold *= threshold_correction_factor
    if not threshold_range_min is None:
        global_threshold = max(global_threshold, threshold_range_min)
    if not threshold_range_max is None:
        global_threshold = min(global_threshold, threshold_range_max)
    if threshold_modifier == TM_GLOBAL:
        local_threshold=global_threshold
    elif threshold_modifier == TM_ADAPTIVE:
        local_threshold = get_adaptive_threshold(
            threshold_method, image, global_threshold,
            mask, adaptive_window_size = adaptive_window_size, **kwargs)
        local_threshold = local_threshold * threshold_correction_factor
    elif threshold_modifier == TM_PER_OBJECT:
        local_threshold = get_per_object_threshold(
            threshold_method, image, global_threshold, mask, labels,
            threshold_range_min, threshold_range_max,**kwargs)
        local_threshold = local_threshold * threshold_correction_factor
    else:
        raise NotImplementedError("%s thresholding is not implemented"%(threshold_modifier))
    if isinstance(local_threshold, np.ndarray):
        #
        # Constrain thresholds to within .7 to 1.5 of the global threshold.
        #
        threshold_range_min = max(threshold_range_min, global_threshold * .7)
        threshold_range_max = min(threshold_range_max, global_threshold * 1.5)
        if not threshold_range_min is None:
            local_threshold[local_threshold < threshold_range_min] = \
                threshold_range_min
        if not threshold_range_max is None:
            local_threshold[local_threshold > threshold_range_max] = \
            threshold_range_max
        if (threshold_modifier == TM_PER_OBJECT) and (labels is not None):
            local_threshold[labels == 0] = 1.0
    else:
        if not threshold_range_min is None:
            local_threshold = max(local_threshold, threshold_range_min)
        if not threshold_range_max is None:
            local_threshold = min(local_threshold, threshold_range_max)
    return local_threshold, global_threshold

Response: Compute a threshold for an image.


Instruction:
Can you create a Python 3 function that
computes a single threshold over the whole image

Response:
def get_global_threshold(threshold_method, image, mask = None, **kwargs):
    """Compute a single threshold over the whole image"""
    if mask is not None and not np.any(mask):
        return 1

    if threshold_method == TM_OTSU:
        fn = get_otsu_threshold
    elif threshold_method == TM_MOG:
        fn = get_mog_threshold
    elif threshold_method == TM_BACKGROUND:
        fn = get_background_threshold
    elif threshold_method == TM_ROBUST_BACKGROUND:
        fn = get_robust_background_threshold
    elif threshold_method == TM_RIDLER_CALVARD:
        fn = get_ridler_calvard_threshold
    elif threshold_method == TM_KAPUR:
        fn = get_kapur_threshold
    elif threshold_method == TM_MCT:
        fn = get_maximum_correlation_threshold
    else:
        raise NotImplementedError("%s algorithm not implemented"%(threshold_method))
    kwargs = dict([(k, v) for k, v in kwargs.items()
                   if k in fn.args])
    return fn(image, mask, **kwargs)


Instruction:
Create a Python 3 function to
give an image and a threshold compute a global threshold per pixel.

Response:
def get_adaptive_threshold(threshold_method, image, threshold,
                           mask = None,
                           adaptive_window_size = 10,
                           **kwargs):

    """Given a global threshold, compute a threshold per pixel

    Break the image into blocks, computing the threshold per block.
    Afterwards, constrain the block threshold to .7 T < t < 1.5 T.

    Block sizes must be at least 50x50. Images > 500 x 500 get 10x10
    blocks.
    """
    # for the X and Y direction, find the # of blocks, given the
    # size constraints
    image_size = np.array(image.shape[:2],dtype=int)
    nblocks = image_size // adaptive_window_size
    #
    # Use a floating point block size to apportion the roundoff
    # roughly equally to each block
    #
    increment = ( np.array(image_size,dtype=float) /
                  np.array(nblocks,dtype=float))
    #
    # Put the answer here
    #
    thresh_out = np.zeros(image_size, image.dtype)
    #
    # Loop once per block, computing the "global" threshold within the
    # block.
    #
    block_threshold = np.zeros([nblocks[0],nblocks[1]])
    for i in range(nblocks[0]):
        i0 = int(i*increment[0])
        i1 = int((i+1)*increment[0])
        for j in range(nblocks[1]):
            j0 = int(j*increment[1])
            j1 = int((j+1)*increment[1])
            block = image[i0:i1,j0:j1]
            block_mask = None if mask is None else mask[i0:i1,j0:j1]
            block_threshold[i,j] = get_global_threshold(
                threshold_method,
                block, mask = block_mask,
                **kwargs)
    #
    # Use a cubic spline to blend the thresholds across the image to avoid image artifacts
    #
    spline_order = min(3, np.min(nblocks) - 1)
    xStart = int(increment[0] / 2)
    xEnd = int((nblocks[0] - 0.5) * increment[0])
    yStart = int(increment[1] / 2)
    yEnd = int((nblocks[1] - 0.5) * increment[1])
    xtStart = .5
    xtEnd = image.shape[0] - .5
    ytStart = .5
    ytEnd = image.shape[1] - .5
    block_x_coords = np.linspace(xStart,xEnd, nblocks[0])
    block_y_coords = np.linspace(yStart,yEnd, nblocks[1])
    adaptive_interpolation = scipy.interpolate.RectBivariateSpline(
        block_x_coords, block_y_coords, block_threshold,
        bbox = (xtStart, xtEnd, ytStart, ytEnd),
        kx = spline_order, ky = spline_order)
    thresh_out_x_coords = np.linspace(.5, int(nblocks[0] * increment[0]) - .5, thresh_out.shape[0])
    thresh_out_y_coords = np.linspace(.5, int(nblocks[1] * increment[1]) - .5 , thresh_out.shape[1])

    thresh_out = adaptive_interpolation(thresh_out_x_coords, thresh_out_y_coords)

    return thresh_out


Instruction:
How would you implement a function in Python 3 that
calculates the threshold per object for the specified method on the image.

Response:
def get_per_object_threshold(method, image, threshold, mask=None, labels=None,
                             threshold_range_min = None,
                             threshold_range_max = None,
                             **kwargs):
    """Return a matrix giving threshold per pixel calculated per-object

    image - image to be thresholded
    mask  - mask out "don't care" pixels
    labels - a label mask indicating object boundaries
    threshold - the global threshold
    """
    if labels is None:
        labels = np.ones(image.shape,int)
        if not mask is None:
            labels[np.logical_not(mask)] = 0
    label_extents = scipy.ndimage.find_objects(labels,np.max(labels))
    local_threshold = np.ones(image.shape,image.dtype)
    for i, extent in enumerate(label_extents, start=1):
        label_mask = labels[extent]==i
        if not mask is None:
            label_mask = np.logical_and(mask[extent], label_mask)
        values = image[extent]
        per_object_threshold = get_global_threshold(
            method, values, mask = label_mask, **kwargs)
        local_threshold[extent][label_mask] = per_object_threshold
    return local_threshold


Instruction:
Here you have a function in Python 3, explain what it does
def get_mog_threshold(image, mask=None, object_fraction = 0.2):
    cropped_image = np.array(image.flat) if mask is None else image[mask]
    pixel_count = np.product(cropped_image.shape)
    max_count   = 512**2 # maximum # of pixels analyzed
    #
    # We need at least 3 pixels to keep from crashing because the highest
    # and lowest are chopped out below.
    #
    object_fraction = float(object_fraction)
    background_fraction = 1.0-object_fraction
    if pixel_count < 3/min(object_fraction,background_fraction):
        return 1
    if np.max(cropped_image)==np.min(cropped_image):
        return cropped_image[0]
    number_of_classes = 3
    if pixel_count > max_count:
        np.random.seed(0)
        pixel_indices = np.random.permutation(pixel_count)[:max_count]
        cropped_image = cropped_image[pixel_indices]
    # Initialize mean and standard deviations of the three Gaussian
    # distributions by looking at the pixel intensities in the original
    # image and by considering the percentage of the image that is
    # covered by object pixels. Class 1 is the background class and Class
    # 3 is the object class. Class 2 is an intermediate class and we will
    # decide later if it encodes background or object pixels. Also, for
    # robustness the we remove 1% of the smallest and highest intensities
    # in case there are any quantization effects that have resulted in
    # unnaturally many 0:s or 1:s in the image.
    cropped_image.sort()
    one_percent = (np.product(cropped_image.shape) + 99)//100
    cropped_image=cropped_image[one_percent:-one_percent]
    pixel_count = np.product(cropped_image.shape)
    # Guess at the class means for the 3 classes: background,
    # in-between and object
    bg_pixel = cropped_image[int(round(pixel_count * background_fraction/2.0))]
    fg_pixel = cropped_image[int(round(pixel_count * (1-object_fraction/2)))]
    class_mean = np.array([bg_pixel, (bg_pixel+fg_pixel)/2,fg_pixel])
    class_std = np.ones((3,)) * 0.15
    # Initialize prior probabilities of a pixel belonging to each class.
    # The intermediate class steals some probability from the background
    # and object classes.
    class_prob = np.array([3.0/4.0 * background_fraction ,
                              1.0/4.0,
                              3.0/4.0 * object_fraction])
    # Expectation-Maximization algorithm for fitting the three Gaussian
    # distributions/classes to the data. Note, the code below is general
    # and works for any number of classes. Iterate until parameters don't
    # change anymore.
    class_count = np.prod(class_mean.shape)
    #
    # Do a coarse iteration on subsampled data and a fine iteration on the real
    # data
    #
    r = np.random.RandomState()
    r.seed(np.frombuffer(cropped_image[:100].data, np.uint8).tolist())
    for data in (
        r.permutation(cropped_image)[0:(len(cropped_image) // 10)],
        cropped_image):
        delta = 1
        pixel_count = len(data)
        while delta > 0.001:
            old_class_mean = class_mean.copy()
            # Update probabilities of a pixel belonging to the background or
            # object1 or object2
            pixel_class_prob = np.ndarray((pixel_count,class_count))
            for k in range(class_count):
                norm = scipy.stats.norm(class_mean[k],class_std[k])
                pixel_class_prob[:,k] = class_prob[k] * norm.pdf(data)
            pixel_class_normalizer = np.sum(pixel_class_prob,1)+.000000000001
            for k in range(class_count):
                pixel_class_prob[:,k] = pixel_class_prob[:,k] / pixel_class_normalizer
                # Update parameters in Gaussian distributions
                class_prob[k] = np.mean(pixel_class_prob[:,k])
                class_mean[k] = (np.sum(pixel_class_prob[:,k] * data) /
                                 (class_prob[k] * pixel_count))
                class_std[k] = \
                    math.sqrt(np.sum(pixel_class_prob[:,k] *
                                        (data-class_mean[k])**2)/
                              (pixel_count * class_prob[k])) + .000001
            delta = np.sum(np.abs(old_class_mean - class_mean))
    # Now the Gaussian distributions are fitted and we can describe the
    # histogram of the pixel intensities as the sum of these Gaussian
    # distributions. To find a threshold we first have to decide if the
    # intermediate class 2 encodes background or object pixels. This is
    # done by choosing the combination of class probabilities "class_prob"
    # that best matches the user input "object_fraction".

    # Construct an equally spaced array of values between the background
    # and object mean
    ndivisions = 10000
    level = (np.arange(ndivisions) *
             ((class_mean[2]-class_mean[0]) / ndivisions)
             + class_mean[0])
    class_gaussian = np.ndarray((ndivisions,class_count))
    for k in range(class_count):
        norm = scipy.stats.norm(class_mean[k],class_std[k])
        class_gaussian[:,k] = class_prob[k] * norm.pdf(level)
    if (abs(class_prob[1]+class_prob[2]-object_fraction) <
        abs(class_prob[2]-object_fraction)):
        # classifying the intermediate as object more closely models
        # the user's desired object fraction
        background_distribution = class_gaussian[:,0]
        object_distribution = class_gaussian[:,1]+class_gaussian[:,2]
    else:
        background_distribution = class_gaussian[:,0]+class_gaussian[:,1]
        object_distribution = class_gaussian[:,2]
    # Now, find the threshold at the intersection of the background
    # distribution and the object distribution.
    index = np.argmin(np.abs(background_distribution - object_distribution))
    return level[index]

Response: Compute a suitable background using a mixture of gaussians


Instruction:
Create a Python 3 function to
get the threshold based on the mode of the image.

Response:
def get_background_threshold(image, mask = None):
    """Get threshold based on the mode of the image
    The threshold is calculated by calculating the mode and multiplying by
    2 (an arbitrary empirical factor). The user will presumably adjust the
    multiplication factor as needed."""
    cropped_image = np.array(image.flat) if mask is None else image[mask]
    if np.product(cropped_image.shape)==0:
        return 0
    img_min = np.min(cropped_image)
    img_max = np.max(cropped_image)
    if img_min == img_max:
        return cropped_image[0]

    # Only do the histogram between values a bit removed from saturation
    robust_min = 0.02 * (img_max - img_min) + img_min
    robust_max = 0.98 * (img_max - img_min) + img_min
    nbins = 256
    cropped_image = cropped_image[np.logical_and(cropped_image > robust_min,
                                                 cropped_image < robust_max)]
    if len(cropped_image) == 0:
        return robust_min

    h = scipy.ndimage.histogram(cropped_image, robust_min, robust_max, nbins)
    index = np.argmax(h)
    cutoff = float(index) / float(nbins-1)
    #
    # If we have a low (or almost no) background, the cutoff will be
    # zero since the background falls into the lowest bin. We want to
    # offset by the robust cutoff factor of .02. We rescale by 1.04
    # to account for the 0.02 at the top and bottom.
    #
    cutoff = (cutoff + 0.02) / 1.04
    return img_min + cutoff * 2 * (img_max - img_min)


Instruction:
Given the following Python 3 function, write the documentation
def get_robust_background_threshold(image,
                                    mask = None,
                                    lower_outlier_fraction = 0.05,
                                    upper_outlier_fraction = 0.05,
                                    deviations_above_average = 2.0,
                                    average_fn = np.mean,
                                    variance_fn = np.std):
    """Calculate threshold based on mean & standard deviation
       The threshold is calculated by trimming the top and bottom 5% of
       pixels off the image, then calculating the mean and standard deviation
       of the remaining image. The threshold is then set at 2 (empirical
       value) standard deviations above the mean.

       image - the image to threshold
       mask - mask of pixels to consider (default = all pixels)
       lower_outlier_fraction - after ordering the pixels by intensity, remove
           the pixels from 0 to len(image) * lower_outlier_fraction from
           the threshold calculation (default = .05).
        upper_outlier_fraction - remove the pixels from
           len(image) * (1 - upper_outlier_fraction) to len(image) from
           consideration (default = .05).
        deviations_above_average - calculate the standard deviation or MAD and
           multiply by this number and add to the average to get the final
           threshold (default = 2)
        average_fn - function used to calculate the average intensity (e.g.
           np.mean, np.median or some sort of mode function). Default = np.mean
        variance_fn - function used to calculate the amount of variance.
                      Default = np.sd
    """

    cropped_image = np.array(image.flat) if mask is None else image[mask]
    n_pixels = np.product(cropped_image.shape)
    if n_pixels<3:
        return 0

    cropped_image.sort()
    if cropped_image[0] == cropped_image[-1]:
        return cropped_image[0]
    low_chop = int(round(n_pixels * lower_outlier_fraction))
    hi_chop = n_pixels - int(round(n_pixels * upper_outlier_fraction))
    im   = cropped_image if low_chop == 0 else cropped_image[low_chop:hi_chop]
    mean = average_fn(im)
    sd   = variance_fn(im)
    return mean+sd*deviations_above_average

Response: Calculate the robust background threshold for a given image.


Instruction:
Here you have a function in Python 3, explain what it does
def mad(a):
    '''Calculate the median absolute deviation of a sample

    a - a numpy array-like collection of values

    returns the median of the deviation of a from its median.
    '''
    a = np.asfarray(a).flatten()
    return np.median(np.abs(a - np.median(a)))

Response:
Calculate the median absolute deviation of a sample

    returns the median of the deviation of a from its median.


Instruction:
Make a summary of the following Python 3 code
def binned_mode(a):
    '''Calculate a binned mode of a sample

    a - array of values

    This routine bins the sample into np.sqrt(len(a)) bins. This is a
    number that is a compromise between fineness of measurement and
    the stochastic nature of counting which roughly scales as the
    square root of the sample size.
    '''
    a = np.asarray(a).flatten()
    a_min = np.min(a)
    a_max = np.max(a)
    n_bins = np.ceil(np.sqrt(len(a)))
    b = ((a - a_min) / (a_max - a_min) * n_bins).astype(int)
    idx = np.argmax(np.bincount(b))
    return np.percentile(a, 100 * float(idx+.5) / n_bins)

Response: Calculate a binned mode of a sample


Instruction:
Can you write a function in Python 3 where it
finds a threshold using the method of Ridler and Calvard.

Response:
def get_ridler_calvard_threshold(image, mask = None):
    """Find a threshold using the method of Ridler and Calvard

    The reference for this method is:
    "Picture Thresholding Using an Iterative Selection Method"
    by T. Ridler and S. Calvard, in IEEE Transactions on Systems, Man and
    Cybernetics, vol. 8, no. 8, August 1978.
    """
    cropped_image = np.array(image.flat) if mask is None else image[mask]
    if np.product(cropped_image.shape)<3:
        return 0
    if np.min(cropped_image) == np.max(cropped_image):
        return cropped_image[0]

    # We want to limit the dynamic range of the image to 256. Otherwise,
    # an image with almost all values near zero can give a bad result.
    min_val = np.max(cropped_image)/256;
    cropped_image[cropped_image < min_val] = min_val;
    im = np.log(cropped_image);
    min_val = np.min(im);
    max_val = np.max(im);
    im = (im - min_val)/(max_val - min_val);
    pre_thresh = 0;
    # This method needs an initial value to start iterating. Using
    # graythresh (Otsu's method) is probably not the best, because the
    # Ridler Calvard threshold ends up being too close to this one and in
    # most cases has the same exact value.
    new_thresh = otsu(im)
    delta = 0.00001;
    while abs(pre_thresh - new_thresh)>delta:
        pre_thresh = new_thresh;
        mean1 = np.mean(im[im<pre_thresh]);
        mean2 = np.mean(im[im>=pre_thresh]);
        new_thresh = np.mean([mean1,mean2]);
    return math.exp(min_val + (max_val-min_val)*new_thresh);


Instruction:
Here you have a function in Python 3, explain what it does
def get_kapur_threshold(image, mask=None):
    cropped_image = np.array(image.flat) if mask is None else image[mask]
    if np.product(cropped_image.shape)<3:
        return 0
    if np.min(cropped_image) == np.max(cropped_image):
        return cropped_image[0]
    log_image = np.log2(smooth_with_noise(cropped_image, 8))
    min_log_image = np.min(log_image)
    max_log_image = np.max(log_image)
    histogram = scipy.ndimage.histogram(log_image,
                                        min_log_image,
                                        max_log_image,
                                        256)
    histogram_values = (min_log_image + (max_log_image - min_log_image)*
                        np.arange(256, dtype=float) / 255)
    # drop any zero bins
    keep = histogram != 0
    histogram = histogram[keep]
    histogram_values = histogram_values[keep]
    # check for corner cases
    if np.product(histogram_values)==1:
        return 2**histogram_values[0]
    # Normalize to probabilities
    p = histogram.astype(float) / float(np.sum(histogram))
    # Find the probabilities totals up to and above each possible threshold.
    lo_sum = np.cumsum(p);
    hi_sum = lo_sum[-1] - lo_sum;
    lo_e = np.cumsum(p * np.log2(p));
    hi_e = lo_e[-1] - lo_e;

    # compute the entropies
    lo_entropy = lo_e / lo_sum - np.log2(lo_sum);
    hi_entropy = hi_e / hi_sum - np.log2(hi_sum);

    sum_entropy = lo_entropy[:-1] + hi_entropy[:-1];
    sum_entropy[np.logical_not(np.isfinite(sum_entropy))] = np.Inf
    entry = np.argmin(sum_entropy);
    return 2**((histogram_values[entry] + histogram_values[entry+1]) / 2);

Response: The Kapur Sahoo & Wong method of thresholding adapted to log - space.


Instruction:
How would you explain what the following Python 3 function does
def get_maximum_correlation_threshold(image, mask = None, bins = 256):
    '''Return the maximum correlation threshold of the image

    image - image to be thresholded

    mask - mask of relevant pixels

    bins - # of value bins to use

    This is an implementation of the maximum correlation threshold as
    described in Padmanabhan, "A novel algorithm for optimal image thresholding
    of biological data", Journal of Neuroscience Methods 193 (2010) p 380-384
    '''

    if mask is not None:
        image = image[mask]
    image = image.ravel()
    nm = len(image)
    if nm == 0:
        return 0

    #
    # Bin the image
    #
    min_value = np.min(image)
    max_value = np.max(image)
    if min_value == max_value:
        return min_value
    image = ((image - min_value) * (bins - 1) /
             (max_value - min_value)).astype(int)
    histogram = np.bincount(image)
    #
    # Compute (j - mean) and (j - mean) **2
    mean_value = np.mean(image)
    diff = np.arange(len(histogram)) - mean_value
    diff2 = diff * diff
    ndiff = histogram * diff
    ndiff2 = histogram * diff2
    #
    # This is the sum over all j of (j-mean)**2. It's a constant that could
    # be factored out, but I follow the method and use it anyway.
    #
    sndiff2 = np.sum(ndiff2)
    #
    # Compute the cumulative sum from i to m which is the cumsum at m
    # minus the cumsum at i-1
    cndiff = np.cumsum(ndiff)
    numerator = np.hstack([[cndiff[-1]], cndiff[-1] - cndiff[:-1]])
    #
    # For the bottom, we need (Nm - Ni) * Ni / Nm
    #
    ni = nm - np.hstack([[0], np.cumsum(histogram[:-1])]) # number of pixels above i-1
    denominator = np.sqrt(sndiff2 * (nm - ni) * ni / nm)
    #
    mct = numerator / denominator
    mct[denominator == 0] = 0
    my_bin = np.argmax(mct)-1
    return min_value + my_bin * (max_value - min_value) / (bins - 1)

Response: Return the maximum correlation threshold of the image


Instruction:
Write a Python 3 script for
computing the log - transformed variance of foreground and background images.

Response:
def weighted_variance(image, mask, binary_image):
    """Compute the log-transformed variance of foreground and background

    image - intensity image used for thresholding

    mask - mask of ignored pixels

    binary_image - binary image marking foreground and background
    """
    if not np.any(mask):
        return 0
    #
    # Clamp the dynamic range of the foreground
    #
    minval = np.max(image[mask])/256
    if minval == 0:
        return 0

    fg = np.log2(np.maximum(image[binary_image & mask], minval))
    bg = np.log2(np.maximum(image[(~ binary_image) & mask], minval))
    nfg = np.product(fg.shape)
    nbg = np.product(bg.shape)
    if nfg == 0:
        return np.var(bg)
    elif nbg == 0:
        return np.var(fg)
    else:
        return (np.var(fg) * nfg + np.var(bg)*nbg) / (nfg+nbg)


Instruction:
Here you have a function in Python 3, explain what it does
def sum_of_entropies(image, mask, binary_image):
    mask=mask.copy()
    mask[np.isnan(image)] = False
    if not np.any(mask):
        return 0
    #
    # Clamp the dynamic range of the foreground
    #
    minval = np.max(image[mask])/256
    if minval == 0:
        return 0
    clamped_image = image.copy()
    clamped_image[clamped_image < minval] = minval
    #
    # Smooth image with -8 bits of noise
    #
    image = smooth_with_noise(clamped_image, 8)
    im_min = np.min(image)
    im_max = np.max(image)
    #
    # Figure out the bounds for the histogram
    #
    upper = np.log2(im_max)
    lower = np.log2(im_min)
    if upper == lower:
        # All values are the same, answer is log2 of # of pixels
        return math.log(np.sum(mask),2)
    #
    # Create log-transformed lists of points in the foreground and background
    #
    fg = image[binary_image & mask]
    bg = image[(~ binary_image) & mask]
    if len(fg) == 0 or len(bg) == 0:
        return 0
    log_fg = np.log2(fg)
    log_bg = np.log2(bg)
    #
    # Make these into histograms
    hfg = numpy_histogram(log_fg, 256, range=(lower,upper))[0]
    hbg = numpy_histogram(log_bg, 256, range=(lower,upper))[0]
    #hfg = scipy.ndimage.histogram(log_fg,lower,upper,256)
    #hbg = scipy.ndimage.histogram(log_bg,lower,upper,256)
    #
    # Drop empty bins
    #
    hfg = hfg[hfg>0]
    hbg = hbg[hbg>0]
    if np.product(hfg.shape) == 0:
        hfg = np.ones((1,),int)
    if np.product(hbg.shape) == 0:
        hbg = np.ones((1,),int)
    #
    # Normalize
    #
    hfg = hfg.astype(float) / float(np.sum(hfg))
    hbg = hbg.astype(float) / float(np.sum(hbg))
    #
    # Compute sum of entropies
    #
    return np.sum(hfg * np.log2(hfg)) + np.sum(hbg*np.log2(hbg))

Response: Compute the entropy of the entropies of the image.


Instruction:
Can you tell what is the following Python 3 function doing
def log_transform(image):
    '''Renormalize image intensities to log space

    Returns a tuple of transformed image and a dictionary to be passed into
    inverse_log_transform. The minimum and maximum from the dictionary
    can be applied to an image by the inverse_log_transform to
    convert it back to its former intensity values.
    '''
    orig_min, orig_max = scipy.ndimage.extrema(image)[:2]
    #
    # We add 1/2 bit noise to an 8 bit image to give the log a bottom
    #
    limage = image.copy()
    noise_min = orig_min + (orig_max-orig_min)/256.0+np.finfo(image.dtype).eps
    limage[limage < noise_min] = noise_min
    d = { "noise_min":noise_min}
    limage = np.log(limage)
    log_min, log_max = scipy.ndimage.extrema(limage)[:2]
    d["log_min"] = log_min
    d["log_max"] = log_max
    return stretch(limage), d

Response:
Renormalize image intensities to log space

    Returns a tuple of transformed image and a dictionary to be passed into inverse_log_transform.


Instruction:
Create a Python 3 function to
rank the image in order of the corresponding pixel in the image.

Response:
def rank_order(image, nbins=None):
    """Return an image of the same shape where each pixel has the
    rank-order value of the corresponding pixel in the image.
    The returned image's elements are of type np.uint32 which
    simplifies processing in C code.
    """
    flat_image = image.ravel()
    sort_order = flat_image.argsort().astype(np.uint32)
    flat_image = flat_image[sort_order]
    sort_rank  = np.zeros_like(sort_order)
    is_different = flat_image[:-1] != flat_image[1:]
    np.cumsum(is_different, out=sort_rank[1:])
    original_values = np.zeros((sort_rank[-1]+1,),image.dtype)
    original_values[0] = flat_image[0]
    original_values[1:] = flat_image[1:][is_different]
    int_image = np.zeros_like(sort_order)
    int_image[sort_order] = sort_rank
    if nbins is not None:
        max_ranked_data = np.max(int_image)
        while max_ranked_data >= nbins:
            #
            # Decimate the bins until there are fewer than nbins
            #
            hist = np.bincount(int_image)
            #
            # Rank the bins from lowest count to highest
            order = np.argsort(hist)
            #
            # find enough to maybe decimate to nbins
            #
            candidates = order[:max_ranked_data+2-nbins]
            to_delete = np.zeros(max_ranked_data+2, bool)
            to_delete[candidates] = True
            #
            # Choose candidates that are either not next to others
            # or have an even index so as not to delete adjacent bins
            #
            td_mask = to_delete[:-1] & (
                ((np.arange(max_ranked_data+1) & 2) == 0) |
                (~ to_delete[1:]))
            if td_mask[0]:
                td_mask[0] = False
            #
            # A value to be deleted has the same index as the following
            # value and the two end up being merged
            #
            rd_translation = np.cumsum(~td_mask)-1
            #
            # Translate the rankings to the new space
            #
            int_image = rd_translation[int_image]
            #
            # Eliminate the bins with low counts
            #
            original_values = original_values[~td_mask]
            max_ranked_data = len(original_values)-1

    return (int_image.reshape(image.shape), original_values)


Instruction:
Create a Python 3 function to
normalize the intensities of each object to the [ 0 1 ) range.

Response:
def normalized_per_object(image, labels):
    """Normalize the intensities of each object to the [0, 1] range."""
    nobjects = labels.max()
    objects = np.arange(nobjects + 1)
    lmin, lmax = scind.extrema(image, labels, objects)[:2]
    # Divisor is the object's max - min, or 1 if they are the same.
    divisor = np.ones((nobjects + 1,))
    divisor[lmax > lmin] = (lmax - lmin)[lmax > lmin]
    return (image - lmin[labels]) / divisor[labels]


Instruction:
Make a summary of the following Python 3 code
def cooccurrence(quantized_image, labels, scale_i=3, scale_j=0):
    labels = labels.astype(int)
    nlevels = quantized_image.max() + 1
    nobjects = labels.max()
    if scale_i < 0:
        scale_i = -scale_i
        scale_j = -scale_j
    if scale_i == 0 and scale_j > 0:
        image_a = quantized_image[:, :-scale_j]
        image_b = quantized_image[:, scale_j:]
        labels_ab = labels_a = labels[:, :-scale_j]
        labels_b = labels[:, scale_j:]
    elif scale_i > 0 and scale_j == 0:
        image_a = quantized_image[:-scale_i, :]
        image_b = quantized_image[scale_i:, :]
        labels_ab = labels_a = labels[:-scale_i, :]
        labels_b = labels[scale_i:, :]
    elif scale_i > 0 and scale_j > 0:
        image_a = quantized_image[:-scale_i, :-scale_j]
        image_b = quantized_image[scale_i:, scale_j:]
        labels_ab = labels_a = labels[:-scale_i, :-scale_j]
        labels_b = labels[scale_i:, scale_j:]
    else:
        # scale_j should be negative
        image_a = quantized_image[:-scale_i, -scale_j:]
        image_b = quantized_image[scale_i:, :scale_j]
        labels_ab = labels_a = labels[:-scale_i, -scale_j:]
        labels_b = labels[scale_i:, :scale_j]
    equilabel = ((labels_a == labels_b) & (labels_a > 0))
    if np.any(equilabel):

        Q = (nlevels*nlevels*(labels_ab[equilabel]-1)+
             nlevels*image_a[equilabel]+image_b[equilabel])
        R = np.bincount(Q)
        if R.size != nobjects*nlevels*nlevels:
            S = np.zeros(nobjects*nlevels*nlevels-R.size)
            R = np.hstack((R, S))
        P = R.reshape(nobjects, nlevels, nlevels)
        pixel_count = fix(scind.sum(equilabel, labels_ab,
                                    np.arange(nobjects, dtype=np.int32)+1))
        pixel_count = np.tile(pixel_count[:,np.newaxis,np.newaxis],
                              (1,nlevels,nlevels))
        return (P.astype(float) / pixel_count.astype(float), nlevels)
    else:
        return np.zeros((nobjects, nlevels, nlevels)), nlevels

Response: Calculates the co - occurrence matrix for all the objects in the image.


Instruction:
Write a Python 3 function for
summing variance ( error in Haralick s original paper here.

Response:
def H7(self):
        "Sum variance (error in Haralick's original paper here)."
        h6 = np.tile(self.H6(), (self.rlevels2.shape[1], 1)).transpose()
        return (((self.rlevels2 + 2) - h6) ** 2 * self.p_xplusy).sum(1)


Instruction:
Write a Python 3 script for
constructing a lookup table of the sum - of - factorial part of the radial polynomial of the zernike indexes passed in.

Response:
def construct_zernike_lookuptable(zernike_indexes):
    """Return a lookup table of the sum-of-factorial part of the radial
    polynomial of the zernike indexes passed

    zernike_indexes - an Nx2 array of the Zernike polynomials to be
                      computed.
    """
    n_max = np.max(zernike_indexes[:,0])
    factorial = np.ones((1 + n_max,), dtype=float)
    factorial[1:] = np.cumproduct(np.arange(1, 1 + n_max, dtype=float))
    width = int(n_max//2 + 1)
    lut = np.zeros((zernike_indexes.shape[0],width), dtype=float)
    for idx, (n, m) in enumerate(zernike_indexes):
        alt = 1
        npmh = (n+m)//2
        nmmh = (n-m)//2
        for k in range(0,nmmh+1):
            lut[idx,k] = \
                (alt * factorial[n-k] /
                 (factorial[k]*factorial[npmh-k]*factorial[nmmh-k]))
            alt = -alt
    return lut


Instruction:
Can you create a Python 3 function that
constructs the zerike polynomials for all objects in an image.

Response:
def construct_zernike_polynomials(x, y, zernike_indexes, mask=None, weight=None):
    """Return the zerike polynomials for all objects in an image

    x - the X distance of a point from the center of its object
    y - the Y distance of a point from the center of its object
    zernike_indexes - an Nx2 array of the Zernike polynomials to be computed.
    mask - a mask with same shape as X and Y of the points to consider
    weight - weightings of points with the same shape as X and Y (default
             weight on each point is 1).

    returns a height x width x N array of complex numbers which are the
    e^i portion of the sine and cosine of the Zernikes
    """
    if x.shape != y.shape:
        raise ValueError("X and Y must have the same shape")
    if mask is None:
        pass
    elif mask.shape != x.shape:
        raise ValueError("The mask must have the same shape as X and Y")
    else:
        x = x[mask]
        y = y[mask]
        if weight is not None:
            weight = weight[mask]
    lut = construct_zernike_lookuptable(zernike_indexes) # precompute poly. coeffs.
    nzernikes = zernike_indexes.shape[0]
    # compute radii
    r_square = np.square(x) # r_square = x**2
    np.add(r_square, np.square(y), out=r_square) # r_square = x**2 + y**2
    # z = y + 1j*x
    # each Zernike polynomial is poly(r)*(r**m * np.exp(1j*m*phi)) ==
    #                            poly(r)*(y + 1j*x)**m
    z = np.empty(x.shape, np.complex)
    np.copyto(z.real, y)
    np.copyto(z.imag, x)
    # preallocate buffers
    s = np.empty_like(x)
    zf = np.zeros((nzernikes,) + x.shape, np.complex)
    z_pows = {}
    for idx, (n, m) in enumerate(zernike_indexes):
        s[:]=0
        if not m in z_pows:
            if m == 0:
                z_pows[m] = np.complex(1.0)
            else:
                z_pows[m] = z if m == 1 else (z ** m)
        z_pow = z_pows[m]
        # use Horner scheme
        for k in range((n-m)//2+1):
            s *= r_square
            s += lut[idx, k]
        s[r_square>1]=0
        if weight is not None:
            s *= weight.astype(s.dtype)
        if m == 0:
            np.copyto(zf[idx], s) # zf[idx] = s
        else:
            np.multiply(s, z_pow, out=zf[idx]) # zf[idx] = s*exp_term

    if mask is None:
        result = zf.transpose( tuple(range(1, 1+x.ndim)) + (0, ))
    else:
        result = np.zeros( mask.shape + (nzernikes,), np.complex)
        result[mask] = zf.transpose( tuple(range(1, 1 + x.ndim)) + (0, ))
    return result


Instruction:
How would you explain what the following Python 3 function does
def score_zernike(zf, radii, labels, indexes=None):
    if indexes is None:
        indexes = np.arange(1,np.max(labels)+1,dtype=np.int32)
    else:
        indexes = np.array(indexes, dtype=np.int32)
    radii = np.asarray(radii, dtype=float)
    n = radii.size
    k = zf.shape[2]
    score = np.zeros((n,k))
    if n == 0:
        return score
    areas = np.square(radii)
    areas *= np.pi
    for ki in range(k):
        zfk=zf[:,:,ki]
        real_score = scipy.ndimage.sum(zfk.real,labels,indexes)
        real_score = fixup_scipy_ndimage_result(real_score)

        imag_score = scipy.ndimage.sum(zfk.imag,labels,indexes)
        imag_score = fixup_scipy_ndimage_result(imag_score)
        # one_score = np.sqrt(real_score**2+imag_score**2) / areas
        np.square(real_score, out=real_score)
        np.square(imag_score, out=imag_score)
        one_score = real_score + imag_score
        np.sqrt(one_score, out=one_score)
        one_score /= areas
        score[:,ki] = one_score
    return score

Response:
Score the output of construct_zernike_polynomials

   .


Instruction:
Write a Python 3 function for
returning a list of all Zernike indexes up to the given limit

Response:
def get_zernike_indexes(limit=10):
    """Return a list of all Zernike indexes up to the given limit

    limit - return all Zernike indexes with N less than this limit

    returns an array of 2-tuples. Each tuple is organized as (N,M).
    The Zernikes are stored as complex numbers with the real part
    being (N,M) and the imaginary being (N,-M)
    """
    def zernike_indexes_iter(n_max):
        for n in range(0, n_max):
            for m in range(n%2, n+1, 2):
                yield n
                yield m

    z_ind = np.fromiter(zernike_indexes_iter(limit), np.intc)
    z_ind = z_ind.reshape( (len(z_ind) // 2, 2) )
    return z_ind


Instruction:
Can you generate the documentation for the following Python 3 function
def propagate(image, labels, mask, weight):
    if image.shape != labels.shape:
        raise ValueError("Image shape %s != label shape %s" % (repr(image.shape), repr(labels.shape)))
    if image.shape != mask.shape:
        raise ValueError("Image shape %s != mask shape %s" % (repr(image.shape), repr(mask.shape)))
    labels_out = np.zeros(labels.shape, np.int32)
    distances = -np.ones(labels.shape, np.float64)
    distances[labels > 0] = 0
    labels_and_mask = np.logical_and(labels != 0, mask)
    coords = np.argwhere(labels_and_mask)
    i1,i2 = _propagate.convert_to_ints(0.0)
    ncoords = coords.shape[0]
    pq = np.column_stack((np.ones((ncoords,), int) * i1, np.ones((ncoords,),int) * i2, labels[labels_and_mask], coords))
    _propagate.propagate(np.ascontiguousarray(image, np.float64), np.ascontiguousarray(pq,np.int32), np.ascontiguousarray(mask, np.int8), labels_out, distances, float(weight))
    labels_out[labels > 0] = labels[labels > 0]
    return labels_out, distances

Response: Propagate the labels to the nearest pixels


Instruction:
Make a summary of the following Python 3 code
def slow_reduction_transfer(ii, j, idx, count, x, u, v, c):
    '''Perform the reduction transfer step from the Jonker-Volgenant algorithm

    The data is input in a ragged array in terms of "i" structured as a
    vector of values for each i,j combination where:

    ii - the i to be reduced
    j - the j-index of every entry
    idx - the index of the first entry for each i
    count - the number of entries for each i
    x - the assignment of j to i
    u - the dual variable "u" which will be updated. It should be
        initialized to zero for the first reduction transfer.
    v - the dual variable "v" which will be reduced in-place
    c - the cost for each entry.

    The code described in the paper is:

    for each assigned row i do
    begin
       j1:=x[i]; u=min {c[i,j]-v[j] | j=1..n, j != j1};
       v[j1]:=v[j1]-(u-u[i]);
       u[i] = u;
    end;

    The authors note that reduction transfer can be applied in later stages
    of the algorithm but does not seem to provide a substantial benefit
    in speed.
    '''
    for i in ii:
        j1 = x[i]
        jj = j[idx[i]:(idx[i]+count[i])]
        uu = np.min((c[idx[i]:(idx[i]+count[i])] - v[jj])[jj != j1])
        v[j1] = v[j1] - uu + u[i]
        u[i] = uu

Response: Perform the reduction transfer step from the Jonker - Volgenant algorithm.


Instruction:
Here you have a function in Python 3, explain what it does
def slow_augmenting_row_reduction(n, ii, jj, idx, count, x, y, u, v, c):
    '''Perform the augmenting row reduction step from the Jonker-Volgenaut algorithm

    n - the number of i and j in the linear assignment problem
    ii - the unassigned i
    jj - the j-index of every entry in c
    idx - the index of the first entry for each i
    count - the number of entries for each i
    x - the assignment of j to i
    y - the assignment of i to j
    u - the dual variable "u" which will be updated. It should be
        initialized to zero for the first reduction transfer.
    v - the dual variable "v" which will be reduced in-place
    c - the cost for each entry.

    returns the new unassigned i
    '''

    #######################################
    #
    # From Jonker:
    #
    # procedure AUGMENTING ROW REDUCTION;
    # begin
    # LIST: = {all unassigned rows};
    # for all i in LIST do
    #    repeat
    #    ul:=min {c[i,j]-v[j] for j=l ...n};
    #    select j1 with c [i,j 1] - v[j 1] = u1;
    #    u2:=min {c[i,j]-v[j] for j=l ...n,j< >jl} ;
    #    select j2 with c [i,j2] - v [j2] = u2 and j2 < >j 1 ;
    #    u[i]:=u2;
    #    if ul <u2 then v[jl]:=v[jl]-(u2-ul)
    #    else if jl is assigned then jl : =j2;
    #    k:=y [jl]; if k>0 then x [k]:=0; x[i]:=jl; y [ j l ] : = i ; i:=k
    #  until ul =u2 (* no reduction transfer *) or k=0 i~* augmentation *)
    #  end
    ii = list(ii)
    k = 0
    limit = len(ii)
    free = []
    while k < limit:
        i = ii[k]
        k += 1
        j = jj[idx[i]:(idx[i] + count[i])]
        uu = c[idx[i]:(idx[i] + count[i])] - v[j]
        order = np.lexsort([uu])
        u1, u2 = uu[order[:2]]
        j1,j2 = j[order[:2]]
        i1 = y[j1]
        if u1 < u2:
            v[j1] = v[j1] - u2 + u1
        elif i1 != n:
            j1 = j2
            i1 = y[j1]
        if i1 != n:
            if u1 < u2:
                k -= 1
                ii[k] = i1
            else:
                free.append(i1)
        x[i] = j1
        y[j1] = i
    return np.array(free,np.uint32)

Response: This function performs the augmenting row reduction step from the Jonker - Volgenaut algorithm.


Instruction:
Write a Python 3 script for
augmenting the unassigned i and j by a given number of variables.

Response:
def slow_augment(n, ii, jj, idx, count, x, y, u, v, c):
    '''Perform the augmentation step to assign unassigned i and j

    n - the # of i and j, also the marker of unassigned x and y
    ii - the unassigned i
    jj - the ragged arrays of j for each i
    idx - the index of the first j for each i
    count - the number of j for each i
    x - the assignments of j for each i
    y - the assignments of i for each j
    u,v - the dual variables
    c - the costs
    '''

    ##################################################
    #
    # Augment procedure: from the Jonker paper.
    #
    # Note:
    #    cred [i,j] = c [i,j] - u [i] - v[j]
    #
    # procedure AUGMENT;
    # begin
    #   for all unassigned i* do
    #   begin
    #     for j:= 1 ... n do
    #       begin d[j] := c[i*,j] - v[j] ; pred[j] := i* end;
    #     READY: = { ) ; SCAN: = { } ; TODO: = { 1 ... n} ;
    #     repeat
    #       if SCAN = { } then
    #       begin
    #         u = min {d[j] for j in TODO} ;
    #         SCAN: = {j | d[j] = u} ;
    #         TODO: = TODO - SCAN;
    #         for j in SCAN do if y[j]==0 then go to augment
    #       end;
    #       select any j* in SCAN;
    #       i := y[j*]; SCAN := SCAN - {j*} ; READY: = READY + {j*} ;
    #       for all j in TODO do if u + cred[i,j] < d[j] then
    #       begin
    #         d[j] := u + cred[i,j]; pred[j] := i;
    #         if d[j] = u then
    #           if y[j] is unassigned then go to augment else
    #           begin SCAN: = SCAN + {j} ; TODO: = TODO - {j} end
    #       end
    #    until false; (* repeat always ends with go to augment *)
    #augment:
    #   (* price updating *)
    #   for k in READY do v[k]: = v[k] + d[k] - u;
    #   (* augmentation *)
    #   repeat
    #     i: = pred[j]; y[ j ] := i ; k:=j; j:=x[i]; x[i]:= k
    #   until i = i*
    #  end
    #end
    inf = np.sum(c) + 1
    d = np.zeros(n)
    cc = np.zeros((n,n))
    cc[:,:] = inf
    for i in range(n):
        cc[i,jj[idx[i]:(idx[i]+count[i])]] = c[idx[i]:(idx[i]+count[i])]
    c = cc
    for i in ii:
        print("Processing i=%d" % i)
        j = jj[idx[i]:(idx[i] + count[i])]
        d = c[i,:] - v
        pred = np.ones(n, int) * i
        on_deck = []
        ready = []
        scan = []
        to_do = list(range(n))
        try:
            while True:
                print("Evaluating i=%d, n_scan = %d" % (i, len(scan)))
                if len(scan) == 0:
                    ready += on_deck
                    on_deck = []
                    umin = np.min([d[jjj] for jjj in to_do])
                    print("umin = %f" % umin)
                    scan = [jjj for jjj in to_do if d[jjj] == umin]
                    to_do = [jjj for jjj in to_do if d[jjj] != umin]
                    for j1 in scan:
                        if y[j1] == n:
                            raise StopIteration()
                j1 = scan[0]
                iii = y[j1]
                print("Consider replacing i=%d, j=%d" % (iii, j1))
                scan = scan[1:]
                on_deck += [j1]
                u1 = c[iii, j1] - v[j1] - umin
                for j1 in list(to_do):
                    h = c[iii, j1] - v[j1] - u1
                    print("Consider j=%d as replacement, c[%d,%d]=%f,v[%d]=%f,h=%f, d[j]= %f" % (j1,iii,j1,c[iii,j1],j1,v[j1],h,d[j1]))
                    if h < d[j1]:
                        print("Add to chain")
                        pred[j1] = iii
                        if h == umin:
                            if y[j1] == n:
                                raise StopIteration()
                            print("Add to scan")
                            scan += [j1]
                            to_do.remove(j1)
                        d[j1] = h

        except StopIteration:
            # Augment
            print("Augmenting %d" % j1)
            for k in ready:
                temp = v[k]
                v[k] = v[k] + d[k] - umin
                print("v[%d] %f -> %f" % (k, temp, v[k]))
            while True:
                iii = pred[j1]
                print("y[%d] %d -> %d" % (j1, y[j1], iii))
                y[j1] = iii
                j1, x[iii] = x[iii], j1
                if iii == i:
                    break
    #
    # Re-establish slackness since we didn't pay attention to u
    #
    for i in range(n):
        j = x[i]
        u[i] = c[i,j] - v[j]


Instruction:
Here you have a function in Python 3, explain what it does
def estimate_parameters_rippe(
        self, max_dist_kb, size_bin_kb, display_graph
    ):
        """
        estimation by least square optimization of Rippe parameters on the
        experimental data
        :param max_dist_kb:
        :param size_bin_kb:
        """
        logger.info("estimation of the parameters of the model")
        self.bins = np.arange(
            size_bin_kb, max_dist_kb + size_bin_kb, size_bin_kb
        )

        self.mean_contacts = np.zeros_like(self.bins, dtype=np.float32)
        self.dict_collect = dict()
        self.gpu_vect_frags.copy_from_gpu()
        epsi = self.mean_value_trans
        for k in self.bins:
            self.dict_collect[k] = []
        for i in range(0, self.n_frags // 10):
            # print "frag i = ", i
            start = self.sparse_matrix.indptr[i]
            end = self.sparse_matrix.indptr[i + 1]
            id_j = self.sparse_matrix.indices[start:end]
            data = self.sparse_matrix.data[start:end]

            info_i = self.np_sub_frags_2_frags[i]
            init_id_fi = int(info_i[0])
            # pos_i = self.S_o_A_frags["pos"][init_id_fi]
            id_c_i = self.S_o_A_frags["id_c"][init_id_fi]
            s_i = (
                self.S_o_A_frags["start_bp"][init_id_fi] / 1000.0
                + self.np_sub_frags_2_frags[i][1]
            )
            len_kb_c_i = self.S_o_A_frags["l_cont_bp"][init_id_fi] / 1000
            # local_bins = np.arange(size_bin_kb, min(len_kb_c_i, max_dist_kb)
            # + size_bin_kb, size_bin_kb) local_storage =
            # np.zeros_like(local_bins, dtype=np.int32)
            #
            # for fj, dj in zip(id_j, data):
            #     info_j = self.np_sub_frags_2_frags[fj]
            #     init_id_fj = info_j[0]
            #     id_c_j = self.S_o_A_frags['id_c'][init_id_fj]
            #     if id_c_i == id_c_j:
            #         pos_j = self.S_o_A_frags['pos'][init_id_fj]
            #         s_j = self.S_o_A_frags['start_bp'][init_id_fj]/1000.0 +
            #         self.np_sub_frags_2_frags[fj][1]
            #         d = np.abs(s_i - s_j)
            #         if d < max_dist_kb:
            #             id_bin = d / size_bin_kb
            #             local_storage[id_bin] += dj
            #             # self.dict_collect[self.bins[id_bin]].append(dj)
            # # we have to add also the zeros
            # for bin, val in zip(local_bins, local_storage):
            #     # print "bin = ", bin
            #     self.dict_collect[bin].append(val)

            if size_bin_kb < len_kb_c_i:
                # local_bins = np.arange(size_bin_kb, min(len_kb_c_i,
                # max_dist_kb) + size_bin_kb, size_bin_kb)
                local_bins = np.arange(
                    size_bin_kb, max_dist_kb + size_bin_kb, size_bin_kb
                )
                local_storage = np.zeros_like(local_bins, dtype=np.int32)

                for fj, dj in zip(id_j, data):
                    info_j = self.np_sub_frags_2_frags[fj]
                    init_id_fj = int(info_j[0])
                    id_c_j = self.S_o_A_frags["id_c"][init_id_fj]
                    if id_c_i == id_c_j:
                        # pos_j = self.S_o_A_frags["pos"][init_id_fj]
                        s_j = (
                            self.S_o_A_frags["start_bp"][init_id_fj] / 1000.0
                            + self.np_sub_frags_2_frags[fj][1]
                        )
                        d = np.abs(s_i - s_j)
                        if d < max_dist_kb:
                            id_bin = int(d / size_bin_kb)
                            local_storage[id_bin] += dj
                            # self.dict_collect[self.bins[id_bin]].append(dj)
                # we have to add also the zeros
                for bin, val in zip(local_bins, local_storage):
                    # print "bin = ", bin
                    self.dict_collect[bin].append(val)

        for id_bin in range(0, len(self.bins)):
            k = self.bins[id_bin]
            self.mean_contacts[id_bin] = np.mean(self.dict_collect[k])

        for id_bin in range(0, len(self.bins)):
            k = self.bins[id_bin]
            tmp = np.mean(self.dict_collect[k])
            if np.isnan(tmp) or tmp == 0:
                # if np.isnan(tmp):
                # if np.isnan(tmp):
                #     print "removing nan"
                self.mean_contacts[id_bin] = np.nan
            else:
                self.mean_contacts[id_bin] = tmp + epsi

        self.mean_contacts_upd = []
        self.bins_upd = []

        for count, ele in zip(self.mean_contacts, self.bins):
            if not np.isnan(count):
                self.bins_upd.append(ele)
                self.mean_contacts_upd.append(count)

        self.bins_upd = np.array(self.bins_upd)
        # self.mean_contacts_upd.sort()
        # self.mean_contacts_upd.reverse()
        self.mean_contacts_upd = np.array(self.mean_contacts_upd)

        # self.mean_contacts_upd =
        # ndi.filters.gaussian_filter1d(self.mean_contacts_upd,
        # sigma=len(self.bins_upd) / 5.)

        # print "size mean contacts vector = ", self.mean_contacts_upd.shape
        # print "mean contacts vector = ", self.mean_contacts_upd

        p, self.y_estim = opti.estimate_param_rippe(
            self.mean_contacts_upd, self.bins_upd
        )
        ##########################################
        logger.info("p from estimate parameters  = {}".format(p))
        # p = list(p[0])
        # p[3] = 2
        # p = tuple(p)
        ##########################################
        fit_param = p
        logger.info("mean value trans = {}".format(self.mean_value_trans))
        ##########################################
        logger.info("BEWARE!!! : I will lower mean value trans  !!!")
        self.mean_value_trans = self.mean_value_trans / 10.0
        ##########################################

        estim_max_dist = opti.estimate_max_dist_intra(
            fit_param, self.mean_value_trans
        )
        logger.info("estimate max dist cis trans = {}".format(estim_max_dist))
        self.param_simu = self.setup_rippe_parameters(
            fit_param, estim_max_dist
        )
        self.param_simu_test = self.param_simu
        self.gpu_param_simu = cuda.mem_alloc(self.param_simu.nbytes)
        self.gpu_param_simu_test = cuda.mem_alloc(self.param_simu.nbytes)

        cuda.memcpy_htod(self.gpu_param_simu, self.param_simu)
        cuda.memcpy_htod(self.gpu_param_simu_test, self.param_simu_test)
        display_graph = False
        if display_graph:
            plt.figure()
            plt.loglog(self.bins_upd, self.mean_contacts_upd, "-*b")
            plt.loglog(self.bins_upd, self.y_estim, "-*r")
            plt.xlabel("genomic distance (kb)")
            plt.ylabel("frequency of contact")
            plt.title(
                r"$\mathrm{Frequency\ of\ contact\ versus\ genomic\ distance"
                " \(data\ tricho test):}\ slope=%.3f,\ max\ cis\ distance(kb)"
                "=%.3f\ d=%.3f\ scale\ factor=%.3f\ $"
                % (
                    self.param_simu["slope"],
                    estim_max_dist,
                    self.param_simu["d"],
                    self.param_simu["fact"],
                )
            )
            plt.legend(["obs", "fit"])
            plt.savefig()
            plt.close()

        self.eval_likelihood_init()

Response: estimate the parameters of the Rippe model


Instruction:
How would you code a function in Python 3 to
estimate parameters of the Rippe model

Response:
def estimate_parameters(self, max_dist_kb, size_bin_kb, display_graph):
        """
        estimation by least square optimization of Rippe parameters on the
        experimental data
        :param max_dist_kb:
        :param size_bin_kb:
        """
        logger.info("estimation of the parameters of the model")
        self.bins = np.arange(
            size_bin_kb, max_dist_kb + size_bin_kb, size_bin_kb
        )

        self.mean_contacts = np.zeros_like(self.bins, dtype=np.float32)
        self.dict_collect = dict()
        self.gpu_vect_frags.copy_from_gpu()
        epsi = self.mean_value_trans
        for k in self.bins:
            self.dict_collect[k] = []
        for i in range(0, 2000):
            # print "frag i = ", i
            start = self.sparse_matrix.indptr[i]
            end = self.sparse_matrix.indptr[i + 1]
            id_j = self.sparse_matrix.indices[start:end]
            data = self.sparse_matrix.data[start:end]

            info_i = self.np_sub_frags_2_frags[i]
            init_id_fi = info_i[0]
            # pos_i = self.S_o_A_frags["pos"][init_id_fi]
            id_c_i = self.S_o_A_frags["id_c"][init_id_fi]
            s_i = (
                self.S_o_A_frags["start_bp"][init_id_fi] / 1000.0
                + self.np_sub_frags_2_frags[i][1]
            )
            len_kb_c_i = self.S_o_A_frags["l_cont_bp"][init_id_fi] / 1000
            local_bins = np.arange(
                size_bin_kb,
                min(len_kb_c_i, max_dist_kb) + size_bin_kb,
                size_bin_kb,
            )
            local_storage = np.zeros_like(local_bins, dtype=np.int32)

            for fj, dj in zip(id_j, data):
                info_j = self.np_sub_frags_2_frags[fj]
                init_id_fj = info_j[0]
                id_c_j = self.S_o_A_frags["id_c"][init_id_fj]
                if id_c_i == id_c_j:
                    # pos_j = self.S_o_A_frags["pos"][init_id_fj]
                    s_j = (
                        self.S_o_A_frags["start_bp"][init_id_fj] / 1000.0
                        + self.np_sub_frags_2_frags[fj][1]
                    )
                    d = np.abs(s_i - s_j)
                    if d < max_dist_kb:
                        id_bin = d / size_bin_kb
                        local_storage[id_bin] += dj
                        # self.dict_collect[self.bins[id_bin]].append(dj)
            # we have to add also the zeros
            for my_bin, val in zip(local_bins, local_storage):
                # print "bin = ", bin
                self.dict_collect[my_bin].append(val)

        for id_bin in range(0, len(self.bins)):
            k = self.bins[id_bin]
            self.mean_contacts[id_bin] = np.mean(self.dict_collect[k])

        for id_bin in range(0, len(self.bins)):
            k = self.bins[id_bin]
            tmp = np.mean(self.dict_collect[k])
            if np.isnan(tmp) or tmp == 0:
                # if np.isnan(tmp):
                # if np.isnan(tmp):
                #     print "removing nan"
                self.mean_contacts[id_bin] = np.nan
            else:
                self.mean_contacts[id_bin] = tmp + epsi

        self.mean_contacts_upd = []
        self.bins_upd = []

        for count, ele in zip(self.mean_contacts, self.bins):
            if not np.isnan(count):
                self.bins_upd.append(ele)
                self.mean_contacts_upd.append(count)

        self.bins_upd = np.array(self.bins_upd)
        self.mean_contacts_upd = np.array(self.mean_contacts_upd)

        # self.mean_contacts_upd =
        # ndi.filters.gaussian_filter1d(self.mean_contacts_upd,
        # sigma=len(self.bins_upd) / 5.)

        p, self.y_estim = nuis.estimate_param_hic(
            self.mean_contacts_upd, self.bins_upd
        )
        ##########################################
        fit_param = p.x
        ##########################################
        logger.info("mean value trans = {}".format(self.mean_value_trans))
        ##########################################
        estim_max_dist = nuis.estimate_max_dist_intra(
            fit_param, self.mean_value_trans
        )
        logger.info("max distance cis/trans = {}".format(estim_max_dist))
        ##########################################
        self.param_simu = self.setup_model_parameters(
            fit_param, estim_max_dist
        )
        self.gpu_param_simu = cuda.mem_alloc(self.param_simu.nbytes)
        self.gpu_param_simu_test = cuda.mem_alloc(self.param_simu.nbytes)

        cuda.memcpy_htod(self.gpu_param_simu, self.param_simu)

        if display_graph:
            plt.loglog(self.bins_upd, self.mean_contacts_upd, "-*b")
            plt.loglog(self.bins_upd, self.y_estim, "-*r")
            plt.xlabel("genomic distance (kb)")
            plt.ylabel("frequency of contact")
            plt.legend(["obs", "fit"])
            plt.show()