Instruction:
Implement a Python 3 function for
calculating an rms image and a bkg image. Parameters ---------- mesh_size : int Number of beams per box default = 20 forced_rms : float The rms of the image. If None: calculate the rms level (default). Otherwise assume a constant rms. forced_bkg : float The background level of the image. If None: calculate the background level (default). Otherwise assume a constant background. cores: int Number of cores to use if different from what is autodetected.

Response:
def _make_bkg_rms(self, mesh_size=20, forced_rms=None, forced_bkg=None, cores=None):
        """
        Calculate an rms image and a bkg image.

        Parameters
        ----------
        mesh_size : int
            Number of beams per box default = 20

        forced_rms : float
            The rms of the image.
            If None:  calculate the rms level (default).
            Otherwise assume a constant rms.

        forced_bkg : float
            The background level of the image.
            If None: calculate the background level (default).
            Otherwise assume a constant background.

        cores: int
            Number of cores to use if different from what is autodetected.

        """
        if (forced_rms is not None):
            self.log.info("Forcing rms = {0}".format(forced_rms))
            self.global_data.rmsimg[:] = forced_rms
        if (forced_bkg is not None):
            self.log.info("Forcing bkg = {0}".format(forced_bkg))
            self.global_data.bkgimg[:] = forced_bkg

        # If we known both the rms and the bkg then there is nothing to compute
        if (forced_rms is not None) and (forced_bkg is not None):
            return

        data = self.global_data.data_pix
        beam = self.global_data.beam

        img_x, img_y = data.shape
        xcen = int(img_x / 2)
        ycen = int(img_y / 2)

        # calculate a local beam from the center of the data
        pixbeam = self.global_data.psfhelper.get_pixbeam_pixel(xcen, ycen)
        if pixbeam is None:
            self.log.error("Cannot determine the beam shape at the image center")
            sys.exit(1)

        width_x = mesh_size * max(abs(math.cos(np.radians(pixbeam.pa)) * pixbeam.a),
                                  abs(math.sin(np.radians(pixbeam.pa)) * pixbeam.b))
        width_x = int(width_x)
        width_y = mesh_size * max(abs(math.sin(np.radians(pixbeam.pa)) * pixbeam.a),
                                  abs(math.cos(np.radians(pixbeam.pa)) * pixbeam.b))
        width_y = int(width_y)

        self.log.debug("image size x,y:{0},{1}".format(img_x, img_y))
        self.log.debug("beam: {0}".format(beam))
        self.log.debug("mesh width (pix) x,y: {0},{1}".format(width_x, width_y))

        # box centered at image center then tilling outwards
        xstart = int(xcen - width_x / 2) % width_x  # the starting point of the first "full" box
        ystart = int(ycen - width_y / 2) % width_y

        xend = img_x - int(img_x - xstart) % width_x  # the end point of the last "full" box
        yend = img_y - int(img_y - ystart) % width_y

        xmins = [0]
        xmins.extend(list(range(xstart, xend, width_x)))
        xmins.append(xend)

        xmaxs = [xstart]
        xmaxs.extend(list(range(xstart + width_x, xend + 1, width_x)))
        xmaxs.append(img_x)

        ymins = [0]
        ymins.extend(list(range(ystart, yend, width_y)))
        ymins.append(yend)

        ymaxs = [ystart]
        ymaxs.extend(list(range(ystart + width_y, yend + 1, width_y)))
        ymaxs.append(img_y)

        # if the image is smaller than our ideal mesh size, just use the whole image instead
        if width_x >= img_x:
            xmins = [0]
            xmaxs = [img_x]
        if width_y >= img_y:
            ymins = [0]
            ymaxs = [img_y]

        if cores > 1:
            # set up the queue
            queue = pprocess.Queue(limit=cores, reuse=1)
            estimate = queue.manage(pprocess.MakeReusable(self._estimate_bkg_rms))
            # populate the queue
            for xmin, xmax in zip(xmins, xmaxs):
                for ymin, ymax in zip(ymins, ymaxs):
                    estimate(ymin, ymax, xmin, xmax)
        else:
            queue = []
            for xmin, xmax in zip(xmins, xmaxs):
                for ymin, ymax in zip(ymins, ymaxs):
                    queue.append(self._estimate_bkg_rms(xmin, xmax, ymin, ymax))

        # only copy across the bkg/rms if they are not already set
        # queue can only be traversed once so we have to put the if inside the loop
        for ymin, ymax, xmin, xmax, bkg, rms in queue:
            if (forced_rms is None):
                self.global_data.rmsimg[ymin:ymax, xmin:xmax] = rms
            if (forced_rms is None):
                self.global_data.bkgimg[ymin:ymax, xmin:xmax] = bkg

        return


Instruction:
Create a Python 3 function for
estimating the background noise mean and RMS.

Response:
def _estimate_bkg_rms(self, xmin, xmax, ymin, ymax):
        """
        Estimate the background noise mean and RMS.
        The mean is estimated as the median of data.
        The RMS is estimated as the IQR of data / 1.34896.

        Parameters
        ----------
        xmin, xmax, ymin, ymax : int
            The bounding region over which the bkg/rms will be calculated.

        Returns
        -------
        ymin, ymax, xmin, xmax : int
            A copy of the input parameters

        bkg, rms : float
            The calculated background and noise.
        """
        data = self.global_data.data_pix[ymin:ymax, xmin:xmax]
        pixels = np.extract(np.isfinite(data), data).ravel()
        if len(pixels) < 4:
            bkg, rms = np.NaN, np.NaN
        else:
            pixels.sort()
            p25 = pixels[int(pixels.size / 4)]
            p50 = pixels[int(pixels.size / 2)]
            p75 = pixels[int(pixels.size / 4 * 3)]
            iqr = p75 - p25
            bkg, rms = p50, iqr / 1.34896
        # return the input and output data so we know what we are doing
        # when compiling the results of multiple processes
        return ymin, ymax, xmin, xmax, bkg, rms


Instruction:
Here you have a function in Python 3, explain what it does
def _load_aux_image(self, image, auxfile):
        auximg = FitsImage(auxfile, beam=self.global_data.beam).get_pixels()
        if auximg.shape != image.get_pixels().shape:
            self.log.error("file {0} is not the same size as the image map".format(auxfile))
            self.log.error("{0}= {1}, image = {2}".format(auxfile, auximg.shape, image.get_pixels().shape))
            sys.exit(1)
        return auximg

Response: Load a main image and auxiliary file and make sure that it is the same shape as the main image.


Instruction:
Write a Python 3 function that can
refit islands on a group of components.

Response:
def _refit_islands(self, group, stage, outerclip=None, istart=0):
        """
        Do island refitting (priorized fitting) on a group of islands.

        Parameters
        ----------
        group : list
            A list of components grouped by island.

        stage : int
            Refitting stage.

        outerclip : float
            Ignored, placed holder for future development.

        istart : int
            The starting island number.

        Returns
        -------
        sources : list
            List of sources (and islands).
        """
        global_data = self.global_data
        sources = []

        data = global_data.data_pix
        rmsimg = global_data.rmsimg

        for inum, isle in enumerate(group, start=istart):
            self.log.debug("-=-")
            self.log.debug("input island = {0}, {1} components".format(isle[0].island, len(isle)))

            # set up the parameters for each of the sources within the island
            i = 0
            params = lmfit.Parameters()
            shape = data.shape
            xmin, ymin = shape
            xmax = ymax = 0

            # island_mask = []
            src_valid_psf = None
            # keep track of the sources that are actually being refit
            # this may be a subset of all sources in the island
            included_sources = []
            for src in isle:
                pixbeam = global_data.psfhelper.get_pixbeam(src.ra, src.dec)
                # find the right pixels from the ra/dec
                source_x, source_y = global_data.wcshelper.sky2pix([src.ra, src.dec])
                source_x -= 1
                source_y -= 1
                x = int(round(source_x))
                y = int(round(source_y))

                self.log.debug("pixel location ({0:5.2f},{1:5.2f})".format(source_x, source_y))
                # reject sources that are outside the image bounds, or which have nan data/rms values
                if not 0 <= x < shape[0] or not 0 <= y < shape[1] or \
                        not np.isfinite(data[x, y]) or \
                        not np.isfinite(rmsimg[x, y]) or \
                        pixbeam is None:
                    self.log.debug("Source ({0},{1}) not within usable region: skipping".format(src.island, src.source))
                    continue
                else:
                    # Keep track of the last source to have a valid psf so that we can use it later on
                    src_valid_psf = src
                # determine the shape parameters in pixel values
                _, _, sx, sy, theta = global_data.wcshelper.sky2pix_ellipse([src.ra, src.dec], src.a / 3600,
                                                                            src.b / 3600, src.pa)
                sx *= FWHM2CC
                sy *= FWHM2CC

                self.log.debug("Source shape [sky coords]  {0:5.2f}x{1:5.2f}@{2:05.2f}".format(src.a, src.b, src.pa))
                self.log.debug("Source shape [pixel coords] {0:4.2f}x{1:4.2f}@{2:05.2f}".format(sx, sy, theta))

                # choose a region that is 2x the major axis of the source, 4x semimajor axis a
                width = 4 * sx
                ywidth = int(round(width)) + 1
                xwidth = int(round(width)) + 1

                # adjust the size of the island to include this source
                xmin = min(xmin, max(0, x - xwidth / 2))
                ymin = min(ymin, max(0, y - ywidth / 2))
                xmax = max(xmax, min(shape[0], x + xwidth / 2 + 1))
                ymax = max(ymax, min(shape[1], y + ywidth / 2 + 1))

                s_lims = [0.8 * min(sx, pixbeam.b * FWHM2CC), max(sy, sx) * 1.25]

                # Set up the parameters for the fit, including constraints
                prefix = "c{0}_".format(i)
                params.add(prefix + 'amp', value=src.peak_flux, vary=True)
                # for now the xo/yo are locations within the main image, we correct this later
                params.add(prefix + 'xo', value=source_x, min=source_x - sx / 2., max=source_x + sx / 2.,
                           vary=stage >= 2)
                params.add(prefix + 'yo', value=source_y, min=source_y - sy / 2., max=source_y + sy / 2.,
                           vary=stage >= 2)
                params.add(prefix + 'sx', value=sx, min=s_lims[0], max=s_lims[1], vary=stage >= 3)
                params.add(prefix + 'sy', value=sy, min=s_lims[0], max=s_lims[1], vary=stage >= 3)
                params.add(prefix + 'theta', value=theta, vary=stage >= 3)
                params.add(prefix + 'flags', value=0, vary=False)
                # this source is being refit so add it to the list
                included_sources.append(src)
                i += 1

                # TODO: Allow this mask to be used in conjunction with the FWHM mask that is defined further on
                # # Use pixels above outerclip sigmas..
                # if outerclip>=0:
                #     mask = np.where(data[xmin:xmax,ymin:ymax]-outerclip*rmsimg[xmin:xmax,ymin:ymax]>0)
                # else: # negative outer clip means use all the pixels
                #     mask = np.where(data[xmin:xmax,ymin:ymax])
                #
                # # convert the pixel indices to be pixels within the parent data set
                # xmask = mask[0] + xmin
                # ymask = mask[1] + ymin
                # island_mask.extend(zip(xmask,ymask))

            if i == 0:
                self.log.debug("No sources found in island {0}".format(src.island))
                continue
            params.add('components', value=i, vary=False)
            # params.components = i
            self.log.debug(" {0} components being fit".format(i))
            # now we correct the xo/yo positions to be relative to the sub-image
            self.log.debug("xmxxymyx {0} {1} {2} {3}".format(xmin, xmax, ymin, ymax))
            for i in range(params['components'].value):
                prefix = "c{0}_".format(i)
                params[prefix + 'xo'].value -= xmin
                params[prefix + 'xo'].min -= xmin
                params[prefix + 'xo'].max -= xmin
                params[prefix + 'yo'].value -= ymin
                params[prefix + 'yo'].min -= ymin
                params[prefix + 'yo'].max -= ymin
            # self.log.debug(params)
            # don't fit if there are no sources
            if params['components'].value < 1:
                self.log.info("Island {0} has no components".format(src.island))
                continue

            # this .copy() will stop us from modifying the parent region when we later apply our mask.
            idata = data[int(xmin):int(xmax), int(ymin):int(ymax)].copy()
            # now convert these back to indices within the idata region
            # island_mask = np.array([(x-xmin, y-ymin) for x, y in island_mask])

            allx, ally = np.indices(idata.shape)
            # mask to include pixels that are withn the FWHM of the sources being fit
            mask_params = copy.deepcopy(params)
            for i in range(mask_params['components'].value):
                prefix = 'c{0}_'.format(i)
                mask_params[prefix + 'amp'].value = 1
            mask_model = ntwodgaussian_lmfit(mask_params)
            mask = np.where(mask_model(allx.ravel(), ally.ravel()) <= 0.1)
            mask = allx.ravel()[mask], ally.ravel()[mask]
            del mask_params

            idata[mask] = np.nan

            mx, my = np.where(np.isfinite(idata))
            non_nan_pix = len(mx)
            total_pix = len(allx.ravel())
            self.log.debug("island extracted:")
            self.log.debug(" x[{0}:{1}] y[{2}:{3}]".format(xmin, xmax, ymin, ymax))
            self.log.debug(" max = {0}".format(np.nanmax(idata)))
            self.log.debug(
                " total {0}, masked {1}, not masked {2}".format(total_pix, total_pix - non_nan_pix, non_nan_pix))

            # Check to see that each component has some data within the central 3x3 pixels of it's location
            # If not then we don't fit that component
            for i in range(params['components'].value):
                prefix = "c{0}_".format(i)
                # figure out a box around the center of this
                cx, cy = params[prefix + 'xo'].value, params[prefix + 'yo'].value  # central pixel coords
                self.log.debug(" comp {0}".format(i))
                self.log.debug("  x0, y0 {0} {1}".format(cx, cy))
                xmx = int(round(np.clip(cx + 2, 0, idata.shape[0])))
                xmn = int(round(np.clip(cx - 1, 0, idata.shape[0])))
                ymx = int(round(np.clip(cy + 2, 0, idata.shape[1])))
                ymn = int(round(np.clip(cy - 1, 0, idata.shape[1])))
                square = idata[xmn:xmx, ymn:ymx]
                # if there are no not-nan pixels in this region then don't vary any parameters
                if not np.any(np.isfinite(square)):
                    self.log.debug(" not fitting component {0}".format(i))
                    params[prefix + 'amp'].value = np.nan
                    for p in ['amp', 'xo', 'yo', 'sx', 'sy', 'theta']:
                        params[prefix + p].vary = False
                        params[prefix + p].stderr = np.nan  # this results in an error of -1 later on
                    params[prefix + 'flags'].value |= flags.NOTFIT

            # determine the number of free parameters and if we have enough data for a fit
            nfree = np.count_nonzero([params[p].vary for p in params.keys()])
            self.log.debug(params)
            if nfree < 1:
                self.log.debug(" Island has no components to fit")
                result = DummyLM()
                model = params
            else:
                if non_nan_pix < nfree:
                    self.log.debug("More free parameters {0} than available pixels {1}".format(nfree, non_nan_pix))
                    if non_nan_pix >= params['components'].value:
                        self.log.debug("Fixing all parameters except amplitudes")
                        for p in params.keys():
                            if 'amp' not in p:
                                params[p].vary = False
                    else:
                        self.log.debug(" no not-masked pixels, skipping")
                    continue

                # do the fit
                # if the pixel beam is not valid, then recalculate using the location of the last source to have a valid psf
                if pixbeam is None:
                    if src_valid_psf is not None:
                        pixbeam = global_data.psfhelper.get_pixbeam(src_valid_psf.ra, src_valid_psf.dec)
                    else:
                        self.log.critical("Cannot determine pixel beam")
                fac = 1 / np.sqrt(2)
                if self.global_data.docov:
                    C = Cmatrix(mx, my, pixbeam.a * FWHM2CC * fac, pixbeam.b * FWHM2CC * fac, pixbeam.pa)
                    B = Bmatrix(C)
                else:
                    C = B = None
                errs = np.nanmax(rmsimg[int(xmin):int(xmax), int(ymin):int(ymax)])
                result, _ = do_lmfit(idata, params, B=B)
                model = covar_errors(result.params, idata, errs=errs, B=B, C=C)

            # convert the results to a source object
            offsets = (xmin, xmax, ymin, ymax)
            # TODO allow for island fluxes in the refitting.
            island_data = IslandFittingData(inum, i=idata, offsets=offsets, doislandflux=False, scalars=(4, 4, None))
            new_src = self.result_to_components(result, model, island_data, src.flags)

            for ns, s in zip(new_src, included_sources):
                # preserve the uuid so we can do exact matching between catalogs
                ns.uuid = s.uuid

                # flag the sources as having been priorized
                ns.flags |= flags.PRIORIZED

                # if the position wasn't fit then copy the errors from the input catalog
                if stage < 2:
                    ns.err_ra = s.err_ra
                    ns.err_dec = s.err_dec
                    ns.flags |= flags.FIXED2PSF

                # if the shape wasn't fit then copy the errors from the input catalog
                if stage < 3:
                    ns.err_a = s.err_a
                    ns.err_b = s.err_b
                    ns.err_pa = s.err_pa
            sources.extend(new_src)
        return sources


Instruction:
Can you tell what is the following Python 3 function doing
def _fit_island(self, island_data):
        global_data = self.global_data

        # global data
        dcurve = global_data.dcurve
        rmsimg = global_data.rmsimg

        # island data
        isle_num = island_data.isle_num
        idata = island_data.i
        innerclip, outerclip, max_summits = island_data.scalars
        xmin, xmax, ymin, ymax = island_data.offsets

        # get the beam parameters at the center of this island
        midra, middec = global_data.wcshelper.pix2sky([0.5 * (xmax + xmin), 0.5 * (ymax + ymin)])
        beam = global_data.psfhelper.get_psf_pix(midra, middec)
        del middec, midra

        icurve = dcurve[xmin:xmax, ymin:ymax]
        rms = rmsimg[xmin:xmax, ymin:ymax]

        is_flag = 0
        pixbeam = global_data.psfhelper.get_pixbeam_pixel((xmin + xmax) / 2., (ymin + ymax) / 2.)
        if pixbeam is None:
            # This island is not 'on' the sky, ignore it
            return []

        self.log.debug("=====")
        self.log.debug("Island ({0})".format(isle_num))

        params = self.estimate_lmfit_parinfo(idata, rms, icurve, beam, innerclip, outerclip, offsets=[xmin, ymin],
                                             max_summits=max_summits)

        # islands at the edge of a region of nans
        # result in no components
        if params is None or params['components'].value < 1:
            return []

        self.log.debug("Rms is {0}".format(np.shape(rms)))
        self.log.debug("Isle is {0}".format(np.shape(idata)))
        self.log.debug(" of which {0} are masked".format(sum(np.isnan(idata).ravel() * 1)))

        # Check that there is enough data to do the fit
        mx, my = np.where(np.isfinite(idata))
        non_blank_pix = len(mx)
        free_vars = len([1 for a in params.keys() if params[a].vary])
        if non_blank_pix < free_vars or free_vars == 0:
            self.log.debug("Island {0} doesn't have enough pixels to fit the given model".format(isle_num))
            self.log.debug("non_blank_pix {0}, free_vars {1}".format(non_blank_pix, free_vars))
            result = DummyLM()
            model = params
            is_flag |= flags.NOTFIT
        else:
            # Model is the fitted parameters
            fac = 1 / np.sqrt(2)
            if self.global_data.docov:
                C = Cmatrix(mx, my, pixbeam.a * FWHM2CC * fac, pixbeam.b * FWHM2CC * fac, pixbeam.pa)
                B = Bmatrix(C)
            else:
                C = B = None
            self.log.debug(
                "C({0},{1},{2},{3},{4})".format(len(mx), len(my), pixbeam.a * FWHM2CC, pixbeam.b * FWHM2CC, pixbeam.pa))
            errs = np.nanmax(rms)
            self.log.debug("Initial params")
            self.log.debug(params)
            result, _ = do_lmfit(idata, params, B=B)
            if not result.errorbars:
                is_flag |= flags.FITERR
            # get the real (sky) parameter errors
            model = covar_errors(result.params, idata, errs=errs, B=B, C=C)

            if self.global_data.dobias and self.global_data.docov:
                x, y = np.indices(idata.shape)
                acf = elliptical_gaussian(x, y, 1, 0, 0, pixbeam.a * FWHM2CC * fac, pixbeam.b * FWHM2CC * fac,
                                          pixbeam.pa)
                bias_correct(model, idata, acf=acf * errs ** 2)

            if not result.success:
                is_flag |= flags.FITERR

        self.log.debug("Final params")
        self.log.debug(model)

        # convert the fitting results to a list of sources [and islands]
        sources = self.result_to_components(result, model, island_data, is_flag)

        return sources

Response: This function is called by the island fit method. It is called by the island fit method.


Instruction:
Write a Python 3 script for
executing fitting on a list of islands and return a list of sources.

Response:
def _fit_islands(self, islands):
        """
        Execute fitting on a list of islands
        This function just wraps around fit_island, so that when we do multiprocesing
        a single process will fit multiple islands before returning results.

        Parameters
        ----------
        islands : list of :class:`AegeanTools.models.IslandFittingData`
            The islands to be fit.

        Returns
        -------
        sources : list
            The sources that were fit.
        """
        self.log.debug("Fitting group of {0} islands".format(len(islands)))
        sources = []
        for island in islands:
            res = self._fit_island(island)
            sources.extend(res)
        return sources


Instruction:
Given the following Python 3 function, write the documentation
def find_sources_in_image(self, filename, hdu_index=0, outfile=None, rms=None, bkg=None, max_summits=None, innerclip=5,
                              outerclip=4, cores=None, rmsin=None, bkgin=None, beam=None, doislandflux=False,
                              nopositive=False, nonegative=False, mask=None, lat=None, imgpsf=None, blank=False,
                              docov=True, cube_index=None):
        """
        Run the Aegean source finder.

        Parameters
        ----------
        filename : str or HDUList
            Image filename or HDUList.

        hdu_index : int
            The index of the FITS HDU (extension).

        outfile : str
            file for printing catalog (NOT a table, just a text file of my own design)

        rms : float
            Use this rms for the entire image (will also assume that background is 0)

        max_summits : int
            Fit up to this many components to each island (extras are included but not fit)

        innerclip, outerclip : float
            The seed (inner) and flood (outer) clipping level (sigmas).

        cores : int
            Number of CPU cores to use. None means all cores.

        rmsin, bkgin : str or HDUList
            Filename or HDUList for the noise and background images.
            If either are None, then it will be calculated internally.

        beam : (major, minor, pa)
            Floats representing the synthesised beam (degrees).
            Replaces whatever is given in the FITS header.
            If the FITS header has no BMAJ/BMIN then this is required.

        doislandflux : bool
            If True then each island will also be characterized.

        nopositive, nonegative : bool
            Whether to return positive or negative sources.
            Default nopositive=False, nonegative=True.

        mask : str
            The filename of a region file created by MIMAS.
            Islands outside of this region will be ignored.

        lat : float
            The latitude of the telescope (declination of zenith).

        imgpsf : str or HDUList
             Filename or HDUList for a psf image.

        blank : bool
            Cause the output image to be blanked where islands are found.

        docov : bool
            If True then include covariance matrix in the fitting process. (default=True)

        cube_index : int
            For image cubes, cube_index determines which slice is used.

        Returns
        -------
        sources : list
            List of sources found.
        """

        # Tell numpy to be quiet
        np.seterr(invalid='ignore')
        if cores is not None:
            if not (cores >= 1): raise AssertionError("cores must be one or more")

        self.load_globals(filename, hdu_index=hdu_index, bkgin=bkgin, rmsin=rmsin, beam=beam, rms=rms, bkg=bkg, cores=cores,
                          verb=True, mask=mask, lat=lat, psf=imgpsf, blank=blank, docov=docov, cube_index=cube_index)
        global_data = self.global_data
        rmsimg = global_data.rmsimg
        data = global_data.data_pix

        self.log.info("beam = {0:5.2f}'' x {1:5.2f}'' at {2:5.2f}deg".format(
            global_data.beam.a * 3600, global_data.beam.b * 3600, global_data.beam.pa))
        # stop people from doing silly things.
        if outerclip > innerclip:
            outerclip = innerclip
        self.log.info("seedclip={0}".format(innerclip))
        self.log.info("floodclip={0}".format(outerclip))

        isle_num = 0

        if cores == 1:  # single-threaded, no parallel processing
            queue = []
        else:
            queue = pprocess.Queue(limit=cores, reuse=1)
            fit_parallel = queue.manage(pprocess.MakeReusable(self._fit_islands))

        island_group = []
        group_size = 20
        for i, xmin, xmax, ymin, ymax in self._gen_flood_wrap(data, rmsimg, innerclip, outerclip, domask=True):
            # ignore empty islands
            # This should now be impossible to trigger
            if np.size(i) < 1:
                self.log.warn("Empty island detected, this should be imposisble.")
                continue
            isle_num += 1
            scalars = (innerclip, outerclip, max_summits)
            offsets = (xmin, xmax, ymin, ymax)
            island_data = IslandFittingData(isle_num, i, scalars, offsets, doislandflux)
            # If cores==1 run fitting in main process. Otherwise build up groups of islands
            # and submit to queue for subprocesses. Passing a group of islands is more
            # efficient than passing single islands to the subprocesses.
            if cores == 1:
                res = self._fit_island(island_data)
                queue.append(res)
            else:
                island_group.append(island_data)
                # If the island group is full queue it for the subprocesses to fit
                if len(island_group) >= group_size:
                    fit_parallel(island_group)
                    island_group = []

        # The last partially-filled island group also needs to be queued for fitting
        if len(island_group) > 0:
            fit_parallel(island_group)

        # Write the output to the output file
        if outfile:
            print(header.format("{0}-({1})".format(__version__, __date__), filename), file=outfile)
            print(OutputSource.header, file=outfile)

        sources = []
        for srcs in queue:
            if srcs:  # ignore empty lists
                for src in srcs:
                    # ignore sources that we have been told to ignore
                    if (src.peak_flux > 0 and nopositive) or (src.peak_flux < 0 and nonegative):
                        continue
                    sources.append(src)
                    if outfile:
                        print(str(src), file=outfile)
        self.sources.extend(sources)
        return sources

Response: This function will find sources in an image.


Instruction:
Write a Python 3 function for
taking an input catalog, and image, and optional background/noise images fit the flux and ra/dec for each of the given sources, keeping the morphology fixed if doregroup is true the groups will be recreated based on a matching radius/probability. if doregroup is false then the islands of the input catalog will be preserved. Multiple cores can be specified, and will be used. Parameters ---------- filename : str or HDUList Image filename or HDUList. catalogue : str or list Input catalogue file name or list of OutputSource objects. hdu_index : int The index of the FITS HDU (extension). outfile : str file for printing catalog (NOT a table, just a text file of my own design) rmsin, bkgin : str or HDUList Filename or HDUList for the noise and background images. If either are None, then it will be calculated internally. cores : int Number of CPU cores to use. None means all cores. rms : float Use this rms for the entire image (will also assume that background is 0) beam : (major, minor, pa) Floats representing the synthesised beam (degrees). Replaces whatever is given in the FITS header. If the FITS header has no BMAJ/BMIN then this is required. lat : float The latitude of the telescope (declination of zenith). imgpsf : str or HDUList Filename or HDUList for a psf image. catpsf : str or HDUList Filename or HDUList for the catalogue psf image. stage : int Refitting stage ratio : float If not None - ratio of image psf to catalog psf, otherwise interpret from catalogue or image if possible innerclip, outerclip : float The seed (inner) and flood (outer) clipping level (sigmas). docov : bool If True then include covariance matrix in the fitting process. (default=True) cube_index : int For image cubes, slice determines which slice is used. Returns ------- sources : list List of sources measured.

Response:
def priorized_fit_islands(self, filename, catalogue, hdu_index=0, outfile=None, bkgin=None, rmsin=None, cores=1,
                              rms=None, bkg=None, beam=None, lat=None, imgpsf=None, catpsf=None, stage=3, ratio=None, outerclip=3,
                              doregroup=True, docov=True, cube_index=None):
        """
        Take an input catalog, and image, and optional background/noise images
        fit the flux and ra/dec for each of the given sources, keeping the morphology fixed

        if doregroup is true the groups will be recreated based on a matching radius/probability.
        if doregroup is false then the islands of the input catalog will be preserved.

        Multiple cores can be specified, and will be used.

        Parameters
        ----------
        filename : str or HDUList
            Image filename or HDUList.

        catalogue : str or list
            Input catalogue file name or list of OutputSource objects.

        hdu_index : int
            The index of the FITS HDU (extension).

        outfile : str
            file for printing catalog (NOT a table, just a text file of my own design)

        rmsin, bkgin : str or HDUList
            Filename or HDUList for the noise and background images.
            If either are None, then it will be calculated internally.

        cores : int
            Number of CPU cores to use. None means all cores.

        rms : float
            Use this rms for the entire image (will also assume that background is 0)

        beam : (major, minor, pa)
            Floats representing the synthesised beam (degrees).
            Replaces whatever is given in the FITS header.
            If the FITS header has no BMAJ/BMIN then this is required.

        lat : float
            The latitude of the telescope (declination of zenith).

        imgpsf : str or HDUList
             Filename or HDUList for a psf image.

        catpsf : str or HDUList
             Filename or HDUList for the catalogue psf image.

        stage : int
            Refitting stage

        ratio : float
            If not None - ratio of image psf to catalog psf, otherwise interpret from catalogue or image if possible

        innerclip, outerclip : float
            The seed (inner) and flood (outer) clipping level (sigmas).

        docov : bool
            If True then include covariance matrix in the fitting process. (default=True)

        cube_index : int
            For image cubes, slice determines which slice is used.

        Returns
        -------
        sources : list
            List of sources measured.

        """

        from AegeanTools.cluster import regroup

        self.load_globals(filename, hdu_index=hdu_index, bkgin=bkgin, rmsin=rmsin, rms=rms, bkg=bkg, cores=cores, verb=True,
                          do_curve=False, beam=beam, lat=lat, psf=imgpsf, docov=docov, cube_index=cube_index)

        global_data = self.global_data
        far = 10 * global_data.beam.a  # degrees
        # load the table and convert to an input source list
        if isinstance(catalogue, six.string_types):
            input_table = load_table(catalogue)
            input_sources = np.array(table_to_source_list(input_table))
        else:
            input_sources = np.array(catalogue)

        if len(input_sources) < 1:
            self.log.debug("No input sources for priorized fitting")
            return []

        # reject sources with missing params
        ok = True
        for param in ['ra', 'dec', 'peak_flux', 'a', 'b', 'pa']:
            if np.isnan(getattr(input_sources[0], param)):
                self.log.info("Source 0, is missing param '{0}'".format(param))
                ok = False
        if not ok:
            self.log.error("Missing parameters! Not fitting.")
            self.log.error("Maybe your table is missing or mis-labeled columns?")
            return []
        del ok

        src_mask = np.ones(len(input_sources), dtype=bool)

        # check to see if the input catalog contains psf information
        has_psf = getattr(input_sources[0], 'psf_a', None) is not None

        # the input sources are the initial conditions for our fits.
        # Expand each source size if needed.

        # If ratio is provided we just the psf by this amount
        if ratio is not None:
            self.log.info("Using ratio of {0} to scale input source shapes".format(ratio))
            far *= ratio
            for i, src in enumerate(input_sources):
                # Sources with an unknown psf are rejected as they are either outside the image
                # or outside the region covered by the psf
                skybeam = global_data.psfhelper.get_beam(src.ra, src.dec)
                if skybeam is None:
                    src_mask[i] = False
                    self.log.info("Excluding source ({0.island},{0.source}) due to lack of psf knowledge".format(src))
                    continue
                # the new source size is the previous size, convolved with the expanded psf
                src.a = np.sqrt(src.a ** 2 + (skybeam.a * 3600) ** 2 * (1 - 1 / ratio ** 2))
                src.b = np.sqrt(src.b ** 2 + (skybeam.b * 3600) ** 2 * (1 - 1 / ratio ** 2))
                # source with funky a/b are also rejected
                if not np.all(np.isfinite((src.a, src.b))):
                    self.log.info("Excluding source ({0.island},{0.source}) due to funky psf ({0.a},{0.b},{0.pa})".format(src))
                    src_mask[i] = False

        # if we know the psf from the input catalogue (has_psf), or if it was provided via a psf map
        # then we use that psf.
        elif catpsf is not None or has_psf:
            if catpsf is not None:
                self.log.info("Using catalog PSF from {0}".format(catpsf))
                psf_helper = PSFHelper(catpsf, None)  # might need to set the WCSHelper to be not None
            else:
                self.log.info("Using catalog PSF from input catalog")
                psf_helper = None
            for i, src in enumerate(input_sources):
                if (src.psf_a <=0) or (src.psf_b <=0):
                    src_mask[i] = False
                    self.log.info("Excluding source ({0.island},{0.source}) due to psf_a/b <=0".format(src))
                    continue
                if has_psf:
                    catbeam = Beam(src.psf_a / 3600, src.psf_b / 3600, src.psf_pa)
                else:
                    catbeam = psf_helper.get_beam(src.ra, src.dec)
                imbeam = global_data.psfhelper.get_beam(src.ra, src.dec)
                # If either of the above are None then we skip this source.
                if catbeam is None or imbeam is None:
                    src_mask[i] = False
                    self.log.info("Excluding source ({0.island},{0.source}) due to lack of psf knowledge".format(src))
                    continue

                # TODO: The following assumes that the various psf's are scaled versions of each other
                # and makes no account for differing position angles. This needs to be checked and/or addressed.

                # deconvolve the source shape from the catalogue psf
                src.a = (src.a / 3600) ** 2 - catbeam.a ** 2 + imbeam.a ** 2  # degrees

                # clip the minimum source shape to be the image psf
                if src.a < 0:
                    src.a = imbeam.a * 3600  # arcsec
                else:
                    src.a = np.sqrt(src.a) * 3600  # arcsec

                src.b = (src.b / 3600) ** 2 - catbeam.b ** 2 + imbeam.b ** 2
                if src.b < 0:
                    src.b = imbeam.b * 3600  # arcsec
                else:
                    src.b = np.sqrt(src.b) * 3600  # arcsec
        else:
            self.log.info("Not scaling input source sizes")

        self.log.info("{0} sources in catalog".format(len(input_sources)))
        self.log.info("{0} sources accepted".format(sum(src_mask)))

        if len(src_mask) < 1:
            self.log.debug("No sources accepted for priorized fitting")
            return []

        input_sources = input_sources[src_mask]
        # redo the grouping if required
        if doregroup:
            groups = regroup(input_sources, eps=np.sqrt(2), far=far)
        else:
            groups = list(island_itergen(input_sources))

        if cores == 1:  # single-threaded, no parallel processing
            queue = []
        else:
            queue = pprocess.Queue(limit=cores, reuse=1)
            fit_parallel = queue.manage(pprocess.MakeReusable(self._refit_islands))

        sources = []
        island_group = []
        group_size = 20

        for i, island in enumerate(groups):
            island_group.append(island)
            # If the island group is full queue it for the subprocesses to fit
            if len(island_group) >= group_size:
                if cores > 1:
                    fit_parallel(island_group, stage, outerclip, istart=i)
                else:
                    res = self._refit_islands(island_group, stage, outerclip, istart=i)
                    queue.append(res)
                island_group = []

        # The last partially-filled island group also needs to be queued for fitting
        if len(island_group) > 0:
            if cores > 1:
                fit_parallel(island_group, stage, outerclip, istart=i)
            else:
                res = self._refit_islands(island_group, stage, outerclip, istart=i)
                queue.append(res)

        # now unpack the fitting results in to a list of sources
        for s in queue:
            sources.extend(s)

        sources = sorted(sources)

        # Write the output to the output file
        if outfile:
            print(header.format("{0}-({1})".format(__version__, __date__), filename), file=outfile)
            print(OutputSource.header, file=outfile)

        components = 0
        for source in sources:
            if isinstance(source, OutputSource):
                components += 1
                if outfile:
                    print(str(source), file=outfile)

        self.log.info("fit {0} components".format(components))
        self.sources.extend(sources)
        return sources


Instruction:
Can you generate the documentation for the following Python 3 function
def check_table_formats(files):
    cont = True
    formats = get_table_formats()
    for t in files.split(','):
        _, ext = os.path.splitext(t)
        ext = ext[1:].lower()
        if ext not in formats:
            cont = False
            log.warn("Format not supported for {0} ({1})".format(t, ext))
    if not cont:
        log.error("Invalid table format specified.")
    return cont

Response: Checks if a list of files are of a recognizable output type.


Instruction:
Create a Python 3 function for
printing a list of all the file formats that are supported for writing.

Response:
def show_formats():
    """
    Print a list of all the file formats that are supported for writing.
    The file formats are determined by their extensions.

    Returns
    -------
    None
    """
    fmts = {
        "ann": "Kvis annotation",
        "reg": "DS9 regions file",
        "fits": "FITS Binary Table",
        "csv": "Comma separated values",
        "tab": "tabe separated values",
        "tex": "LaTeX table format",
        "html": "HTML table",
        "vot": "VO-Table",
        "xml": "VO-Table",
        "db": "Sqlite3 database",
        "sqlite": "Sqlite3 database"}
    supported = get_table_formats()
    print("Extension |     Description       | Supported?")
    for k in sorted(fmts.keys()):
        print("{0:10s} {1:24s} {2}".format(k, fmts[k], k in supported))
    return


Instruction:
Make a summary of the following Python 3 code
def update_meta_data(meta=None):
    if meta is None:
        meta = {}
    if 'DATE' not in meta:
        meta['DATE'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
    if 'PROGRAM' not in meta:
        meta['PROGRAM'] = "AegeanTools.catalogs"
        meta['PROGVER'] = "{0}-({1})".format(__version__, __date__)
    return meta

Response: Modify the metadata dictionary.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def save_catalog(filename, catalog, meta=None, prefix=None):
    ascii_table_formats = {'csv': 'csv', 'tab': 'tab', 'tex': 'latex', 'html': 'html'}
    # .ann and .reg are handled by me
    meta = update_meta_data(meta)
    extension = os.path.splitext(filename)[1][1:].lower()
    if extension in ['ann', 'reg']:
        writeAnn(filename, catalog, extension)
    elif extension in ['db', 'sqlite']:
        writeDB(filename, catalog, meta)
    elif extension in ['hdf5', 'fits', 'vo', 'vot', 'xml']:
        write_catalog(filename, catalog, extension, meta, prefix=prefix)
    elif extension in ascii_table_formats.keys():
        write_catalog(filename, catalog, fmt=ascii_table_formats[extension], meta=meta, prefix=prefix)
    else:
        log.warning("extension not recognised {0}".format(extension))
        log.warning("You get tab format")
        write_catalog(filename, catalog, fmt='tab', prefix=prefix)
    return

Response: Save a catalogue of sources using filename as a model.


Instruction:
Can you write a function in Python 3 where it
loads a catalogue and extract the source positions

Response:
def load_catalog(filename):
    """
    Load a catalogue and extract the source positions (only)

    Parameters
    ----------
    filename : str
        Filename to read. Supported types are csv, tab, tex, vo, vot, and xml.

    Returns
    -------
    catalogue : list
        A list of [ (ra, dec), ...]

    """
    supported = get_table_formats()

    fmt = os.path.splitext(filename)[-1][1:].lower()  # extension sans '.'

    if fmt in ['csv', 'tab', 'tex'] and fmt in supported:
        log.info("Reading file {0}".format(filename))
        t = ascii.read(filename)
        catalog = list(zip(t.columns['ra'], t.columns['dec']))

    elif fmt in ['vo', 'vot', 'xml'] and fmt in supported:
        log.info("Reading file {0}".format(filename))
        t = parse_single_table(filename)
        catalog = list(zip(t.array['ra'].tolist(), t.array['dec'].tolist()))

    else:
        log.info("Assuming ascii format, reading first two columns")
        lines = [a.strip().split() for a in open(filename, 'r').readlines() if not a.startswith('#')]
        try:
            catalog = [(float(a[0]), float(a[1])) for a in lines]
        except:
            log.error("Expecting two columns of floats but failed to parse")
            log.error("Catalog file {0} not loaded".format(filename))
            raise Exception("Could not determine file format")

    return catalog


Instruction:
Can you tell what is the following Python 3 function doing
def load_table(filename):
    supported = get_table_formats()

    fmt = os.path.splitext(filename)[-1][1:].lower()  # extension sans '.'

    if fmt in ['csv', 'tab', 'tex'] and fmt in supported:
        log.info("Reading file {0}".format(filename))
        t = ascii.read(filename)
    elif fmt in ['vo', 'vot', 'xml', 'fits', 'hdf5'] and fmt in supported:
        log.info("Reading file {0}".format(filename))
        t = Table.read(filename)
    else:
        log.error("Table format not recognized or supported")
        log.error("{0} [{1}]".format(filename, fmt))
        raise Exception("Table format not recognized or supported")
    return t

Response: Load a table from a given file.


Instruction:
Create a Python 3 function for
writing a table to a file.

Response:
def write_table(table, filename):
    """
    Write a table to a file.

    Parameters
    ----------
    table : Table
        Table to be written

    filename : str
        Destination for saving table.

    Returns
    -------
    None
    """
    try:
        if os.path.exists(filename):
            os.remove(filename)
        table.write(filename)
        log.info("Wrote {0}".format(filename))
    except Exception as e:
        if "Format could not be identified" not in e.message:
            raise e
        else:
            fmt = os.path.splitext(filename)[-1][1:].lower()  # extension sans '.'
            raise Exception("Cannot auto-determine format for {0}".format(fmt))
    return


Instruction:
Given the following Python 3 function, write the documentation
def table_to_source_list(table, src_type=OutputSource):
    source_list = []
    if table is None:
        return source_list

    for row in table:
        # Initialise our object
        src = src_type()
        # look for the columns required by our source object
        for param in src_type.names:
            if param in table.colnames:
                # copy the value to our object
                val = row[param]
                # hack around float32's broken-ness
                if isinstance(val, np.float32):
                    val = np.float64(val)
                setattr(src, param, val)
        # save this object to our list of sources
        source_list.append(src)
    return source_list

Response: Convert a table of data into a list of sources.


Instruction:
Here you have a function in Python 3, explain what it does
def write_catalog(filename, catalog, fmt=None, meta=None, prefix=None):
    if meta is None:
        meta = {}

    if prefix is None:
        pre=''
    else:
        pre = prefix + '_'

    def writer(filename, catalog, fmt=None):
        """
        construct a dict of the data
        this method preserves the data types in the VOTable
        """
        tab_dict = {}
        name_list = []
        for name in catalog[0].names:
            col_name = name
            if catalog[0].galactic:
                if name.startswith('ra'):
                    col_name = 'lon'+name[2:]
                elif name.endswith('ra'):
                    col_name = name[:-2] + 'lon'
                elif name.startswith('dec'):
                    col_name = 'lat'+name[3:]
                elif name.endswith('dec'):
                    col_name = name[:-3] + 'lat'
            col_name = pre + col_name
            tab_dict[col_name] = [getattr(c, name, None) for c in catalog]
            name_list.append(col_name)
        t = Table(tab_dict, meta=meta)
        # re-order the columns
        t = t[[n for n in name_list]]

        if fmt is not None:
            if fmt in ["vot", "vo", "xml"]:
                vot = from_table(t)
                # description of this votable
                vot.description = repr(meta)
                writetoVO(vot, filename)
            elif fmt in ['hdf5']:
                t.write(filename, path='data', overwrite=True)
            elif fmt in ['fits']:
                writeFITSTable(filename, t)
            else:
                ascii.write(t, filename, fmt, overwrite=True)
        else:
            ascii.write(t, filename, overwrite=True)
        return

    # sort the sources into types and then write them out individually
    components, islands, simples = classify_catalog(catalog)

    if len(components) > 0:
        new_name = "{1}{0}{2}".format('_comp', *os.path.splitext(filename))
        writer(new_name, components, fmt)
        log.info("wrote {0}".format(new_name))
    if len(islands) > 0:
        new_name = "{1}{0}{2}".format('_isle', *os.path.splitext(filename))
        writer(new_name, islands, fmt)
        log.info("wrote {0}".format(new_name))
    if len(simples) > 0:
        new_name = "{1}{0}{2}".format('_simp', *os.path.splitext(filename))
        writer(new_name, simples, fmt)
        log.info("wrote {0}".format(new_name))
    return

Response: Writes a list of sources to a file with format determined by extension.


Instruction:
Write a Python 3 script to
convert a table into a FITSTable and then write to disk.

Response:
def writeFITSTable(filename, table):
    """
    Convert a table into a FITSTable and then write to disk.

    Parameters
    ----------
    filename : str
        Filename to write.

    table : Table
        Table to write.

    Returns
    -------
    None

    Notes
    -----
    Due to a bug in numpy, `int32` and `float32` are converted to `int64` and `float64` before writing.
    """
    def FITSTableType(val):
        """
        Return the FITSTable type corresponding to each named parameter in obj
        """
        if isinstance(val, bool):
            types = "L"
        elif isinstance(val, (int, np.int64, np.int32)):
            types = "J"
        elif isinstance(val, (float, np.float64, np.float32)):
            types = "E"
        elif isinstance(val, six.string_types):
            types = "{0}A".format(len(val))
        else:
            log.warning("Column {0} is of unknown type {1}".format(val, type(val)))
            log.warning("Using 5A")
            types = "5A"
        return types

    cols = []
    for name in table.colnames:
        cols.append(fits.Column(name=name, format=FITSTableType(table[name][0]), array=table[name]))
    cols = fits.ColDefs(cols)
    tbhdu = fits.BinTableHDU.from_columns(cols)
    for k in table.meta:
        tbhdu.header['HISTORY'] = ':'.join((k, table.meta[k]))
    tbhdu.writeto(filename, overwrite=True)


Instruction:
Make a summary of the following Python 3 code
def writeIslandContours(filename, catalog, fmt='reg'):
    if fmt != 'reg':
        log.warning("Format {0} not yet supported".format(fmt))
        log.warning("not writing anything")
        return

    out = open(filename, 'w')
    print("#Aegean island contours", file=out)
    print("#AegeanTools.catalogs version {0}-({1})".format(__version__, __date__), file=out)
    line_fmt = 'image;line({0},{1},{2},{3})'
    text_fmt = 'fk5; text({0},{1}) # text={{{2}}}'
    mas_fmt = 'image; line({1},{0},{3},{2}) #color = yellow'
    x_fmt = 'image; point({1},{0}) # point=x'
    for c in catalog:
        contour = c.contour
        if len(contour) > 1:
            for p1, p2 in zip(contour[:-1], contour[1:]):
                print(line_fmt.format(p1[1] + 0.5, p1[0] + 0.5, p2[1] + 0.5, p2[0] + 0.5), file=out)
            print(line_fmt.format(contour[-1][1] + 0.5, contour[-1][0] + 0.5, contour[0][1] + 0.5,
                                          contour[0][0] + 0.5), file=out)
        # comment out lines that have invalid ra/dec (WCS problems)
        if np.nan in [c.ra, c.dec]:
            print('#', end=' ', file=out)
        # some islands may not have anchors because they don't have any contours
        if len(c.max_angular_size_anchors) == 4:
            print(text_fmt.format(c.ra, c.dec, c.island), file=out)
            print(mas_fmt.format(*[a + 0.5 for a in c.max_angular_size_anchors]), file=out)
        for p1, p2 in c.pix_mask:
            # DS9 uses 1-based instead of 0-based indexing
            print(x_fmt.format(p1 + 1, p2 + 1), file=out)
    out.close()
    return

Response: Writes an output file in ds9. reg format that outlines the boundaries of each island.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def writeIslandBoxes(filename, catalog, fmt):
    if fmt not in ['reg', 'ann']:
        log.warning("Format not supported for island boxes{0}".format(fmt))
        return  # fmt not supported

    out = open(filename, 'w')
    print("#Aegean Islands", file=out)
    print("#Aegean version {0}-({1})".format(__version__, __date__), file=out)

    if fmt == 'reg':
        print("IMAGE", file=out)
        box_fmt = 'box({0},{1},{2},{3}) #{4}'
    else:
        print("COORD P", file=out)
        box_fmt = 'box P {0} {1} {2} {3} #{4}'

    for c in catalog:
        # x/y swap for pyfits/numpy translation
        ymin, ymax, xmin, xmax = c.extent
        # +1 for array/image offset
        xcen = (xmin + xmax) / 2.0 + 1
        # + 0.5 in each direction to make lines run 'between' DS9 pixels
        xwidth = xmax - xmin + 1
        ycen = (ymin + ymax) / 2.0 + 1
        ywidth = ymax - ymin + 1
        print(box_fmt.format(xcen, ycen, xwidth, ywidth, c.island), file=out)
    out.close()
    return

Response: Writes an output file in ds9. reg or kvis. ann format that contains bounding boxes for all the islands.


Instruction:
Write a Python 3 script for
writing an annotation file that can be read by Kvis or DS9.

Response:
def writeAnn(filename, catalog, fmt):
    """
    Write an annotation file that can be read by Kvis (.ann) or DS9 (.reg).
    Uses ra/dec from catalog.
    Draws ellipses if bmaj/bmin/pa are in catalog. Draws 30" circles otherwise.

    Only :class:`AegeanTools.models.OutputSource` will appear in the annotation file
    unless there are none, in which case :class:`AegeanTools.models.SimpleSource` (if present)
    will be written. If any :class:`AegeanTools.models.IslandSource` objects are present then
    an island contours file will be written.

    Parameters
    ----------
    filename : str
        Output filename base.

    catalog : list
        List of sources.

    fmt : ['ann', 'reg']
        Output file type.

    Returns
    -------
    None

    See Also
    --------
    AegeanTools.catalogs.writeIslandContours
    """
    if fmt not in ['reg', 'ann']:
        log.warning("Format not supported for island boxes{0}".format(fmt))
        return  # fmt not supported

    components, islands, simples = classify_catalog(catalog)
    if len(components) > 0:
        cat = sorted(components)
        suffix = "comp"
    elif len(simples) > 0:
        cat = simples
        suffix = "simp"
    else:
        cat = []

    if len(cat) > 0:
        ras = [a.ra for a in cat]
        decs = [a.dec for a in cat]
        if not hasattr(cat[0], 'a'):  # a being the variable that I used for bmaj.
            bmajs = [30 / 3600.0 for a in cat]
            bmins = bmajs
            pas = [0 for a in cat]
        else:
            bmajs = [a.a / 3600.0 for a in cat]
            bmins = [a.b / 3600.0 for a in cat]
            pas = [a.pa for a in cat]

        names = [a.__repr__() for a in cat]
        if fmt == 'ann':
            new_file = re.sub('.ann$', '_{0}.ann'.format(suffix), filename)
            out = open(new_file, 'w')
            print("#Aegean version {0}-({1})".format(__version__, __date__), file=out)
            print('PA SKY', file=out)
            print('FONT hershey12', file=out)
            print('COORD W', file=out)
            formatter = "ELLIPSE W {0} {1} {2} {3} {4:+07.3f} #{5}\nTEXT W {0} {1} {5}"
        else:  # reg
            new_file = re.sub('.reg$', '_{0}.reg'.format(suffix), filename)
            out = open(new_file, 'w')
            print("#Aegean version {0}-({1})".format(__version__, __date__), file=out)
            print("fk5", file=out)
            formatter = 'ellipse {0} {1} {2:.9f}d {3:.9f}d {4:+07.3f}d # text="{5}"'
            # DS9 has some strange ideas about position angle
            pas = [a - 90 for a in pas]

        for ra, dec, bmaj, bmin, pa, name in zip(ras, decs, bmajs, bmins, pas, names):
            # comment out lines that have invalid or stupid entries
            if np.nan in [ra, dec, bmaj, bmin, pa] or bmaj >= 180:
                print('#', end=' ', file=out)
            print(formatter.format(ra, dec, bmaj, bmin, pa, name), file=out)
        out.close()
        log.info("wrote {0}".format(new_file))
    if len(islands) > 0:
        if fmt == 'reg':
            new_file = re.sub('.reg$', '_isle.reg', filename)
        elif fmt == 'ann':
            log.warning('kvis islands are currently not working')
            return
        else:
            log.warning('format {0} not supported for island annotations'.format(fmt))
            return
        writeIslandContours(new_file, islands, fmt)
        log.info("wrote {0}".format(new_file))

    return


Instruction:
Explain what the following Python 3 code does
def writeDB(filename, catalog, meta=None):

    def sqlTypes(obj, names):
        """
        Return the sql type corresponding to each named parameter in obj
        """
        types = []
        for n in names:
            val = getattr(obj, n)
            if isinstance(val, bool):
                types.append("BOOL")
            elif isinstance(val, (int, np.int64, np.int32)):
                types.append("INT")
            elif isinstance(val, (float, np.float64, np.float32)):  # float32 is bugged and claims not to be a float
                types.append("FLOAT")
            elif isinstance(val, six.string_types):
                types.append("VARCHAR")
            else:
                log.warning("Column {0} is of unknown type {1}".format(n, type(n)))
                log.warning("Using VARCHAR")
                types.append("VARCHAR")
        return types

    if os.path.exists(filename):
        log.warning("overwriting {0}".format(filename))
        os.remove(filename)
    conn = sqlite3.connect(filename)
    db = conn.cursor()
    # determine the column names by inspecting the catalog class
    for t, tn in zip(classify_catalog(catalog), ["components", "islands", "simples"]):
        if len(t) < 1:
            continue  #don't write empty tables
        col_names = t[0].names
        col_types = sqlTypes(t[0], col_names)
        stmnt = ','.join(["{0} {1}".format(a, b) for a, b in zip(col_names, col_types)])
        db.execute('CREATE TABLE {0} ({1})'.format(tn, stmnt))
        stmnt = 'INSERT INTO {0} ({1}) VALUES ({2})'.format(tn, ','.join(col_names), ','.join(['?' for i in col_names]))
        # expend the iterators that are created by python 3+
        data = list(map(nulls, list(r.as_list() for r in t)))
        db.executemany(stmnt, data)
        log.info("Created table {0}".format(tn))
    # metadata add some meta data
    db.execute("CREATE TABLE meta (key VARCHAR, val VARCHAR)")
    for k in meta:
        db.execute("INSERT INTO meta (key, val) VALUES (?,?)", (k, meta[k]))
    conn.commit()
    log.info(db.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall())
    conn.close()
    log.info("Wrote file {0}".format(filename))
    return

Response: Writes a sqlite3 database containing one table for each source type and each source type in the catalog.


Instruction:
How would you code a function in Python 3 to
calculate the normalised distance between two sources.

Response:
def norm_dist(src1, src2):
    """
    Calculate the normalised distance between two sources.
    Sources are elliptical Gaussians.

    The normalised distance is calculated as the GCD distance between the centers,
    divided by quadrature sum of the radius of each ellipse along a line joining the two ellipses.

    For ellipses that touch at a single point, the normalized distance will be 1/sqrt(2).

    Parameters
    ----------
    src1, src2 : object
        The two positions to compare. Objects must have the following parameters: (ra, dec, a, b, pa).

    Returns
    -------
    dist: float
        The normalised distance.

    """
    if np.all(src1 == src2):
        return 0
    dist = gcd(src1.ra, src1.dec, src2.ra, src2.dec) # degrees

    # the angle between the ellipse centers
    phi = bear(src1.ra, src1.dec, src2.ra, src2.dec) # Degrees
    # Calculate the radius of each ellipse along a line that joins their centers.
    r1 = src1.a*src1.b / np.hypot(src1.a * np.sin(np.radians(phi - src1.pa)),
                                  src1.b * np.cos(np.radians(phi - src1.pa)))
    r2 = src2.a*src2.b / np.hypot(src2.a * np.sin(np.radians(180 + phi - src2.pa)),
                                  src2.b * np.cos(np.radians(180 + phi - src2.pa)))
    R = dist / (np.hypot(r1, r2) / 3600)
    return R


Instruction:
Here you have a function in Python 3, explain what it does
def sky_dist(src1, src2):

    if np.all(src1 == src2):
        return 0
    return gcd(src1.ra, src1.dec, src2.ra, src2.dec)

Response: Returns the approximate circle distance between two sources.


Instruction:
Create a Python 3 function for
doing a pairwise comparison of all sources and determine if they have a normalized distance within eps.

Response:
def pairwise_ellpitical_binary(sources, eps, far=None):
    """
    Do a pairwise comparison of all sources and determine if they have a normalized distance within
    eps.

    Form this into a matrix of shape NxN.

    Parameters
    ----------
    sources : list
        A list of sources (objects with parameters: ra,dec,a,b,pa)

    eps : float
        Normalised distance constraint.

    far : float
        If sources have a dec that differs by more than this amount then they are considered to be not matched.
        This is a short-cut around performing GCD calculations.

    Returns
    -------
    prob : numpy.ndarray
        A 2d array of True/False.

    See Also
    --------
    :func:`AegeanTools.cluster.norm_dist`
    """
    if far is None:
        far = max(a.a/3600 for a in sources)
    l = len(sources)
    distances = np.zeros((l, l), dtype=bool)
    for i in range(l):
        for j in range(i, l):
            if i == j:
                distances[i, j] = False
                continue
            src1 = sources[i]
            src2 = sources[j]
            if src2.dec - src1.dec > far:
                break
            if abs(src2.ra - src1.ra)*np.cos(np.radians(src1.dec)) > far:
                continue
            distances[i, j] = norm_dist(src1, src2) > eps
            distances[j, i] = distances[i, j]
    return distances


Instruction:
How would you code a function in Python 3 to
return a list of islands that are vectorized within a catalog.

Response:
def regroup_vectorized(srccat, eps, far=None, dist=norm_dist):
    """
    Regroup the islands of a catalog according to their normalised distance.

    Assumes srccat is recarray-like for efficiency.
    Return a list of island groups.

    Parameters
    ----------
    srccat : np.rec.arry or pd.DataFrame
        Should have the following fields[units]:
        ra[deg],dec[deg], a[arcsec],b[arcsec],pa[deg], peak_flux[any]
    eps : float
        maximum normalised distance within which sources are considered to be
        grouped
    far : float
        (degrees) sources that are further than this distance apart will not
        be grouped, and will not be tested.
        Default = 0.5.
    dist : func
        a function that calculates the distance between a source and each
        element of an array of sources.
        Default = :func:`AegeanTools.cluster.norm_dist`

    Returns
    -------
    islands : list of lists
        Each island contians integer indices for members from srccat
        (in descending dec order).
    """
    if far is None:
        far = 0.5  # 10*max(a.a/3600 for a in srccat)

    # most negative declination first
    # XXX: kind='mergesort' ensures stable sorting for determinism.
    #      Do we need this?
    order = np.argsort(srccat.dec, kind='mergesort')[::-1]
    # TODO: is it better to store groups as arrays even if appends are more
    #       costly?
    groups = [[order[0]]]
    for idx in order[1:]:
        rec = srccat[idx]
        # TODO: Find out if groups are big enough for this to give us a speed
        #       gain. If not, get distance to all entries in groups above
        #       decmin simultaneously.
        decmin = rec.dec - far
        for group in reversed(groups):
            # when an island's largest (last) declination is smaller than
            # decmin, we don't need to look at any more islands
            if srccat.dec[group[-1]] < decmin:
                # new group
                groups.append([idx])
            rafar = far / np.cos(np.radians(rec.dec))
            group_recs = np.take(srccat, group, mode='clip')
            group_recs = group_recs[abs(rec.ra - group_recs.ra) <= rafar]
            if len(group_recs) and dist(rec, group_recs).min() < eps:
                group.append(idx)
                break
        else:
            # new group
            groups.append([idx])

    # TODO?: a more numpy-like interface would return only an array providing
    #        the mapping:
    #    group_idx = np.empty(len(srccat), dtype=int)
    #    for i, group in enumerate(groups):
    #        group_idx[group] = i
    #    return group_idx
    return groups


Instruction:
Can you tell what is the following Python 3 function doing
def regroup(catalog, eps, far=None, dist=norm_dist):

    if isinstance(catalog, str):
        table = load_table(catalog)
        srccat = table_to_source_list(table)
    else:
        try:
            srccat = catalog
            _ = catalog[0].ra, catalog[0].dec, catalog[0].a, catalog[0].b, catalog[0].pa, catalog[0].peak_flux

        except AttributeError as e:
            log.error("catalog is not understood.")
            log.error("catalog: Should be a list of objects with the following properties[units]:\n" +
                      "ra[deg],dec[deg], a[arcsec],b[arcsec],pa[deg], peak_flux[any]")
            raise e

    log.info("Regrouping islands within catalog")
    log.debug("Calculating distances")

    if far is None:
        far = 0.5  # 10*max(a.a/3600 for a in srccat)

    srccat_array = np.rec.fromrecords(
        [(s.ra, s.dec, s.a, s.b, s.pa, s.peak_flux)
         for s in srccat],
        names=['ra', 'dec', 'a', 'b', 'pa', 'peak_flux'])
    groups = regroup_vectorized(srccat_array, eps=eps, far=far, dist=dist)
    groups = [[srccat[idx] for idx in group]
              for group in groups]

    islands = []
    # now that we have the groups, we relabel the sources to have (island,component) in flux order
    # note that the order of sources within an island list is not changed - just their labels
    for isle, group in enumerate(groups):
        for comp, src in enumerate(sorted(group, key=lambda x: -1*x.peak_flux)):
            src.island = isle
            src.source = comp
        islands.append(group)
    return islands

Response: Regroups the islands of a catalog according to their normalised distance.


Instruction:
Can you implement a function in Python 3 that
loads a file from disk and return an HDUList if it is already an HDUList

Response:
def load_file_or_hdu(filename):
    """
    Load a file from disk and return an HDUList
    If filename is already an HDUList return that instead

    Parameters
    ----------
    filename : str or HDUList
        File or HDU to be loaded

    Returns
    -------
    hdulist : HDUList
    """
    if isinstance(filename, fits.HDUList):
        hdulist = filename
    else:
        hdulist = fits.open(filename, ignore_missing_end=True)
    return hdulist


Instruction:
Can you create a Python 3 function that
compresses a file using decimation factor.

Response:
def compress(datafile, factor, outfile=None):
    """
    Compress a file using decimation.

    Parameters
    ----------
    datafile : str or HDUList
        Input data to be loaded. (HDUList will be modified if passed).

    factor : int
        Decimation factor.

    outfile : str
        File to be written. Default = None, which means don't write a file.

    Returns
    -------
    hdulist : HDUList
        A decimated HDUList

    See Also
    --------
    :func:`AegeanTools.fits_interp.expand`
    """
    if not (factor > 0 and isinstance(factor, int)):
        logging.error("factor must be a positive integer")
        return None

    hdulist = load_file_or_hdu(datafile)

    header = hdulist[0].header
    data = np.squeeze(hdulist[0].data)
    cx, cy = data.shape[0], data.shape[1]

    nx = cx // factor
    ny = cy // factor
    # check to see if we will have some residual data points
    lcx = cx % factor
    lcy = cy % factor
    if lcx > 0:
        nx += 1
    if lcy > 0:
        ny += 1
    # decimate the data
    new_data = np.empty((nx + 1, ny + 1))
    new_data[:nx, :ny] = data[::factor, ::factor]
    # copy the last row/col across
    new_data[-1, :ny] = data[-1, ::factor]
    new_data[:nx, -1] = data[::factor, -1]
    new_data[-1, -1] = data[-1, -1]

    # TODO: Figure out what to do when CD2_1 and CD1_2 are non-zero
    if 'CDELT1' in header:
        header['CDELT1'] *= factor
    elif 'CD1_1' in header:
        header['CD1_1'] *= factor
    else:
        logging.error("Error: Can't find CDELT1 or CD1_1")
        return None
    if 'CDELT2' in header:
        header['CDELT2'] *= factor
    elif "CD2_2" in header:
        header['CD2_2'] *= factor
    else:
        logging.error("Error: Can't find CDELT2 or CD2_2")
        return None
    # Move the reference pixel so that the WCS is correct
    header['CRPIX1'] = (header['CRPIX1'] + factor - 1) / factor
    header['CRPIX2'] = (header['CRPIX2'] + factor - 1) / factor

    # Update the header so that we can do the correct interpolation later on
    header['BN_CFAC'] = (factor, "Compression factor (grid size) used by BANE")
    header['BN_NPX1'] = (header['NAXIS1'], 'original NAXIS1 value')
    header['BN_NPX2'] = (header['NAXIS2'], 'original NAXIS2 value')
    header['BN_RPX1'] = (lcx, 'Residual on axis 1')
    header['BN_RPX2'] = (lcy, 'Residual on axis 2')
    header['HISTORY'] = "Compressed by a factor of {0}".format(factor)

    # save the changes
    hdulist[0].data = np.array(new_data, dtype=np.float32)
    hdulist[0].header = header
    if outfile is not None:
        hdulist.writeto(outfile, overwrite=True)
        logging.info("Wrote: {0}".format(outfile))
    return hdulist


Instruction:
Can you implement a function in Python 3 that
expands and interpolate the given data file using the given method.

Response:
def expand(datafile, outfile=None):
    """
    Expand and interpolate the given data file using the given method.
    Datafile can be a filename or an HDUList

    It is assumed that the file has been compressed and that there are `BN_?` keywords in the
    fits header that describe how the compression was done.

    Parameters
    ----------
    datafile : str or HDUList
        filename or HDUList of file to work on

    outfile : str
        filename to write to (default = None)

    Returns
    -------
    hdulist : HDUList
        HDUList of the expanded data.

    See Also
    --------
    :func:`AegeanTools.fits_interp.compress`

    """
    hdulist = load_file_or_hdu(datafile)

    header = hdulist[0].header
    data = hdulist[0].data
    # Check for the required key words, only expand if they exist
    if not all(a in header for a in ['BN_CFAC', 'BN_NPX1', 'BN_NPX2', 'BN_RPX1', 'BN_RPX2']):
        return hdulist

    factor = header['BN_CFAC']
    (gx, gy) = np.mgrid[0:header['BN_NPX2'], 0:header['BN_NPX1']]
    # fix the last column of the grid to account for residuals
    lcx = header['BN_RPX2']
    lcy = header['BN_RPX1']

    rows = (np.arange(data.shape[0]) + int(lcx/factor))*factor
    cols = (np.arange(data.shape[1]) + int(lcy/factor))*factor

    # Do the interpolation
    hdulist[0].data = np.array(RegularGridInterpolator((rows,cols), data)((gx, gy)), dtype=np.float32)

    # update the fits keywords so that the WCS is correct
    header['CRPIX1'] = (header['CRPIX1'] - 1) * factor + 1
    header['CRPIX2'] = (header['CRPIX2'] - 1) * factor + 1

    if 'CDELT1' in header:
        header['CDELT1'] /= factor
    elif 'CD1_1' in header:
        header['CD1_1'] /= factor
    else:
        logging.error("Error: Can't find CD1_1 or CDELT1")
        return None

    if 'CDELT2' in header:
        header['CDELT2'] /= factor
    elif "CD2_2" in header:
        header['CD2_2'] /= factor
    else:
        logging.error("Error: Can't find CDELT2 or CD2_2")
        return None

    header['HISTORY'] = 'Expanded by factor {0}'.format(factor)

    # don't need these any more so delete them.
    del header['BN_CFAC'], header['BN_NPX1'], header['BN_NPX2'], header['BN_RPX1'], header['BN_RPX2']
    hdulist[0].header = header
    if outfile is not None:
        hdulist.writeto(outfile, overwrite=True)
        logging.info("Wrote: {0}".format(outfile))
    return hdulist


Instruction:
Create a Python 3 function to
change autocommit mode of the current database entry.

Response:
def change_autocommit_mode(self, switch):
        """
        Strip and make a string case insensitive and ensure it is either 'true' or 'false'.

        If neither, prompt user for either value.
        When 'true', return True, and when 'false' return False.
        """
        parsed_switch = switch.strip().lower()
        if not parsed_switch in ['true', 'false']:
            self.send_response(
                self.iopub_socket, 'stream', {
                    'name': 'stderr',
                    'text': 'autocommit must be true or false.\n\n'
                }
            )

        switch_bool = (parsed_switch == 'true')
        committed = self.switch_autocommit(switch_bool)
        message = (
            'committed current transaction & ' if committed else '' +
            'switched autocommit mode to ' +
            str(self._autocommit)
        )
        self.send_response(
            self.iopub_socket, 'stream', {
                'name': 'stderr',
                'text': message,
            }
        )


Instruction:
Can you implement a function in Python 3 that
returns an ASCII Armor format for the given binary data.


Instruction:
How would you explain what the following Python 3 function does
def dearmor(text, verify=True):
    lines = text.strip().split('\n')
    data_lines = []
    check_data = None
    started = False
    in_body = False
    for line in lines:
        if line.startswith('-----BEGIN'):
            started = True
        elif line.startswith('-----END'):
            break
        elif started:
            if in_body:
                if line.startswith('='):
                    # Once we get the checksum data, we're done.
                    check_data = line[1:5].encode('ascii')
                    break
                else:
                    # This is part of the base64-encoded data.
                    data_lines.append(line)
            else:
                if line.strip():
                    # This is a header line, which we basically ignore for now.
                    pass
                else:
                    # The data starts after an empty line.
                    in_body = True
    b64_str = ''.join(data_lines)
    # Python 3's b64decode expects bytes, not a string. We know base64 is ASCII, though.
    data = base64.b64decode(b64_str.encode('ascii'))
    if verify and check_data:
        # The 24-bit CRC is in big-endian, so we add a null byte to the beginning.
        crc = struct.unpack('>L', b'\0' + base64.b64decode(check_data))[0]
        if crc != crc24(data):
            raise BadChecksumError()
    return data

Response: Given a string in ASCII Armor format returns the decoded binary data.


Instruction:
Here you have a function in Python 3, explain what it does
def unpad(text, block_size):
    end = len(text)
    if end == 0:
        return text
    padch = ord_safe(text[end - 1])
    if padch > block_size:
        # If the last byte value is larger than the block size, it's not padded.
        return text
    while end > 0 and ord_safe(text[end - 1]) in (0, padch):
        end -= 1
    return text[:end]

Response: Unpads the given text into a block of size block_size.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def pad(text, block_size, zero=False):
    num = block_size - (len(text) % block_size)
    ch = b'\0' if zero else chr(num).encode('latin-1')
    return text + (ch * num)

Response: Given a text string and a block size pads the text with zeros to the specified block size.


Instruction:
Explain what the following Python 3 code does
def deconstruct(self):
        name, path, args, kwargs = super(BaseEncryptedField, self).deconstruct()
        kwargs.update({
            #'key': self.cipher_key,
            'cipher': self.cipher_name,
            'charset': self.charset,
            'check_armor': self.check_armor,
            'versioned': self.versioned,
        })
        return name, path, args, kwargs

Response: Deconstruct the field for Django 1. 7 + migrations.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_cipher(self):
        return self.cipher_class.new(self.cipher_key, self.cipher_class.MODE_CBC, b'\0' * self.cipher_class.block_size)

Response: Returns a new Cipher object for each time we want to encrypt or decrypt.


Instruction:
Make a summary of the following Python 3 code
def find_packages_by_root_package(where):
    root_package = os.path.basename(where)
    packages = [ "%s.%s" % (root_package, sub_package)
                 for sub_package in find_packages(where)]
    packages.insert(0, root_package)
    return packages

Response: Better than excluding everything that is not needed collect only what is needed.


Instruction:
Can you create a Python 3 function that
clicks is a framework to simplify writing composable commands for command-line tools. This package extends the click_ functionality by adding support for commands that use configuration files. .. _click: https://click.pocoo.org/ EXAMPLE: A configuration file, like: .. code-block:: INI # -- FILE: foo.ini [foo] flag = yes name = Alice and Bob numbers = 1 4 9 16 25 filenames = foo/xxx.txt bar/baz/zzz.txt [person.alice] name = Alice birthyear = 1995 [person.bob] name = Bob birthyear = 2001 can be processed with: .. code-block:: python # EXAMPLE:

Response:
def make_long_description(marker=None, intro=None):
    """
    click_ is a framework to simplify writing composable commands for
    command-line tools. This package extends the click_ functionality
    by adding support for commands that use configuration files.

    .. _click: https://click.pocoo.org/

    EXAMPLE:

    A configuration file, like:

    .. code-block:: INI

        # -- FILE: foo.ini
        [foo]
        flag = yes
        name = Alice and Bob
        numbers = 1 4 9 16 25
        filenames = foo/xxx.txt
            bar/baz/zzz.txt

        [person.alice]
        name = Alice
        birthyear = 1995

        [person.bob]
        name = Bob
        birthyear = 2001

    can be processed with:

    .. code-block:: python

        # EXAMPLE:
    """
    if intro is None:
        intro = inspect.getdoc(make_long_description)

    with open("README.rst", "r") as infile:
        line = infile.readline()
        while not line.strip().startswith(marker):
            line = infile.readline()

        # -- COLLECT REMAINING: Usage example
        contents = infile.read()

    text = intro +"\n" + contents
    return text


Instruction:
Can you generate the documentation for the following Python 3 function
def pubsub_pop_message(self, deadline=None):
        if not self.subscribed:
            excep = ClientError("you must subscribe before using "
                                "pubsub_pop_message")
            raise tornado.gen.Return(excep)
        reply = None
        try:
            reply = self._reply_list.pop(0)
            raise tornado.gen.Return(reply)
        except IndexError:
            pass
        if deadline is not None:
            td = timedelta(seconds=deadline)
            yield self._condition.wait(timeout=td)
        else:
            yield self._condition.wait()
        try:
            reply = self._reply_list.pop(0)
        except IndexError:
            pass
        raise tornado.gen.Return(reply)

Response: Pops a message for a subscribed client.


Instruction:
Implement a function in Python 3 to
calculate the slopes and directions based on Tarboton s algorithm.

Response:
def _tarboton_slopes_directions(data, dX, dY, facets, ang_adj):
    """
    Calculate the slopes and directions based on the 8 sections from
    Tarboton http://www.neng.usu.edu/cee/faculty/dtarb/96wr03137.pdf
    """
    shp = np.array(data.shape) - 1

    direction = np.full(data.shape, FLAT_ID_INT, 'float64')
    mag = np.full(data.shape, FLAT_ID_INT, 'float64')

    slc0 = [slice(1, -1), slice(1, -1)]
    for ind in xrange(8):

        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(1 + e1[0], shp[0] + e1[0]),
                slice(1 + e1[1], shp[1] + e1[1])]
        slc2 = [slice(1 + e2[0], shp[0] + e2[0]),
                slice(1 + e2[1], shp[1] + e2[1])]

        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp)
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)

    # %%Now do the edges
    # if the edge is lower than the interior, we need to copy the value
    # from the interior (as an approximation)
    ids1 = (direction[:, 1] > np.pi / 2) \
        & (direction[:, 1] < 3 * np.pi / 2)
    direction[ids1, 0] = direction[ids1, 1]
    mag[ids1, 0] = mag[ids1, 1]
    ids1 = (direction[:, -2] < np.pi / 2) \
        | (direction[:, -2] > 3 * np.pi / 2)
    direction[ids1, -1] = direction[ids1, -2]
    mag[ids1, -1] = mag[ids1, -2]
    ids1 = (direction[1, :] > 0) & (direction[1, :] < np.pi)
    direction[0, ids1] = direction[1, ids1]
    mag[0, ids1] = mag[1, ids1]
    ids1 = (direction[-2, :] > np.pi) & (direction[-2, :] < 2 * np.pi)
    direction[-1, ids1] = direction[-2, ids1]
    mag[-1, ids1] = mag[-2, ids1]

    # Now update the edges in case they are higher than the interior (i.e.
    # look at the downstream angle)

    # left edge
    slc0 = [slice(1, -1), slice(0, 1)]
    for ind in [0, 1, 6, 7]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(1 + e1[0], shp[0] + e1[0]), slice(e1[1], 1 + e1[1])]
        slc2 = [slice(1 + e2[0], shp[0] + e2[0]), slice(e2[1], 1 + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp)
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # right edge
    slc0 = [slice(1, -1), slice(-1, None)]
    for ind in [2, 3, 4, 5]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(1 + e1[0], shp[0] + e1[0]),
                slice(shp[1] + e1[1], shp[1] + 1 + e1[1])]
        slc2 = [slice(1 + e2[0], shp[0] + e2[0]),
                slice(shp[1] + e2[1], shp[1] + 1 + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp)
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # top edge
    slc0 = [slice(0, 1), slice(1, -1)]
    for ind in [4, 5, 6, 7]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(e1[0], 1 + e1[0]), slice(1 + e1[1], shp[1] + e1[1])]
        slc2 = [slice(e2[0], 1 + e2[0]), slice(1 + e2[1], shp[1] + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp, 'top')
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # bottom edge
    slc0 = [slice(-1, None), slice(1, -1)]
    for ind in [0, 1, 2, 3]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(shp[0] + e1[0], shp[0] + 1 + e1[0]),
                slice(1 + e1[1], shp[1] + e1[1])]
        slc2 = [slice(shp[0] + e2[0], shp[0] + 1 + e2[0]),
                slice(1 + e2[1], shp[1] + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp, 'bot')
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # top-left corner
    slc0 = [slice(0, 1), slice(0, 1)]
    for ind in [6, 7]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(e1[0], 1 + e1[0]), slice(e1[1], 1 + e1[1])]
        slc2 = [slice(e2[0], 1 + e2[0]), slice(e2[1], 1 + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp, 'top')
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # top-right corner
    slc0 = [slice(0, 1), slice(-1, None)]
    for ind in [4, 5]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(e1[0], 1 + e1[0]),
                slice(shp[1] + e1[1], shp[1] + 1 + e1[1])]
        slc2 = [slice(e2[0], 1 + e2[0]),
                slice(shp[1] + e2[1], shp[1] + 1 + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp, 'top')
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # bottom-left corner
    slc0 = [slice(-1, None), slice(0, 1)]
    for ind in [0, 1]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(shp[0] + e1[0], shp[0] + 1 + e1[0]),
                slice(e1[1], 1 + e1[1])]
        slc2 = [slice(shp[0] + e2[0], shp[0] + 1 + e2[0]),
                slice(e2[1], 1 + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp, 'bot')
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)
    # bottom-right corner
    slc0 = [slice(-1, None), slice(-1, None)]
    for ind in [3, 4]:
        e1 = facets[ind][1]
        e2 = facets[ind][2]
        ang = ang_adj[ind]
        slc1 = [slice(shp[0] + e1[0], shp[0] + 1 + e1[0]),
                slice(shp[1] + e1[1], shp[1] + 1 + e1[1])]
        slc2 = [slice(shp[0] + e2[0], shp[0] + 1 + e2[0]),
                slice(shp[1] + e2[1], shp[1] + 1 + e2[1])]
        d1, d2, theta = _get_d1_d2(dX, dY, ind, e1, e2, shp, 'bot')
        mag, direction = _calc_direction(data, mag, direction, ang, d1, d2,
                                         theta, slc0, slc1, slc2)

    mag[mag > 0] = np.sqrt(mag[mag > 0])

    return mag, direction


Instruction:
Can you generate the documentation for the following Python 3 function
def _calc_direction(data, mag, direction, ang, d1, d2, theta,
                    slc0, slc1, slc2):
    """
    This function gives the magnitude and direction of the slope based on
    Tarboton's D_\infty method. This is a helper-function to
    _tarboton_slopes_directions
    """

    data0 = data[slc0]
    data1 = data[slc1]
    data2 = data[slc2]

    s1 = (data0 - data1) / d1
    s2 = (data1 - data2) / d2
    s1_2 = s1**2

    sd = (data0 - data2) / np.sqrt(d1**2 + d2**2)
    r = np.arctan2(s2, s1)
    rad2 = s1_2 + s2**2

    # Handle special cases
    # should be on diagonal
    b_s1_lte0 = s1 <= 0
    b_s2_lte0 = s2 <= 0
    b_s1_gt0 = s1 > 0
    b_s2_gt0 = s2 > 0

    I1 = (b_s1_lte0 & b_s2_gt0) | (r > theta)
    if I1.any():
        rad2[I1] = sd[I1] ** 2
        r[I1] = theta.repeat(I1.shape[1], 1)[I1]

    I2 = (b_s1_gt0 & b_s2_lte0) | (r < 0)  # should be on straight section
    if I2.any():
        rad2[I2] = s1_2[I2]
        r[I2] = 0

    I3 = b_s1_lte0 & (b_s2_lte0 | (b_s2_gt0 & (sd <= 0)))  # upslope or flat
    rad2[I3] = -1

    I4 = rad2 > mag[slc0]
    if I4.any():
        mag[slc0][I4] = rad2[I4]
        direction[slc0][I4] = r[I4] * ang[1] + ang[0] * np.pi/2

    return mag, direction

Response: This function calculates the magnitude and direction of the current Tarboton object based on Tarboton s D_\infty method.


Instruction:
Explain what the following Python 3 code does
def get(self, key, side):
        return getattr(self, side).ravel()[self.keys[key]]

Response: Returns an edge given a particular key


Instruction:
Implement a Python 3 function for
assigning data on the ith tile to the field of that tile.

Response:
def set_i(self, i, data, field, side):
        """ Assigns data on the i'th tile to the data 'field' of the 'side'
        edge of that tile
        """
        edge = self.get_i(i, side)
        setattr(edge, field, data[edge.slice])


Instruction:
How would you code a function in Python 3 to
assign data on the key tile to all the edges that are on the key tile.

Response:
def set_sides(self, key, data, field, local=False):
        """
        Assign data on the 'key' tile to all the edges
        """
        for side in ['left', 'right', 'top', 'bottom']:
            self.set(key, data, field, side, local)


Instruction:
Create a Python 3 function to
assign data to the edge on the neihboring tile which is on the neighbor_side of the key tile.

Response:
def set_neighbor_data(self, neighbor_side, data, key, field):
        """
        Assign data from the 'key' tile to the edge on the
        neighboring tile which is on the 'neighbor_side' of the 'key' tile.
        The data is assigned to the 'field' attribute of the neihboring tile's
        edge.
        """
        i = self.keys[key]
        found = False
        sides = []
        if 'left' in neighbor_side:
            if i % self.n_cols == 0:
                return None
            i -= 1
            sides.append('right')
            found = True
        if 'right' in neighbor_side:
            if i % self.n_cols == self.n_cols - 1:
                return None
            i += 1
            sides.append('left')
            found = True
        if 'top' in neighbor_side:
            sides.append('bottom')
            i -= self.n_cols
            found = True
        if 'bottom' in neighbor_side:
            sides.append('top')
            i += self.n_cols
            found = True
        if not found:
            print "Side '%s' not found" % neighbor_side
        # Check if i is in range
        if i < 0 or i >= self.n_chunks:
            return None
        # Otherwise, set the data
        for side in sides:
            self.set_i(i, data, field, side)


Instruction:
Make a summary of the following Python 3 code
def set_all_neighbors_data(self, data, done,  key):
        # The order of this for loop is important because the topleft gets
        # it's data from the left neighbor, which should have already been
        # updated...
        for side in ['left', 'right', 'top', 'bottom', 'topleft',
                     'topright', 'bottomleft', 'bottomright']:
            self.set_neighbor_data(side, data, key, 'data')
#            self.set_neighbor_data(side, todo, key, 'todo')
            self.set_neighbor_data(side, done, key, 'done')

Response: Assign this data to all the neighbors of the key tile s data.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fill_n_todo(self):
        left = self.left
        right = self.right
        top = self.top
        bottom = self.bottom
        for i in xrange(self.n_chunks):
            self.n_todo.ravel()[i] = np.sum([left.ravel()[i].n_todo,
                                            right.ravel()[i].n_todo,
                                            top.ravel()[i].n_todo,
                                            bottom.ravel()[i].n_todo])

Response: Calculate and record the number of edge pixels left to do on each tile


Instruction:
Make a summary of the following Python 3 code
def fill_n_done(self):
        left = self.left
        right = self.right
        top = self.top
        bottom = self.bottom
        for i in xrange(self.n_chunks):
            self.n_done.ravel()[i] = np.sum([left.ravel()[i].n_done,
                                            right.ravel()[i].n_done,
                                            top.ravel()[i].n_done,
                                            bottom.ravel()[i].n_done])

Response:
Calculates and record the number of edge pixels that are done one each
        tile.


Instruction:
Can you write a function in Python 3 where it
calculates the percentage of edge pixels that would be done if the tile was reprocessed. This is done for each tile.

Response:
def fill_percent_done(self):
        """
        Calculate the percentage of edge pixels that would be done if the tile
        was reprocessed. This is done for each tile.
        """
        left = self.left
        right = self.right
        top = self.top
        bottom = self.bottom
        for i in xrange(self.n_chunks):
            self.percent_done.ravel()[i] = \
                np.sum([left.ravel()[i].percent_done,
                        right.ravel()[i].percent_done,
                        top.ravel()[i].percent_done,
                        bottom.ravel()[i].percent_done])
            self.percent_done.ravel()[i] /= \
                np.sum([left.ravel()[i].percent_done > 0,
                        right.ravel()[i].percent_done > 0,
                        top.ravel()[i].percent_done > 0,
                        bottom.ravel()[i].percent_done > 0, 1e-16])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def fill_array(self, array, field, add=False, maximize=False):
        self.fix_shapes()
        for i in xrange(self.n_chunks):
            for side in ['left', 'right', 'top', 'bottom']:
                edge = getattr(self, side).ravel()[i]
                if add:
                    array[edge.slice] += getattr(edge, field)
                elif maximize:
                    array[edge.slice] = np.maximum(array[edge.slice],
                                                   getattr(edge, field))
                else:
                    array[edge.slice] = getattr(edge, field)
        return array

Response: Given a full array with the data on the while image fill it with the data on the while image edges.


Instruction:
Write a Python 3 function for
fixing the shape of the data fields on edges.

Response:
def fix_shapes(self):
        """
        Fixes the shape of the data fields on edges. Left edges should be
        column vectors, and top edges should be row vectors, for example.
        """
        for i in xrange(self.n_chunks):
            for side in ['left', 'right', 'top', 'bottom']:
                edge = getattr(self, side).ravel()[i]
                if side in ['left', 'right']:
                    shp = [edge.todo.size, 1]
                else:
                    shp = [1, edge.todo.size]
                edge.done = edge.done.reshape(shp)
                edge.data = edge.data.reshape(shp)
                edge.todo = edge.todo.reshape(shp)


Instruction:
How would you explain what the following Python 3 function does
def find_best_candidate(self):
        self.fill_percent_done()
        i_b = np.argmax(self.percent_done.ravel())
        if self.percent_done.ravel()[i_b] <= 0:
            return None

        # check for ties
        I = self.percent_done.ravel() == self.percent_done.ravel()[i_b]
        if I.sum() == 1:
            return i_b
        else:
            I2 = np.argmax(self.max_elev.ravel()[I])
            return I.nonzero()[0][I2]

Response:
Determine which tile when processed would complete the largest
        percentage of unresolved edge pixels.