Instruction:
Implement a Python 3 function for
crossing - validate the current set of curves.

Response:
def cross_validate(self, ax, info=''):
        '''
        Cross-validate to find the optimal value of :py:obj:`lambda`.

        :param ax: The current :py:obj:`matplotlib.pyplot` axis instance to \
               plot the cross-validation results.
        :param str info: The label to show in the bottom right-hand corner \
               of the plot. Default `''`

        '''

        # Loop over all chunks
        ax = np.atleast_1d(ax)
        for b, brkpt in enumerate(self.breakpoints):

            log.info("Cross-validating chunk %d/%d..." %
                     (b + 1, len(self.breakpoints)))
            med_training = np.zeros_like(self.lambda_arr)
            med_validation = np.zeros_like(self.lambda_arr)

            # Mask for current chunk
            m = self.get_masked_chunk(b)

            # Check that we have enough data
            if len(m) < 3 * self.cdivs:
                self.cdppv_arr[b] = np.nan
                self.lam[b][self.lam_idx] = 0.
                log.info(
                    "Insufficient data to run cross-validation on this chunk.")
                continue

            # Mask transits and outliers
            time = self.time[m]
            flux = self.fraw[m]
            ferr = self.fraw_err[m]
            med = np.nanmedian(flux)

            # The precision in the validation set
            validation = [[] for k, _ in enumerate(self.lambda_arr)]

            # The precision in the training set
            training = [[] for k, _ in enumerate(self.lambda_arr)]

            # Setup the GP
            gp = GP(self.kernel, self.kernel_params, white=False)
            gp.compute(time, ferr)

            # The masks
            masks = list(Chunks(np.arange(0, len(time)),
                                len(time) // self.cdivs))

            # Loop over the different masks
            for i, mask in enumerate(masks):

                log.info("Section %d/%d..." % (i + 1, len(masks)))

                # Pre-compute (training set)
                pre_t = self.cv_precompute([], b)

                # Pre-compute (validation set)
                pre_v = self.cv_precompute(mask, b)

                # Iterate over lambda
                for k, lam in enumerate(self.lambda_arr):

                    # Update the lambda matrix
                    self.lam[b][self.lam_idx] = lam

                    # Training set
                    model = self.cv_compute(b, *pre_t)
                    training[k].append(
                        self.fobj(flux - model, med, time, gp, mask))

                    # Validation set
                    model = self.cv_compute(b, *pre_v)
                    validation[k].append(
                        self.fobj(flux - model, med, time, gp, mask))

            # Finalize
            training = np.array(training)
            validation = np.array(validation)
            for k, _ in enumerate(self.lambda_arr):

                # Take the mean
                med_validation[k] = np.nanmean(validation[k])
                med_training[k] = np.nanmean(training[k])

            # Compute best model
            i = self.optimize_lambda(validation)
            v_best = med_validation[i]
            t_best = med_training[i]
            self.cdppv_arr[b] = v_best / t_best
            self.lam[b][self.lam_idx] = self.lambda_arr[i]
            log.info("Found optimum solution at log(lambda) = %.1f." %
                     np.log10(self.lam[b][self.lam_idx]))

            # Plotting: There's not enough space in the DVS to show the
            # cross-val results for more than three light curve segments.
            if len(self.breakpoints) <= 3:

                # Plotting hack: first x tick will be -infty
                lambda_arr = np.array(self.lambda_arr)
                lambda_arr[0] = 10 ** (np.log10(lambda_arr[1]) - 3)

                # Plot cross-val
                for n in range(len(masks)):
                    ax[b].plot(np.log10(lambda_arr),
                               validation[:, n], 'r-', alpha=0.3)

                ax[b].plot(np.log10(lambda_arr),
                           med_training, 'b-', lw=1., alpha=1)
                ax[b].plot(np.log10(lambda_arr),
                           med_validation, 'r-', lw=1., alpha=1)
                ax[b].axvline(np.log10(self.lam[b][self.lam_idx]),
                              color='k', ls='--', lw=0.75, alpha=0.75)
                ax[b].axhline(v_best, color='k', ls='--', lw=0.75, alpha=0.75)
                ax[b].set_ylabel(r'Scatter (ppm)', fontsize=5)
                hi = np.max(validation[0])
                lo = np.min(training)
                rng = (hi - lo)
                ax[b].set_ylim(lo - 0.15 * rng, hi + 0.15 * rng)
                if rng > 2:
                    ax[b].get_yaxis().set_major_formatter(Formatter.CDPP)
                    ax[b].get_yaxis().set_major_locator(
                        MaxNLocator(4, integer=True))
                elif rng > 0.2:
                    ax[b].get_yaxis().set_major_formatter(Formatter.CDPP1F)
                    ax[b].get_yaxis().set_major_locator(MaxNLocator(4))
                else:
                    ax[b].get_yaxis().set_major_formatter(Formatter.CDPP2F)
                    ax[b].get_yaxis().set_major_locator(MaxNLocator(4))

                # Fix the x ticks
                xticks = [np.log10(lambda_arr[0])] + list(np.linspace(
                    np.log10(lambda_arr[1]), np.log10(lambda_arr[-1]), 6))
                ax[b].set_xticks(xticks)
                ax[b].set_xticklabels(['' for x in xticks])
                pad = 0.01 * \
                    (np.log10(lambda_arr[-1]) - np.log10(lambda_arr[0]))
                ax[b].set_xlim(np.log10(lambda_arr[0]) - pad,
                               np.log10(lambda_arr[-1]) + pad)
                ax[b].annotate('%s.%d' % (info, b), xy=(0.02, 0.025),
                               xycoords='axes fraction',
                               ha='left', va='bottom', fontsize=7, alpha=0.25,
                               fontweight='bold')

        # Finally, compute the model
        self.compute()

        # Tidy up
        if len(ax) == 2:
            ax[0].xaxis.set_ticks_position('top')
        for axis in ax[1:]:
            axis.spines['top'].set_visible(False)
            axis.xaxis.set_ticks_position('bottom')

        if len(self.breakpoints) <= 3:

            # A hack to mark the first xtick as -infty
            labels = ['%.1f' % x for x in xticks]
            labels[0] = r'$-\infty$'
            ax[-1].set_xticklabels(labels)
            ax[-1].set_xlabel(r'Log $\Lambda$', fontsize=5)

        else:

            # We're just going to plot lambda as a function of chunk number
            bs = np.arange(len(self.breakpoints))
            ax[0].plot(bs + 1, [np.log10(self.lam[b][self.lam_idx])
                                for b in bs], 'r.')
            ax[0].plot(bs + 1, [np.log10(self.lam[b][self.lam_idx])
                                for b in bs], 'r-', alpha=0.25)
            ax[0].set_ylabel(r'$\log\Lambda$', fontsize=5)
            ax[0].margins(0.1, 0.1)
            ax[0].set_xticks(np.arange(1, len(self.breakpoints) + 1))
            ax[0].set_xticklabels([])

            # Now plot the CDPP and approximate validation CDPP
            cdpp_arr = self.get_cdpp_arr()
            cdppv_arr = self.cdppv_arr * cdpp_arr
            ax[1].plot(bs + 1, cdpp_arr, 'b.')
            ax[1].plot(bs + 1, cdpp_arr, 'b-', alpha=0.25)
            ax[1].plot(bs + 1, cdppv_arr, 'r.')
            ax[1].plot(bs + 1, cdppv_arr, 'r-', alpha=0.25)
            ax[1].margins(0.1, 0.1)
            ax[1].set_ylabel(r'Scatter (ppm)', fontsize=5)
            ax[1].set_xlabel(r'Chunk', fontsize=5)
            if len(self.breakpoints) < 15:
                ax[1].set_xticks(np.arange(1, len(self.breakpoints) + 1))
            else:
                ax[1].set_xticks(np.arange(1, len(self.breakpoints) + 1, 2))


Instruction:
Can you tell what is the following Python 3 function doing
def get_ylim(self):
        '''
        Computes the ideal y-axis limits for the light curve plot. Attempts to
        set the limits equal to those of the raw light curve, but if more than
        1% of the flux lies either above or below these limits, auto-expands
        to include those points. At the end, adds 5% padding to both the
        top and the bottom.

        '''

        bn = np.array(
            list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)
        fraw = np.delete(self.fraw, bn)
        lo, hi = fraw[np.argsort(fraw)][[3, -3]]
        flux = np.delete(self.flux, bn)
        fsort = flux[np.argsort(flux)]
        if fsort[int(0.01 * len(fsort))] < lo:
            lo = fsort[int(0.01 * len(fsort))]
        if fsort[int(0.99 * len(fsort))] > hi:
            hi = fsort[int(0.99 * len(fsort))]
        pad = (hi - lo) * 0.05
        ylim = (lo - pad, hi + pad)
        return ylim

Response: Computes the ideal y - axis limits for the light curve plot.


Instruction:
Create a Python 3 function to
plot the current light curve.

Response:
def plot_lc(self, ax, info_left='', info_right='', color='b'):
        '''
        Plots the current light curve. This is called at several stages to
        plot the de-trending progress as a function of the different
        *PLD* orders.

        :param ax: The current :py:obj:`matplotlib.pyplot` axis instance
        :param str info_left: Information to display at the left of the \
               plot. Default `''`
        :param str info_right: Information to display at the right of the \
               plot. Default `''`
        :param str color: The color of the data points. Default `'b'`

        '''

        # Plot
        if (self.cadence == 'lc') or (len(self.time) < 4000):
            ax.plot(self.apply_mask(self.time), self.apply_mask(self.flux),
                    ls='none', marker='.', color=color,
                    markersize=2, alpha=0.5)
            ax.plot(self.time[self.transitmask], self.flux[self.transitmask],
                    ls='none', marker='.', color=color,
                    markersize=2, alpha=0.5)
        else:
            ax.plot(self.apply_mask(self.time), self.apply_mask(
                    self.flux), ls='none', marker='.', color=color,
                    markersize=2, alpha=0.03, zorder=-1)
            ax.plot(self.time[self.transitmask], self.flux[self.transitmask],
                    ls='none', marker='.', color=color,
                    markersize=2, alpha=0.03, zorder=-1)
            ax.set_rasterization_zorder(0)
        ylim = self.get_ylim()

        # Plot the outliers, but not the NaNs
        badmask = [i for i in self.badmask if i not in self.nanmask]

        def O1(x): return x[self.outmask]

        def O2(x): return x[badmask]
        if self.cadence == 'lc':
            ax.plot(O1(self.time), O1(self.flux), ls='none',
                    color="#777777", marker='.', markersize=2, alpha=0.5)
            ax.plot(O2(self.time), O2(self.flux),
                    'r.', markersize=2, alpha=0.25)
        else:
            ax.plot(O1(self.time), O1(self.flux), ls='none', color="#777777",
                    marker='.', markersize=2, alpha=0.25, zorder=-1)
            ax.plot(O2(self.time), O2(self.flux), 'r.',
                    markersize=2, alpha=0.125, zorder=-1)
        for i in np.where(self.flux < ylim[0])[0]:
            if i in badmask:
                color = "#ffcccc"
            elif i in self.outmask:
                color = "#cccccc"
            elif i in self.nanmask:
                continue
            else:
                color = "#ccccff"
            ax.annotate('', xy=(self.time[i], ylim[0]), xycoords='data',
                        xytext=(0, 15), textcoords='offset points',
                        arrowprops=dict(arrowstyle="-|>", color=color))
        for i in np.where(self.flux > ylim[1])[0]:
            if i in badmask:
                color = "#ffcccc"
            elif i in self.outmask:
                color = "#cccccc"
            elif i in self.nanmask:
                continue
            else:
                color = "#ccccff"
            ax.annotate('', xy=(self.time[i], ylim[1]), xycoords='data',
                        xytext=(0, -15), textcoords='offset points',
                        arrowprops=dict(arrowstyle="-|>", color=color))

        # Plot the breakpoints
        for brkpt in self.breakpoints[:-1]:
            if len(self.breakpoints) <= 5:
                ax.axvline(self.time[brkpt], color='r', ls='--', alpha=0.5)
            else:
                ax.axvline(self.time[brkpt], color='r', ls='-', alpha=0.025)

        # Appearance
        if len(self.cdpp_arr) == 2:
            ax.annotate('%.2f ppm' % self.cdpp_arr[0], xy=(0.02, 0.975),
                        xycoords='axes fraction',
                        ha='left', va='top', fontsize=10)
            ax.annotate('%.2f ppm' % self.cdpp_arr[1], xy=(0.98, 0.975),
                        xycoords='axes fraction',
                        ha='right', va='top', fontsize=10)
        elif len(self.cdpp_arr) < 6:
            for n in range(len(self.cdpp_arr)):
                if n > 0:
                    x = (self.time[self.breakpoints[n - 1]] - self.time[0]
                         ) / (self.time[-1] - self.time[0]) + 0.02
                else:
                    x = 0.02
                ax.annotate('%.2f ppm' % self.cdpp_arr[n], xy=(x, 0.975),
                            xycoords='axes fraction',
                            ha='left', va='top', fontsize=8)
        else:
            ax.annotate('%.2f ppm' % self.cdpp, xy=(0.02, 0.975),
                        xycoords='axes fraction',
                        ha='left', va='top', fontsize=10)
        ax.annotate(info_right, xy=(0.98, 0.025), xycoords='axes fraction',
                    ha='right', va='bottom', fontsize=10, alpha=0.5,
                    fontweight='bold')
        ax.annotate(info_left, xy=(0.02, 0.025), xycoords='axes fraction',
                    ha='left', va='bottom', fontsize=8)
        ax.set_xlabel(r'Time (%s)' % self._mission.TIMEUNITS, fontsize=5)
        ax.margins(0.01, 0.1)
        ax.set_ylim(*ylim)
        ax.get_yaxis().set_major_formatter(Formatter.Flux)


Instruction:
Can you tell what is the following Python 3 function doing
def plot_final(self, ax):
        '''
        Plots the final de-trended light curve.

        '''

        # Plot the light curve
        bnmask = np.array(
            list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)

        def M(x): return np.delete(x, bnmask)
        if (self.cadence == 'lc') or (len(self.time) < 4000):
            ax.plot(M(self.time), M(self.flux), ls='none',
                    marker='.', color='k', markersize=2, alpha=0.3)
        else:
            ax.plot(M(self.time), M(self.flux), ls='none', marker='.',
                    color='k', markersize=2, alpha=0.03, zorder=-1)
            ax.set_rasterization_zorder(0)
        # Hack: Plot invisible first and last points to ensure
        # the x axis limits are the
        # same in the other plots, where we also plot outliers!
        ax.plot(self.time[0], np.nanmedian(M(self.flux)), marker='.', alpha=0)
        ax.plot(self.time[-1], np.nanmedian(M(self.flux)), marker='.', alpha=0)

        # Plot the GP (long cadence only)
        if self.cadence == 'lc':
            gp = GP(self.kernel, self.kernel_params, white=False)
            gp.compute(self.apply_mask(self.time),
                       self.apply_mask(self.fraw_err))
            med = np.nanmedian(self.apply_mask(self.flux))
            y, _ = gp.predict(self.apply_mask(self.flux) - med, self.time)
            y += med
            ax.plot(M(self.time), M(y), 'r-', lw=0.5, alpha=0.5)

            # Compute the CDPP of the GP-detrended flux
            self.cdppg = self._mission.CDPP(self.apply_mask(
                self.flux - y + med), cadence=self.cadence)

        else:

            # We're not going to calculate this
            self.cdppg = 0.

        # Appearance
        ax.annotate('Final', xy=(0.98, 0.025), xycoords='axes fraction',
                    ha='right', va='bottom', fontsize=10, alpha=0.5,
                    fontweight='bold')
        ax.margins(0.01, 0.1)

        # Get y lims that bound 99% of the flux
        flux = np.delete(self.flux, bnmask)
        N = int(0.995 * len(flux))
        hi, lo = flux[np.argsort(flux)][[N, -N]]
        fsort = flux[np.argsort(flux)]
        pad = (hi - lo) * 0.1
        ylim = (lo - pad, hi + pad)
        ax.set_ylim(ylim)
        ax.get_yaxis().set_major_formatter(Formatter.Flux)

Response: Plots the final de - trended light curve.


Instruction:
How would you explain what the following Python 3 function does
def plot_cbv(self, ax, flux, info, show_cbv=False):
        '''
        Plots the final CBV-corrected light curve.

        '''

        # Plot the light curve
        bnmask = np.array(
            list(set(np.concatenate([self.badmask, self.nanmask]))), dtype=int)

        def M(x): return np.delete(x, bnmask)
        if self.cadence == 'lc':
            ax.plot(M(self.time), M(flux), ls='none', marker='.',
                    color='k', markersize=2, alpha=0.45)
        else:
            ax.plot(M(self.time), M(flux), ls='none', marker='.',
                    color='k', markersize=2, alpha=0.03, zorder=-1)
            ax.set_rasterization_zorder(0)
        # Hack: Plot invisible first and last points to ensure
        # the x axis limits are the
        # same in the other plots, where we also plot outliers!
        ax.plot(self.time[0], np.nanmedian(M(flux)), marker='.', alpha=0)
        ax.plot(self.time[-1], np.nanmedian(M(flux)), marker='.', alpha=0)

        # Show CBV fit?
        if show_cbv:
            ax.plot(self.time, self._mission.FitCBVs(
                self) + np.nanmedian(flux), 'r-', alpha=0.2)

        # Appearance
        ax.annotate(info, xy=(0.98, 0.025), xycoords='axes fraction',
                    ha='right', va='bottom', fontsize=10, alpha=0.5,
                    fontweight='bold')
        ax.margins(0.01, 0.1)

        # Get y lims that bound 99% of the flux
        flux = np.delete(flux, bnmask)
        N = int(0.995 * len(flux))
        hi, lo = flux[np.argsort(flux)][[N, -N]]
        fsort = flux[np.argsort(flux)]
        pad = (hi - lo) * 0.2
        ylim = (lo - pad, hi + pad)
        ax.set_ylim(ylim)
        ax.get_yaxis().set_major_formatter(Formatter.Flux)
        ax.set_xlabel(r'Time (%s)' % self._mission.TIMEUNITS, fontsize=9)
        for tick in ax.get_xticklabels() + ax.get_yticklabels():
            tick.set_fontsize(7)

Response: Plots the final CBV - correcteded light curve.


Instruction:
Make a summary of the following Python 3 code
def load_tpf(self):
        '''
        Loads the target pixel file.

        '''

        if not self.loaded:
            if self._data is not None:
                data = self._data
            else:
                data = self._mission.GetData(
                         self.ID, season=self.season,
                         cadence=self.cadence,
                         clobber=self.clobber_tpf,
                         aperture_name=self.aperture_name,
                         saturated_aperture_name=self.saturated_aperture_name,
                         max_pixels=self.max_pixels,
                         saturation_tolerance=self.saturation_tolerance,
                         get_hires=self.get_hires,
                         get_nearby=self.get_nearby)
                if data is None:
                    raise Exception("Unable to retrieve target data.")
            self.cadn = data.cadn
            self.time = data.time
            self.model = np.zeros_like(self.time)
            self.fpix = data.fpix
            self.fraw = np.sum(self.fpix, axis=1)
            self.fpix_err = data.fpix_err
            self.fraw_err = np.sqrt(np.sum(self.fpix_err ** 2, axis=1))
            self.nanmask = data.nanmask
            self.badmask = data.badmask
            self.transitmask = np.array([], dtype=int)
            self.outmask = np.array([], dtype=int)
            self.aperture = data.aperture
            self.aperture_name = data.aperture_name
            self.apertures = data.apertures
            self.quality = data.quality
            self.Xpos = data.Xpos
            self.Ypos = data.Ypos
            self.mag = data.mag
            self.pixel_images = data.pixel_images
            self.nearby = data.nearby
            self.hires = data.hires
            self.saturated = data.saturated
            self.meta = data.meta
            self.bkg = data.bkg

            # Update the last breakpoint to the correct value
            self.breakpoints[-1] = len(self.time) - 1

            # Get PLD normalization
            self.get_norm()

            self.loaded = True

Response: Loads the target pixel file.


Instruction:
Can you tell what is the following Python 3 function doing
def load_model(self, name=None):
        '''
        Loads a saved version of the model.

        '''

        if self.clobber:
            return False

        if name is None:
            name = self.name
        file = os.path.join(self.dir, '%s.npz' % name)
        if os.path.exists(file):
            if not self.is_parent:
                log.info("Loading '%s.npz'..." % name)
            try:
                data = np.load(file)
                for key in data.keys():
                    try:
                        setattr(self, key, data[key][()])
                    except NotImplementedError:
                        pass

                # HACK: Backwards compatibility. Previous version stored
                # the CDPP in the `cdpp6`
                # and `cdpp6_arr` attributes. Let's move them over.
                if hasattr(self, 'cdpp6'):
                    self.cdpp = self.cdpp6
                    del self.cdpp6
                if hasattr(self, 'cdpp6_arr'):
                    self.cdpp_arr = np.array(self.cdpp6_arr)
                    del self.cdpp6_arr
                if hasattr(self, 'gppp'):
                    self.cdppg = self.gppp
                    del self.gppp

                # HACK: At one point we were saving the figure instances,
                # so loading the .npz
                # opened a plotting window. I don't think this is the case
                # any more, so this
                # next line should be removed in the future...
                pl.close()

                return True
            except:
                log.warn("Error loading '%s.npz'." % name)
                exctype, value, tb = sys.exc_info()
                for line in traceback.format_exception_only(exctype, value):
                    ln = line.replace('\n', '')
                    log.warn(ln)
                os.rename(file, file + '.bad')

        if self.is_parent:
            raise Exception(
                'Unable to load `%s` model for target %d.'
                % (self.name, self.ID))

        return False

Response: Loads a saved version of the model.


Instruction:
Can you generate the documentation for the following Python 3 function
def save_model(self):
        '''
        Saves all of the de-trending information to disk in an `npz` file
        and saves the DVS as a `pdf`.

        '''

        # Save the data
        log.info("Saving data to '%s.npz'..." % self.name)
        d = dict(self.__dict__)
        d.pop('_weights', None)
        d.pop('_A', None)
        d.pop('_B', None)
        d.pop('_f', None)
        d.pop('_mK', None)
        d.pop('K', None)
        d.pop('dvs', None)
        d.pop('clobber', None)
        d.pop('clobber_tpf', None)
        d.pop('_mission', None)
        d.pop('debug', None)
        d.pop('transit_model', None)
        d.pop('_transit_model', None)
        np.savez(os.path.join(self.dir, self.name + '.npz'), **d)

        # Save the DVS
        pdf = PdfPages(os.path.join(self.dir, self.name + '.pdf'))
        pdf.savefig(self.dvs.fig)
        pl.close(self.dvs.fig)
        d = pdf.infodict()
        d['Title'] = 'EVEREST: %s de-trending of %s %d' % (
            self.name, self._mission.IDSTRING, self.ID)
        d['Author'] = 'Rodrigo Luger'
        pdf.close()

Response: Save all of the de - trending information to disk in an npz file and saves the DVS as a pdf.


Instruction:
How would you explain what the following Python 3 function does
def update_gp(self):
        '''
        Calls :py:func:`gp.GetKernelParams` to optimize the GP and obtain the
        covariance matrix for the regression.

        '''

        self.kernel_params = GetKernelParams(self.time, self.flux,
                                             self.fraw_err,
                                             mask=self.mask,
                                             guess=self.kernel_params,
                                             kernel=self.kernel,
                                             giter=self.giter,
                                             gmaxf=self.gmaxf)

Response: Updates the kernel parameters of the current object.


Instruction:
Make a summary of the following Python 3 code
def init_kernel(self):
        '''
        Initializes the covariance matrix with a guess at
        the GP kernel parameters.

        '''

        if self.kernel_params is None:
            X = self.apply_mask(self.fpix / self.flux.reshape(-1, 1))
            y = self.apply_mask(self.flux) - np.dot(X, np.linalg.solve(
                np.dot(X.T, X), np.dot(X.T, self.apply_mask(self.flux))))
            white = np.nanmedian([np.nanstd(c) for c in Chunks(y, 13)])
            amp = self.gp_factor * np.nanstd(y)
            tau = 30.0
            if self.kernel == 'Basic':
                self.kernel_params = [white, amp, tau]
            elif self.kernel == 'QuasiPeriodic':
                self.kernel_params = [white, amp, 1., 20.]

Response:
Initializes the covariance matrix with a guess at
        the GP kernel parameters.


Instruction:
Can you tell what is the following Python 3 function doing
def run(self):
        '''
        Runs the de-trending step.

        '''

        try:

            # Load raw data
            log.info("Loading target data...")
            self.load_tpf()
            self.mask_planets()
            self.plot_aperture([self.dvs.top_right() for i in range(4)])
            self.init_kernel()
            M = self.apply_mask(np.arange(len(self.time)))
            self.cdppr_arr = self.get_cdpp_arr()
            self.cdpp_arr = np.array(self.cdppr_arr)
            self.cdppv_arr = np.array(self.cdppr_arr)
            self.cdppr = self.get_cdpp()
            self.cdpp = self.cdppr
            self.cdppv = self.cdppr

            log.info("%s (Raw): CDPP = %s" % (self.name, self.cdpps))
            self.plot_lc(self.dvs.left(), info_right='Raw', color='k')

            # Loop
            for n in range(self.pld_order):
                self.lam_idx += 1
                self.get_outliers()
                if n > 0 and self.optimize_gp:
                    self.update_gp()
                self.cross_validate(self.dvs.right(), info='CV%d' % n)
                self.cdpp_arr = self.get_cdpp_arr()
                self.cdppv_arr *= self.cdpp_arr
                self.cdpp = self.get_cdpp()
                self.cdppv = np.nanmean(self.cdppv_arr)
                log.info("%s (%d/%d): CDPP = %s" %
                         (self.name, n + 1, self.pld_order, self.cdpps))
                self.plot_lc(self.dvs.left(), info_right='LC%d' % (
                    n + 1), info_left='%d outliers' % len(self.outmask))

            # Save
            self.finalize()
            self.plot_final(self.dvs.top_left())
            self.plot_info(self.dvs)
            self.save_model()

        except:

            self.exception_handler(self.debug)

Response: Runs the de - trending step.


Instruction:
Can you implement a function in Python 3 that
publishes the current state of the object to the DVS figure and generates a FITS file for the publication.

Response:
def publish(self, **kwargs):
        '''
        Correct the light curve with the CBVs, generate a
        cover page for the DVS figure,
        and produce a FITS file for publication.

        '''

        try:

            # HACK: Force these params for publication
            self.cbv_win = 999
            self.cbv_order = 3
            self.cbv_num = 1

            # Get the CBVs
            self._mission.GetTargetCBVs(self)

            # Plot the final corrected light curve
            cbv = CBV()
            self.plot_info(cbv)
            self.plot_cbv(cbv.body(), self.fcor, 'Corrected')
            self.plot_cbv(cbv.body(), self.flux, 'De-trended', show_cbv=True)
            self.plot_cbv(cbv.body(), self.fraw, 'Raw')

            # Save the CBV pdf
            pdf = PdfPages(os.path.join(self.dir, 'cbv.pdf'))
            pdf.savefig(cbv.fig)
            pl.close(cbv.fig)
            d = pdf.infodict()
            d['Title'] = 'EVEREST: %s de-trending of %s %d' % (
                self.name, self._mission.IDSTRING, self.ID)
            d['Author'] = 'Rodrigo Luger'
            pdf.close()

            # Now merge the two PDFs
            assert os.path.exists(os.path.join(
                self.dir, self.name + '.pdf')), \
                "Unable to locate %s.pdf." % self.name
            output = PdfFileWriter()
            pdfOne = PdfFileReader(os.path.join(self.dir, 'cbv.pdf'))
            pdfTwo = PdfFileReader(os.path.join(self.dir, self.name + '.pdf'))
            # Add the CBV page
            output.addPage(pdfOne.getPage(0))
            # Add the original DVS page
            output.addPage(pdfTwo.getPage(pdfTwo.numPages - 1))
            # Write the final PDF
            outputStream = open(os.path.join(self.dir, self._mission.DVSFile(
                self.ID, self.season, self.cadence)), "wb")
            output.write(outputStream)
            outputStream.close()
            os.remove(os.path.join(self.dir, 'cbv.pdf'))

            # Make the FITS file
            MakeFITS(self)

        except:

            self.exception_handler(self.debug)


Instruction:
Can you generate the documentation for the following Python 3 function
def setup(self, **kwargs):
        '''
        This is called during production de-trending, prior to
        calling the :py:obj:`Detrender.run()` method.

        :param tuple cdpp_range: If :py:obj:`parent_model` is set, \
               neighbors are selected only if \
               their de-trended CDPPs fall within this range. Default `None`
        :param tuple mag_range: Only select neighbors whose magnitudes are \
               within this range. Default (11., 13.)
        :param int neighbors: The number of neighboring stars to use in \
               the de-trending. The higher this number, the more signals \
               there are and hence the more de-trending information there is. \
               However, the neighboring star signals are regularized together \
               with the target's signals, so adding too many neighbors will \
               inevitably reduce the contribution of the target's own \
               signals, which may reduce performance. Default `10`
        :param str parent_model: By default, :py:class:`nPLD` is run in \
               stand-alone mode. The neighbor signals are computed directly \
               from their TPFs, so there is no need to have run *PLD* on them \
               beforehand. However, if :py:obj:`parent_model` \
               is set, :py:class:`nPLD` will use information from the \
               :py:obj:`parent_model` model of each neighboring star when \
               de-trending. This is particularly useful for identifying \
               outliers in the neighbor signals and preventing them from \
               polluting the current target. Setting :py:obj:`parent_model` \
               to :py:class:`rPLD`, for instance, will use the \
               outlier information in the :py:class:`rPLD` model of the \
               neighbors (this must have been run ahead of time). \
               Note, however, that tests with *K2* data show that including \
               outliers in the neighbor signals actually \
               *improves* the performance, since many of these outliers \
               are associated with events such as thruster firings and are \
               present in all light curves, and therefore *help* in the \
               de-trending. Default `None`

        ..note :: Optionally, the :py:obj:`neighbors` may be specified \
                  directly as a list of target IDs to use. \
                  In this case, users may also provide a list of \
                  :py:class:`everest.utils.DataContainer` instances \
                  corresponding to each of the neighbors in the \
                  :py:obj:`neighbors_data` kwarg.
        '''

        # Get neighbors
        self.parent_model = kwargs.get('parent_model', None)
        neighbors = kwargs.get('neighbors', 10)
        neighbors_data = kwargs.get('neighbors_data', None)
        if hasattr(neighbors, '__len__'):
            self.neighbors = neighbors
        else:
            num_neighbors = neighbors
            self.neighbors = \
                self._mission.GetNeighbors(self.ID,
                                           season=self.season,
                                           cadence=self.cadence,
                                           model=self.parent_model,
                                           neighbors=num_neighbors,
                                           mag_range=kwargs.get(
                                               'mag_range', (11., 13.)),
                                           cdpp_range=kwargs.get(
                                               'cdpp_range', None),
                                           aperture_name=self.aperture_name)
            if len(self.neighbors):
                if len(self.neighbors) < num_neighbors:
                    log.warn("%d neighbors requested, but only %d found." %
                             (num_neighbors, len(self.neighbors)))
            elif num_neighbors > 0:
                log.warn("No neighbors found! Running standard PLD...")

        for n, neighbor in enumerate(self.neighbors):
            log.info("Loading data for neighboring target %d..." % neighbor)
            if neighbors_data is not None:
                data = neighbors_data[n]
                data.mask = np.array(
                    list(set(np.concatenate([data.badmask, data.nanmask]))),
                    dtype=int)
                data.fraw = np.sum(data.fpix, axis=1)
            elif self.parent_model is not None and self.cadence == 'lc':
                # We load the `parent` model. The advantage here is
                # that outliers have properly been identified and masked.
                # I haven't tested this on short
                # cadence data, so I'm going to just forbid it...
                data = eval(self.parent_model)(
                    neighbor, mission=self.mission, is_parent=True)
            else:
                # We load the data straight from the TPF. Much quicker,
                # since no model must be run in advance. Downside is we
                # don't know where the outliers are. But based
                # on tests with K2 data, the de-trending is actually
                # *better* if the outliers are
                # included! These are mostly thruster fire events and other
                #  artifacts common to
                # all the stars, so it makes sense that we might want
                # to keep them in the design matrix.
                data = self._mission.GetData(neighbor, season=self.season,
                                             clobber=self.clobber_tpf,
                                             cadence=self.cadence,
                                             aperture_name=self.aperture_name,
                                             saturated_aperture_name=
                                             self.saturated_aperture_name,
                                             max_pixels=self.max_pixels,
                                             saturation_tolerance=
                                             self.saturation_tolerance,
                                             get_hires=False, get_nearby=False)
                if data is None:
                    raise Exception(
                        "Unable to retrieve data for neighboring target.")
                data.mask = np.array(
                    list(set(np.concatenate([data.badmask, data.nanmask]))),
                    dtype=int)
                data.fraw = np.sum(data.fpix, axis=1)

            # Compute the linear PLD vectors and interpolate over
            # outliers, NaNs and bad timestamps
            X1 = data.fpix / data.fraw.reshape(-1, 1)
            X1 = Interpolate(data.time, data.mask, X1)
            if self.X1N is None:
                self.X1N = np.array(X1)
            else:
                self.X1N = np.hstack([self.X1N, X1])
            del X1
            del data

Response: This method is called by the detector to setup the internal state of the detector.


Instruction:
How would you explain what the following Python 3 function does
def run(self):
        '''
        Runs the de-trending.

        '''

        try:

            # Plot original
            self.plot_aperture([self.dvs.top_right() for i in range(4)])
            self.plot_lc(self.dvs.left(), info_right='nPLD', color='k')

            # Cross-validate
            self.cross_validate(self.dvs.right())
            self.compute()
            self.cdpp_arr = self.get_cdpp_arr()
            self.cdpp = self.get_cdpp()

            # Plot new
            self.plot_lc(self.dvs.left(), info_right='Powell', color='k')

            # Save
            self.plot_final(self.dvs.top_left())
            self.plot_info(self.dvs)
            self.save_model()

        except:

            self.exception_handler(self.debug)

Response: Runs the de - trending.


Instruction:
How would you implement a function in Python 3 that
performs the cross - validation step.

Response:
def cross_validate(self, ax):
        '''
        Performs the cross-validation step.

        '''

        # The CDPP to beat
        cdpp_opt = self.get_cdpp_arr()

        # Loop over all chunks
        for b, brkpt in enumerate(self.breakpoints):

            log.info("Cross-validating chunk %d/%d..." %
                     (b + 1, len(self.breakpoints)))

            # Mask for current chunk
            m = self.get_masked_chunk(b)

            # Mask transits and outliers
            time = self.time[m]
            flux = self.fraw[m]
            ferr = self.fraw_err[m]
            med = np.nanmedian(self.fraw)

            # Setup the GP
            gp = GP(self.kernel, self.kernel_params, white=False)
            gp.compute(time, ferr)

            # The masks
            masks = list(Chunks(np.arange(0, len(time)),
                                len(time) // self.cdivs))

            # The pre-computed matrices
            pre_v = [self.cv_precompute(mask, b) for mask in masks]

            # Initialize with the nPLD solution
            log_lam_opt = np.log10(self.lam[b])
            scatter_opt = self.validation_scatter(
                log_lam_opt, b, masks, pre_v, gp, flux, time, med)
            log.info("Iter 0/%d: " % (self.piter) +
                     "logL = (%s), s = %.3f" %
                     (", ".join(["%.3f" % l for l in log_lam_opt]),
                      scatter_opt))

            # Do `piter` iterations
            for p in range(self.piter):

                # Perturb the initial condition a bit
                log_lam = np.array(
                    np.log10(self.lam[b])) * \
                    (1 + self.ppert * np.random.randn(len(self.lam[b])))
                scatter = self.validation_scatter(
                    log_lam, b, masks, pre_v, gp, flux, time, med)
                log.info("Initializing at: " +
                         "logL = (%s), s = %.3f" %
                         (", ".join(["%.3f" % l for l in log_lam]), scatter))

                # Call the minimizer
                log_lam, scatter, _, _, _, _ = \
                    fmin_powell(self.validation_scatter, log_lam,
                                args=(b, masks, pre_v, gp, flux, time, med),
                                maxfun=self.pmaxf, disp=False,
                                full_output=True)

                # Did it improve the CDPP?
                tmp = np.array(self.lam[b])
                self.lam[b] = 10 ** log_lam
                self.compute()
                cdpp = self.get_cdpp_arr()[b]
                self.lam[b] = tmp
                if cdpp < cdpp_opt[b]:
                    cdpp_opt[b] = cdpp
                    log_lam_opt = log_lam

                # Log it
                log.info("Iter %d/%d: " % (p + 1, self.piter) +
                         "logL = (%s), s = %.3f" %
                         (", ".join(["%.3f" % l for l in log_lam]), scatter))

            # The best solution
            log.info("Found minimum: logL = (%s), s = %.3f" %
                     (", ".join(["%.3f" % l for l in log_lam_opt]),
                      scatter_opt))
            self.lam[b] = 10 ** log_lam_opt

        # We're just going to plot lambda as a function of chunk number
        bs = np.arange(len(self.breakpoints))
        color = ['k', 'b', 'r', 'g', 'y']
        for n in range(self.pld_order):
            ax[0].plot(bs + 1, [np.log10(self.lam[b][n])
                                for b in bs], '.', color=color[n])
            ax[0].plot(bs + 1, [np.log10(self.lam[b][n])
                                for b in bs], '-', color=color[n], alpha=0.25)
        ax[0].set_ylabel(r'$\log\Lambda$', fontsize=5)
        ax[0].margins(0.1, 0.1)
        ax[0].set_xticks(np.arange(1, len(self.breakpoints) + 1))
        ax[0].set_xticklabels([])

        # Now plot the CDPP
        cdpp_arr = self.get_cdpp_arr()
        ax[1].plot(bs + 1, cdpp_arr, 'b.')
        ax[1].plot(bs + 1, cdpp_arr, 'b-', alpha=0.25)
        ax[1].margins(0.1, 0.1)
        ax[1].set_ylabel(r'Scatter (ppm)', fontsize=5)
        ax[1].set_xlabel(r'Chunk', fontsize=5)
        ax[1].set_xticks(np.arange(1, len(self.breakpoints) + 1))


Instruction:
Can you tell what is the following Python 3 function doing
def validation_scatter(self, log_lam, b, masks, pre_v, gp, flux,
                           time, med):
        '''
        Computes the scatter in the validation set.

        '''

        # Update the lambda matrix
        self.lam[b] = 10 ** log_lam

        # Validation set scatter
        scatter = [None for i in range(len(masks))]
        for i in range(len(masks)):
            model = self.cv_compute(b, *pre_v[i])
            try:
                gpm, _ = gp.predict(flux - model - med, time[masks[i]])
            except ValueError:
                # Sometimes the model can have NaNs if
                # `lambda` is a crazy value
                return 1.e30
            fdet = (flux - model)[masks[i]] - gpm
            scatter[i] = 1.e6 * (1.4826 * np.nanmedian(np.abs(fdet / med -
                                 np.nanmedian(fdet / med))) /
                                 np.sqrt(len(masks[i])))

        return np.max(scatter)

Response: Compute the scatter for a given set of masks and the given time.


Instruction:
Implement a function in Python 3 to
search for a given star in a random univariate light curve.

Response:
def Search(star, pos_tol=2.5, neg_tol=50., **ps_kwargs):
    '''
    NOTE: `pos_tol` is the positive (i.e., above the median)
    outlier tolerance in standard deviations.
    NOTE: `neg_tol` is the negative (i.e., below the median)
    outlier tolerance in standard deviations.

    '''

    # Smooth the light curve
    t = np.delete(star.time, np.concatenate([star.nanmask, star.badmask]))
    f = np.delete(star.flux, np.concatenate([star.nanmask, star.badmask]))
    f = SavGol(f)
    med = np.nanmedian(f)

    # Kill positive outliers
    MAD = 1.4826 * np.nanmedian(np.abs(f - med))
    pos_inds = np.where((f > med + pos_tol * MAD))[0]
    pos_inds = np.array([np.argmax(star.time == t[i]) for i in pos_inds])

    # Kill negative outliers
    MAD = 1.4826 * np.nanmedian(np.abs(f - med))
    neg_inds = np.where((f < med - neg_tol * MAD))[0]
    neg_inds = np.array([np.argmax(star.time == t[i]) for i in neg_inds])

    # Replace the star.outmask array
    star.outmask = np.concatenate([neg_inds, pos_inds])
    star.transitmask = np.array([], dtype=int)

    # Delta chi squared
    TIME = np.array([])
    DEPTH = np.array([])
    VARDEPTH = np.array([])
    DELCHISQ = np.array([])
    for b, brkpt in enumerate(star.breakpoints):

        # Log
        log.info('Running chunk %d/%d...' % (b + 1, len(star.breakpoints)))

        # Masks for current chunk
        m = star.get_masked_chunk(b, pad=False)

        # This block of the masked covariance matrix
        K = GetCovariance(star.kernel, star.kernel_params,
                          star.time[m], star.fraw_err[m])

        # The masked X.L.X^T term
        A = np.zeros((len(m), len(m)))
        for n in range(star.pld_order):
            XM = star.X(n, m)
            A += star.lam[b][n] * np.dot(XM, XM.T)
        K += A
        CDK = cho_factor(K)

        # Baseline
        med = np.nanmedian(star.fraw[m])
        lnL0 = -0.5 * np.dot(star.fraw[m], cho_solve(CDK, star.fraw[m]))
        dt = np.median(np.diff(star.time[m]))

        # Create a uniform time array and get indices of missing cadences
        tol = np.nanmedian(np.diff(star.time[m])) / 5.
        tunif = np.arange(star.time[m][0], star.time[m][-1] + tol, dt)
        tnogaps = np.array(tunif)
        gaps = []
        j = 0
        for i, t in enumerate(tunif):
            if np.abs(star.time[m][j] - t) < tol:
                tnogaps[i] = star.time[m][j]
                j += 1
                if j == len(star.time[m]):
                    break
            else:
                gaps.append(i)
        gaps = np.array(gaps, dtype=int)

        # Compute the normalized transit model for a single transit
        transit_model = TransitShape(**ps_kwargs)

        # Now roll the transit model across each cadence
        dchisq = np.zeros(len(tnogaps))
        d = np.zeros(len(tnogaps))
        vard = np.zeros(len(tnogaps))
        for i in prange(len(tnogaps)):
            trn = transit_model(tnogaps, tnogaps[i])
            trn = np.delete(trn, gaps)
            trn *= med
            vard[i] = 1. / np.dot(trn, cho_solve(CDK, trn))
            if not np.isfinite(vard[i]):
                vard[i] = np.nan
                d[i] = np.nan
                dchisq[i] = np.nan
                continue
            d[i] = vard[i] * np.dot(trn, cho_solve(CDK, star.fraw[m]))
            r = star.fraw[m] - trn * d[i]
            lnL = -0.5 * np.dot(r, cho_solve(CDK, r))
            dchisq[i] = -2 * (lnL0 - lnL)

        TIME = np.append(TIME, tnogaps)
        DEPTH = np.append(DEPTH, d)
        VARDEPTH = np.append(VARDEPTH, vard)
        DELCHISQ = np.append(DELCHISQ, dchisq)

    return TIME, DEPTH, VARDEPTH, DELCHISQ


Instruction:
Write a Python 3 function for
ordering iterator over dirty elements.

Response:
def iterdirty(self):
        '''Ordered iterator over dirty elements.'''
        return iter(chain(itervalues(self._new), itervalues(self._modified)))


Instruction:
How would you implement a function in Python 3 that
adds a new instance to the session.

Response:
def add(self, instance, modified=True, persistent=None,
            force_update=False):
        '''Add a new instance to this :class:`SessionModel`.

:param modified: Optional flag indicating if the ``instance`` has been
    modified. By default its value is ``True``.
:param force_update: if ``instance`` is persistent, it forces an update of the
    data rather than a full replacement. This is used by the
    :meth:`insert_update_replace` method.
:rtype: The instance added to the session'''
        if instance._meta.type == 'structure':
            return self._add_structure(instance)
        state = instance.get_state()
        if state.deleted:
            raise ValueError('State is deleted. Cannot add.')
        self.pop(state.iid)
        pers = persistent if persistent is not None else state.persistent
        pkname = instance._meta.pkname()
        if not pers:
            instance._dbdata.pop(pkname, None)  # to make sure it is add action
            state = instance.get_state(iid=None)
        elif persistent:
            instance._dbdata[pkname] = instance.pkvalue()
            state = instance.get_state(iid=instance.pkvalue())
        else:
            action = 'update' if force_update else None
            state = instance.get_state(action=action, iid=state.iid)
        iid = state.iid
        if state.persistent:
            if modified:
                self._modified[iid] = instance
        else:
            self._new[iid] = instance
        return instance


Instruction:
Given the following Python 3 function, write the documentation
def pop(self, instance):
        '''Remove ``instance`` from the :class:`SessionModel`. Instance
could be a :class:`Model` or an id.

:parameter instance: a :class:`Model` or an ``id``.
:rtype: the :class:`Model` removed from session or ``None`` if
    it was not in the session.
'''
        if isinstance(instance, self.model):
            iid = instance.get_state().iid
        else:
            iid = instance
        instance = None
        for d in (self._new, self._modified, self._deleted):
            if iid in d:
                inst = d.pop(iid)
                if instance is None:
                    instance = inst
                elif inst is not instance:
                    raise ValueError('Critical error: %s is duplicated' % iid)
        return instance

Response: Remove an instance from the session or None if the instance is not in the session.


Instruction:
Can you tell what is the following Python 3 function doing
def expunge(self, instance):
        '''Remove *instance* from the :class:`Session`. Instance could be a
:class:`Model` or an id.

:parameter instance: a :class:`Model` or an *id*
:rtype: the :class:`Model` removed from session or ``None`` if
    it was not in the session.
'''
        instance = self.pop(instance)
        instance.session = None
        return instance

Response: Removes the given instance from the session.


Instruction:
Write a Python 3 function for
posting - commit the results of a single instance session.

Response:
def post_commit(self, results):
        '''\
Process results after a commit.

:parameter results: iterator over :class:`stdnet.instance_session_result`
    items.
:rtype: a two elements tuple containing a list of instances saved and
    a list of ids of instances deleted.'''
        tpy = self._meta.pk_to_python
        instances = []
        deleted = []
        errors = []
        # The length of results must be the same as the length of
        # all committed instances
        for result in results:
            if isinstance(result, Exception):
                errors.append(result.__class__('Exception while committing %s.'
                                               ' %s' % (self._meta, result)))
                continue
            instance = self.pop(result.iid)
            id = tpy(result.id, self.backend)
            if result.deleted:
                deleted.append(id)
            else:
                if instance is None:
                    raise InvalidTransaction('{0} session received id "{1}"\
 which is not in the session.'.format(self, result.iid))
                setattr(instance, instance._meta.pkname(), id)
                instance = self.add(instance,
                                    modified=False,
                                    persistent=result.persistent)
                instance.get_state().score = result.score
                if instance.get_state().persistent:
                    instances.append(instance)
        return instances, deleted, errors


Instruction:
Explain what the following Python 3 code does
def commit(self, callback=None):
        '''Close the transaction and commit session to the backend.'''
        if self.executed:
            raise InvalidTransaction('Invalid operation. '
                                     'Transaction already executed.')
        session = self.session
        self.session = None
        self.on_result = self._commit(session, callback)
        return self.on_result

Response: Close the transaction and commit the session to the backend.


Instruction:
Explain what the following Python 3 code does
def begin(self, **options):
        '''Begin a new :class:`Transaction`. If this :class:`Session`
is already in a :ref:`transactional state <transactional-state>`,
an error will occur. It returns the :attr:`transaction` attribute.

This method is mostly used within a ``with`` statement block::

    with session.begin() as t:
        t.add(...)
        ...

which is equivalent to::

    t = session.begin()
    t.add(...)
    ...
    session.commit()

``options`` parameters are passed to the :class:`Transaction` constructor.
'''
        if self.transaction is not None:
            raise InvalidTransaction("A transaction is already begun.")
        else:
            self.transaction = Transaction(self, **options)
        return self.transaction

Response: Begin a new transaction.


Instruction:
Implement a function in Python 3 to
create a new : class : Query for model *model*.

Response:
def query(self, model, **kwargs):
        '''Create a new :class:`Query` for *model*.'''
        sm = self.model(model)
        query_class = sm.manager.query_class or Query
        return query_class(sm._meta, self, **kwargs)


Instruction:
Can you implement a function in Python 3 that
updates or create a new instance of model.

Response:
def update_or_create(self, model, **kwargs):
        '''Update or create a new instance of ``model``.

        This method can raise an exception if the ``kwargs`` dictionary
        contains field data that does not validate.

        :param model: a :class:`StdModel`
        :param kwargs: dictionary of parameters.
        :returns: A two elements tuple containing the instance and a boolean
            indicating if the instance was created or not.
        '''
        backend = self.model(model).backend
        return backend.execute(self._update_or_create(model, **kwargs))


Instruction:
Make a summary of the following Python 3 code
def add(self, instance, modified=True, **params):
        '''Add an ``instance`` to the session.

        If the session is not in a
        :ref:`transactional state <transactional-state>`, this operation
        commits changes to the back-end server immediately.

        :parameter instance: a :class:`Model` instance. It must be registered
            with the :attr:`router` which created this :class:`Session`.
        :parameter modified: a boolean flag indicating if the instance was
            modified.
        :return: the ``instance``.

        If the instance is persistent (it is already stored in the database),
        an updated will be performed, otherwise a new entry will be created
        once the :meth:`commit` method is invoked.
        '''
        sm = self.model(instance)
        instance.session = self
        o = sm.add(instance, modified=modified, **params)
        if modified and not self.transaction:
            transaction = self.begin()
            return transaction.commit(lambda: o)
        else:
            return o

Response: Add an instance to the session.


Instruction:
How would you implement a function in Python 3 that
deletes an instance or a query.

Response:
def delete(self, instance_or_query):
        '''Delete an ``instance`` or a ``query``.

        Adds ``instance_or_query`` to this :class:`Session` list
        of data to be deleted. If the session is not in a
        :ref:`transactional state <transactional-state>`, this operation
        commits changes to the backend server immediately.

        :parameter instance_or_query: a :class:`Model` instance or
            a :class:`Query`.
        '''
        sm = self.model(instance_or_query)
        # not an instance of a Model. Assume it is a query.
        if is_query(instance_or_query):
            if instance_or_query.session is not self:
                raise ValueError('Adding a query generated by another session')
            sm._delete_query.append(instance_or_query)
        else:
            instance_or_query = sm.delete(instance_or_query, self)
        if not self.transaction:
            transaction = self.begin()
            return transaction.commit(
                lambda: transaction.deleted.get(sm._meta))
        else:
            return instance_or_query


Instruction:
Make a summary of the following Python 3 code
def model(self, model, create=True):
        '''Returns the :class:`SessionModel` for ``model`` which
can be :class:`Model`, or a :class:`MetaClass`, or an instance
of :class:`Model`.'''
        manager = self.manager(model)
        sm = self._models.get(manager)
        if sm is None and create:
            sm = SessionModel(manager)
            self._models[manager] = sm
        return sm

Response: Returns the SessionModel for the given model which is a MetaClass or an instance of Model.


Instruction:
Create a Python 3 function for
removing instance from this Session.

Response:
def expunge(self, instance=None):
        '''Remove ``instance`` from this :class:`Session`. If ``instance``
is not given, it removes all instances from this :class:`Session`.'''
        if instance is not None:
            sm = self._models.get(instance._meta)
            if sm:
                return sm.expunge(instance)
        else:
            self._models.clear()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def manager(self, model):
        '''Retrieve the :class:`Manager` for ``model`` which can be any of the
values valid for the :meth:`model` method.'''
        try:
            return self.router[model]
        except KeyError:
            meta = getattr(model, '_meta', model)
            if meta.type == 'structure':
                # this is a structure
                if hasattr(model, 'model'):
                    structure_model = model.model
                    if structure_model:
                        return self.manager(structure_model)
                    else:
                        manager = self.router.structure(model)
                        if manager:
                            return manager
            raise InvalidTransaction('"%s" not valid in this session' % meta)

Response:
Retrieve the manager for the given model which can be any of the the
values valid for the model method.


Instruction:
Explain what the following Python 3 code does
def new(self, *args, **kwargs):
        '''Create a new instance of :attr:`model` and commit it to the backend
server. This a shortcut method for the more verbose::

    instance = manager.session().add(MyModel(**kwargs))
'''
        return self.session().add(self.model(*args, **kwargs))

Response:
Create a new instance of the model and commit it to the backend
server.


Instruction:
Given the following Python 3 function, write the documentation
def query(self, session=None):
        '''Returns a new :class:`Query` for :attr:`Manager.model`.'''
        if session is None or session.router is not self.router:
            session = self.session()
        return session.query(self.model)

Response: Returns a new : class : Query object for the current model.


Instruction:
Can you write a function in Python 3 where it
returns a new Query object for the given text search value.

Response:
def search(self, text, lookup=None):
        '''Returns a new :class:`Query` for :attr:`Manager.model` with
a full text search value.'''
        return self.query().search(text, lookup=lookup)


Instruction:
Can you generate the documentation for the following Python 3 function
def pairs_to_dict(response, encoding):
    "Create a dict given a list of key/value pairs"
    it = iter(response)
    return dict(((k.decode(encoding), v) for k, v in zip(it, it)))

Response: Create a dict given a list of key / value pairs


Instruction:
How would you explain what the following Python 3 function does
def load_related(self, meta, fname, data, fields, encoding):
        '''Parse data for related objects.'''
        field = meta.dfields[fname]
        if field in meta.multifields:
            fmeta = field.structure_class()._meta
            if fmeta.name in ('hashtable', 'zset'):
                return ((native_str(id, encoding),
                         pairs_to_dict(fdata, encoding)) for
                        id, fdata in data)
            else:
                return ((native_str(id, encoding), fdata) for
                        id, fdata in data)
        else:
            # this is data for stdmodel instances
            return self.build(data, meta, fields, fields, encoding)

Response: Parse data for related objects.


Instruction:
Given the following Python 3 function, write the documentation
def _execute_query(self):
        '''Execute the query without fetching data. Returns the number of
elements in the query.'''
        pipe = self.pipe
        if not self.card:
            if self.meta.ordering:
                self.ismember = getattr(self.backend.client, 'zrank')
                self.card = getattr(pipe, 'zcard')
                self._check_member = self.zism
            else:
                self.ismember = getattr(self.backend.client, 'sismember')
                self.card = getattr(pipe, 'scard')
                self._check_member = self.sism
        else:
            self.ismember = None
        self.card(self.query_key)
        result = yield pipe.execute()
        yield result[-1]

Response:
Execute the query without fetching data. Returns the number of
elements in the query.


Instruction:
Write a Python 3 function that can
perform ordering with respect model fields.

Response:
def order(self, last):
        '''Perform ordering with respect model fields.'''
        desc = last.desc
        field = last.name
        nested = last.nested
        nested_args = []
        while nested:
            meta = nested.model._meta
            nested_args.extend((self.backend.basekey(meta), nested.name))
            last = nested
            nested = nested.nested
        method = 'ALPHA' if last.field.internal_type == 'text' else ''
        if field == last.model._meta.pkname():
            field = ''
        return {'field': field,
                'method': method,
                'desc': desc,
                'nested': nested_args}


Instruction:
Implement a Python 3 function for
removing and return a range from the ordered set by rank ( index.

Response:
def ipop_range(self, start, stop=None, withscores=True, **options):
        '''Remove and return a range from the ordered set by rank (index).'''
        return self.backend.execute(
            self.client.zpopbyrank(self.id, start, stop,
                                   withscores=withscores, **options),
            partial(self._range, withscores))


Instruction:
How would you code a function in Python 3 to
remove and return a range from the ordered set by score.

Response:
def pop_range(self, start, stop=None, withscores=True, **options):
        '''Remove and return a range from the ordered set by score.'''
        return self.backend.execute(
            self.client.zpopbyscore(self.id, start, stop,
                                    withscores=withscores, **options),
            partial(self._range, withscores))


Instruction:
Make a summary of the following Python 3 code
def meta(self, meta):
        '''Extract model metadata for lua script stdnet/lib/lua/odm.lua'''
        data = meta.as_dict()
        data['namespace'] = self.basekey(meta)
        return data

Response: Extract model metadata for lua script stdnet / lib / lua. lua


Instruction:
Write a Python 3 script for
executing a single session in redis.

Response:
def execute_session(self, session_data):
        '''Execute a session in redis.'''
        pipe = self.client.pipeline()
        for sm in session_data:  # loop through model sessions
            meta = sm.meta
            if sm.structures:
                self.flush_structure(sm, pipe)
            delquery = None
            if sm.deletes is not None:
                delquery = sm.deletes.backend_query(pipe=pipe)
            self.accumulate_delete(pipe, delquery)
            if sm.dirty:
                meta_info = json.dumps(self.meta(meta))
                lua_data = [len(sm.dirty)]
                processed = []
                for instance in sm.dirty:
                    state = instance.get_state()
                    if not meta.is_valid(instance):
                        raise FieldValueError(
                            json.dumps(instance._dbdata['errors']))
                    score = MIN_FLOAT
                    if meta.ordering:
                        if meta.ordering.auto:
                            score = meta.ordering.name.incrby
                        else:
                            v = getattr(instance, meta.ordering.name, None)
                            if v is not None:
                                score = meta.ordering.field.scorefun(v)
                    data = instance._dbdata['cleaned_data']
                    action = state.action
                    prev_id = state.iid if state.persistent else ''
                    id = instance.pkvalue() or ''
                    data = flat_mapping(data)
                    lua_data.extend((action, prev_id, id, score, len(data)))
                    lua_data.extend(data)
                    processed.append(state.iid)
                self.odmrun(pipe, 'commit', meta, (), meta_info,
                            *lua_data, iids=processed)
        return pipe.execute()


Instruction:
Write a Python 3 script to
flush all model keys from the database

Response:
def flush(self, meta=None):
        '''Flush all model keys from the database'''
        pattern = self.basekey(meta) if meta else self.namespace
        return self.client.delpattern('%s*' % pattern)


Instruction:
Given the following Python 3 function, write the documentation
def GetCovariance(kernel, kernel_params, time, errors):
    '''
    Returns the covariance matrix for a given light curve
    segment.

    :param array_like kernel_params: A list of kernel parameters \
          (white noise amplitude, red noise amplitude, and red noise timescale)
    :param array_like time: The time array (*N*)
    :param array_like errors: The data error array (*N*)

    :returns: The covariance matrix :py:obj:`K` (*N*,*N*)

    '''

    # NOTE: We purposefully compute the covariance matrix
    # *without* the GP white noise term
    K = np.diag(errors ** 2)
    K += GP(kernel, kernel_params, white=False).get_matrix(time)
    return K

Response:
Calculates the covariance matrix for a given light curve
    segment.


Instruction:
How would you code a function in Python 3 to
optimize the GP by training it on the current de-trended light curve. Returns the white noise amplitude, red noise amplitude, and red noise timescale. :param array_like time: The time array :param array_like flux: The flux array :param array_like errors: The flux errors array :param array_like mask: The indices to be masked when training the GP. \ Default `[]` :param int giter: The number of iterations. Default 3 :param int gmaxf: The maximum number of function evaluations. Default 200 :param tuple guess: The guess to initialize the minimization with. \ Default :py:obj:`None`

Response:
def GetKernelParams(time, flux, errors, kernel='Basic', mask=[],
                    giter=3, gmaxf=200, guess=None):
    '''
    Optimizes the GP by training it on the current de-trended light curve.
    Returns the white noise amplitude, red noise amplitude,
    and red noise timescale.

    :param array_like time: The time array
    :param array_like flux: The flux array
    :param array_like errors: The flux errors array
    :param array_like mask: The indices to be masked when training the GP. \
           Default `[]`
    :param int giter: The number of iterations. Default 3
    :param int gmaxf: The maximum number of function evaluations. Default 200
    :param tuple guess: The guess to initialize the minimization with. \
           Default :py:obj:`None`

    '''

    log.info("Optimizing the GP...")

    # Save a copy of time and errors for later
    time_copy = np.array(time)
    errors_copy = np.array(errors)

    # Apply the mask
    time = np.delete(time, mask)
    flux = np.delete(flux, mask)
    errors = np.delete(errors, mask)

    # Remove 5-sigma outliers to be safe
    f = flux - savgol_filter(flux, 49, 2) + np.nanmedian(flux)
    med = np.nanmedian(f)
    MAD = 1.4826 * np.nanmedian(np.abs(f - med))
    mask = np.where((f > med + 5 * MAD) | (f < med - 5 * MAD))[0]
    time = np.delete(time, mask)
    flux = np.delete(flux, mask)
    errors = np.delete(errors, mask)

    # Initial guesses and bounds
    white = np.nanmedian([np.nanstd(c) for c in Chunks(flux, 13)])
    amp = np.nanstd(flux)
    tau = 30.0
    if kernel == 'Basic':
        if guess is None:
            guess = [white, amp, tau]
        bounds = [[0.1 * white, 10. * white],
                  [1., 10000. * amp],
                  [0.5, 100.]]
    elif kernel == 'QuasiPeriodic':
        if guess is None:
            guess = [white, amp, tau, 1., 20.]
        bounds = [[0.1 * white, 10. * white],
                  [1., 10000. * amp],
                  [1e-5, 1e2],
                  [0.02, 100.]]
    else:
        raise ValueError('Invalid value for `kernel`.')

    # Loop
    llbest = -np.inf
    xbest = np.array(guess)
    for i in range(giter):

        # Randomize an initial guess
        iguess = [np.inf for g in guess]
        for j, b in enumerate(bounds):
            tries = 0
            while (iguess[j] < b[0]) or (iguess[j] > b[1]):
                iguess[j] = (1 + 0.5 * np.random.randn()) * guess[j]
                tries += 1
                if tries > 100:
                    iguess[j] = b[0] + np.random.random() * (b[1] - b[0])
                    break

        # Optimize
        x = fmin_l_bfgs_b(NegLnLike, iguess, approx_grad=False,
                          bounds=bounds, args=(time, flux, errors, kernel),
                          maxfun=gmaxf)
        log.info('Iteration #%d/%d:' % (i + 1, giter))
        log.info('   ' + x[2]['task'].decode('utf-8'))
        log.info('   ' + 'Function calls: %d' % x[2]['funcalls'])
        log.info('   ' + 'Log-likelihood: %.3e' % -x[1])
        if kernel == 'Basic':
            log.info('   ' + 'White noise   : %.3e (%.1f x error bars)' %
                     (x[0][0], x[0][0] / np.nanmedian(errors)))
            log.info('   ' + 'Red amplitude : %.3e (%.1f x stand dev)' %
                     (x[0][1], x[0][1] / np.nanstd(flux)))
            log.info('   ' + 'Red timescale : %.2f days' % x[0][2])
        elif kernel == 'QuasiPeriodic':
            log.info('   ' + 'White noise   : %.3e (%.1f x error bars)' %
                     (x[0][0], x[0][0] / np.nanmedian(errors)))
            log.info('   ' + 'Red amplitude : %.3e (%.1f x stand dev)' %
                     (x[0][1], x[0][1] / np.nanstd(flux)))
            log.info('   ' + 'Gamma         : %.3e' % x[0][2])
            log.info('   ' + 'Period        : %.2f days' % x[0][3])
        if -x[1] > llbest:
            llbest = -x[1]
            xbest = np.array(x[0])

    return xbest


Instruction:
Can you create a Python 3 function that
returns the negative log - likelihood function and its gradient.

Response:
def NegLnLike(x, time, flux, errors, kernel):
    '''
    Returns the negative log-likelihood function and its gradient.

    '''

    gp = GP(kernel, x, white=True)
    gp.compute(time, errors)
    if OLDGEORGE:
        nll = -gp.lnlikelihood(flux)
        # NOTE: There was a bug on this next line! Used to be
        #
        #    ngr = -gp.grad_lnlikelihood(flux) / gp.kernel.pars
        #
        # But I think we want
        #
        # dlogL/dx =     dlogL/dlogx^2       * dlogx^2/dx^2 * dx^2/dx
        #          = gp.grad_lnlikelihood()  *     1/x^2    *   2x
        #          = 2 * gp.grad_lnlikelihood() / x
        #          = 2 * gp.grad_lnlikelihood() / np.sqrt(x^2)
        #          = 2 * gp.grad_lnlikelihood() / np.sqrt(gp.kernel.pars)
        #
        # (with a negative sign out front for the negative gradient).
        # So we probably weren't optimizing the GP correctly! This affects
        # all campaigns through C13. It's not a *huge* deal, since the sign
        # of the gradient was correct and the model isn't that sensitive to
        # the value of the hyperparameters, but it may have contributed to
        # the poor performance on super variable stars. In most cases it means
        # the solver takes longer to converge and isn't as good at finding
        # the minimum.
        ngr = -2 * gp.grad_lnlikelihood(flux) / np.sqrt(gp.kernel.pars)
    else:
        nll = -gp.log_likelihood(flux)
        ngr = -2 * gp.grad_log_likelihood(flux) / \
            np.sqrt(np.exp(gp.get_parameter_vector()))

    return nll, ngr


Instruction:
Write a Python 3 script to
give a ``startdate`` and an ``enddate`` dates, evaluate the date intervals from which data is not available. It return a list of two-dimensional tuples containing start and end date for the interval. The list could countain 0,1 or 2 tuples.

Response:
def missing_intervals(startdate, enddate, start, end,
                      dateconverter=None,
                      parseinterval=None,
                      intervals=None):
    '''Given a ``startdate`` and an ``enddate`` dates, evaluate the
date intervals from which data is not available. It return a list of
two-dimensional tuples containing start and end date for the interval.
The list could countain 0,1 or 2 tuples.'''
    parseinterval = parseinterval or default_parse_interval
    dateconverter = dateconverter or todate
    startdate = dateconverter(parseinterval(startdate, 0))
    enddate = max(startdate, dateconverter(parseinterval(enddate, 0)))
    if intervals is not None and not isinstance(intervals, Intervals):
        intervals = Intervals(intervals)

    calc_intervals = Intervals()
    # we have some history already
    if start:
        # the startdate not available
        if startdate < start:
            calc_start = startdate
            calc_end = parseinterval(start, -1)
            if calc_end >= calc_start:
                calc_intervals.append(Interval(calc_start, calc_end))

        if enddate > end:
            calc_start = parseinterval(end, 1)
            calc_end = enddate
            if calc_end >= calc_start:
                calc_intervals.append(Interval(calc_start, calc_end))
    else:
        start = startdate
        end = enddate
        calc_intervals.append(Interval(startdate, enddate))

    if calc_intervals:
        if intervals:
            calc_intervals.extend(intervals)
    elif intervals:
        calc_intervals = intervals

    return calc_intervals


Instruction:
Explain what the following Python 3 code does
def dategenerator(start, end, step=1, desc=False):
    '''Generates dates between *atrt* and *end*.'''
    delta = timedelta(abs(step))
    end = max(start, end)
    if desc:
        dt = end
        while dt >= start:
            yield dt
            dt -= delta
    else:
        dt = start
        while dt <= end:
            yield dt
            dt += delta

Response: Generates dates between start and end.


Instruction:
How would you code a function in Python 3 to
initialize the logging functionality.

Response:
def InitLog(file_name=None, log_level=logging.DEBUG,
            screen_level=logging.CRITICAL, pdb=False):
    '''
    A little routine to initialize the logging functionality.

    :param str file_name: The name of the file to log to. \
           Default :py:obj:`None` (set internally by :py:mod:`everest`)
    :param int log_level: The file logging level (0-50). Default 10 (debug)
    :param int screen_level: The screen logging level (0-50). \
           Default 50 (critical)

    '''

    # Initialize the logging
    root = logging.getLogger()
    root.handlers = []
    root.setLevel(logging.DEBUG)

    # File handler
    if file_name is not None:
        if not os.path.exists(os.path.dirname(file_name)):
            os.makedirs(os.path.dirname(file_name))
        fh = logging.FileHandler(file_name)
        fh.setLevel(log_level)
        fh_formatter = logging.Formatter(
            "%(asctime)s %(levelname)-5s [%(name)s.%(funcName)s()]: %(message)s",
            datefmt="%m/%d/%y %H:%M:%S")
        fh.setFormatter(fh_formatter)
        fh.addFilter(NoPILFilter())
        root.addHandler(fh)

    # Screen handler
    sh = logging.StreamHandler(sys.stdout)
    if pdb:
        sh.setLevel(logging.DEBUG)
    else:
        sh.setLevel(screen_level)
    sh_formatter = logging.Formatter(
        "%(levelname)-5s [%(name)s.%(funcName)s()]: %(message)s")
    sh.setFormatter(sh_formatter)
    sh.addFilter(NoPILFilter())
    root.addHandler(sh)

    # Set exception hook
    if pdb:
        sys.excepthook = ExceptionHookPDB
    else:
        sys.excepthook = ExceptionHook


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sort_like(l, col1, col2):
    '''
    Sorts the list :py:obj:`l` by comparing :py:obj:`col2` to :py:obj:`col1`.
    Specifically, finds the indices :py:obj:`i` such that ``col2[i] = col1``
    and returns ``l[i]``. This is useful when comparing the CDPP values of
    catalogs generated by different pipelines. The
    target IDs are all the same, but won't necessarily be in the same order.
    This allows :py:obj:`everest` to sort the CDPP arrays so that the
    targets match.

    :param array_like l: The list or array to sort
    :param array_like col1: A list or array (same length as :py:obj:`l`)
    :param array_like col2: A second list or array containing the same \
           elements as :py:obj:`col1` but in a different order

    '''

    s = np.zeros_like(col1) * np.nan
    for i, c in enumerate(col1):
        j = np.argmax(col2 == c)
        if j == 0:
            if col2[0] != c:
                continue
        s[i] = l[j]
    return s

Response: Sorts the list l by comparing col1 to col2.


Instruction:
Write a Python 3 function for
progressing bar range with tqdm

Response:
def prange(*x):
    '''
    Progress bar range with `tqdm`

    '''

    try:
        root = logging.getLogger()
        if len(root.handlers):
            for h in root.handlers:
                if (type(h) is logging.StreamHandler) and \
                        (h.level != logging.CRITICAL):
                    from tqdm import tqdm
                    return tqdm(range(*x))
            return range(*x)
        else:
            from tqdm import tqdm
            return tqdm(range(*x))
    except ImportError:
        return range(*x)


Instruction:
Explain what the following Python 3 code does
def front(self, *fields):
        '''Return the front pair of the structure'''
        ts = self.irange(0, 0, fields=fields)
        if ts:
            return ts.start(), ts[0]

Response: Return the front pair of the structure


Instruction:
Can you implement a function in Python 3 that
returns the back pair of the structure

Response:
def back(self, *fields):
        '''Return the back pair of the structure'''
        ts = self.irange(-1, -1, fields=fields)
        if ts:
            return ts.end(), ts[0]


Instruction:
How would you implement a function in Python 3 that
converts the backend into the database connection parameters. It returns a tuple of scheme host params.

Response:
def parse_backend(backend):
    """Converts the "backend" into the database connection parameters.
It returns a (scheme, host, params) tuple."""
    r = urlparse.urlsplit(backend)
    scheme, host = r.scheme, r.netloc
    path, query = r.path, r.query
    if path and not query:
        query, path = path, ''
        if query:
            if query.find('?'):
                path = query
            else:
                query = query[1:]
    if query:
        params = dict(urlparse.parse_qsl(query))
    else:
        params = {}

    return scheme, host, params


Instruction:
Write a Python 3 script to
get a : class : BackendDataServer.

Response:
def getdb(backend=None, **kwargs):
    '''get a :class:`BackendDataServer`.'''
    if isinstance(backend, BackendDataServer):
        return backend
    backend = backend or settings.DEFAULT_BACKEND
    if not backend:
        return None
    scheme, address, params = parse_backend(backend)
    params.update(kwargs)
    if 'timeout' in params:
        params['timeout'] = int(params['timeout'])
    return _getdb(scheme, address, params)


Instruction:
Can you generate the documentation for the following Python 3 function
def basekey(self, meta, *args):
        key = '%s%s' % (self.namespace, meta.modelkey)
        postfix = ':'.join((str(p) for p in args if p is not None))
        return '%s:%s' % (key, postfix) if postfix else key

Response: Calculate the basekey to access model data.


Instruction:
Here you have a function in Python 3, explain what it does
def make_objects(self, meta, data, related_fields=None):
        '''Generator of :class:`stdnet.odm.StdModel` instances with data
from database.

:parameter meta: instance of model :class:`stdnet.odm.Metaclass`.
:parameter data: iterator over instances data.
'''
        make_object = meta.make_object
        related_data = []
        if related_fields:
            for fname, fdata in iteritems(related_fields):
                field = meta.dfields[fname]
                if field in meta.multifields:
                    related = dict(fdata)
                    multi = True
                else:
                    multi = False
                    relmodel = field.relmodel
                    related = dict(((obj.id, obj) for obj in
                                    self.make_objects(relmodel._meta, fdata)))
                related_data.append((field, related, multi))
        for state in data:
            instance = make_object(state, self)
            for field, rdata, multi in related_data:
                if multi:
                    field.set_cache(instance, rdata.get(str(instance.id)))
                else:
                    rid = getattr(instance, field.attname, None)
                    if rid is not None:
                        value = rdata.get(rid)
                        setattr(instance, field.name, value)
            yield instance

Response:
Generator of objects with data
from database.


Instruction:
Write a Python 3 function that can
create a new backend : class : stdnet. odm. Structure handler.

Response:
def structure(self, instance, client=None):
        '''Create a backend :class:`stdnet.odm.Structure` handler.

        :param instance: a :class:`stdnet.odm.Structure`
        :param client: Optional client handler.
        '''
        struct = self.struct_map.get(instance._meta.name)
        if struct is None:
            raise ModelNotAvailable('"%s" is not available for backend '
                                    '"%s"' % (instance._meta.name, self))
        client = client if client is not None else self.client
        return struct(instance, self, client)


Instruction:
Here you have a function in Python 3, explain what it does
def Search(ID, mission='k2'):
    # Only K2 supported for now
    assert mission == 'k2', "Only the K2 mission is supported for now."
    print("Searching for target %d..." % ID)

    # First check if it is in the database
    season = missions.k2.Season(ID)
    if season in [91, 92, [91, 92]]:
        print("Campaign 9 is currently not part of the EVEREST catalog.")
        return
    elif season == 101:
        print("The first half of campaign 10 is not currently part of " +
              "the EVEREST catalog.")
        return
    elif season is not None:
        print("Target is in campaign %d of the EVEREST catalog." % season)
        return

    # Get the kplr object
    star = k2plr_client.k2_star(ID)

    # First check if this is a star
    if star.objtype.lower() != "star":
        print("Target is of type %s, not STAR, " % star.objtype +
              "and is therefore not included in the EVEREST catalog.")
        return

    # Let's try to download the pixel data and see what happens
    try:
        tpf = star.get_target_pixel_files()
    except:
        print("Unable to download the raw pixel files for this target.")
        return
    if len(tpf) == 0:
        print("Raw pixel files are not available for this target. Looks like " +
              "data may not have been collected for it.")
        return

    # Perhaps it's in a campaign we haven't gotten to yet
    if tpf[0].sci_campaign not in missions.k2.SEASONS:
        print("Targets for campaign %d are not yet available."
              % tpf[0].sci_campaign)
        return

    # Let's try to download the K2SFF data
    try:
        k2sff = k2plr.K2SFF(ID)
    except:
        print("Error downloading the K2SFF light curve for this target. " +
              "Currently, EVEREST uses the K2SFF apertures to perform " +
              "photometry. This is likely to change in the next version.")
        return

    # Let's try to get the aperture
    try:
        assert np.count_nonzero(k2sff.apertures[15]), "Invalid aperture."
    except:
        print("Unable to retrieve the K2SFF aperture for this target. " +
              "Currently, EVEREST uses the K2SFF apertures to perform " +
              "photometry. This is likely to change in the next version.")
        return

    # Perhaps the star is *super* saturated and we didn't bother
    # de-trending it?
    if star.kp < 8:
        print("Target has Kp = %.1f and is too saturated " +
              "for proper de-trending with EVEREST.")
        return

    # I'm out of ideas
    print("I'm not sure why this target isn't in the EVEREST catalog." +
          "You can try de-trending it yourself:")
    print("http://faculty.washington.edu/rodluger/everest/pipeline.html")
    return

Response: Search for a target in the EVEREST database.


Instruction:
Implement a Python 3 function for
downloading a given file from MAST.

Response:
def DownloadFile(ID, season=None, mission='k2', cadence='lc',
                 filename=None, clobber=False):
    '''
    Download a given :py:mod:`everest` file from MAST.

    :param str mission: The mission name. Default `k2`
    :param str cadence: The light curve cadence. Default `lc`
    :param str filename: The name of the file to download. Default \
           :py:obj:`None`, in which case the default \
           FITS file is retrieved.
    :param bool clobber: If :py:obj:`True`, download and overwrite \
           existing files. Default :py:obj:`False`

    '''

    # Get season
    if season is None:
        season = getattr(missions, mission).Season(ID)
    if hasattr(season, '__len__'):
        raise AttributeError(
            "Please choose a `season` for this target: %s." % season)
    if season is None:
        if getattr(missions, mission).ISTARGET(ID):
            raise ValueError('Target not found in local database. ' +
                             'Run `everest.Search(%d)` for more information.'
                             % ID)
        else:
            raise ValueError('Invalid target ID.')
    path = getattr(missions, mission).TargetDirectory(ID, season)
    relpath = getattr(missions, mission).TargetDirectory(
        ID, season, relative=True)
    if filename is None:
        filename = getattr(missions, mission).FITSFile(ID, season, cadence)

    # Check if file exists
    if not os.path.exists(path):
        os.makedirs(path)
    elif os.path.exists(os.path.join(path, filename)) and not clobber:
        log.info('Found cached file.')
        return os.path.join(path, filename)

    # Get file URL
    log.info('Downloading the file...')
    fitsurl = getattr(missions, mission).FITSUrl(ID, season)
    if not fitsurl.endswith('/'):
        fitsurl += '/'

    # Download the data
    r = urllib.request.Request(fitsurl + filename)
    try:
        handler = urllib.request.urlopen(r)
        code = handler.getcode()
    except (urllib.error.HTTPError, urllib.error.URLError):
        code = 0
    if int(code) == 200:

        # Read the data
        data = handler.read()

        # Atomically save to disk
        f = NamedTemporaryFile("wb", delete=False)
        f.write(data)
        f.flush()
        os.fsync(f.fileno())
        f.close()
        shutil.move(f.name, os.path.join(path, filename))

    else:

        # Something went wrong!
        log.error("Error code {0} for URL '{1}'".format(
            code, fitsurl + filename))

        # If the files can be accessed by `ssh`, let's try that
        # (development version only!)
        if EVEREST_FITS is None:
            raise Exception("Unable to locate the file.")

        # Get the url
        inpath = os.path.join(EVEREST_FITS, relpath, filename)
        outpath = os.path.join(path, filename)

        # Download the data
        log.info("Accessing file via `scp`...")
        subprocess.call(['scp', inpath, outpath])

    # Success?
    if os.path.exists(os.path.join(path, filename)):
        return os.path.join(path, filename)
    else:
        raise Exception("Unable to download the file." +
                        "Run `everest.Search(%d)` to troubleshoot." % ID)


Instruction:
Can you create a Python 3 function that
shows the data validation summary for a given target.

Response:
def DVS(ID, season=None, mission='k2', clobber=False,
        cadence='lc', model='nPLD'):
    '''
    Show the data validation summary (DVS) for a given target.

    :param str mission: The mission name. Default `k2`
    :param str cadence: The light curve cadence. Default `lc`
    :param bool clobber: If :py:obj:`True`, download and overwrite \
           existing files. Default :py:obj:`False`

    '''

    # Get season
    if season is None:
        season = getattr(missions, mission).Season(ID)
    if hasattr(season, '__len__'):
        raise AttributeError(
            "Please choose a `season` for this target: %s." % season)

    # Get file name
    if model == 'nPLD':
        filename = getattr(missions, mission).DVSFile(ID, season, cadence)
    else:
        if cadence == 'sc':
            filename = model + '.sc.pdf'
        else:
            filename = model + '.pdf'

    file = DownloadFile(ID, season=season,
                        mission=mission,
                        filename=filename,
                        clobber=clobber)

    try:
        if platform.system().lower().startswith('darwin'):
            subprocess.call(['open', file])
        elif os.name == 'nt':
            os.startfile(file)
        elif os.name == 'posix':
            subprocess.call(['xdg-open', file])
        else:
            raise Exception("")
    except:
        log.info("Unable to open the pdf. Try opening it manually:")
        log.info(file)


Instruction:
Can you generate the documentation for the following Python 3 function
def compute(self):
        '''
        Re-compute the :py:mod:`everest` model for the given
        value of :py:obj:`lambda`.
        For long cadence `k2` light curves, this should take several
        seconds. For short cadence `k2` light curves, it may take a
        few minutes. Note that this is a simple wrapper around
        :py:func:`everest.Basecamp.compute`.

        '''

        # If we're doing iterative PLD, get the normalization
        if self.model_name == 'iPLD':
            self._get_norm()

        # Compute as usual
        super(Everest, self).compute()

        # Make NaN cadences NaNs
        self.flux[self.nanmask] = np.nan

Response: Compute the current cadence model for the given base cadence class.


Instruction:
Make a summary of the following Python 3 code
def _get_norm(self):
        '''
        Computes the PLD flux normalization array.

        ..note :: `iPLD` model **only**.

        '''

        log.info('Computing the PLD normalization...')

        # Loop over all chunks
        mod = [None for b in self.breakpoints]
        for b, brkpt in enumerate(self.breakpoints):

            # Unmasked chunk
            c = self.get_chunk(b)

            # Masked chunk (original mask plus user transit mask)
            inds = np.array(
                list(set(np.concatenate([self.transitmask,
                                         self.recmask]))), dtype=int)
            M = np.delete(np.arange(len(self.time)), inds, axis=0)
            if b > 0:
                m = M[(M > self.breakpoints[b - 1] - self.bpad)
                      & (M <= self.breakpoints[b] + self.bpad)]
            else:
                m = M[M <= self.breakpoints[b] + self.bpad]

            # This block of the masked covariance matrix
            mK = GetCovariance(self.kernel, self.kernel_params,
                               self.time[m], self.fraw_err[m])

            # Get median
            med = np.nanmedian(self.fraw[m])

            # Normalize the flux
            f = self.fraw[m] - med

            # The X^2 matrices
            A = np.zeros((len(m), len(m)))
            B = np.zeros((len(c), len(m)))

            # Loop over all orders
            for n in range(self.pld_order):
                XM = self.X(n, m)
                XC = self.X(n, c)
                A += self.reclam[b][n] * np.dot(XM, XM.T)
                B += self.reclam[b][n] * np.dot(XC, XM.T)
                del XM, XC

            W = np.linalg.solve(mK + A, f)
            mod[b] = np.dot(B, W)
            del A, B, W

        # Join the chunks after applying the correct offset
        if len(mod) > 1:

            # First chunk
            model = mod[0][:-self.bpad]

            # Center chunks
            for m in mod[1:-1]:
                offset = model[-1] - m[self.bpad - 1]
                model = np.concatenate(
                    [model, m[self.bpad:-self.bpad] + offset])

            # Last chunk
            offset = model[-1] - mod[-1][self.bpad - 1]
            model = np.concatenate([model, mod[-1][self.bpad:] + offset])

        else:

            model = mod[0]

        # Subtract the global median
        model -= np.nanmedian(model)

        # Save the norm
        self._norm = self.fraw - model

Response: Compute the PLD flux normalization array.