Instruction:
Can you tell what is the following Python 3 function doing
def bces(x1, x2, x1err=[], x2err=[], cerr=[], logify=True, model='yx', \
         bootstrap=5000, verbose='normal', full_output=True):
    """
    Bivariate, Correlated Errors and intrinsic Scatter (BCES)
    translated from the FORTRAN code by Christina Bird and Matthew Bershady
    (Akritas & Bershady, 1996)

    Linear regression in the presence of heteroscedastic errors on both
    variables and intrinsic scatter

    Parameters
    ----------
      x1        : array of floats
                  Independent variable, or observable
      x2        : array of floats
                  Dependent variable
      x1err     : array of floats (optional)
                  Uncertainties on the independent variable
      x2err     : array of floats (optional)
                  Uncertainties on the dependent variable
      cerr      : array of floats (optional)
                  Covariances of the uncertainties in the dependent and
                  independent variables
      logify    : bool (default True)
                  Whether to take the log of the measurements in order to
                  estimate the best-fit power law instead of linear relation
      model     : {'yx', 'xy', 'bi', 'orth'}
                  BCES model with which to calculate regression. See Notes
                  below for details.
      bootstrap : False or int (default 5000)
                  get the errors from bootstrap resampling instead of the
                  analytical prescription? if bootstrap is an int, it is the
                  number of bootstrap resamplings
      verbose   : str (default 'normal')
                  Verbose level. Options are {'quiet', 'normal', 'debug'}
      full_output : bool (default True)
                  If True, return also the covariance between the
                  normalization and slope of the regression.

    Returns
    -------
      a         : tuple of length 2
                  Best-fit normalization and its uncertainty (a, da)
      b         : tuple of length 2
                  Best-fit slope and its uncertainty (b, db)

    Optional outputs
    ----------------
      cov_ab    : 2x2 array of floats
                  covariance between a and b. Returned if full_output is set to
                  True.

    Notes
    -----
      If verbose is normal or debug, the results from all the BCES models will
      be printed (still, only the one selected in *model* will be returned).

      the *model* parameter:
        -'yx' stands for BCES(Y|X)
        -'xy' stands for BCES(X|Y)
        -'bi' stands for BCES Bisector
        -'orth' stands for BCES Orthogonal

    """

    def _bess_bootstrap(npts, x1, x2, x1err, x2err, cerr,nsim):
        ##added by Gerrit, July 2014
        ##Unfortunately I needed a copy of the _bess function for bootstrapping.
        #Would be nicer if those two could be combined
        """
        Do the entire regression calculation for 4 slopes:
        OLS(Y|X), OLS(X|Y), bisector, orthogonal
        """
        #calculate sigma's for datapoints using length of confidence intervals
        sig11var = numpy.sum(x1err ** 2,axis=1,keepdims=True) / npts
        sig22var = numpy.sum(x2err ** 2,axis=1,keepdims=True) / npts
        sig12var = numpy.sum(cerr,axis=1,keepdims=True) / npts

        # calculate means and variances
        x1av = numpy.mean(x1,axis=1,keepdims=True)
        x1var = x1.var(axis=1,keepdims=True)
        x2av = numpy.mean(x2,axis=1,keepdims=True)
        x2var = x2.var(axis=1,keepdims=True)
        covar_x1x2 = numpy.mean((x1-numpy.mean(x1,axis=1,keepdims=True)) * \
                             (x2-numpy.mean(x2,axis=1,keepdims=True)),
                             axis=1,keepdims=True)

        # compute the regression slopes for OLS(X2|X1), OLS(X1|X2),
        # bisector and orthogonal
        if model == 'yx':
            modelint = 1
        else:
            modelint = 4
        b = numpy.zeros((modelint,nsim))
        b[0] = ((covar_x1x2 - sig12var) / (x1var - sig11var)).flatten()
        if model != 'yx':
            b[1] = ((x2var - sig22var) / (covar_x1x2 - sig12var)).flatten()
            b[2] = ((b[0] * b[1] - 1 + numpy.sqrt((1 + b[0] ** 2) * \
                   (1 + b[1] ** 2))) / (b[0] + b[1])).flatten()
            b[3] = 0.5 * ((b[1] - 1 / b[0]) + numpy.sign(covar_x1x2).flatten()* \
                   numpy.sqrt(4 + (b[1] - 1 / b[0]) ** 2))

        # compute intercepts for above 4 cases:
        a = x2av.flatten() - b * x1av.flatten()

        # set up variables to calculate standard deviations of slope and
        # intercept
        xi = []
        xi.append(((x1 - x1av) * (x2 - b[0].reshape(nsim,1) * x1 - \
                                  a[0].reshape(nsim,1)) + \
                   b[0].reshape(nsim,1) * x1err ** 2) / \
                  (x1var - sig11var))
        if model != 'yx':
            xi.append(((x2 - x2av) * (x2 - b[1].reshape(nsim,1) * x1 - \
                                      a[1].reshape(nsim,1)) + x2err ** 2) / \
                      covar_x1x2)
            xi.append((xi[0] * (1 + b[1].reshape(nsim,1) ** 2) + \
                       xi[1] * (1 + b[0].reshape(nsim,1) ** 2)) / \
                      ((b[0].reshape(nsim,1) + \
                       b[1].reshape(nsim,1)) * \
                       numpy.sqrt((1 + b[0].reshape(nsim,1) ** 2) * \
                               (1 + b[1].reshape(nsim,1) ** 2))))
            xi.append((xi[0] / b[0].reshape(nsim,1) ** 2 + xi[1]) * \
                      b[3].reshape(nsim,1) / \
                      numpy.sqrt(4 + (b[1].reshape(nsim,1) - \
                              1 / b[0].reshape(nsim,1)) ** 2))
        zeta = []
        for i in xrange(modelint):
            zeta.append(x2 - b[i].reshape(nsim,1) * x1 - x1av * xi[i])

        # calculate  variance for all a and b
        bvar = numpy.zeros((4,nsim))
        avar = numpy.zeros((4,nsim))
        for i in xrange(modelint):
            bvar[i] = xi[i].var(axis=1,keepdims=False)/ npts
            avar[i] = zeta[i].var(axis=1,keepdims=False) / npts
        return a, b, avar, bvar, xi, zeta

    def _bess(npts, x1, x2, x1err, x2err, cerr):
        """
        Do the entire regression calculation for 4 slopes:
          OLS(Y|X), OLS(X|Y), bisector, orthogonal
        """
        # calculate sigma's for datapoints using length of confidence
        # intervals
        sig11var = sum(x1err ** 2) / npts
        sig22var = sum(x2err ** 2) / npts
        sig12var = sum(cerr) / npts
        # calculate means and variances
        x1av = numpy.average(x1)
        x1var = numpy.std(x1) ** 2
        x2av = numpy.average(x2)
        x2var = numpy.std(x2) ** 2
        covar_x1x2 = sum((x1 - x1av) * (x2 - x2av)) / npts
        # compute the regression slopes for OLS(X2|X1), OLS(X1|X2),
        # bisector and orthogonal
        b = numpy.zeros(4)
        b[0] = (covar_x1x2 - sig12var) / (x1var - sig11var)
        b[1] = (x2var - sig22var) / (covar_x1x2 - sig12var)
        b[2] = (b[0] * b[1] - 1 + numpy.sqrt((1 + b[0] ** 2) * \
               (1 + b[1] ** 2))) / (b[0] + b[1])
        b[3] = 0.5 * ((b[1] - 1 / b[0]) + numpy.sign(covar_x1x2) * \
               numpy.sqrt(4 + (b[1] - 1 / b[0]) ** 2))
        # compute intercepts for above 4 cases:
        a = x2av - b * x1av
        # set up variables to calculate standard deviations of slope
        # and intercept
        xi = []
        xi.append(((x1 - x1av) * \
                   (x2 - b[0] * x1 - a[0]) + b[0] * x1err ** 2) / \
                  (x1var - sig11var))
        xi.append(((x2 - x2av) * (x2 - b[1] * x1 - a[1]) + x2err ** 2) / \
                  covar_x1x2)
        xi.append((xi[0] * (1 + b[1] ** 2) + xi[1] * (1 + b[0] ** 2)) / \
                  ((b[0] + b[1]) * \
                   numpy.sqrt((1 + b[0] ** 2) * (1 + b[1] ** 2))))
        xi.append((xi[0] / b[0] ** 2 + xi[1]) * b[3] / \
                  numpy.sqrt(4 + (b[1] - 1 / b[0]) ** 2))
        zeta = []
        for i in xrange(4):
            zeta.append(x2 - b[i]*x1 - x1av*xi[i])
        # calculate  variance for all a and b
        bvar = numpy.zeros(4)
        avar = numpy.zeros(4)
        for i in xrange(4):
            bvar[i] = numpy.std(xi[i]) ** 2 / npts
            avar[i] = numpy.std(zeta[i]) ** 2 / npts
        return a, b, avar, bvar, xi, zeta

    def _bootspbec(npts, x, y, xerr, yerr, cerr):
        """
        Bootstrap samples
        """
        j = numpy.random.randint(npts, size = npts)
        xboot = x[j]
        xerrboot = xerr[j]
        yboot = y[j]
        yerrboot = yerr[j]
        cerrboot = cerr[j]
        return xboot, yboot, xerrboot, yerrboot, cerrboot

    # ----  Main routine starts here  ---- #
    # convert to numpy arrays just in case
    x1 = numpy.array(x1)
    x2 = numpy.array(x2)
    x1err = numpy.array(x1err)
    x2err = numpy.array(x2err)
    if logify:
        x1, x2, x1err, x2err = to_log(x1, x2, x1err, x2err)
    cerr = numpy.array(cerr)
    models = [['yx', 'xy', 'bi', 'orth'],
              ['BCES(Y|X)', 'BCES(X|Y)', 'BCES Bisector', 'BCES Orthogonal']]
    # which to return?
    j = models[0].index(model)
    npts = len(x1)
    # are the errors defined?
    if len(x1err) == 0:
        x1err = numpy.zeros(npts)
    if len(x2err) == 0:
        x2err = numpy.zeros(npts)
    if len(cerr) == 0:
        cerr = numpy.zeros(npts)
    if verbose == 'debug':
        print 'x1 =', x1
        print 'x1err =', x1err
        print 'x2 =', x2
        print 'x2err =', x2err
        print 'cerr =', cerr
        print '\n ** Returning values for', models[1][j], '**'
        if bootstrap is not False:
            print '    with errors from %d bootstrap resamplings' %bootstrap
        print ''

    # calculate nominal fits
    bessresults = _bess(npts, x1, x2, x1err, x2err, cerr)
    (a, b, avar, bvar, xi, zeta) = bessresults
    # covariance between normalization and slope
    if full_output:
        covar_ab = numpy.cov(xi[j], zeta[j])

    if bootstrap is not False:
        # make bootstrap simulated datasets, and compute averages and
        # standard deviations of regression coefficients
        asum = numpy.zeros(4)
        assum = numpy.zeros(4)
        bsum = numpy.zeros(4)
        bssum = numpy.zeros(4)
        sda = numpy.zeros(4)
        sdb = numpy.zeros(4)
        for i in xrange(bootstrap):
            samples = _bootspbec(npts, x1, x2, x1err, x2err, cerr)
            (x1sim, x2sim, x1errsim, x2errsim, cerrsim) = samples
            besssim = _bess(npts, x1sim, x2sim, x1errsim, x2errsim, cerrsim)
            (asim, bsim, avarsim, bvarsim, xi, zeta) = besssim
            asum += asim
            assum += asim ** 2
            bsum += bsim
            bssum += bsim ** 2

        aavg = asum / bootstrap
        bavg = bsum / bootstrap
        for i in range(4):
            sdtest = assum[i] - bootstrap * aavg[i] ** 2
            if sdtest > 0:
                sda[i] = numpy.sqrt(sdtest / (bootstrap - 1))
            sdtest = bssum[i] - bootstrap * bavg[i] ** 2
            if sdtest > 0:
                sdb[i] = numpy.sqrt(sdtest / (bootstrap - 1))

    if verbose in ('normal', 'debug'):
        print '%s   B          err(B)' %('Fit'.ljust(19)),
        print '         A          err(A)'
        for i in range(4):
            print '%s  %9.2e +/- %8.2e    %10.3e +/- %9.3e' \
                  %(models[1][i].ljust(16), b[i],
                    numpy.sqrt(bvar[i]), a[i], numpy.sqrt(avar[i]))
            if bootstrap is not False:
                print '%s  %9.2e +/- %8.2e    %10.3e +/- %9.3e' \
                      %('bootstrap'.ljust(16), bavg[i],
                        sdb[i], aavg[i], sda[i])
            print ''
        if verbose == 'debug':
            print 'cov[%s] =' %models[model]
            print covar_ab

    if bootstrap is not False:
        if full_output:
            return (a[j], sda[j]), (b[j], sdb[j]), covar_ab
        else:
            return (a[j], sda[j]), (b[j], sdb[j])
    if full_output:
        out = ((a[j], numpy.sqrt(avar[j])),
               (b[j], numpy.sqrt(bvar[j])),
               covar_ab)
    else:
        out = ((a[j], numpy.sqrt(avar[j])),
               (b[j], numpy.sqrt(bvar[j])))
    return out

Response: Returns the Bivariate Correlated Errors and intrinsic scattering of two uncertainties.


Instruction:
Implement a function in Python 3 to
scatter function for the BCES best fit.

Response:
def scatter(slope, zero, x1, x2, x1err=[], x2err=[]):
    """
    Used mainly to measure scatter for the BCES best-fit

    """
    n = len(x1)
    x2pred = zero + slope * x1
    s = sum((x2 - x2pred) ** 2) / (n - 1)
    if len(x2err) == n:
        s_obs = sum((x2err / x2) ** 2) / n
        s0 = s - s_obs
    print numpy.sqrt(s), numpy.sqrt(s_obs), numpy.sqrt(s0)
    return numpy.sqrt(s0)


Instruction:
Can you generate the documentation for the following Python 3 function
def kelly(x1, x2, x1err=[], x2err=[], cerr=[], logify=True,
          miniter=5000, maxiter=1e5, metro=True,
          silent=True):
    """
    Python wrapper for the linear regression MCMC of Kelly (2007).
    Requires pidly (http://astronomy.sussex.ac.uk/~anthonys/pidly/) and
    an IDL license.

    Parameters
    ----------
      x1        : array of floats
                  Independent variable, or observable
      x2        : array of floats
                  Dependent variable
      x1err     : array of floats (optional)
                  Uncertainties on the independent variable
      x2err     : array of floats (optional)
                  Uncertainties on the dependent variable
      cerr      : array of floats (optional)
                  Covariances of the uncertainties in the dependent and
                  independent variables
    """
    import pidly

    n = len(x1)
    if len(x2) != n:
        raise ValueError('x1 and x2 must have same length')
    if len(x1err) == 0:
        x1err = numpy.zeros(n)
    if len(x2err) == 0:
        x2err = numpy.zeros(n)
    if logify:
        x1, x2, x1err, x2err = to_log(x1, x2, x1err, x2err)
    idl = pidly.IDL()
    idl('x1 = %s' %list(x1))
    idl('x2 = %s' %list(x2))
    cmd = 'linmix_err, x1, x2, fit'
    if len(x1err) == n:
        idl('x1err = %s' %list(x1err))
        cmd += ', xsig=x1err'
    if len(x2err) == n:
        idl('x2err = %s' %list(x2err))
        cmd += ', ysig=x2err'
    if len(cerr) == n:
        idl('cerr = %s' %list(cerr))
        cmd += ', xycov=cerr'
    cmd += ', miniter=%d, maxiter=%d' %(miniter, maxiter)
    if metro:
        cmd += ', /metro'
    if silent:
        cmd += ', /silent'
    idl(cmd)
    alpha = idl.ev('fit.alpha')
    beta = idl.ev('fit.beta')
    sigma = numpy.sqrt(idl.ev('fit.sigsqr'))
    return alpha, beta, sigma

Response: Returns a new Kelly model with linear regression.


Instruction:
Can you create a Python 3 function that
uses emcee to find the best-fit linear relation or power law accounting for measurement uncertainties and intrinsic scatter Parameters ---------- x1 : array of floats Independent variable, or observable x2 : array of floats Dependent variable x1err : array of floats (optional) Uncertainties on the independent variable x2err : array of floats (optional) Uncertainties on the dependent variable po : tuple of 3 floats (optional) Initial guesses for zero point, slope, and intrinsic scatter. Results are not very sensitive to these values so they shouldn't matter a lot. logify : bool (default True) Whether to take the log of the measurements in order to estimate the best-fit power law instead of linear relation nsteps : int (default 5000) Number of steps each walker should take in the MCMC nwalkers : int (default 100) Number of MCMC walkers nburn : int (default 500) Number of samples to discard to give the MCMC enough time to converge. output : list of ints or 'full' (default 'full') If 'full', then return the full samples (except for burn-in section) for each parameter. Otherwise, each float corresponds to a percentile that will be returned for each parameter. Returns ------- See *output* argument above for return options.

Response:
def mcmc(x1, x2, x1err=[], x2err=[], po=(1,1,0.5), logify=True,
         nsteps=5000, nwalkers=100, nburn=500, output='full'):
    """
    Use emcee to find the best-fit linear relation or power law
    accounting for measurement uncertainties and intrinsic scatter

    Parameters
    ----------
      x1        : array of floats
                  Independent variable, or observable
      x2        : array of floats
                  Dependent variable
      x1err     : array of floats (optional)
                  Uncertainties on the independent variable
      x2err     : array of floats (optional)
                  Uncertainties on the dependent variable
      po        : tuple of 3 floats (optional)
                  Initial guesses for zero point, slope, and intrinsic
                  scatter. Results are not very sensitive to these values
                  so they shouldn't matter a lot.
      logify    : bool (default True)
                  Whether to take the log of the measurements in order to
                  estimate the best-fit power law instead of linear relation
      nsteps    : int (default 5000)
                  Number of steps each walker should take in the MCMC
      nwalkers  : int (default 100)
                  Number of MCMC walkers
      nburn     : int (default 500)
                  Number of samples to discard to give the MCMC enough time
                  to converge.
      output    : list of ints or 'full' (default 'full')
                  If 'full', then return the full samples (except for burn-in
                  section) for each parameter. Otherwise, each float
                  corresponds to a percentile that will be returned for
                  each parameter.

    Returns
    -------
      See *output* argument above for return options.

    """
    import emcee
    if len(x1err) == 0:
        x1err = numpy.ones(len(x1))
    if len(x2err) == 0:
        x2err = numpy.ones(len(x1))
    def lnlike(theta, x, y, xerr, yerr):
        a, b, s = theta
        model = a + b*x
        sigma = numpy.sqrt((b*xerr)**2 + yerr*2 + s**2)
        lglk = 2 * sum(numpy.log(sigma)) + \
               sum(((y-model) / sigma) ** 2) + \
               numpy.log(len(x)) * numpy.sqrt(2*numpy.pi) / 2
        return -lglk
    def lnprior(theta):
        a, b, s = theta
        if s >= 0:
            return 0
        return -numpy.inf
    def lnprob(theta, x, y, xerr, yerr):
        lp = lnprior(theta)
        return lp + lnlike(theta, x, y, xerr, yerr)
    if logify:
        x1, x2, x1err, x2err = to_log(x1, x2, x1err, x2err)
    start = numpy.array(po)
    ndim = len(start)
    pos = [start + 1e-4*numpy.random.randn(ndim) for i in range(nwalkers)]
    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob,
                                    args=(x1,x2,x1err,x2err))
    sampler.run_mcmc(pos, nsteps)
    samples = numpy.array([sampler.chain[:,nburn:,i].reshape(-1) \
                           for i in xrange(ndim)])
    if logify:
        samples[2] *= numpy.log(10)
    if output == 'full':
        return samples
    else:
        try:
            values = [[numpy.percentile(s, o) for o in output]
                      for s in samples]
            return values
        except TypeError:
            msg = 'ERROR: wrong value for argument output in mcmc().'
            msg += ' Must be "full" or list of ints.'
            print msg
            exit()
    return


Instruction:
Given the following Python 3 function, write the documentation
def mle(x1, x2, x1err=[], x2err=[], cerr=[], s_int=True,
        po=(1,0,0.1), verbose=False, logify=True, full_output=False):
    """
    Maximum Likelihood Estimation of best-fit parameters

    Parameters
    ----------
      x1, x2    : float arrays
                  the independent and dependent variables.
      x1err, x2err : float arrays (optional)
                  measurement uncertainties on independent and dependent
                  variables. Any of the two, or both, can be supplied.
      cerr      : float array (same size as x1)
                  covariance on the measurement errors
      s_int     : boolean (default True)
                  whether to include intrinsic scatter in the MLE.
      po        : tuple of floats
                  initial guess for free parameters. If s_int is True, then
                  po must have 3 elements; otherwise it can have two (for the
                  zero point and the slope)
      verbose   : boolean (default False)
                  verbose?
      logify    : boolean (default True)
                  whether to convert the values to log10's. This is to
                  calculate the best-fit power law. Note that the result is
                  given for the equation log(y)=a+b*log(x) -- i.e., the
                  zero point must be converted to 10**a if logify=True
      full_output : boolean (default False)
                  numpy.optimize.fmin's full_output argument

    Returns
    -------
      a         : float
                  Maximum Likelihood Estimate of the zero point. Note that
                  if logify=True, the power-law intercept is 10**a
      b         : float
                  Maximum Likelihood Estimate of the slope
      s         : float (optional, if s_int=True)
                  Maximum Likelihood Estimate of the intrinsic scatter

    """
    from scipy import optimize
    n = len(x1)
    if len(x2) != n:
        raise ValueError('x1 and x2 must have same length')
    if len(x1err) == 0:
        x1err = numpy.ones(n)
    if len(x2err) == 0:
        x2err = numpy.ones(n)
    if logify:
        x1, x2, x1err, x2err = to_log(x1, x2, x1err, x2err)

    f = lambda a, b: a + b * x1
    if s_int:
        w = lambda b, s: numpy.sqrt(b**2 * x1err**2 + x2err**2 + s**2)
        loglike = lambda p: 2 * sum(numpy.log(w(p[1],p[2]))) + \
                            sum(((x2 - f(p[0],p[1])) / w(p[1],p[2])) ** 2) + \
                            numpy.log(n * numpy.sqrt(2*numpy.pi)) / 2
    else:
        w = lambda b: numpy.sqrt(b**2 * x1err**2 + x2err**2)
        loglike = lambda p: sum(numpy.log(w(p[1]))) + \
                            sum(((x2 - f(p[0],p[1])) / w(p[1])) ** 2) / 2 + \
                            numpy.log(n * numpy.sqrt(2*numpy.pi)) / 2
        po = po[:2]
    out = optimize.fmin(loglike, po, disp=verbose, full_output=full_output)
    return out

Response: This function calculates the MLE of two sets of measurement uncertainties and returns the MLE of the two sets of free parameters.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def to_log(x1, x2, x1err, x2err):
    logx1 = numpy.log10(numpy.array(x1))
    logx2 = numpy.log10(numpy.array(x2))
    x1err = numpy.log10(numpy.array(x1)+numpy.array(x1err)) - logx1
    x2err = numpy.log10(numpy.array(x2)+numpy.array(x2err)) - logx2
    return logx1, logx2, x1err, x2err

Response: Transform two linear measurements and uncertainties and transform to log values.


Instruction:
Can you implement a function in Python 3 that
wraps all paths in quotes and join them with space in - between.

Response:
def wrap_paths(paths):
    # type: (list[str]) -> str
    """ Put quotes around all paths and join them with space in-between. """
    if isinstance(paths, string_types):
        raise ValueError(
            "paths cannot be a string. "
            "Use array with one element instead."
        )
    return ' '.join('"' + path + '"' for path in paths)


Instruction:
How would you implement a function in Python 3 that
walks a directory and returns a generator yielding all the files that match any of the patterns in include and exclude.

Response:
def filtered_walk(path, include=None, exclude=None):
    # type: (str, List[str], List[str]) -> Generator[str]
    """ Walk recursively starting at *path* excluding files matching *exclude*

    Args:
        path (str):
            A starting path. This has to be an existing directory.
        include (list[str]):
            A white list of glob patterns. If given, only files that match those
            globs will be yielded (filtered by exclude).
        exclude (list[str]):
            A list of glob string patterns to test against. If the file/path
            matches any of those patters, it will be filtered out.

    Returns:
        Generator[str]: A generator yielding all the files that do not match any
        pattern in ``exclude``.
    """
    exclude = exclude or []

    if not isdir(path):
        raise ValueError("Cannot walk files, only directories")

    files = os.listdir(path)
    for name in files:
        filename = normpath(join(path, name))

        # If excluded, completely skip it. Will not recurse into directories
        if search_globs(filename, exclude):
            continue

        # If we have a whitelist and the pattern matches, yield it. If the
        # pattern didn't match and it's a dir, it will still be recursively
        # processed.
        if include is None or match_globs(filename, include):
            yield filename

        if isdir(filename):
            for p in filtered_walk(filename, include, exclude):
                yield p


Instruction:
Implement a function in Python 3 to
test whether the given path matches any of the given glob strings.

Response:
def match_globs(path, patterns):
    # type: (str, List[str]) -> bool
    """ Test whether the given *path* matches any patterns in *patterns*

    Args:
        path (str):
            A file path to test for matches.
        patterns (list[str]):
            A list of glob string patterns to test against. If *path* matches
            any of those patters, it will return True.

    Returns:
        bool: **True** if the *path* matches any pattern in *patterns*.
    """
    for pattern in (p for p in patterns if p):
        if pattern.startswith('/'):
            regex = fnmatch.translate(pattern[1:])

            temp_path = path[1:] if path.startswith('/') else path

            m = re.search(regex, temp_path)

            if m and m.start() == 0:
                return True

        elif fnmatch.fnmatch(path, pattern):
                return True

    return False


Instruction:
Make a summary of the following Python 3 code
def search_globs(path, patterns):
    # type: (str, List[str]) -> bool
    """ Test whether the given *path* contains any patterns in *patterns*

    Args:
        path (str):
            A file path to test for matches.
        patterns (list[str]):
            A list of glob string patterns to test against. If *path* matches
            any of those patters, it will return True.

    Returns:
        bool: **True** if the ``path`` matches any pattern in *patterns*.
    """
    for pattern in (p for p in patterns if p):
        if pattern.startswith('/'):
            # If pattern starts with root it means it match from root only
            regex = fnmatch.translate(pattern[1:])
            regex = regex.replace('\\Z', '')

            temp_path = path[1:] if path.startswith('/') else path
            m = re.search(regex, temp_path)

            if m and m.start() == 0:
                return True

        else:
            regex = fnmatch.translate(pattern)
            regex = regex.replace('\\Z', '')

            if re.search(regex, path):
                return True

    return False

Response: Test whether the given path contains any of the given glob strings.


Instruction:
Can you tell what is the following Python 3 function doing
def write_file(path, content, mode='w'):
    # type: (Text, Union[Text,bytes], Text) -> None
    """ --pretend aware file writing.

    You can always write files manually but you should always handle the
    --pretend case.

    Args:
        path (str):
        content (str):
        mode (str):
    """
    from peltak.core import context
    from peltak.core import log

    if context.get('pretend', False):
        log.info("Would overwrite <34>{path}<32> with:\n<90>{content}",
                 path=path,
                 content=content)
    else:
        with open(path, mode) as fp:
            fp.write(content)

Response: Writes the content of the current node to the file at the given path.


Instruction:
Implement a Python 3 function for
running pep8 and pylint on all project files. You can configure the linting paths using the lint.paths config variable. This should be a list of paths that will be linted. If a path to a directory is given, all files in that directory and it's subdirectories will be used. The pep8 and pylint config paths are by default stored in ops/tools/pep8.ini and ops/tools/pylint.ini. You can customise those paths in your config with lint.pep8_cfg and lint.pylint_cfg variables. **Config Example**:: \b lint: pylint_cfg: 'ops/tools/pylint.ini' pep8_cfg: 'ops/tools/pep8.ini' paths: - 'src/mypkg' **Examples**:: \b $ peltak lint # Run linter in default mode, skip untracked $ peltak lint --commit # Lint only files staged for commit $ peltak lint --all # Lint all files, including untracked. $ peltak lint --pretend # Print the list of files to lint $ peltak lint -e "*.tox*" # Don't lint files inside .tox directory

Response:
def lint_cli(ctx, exclude, skip_untracked, commit_only):
    # type: (click.Context, List[str], bool, bool) -> None
    """ Run pep8 and pylint on all project files.

    You can configure the linting paths using the lint.paths config variable.
    This should be a list of paths that will be linted. If a path to a directory
    is given, all files in that directory and it's subdirectories will be
    used.

    The pep8 and pylint config paths are by default stored in ops/tools/pep8.ini
    and ops/tools/pylint.ini. You can customise those paths in your config with
    lint.pep8_cfg and lint.pylint_cfg variables.

    **Config Example**::

        \b
        lint:
          pylint_cfg: 'ops/tools/pylint.ini'
          pep8_cfg: 'ops/tools/pep8.ini'
          paths:
            - 'src/mypkg'

    **Examples**::

        \b
        $ peltak lint               # Run linter in default mode, skip untracked
        $ peltak lint --commit      # Lint only files staged for commit
        $ peltak lint --all         # Lint all files, including untracked.
        $ peltak lint --pretend     # Print the list of files to lint
        $ peltak lint -e "*.tox*"   # Don't lint files inside .tox directory

    """
    if ctx.invoked_subcommand:
        return

    from peltak.logic import lint
    lint.lint(exclude, skip_untracked, commit_only)


Instruction:
Write a Python 3 function for
running a command using subprocess module .

Response:
def run_command(command, timeout_sec=3600.0, output=True):
    """Runs a command using the subprocess module

    :param command: List containing the command and all args
    :param timeout_sec (float) seconds to wait before killing
        the command.
    :param output (bool) True collects output, False ignores output
    :return: Dict containing the command output and return code
    :raises CommandError
    """
    log = logging.getLogger(mod_logger + '.run_command')
    if not isinstance(command, list):
        msg = 'command arg must be a list'
        log.error(msg)
        raise CommandError(msg)
    if output:
        subproc_stdout = subprocess.PIPE
        subproc_stderr = subprocess.STDOUT
    else:
        subproc_stdout = None
        subproc_stderr = None
    command = map(str, command)
    command_str = ' '.join(command)
    timer = None
    log.debug('Running command: {c}'.format(c=command_str))
    output_collector = ''
    try:
        log.debug('Opening subprocess...')
        subproc = subprocess.Popen(
            command,
            bufsize=1,
            stdin=open(os.devnull),
            stdout=subproc_stdout,
            stderr=subproc_stderr
        )
        log.debug('Opened subprocess wih PID: {p}'.format(p=subproc.pid))
        log.debug('Setting up process kill timer for PID {p} at {s} sec...'.format(p=subproc.pid, s=timeout_sec))
        kill_proc = process_killer
        timer = Timer(timeout_sec, kill_proc, [subproc])
        timer.start()
        if output:
            log.debug('Collecting and logging output...')
            with subproc.stdout:
                for line in iter(subproc.stdout.readline, b''):
                    output_collector += line.rstrip() + '\n'
                    print(">>> " + line.rstrip())
        log.debug('Waiting for process completion...')
        subproc.wait()
        log.debug('Collecting the exit code...')
        code = subproc.poll()
    except ValueError:
        _, ex, trace = sys.exc_info()
        msg = 'Bad command supplied: {c}\n{e}'.format(
            c=command_str, e=str(ex)
        )
        log.error(msg)
        raise CommandError, msg, trace
    except (OSError, IOError):
        _, ex, trace = sys.exc_info()
        msg = 'There was a problem running command: {c}\n{e}'.format(
            c=command_str, e=str(ex))
        log.error(msg)
        raise CommandError, msg, trace
    except subprocess.CalledProcessError:
        _, ex, trace = sys.exc_info()
        msg = 'Command returned a non-zero exit code: {c}, return code: {cde}\n{e}'.format(
            c=command_str, cde=ex.returncode, e=ex)
        log.error(msg)
        raise CommandError, msg, trace
    finally:
        if timer is not None:
            log.debug('Cancelling the timer...')
            timer.cancel()
        else:
            log.debug('No need to cancel the timer.')
    # Collect exit code and output for return
    output = output_collector.strip()
    try:
        code = int(code)
    except ValueError:
        _, ex, trace = sys.exc_info()
        msg = 'Return code {c} could not be parsed into an int\n{e}'.format(
            c=code, e=str(ex))
        log.error(msg)
        raise CommandError, msg, trace
    else:
        log.debug('Command executed and returned code: {c} with output:\n{o}'.format(c=code, o=output))
        output = {
            'output': output,
            'code': code
        }
    return output


Instruction:
Create a Python 3 function for
getting the ip addresses from ifconfig

Response:
def get_ip_addresses():
    """Gets the ip addresses from ifconfig

    :return: (dict) of devices and aliases with the IPv4 address
    """
    log = logging.getLogger(mod_logger + '.get_ip_addresses')

    command = ['/sbin/ifconfig']
    try:
        result = run_command(command)
    except CommandError:
        raise
    ifconfig = result['output'].strip()

    # Scan the ifconfig output for IPv4 addresses
    devices = {}
    parts = ifconfig.split()
    device = None
    for part in parts:

        if device is None:
            if 'eth' in part or 'eno' in part:
                device = part
        else:
            test = part.split(':', 1)
            if len(test) == 2:
                if test[0] == 'addr':
                    ip_address = test[1]
                    log.info('Found IP address %s on device %s', ip_address,
                             device)
                    devices[device] = ip_address
                    device = None
    return devices


Instruction:
How would you explain what the following Python 3 function does
def get_mac_address(device_index=0):
    log = logging.getLogger(mod_logger + '.get_mac_address')
    command = ['ip', 'addr', 'show', 'eth{d}'.format(d=device_index)]
    log.info('Attempting to find a mac address at device index: {d}'.format(d=device_index))
    try:
        result = run_command(command)
    except CommandError:
        _, ex, trace = sys.exc_info()
        log.error('There was a problem running command, unable to determine mac address: {c}\n{e}'.format(
                c=command, e=str(ex)))
        return
    ipaddr = result['output'].split()
    get_next = False
    mac_address = None
    for part in ipaddr:
        if get_next:
            mac_address = part
            log.info('Found mac address: {m}'.format(m=mac_address))
            break
        if 'link' in part:
            get_next = True
    if not mac_address:
        log.info('mac address not found for device: {d}'.format(d=device_index))
    return mac_address

Response: Returns the Mac Address given a device index


Instruction:
Write a Python 3 script to
emulate bash chmod command

Response:
def chmod(path, mode, recursive=False):
    """Emulates bash chmod command

    This method sets the file permissions to the specified mode.

    :param path: (str) Full path to the file or directory
    :param mode: (str) Mode to be set (e.g. 0755)
    :param recursive: (bool) Set True to make a recursive call
    :return: int exit code of the chmod command
    :raises CommandError
    """
    log = logging.getLogger(mod_logger + '.chmod')

    # Validate args
    if not isinstance(path, basestring):
        msg = 'path argument is not a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(mode, basestring):
        msg = 'mode argument is not a string'
        log.error(msg)
        raise CommandError(msg)

    # Ensure the item exists
    if not os.path.exists(path):
        msg = 'Item not found: {p}'.format(p=path)
        log.error(msg)
        raise CommandError(msg)

    # Create the chmod command
    command = ['chmod']
    # Make it recursive if specified
    if recursive:
        command.append('-R')
    command.append(mode)
    command.append(path)
    try:
        result = run_command(command)
    except CommandError:
        raise
    log.info('chmod command exited with code: {c}'.format(c=result['code']))
    return result['code']


Instruction:
Create a Python 3 function to
emulate mkdir - p in bash

Response:
def mkdir_p(path):
    """Emulates 'mkdir -p' in bash

    :param path: (str) Path to create
    :return: None
    :raises CommandError
    """
    log = logging.getLogger(mod_logger + '.mkdir_p')
    if not isinstance(path, basestring):
        msg = 'path argument is not a string'
        log.error(msg)
        raise CommandError(msg)
    log.info('Attempting to create directory: %s', path)
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            msg = 'Unable to create directory: {p}'.format(p=path)
            log.error(msg)
            raise CommandError(msg)


Instruction:
How would you explain what the following Python 3 function does
def source(script):
    log = logging.getLogger(mod_logger + '.source')
    if not isinstance(script, basestring):
        msg = 'script argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    log.info('Attempting to source script: %s', script)
    try:
        pipe = subprocess.Popen(". %s; env" % script, stdout=subprocess.PIPE, shell=True)
        data = pipe.communicate()[0]
    except ValueError:
        _, ex, trace = sys.exc_info()
        msg = 'Invalid argument:\n{e}'.format(e=str(ex))
        log.error(msg)
        raise CommandError, msg, trace
    except OSError:
        _, ex, trace = sys.exc_info()
        msg = 'File not found: {s}\n{e}'.format(s=script, e=str(ex))
        raise CommandError, msg, trace
    except subprocess.CalledProcessError:
        _, ex, trace = sys.exc_info()
        msg = 'Script {s} returned a non-zero exit code: {c}\n{e}'.format(
            s=script, e=str(ex), c=ex.returncode)
        log.error(msg)
        raise CommandError, msg, trace
    env = {}
    log.debug('Adding environment variables from data: {d}'.format(d=data))
    for line in data.splitlines():
        entry = line.split("=", 1)
        if len(entry) != 2:
            log.warn('This property is not in prop=value format, and will be skipped: {p}'.format(p=line))
            continue
        try:
            env[entry[0]] = entry[1]
        except IndexError:
            _, ex, trace = sys.exc_info()
            log.warn('IndexError: There was a problem setting environment variables from line: {p}\n{e}'.format(
                p=line, e=str(ex)))
            continue
        else:
            log.debug('Added environment variable {p}={v}'.format(p=entry[0], v=entry[1]))
    os.environ.update(env)
    return env

Response: Emulates source command in bash


Instruction:
Explain what the following Python 3 code does
def yum_update(downloadonly=False, dest_dir='/tmp'):
    log = logging.getLogger(mod_logger + '.yum_update')

    # Type checks on the args
    if not isinstance(dest_dir, basestring):
        msg = 'dest_dir argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(downloadonly, bool):
        msg = 'downloadonly argument must be a bool'
        log.error(msg)
        raise CommandError(msg)

    # If downloadonly was True, download packages to dest_dir
    if downloadonly:
        # Create the destination directory if it does not exist
        log.info('Creating directory: %s', dest_dir)
        try:
            mkdir_p(dest_dir)
        except OSError:
            _, ex, trace = sys.exc_info()
            msg = 'Unable to create destination directory: {d}'.format(
                d=dest_dir)
            log.error(msg)
            raise CommandError, msg, trace

        # Build command string with downloadonly options specified
        command = ['yum', '-y', 'update', '--downloadonly',
                   '--downloaddir={d}'.format(d=dest_dir)]
        log.info('Downloading updates from yum to %s...', dest_dir)
    else:
        # Build command string to update directly
        command = ['yum', '-y', 'update']
        log.info('Installing yum updates from RHN...')

    # Run the command
    try:
        result = run_command(command)
    except CommandError:
        raise
    log.info('Yum update completed and exit with code: {c}'.format(
        c=result['code']))
    return result['code']

Response: This method runs the yum update command to update the packages in the system.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def yum_install(packages, downloadonly=False, dest_dir='/tmp'):
    log = logging.getLogger(mod_logger + '.yum_install')

    # Type checks on the args
    if not isinstance(dest_dir, basestring):
        msg = 'dest_dir argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(packages, list):
        msg = 'packages argument must be a list'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(downloadonly, bool):
        msg = 'downloadonly argument must be a bool'
        log.error(msg)
        raise CommandError(msg)
    if not packages:
        msg = 'Empty list of packages provided'
        log.error(msg)
        raise CommandError(msg)
    for package in packages:
        # Ensure the package is specified as a string
        if not isinstance(package, basestring):
            msg = 'One of the packages was not specified as a string'
            log.error(msg)
            raise CommandError(msg)

    # Build the yum install command string
    command = ['yum', '-y', 'install'] + packages

    # If downloadonly was True, download packages to dest_dir
    if downloadonly:
        log.info('yum downloadonly was specified, adding additional options...')

        # Append downloadonly args to the command
        command += ['--downloadonly', '--downloaddir={d}'.format(d=dest_dir)]

        # Create the destination directory if it does not exist
        log.info('Creating directory: %s', dest_dir)
        try:
            mkdir_p(dest_dir)
        except CommandError:
            _, ex, trace = sys.exc_info()
            msg = 'Unable to create destination directory: {d}'.format(d=dest_dir)
            log.error(msg)
            raise CommandError, msg, trace
        log.info('Downloading packages from yum to %s...', dest_dir)
    else:
        log.info('Installing yum packages from RHN...')

    # Run the yum install command
    try:
        result = run_command(command)
    except CommandError:
        raise
    log.info('Yum update completed and exit with code: {c}'.format(
        c=result['code']))
    return result['code']

Response: This function installs a list of packages from yum or downloads the packages to dest_dir using the yum - y plugin.


Instruction:
Make a summary of the following Python 3 code
def sed(file_path, pattern, replace_str, g=0):
    log = logging.getLogger(mod_logger + '.sed')

    # Type checks on the args
    if not isinstance(file_path, basestring):
        msg = 'file_path argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(pattern, basestring):
        msg = 'pattern argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(replace_str, basestring):
        msg = 'replace_str argument must be a string'
        log.error(msg)
        raise CommandError(msg)

    # Ensure the file_path file exists
    if not os.path.isfile(file_path):
        msg = 'File not found: {f}'.format(f=file_path)
        log.error(msg)
        raise CommandError(msg)

    # Search for a matching pattern and replace matching patterns
    log.info('Updating file: %s...', file_path)
    for line in fileinput.input(file_path, inplace=True):
        if re.search(pattern, line):
            log.info('Updating line: %s', line)
            new_line = re.sub(pattern, replace_str, line, count=g)
            log.info('Replacing with line: %s', new_line)
            sys.stdout.write(new_line)
        else:
            sys.stdout.write(line)

Response: This function emulates the functionality of the bash sed command.


Instruction:
Can you tell what is the following Python 3 function doing
def zip_dir(dir_path, zip_file):
    log = logging.getLogger(mod_logger + '.zip_dir')

    # Validate args
    if not isinstance(dir_path, basestring):
        msg = 'dir_path argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(zip_file, basestring):
        msg = 'zip_file argument must be a string'
        log.error(msg)
        raise CommandError(msg)

    # Ensure the dir_path file exists
    if not os.path.isdir(dir_path):
        msg = 'Directory not found: {f}'.format(f=dir_path)
        log.error(msg)
        raise CommandError(msg)

    try:
        with contextlib.closing(zipfile.ZipFile(zip_file, 'w', allowZip64=True)) as zip_w:
            for root, dirs, files in os.walk(dir_path):
                for f in files:
                    log.debug('Adding file to zip: %s', f)
                    strip = len(dir_path) - len(os.path.split(dir_path)[-1])
                    file_name = os.path.join(root, f)
                    archive_name = os.path.join(root[strip:], f)
                    zip_w.write(file_name, archive_name)
    except Exception:
        _, ex, trace = sys.exc_info()
        msg = 'Unable to create zip file: {f}\n{e}'.format(
            f=zip_file, e=str(ex))
        log.error(msg)
        raise CommandError, msg, trace
    log.info('Successfully created zip file: %s', zip_file)

Response: Creates a zip file of a directory tree


Instruction:
Write a Python 3 function for
updating the /etc/hosts file for the specified ip This method updates the /etc/hosts file for the specified IP address with the specified entry. :param ip: (str) IP address to be added or updated :param entry: (str) Hosts file entry to be added :return: None :raises CommandError

Response:
def update_hosts_file(ip, entry):
    """Updates the /etc/hosts file for the specified ip

    This method updates the /etc/hosts file for the specified IP
    address with the specified entry.

    :param ip: (str) IP address to be added or updated
    :param entry: (str) Hosts file entry to be added
    :return: None
    :raises CommandError
    """
    log = logging.getLogger(mod_logger + '.update_hosts_file')

    # Validate args
    if not isinstance(ip, basestring):
        msg = 'ip argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    if not isinstance(entry, basestring):
        msg = 'entry argument must be a string'
        log.error(msg)
        raise CommandError(msg)

    # Ensure the file_path file exists
    hosts_file = '/etc/hosts'
    if not os.path.isfile(hosts_file):
        msg = 'File not found: {f}'.format(f=hosts_file)
        log.error(msg)
        raise CommandError(msg)

    # Updating /etc/hosts file
    log.info('Updating hosts file: {f} with IP {i} and entry: {e}'.format(f=hosts_file, i=ip, e=entry))
    full_entry = ip + ' ' + entry.strip() + '\n'
    updated = False
    for line in fileinput.input(hosts_file, inplace=True):
        if re.search(ip, line):
            if line.split()[0] == ip:
                log.info('Found IP {i} in line: {li}, updating...'.format(i=ip, li=line))
                log.info('Replacing with new line: {n}'.format(n=full_entry))
                sys.stdout.write(full_entry)
                updated = True
            else:
                log.debug('Found ip {i} in line {li} but not an exact match, adding line back to hosts file {f}...'.
                          format(i=ip, li=line, f=hosts_file))
                sys.stdout.write(line)
        else:
            log.debug('IP address {i} not found in line, adding line back to hosts file {f}: {li}'.format(
                    i=ip, li=line, f=hosts_file))
            sys.stdout.write(line)

    # Append the entry if the hosts file was not updated
    if updated is False:
        with open(hosts_file, 'a') as f:
            log.info('Appending hosts file entry to {f}: {e}'.format(f=hosts_file, e=full_entry))
            f.write(full_entry)


Instruction:
How would you code a function in Python 3 to
set this hosts hostname This method updates /etc/sysconfig/network and calls the hostname command to set a hostname on a Linux system. :param new_hostname: (str) New hostname :param pretty_hostname: (str) new pretty hostname, set to the same as new_hostname if not provided :return (int) exit code of the hostname command :raises CommandError

Response:
def set_hostname(new_hostname, pretty_hostname=None):
    """Sets this hosts hostname

    This method updates /etc/sysconfig/network and calls the hostname
    command to set a hostname on a Linux system.

    :param new_hostname: (str) New hostname
    :param pretty_hostname: (str) new pretty hostname, set to the same as
        new_hostname if not provided
    :return (int) exit code of the hostname command
    :raises CommandError
    """
    log = logging.getLogger(mod_logger + '.set_hostname')

    # Ensure the hostname is a str
    if not isinstance(new_hostname, basestring):
        msg = 'new_hostname argument must be a string'
        raise CommandError(msg)

    # Update the network config file
    network_file = '/etc/sysconfig/network'
    if os.path.isfile(network_file):
        log.info('Updating {f} with the new hostname: {h}...'.format(f=network_file, h=new_hostname))
        try:
            sed(network_file, '^HOSTNAME=.*', 'HOSTNAME=' + new_hostname)
        except CommandError:
            _, ex, trace = sys.exc_info()
            msg = 'Unable to update [{f}], produced output:\n{e}'.format(f=network_file, e=str(ex))
            raise CommandError, msg, trace
    else:
        log.info('Network file not found, will not be updated: {f}'.format(f=network_file))

    # Update the hostname
    if is_systemd():
        hostname_file = '/etc/hostname'
        pretty_hostname_file = '/etc/machine-info'
        log.info('This is systemd, updating files: {h} and {p}'.format(h=hostname_file, p=pretty_hostname_file))

        # Update the hostname file
        log.info('Updating hostname file: {h}...'.format(h=hostname_file))
        if os.path.isfile(hostname_file):
            os.remove(hostname_file)
        with open(hostname_file, 'w') as f:
            f.write(new_hostname)
        log.info('Updating pretty hostname file: {p}'.format(p=pretty_hostname_file))

        # Use the same thing if pretty hostname is not provided
        if pretty_hostname is None:
            log.info('Pretty hostname not provided, using: {p}'.format(p=pretty_hostname))
            pretty_hostname = new_hostname

        # Update the pretty hostname file
        if os.path.isfile(pretty_hostname_file):
            os.remove(pretty_hostname_file)
        with open(pretty_hostname_file, 'w') as f:
            f.write('PRETTY_HOSTNAME={p}'.format(p=pretty_hostname))
        return 0
    else:
        command = ['/bin/hostname', new_hostname]

        # Run the hostname command
        log.info('Running hostname command to set the hostname: [{c}]'.format(c=' '.join(command)))
        try:
            result = run_command(command)
        except CommandError:
            raise
        log.info('Hostname command completed with code: {c} and output:\n{o}'.format(
            c=result['code'], o=result['output']))
        return result['code']


Instruction:
Can you generate the documentation for the following Python 3 function
def set_ntp_server(server):
    log = logging.getLogger(mod_logger + '.set_ntp_server')

    # Ensure the hostname is a str
    if not isinstance(server, basestring):
        msg = 'server argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    # Ensure the ntp.conf file exists
    ntp_conf = '/etc/ntp.conf'
    if not os.path.isfile(ntp_conf):
        msg = 'File not found: {f}'.format(f=ntp_conf)
        log.error(msg)
        raise CommandError(msg)
    log.info('Clearing out existing server entries from %s...', ntp_conf)
    try:
        sed(ntp_conf, '^server.*', '', g=0)
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'Unable to update file: {f}\n{e}'.format(f=ntp_conf, e=str(ex))
        log.error(msg)
        raise CommandError, msg, trace
    out_str = 'server ' + server
    log.info('Appending server: %s', out_str)
    with open(ntp_conf, 'a') as f:
        f.write(out_str)
    log.info('Successfully updated file: {f}'.format(f=ntp_conf))

Response: Sets the NTP server on Linux and returns None


Instruction:
Explain what the following Python 3 code does
def copy_ifcfg_file(source_interface, dest_interface):
    log = logging.getLogger(mod_logger + '.copy_ifcfg_file')
    # Validate args
    if not isinstance(source_interface, basestring):
        msg = 'source_interface argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(dest_interface, basestring):
        msg = 'dest_interface argument must be a string'
        log.error(msg)
        raise TypeError(msg)

    network_script = '/etc/sysconfig/network-scripts/ifcfg-eth'
    source_file = network_script + source_interface
    dest_file = network_script + dest_interface
    command = ['cp', '-f', source_file, dest_file]
    try:
        result = run_command(command)
        code = result['code']
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'Unable to copy the ifcfg file from interface {s} to interface {d}\n{e}'.format(
            s=source_interface, d=dest_interface, e=str(ex))
        raise OSError, msg, trace
    log.info('Copy command exited with code: {c}'.format(c=code))

    if code != 0:
        msg = 'There was a problem copying file {s} file to {d}'.format(s=source, d=dest_file)
        log.error(msg)
        raise OSError(msg)

    # Updating the destination network script DEVICE property
    try:
        sed(file_path=dest_file, pattern='^DEVICE=.*',
            replace_str='DEVICE="eth{i}"'.format(i=dest_interface))
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'Unable to update DEVICE in file: {d}\n{e}'.format(
            d=dest_file, e=str(ex))
        log.error(msg)
        raise CommandError, msg, trace
    log.info('Successfully created file: {d}'.format(d=dest_file))

    log.info('Restarting networking in 10 seconds to ensure the changes take effect...')
    time.sleep(10)
    retry_time = 10
    max_retries = 10
    for i in range(1, max_retries+2):
        if i > max_retries:
            msg = 'Unable to successfully start the networking service after {m} attempts'.format(m=max_retries)
            log.error(msg)
            raise OSError(msg)
        log.info('Attempting to restart the networking service, attempt #{i} of {m}'.format(i=i, m=max_retries))
        try:
            service_network_restart()
        except CommandError:
            _, ex, trace = sys.exc_info()
            log.warn('Attempted unsuccessfully to restart networking on attempt #{i} of {m}, trying again in {t} '
                     'seconds\n{e}'.format(i=i, m=max_retries, t=retry_time, e=str(ex)))
            time.sleep(retry_time)
        else:
            log.info('Successfully restarted networking')
            break
    log.info('Successfully configured interface: {d}'.format(d=dest_interface))

Response: Copies an existing ifcfg network script to another ifcfg network script


Instruction:
Given the following Python 3 function, write the documentation
def remove_ifcfg_file(device_index='0'):
    log = logging.getLogger(mod_logger + '.remove_ifcfg_file')
    if not isinstance(device_index, basestring):
        msg = 'device_index argument must be a string'
        log.error(msg)
        raise CommandError(msg)
    network_script = '/etc/sysconfig/network-scripts/ifcfg-eth{d}'.format(d=device_index)
    if not os.path.isfile(network_script):
        log.info('File does not exist, nothing will be removed: {n}'.format(n=network_script))
        return

    # Remove the network config script
    log.info('Attempting to remove file: {n}'.format(n=network_script))
    try:
        os.remove(network_script)
    except(IOError, OSError):
        _, ex, trace = sys.exc_info()
        msg = 'There was a problem removing network script file: {n}\n{e}'.format(n=network_script, e=str(ex))
        log.error(msg)
        raise OSError, msg, trace
    else:
        log.info('Successfully removed file: {n}'.format(n=network_script))

    # Restart the network service
    log.info('Restarting the network service...')
    try:
        service_network_restart()
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'There was a problem restarting the network service\n{e}'.format(e=str(ex))
        log.error(msg)
        raise OSError, msg, trace
    else:
        log.info('Successfully restarted the network service')

Response: Removes the ifcfg file at the specified device index and restarts the network service if it is still running.


Instruction:
How would you explain what the following Python 3 function does
def add_nat_rule(port, source_interface, dest_interface):
    log = logging.getLogger(mod_logger + '.add_nat_rule')
    # Validate args
    if not isinstance(source_interface, basestring):
        msg = 'source_interface argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(dest_interface, basestring):
        msg = 'dest_interface argument must be a string'
        log.error(msg)
        raise TypeError(msg)

    ip_addresses = ip_addr()
    destination_ip = ip_addresses['eth{i}'.format(i=dest_interface)]
    log.info('Using destination IP address: {d}'.format(d=destination_ip))

    command = ['iptables', '-t', 'nat', '-A', 'PREROUTING', '-i',
               'eth{s}'.format(s=source_interface), '-p', 'tcp',
               '--dport', str(port), '-j', 'DNAT', '--to',
               '{d}:{p}'.format(p=port, d=destination_ip)]
    log.info('Running command: {c}'.format(c=command))
    try:
        subprocess.check_call(command)
    except OSError:
        _, ex, trace = sys.exc_info()
        msg = 'There was a problem running command: {c}\n{e}'.format(c=command, e=str(ex))
        log.error(msg)
        raise OSError, msg, trace
    except subprocess.CalledProcessError:
        _, ex, trace = sys.exc_info()
        msg = 'Command returned a non-zero exit code: {c}\n{e}'.format(c=command, e=str(ex))
        log.error(msg)
        raise OSError, msg, trace
    else:
        log.info('Successfully ran command: {c}'.format(c=command))

    # Save the iptables with the new NAT rule
    try:
        save_iptables()
    except OSError:
        _, ex, trace = sys.exc_info()
        msg = 'OSError: There was a problem saving iptables rules\n{e}'.format(e=str(ex))
        raise OSError, msg, trace
    log.info('Successfully saved iptables rules with the NAT rule')

Response:
Adds a NAT rule to iptables
   .


Instruction:
Explain what the following Python 3 code does
def service_network_restart():
    log = logging.getLogger(mod_logger + '.service_network_restart')
    command = ['service', 'network', 'restart']
    time.sleep(5)
    try:
        result = run_command(command)
        time.sleep(5)
        code = result['code']
    except CommandError:
        raise
    log.info('Network restart produced output:\n{o}'.format(o=result['output']))

    if code != 0:
        msg = 'Network services did not restart cleanly, exited with code: {c}'.format(c=code)
        log.error(msg)
        raise CommandError(msg)
    else:
        log.info('Successfully restarted networking!')

Response: Restarts the linux network service on linux


Instruction:
Write a Python 3 script for
saving iptables rules to the provided rules file.

Response:
def save_iptables(rules_file='/etc/sysconfig/iptables'):
    """Saves iptables rules to the provided rules file

    :return: None
    :raises OSError
    """
    log = logging.getLogger(mod_logger + '.save_iptables')

    # Run iptables-save to get the output
    command = ['iptables-save']
    log.debug('Running command: iptables-save')
    try:
        iptables_out = run_command(command, timeout_sec=20)
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'There was a problem running iptables command: {c}\n{e}'.format(c=' '.join(command), e=str(ex))
        raise OSError, msg, trace

    # Error if iptables-save did not exit clean
    if int(iptables_out['code']) != 0:
        raise OSError('Command [{g}] exited with code [{c}] and output:\n{o}'.format(
            g=' '.join(command), c=iptables_out['code'], o=iptables_out['output']))

    # Back up the existing rules file if it exists
    if os.path.isfile(rules_file):
        time_now = datetime.now().strftime('%Y%m%d-%H%M%S')
        backup_file = '{f}.{d}'.format(f=rules_file, d=time_now)
        log.debug('Creating backup file: {f}'.format(f=backup_file))
        shutil.copy2(rules_file, backup_file)

    # Save the output to the rules file
    log.debug('Creating file: {f}'.format(f=rules_file))
    with open(rules_file, 'w') as f:
        f.write(iptables_out['output'])


Instruction:
Create a Python 3 function for
retrieving the value of an environment variable of a remote host over SSH

Response:
def get_remote_host_environment_variable(host, environment_variable):
    """Retrieves the value of an environment variable of a
    remote host over SSH

    :param host: (str) host to query
    :param environment_variable: (str) variable to query
    :return: (str) value of the environment variable
    :raises: TypeError, CommandError
    """
    log = logging.getLogger(mod_logger + '.get_remote_host_environment_variable')
    if not isinstance(host, basestring):
        msg = 'host argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(environment_variable, basestring):
        msg = 'environment_variable argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    log.info('Checking host {h} for environment variable: {v}...'.format(h=host, v=environment_variable))
    command = ['ssh', '{h}'.format(h=host), 'echo ${v}'.format(v=environment_variable)]
    try:
        result = run_command(command, timeout_sec=5.0)
        code = result['code']
    except CommandError:
        raise
    if code != 0:
        msg = 'There was a problem checking the remote host {h} over SSH, return code: {c}'.format(
                h=host, c=code)
        log.error(msg)
        raise CommandError(msg)
    else:
        value = result['output'].strip()
        log.info('Environment variable {e} on host {h} value is: {v}'.format(
                e=environment_variable, h=host, v=value))
    return value


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_remote_host_environment_variable(host, variable_name, variable_value, env_file='/etc/bashrc'):
    log = logging.getLogger(mod_logger + '.set_remote_host_environment_variable')
    if not isinstance(host, basestring):
        msg = 'host argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(variable_name, basestring):
        msg = 'variable_name argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(variable_value, basestring):
        msg = 'variable_value argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(env_file, basestring):
        msg = 'env_file argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    log.info('Creating the environment file if it does not exist...')
    command = ['ssh', host, 'touch {f}'.format(f=env_file)]
    try:
        result = run_command(command, timeout_sec=5.0)
        code = result['code']
        output = result['output']
    except CommandError:
        raise
    if code != 0:
        msg = 'There was a problem creating environment file {f} on remote host {h} over SSH, ' \
              'exit code {c} and output:\n{o}'.format(h=host, c=code, f=env_file, o=output)
        log.error(msg)
        raise CommandError(msg)

    log.info('Creating ensuring the environment file is executable...')
    command = ['ssh', host, 'chmod +x {f}'.format(f=env_file)]
    try:
        result = run_command(command, timeout_sec=5.0)
        code = result['code']
        output = result['output']
    except CommandError:
        raise
    if code != 0:
        msg = 'There was a problem setting permissions on environment file {f} on remote host {h} over SSH, ' \
              'exit code {c} and output:\n{o}'.format(h=host, c=code, f=env_file, o=output)
        log.error(msg)
        raise CommandError(msg)

    log.info('Adding environment variable {v} with value {n} to file {f}...'.format(
            v=variable_name, n=variable_value, f=env_file))
    command = ['ssh', host, 'echo "export {v}=\\"{n}\\"" >> {f}'.format(f=env_file, v=variable_name, n=variable_value)]
    try:
        result = run_command(command, timeout_sec=5.0)
        code = result['code']
        output = result['output']
    except CommandError:
        raise
    if code != 0:
        msg = 'There was a problem adding variable {v} to environment file {f} on remote host {h} over SSH, ' \
              'exit code {c} and output:\n{o}'.format(h=host, c=code, f=env_file, o=output, v=variable_name)
        log.error(msg)
        raise CommandError(msg)
    else:
        log.info('Environment variable {v} set to {n} on host {h}'.format(v=variable_name, n=variable_value, h=host))

Response: Sets an environment variable on the remote host in the specified environment file


Instruction:
Implement a function in Python 3 to
run a command on the specified host over SSH and returns the output of the command.

Response:
def run_remote_command(host, command, timeout_sec=5.0):
    """Retrieves the value of an environment variable of a
    remote host over SSH

    :param host: (str) host to query
    :param command: (str) command
    :param timeout_sec (float) seconds to wait before killing the command.
    :return: (str) command output
    :raises: TypeError, CommandError
    """
    log = logging.getLogger(mod_logger + '.run_remote_command')
    if not isinstance(host, basestring):
        msg = 'host argument must be a string'
        raise TypeError(msg)
    if not isinstance(command, basestring):
        msg = 'command argument must be a string'
        raise TypeError(msg)
    log.debug('Running remote command on host: {h}: {c}...'.format(h=host, c=command))
    command = ['ssh', '{h}'.format(h=host), '{c}'.format(c=command)]
    try:
        result = run_command(command, timeout_sec=timeout_sec)
        code = result['code']
    except CommandError:
        raise
    if code != 0:
        msg = 'There was a problem running command [{m}] on host {h} over SSH, return code: {c}, and ' \
              'produced output:\n{o}'.format(h=host, c=code, m=' '.join(command), o=result['output'])
        raise CommandError(msg)
    else:
        output_text = result['output'].strip()
        log.debug('Running command [{m}] host {h} over SSH produced output: {o}'.format(
            m=command, h=host, o=output_text))
        output = {
            'output': output_text,
            'code': code
        }
    return output


Instruction:
Can you write a function in Python 3 where it
queries a remote host to check for existence of a marker file

Response:
def check_remote_host_marker_file(host, file_path):
    """Queries a remote host over SSH to check for existence
    of a marker file

    :param host: (str) host to query
    :param file_path: (str) path to the marker file
    :return: (bool) True if the marker file exists
    :raises: TypeError, CommandError
    """
    log = logging.getLogger(mod_logger + '.check_remote_host_marker_file')
    if not isinstance(host, basestring):
        msg = 'host argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    if not isinstance(file_path, basestring):
        msg = 'file_path argument must be a string'
        log.error(msg)
        raise TypeError(msg)
    log.debug('Checking host {h} for marker file: {f}...'.format(h=host, f=file_path))
    command = ['ssh', '{h}'.format(h=host), 'if [ -f {f} ] ; then exit 0 ; else exit 1 ; fi'.format(f=file_path)]
    try:
        result = run_command(command, timeout_sec=5.0)
        code = result['code']
        output = result['output']
    except CommandError:
        raise
    if code == 0:
        log.debug('Marker file <{f}> was found on host {h}'.format(f=file_path, h=host))
        return True
    elif code == 1 and output == '':
        log.debug('Marker file <{f}> was not found on host {h}'.format(f=file_path, h=host))
        return False
    else:
        msg = 'There was a problem checking the remote host {h} over SSH for marker file {f}, ' \
              'command returned code {c} and produced output: {o}'.format(
                h=host, f=file_path, c=code, o=output)
        log.debug(msg)
        raise CommandError(msg)


Instruction:
How would you explain what the following Python 3 function does
def restore_iptables(firewall_rules):
    log = logging.getLogger(mod_logger + '.restore_iptables')
    log.info('Restoring firewall rules from file: {f}'.format(f=firewall_rules))

    # Ensure the firewall rules file exists
    if not os.path.isfile(firewall_rules):
        msg = 'Unable to restore iptables, file not found: {f}'.format(f=firewall_rules)
        log.error(msg)
        raise OSError(msg)

    # Restore the firewall rules
    log.info('Restoring iptables from file: {f}'.format(f=firewall_rules))
    command = ['/sbin/iptables-restore', firewall_rules]
    try:
        result = run_command(command)
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'Unable to restore firewall rules from file: {f}\n{e}'.format(f=firewall_rules, e=str(ex))
        log.error(msg)
        raise OSError(msg)
    log.info('Restoring iptables produced output:\n{o}'.format(o=result['output']))

    # Save iptables
    log.info('Saving iptables...')
    command = ['/etc/init.d/iptables', 'save']
    try:
        result = run_command(command)
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'Unable to save firewall rules\n{e}'.format(e=str(ex))
        log.error(msg)
        raise OSError(msg)
    log.info('Saving iptables produced output:\n{o}'.format(o=result['output']))

Response: Restores and saves firewall rules from the firewall_rules file


Instruction:
Write a Python 3 script for
removing default gateway from the system config file and restarts the network service if it is still running

Response:
def remove_default_gateway():
    """Removes Default Gateway configuration from /etc/sysconfig/network
    and restarts networking

    :return: None
    :raises: OSError
    """
    log = logging.getLogger(mod_logger + '.remove_default_gateway')

    # Ensure the network script exists
    network_script = '/etc/sysconfig/network'
    if not os.path.isfile(network_script):
        log.info('Network script not found, nothing to do: {f}'.format(f=network_script))
        return
    log.debug('Found network script: {f}'.format(f=network_script))

    # Remove settings for GATEWAY and GATEWAYDEV
    log.info('Attempting to remove any default gateway configurations...')
    for line in fileinput.input(network_script, inplace=True):
        if re.search('^GATEWAY=.*', line):
            log.info('Removing GATEWAY line: {li}'.format(li=line))
        elif re.search('^GATEWAYDEV=.*', line):
            log.info('Removing GATEWAYDEV line: {li}'.format(li=line))
        else:
            log.debug('Keeping line: {li}'.format(li=line))
            sys.stdout.write(line)

    # Restart networking for the changes to take effect
    log.info('Restarting the network service...')
    try:
        service_network_restart()
    except CommandError:
        _, ex, trace = sys.exc_info()
        raise OSError('{n}: Attempted unsuccessfully to restart networking\n{e}'.format(
            n=ex.__class__.__name__, e=str(ex)))
    else:
        log.info('Successfully restarted networking')


Instruction:
Here you have a function in Python 3, explain what it does
def is_systemd():
    os_family = platform.system()
    if os_family != 'Linux':
        raise OSError('This method is only supported on Linux, found OS: {o}'.format(o=os_family))
    linux_distro, linux_version, distro_name = platform.linux_distribution()

    # Determine when to use systemd
    systemd = False
    if 'ubuntu' in linux_distro.lower() and '16' in linux_version:
        systemd = True
    elif 'red' in linux_distro.lower() and '7' in linux_version:
        systemd = True
    elif 'cent' in linux_distro.lower() and '7' in linux_version:
        systemd = True
    return systemd

Response: Determines if this distro has a systemd version of the system.


Instruction:
Write a Python 3 function that can
manage a Linux service.

Response:
def manage_service(service_name, service_action='status', systemd=None, output=True):
    """Use to run Linux sysv or systemd service commands

    :param service_name (str) name of the service to start
    :param service_action (str) action to perform on the service
    :param systemd (bool) True if the command should use systemd
    :param output (bool) True to print output
    :return: None
    :raises: OSError
    """
    log = logging.getLogger(mod_logger + '.manage_service')

    # Ensure the service name is a string
    if not isinstance(service_name, basestring):
        raise OSError('service_name arg must be a string, found: {t}'.format(t=service_name.__class__.__name__))

    # Ensure the service name is a string
    if not isinstance(service_action, basestring):
        raise OSError('service_action arg must be a string, found: {t}'.format(t=service_name.__class__.__name__))

    # Ensure the service action is valid
    valid_actions = ['start', 'stop', 'reload', 'restart', 'status', 'enable', 'disable']
    service_action = service_action.lower().strip()
    if service_action not in valid_actions:
        raise OSError('Invalid service action requested [{a}], valid actions are: [{v}]'.format(
            a=service_action, v=','.join(valid_actions)
        ))
    log.info('Attempting to [{a}] service: {s}'.format(a=service_action, s=service_name))

    # If systemd was not provided, attempt to determine which method to use
    if not systemd:
        log.debug('Systemd not provided, attempting to determine which method to use...')
        systemd = is_systemd()

    # Create commands depending on the method
    command_list = []
    if systemd:
        if not service_name.endswith('.service'):
            service_name = '{s}.service'.format(s=service_name)
        log.info('Attempting to manage service with systemd: {s}'.format(s=service_name))
        command_list.append(['/usr/bin/systemctl', service_action, service_name])
    else:
        log.info('Attempting to manage service with sysv: {s}'.format(s=service_name))

        # Determine the commands to run
        if service_action == 'enable':
            command_list.append(['/sbin/chkconfig', '--add', service_name])
            command_list.append(['/sbin/chkconfig', service_name, 'on'])
        elif service_action == 'disable':
            command_list.append(['/sbin/chkconfig', service_name, 'off'])
        else:
            command_list.append(['/sbin/service', service_name, service_action])

    # Run the commands in the command list
    post_command_wait_time_sec = 3
    for command in command_list:
        log.info('Attempting to run command: [{c}]'.format(c=' '.join(command)))
        try:
            result = run_command(command, timeout_sec=30, output=output)
        except CommandError:
            _, ex, trace = sys.exc_info()
            msg = 'There was a problem running a service management command\n{e}'.format(e=str(ex))
            raise OSError, msg, trace
        log.info('Command exited with code: {c}'.format(c=str(result['code'])))
        if result['code'] != 0:
            msg = 'Command exited with a non-zero code: [{c}], and produced output:\n{o}'.format(
                c=str(result['code']), o=result['output'])
            raise OSError(msg)
        else:
            log.info('Command returned successfully with output:\n{o}'.format(o=result['output']))
        log.info('Waiting {t} sec...'.format(t=str(post_command_wait_time_sec)))
        time.sleep(post_command_wait_time_sec)


Instruction:
Can you generate the documentation for the following Python 3 function
def system_reboot(wait_time_sec=20):
    log = logging.getLogger(mod_logger + '.system_reboot')

    try:
        wait_time_sec = int(wait_time_sec)
    except ValueError:
        raise CommandError('wait_time_sec must be an int, or a string convertible to an int')

    log.info('Waiting {t} seconds before reboot...'.format(t=str(wait_time_sec)))
    time.sleep(wait_time_sec)
    command = ['shutdown', '-r', 'now']
    log.info('Shutting down with command: [{c}]'.format(c=' '.join(command)))
    time.sleep(2)
    log.info('Shutting down...')
    try:
        result = run_command(command=command, timeout_sec=60)
    except CommandError:
        _, ex, trace = sys.exc_info()
        msg = 'There was a problem running shutdown command: [{c}]\n{e}'.format(c=' '.join(command), e=str(ex))
        raise SystemRebootError, msg, trace
    if result['code'] != 0:
        msg = 'Shutdown command exited with a non-zero code: [{c}], and produced output:\n{o}'.format(
            c=str(result['code']), o=result['output'])
        raise SystemRebootError(msg)
    log.info('Waiting 60 seconds to ensure the reboot completes...')
    time.sleep(60)
    msg = 'Reboot has not completed after 60 seconds'
    log.error(msg)
    raise SystemRebootTimeoutError(msg)

Response: Reboot the system after a specified wait time.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def log_event(name: str, request: Request=None, data=None, ip=None):
    log_data = {}
    if not ip and request:
        ip = get_real_ip(request)
    if ip:
        log_data['ip'] = ip
    if data:
        log_data['data'] = data

    msg = 'EVENT_{}: {}'.format(name.upper(), json.dumps(log_data))
    logger.info(msg)

Response: Logs a consistent event for easy parsing and analysis.


Instruction:
Can you generate the documentation for the following Python 3 function
def aes_b64_decrypt(value, secret, block_size=AES.block_size):
    if value is not None:
        iv = value[:block_size]
        cipher = AES.new(secret[:32], AES.MODE_CFB, iv)
        return cipher.decrypt(b64decode(
            uniorbytes(value[block_size * 2:], bytes))).decode('utf-8')

Response: AES decrypt the value with the given secret.


Instruction:
Explain what the following Python 3 code does
def aes_decrypt(value, secret, block_size=AES.block_size):
    if value is not None:
        cipher = AES.new(secret[:32], AES.MODE_CFB, value[:block_size])
        return cipher.decrypt(uniorbytes(value[block_size * 2:], bytes))

Response: AES decrypt the value with the given secret.


Instruction:
How would you explain what the following Python 3 function does
def aes_pad(s, block_size=32, padding='{'):
    return s + (block_size - len(s) % block_size) * padding

Response: Adds padding to get the correct block sizes for AES encryption


Instruction:
Write a Python 3 function that can
compare two strings in a cryptographically safe way.

Response:
def lscmp(a, b):
    """ Compares two strings in a cryptographically safe way:
        Runtime is not affected by length of common prefix, so this
        is helpful against timing attacks.

        ..
            from vital.security import lscmp
            lscmp("ringo", "starr")
            # -> False
            lscmp("ringo", "ringo")
            # -> True
        ..
    """
    l = len
    return not sum(0 if x == y else 1 for x, y in zip(a, b)) and l(a) == l(b)


Instruction:
Create a Python 3 function for
encoding or decodes a signed or unsigned cookie.

Response:
def cookie(data, key_salt='', secret=None, digestmod=None):
    """ Encodes or decodes a signed cookie.
        @data: cookie data
        @key_salt: HMAC key signing salt
        @secret: HMAC signing secret key
        @digestmod: hashing algorithm to sign with, recommended >=sha256

        -> HMAC signed or unsigned cookie data

        ..
            from vital.security import cookie

            cookie("Hello, world.", "saltyDog", secret="alBVlwe")
            # -> '!YuOoKwDp8GhrwwojdjTxSCj1c2Z+7yz7r6cC7E3hBWo=?IkhlbGxvLCB3b3JsZC4i'
            cookie(
                "!YuOoKwDp8GhrwwojdjTxSCj1c2Z+7yz7r6cC7E3hBWo=?IkhlbGxvLCB3b3JsZC4i",
                "saltyDog", secret="alBVlwe")
            # -> 'Hello, world.'
        ..
    """
    digestmod = digestmod or sha256
    if not data:
        return None
    try:
        # Decode signed cookie
        assert cookie_is_encoded(data)
        datab = uniorbytes(data, bytes)
        sig, msg = datab.split(uniorbytes('?', bytes), 1)
        key = ("{}{}").format(secret, key_salt)
        sig_check = hmac.new(
            key=uniorbytes(key, bytes), msg=msg, digestmod=digestmod).digest()
        sig_check = uniorbytes(b64encode(sig_check), bytes)
        if lscmp(sig[1:], sig_check):
            return json.loads(uniorbytes(b64decode(msg)))
        return None
    except:
        # Encode and sign a json-able object. Return a string.
        key = ("{}{}").format(secret, key_salt)
        msg = b64encode(uniorbytes(json.dumps(data), bytes))
        sig = hmac.new(
            key=uniorbytes(key, bytes), msg=msg,
            digestmod=digestmod).digest()
        sig = uniorbytes(b64encode(sig), bytes)
        return uniorbytes('!'.encode() + sig + '?'.encode() + msg)


Instruction:
Given the following Python 3 function, write the documentation
def strkey(val, chaffify=1, keyspace=string.ascii_letters + string.digits):
    chaffify = chaffify or 1
    keylen = len(keyspace)
    try:
        # INT TO STRING
        if val < 0:
            raise ValueError("Input value must be greater than -1.")

        # chaffify the value
        val = val * chaffify

        if val == 0:
            return keyspace[0]

        # output the new string value
        out = []
        out_add = out.append

        while val > 0:
            val, digit = divmod(val, keylen)
            out_add(keyspace[digit])

        return "".join(out)[::-1]
    except TypeError:
        # STRING TO INT
        out = 0
        val = str(val)
        find = str.find
        for c in val:
            out = out * keylen + find(keyspace, c)
        # dechaffify the value
        out = out // chaffify
        return int(out)

Response: Converts an integer to a sequence of strings and reverse.


Instruction:
Explain what the following Python 3 code does
def chars_in(bits, keyspace):
    keyspace = len(keyspace)
    if keyspace < 2:
        raise ValueError("Keyspace size must be >1")
    bits_per_cycle = 512
    if bits > bits_per_cycle:
        chars = 0
        bits_processed = 0
        cycles = ceil(bits / bits_per_cycle)
        for _ in range(int(cycles)):
            if bits_processed + bits_per_cycle > bits:
                bits_per_cycle = bits - bits_processed
            chars += calc_chars_in(bits_per_cycle, keyspace)
            bits_processed += bits_per_cycle
    else:
        chars = calc_chars_in(bits, keyspace)
    return abs(chars)

Response: Calculates the number of characters in a keyspace given the number of bits of entropy given the keyspace.


Instruction:
Can you tell what is the following Python 3 function doing
def bits_in(length, keyspace):
    keyspace = len(keyspace)
    length_per_cycle = 64
    if length > length_per_cycle:
        bits = 0
        length_processed = 0
        cycles = ceil(length / length_per_cycle)
        for _ in range(int(cycles)):
            if length_processed + length_per_cycle > length:
                length_per_cycle = length - length_processed
            bits += calc_bits_in(length_per_cycle, keyspace)
            length_processed += length_per_cycle
    else:
        bits = calc_bits_in(length, keyspace)
    return float(abs(bits))

Response: Calculates the number of bits of entropy in a given length of a given keyspace.


Instruction:
Write a Python 3 function that can
yield a cryptographically secure random key of desired bits of entropy of the keyspace.

Response:
def iter_random_chars(bits,
                      keyspace=string.ascii_letters + string.digits + '#/.',
                      rng=None):
    """ Yields a cryptographically secure random key of desired @bits of
        entropy within @keyspace using :class:random.SystemRandom

        @bits: (#int) minimum bits of entropy
        @keyspace: (#str) or iterable allowed output chars

        ..
            from vital.security import iter_rand

            for char in iter_rand(512):
                do_something_with(char)
    """
    if bits < 8:
        raise ValueError('Bits cannot be <8')
    else:
        chars = chars_in(bits, keyspace)
    rng = rng or random.SystemRandom()
    for char in range(int(ceil(chars))):
        yield rng.choice(keyspace)


Instruction:
Implement a function in Python 3 to
return a cryptographically secure random key of desired bits of entropy within a keyspace.

Response:
def randkey(bits, keyspace=string.ascii_letters + string.digits + '#/.',
            rng=None):
    """ Returns a cryptographically secure random key of desired @bits of
        entropy within @keyspace using :class:random.SystemRandom

        @bits: (#int) minimum bits of entropy
        @keyspace: (#str) or iterable allowed output chars
        @rng: the random number generator to use. Defaults to
            :class:random.SystemRandom. Must have a |choice| method

        -> (#str) random key

        ..
            from vital.security import randkey

            randkey(24)
            # -> '9qaX'
            randkey(48)
            # -> 'iPJ5YWs9'
            randkey(64)
            # - > 'C..VJ.KLdxg'
            randkey(64, keyspace="abc", rng=random)
            # -> 'aabcccbabcaacaccccabcaabbabcacabacbbbaaab'
        ..
    """
    return "".join(char for char in iter_random_chars(bits, keyspace, rng))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def randstr(size, keyspace=string.ascii_letters + string.digits, rng=None):
    rng = rng or random.SystemRandom()
    return "".join(rng.choice(keyspace) for char in range(int(ceil(size))))

Response: Returns a cryptographically secure random string of desired size within a keyspace.


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_requirements(filename):
    lineiter = (line.strip() for line in open(filename))
    return (line for line in lineiter if line and not line.startswith("#"))

Response: load a pip requirements file


Instruction:
Create a Python 3 function to
round a floating point number x to n significant figures

Response:
def round_sig(x, n, scien_notation = False):
    if x < 0:
        x = x * -1
        symbol = '-'
    else:
        symbol = ''

    '''round floating point x to n significant figures'''
    if type(n) is not types.IntType:
        raise TypeError, "n must be an integer"
    try:
        x = float(x)
    except:
        raise TypeError, "x must be a floating point object"
    form = "%0." + str(n-1) + "e"
    st = form % x
    num,expo = epat.findall(st)[0]
    expo = int(expo)
    fs = string.split(num,'.')
    if len(fs) < 2:
        fs = [fs[0],""]
    if expo == 0:
#         print 'One', num
        return symbol + num
    elif expo > 0:
        if len(fs[1]) < expo:
            fs[1] += "0"*(expo-len(fs[1]))
        st = fs[0]+fs[1][0:expo]
        if len(fs[1][expo:]) > 0:
            st += '.'+fs[1][expo:]
#         print 'Two', st
        return symbol + st
    else:
        expo = -expo
        if fs[0][0] == '-':
            fs[0] = fs[0][1:]
            sign = "-"
        else:
            sign = ""

        if scien_notation:
            coso = float(sign+"0."+"0"*(expo-1)+fs[0]+fs[1])
            scient_format = '{:.' + str(n) + 'e}'
            StrScientific = scient_format.format(coso)
#             print 'three', StrScientific
            return symbol + StrScientific

        else:
            formated = sign+"0."+"0"*(expo-1)+fs[0]+fs[1]
            if len(formated) > (n+5):
                formated = '0.' + '0' * n
#             print 'cuantro', formated
            return symbol + formated


Instruction:
Here you have a function in Python 3, explain what it does
def round_sig_error(x, ex, n, paren=False):
    '''Find ex rounded to n sig-figs and make the floating point x
    match the number of decimals.  If [paren], the string is
    returned as quantity(error) format'''
    stex = round_sig(ex,n)
    if stex.find('.') < 0:
        extra_zeros = len(stex) - n
        sigfigs = len(str(int(x))) - extra_zeros
        stx = round_sig(x,sigfigs)
    else:
        num_after_dec = len(string.split(stex,'.')[1])
        stx = ("%%.%df" % num_after_dec) % (x)
    if paren:
        if stex.find('.') >= 0:
            stex = stex[stex.find('.')+1:]
        return "%s(%s)" % (stx,stex)
    return stx,stex

Response:
Find ex rounded to n sig - figs and make the floating point x
    match the number of decimals.


Instruction:
Create a Python 3 function to
format a table such that the errors have n significant figures.

Response:
def format_table(cols, errors, n, labels=None, headers=None, latex=False):
    '''Format a table such that the errors have n significant
    figures.  [cols] and [errors] should be a list of 1D arrays
    that correspond to data and errors in columns.  [n] is the number of
    significant figures to keep in the errors.  [labels] is an optional
    column of strings that will be in the first column.  [headers] is
    an optional list of column headers.  If [latex] is true, myformat
    the table so that it can be included in a LaTeX table '''
    if len(cols) != len(errors):
        raise ValueError, "Error:  cols and errors must have same length"

    ncols = len(cols)
    nrows = len(cols[0])

    if headers is not None:
        if labels is not None:
            if len(headers) == ncols:
                headers = [""] + headers
            elif len(headers) == ncols+1:
                pass
            else:
                raise ValueError, "length of headers should be %d" % (ncols+1)
        else:
            if len(headers) != ncols:
                raise ValueError, "length of headers should be %d" % (ncols)

    if labels is not None:
        if len(labels) != nrows:
            raise ValueError, "length of labels should be %d" % (nrows)

    strcols = []
    for col,error in zip(cols,errors):
        strcols.append([])
        strcols.append([])
        for i in range(nrows):
            val,err = round_sig_error(col[i], error[i], n)
            strcols[-2].append(val)
            strcols[-1].append(err)

    lengths = [max([len(item) for item in strcol]) for strcol in strcols]
    myformat = ""
    if labels is not None:
        myformat += "%%%ds " % (max(map(len, labels)))
        if latex:
            myformat += "& "
    for length in lengths:
        myformat += "%%%ds " % (length)
        if latex:
            myformat += "& "
    if latex:
        myformat = myformat[:-2] + " \\\\"
    output = []
    if headers:
        if labels:
            hs = [headers[0]]
            for head in headers[1:]:
                hs.append(head)
                hs.append('+/-')
        else:
            hs = []
            for head in headers:
                hs.append(head)
                hs.append('+/-')
        output.append(myformat % tuple(hs))
    for i in range(nrows):
        if labels is not None:
            output.append(myformat % tuple([labels[i]] + [strcol[i] for strcol in strcols]))
        else:
            output.append(myformat % tuple([strcol[i] for strcol in strcols]))
    return output


Instruction:
Can you write a function in Python 3 where it
finds the min max and min error and make the floating point x and max ( ex ex1 ex2 match the number of decimals.

Response:
def round_sig_error2(x, ex1, ex2, n):
    '''Find min(ex1,ex2) rounded to n sig-figs and make the floating point x
    and max(ex,ex2) match the number of decimals.'''
    minerr = min(ex1,ex2)
    minstex = round_sig(minerr,n)
    if minstex.find('.') < 0:
        extra_zeros = len(minstex) - n
        sigfigs = len(str(int(x))) - extra_zeros
        stx = round_sig(x,sigfigs)
        maxstex = round_sig(max(ex1,ex2),sigfigs)
    else:
        num_after_dec = len(string.split(minstex,'.')[1])
        stx = ("%%.%df" % num_after_dec) % (x)
        maxstex = ("%%.%df" % num_after_dec) % (max(ex1,ex2))
    if ex1 < ex2:
        return stx,minstex,maxstex
    else:
        return stx,maxstex,minstex


Instruction:
Write a Python 3 function that can
send data to all available Azure threads.

Response:
def send_to_azure_multi_threads(instance, data, nb_threads=4, replace=True, types=None, primary_key=(),
                                sub_commit=False):
    """
    data = {
        "table_name" 	: 'name_of_the_azure_schema' + '.' + 'name_of_the_azure_table' #Must already exist,
        "columns_name" 	: [first_column_name,second_column_name,...,last_column_name],
        "rows"		: [[first_raw_value,second_raw_value,...,last_raw_value],...]
    }
    """

    # Time initialization
    start = datetime.datetime.now()

    # Extract info
    table_name = data["table_name"]
    columns_name = data["columns_name"]
    rows = data["rows"]
    total_len_data = len(rows)

    # Create table if needed
    if not existing_test(instance, table_name) or (types is not None) or (primary_key != ()):
        create.create_table(instance, data, primary_key, types)

    # Clean table if needed
    if replace:
        cleaning_function(instance, table_name)

    # Define batch size
    batch_size = int(total_len_data / nb_threads) + 1
    if total_len_data < nb_threads:
        batch_size = 1

    # Get table info
    table_info = get_table_info(instance, table_name)

    # Split data in batches of batch_size length
    split_data = []

    # global threads_state
    # threads_state = {}

    for i in range(nb_threads):
        batch = create_a_batch(rows, batch_size, i)
        split_data.append(
            {
                "data":
                    {
                        "table_name": table_name,
                        "columns_name": columns_name,
                        "rows": batch
                    },
                "instance": instance,
                "thread_number": i,
                "nb_threads": nb_threads,
                "sub_commit": sub_commit,
                "table_info": table_info,
            }
        )
        write_in_file("threads_state_%s" % str(i), str({
            "iteration": 0,
            "total": len(batch)
        }))

    with concurrent.futures.ProcessPoolExecutor() as executor:
        r = list(executor.map(send_to_azure_from_one_thread, split_data))

    print()
    for num_thread in range(nb_threads):
        insert_query = "INSERT INTO %s SELECT * FROM %s" % (table_name, table_name + "_" + str(num_thread))
        print(insert_query)
        execute_query(instance, insert_query)

    for num_thread in range(nb_threads):
        sub_table = table_name + "_" + str(num_thread)
        print(C.HEADER + "DROP TABLE %s..." % sub_table + C.ENDC)
        execute_query(instance, "DROP TABLE %s" % sub_table)
        print(C.HEADER + "DROP TABLE %s...OK" % sub_table + C.ENDC)

    total_length_data = 0
    for element in split_data:
        total_length_data = total_length_data + len(element["data"]["rows"])

    for i in range(len(r)):
        print("Thread %s : %s seconds" % (str(i), str(r[i])))

    print("Total rows: %s" % str(total_length_data))
    print(C.BOLD + "Total time in seconds : %s" % str((datetime.datetime.now() - start).seconds) + C.ENDC)
    return 0