Instruction:
Make a summary of the following Python 3 code
def get_cache_key(user_or_username, size, prefix):
    if isinstance(user_or_username, get_user_model()):
        user_or_username = user_or_username.username
    return '%s_%s_%s' % (prefix, user_or_username, size)

Response: Returns a cache key that can be used to cache the user_or_username and size.


Instruction:
Make a summary of the following Python 3 code
def cache_result(func):
    def cache_set(key, value):
        cache.set(key, value, AVATAR_CACHE_TIMEOUT)
        return value

    def cached_func(user, size):
        prefix = func.__name__
        cached_funcs.add(prefix)
        key = get_cache_key(user, size, prefix=prefix)
        return cache.get(key) or cache_set(key, func(user, size))
    return cached_func

Response: Decorator to cache the result of a function that take a user and size value.


Instruction:
Implement a function in Python 3 to
function to be called when saving or changing an user s avatars.

Response:
def invalidate_cache(user, size=None):
    """
    Function to be called when saving or changing an user's avatars.
    """
    sizes = set(AUTO_GENERATE_AVATAR_SIZES)
    if size is not None:
        sizes.add(size)
    for prefix in cached_funcs:
        for size in sizes:
            cache.delete(get_cache_key(user, size, prefix))


Instruction:
Can you create a Python 3 function that
returns a field object instance for a given PrefProxy object.

Response:
def get_field_for_proxy(pref_proxy):
    """Returns a field object instance for a given PrefProxy object.

    :param PrefProxy pref_proxy:

    :rtype: models.Field

    """
    field = {
        bool: models.BooleanField,
        int:  models.IntegerField,
        float: models.FloatField,
        datetime: models.DateTimeField,

    }.get(type(pref_proxy.default), models.TextField)()

    update_field_from_proxy(field, pref_proxy)

    return field


Instruction:
Explain what the following Python 3 code does
def update_field_from_proxy(field_obj, pref_proxy):
    attr_names = ('verbose_name', 'help_text', 'default')

    for attr_name in attr_names:
        setattr(field_obj, attr_name, getattr(pref_proxy, attr_name))

Response: Updates a field object with data from a PrefProxy object.


Instruction:
Given the following Python 3 function, write the documentation
def get_pref_model_class(app, prefs, get_prefs_func):

    module = '%s.%s' % (app, PREFS_MODULE_NAME)

    model_dict = {
            '_prefs_app': app,
            '_get_prefs': staticmethod(get_prefs_func),
            '__module__': module,
            'Meta': type('Meta', (models.options.Options,), {
                'verbose_name': _('Preference'),
                'verbose_name_plural': _('Preferences'),
                'app_label': app,
                'managed': False,
            })
    }

    for field_name, val_proxy in prefs.items():
        model_dict[field_name] = val_proxy.field

    model = type('Preferences', (models.Model,), model_dict)

    def fake_save_base(self, *args, **kwargs):

        updated_prefs = {
            f.name: getattr(self, f.name) for f in self._meta.fields if not isinstance(f, models.fields.AutoField)
        }

        app_prefs = self._get_prefs(self._prefs_app)

        for pref in app_prefs.keys():
            if pref in updated_prefs:
                app_prefs[pref].db_value = updated_prefs[pref]

        self.pk = self._prefs_app  # Make Django 1.7 happy.
        prefs_save.send(sender=self, app=self._prefs_app, updated_prefs=updated_prefs)

        return True

    model.save_base = fake_save_base

    return model

Response: Returns a preferences model class dynamically crated for a given app.


Instruction:
How would you code a function in Python 3 to
return a dictionary of locals from a given frame.

Response:
def get_frame_locals(stepback=0):
    """Returns locals dictionary from a given frame.

    :param int stepback:

    :rtype: dict

    """
    with Frame(stepback=stepback) as frame:
        locals_dict = frame.f_locals

    return locals_dict


Instruction:
Here you have a function in Python 3, explain what it does
def traverse_local_prefs(stepback=0):
    locals_dict = get_frame_locals(stepback+1)
    for k in locals_dict:
        if not k.startswith('_') and k.upper() == k:
            yield k, locals_dict

Response: Generator to traverse through preferences available in local frame.


Instruction:
Can you create a Python 3 function that
imports preferences modules from packages and project root.

Response:
def import_prefs():
    """Imports preferences modules from packages (apps) and project root."""

    # settings.py locals if autodiscover_siteprefs() is in urls.py
    settings_locals = get_frame_locals(3)

    if 'self' not in settings_locals:  # If not SiteprefsConfig.ready()
        # Try to import project-wide prefs.

        project_package = settings_locals['__package__']  # Expected project layout introduced in Django 1.4
        if not project_package:
            # Fallback to old layout.
            project_package = os.path.split(os.path.dirname(settings_locals['__file__']))[-1]

        import_module(project_package, PREFS_MODULE_NAME)

    import_project_modules(PREFS_MODULE_NAME)


Instruction:
Implement a Python 3 function for
printing file details in the current directory

Response:
def print_file_info():
    """Prints file details in the current directory"""
    tpl = TableLogger(columns='file,created,modified,size')
    for f in os.listdir('.'):
        size = os.stat(f).st_size
        date_created = datetime.fromtimestamp(os.path.getctime(f))
        date_modified = datetime.fromtimestamp(os.path.getmtime(f))
        tpl(f, date_created, date_modified, size)


Instruction:
Explain what the following Python 3 code does
def _bind_args(sig, param_matchers, args, kwargs):
        '''
        Attempt to bind the args to the type signature. First try to just bind
        to the signature, then ensure that all arguments match the parameter
        types.
        '''
        #Bind to signature. May throw its own TypeError
        bound = sig.bind(*args, **kwargs)

        if not all(param_matcher(bound.arguments[param_name])
                for param_name, param_matcher in param_matchers):
            raise TypeError

        return bound

Response: Bind the args to the type signature.


Instruction:
Can you implement a function in Python 3 that
returns a function which returns true if the argument of the a function matches the given annotation.

Response:
def _make_param_matcher(annotation, kind=None):
        '''
        For a given annotation, return a function which, when called on a
        function argument, returns true if that argument matches the annotation.
        If the annotation is a type, it calls isinstance; if it's a callable,
        it calls it on the object; otherwise, it performs a value comparison.
        If the parameter is variadic (*args) and the annotation is a type, the
        matcher will attempt to match each of the arguments in args
        '''
        if isinstance(annotation, type) or (
                isinstance(annotation, tuple) and
                all(isinstance(a, type) for a in annotation)):
            if kind is Parameter.VAR_POSITIONAL:
                return (lambda args: all(isinstance(x, annotation) for x in args))
            else:
                return (lambda x: isinstance(x, annotation))
        elif callable(annotation):
            return annotation
        else:
            return (lambda x: x == annotation)


Instruction:
Make a summary of the following Python 3 code
def _make_all_matchers(cls, parameters):
        '''
        For every parameter, create a matcher if the parameter has an
        annotation.
        '''
        for name, param in parameters:
            annotation = param.annotation
            if annotation is not Parameter.empty:
                yield name, cls._make_param_matcher(annotation, param.kind)

Response: Create a matcher for all the parameters in the list.


Instruction:
Can you write a function in Python 3 where it
creates a dispatch pair for func - a function that takes args kwargs attempts to bind those args to the type signature of func raises a TypeError

Response:
def _make_dispatch(cls, func):
        '''
        Create a dispatch pair for func- a tuple of (bind_args, func), where
        bind_args is a function that, when called with (args, kwargs), attempts
        to bind those args to the type signature of func, or else raise a
        TypeError
        '''
        sig = signature(func)
        matchers = tuple(cls._make_all_matchers(sig.parameters.items()))
        return (partial(cls._bind_args, sig, matchers), func)


Instruction:
Explain what the following Python 3 code does
def _make_wrapper(self, func):
        '''
        Makes a wrapper function that executes a dispatch call for func. The
        wrapper has the dispatch and dispatch_first attributes, so that
        additional overloads can be added to the group.
        '''

        #TODO: consider using a class to make attribute forwarding easier.
        #TODO: consider using simply another DispatchGroup, with self.callees
        # assigned by reference to the original callees.
        @wraps(func)
        def executor(*args, **kwargs):
            return self.execute(args, kwargs)
        executor.dispatch = self.dispatch
        executor.dispatch_first = self.dispatch_first
        executor.func = func
        executor.lookup = self.lookup
        return executor

Response: Make a wrapper function that executes a dispatch call for func.


Instruction:
Given the following Python 3 function, write the documentation
def dispatch(self, func):
        '''
        Adds the decorated function to this dispatch.
        '''
        self.callees.append(self._make_dispatch(func))
        return self._make_wrapper(func)

Response: Adds the decorated function to this dispatch.


Instruction:
How would you implement a function in Python 3 that
adds the decorated function to this dispatch at the FRONT of the order.

Response:
def dispatch_first(self, func):
        '''
        Adds the decorated function to this dispatch, at the FRONT of the order.
        Useful for allowing third parties to add overloaded functionality
        to be executed before default functionality.
        '''
        self.callees.appendleft(self._make_dispatch(func))
        return self._make_wrapper(func)


Instruction:
Create a Python 3 function for
dispatching a call to the first function whose type signature matches the arguemts.

Response:
def execute(self, args, kwargs):
        '''
        Dispatch a call. Call the first function whose type signature matches
        the arguemts.
        '''
        return self.lookup_explicit(args, kwargs)(*args, **kwargs)


Instruction:
Can you generate the documentation for the following Python 3 function
def setup_formatters(self, *args):
        formatters = []
        col_offset = 0
        # initialize formatters for row-id, timestamp and time-diff columns
        if self.rownum:
            formatters.append(fmt.RowNumberFormatter.setup(0))
            col_offset += 1
        if self.timestamp:
            formatters.append(fmt.DatetimeFormatter.setup(
                datetime.datetime.now(),
                fmt='{:%Y-%m-%d %H:%M:%S.%f}'.format,
                col_width=26))
            col_offset += 1
        if self.time_diff:
            formatters.append(fmt.TimeDeltaFormatter.setup(0))
            col_offset += 1

        # initialize formatters for user-defined columns
        for coli, value in enumerate(args):
            fmt_class = type2fmt.get(type(value), fmt.GenericFormatter)
            kwargs = {}

            # set column width
            if self.default_colwidth is not None:
                kwargs['col_width'] = self.default_colwidth
            if coli in self.column_widths:
                kwargs['col_width'] = self.column_widths[coli]
            elif self.columns and self.columns[coli + col_offset] in self.column_widths:
                kwargs['col_width'] = self.column_widths[self.columns[coli + col_offset]]

            # set formatter function
            if fmt_class == fmt.FloatFormatter and self.float_format is not None:
                kwargs['fmt'] = self.float_format
            if coli in self.column_formatters:
                kwargs['fmt'] = self.column_formatters[coli]
            elif self.columns and self.columns[coli + col_offset] in self.column_formatters:
                kwargs['fmt'] = self.column_formatters[self.columns[coli + col_offset]]

            formatter = fmt_class.setup(value, **kwargs)
            formatters.append(formatter)

        self.formatters = formatters

Response: Setup formatters by observing the first row.


Instruction:
Here you have a function in Python 3, explain what it does
def setup(self, *args):
        self.setup_formatters(*args)
        if self.columns:
            self.print_header()
        elif self.border and not self.csv:
            self.print_line(self.make_horizontal_border())

Response:
Do preparations before printing the first row

oment.


Instruction:
Write a Python 3 script for
converting row values into a csv line

Response:
def csv_format(self, row):
        """Converts row values into a csv line

        Args:
            row: a list of row cells as unicode
        Returns:
            csv_line (unicode)
        """
        if PY2:
            buf = io.BytesIO()
            csvwriter = csv.writer(buf)
            csvwriter.writerow([c.strip().encode(self.encoding) for c in row])
            csv_line = buf.getvalue().decode(self.encoding).rstrip()
        else:
            buf = io.StringIO()
            csvwriter = csv.writer(buf)
            csvwriter.writerow([c.strip() for c in row])
            csv_line = buf.getvalue().rstrip()
        return csv_line


Instruction:
Explain what the following Python 3 code does
def convertShpToExtend(pathToShp):

    driver = ogr.GetDriverByName('ESRI Shapefile')
    dataset = driver.Open(pathToShp)
    if dataset is not None:
        # from Layer
        layer = dataset.GetLayer()
        spatialRef = layer.GetSpatialRef()
        # from Geometry
        feature = layer.GetNextFeature()
        geom = feature.GetGeometryRef()
        spatialRef = geom.GetSpatialReference()

        #WGS84
        outSpatialRef = osr.SpatialReference()
        outSpatialRef.ImportFromEPSG(4326)

        coordTrans = osr.CoordinateTransformation(spatialRef, outSpatialRef)

        env = geom.GetEnvelope()

        pointMAX = ogr.Geometry(ogr.wkbPoint)
        pointMAX.AddPoint(env[1], env[3])
        pointMAX.Transform(coordTrans)

        pointMIN = ogr.Geometry(ogr.wkbPoint)
        pointMIN.AddPoint(env[0], env[2])
        pointMIN.Transform(coordTrans)

        return [pointMAX.GetPoint()[1],pointMIN.GetPoint()[0],pointMIN.GetPoint()[1],pointMAX.GetPoint()[0]]
    else:
        exit(" shapefile not found. Please verify your path to the shapefile")

Response: converts a shapefile to an extend shapefile


Instruction:
Can you implement a function in Python 3 that
creates a request for a single GFS object.

Response:
def create_request_gfs(dateStart,dateEnd,stepList,levelList,grid,extent,paramList,typeData):
    """
        Genere la structure de requete pour le téléchargement de données GFS

        INPUTS:\n
        -date : au format annee-mois-jour\n
        -heure : au format heure:minute:seconde\n
        -coord : une liste des coordonnees au format [N,W,S,E]\n
        -dim_grille : taille de la grille en degree \n
    """

    URLlist=[]

    #Control datetype
    listForcastSurface=['GUST','HINDEX','PRES','HGT','TMP','WEASD','SNOD','CPOFP','WILT','FLDCP','SUNSD','LFTX','CAPE','CIN','4LFTX','HPBL','LAND']
    if (0 not in [int(x) for x in stepList]):
        listForcastSurface=listForcastSurface+['PEVPR','CPRAT','PRATE','APCP','ACPCP','WATR','CSNOW','CICEP','CFPER','CRAIN','LHTFL','SHTFL','SHTFL','GFLUX','UFLX','VFLX','U-GWD','V-GWD','DSWRF','DLWRF','ULWRF','USWRF','ALBDO']
    listAnalyseSurface=['HGT','PRES','LFTX','CAPE','CIN','4LFTX']

    if typeData == 'analyse' and all([x in listAnalyseSurface for x in paramList]):
        typeData= 'analyse'
        validChoice = None
        prbParameters =  None
    else:
        if all([x in listForcastSurface for x in paramList]) and typeData != 'cycleforecast':
            if typeData=='analyse':
                typeData= 'forecast'
                validChoice = typeData
            else:
                validChoice = None
            indexParameters=[i for i, elem in enumerate([x in listAnalyseSurface for x in paramList], 1) if not elem]
            prbParameters=[]
            for i in indexParameters:
                prbParameters.append(paramList[i-1])
        else:
            if typeData != 'cycleforecast':
                typeData= 'cycleforecast'
                validChoice = typeData
            else:
                validChoice = None
            indexParameters=[i for i, elem in enumerate([x in listAnalyseSurface for x in paramList], 1) if not elem]
            prbParameters=[]
            for i in indexParameters:
                prbParameters.append(paramList[i-1])

    #Control si date/timeList disponible
    today=date.today()
    lastData = today - timedelta(days=14)
    if dateStart < lastData or dateEnd > today :
        exit('date are not in 14 days range from today' )
    else:
        #Pour chaque jour souhaité
        nbDays=(dateEnd-dateStart).days+1
        for i in range(0,nbDays):
            #on crontrole pour les timeList
            if dateStart + timedelta(days=i) == today:
                maxT=datetime.now().hour-5
                timeListCorr=[ x for x in stepList if x<maxT ]
            else:
                timeListCorr=stepList

            for t in timeListCorr:
                URL='http://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_'
                #grid
                URL=URL+"{:.2f}".format(grid).replace('.','p')+'.pl?file=gfs.'
                #time ( attention limiter avec décalage horaire for today
                URL=URL+'t'+str(t).zfill(2)+'z.'
                if (grid==0.5):
                    URL=URL+'pgrb2full.'
                else:
                    URL=URL+'pgrb2.'
                URL=URL+"{:.2f}".format(grid).replace('.','p')+'.'

                if typeData=='cycleforecast':
                    URL=URL+'f006&lev_'
                elif typeData=='forecast':
                    URL=URL+'f000&lev_'
                else:
                    URL=URL+'anl&lev_'
                URL=URL+"=on&lev_".join(levelList)+"=on&var_"
                URL=URL+"=on&var_".join(paramList)+"=on&subregion=&"
                URL=URL+"leftlon="+str(round(float(extent[1])-0.05,1))+"&rightlon="+str(round(float(extent[3])+0.05,1))+"&toplat="+str(round(float(extent[0])+0.5,1))+"&bottomlat="+str(round(float(extent[2])-0.5,1))
                URL=URL+"&dir=%2Fgfs."+"{:%Y%m%d}".format(dateStart+timedelta(days=i))+str(t).zfill(2)
                URLlist.append(URL)

        return (URLlist,validChoice,prbParameters)


Instruction:
How would you code a function in Python 3 to
convert GRIB to Tiff.

Response:
def convertGribToTiff(listeFile,listParam,listLevel,liststep,grid,startDate,endDate,outFolder):
    """ Convert GRIB to Tif"""

    dicoValues={}

    for l in listeFile:
        grbs = pygrib.open(l)
        grbs.seek(0)
        index=1
        for j in range(len(listLevel),0,-1):
            for i in range(len(listParam)-1,-1,-1):
                grb = grbs[index]
                p=grb.name.replace(' ','_')
                if grb.level != 0:
                    l=str(grb.level)+'_'+grb.typeOfLevel
                else:
                    l=grb.typeOfLevel
                if p+'_'+l not in dicoValues.keys():
                    dicoValues[p+'_'+l]=[]
                dicoValues[p+'_'+l].append(grb.values)
                shape=grb.values.shape
                lat,lon=grb.latlons()
                geoparam=(lon.min(),lat.max(),grid,grid)
                index+= 1

    nbJour=(endDate-startDate).days+1
    #on joute des arrayNan si il manque des fichiers
    for s in range(0, (len(liststep)*nbJour-len(listeFile))):
        for k in dicoValues.keys():
            dicoValues[k].append(np.full(shape, np.nan))

    #On écrit pour chacune des variables dans un fichier
    for i in range(len(dicoValues.keys())-1,-1,-1):
        dictParam=dict((k,dicoValues[dicoValues.keys()[i]][k]) for k in range(0,len(dicoValues[dicoValues.keys()[i]])))
        sorted(dictParam.items(), key=lambda x: x[0])
        outputImg=outFolder+'/'+dicoValues.keys()[i]+'_'+startDate.strftime('%Y%M%d')+'_'+endDate.strftime('%Y%M%d')+'.tif'
        writeTiffFromDicoArray(dictParam,outputImg,shape,geoparam)

    for f in listeFile:
        os.remove(f)


Instruction:
Here you have a function in Python 3, explain what it does
def on_pref_update(*args, **kwargs):
    Preference.update_prefs(*args, **kwargs)
    Preference.read_prefs(get_prefs())

Response: Triggered when a preference is updated.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_app_prefs(app=None):
    if app is None:

        with Frame(stepback=1) as frame:
            app = frame.f_globals['__name__'].split('.')[0]

    prefs = get_prefs()

    if app not in prefs:
        return {}

    return prefs[app]

Response: Returns a dictionary with preferences for a certain app or module.


Instruction:
Write a Python 3 script for
binding PrefProxy objects to module variables used by apps as preferences.

Response:
def bind_proxy(values, category=None, field=None, verbose_name=None, help_text='', static=True, readonly=False):
    """Binds PrefProxy objects to module variables used by apps as preferences.

    :param list|tuple values: Preference values.

    :param str|unicode category: Category name the preference belongs to.

    :param Field field: Django model field to represent this preference.

    :param str|unicode verbose_name: Field verbose name.

    :param str|unicode help_text: Field help text.

    :param bool static: Leave this preference static (do not store in DB).

    :param bool readonly: Make this field read only.

    :rtype: list

    """
    addrs = OrderedDict()

    depth = 3

    for local_name, locals_dict in traverse_local_prefs(depth):
        addrs[id(locals_dict[local_name])] = local_name

    proxies = []
    locals_dict = get_frame_locals(depth)

    for value in values:  # Try to preserve fields order.

        id_val = id(value)

        if id_val in addrs:
            local_name = addrs[id_val]
            local_val = locals_dict[local_name]

            if isinstance(local_val, PatchedLocal) and not isinstance(local_val, PrefProxy):

                proxy = PrefProxy(
                    local_name, value.val,
                    category=category,
                    field=field,
                    verbose_name=verbose_name,
                    help_text=help_text,
                    static=static,
                    readonly=readonly,
                )

                app_name = locals_dict['__name__'].split('.')[-2]  # x.y.settings -> y
                prefs = get_prefs()

                if app_name not in prefs:
                    prefs[app_name] = OrderedDict()

                prefs[app_name][local_name.lower()] = proxy

                # Replace original pref variable with a proxy.
                locals_dict[local_name] = proxy
                proxies.append(proxy)

    return proxies


Instruction:
Create a Python 3 function for
registering dynamically created preferences models for Admin interface.

Response:
def register_admin_models(admin_site):
    """Registers dynamically created preferences models for Admin interface.

    :param admin.AdminSite admin_site: AdminSite object.

    """
    global __MODELS_REGISTRY

    prefs = get_prefs()

    for app_label, prefs_items in prefs.items():

        model_class = get_pref_model_class(app_label, prefs_items, get_app_prefs)

        if model_class is not None:
            __MODELS_REGISTRY[app_label] = model_class
            admin_site.register(model_class, get_pref_model_admin_class(prefs_items))


Instruction:
Here you have a function in Python 3, explain what it does
def autodiscover_siteprefs(admin_site=None):
    if admin_site is None:
        admin_site = admin.site

    # Do not discover anything if called from manage.py (e.g. executing commands from cli).
    if 'manage' not in sys.argv[0] or (len(sys.argv) > 1 and sys.argv[1] in MANAGE_SAFE_COMMANDS):
        import_prefs()
        Preference.read_prefs(get_prefs())
        register_admin_models(admin_site)

Response: Automatically discovers and registers all preferences available in all apps.


Instruction:
How would you explain what the following Python 3 function does
def patch_locals(depth=2):
    for name, locals_dict in traverse_local_prefs(depth):
        locals_dict[name] = PatchedLocal(name, locals_dict[name])

    get_frame_locals(depth)[__PATCHED_LOCALS_SENTINEL] = True

Response:
Temporarily replaces all module variables
    considered preferences with PatchedLocal objects.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def unpatch_locals(depth=3):
    for name, locals_dict in traverse_local_prefs(depth):
        if isinstance(locals_dict[name], PatchedLocal):
            locals_dict[name] = locals_dict[name].val

    del get_frame_locals(depth)[__PATCHED_LOCALS_SENTINEL]

Response:
Restores the original values of module variables
    considered preferences if they are still PatchedLocal
    and PrefProxy.


Instruction:
Make a summary of the following Python 3 code
def proxy_settings_module(depth=3):
    proxies = []

    modules = sys.modules
    module_name = get_frame_locals(depth)['__name__']

    module_real = modules[module_name]

    for name, locals_dict in traverse_local_prefs(depth):

        value = locals_dict[name]

        if isinstance(value, PrefProxy):
            proxies.append(name)

    new_module = type(module_name, (ModuleType, ModuleProxy), {})(module_name)  # ModuleProxy
    new_module.bind(module_real, proxies)

    modules[module_name] = new_module

Response:
Replaces a settings module with a Module proxy to intercept
    an access to settings.


Instruction:
Implement a Python 3 function for
registering preferences that should be handled by the site.

Response:
def register_prefs(*args, **kwargs):
    """Registers preferences that should be handled by siteprefs.

    Expects preferences as *args.

    Use keyword arguments to batch apply params supported by
    ``PrefProxy`` to all preferences not constructed by ``pref`` and ``pref_group``.

    Batch kwargs:

        :param str|unicode help_text: Field help text.

        :param bool static: Leave this preference static (do not store in DB).

        :param bool readonly: Make this field read only.

    :param bool swap_settings_module: Whether to automatically replace settings module
        with a special ``ProxyModule`` object to access dynamic values of settings
        transparently (so not to bother with calling ``.value`` of ``PrefProxy`` object).

    """
    swap_settings_module = bool(kwargs.get('swap_settings_module', True))

    if __PATCHED_LOCALS_SENTINEL not in get_frame_locals(2):
        raise SitePrefsException('Please call `patch_locals()` right before the `register_prefs()`.')

    bind_proxy(args, **kwargs)

    unpatch_locals()

    swap_settings_module and proxy_settings_module()


Instruction:
Can you generate the documentation for the following Python 3 function
def pref_group(title, prefs, help_text='', static=True, readonly=False):
    bind_proxy(prefs, title, help_text=help_text, static=static, readonly=readonly)

    for proxy in prefs:  # For preferences already marked by pref().
        if isinstance(proxy, PrefProxy):
            proxy.category = title

Response:
Marks preferences group.

    :param str|unicode title: Group title

    :param list|tuple prefs: Preferences to group.

    :param str|unicode help_text: Field help text.

    :param bool static: Leave this preference static (do not store in DB).

    :param bool readonly: Make this field read only.


Instruction:
How would you explain what the following Python 3 function does
def pref(preference, field=None, verbose_name=None, help_text='', static=True, readonly=False):
    try:
        bound = bind_proxy(
            (preference,),
            field=field,
            verbose_name=verbose_name,
            help_text=help_text,
            static=static,
            readonly=readonly,
        )
        return bound[0]

    except IndexError:
        return

Response: Marks a preference variable.


Instruction:
Given the following Python 3 function, write the documentation
def bind(self, module, prefs):
        self._module = module
        self._prefs = set(prefs)

Response: Bind the current module to the given list of preference names.


Instruction:
Make a summary of the following Python 3 code
def generate_versionwarning_data_json(app, config=None, **kwargs):

    # In Sphinx >= 1.8 we use ``config-initied`` signal which comes with the
    # ``config`` object and in Sphinx < 1.8 we use ``builder-initied`` signal
    # that doesn't have the ``config`` object and we take it from the ``app``
    config = config or kwargs.pop('config', None)
    if config is None:
        config = app.config

    if config.versionwarning_project_version in config.versionwarning_messages:
        custom = True
        message = config.versionwarning_messages.get(config.versionwarning_project_version)
    else:
        custom = False
        message = config.versionwarning_default_message

    banner_html = config.versionwarning_banner_html.format(
        id_div=config.versionwarning_banner_id_div,
        banner_title=config.versionwarning_banner_title,
        message=message.format(
            **{config.versionwarning_message_placeholder: ''},
        ),
        admonition_type=config.versionwarning_admonition_type,
    )

    data = json.dumps({
        'meta': {
            'api_url': config.versionwarning_api_url,
        },
        'banner': {
            'html': banner_html,
            'id_div': config.versionwarning_banner_id_div,
            'body_selector': config.versionwarning_body_selector,
            'custom': custom,
        },
        'project': {
            'slug': config.versionwarning_project_slug,
        },
        'version': {
            'slug': config.versionwarning_project_version,
        },
    }, indent=4)

    data_path = os.path.join(STATIC_PATH, 'data')
    if not os.path.exists(data_path):
        os.mkdir(data_path)

    with open(os.path.join(data_path, JSON_DATA_FILENAME), 'w') as f:
        f.write(data)

    # Add the path where ``versionwarning-data.json`` file and
    # ``versionwarning.js`` are saved
    config.html_static_path.append(STATIC_PATH)

Response: Generate the versionwarning - data. json file.


Instruction:
Can you implement a function in Python 3 that
gives objective functions a number of dimensions and parameter range Parameters ---------- param_scales : (int, int) Scale (std. dev.) for choosing each parameter xstar : array_like Optimal parameters

Response:
def objective(param_scales=(1, 1), xstar=None, seed=None):
    """Gives objective functions a number of dimensions and parameter range

    Parameters
    ----------
    param_scales : (int, int)
        Scale (std. dev.) for choosing each parameter

    xstar : array_like
        Optimal parameters
    """
    ndim = len(param_scales)

    def decorator(func):

        @wraps(func)
        def wrapper(theta):
            return func(theta)

        def param_init():
            np.random.seed(seed)
            return np.random.randn(ndim,) * np.array(param_scales)

        wrapper.ndim = ndim
        wrapper.param_init = param_init
        wrapper.xstar = xstar

        return wrapper

    return decorator


Instruction:
Given the following Python 3 function, write the documentation
def doublewell(theta):
    k0, k1, depth = 0.01, 100, 0.5
    shallow = 0.5 * k0 * theta ** 2 + depth
    deep = 0.5 * k1 * theta ** 2
    obj = float(np.minimum(shallow, deep))
    grad = np.where(deep < shallow, k1 * theta, k0 * theta)
    return obj, grad

Response: Pointwise minimum of two quadratic bowls


Instruction:
Can you generate a brief explanation for the following Python 3 code
def camel(theta):
    x, y = theta
    obj = 2 * x ** 2 - 1.05 * x ** 4 + x ** 6 / 6 + x * y + y ** 2
    grad = np.array([
        4 * x - 4.2 * x ** 3 + x ** 5 + y,
        x + 2 * y
    ])
    return obj, grad

Response: Three - hump camel function


Instruction:
Can you generate the documentation for the following Python 3 function
def bohachevsky1(theta):
    x, y = theta
    obj = x ** 2 + 2 * y ** 2 - 0.3 * np.cos(3 * np.pi * x) - 0.4 * np.cos(4 * np.pi * y) + 0.7
    grad = np.array([
        2 * x + 0.3 * np.sin(3 * np.pi * x) * 3 * np.pi,
        4 * y + 0.4 * np.sin(4 * np.pi * y) * 4 * np.pi,
    ])
    return obj, grad

Response: One of the Bohachevsky functions


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_all_buckets(self, *args, **kwargs):
        if kwargs.pop('force', None):
            buckets = super(S3Connection, self).get_all_buckets(*args, **kwargs)

            for bucket in buckets:
                mimicdb.backend.sadd(tpl.connection, bucket.name)

            return buckets

        return [Bucket(self, bucket) for bucket in mimicdb.backend.smembers(tpl.connection)]

Response: Return a list of buckets in MimicDB.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_bucket(self, bucket_name, validate=True, headers=None, force=None):
        if force:
            bucket = super(S3Connection, self).get_bucket(bucket_name, validate, headers)
            mimicdb.backend.sadd(tpl.connection, bucket.name)
            return bucket

        if mimicdb.backend.sismember(tpl.connection, bucket_name):
            return Bucket(self, bucket_name)
        else:
            if validate:
                raise S3ResponseError(404, 'NoSuchBucket')
            else:
                return Bucket(self, bucket_name)

Response: Return a bucket from MimicDB if it exists.


Instruction:
Write a Python 3 function that can
add the bucket to MimicDB after successful creation.

Response:
def create_bucket(self, *args, **kwargs):
        """Add the bucket to MimicDB after successful creation.
        """
        bucket = super(S3Connection, self).create_bucket(*args, **kwargs)

        if bucket:
            mimicdb.backend.sadd(tpl.connection, bucket.name)

        return bucket


Instruction:
Make a summary of the following Python 3 code
def delete_bucket(self, *args, **kwargs):
        super(S3Connection, self).delete_bucket(*args, **kwargs)

        bucket = kwargs.get('bucket_name', args[0] if args else None)

        if bucket:
            mimicdb.backend.srem(tpl.connection, bucket)

Response: Delete the bucket on S3 before removing it from MimicDB.


Instruction:
Write a Python 3 script to
return the key from MimicDB.

Response:
def get_key(self, *args, **kwargs):
        """Return the key from MimicDB.

        :param boolean force: If true, API call is forced to S3
        """
        if kwargs.pop('force', None):
            headers = kwargs.get('headers', {})
            headers['force'] = True
            kwargs['headers'] = headers

        return super(Bucket, self).get_key(*args, **kwargs)


Instruction:
Explain what the following Python 3 code does
def _get_key_internal(self, *args, **kwargs):
        if args[1] is not None and 'force' in args[1]:
            key, res = super(Bucket, self)._get_key_internal(*args, **kwargs)

            if key:
                mimicdb.backend.sadd(tpl.bucket % self.name, key.name)
                mimicdb.backend.hmset(tpl.key % (self.name, key.name),
                                    dict(size=key.size,
                                         md5=key.etag.strip('"')))
            return key, res

        key = None

        if mimicdb.backend.sismember(tpl.bucket % self.name, args[0]):
            key = Key(self)
            key.name = args[0]

        return key, None

Response: Internal method that checks the S3 for the key and saves the metadata and key to the bucket set.


Instruction:
How would you explain what the following Python 3 function does
def get_all_keys(self, *args, **kwargs):
        if kwargs.pop('force', None):
            headers = kwargs.get('headers', args[0] if len(args) else None) or dict()
            headers['force'] = True
            kwargs['headers'] = headers

        return super(Bucket, self).get_all_keys(*args, **kwargs)

Response: Return a list of keys from MimicDB.


Instruction:
Can you tell what is the following Python 3 function doing
def delete_keys(self, *args, **kwargs):
        ikeys = iter(kwargs.get('keys', args[0] if args else []))

        while True:
            try:
                key = ikeys.next()
            except StopIteration:
                break

            if isinstance(key, basestring):
                mimicdb.backend.srem(tpl.bucket % self.name, key)
                mimicdb.backend.delete(tpl.key % (self.name, key))
            elif isinstance(key, BotoKey) or isinstance(key, Key):
                mimicdb.backend.srem(tpl.bucket % self.name, key.name)
                mimicdb.backend.delete(tpl.key % (self.name, key.name))

        return super(Bucket, self).delete_keys(*args, **kwargs)

Response: Remove each key or key name in an iterable from the bucket set.


Instruction:
Write a Python 3 function that can
remove key from bucket set.

Response:
def _delete_key_internal(self, *args, **kwargs):
        """Remove key name from bucket set.
        """
        mimicdb.backend.srem(tpl.bucket % self.name, args[0])
        mimicdb.backend.delete(tpl.key % (self.name, args[0]))

        return super(Bucket, self)._delete_key_internal(*args, **kwargs)


Instruction:
Given the following Python 3 function, write the documentation
def list(self, *args, **kwargs):
        if kwargs.pop('force', None):
            headers = kwargs.get('headers', args[4] if len(args) > 4 else None) or dict()
            headers['force'] = True
            kwargs['headers'] = headers

            for key in super(Bucket, self).list(*args, **kwargs):
                yield key

        else:
            prefix = kwargs.get('prefix', args[0] if args else '')

            for key in mimicdb.backend.smembers(tpl.bucket % self.name):
                if key.startswith(prefix):
                    k = Key(self, key)

                    meta = mimicdb.backend.hgetall(tpl.key % (self.name, key))

                    if meta:
                        k._load_meta(meta['size'], meta['md5'])

                    yield k

Response: Return an iterable of keys from MimicDB.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_all(self, *args, **kwargs):
        headers = kwargs.get('headers', args[2] if len(args) > 2 else None) or dict()

        if 'force' in headers:
            keys = super(Bucket, self)._get_all(*args, **kwargs)

            for key in keys:
                mimicdb.backend.sadd(tpl.bucket % self.name, key.name)
                mimicdb.backend.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"')))

                key.name = key.name

            return keys

        prefix = kwargs.get('prefix', '')

        return list(self.list(prefix=prefix))

Response: Get all keys from S3 or MimicDB.


Instruction:
How would you explain what the following Python 3 function does
def sync(self):
        for key in mimicdb.backend.smembers(tpl.bucket % self.name):
            mimicdb.backend.delete(tpl.key % (self.name, key))

        mimicdb.backend.delete(tpl.bucket % self.name)
        mimicdb.backend.sadd(tpl.connection, self.name)

        for key in self.list(force=True):
            mimicdb.backend.sadd(tpl.bucket % self.name, key.name)
            mimicdb.backend.hmset(tpl.key % (self.name, key.name), dict(size=key.size, md5=key.etag.strip('"')))

Response:
Sync a bucket.

        Force all API calls to S3 and populate the database with the current state of S3.


Instruction:
Create a Python 3 function to
compute the sparse probability of a single node.

Response:
def sparse(x, rho, penalty):
    """
    Proximal operator for the l1-norm: soft thresholding

    Parameters
    ----------
    penalty : float
        Strength or weight on the l1-norm
    """

    lmbda = penalty / rho
    return (x - lmbda) * (x >= lmbda) + (x + lmbda) * (x <= -lmbda)


Instruction:
How would you explain what the following Python 3 function does
def lbfgs(x, rho, f_df, maxiter=20):

    def f_df_augmented(theta):
        f, df = f_df(theta)
        obj = f + (rho / 2.) * np.linalg.norm(theta - x) ** 2
        grad = df + rho * (theta - x)
        return obj, grad

    res = scipy_minimize(f_df_augmented, x, jac=True, method='L-BFGS-B',
                         options={'maxiter': maxiter, 'disp': False})

    return res.x

Response: Minimize the proximal operator of a given objective using L - BFGS - B method.


Instruction:
Make a summary of the following Python 3 code
def smooth(x, rho, penalty, axis=0, newshape=None):

    orig_shape = x.shape

    if newshape is not None:
        x = x.reshape(newshape)

    # Apply Laplacian smoothing (l2 norm on the parameters multiplied by
    # the laplacian)
    n = x.shape[axis]
    lap_op = spdiags([(2 + rho / penalty) * np.ones(n),
                      -1 * np.ones(n), -1 * np.ones(n)],
                     [0, -1, 1], n, n, format='csc')

    A = penalty * lap_op
    b = rho * np.rollaxis(x, axis, 0)
    return np.rollaxis(spsolve(A, b), axis, 0).reshape(orig_shape)

Response: Applies a smoothing operator along one dimension to a matrix x.


Instruction:
Implement a function in Python 3 to
apply a proximal operator to the columns of a matrix

Response:
def columns(x, rho, proxop):
    """Applies a proximal operator to the columns of a matrix"""

    xnext = np.zeros_like(x)

    for ix in range(x.shape[1]):
        xnext[:, ix] = proxop(x[:, ix], rho)

    return xnext


Instruction:
How would you explain what the following Python 3 function does
def fantope(x, rho, dim, tol=1e-4):

    U, V = np.linalg.eigh(x)

    minval, maxval = np.maximum(U.min(), 0), np.maximum(U.max(), 20 * dim)

    while True:

        theta = 0.5 * (maxval + minval)
        thr_eigvals = np.minimum(np.maximum((U - theta), 0), 1)
        constraint = np.sum(thr_eigvals)

        if np.abs(constraint - dim) <= tol:
            break

        elif constraint < dim:
            maxval = theta

        elif constraint > dim:
            minval = theta

        else:
            break

    return np.linalg.multi_dot((V, np.diag(thr_eigvals), V.T))

Response: Fantope projection and selection of Aracite Aracite.


Instruction:
How would you code a function in Python 3 to
turn a coroutine into a gradient based optimizer.

Response:
def gradient_optimizer(coro):
    """Turns a coroutine into a gradient based optimizer."""

    class GradientOptimizer(Optimizer):

        @wraps(coro)
        def __init__(self, *args, **kwargs):
            self.algorithm = coro(*args, **kwargs)
            self.algorithm.send(None)
            self.operators = []

        def set_transform(self, func):
            self.transform = compose(destruct, func, self.restruct)

        def minimize(self, f_df, x0, display=sys.stdout, maxiter=1e3):

            self.display = display
            self.theta = x0

            # setup
            xk = self.algorithm.send(destruct(x0).copy())
            store = defaultdict(list)
            runtimes = []
            if len(self.operators) == 0:
                self.operators = [proxops.identity()]

            # setup
            obj, grad = wrap(f_df, x0)
            transform = compose(destruct, *reversed(self.operators), self.restruct)

            self.optional_print(tp.header(['Iteration', 'Objective', '||Grad||', 'Runtime']))
            try:
                for k in count():

                    # setup
                    tstart = perf_counter()
                    f = obj(xk)
                    df = grad(xk)
                    xk = transform(self.algorithm.send(df))
                    runtimes.append(perf_counter() - tstart)
                    store['f'].append(f)

                    # Update display
                    self.optional_print(tp.row([k,
                                                f,
                                                np.linalg.norm(destruct(df)),
                                                tp.humantime(runtimes[-1])]))

                    if k >= maxiter:
                        break

            except KeyboardInterrupt:
                pass

            self.optional_print(tp.bottom(4))

            # cleanup
            self.optional_print(u'\u279b Final objective: {}'.format(store['f'][-1]))
            self.optional_print(u'\u279b Total runtime: {}'.format(tp.humantime(sum(runtimes))))
            self.optional_print(u'\u279b Per iteration runtime: {} +/- {}'.format(
                tp.humantime(np.mean(runtimes)),
                tp.humantime(np.std(runtimes)),
            ))

            # result
            return OptimizeResult({
                'x': self.restruct(xk),
                'f': f,
                'df': self.restruct(df),
                'k': k,
                'obj': np.array(store['f']),
            })

    return GradientOptimizer


Instruction:
Make a summary of the following Python 3 code
def add(self, operator, *args):

        if isinstance(operator, str):
            op = getattr(proxops, operator)(*args)
        elif isinstance(operator, proxops.ProximalOperatorBaseClass):
            op = operator
        else:
            raise ValueError("operator must be a string or a subclass of ProximalOperator")

        self.operators.append(op)
        return self

Response: Adds a proximal operator to the list of operators


Instruction:
Write a Python 3 function that can
load the key attributes from the cache.

Response:
def _load_meta(self, size, md5):
        """Set key attributes to retrived metadata. Might be extended in the
        future to support more attributes.
        """
        if not hasattr(self, 'local_hashes'):
            self.local_hashes = {}

        self.size = int(size)

        if (re.match('^[a-fA-F0-9]{32}$', md5)):
            self.md5 = md5


Instruction:
Here you have a function in Python 3, explain what it does
def name(self, value):
        self._name = value

        if value:
            meta = mimicdb.backend.hgetall(tpl.key % (self.bucket.name, value))

            if meta:
                mimicdb.backend.sadd(tpl.bucket % self.bucket.name, value)
                self._load_meta(meta['size'], meta['md5'])

Response: Set the name of the key.


Instruction:
Can you generate the documentation for the following Python 3 function
def _send_file_internal(self, *args, **kwargs):
        super(Key, self)._send_file_internal(*args, **kwargs)

        mimicdb.backend.sadd(tpl.bucket % self.bucket.name, self.name)
        mimicdb.backend.hmset(tpl.key % (self.bucket.name, self.name),
                            dict(size=self.size, md5=self.md5))

Response: Called internally for any type of upload.


Instruction:
Given the following Python 3 function, write the documentation
def wrap(f_df, xref, size=1):
    memoized_f_df = lrucache(lambda x: f_df(restruct(x, xref)), size)
    objective = compose(first, memoized_f_df)
    gradient = compose(destruct, second, memoized_f_df)
    return objective, gradient

Response: Wraps a function f_df into a memoized objective + gradient function and splits it into two functions that return just the objective and gradient respectively.


Instruction:
Can you implement a function in Python 3 that
decorates a function with the given docstring Parameters ---------- docstr : string

Response:
def docstring(docstr):
    """
    Decorates a function with the given docstring

    Parameters
    ----------
    docstr : string
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        wrapper.__doc__ = docstr
        return wrapper
    return decorator


Instruction:
How would you implement a function in Python 3 that
checks the numerical gradient of a function and outputs a table of the correct and incorrect gradients.

Response:
def check_grad(f_df, xref, stepsize=1e-6, tol=1e-6, width=15, style='round', out=sys.stdout):
    """
    Compares the numerical gradient to the analytic gradient

    Parameters
    ----------
    f_df : function
        The analytic objective and gradient function to check

    x0 : array_like
        Parameter values to check the gradient at

    stepsize : float, optional
        Stepsize for the numerical gradient. Too big and this will poorly estimate the gradient.
        Too small and you will run into precision issues (default: 1e-6)

    tol : float, optional
        Tolerance to use when coloring correct/incorrect gradients (default: 1e-5)

    width : int, optional
        Width of the table columns (default: 15)

    style : string, optional
        Style of the printed table, see tableprint for a list of styles (default: 'round')
    """
    CORRECT = u'\x1b[32m\N{CHECK MARK}\x1b[0m'
    INCORRECT = u'\x1b[31m\N{BALLOT X}\x1b[0m'

    obj, grad = wrap(f_df, xref, size=0)
    x0 = destruct(xref)
    df = grad(x0)

    # header
    out.write(tp.header(["Numerical", "Analytic", "Error"], width=width, style=style) + "\n")
    out.flush()

    # helper function to parse a number
    def parse_error(number):

        # colors
        failure = "\033[91m"
        passing = "\033[92m"
        warning = "\033[93m"
        end = "\033[0m"
        base = "{}{:0.3e}{}"

        # correct
        if error < 0.1 * tol:
            return base.format(passing, error, end)

        # warning
        elif error < tol:
            return base.format(warning, error, end)

        # failure
        else:
            return base.format(failure, error, end)

    # check each dimension
    num_errors = 0
    for j in range(x0.size):

        # take a small step in one dimension
        dx = np.zeros(x0.size)
        dx[j] = stepsize

        # compute the centered difference formula
        df_approx = (obj(x0 + dx) - obj(x0 - dx)) / (2 * stepsize)
        df_analytic = df[j]

        # absolute error
        abs_error = np.linalg.norm(df_approx - df_analytic)

        # relative error
        error = abs_error if np.allclose(abs_error, 0) else abs_error / \
            (np.linalg.norm(df_analytic) + np.linalg.norm(df_approx))

        num_errors += error >= tol
        errstr = CORRECT if error < tol else INCORRECT
        out.write(tp.row([df_approx, df_analytic, parse_error(error) + ' ' + errstr],
                         width=width, style=style) + "\n")
        out.flush()

    out.write(tp.bottom(3, width=width, style=style) + "\n")
    return num_errors


Instruction:
Create a Python 3 function to
evaluate the files identified for checksum.

Response:
def evaluate(self, repo, spec, args):
        """
        Evaluate the files identified for checksum.
        """

        status = []

        # Do we have to any thing at all?
        if len(spec['files']) == 0:
            return status

        with cd(repo.rootdir):

            rules = None
            if 'rules-files' in spec and len(spec['rules-files']) > 0:
                rulesfiles = spec['rules-files']
                rules = {}
                for f in rulesfiles:
                    d = json.loads(open(f).read())
                    rules.update(d)
            elif 'rules' in spec:
                rules = {
                    'inline': spec['rules']
                }

            if rules is None or len(rules) == 0:
                print("Regression quality validation has been enabled but no rules file has been specified")
                print("Example: { 'min-r2': 0.25 }. Put this either in file or in dgit.json")
                raise InvalidParameters("Regression quality checking rules missing")

            files = dict([(f, open(f).read()) for f in spec['files']])

            for r in rules:
                if 'min-r2' not in rules[r]:
                    continue
                minr2 = float(rules[r]['min-r2'])
                for f in files:
                    match = re.search(r"R-squared:\s+(\d.\d+)", files[f])
                    if match is None:
                        status.append({
                            'target': f,
                            'validator': self.name,
                            'description': self.description,
                            'rules': r,
                            'status': "ERROR",
                            'message': "Invalid model output"
                            })
                    else:
                        r2 = match.group(1)
                        r2 = float(r2)
                        if r2 > minr2:
                            status.append({
                                'target': f,
                                'validator': self.name,
                                'description': self.description,
                                'rules': r,
                                'status': "OK",
                                'message': "Acceptable R2"
                            })
                        else:
                            status.append({
                                'target': f,
                                'validator': self.name,
                                'description': self.description,
                                'rules': r,
                                'status': "ERROR",
                                'message': "R2 is too low"
                            })

        return status


Instruction:
Can you implement a function in Python 3 that
evaluates the integrity of the datapackage. json

Response:
def evaluate(self, repo, spec, args):
        """
        Check the integrity of the datapackage.json
        """

        status = []
        with cd(repo.rootdir):
            files = spec.get('files', ['*'])
            resource_files = repo.find_matching_files(files)
            files = glob2.glob("**/*")
            disk_files = [f for f in files if os.path.isfile(f) and f != "datapackage.json"]

            allfiles = list(set(resource_files + disk_files))
            allfiles.sort()

            for f in allfiles:
                if f in resource_files and f in disk_files:
                    r = repo.get_resource(f)
                    coded_sha256 = r['sha256']
                    computed_sha256 = compute_sha256(f)
                    if computed_sha256 != coded_sha256:
                        status.append({
                            'target': f,
                            'rules': "",
                            'validator': self.name,
                            'description': self.description,
                            'status': 'ERROR',
                            'message': "Mismatch in checksum on disk and in datapackage.json"
                        })
                    else:
                        status.append({
                            'target': f,
                            'rules': "",
                            'validator': self.name,
                            'description': self.description,
                            'status': 'OK',
                            'message': ""
                        })
                elif f in resource_files:
                    status.append({
                        'target': f,
                        'rules': "",
                        'validator': self.name,
                        'description': self.description,
                        'status': 'ERROR',
                        'message': "In datapackage.json but not in repo"
                    })
                else:
                    status.append({
                        'target': f,
                        'rules': "",
                        'validator': self.name,
                        'description': self.description,
                        'status': 'ERROR',
                        'message': "In repo but not in datapackage.json"
                        })

        return status


Instruction:
How would you implement a function in Python 3 that
reads the file into a table set.

Response:
def read_file(self, filename):
        """
        Guess the filetype and read the file into row sets
        """
        #print("Reading file", filename)

        try:
            fh = open(filename, 'rb')
            table_set = any_tableset(fh) # guess the type...
        except:
            #traceback.print_exc()
            # Cannot find the schema.
            table_set = None

        return table_set


Instruction:
Explain what the following Python 3 code does
def get_schema(self, filename):
        table_set = self.read_file(filename)

        # Have I been able to read the filename
        if table_set is None:
            return []

        # Get the first table as rowset
        row_set = table_set.tables[0]

        offset, headers = headers_guess(row_set.sample)
        row_set.register_processor(headers_processor(headers))
        row_set.register_processor(offset_processor(offset + 1))
        types = type_guess(row_set.sample, strict=True)

        # Get a sample as well..
        sample = next(row_set.sample)

        clean = lambda v: str(v) if not isinstance(v, str) else v
        schema = []
        for i, h in enumerate(headers):
            schema.append([h,
                           str(types[i]),
                           clean(sample[i].value)])

        return schema

Response: Guess schema using messytables


Instruction:
Can you generate a brief explanation for the following Python 3 code
def int2fin_reference(n):
    checksum = 10 - (sum([int(c) * i for c, i in zip(str(n)[::-1], it.cycle((7, 3, 1)))]) % 10)
    if checksum == 10:
        checksum = 0
    return "%s%s" % (n, checksum)

Response: Calculates a checksum for a Finnish national reference number


Instruction:
Can you generate a brief explanation for the following Python 3 code
def iso_reference_str2int(n):
    n = n.upper()
    numbers = []
    for c in n:
        iso_reference_valid_char(c)
        if c in ISO_REFERENCE_VALID_NUMERIC:
            numbers.append(c)
        else:
            numbers.append(str(iso_reference_char2int(c)))
    return int(''.join(numbers))

Response: Creates the huge number from an ISO alphanumeric ISO reference string


Instruction:
Can you create a Python 3 function that
validates ISO reference number

Response:
def iso_reference_isvalid(ref):
    """Validates ISO reference number"""
    ref = str(ref)
    cs_source = ref[4:] + ref[:4]
    return (iso_reference_str2int(cs_source) % 97) == 1


Instruction:
Explain what the following Python 3 code does
def barcode(iban, reference, amount, due=None):

    iban = iban.replace(' ', '')
    reference = reference.replace(' ', '')

    if reference.startswith('RF'):
        version = 5
    else:
        version = 4

    if version == 5:
        reference = reference[2:]  # test RF and add 00 where needed
        if len(reference) < 23:
            reference = reference[:2] + ("0" * (23 - len(reference))) + reference[2:]
    elif version == 4:
        reference = reference.zfill(20)

    if not iban.startswith('FI'):
        raise BarcodeException('Barcodes can be printed only for IBANs starting with FI')

    iban = iban[2:]
    amount = "%08d" % (amount.quantize(Decimal('.01')).shift(2).to_integral_value())
    if len(amount) != 8:
        raise BarcodeException("Barcode payment amount must be less than 1000000.00")

    if due:
        due = due.strftime("%y%m%d")
    else:
        due = "000000"

    if version == 4:
        barcode = "%s%s%s000%s%s" % (version, iban, amount, reference, due)
    elif version == 5:
        barcode = "%s%s%s%s%s" % (version, iban, amount, reference, due)

    return barcode

Response: Calculates virtual barcode for given IBAN account number and ISO reference


Instruction:
Can you generate the documentation for the following Python 3 function
def add_file_normal(f, targetdir, generator,script, source):

    basename = os.path.basename(f)
    if targetdir != ".":
        relativepath = os.path.join(targetdir, basename)
    else:
        relativepath = basename

    relpath = os.path.relpath(f, os.getcwd())
    filetype = 'data'
    if script:
        filetype = 'script'
        if generator:
            filetype = 'generator'

    update = OrderedDict([
        ('type', filetype),
        ('generator', generator),
        ('relativepath', relativepath),
        ('content', ""),
        ('source', source),
        ('localfullpath', f),
        ('localrelativepath', relpath)
    ])

    update = annotate_record(update)

    return (basename, update)

Response: Add a normal file including its source


Instruction:
Here you have a function in Python 3, explain what it does
def extract_files(filename, includes):

    # Load the execution strace log
    lines = open(filename).readlines()

    # Extract only open files - whether for read or write. You often
    # want to capture the json/ini configuration file as well
    files = {}
    lines = [l.strip() for l in lines if 'open(' in l]
    for l in lines:

        # Check both these formats...
        # 20826 open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
        #[28940] access(b'/etc/ld.so.nohwcap', F_OK)      = -2 (No such file or directory)
        matchedfile = re.search('open\([b]["\'](.+?)["\']', l)
        if matchedfile is None:
            matchedfile = re.search('open\("(.+?)\"', l)

        if matchedfile is None:
            continue

        matchedfile = matchedfile.group(1)

        if os.path.exists(matchedfile) and os.path.isfile(matchedfile):

            #print("Looking at ", matchedfile)

            # Check what action is being performed on these
            action = 'input' if 'O_RDONLY' in l else 'output'

            matchedfile = os.path.relpath(matchedfile, ".")
            #print("Matched file's relative path", matchedfile)

            for i in includes:
                if fnmatch.fnmatch(matchedfile, i):
                    # Exclude python libraries
                    if 'site-packages' in matchedfile:
                        continue
                    if matchedfile not in files:
                        files[matchedfile] = [action]
                    else:
                        if action not in files[matchedfile]:
                            files[matchedfile].append(action)

    # A single file may be opened and closed multiple times

    if len(files) == 0:
        print("No input or output files found that match pattern")
        return []

    print('We captured files that matched the pattern you specified.')
    print('Please select files to keep (press ENTER)')

    # Let the user have the final say on which files must be included.
    filenames = list(files.keys())
    filenames.sort()
    with tempfile.NamedTemporaryFile(suffix=".tmp") as temp:
        temp.write(yaml.dump(filenames, default_flow_style=False).encode('utf-8'))
        temp.flush()
        EDITOR = os.environ.get('EDITOR','/usr/bin/vi')
        subprocess.call("%s %s" %(EDITOR,temp.name), shell=True)
        temp.seek(0)
        data = temp.read()
        selected = yaml.load(data)

    print("You selected", len(selected), "file(s)")
    if len(selected) == 0:
        return []

    # Get the action corresponding to the selected files
    filenames = [f for f in filenames if f in selected]

    # Now we know the list of files. Where should they go?
    print('Please select target locations for the various directories we found')
    print('Please make sure you do not delete any rows or edit the keys.')
    input('(press ENTER)')
    prefixes = {}
    for f in filenames:
        dirname = os.path.dirname(f)
        if dirname == "":
            dirname = "."
        prefixes[dirname] = dirname

    while True:
        with tempfile.NamedTemporaryFile(suffix=".tmp") as temp:
            temp.write(yaml.dump(prefixes, default_flow_style=False).encode('utf-8'))
            temp.flush()
            EDITOR = os.environ.get('EDITOR','/usr/bin/vi')
            subprocess.call("%s %s" %(EDITOR,temp.name), shell=True)
            temp.seek(0)
            data = temp.read()
            try:
                revised = yaml.load(data)
            except Exception as e:
                revised = {}

            #print(list(revised.keys()))
            #print(list(prefixes.keys()))

            if set(list(revised.keys())) == set(list(prefixes.keys())):
                prefixes = revised
                break
            else:
                print("Could not process edited file. Either some rows are missing or entry has YAML syntax errors")
                input("Press ENTER to continue")

    # Add the root directory back
    if "." in prefixes:
        prefixes[""] = prefixes["."]

    result = []
    ts = datetime.now().isoformat()
    for f in filenames:
        relativepath = prefixes[os.path.dirname(f)]
        if relativepath == ".":
            relativepath = os.path.basename(f)
        else:
            relativepath = os.path.join(relativepath, os.path.basename(f))

        result.append(OrderedDict([
            ('relativepath', relativepath),
            ('type', 'run-output'),
            ('actions', files[f]),
            ('mimetypes', mimetypes.guess_type(f)[0]),
            ('content', open(f).read(512)),
            ('sha256', compute_sha256(f)),
            ('ts', ts),
            ('localrelativepath', os.path.relpath(f, ".")),
            ('localfullpath', os.path.abspath(f)),
        ]))

    print(json.dumps(result, indent=4))
    return result

Response: Extract the files that are being added based on the includes


Instruction:
Given the following Python 3 function, write the documentation
def run_executable(repo, args, includes):

    # Get platform information
    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='instrumentation', name='platform')
    platform_metadata = repomgr.get_metadata()

    print("Obtaining Commit Information")
    (executable, commiturl) = \
            find_executable_commitpath(repo, args)

    # Create a local directory
    tmpdir = tempfile.mkdtemp()

    # Construct the strace command
    print("Running the command")
    strace_filename = os.path.join(tmpdir,'strace.out.txt')
    cmd = ["strace.py", "-f", "-o", strace_filename,
           "-s", "1024", "-q", "--"] + args

    # Run the command
    p = subprocess.Popen(cmd,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    out, err = p.communicate()

    # Capture the stdout/stderr
    stdout = os.path.join(tmpdir, 'stdout.log.txt')
    with open(stdout, 'w') as fd:
        fd.write(out.decode('utf-8'))

    stderr = os.path.join(tmpdir, 'stderr.log.txt')
    with open(stderr, 'w') as fd:
        fd.write(err.decode('utf-8'))

    # Check the strace output
    files = extract_files(strace_filename, includes)

    # Now insert the execution metadata
    execution_metadata = {
        'likelyexecutable': executable,
        'commitpath': commiturl,
        'args': args,
    }
    execution_metadata.update(platform_metadata)

    for i in range(len(files)):
        files[i]['execution_metadata'] = execution_metadata

    return files

Response: Run the executable and capture the input and output...


Instruction:
Can you tell what is the following Python 3 function doing
def add(repo, args, targetdir,
        execute=False, generator=False,
        includes=[], script=False,
        source=None):
    """
    Add files to the repository by explicitly specifying them or by
    specifying a pattern over files accessed during execution of an
    executable.

    Parameters
    ----------

    repo: Repository

    args: files or command line
         (a) If simply adding files, then the list of files that must
         be added (including any additional arguments to be passed to
         git
         (b) If files to be added are an output of a command line, then
         args is the command lined
    targetdir: Target directory to store the files
    execute: Args are not files to be added but scripts that must be run.
    includes: patterns used to select files to
    script: Is this a script?
    generator: Is this a generator
    source: Link to the original source of the data

    """

    # Gather the files...
    if not execute:
        files = add_files(args=args,
                          targetdir=targetdir,
                          source=source,
                          script=script,
                          generator=generator)
    else:
        files = run_executable(repo, args, includes)

    if files is None or len(files) == 0:
        return repo

    # Update the repo package but with only those that have changed.

    filtered_files = []
    package = repo.package
    for h in files:
        found = False
        for i, r in  enumerate(package['resources']):
            if h['relativepath'] == r['relativepath']:
                found = True
                if h['sha256'] == r['sha256']:
                    change = False
                    for attr in ['source']:
                        if h[attr] != r[attr]:
                            r[attr] = h[attr]
                            change = True
                    if change:
                        filtered_files.append(h)
                    continue
                else:
                    filtered_files.append(h)
                    package['resources'][i] = h
                break
        if not found:
            filtered_files.append(h)
            package['resources'].append(h)

    if len(filtered_files) == 0:
        return 0

    # Copy the files
    repo.manager.add_files(repo, filtered_files)

    # Write to disk...
    rootdir = repo.rootdir
    with cd(rootdir):
        datapath = "datapackage.json"
        with open(datapath, 'w') as fd:
            fd.write(json.dumps(package, indent=4))

    return len(filtered_files)

Response: Add files to the repository.


Instruction:
Make a summary of the following Python 3 code
def find_matching_files(self, includes):

        if len(includes) == 0:
            return []

        files = [f['relativepath'] for f in self.package['resources']]
        includes = r'|'.join([fnmatch.translate(x) for x in includes])

        # Match both the file name as well the path..
        files = [f for f in files if re.match(includes, os.path.basename(f))] + \
                [f for f in files if re.match(includes, f)]
        files = list(set(files))

        return files

Response: Find files that match patterns


Instruction:
Implement a function in Python 3 to
run a specific command using the manager

Response:
def run(self, cmd, *args):
        """
        Run a specific command using the manager
        """
        if self.manager is None:
            raise Exception("Fatal internal error: Missing repository manager")
        if cmd not in dir(self.manager):
            raise Exception("Fatal internal error: Invalid command {} being run".format(cmd))
        func = getattr(self.manager, cmd)
        repo = self
        return func(repo, *args)


Instruction:
Write a Python 3 function for
getting metadata for a given file

Response:
def get_resource(self, p):
        """
        Get metadata for a given file
        """
        for r in self.package['resources']:
            if r['relativepath'] == p:
                r['localfullpath'] = os.path.join(self.rootdir, p)
                return r

        raise Exception("Invalid path")


Instruction:
Given the following Python 3 function, write the documentation
def lookup(self, username=None, reponame=None, key=None):
        if key is None:
            key = self.key(username, reponame)
        if key not in self.repos:
            raise UnknownRepository()

        return self.repos[key]

Response: Lookup a specific user in the registry.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def rootdir(self,  username, reponame, create=True):
        path = os.path.join(self.workspace,
                            'datasets',
                            username,
                            reponame)
        if create:
            try:
                os.makedirs(path)
            except:
                pass

        return path

Response: Return the root directory for the repo.


Instruction:
Can you implement a function in Python 3 that
adds a repository to the internal lookup table...

Response:
def add(self, repo):
        """
        Add repo to the internal lookup table...
        """
        key = self.key(repo.username, repo.reponame)
        repo.key = key
        self.repos[key] = repo
        return key


Instruction:
How would you code a function in Python 3 to
list all repos in the

Response:
def list_repos(remote=False):
    """
    List repos

    Parameters
    ----------

    remote: Flag
    """
    mgr = plugins_get_mgr()

    if not remote:
        repomgr = mgr.get(what='repomanager', name='git')
        repos = repomgr.get_repo_list()
        repos.sort()
        return repos
    else:
        raise Exception("Not supported yet")


Instruction:
Write a Python 3 function for
running a shell command within the repo s rootdir

Response:
def shellcmd(repo, args):
    """
    Run a shell command within the repo's context

    Parameters
    ----------

    repo: Repository object
    args: Shell command
    """
    with cd(repo.rootdir):
        result = run(args)
        return result


Instruction:
Write a Python 3 script for
checking if the datapackage exists...

Response:
def datapackage_exists(repo):
    """
    Check if the datapackage exists...
    """
    datapath = os.path.join(repo.rootdir, "datapackage.json")
    return os.path.exists(datapath)


Instruction:
Can you write a function in Python 3 where it
deletes files in the specified object.

Response:
def delete(repo, args=[]):
    """
    Delete files

    Parameters
    ----------

    repo: Repository object
    args: Arguments to git command
    """

    # Remove the files
    result = generic_repo_cmd(repo, 'delete', args)
    if result['status'] != 'success':
        return status

    with cd(repo.rootdir):

        package = repo.package
        resources = package['resources']

        cleaned_resources = []
        for r in resources:
            relativepath = r['relativepath']
            sha256 = r['sha256']
            if relativepath not in ['', None]:
                if not os.path.exists(relativepath):
                    # This file does not exist on disk.
                    print("Skipping", relativepath)
                    continue
            cleaned_resources.append(r)

        package['resources'] = cleaned_resources
        repo.package = package

        with open('datapackage.json', 'w') as fd:
            fd.write(json.dumps(repo.package, indent=4))

        return {
            'status': 'success',
            'message': ''
        }


Instruction:
Implement a Python 3 function for
bootstrapping the datapackage file for a repository.

Response:
def bootstrap_datapackage(repo, force=False,
                          options=None, noinput=False):
    """
    Create the datapackage file..
    """

    print("Bootstrapping datapackage")

    # get the directory
    tsprefix = datetime.now().date().isoformat()

    # Initial data package json
    package = OrderedDict([
        ('title', ''),
        ('description', ''),
        ('username', repo.username),
        ('reponame', repo.reponame),
        ('name', str(repo)),
        ('title', ""),
        ('description', ""),
        ('keywords', []),
        ('resources', []),
        ('creator', getpass.getuser()),
        ('createdat', datetime.now().isoformat()),
        ('remote-url', repo.remoteurl)
    ])

    if options is not None:
        package['title'] = options['title']
        package['description'] = options['description']
    else:
        if noinput:
            raise IncompleteParameters("Option field with title and description")

        for var in ['title', 'description']:
            value = ''
            while value in ['',None]:
                value = input('Your Repo ' + var.title() + ": ")
                if len(value) == 0:
                    print("{} cannot be empty. Please re-enter.".format(var.title()))

            package[var] = value

    # Now store the package...
    (handle, filename) = tempfile.mkstemp()
    with open(filename, 'w') as fd:
        fd.write(json.dumps(package, indent=4))

    repo.package = package

    return filename


Instruction:
Can you generate the documentation for the following Python 3 function
def init(username, reponame, setup,
         force=False, options=None,
         noinput=False):
    """
    Initialize an empty repository with datapackage.json

    Parameters
    ----------

    username: Name of the user
    reponame: Name of the repo
    setup: Specify the 'configuration' (git only, git+s3 backend etc)
    force: Force creation of the files
    options: Dictionary with content of dgit.json, if available.
    noinput: Automatic operation with no human interaction
    """

    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='repomanager', name='git')

    backendmgr = None
    if setup == 'git+s3':
        backendmgr = mgr.get(what='backend', name='s3')

    repo = repomgr.init(username, reponame, force, backendmgr)

    # Now bootstrap the datapackage.json metadata file and copy it in...

    # Insert a gitignore with .dgit directory in the repo. This
    # directory will be used to store partial results
    (handle, gitignore) = tempfile.mkstemp()
    with open(gitignore, 'w') as fd:
        fd.write(".dgit")

    # Try to bootstrap. If you cant, cleanup and return
    try:
        filename = bootstrap_datapackage(repo, force, options, noinput)
    except Exception as e:
        repomgr.drop(repo,[])
        os.unlink(gitignore)
        raise e

    repo.run('add_files',
             [
                 {
                     'relativepath': 'datapackage.json',
                     'localfullpath': filename,
                 },
                 {
                     'relativepath': '.gitignore',
                     'localfullpath': gitignore,
                 },
             ])

    # Cleanup temp files
    os.unlink(filename)
    os.unlink(gitignore)

    args = ['-a', '-m', 'Bootstrapped the repo']
    repo.run('commit', args)

    return repo

Response: Initialize an empty repository with datapackage. json and return it


Instruction:
Implement a Python 3 function for
cloning a URL and return a new repository.

Response:
def clone(url):
    """
    Clone a URL. Examples include:

        - git@github.com:pingali/dgit.git
        - https://github.com:pingali/dgit.git
        - s3://mybucket/git/pingali/dgit.git

    Parameters
    ----------

    url: URL of the repo

    """
    backend = None
    backendmgr = None
    if url.startswith('s3'):
        backendtype = 's3'
    elif url.startswith("http") or url.startswith("git"):
        backendtype = 'git'
    else:
        backendtype = None

    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='repomanager', name='git')
    backendmgr = mgr.get(what='backend', name=backendtype)

    # print("Testing {} with backend {}".format(url, backendmgr))
    if backendmgr is not None and not backendmgr.url_is_valid(url):
        raise InvalidParameters("Invalid URL")

    key = repomgr.clone(url, backendmgr)

    # Insert a datapackage if it doesnt already exist...
    repo = repomgr.lookup(key=key)
    if not datapackage_exists(repo):
        filename = bootstrap_datapackage(repo)
        repo.run('add_files',
                 [
                     {
                         'relativepath': 'datapackage.json',
                         'localfullpath': filename,
                    },
                 ])
        os.unlink(filename)
        args = ['-a', '-m', 'Bootstrapped the repo']
        repo.run('commit', args)

    return repo


Instruction:
Make a summary of the following Python 3 code
def annotate_metadata_data(repo, task, patterns=["*"], size=0):

    mgr = plugins_get_mgr()
    keys = mgr.search('representation')['representation']
    representations = [mgr.get_by_key('representation', k) for k in keys]

    matching_files = repo.find_matching_files(patterns)
    package = repo.package
    rootdir = repo.rootdir
    files = package['resources']
    for f in files:
        relativepath = f['relativepath']
        if relativepath in matching_files:
            path = os.path.join(rootdir, relativepath)
            if task == 'preview':
                print("Adding preview for ", relativepath)
                f['content'] = open(path).read()[:size]
            elif task == 'schema':
                for r in representations:
                    if r.can_process(path):
                        print("Adding schema for ", path)
                        f['schema'] = r.get_schema(path)
                        break

Response: Update metadata with the content of the files that match the patterns


Instruction:
How would you implement a function in Python 3 that
adds the code to the metadata for the files in files.

Response:
def annotate_metadata_code(repo, files):
    """
    Update metadata with the commit information
    """

    package = repo.package
    package['code'] = []
    for p in files:
        matching_files = glob2.glob("**/{}".format(p))
        for f in matching_files:
            absf = os.path.abspath(f)
            print("Add commit data for {}".format(f))
            package['code'].append(OrderedDict([
                ('script', f),
                ('permalink', repo.manager.permalink(repo, absf)),
                ('mimetypes', mimetypes.guess_type(absf)[0]),
                ('sha256', compute_sha256(absf))
            ]))


Instruction:
Write a Python 3 function for
updating metadata with the actions in the history of actions

Response:
def annotate_metadata_action(repo):
    """
    Update metadata with the action history
    """
    package = repo.package

    print("Including history of actions")
    with cd(repo.rootdir):
        filename = ".dgit/log.json"
        if os.path.exists(filename):
            history = open(filename).readlines()
            actions = []
            for a in history:
                try:
                    a = json.loads(a)
                    for x in ['code']:
                        if x not in a or a[x] == None:
                            a[x] = "..."
                    actions.append(a)
                except:
                    pass
            package['actions'] = actions


Instruction:
Explain what the following Python 3 code does
def annotate_metadata_platform(repo):

    print("Added platform information")
    package = repo.package
    mgr = plugins_get_mgr()
    repomgr = mgr.get(what='instrumentation', name='platform')
    package['platform'] = repomgr.get_metadata()

Response: Add platform information to the metadata host information


Instruction:
Can you tell what is the following Python 3 function doing
def annotate_metadata_dependencies(repo):

    options = repo.options

    if 'dependencies' not in options:
        print("No dependencies")
        return []

    repos = []
    dependent_repos = options['dependencies']
    for d in dependent_repos:
        if "/" not in d:
            print("Invalid dependency specification")
        (username, reponame) = d.split("/")
        try:
            repos.append(repo.manager.lookup(username, reponame))
        except:
            print("Repository does not exist. Please create one", d)

    package = repo.package
    package['dependencies'] = []
    for r in repos:
        package['dependencies'].append({
            'username': r.username,
            'reponame': r.reponame,
            })

Response: Annotate metadata dependencies of the dependent repo with information from the dependent repo s


Instruction:
Explain what the following Python 3 code does
def post(repo, args=[]):

    mgr = plugins_get_mgr()
    keys = mgr.search(what='metadata')
    keys = keys['metadata']

    if len(keys) == 0:
        return

    # Incorporate pipeline information...
    if 'pipeline' in repo.options:
        for name, details in repo.options['pipeline'].items():
            patterns = details['files']
            matching_files = repo.find_matching_files(patterns)
            matching_files.sort()
            details['files'] = matching_files
            for i, f in enumerate(matching_files):
                r = repo.get_resource(f)
                if 'pipeline' not in r:
                    r['pipeline'] = []
                r['pipeline'].append(name + " [Step {}]".format(i))

    if 'metadata-management' in repo.options:

        print("Collecting all the required metadata to post")
        metadata = repo.options['metadata-management']

        # Add data repo history
        if 'include-data-history' in metadata and metadata['include-data-history']:
            repo.package['history'] = get_history(repo.rootdir)

        # Add action history
        if 'include-action-history' in metadata and metadata['include-action-history']:
            annotate_metadata_action(repo)

        # Add data repo history
        if 'include-preview' in metadata:
            annotate_metadata_data(repo,
                                   task='preview',
                                   patterns=metadata['include-preview']['files'],
                                   size=metadata['include-preview']['length'])

        if (('include-schema' in metadata) and metadata['include-schema']):
            annotate_metadata_data(repo,  task='schema')

        if 'include-code-history' in metadata:
            annotate_metadata_code(repo, files=metadata['include-code-history'])

        if 'include-platform' in metadata:
            annotate_metadata_platform(repo)

        if 'include-validation' in metadata:
            annotate_metadata_validation(repo)

        if 'include-dependencies' in metadata:
            annotate_metadata_dependencies(repo)

        history = repo.package.get('history',None)
        if (('include-tab-diffs' in metadata) and
            metadata['include-tab-diffs'] and
            history is not None):
            annotate_metadata_diffs(repo)

        # Insert options as well
        repo.package['config'] = repo.options

    try:
        for k in keys:
            # print("Key", k)
            metadatamgr = mgr.get_by_key('metadata', k)
            url = metadatamgr.url
            o = urlparse(url)
            print("Posting to ", o.netloc)
            response = metadatamgr.post(repo)
            if isinstance(response, str):
                print("Error while posting:", response)
            elif response.status_code in [400]:
                content = response.json()
                print("Error while posting:")
                for k in content:
                    print("   ", k,"- ", ",".join(content[k]))
    except NetworkError as e:
        print("Unable to reach metadata server!")
    except NetworkInvalidConfiguration as e:
        print("Invalid network configuration in the INI file")
        print(e.message)
    except Exception as e:
        print("Could not post. Unknown error")
        print(e)

Response: Post to metadata server


Instruction:
Implement a Python 3 function for
showing details of available plugins

Response:
def plugins_show(what=None, name=None, version=None, details=False):
    """
    Show details of available plugins

    Parameters
    ----------
    what: Class of plugins e.g., backend
    name: Name of the plugin e.g., s3
    version: Version of the plugin
    details: Show details be shown?

    """
    global pluginmgr
    return pluginmgr.show(what, name, version, details)


Instruction:
How would you explain what the following Python 3 function does
def discover_all_plugins(self):
        for v in pkg_resources.iter_entry_points('dgit.plugins'):
            m = v.load()
            m.setup(self)

Response: Load all plugins from dgit extension


Instruction:
Create a Python 3 function for
registering a plugin with the registry

Response:
def register(self, what, obj):
        """
        Registering a plugin

        Params
        ------
        what: Nature of the plugin (backend, instrumentation, repo)
        obj: Instance of the plugin
        """
        # print("Registering pattern", name, pattern)
        name = obj.name
        version = obj.version
        enable = obj.enable
        if enable == 'n':
            return

        key = Key(name, version)
        self.plugins[what][key] = obj


Instruction:
Explain what the following Python 3 code does
def search(self, what, name=None, version=None):
        filtered = {}

        # The search may for a scan (what is None) or
        if what is None:
            whats = list(self.plugins.keys())
        elif what is not None:
            if what not in self.plugins:
                raise Exception("Unknown class of plugins")
            whats = [what]
        for what in whats:
            if what not in filtered:
                filtered[what] = []
            for key in self.plugins[what].keys():
                (k_name, k_version) = key
                if name is not None and k_name != name:
                    continue
                if version is not None and k_version != version:
                    continue
                if self.plugins[what][key].enable == 'n':
                    continue
                filtered[what].append(key)

        # print(filtered)
        return filtered

Response: Search for a plugin in the cache.


Instruction:
Write a Python 3 script to
gather configuration requirements of all plugins

Response:
def gather_configs(self):
        """
        Gather configuration requirements of all plugins
        """
        configs = []
        for what in self.order:
            for key in self.plugins[what]:
                mgr = self.plugins[what][key]
                c = mgr.config(what='get')
                if c is not None:
                    c.update({
                        'description': mgr.description
                    })
                    # print("Gathering configuration from ", c)
                    configs.append(c)
        return configs


Instruction:
Write a Python 3 script to
update all configuration requirements of all plugins

Response:
def update_configs(self, config):
        """
        Gather configuration requirements of all plugins
        """
        for what in self.plugins:  # backend, repo etc.
            for key in self.plugins[what]: # s3, filesystem etc.
                # print("Updating configuration of", what, key)
                self.plugins[what][key].config(what='set', params=config)
        return


Instruction:
Given the following Python 3 function, write the documentation
def run(self):
        run_once = True
        while (run_once or self._threaded) and self.end is False:
            self.service_tx_queue()
            self.parse_messages()

            run_once = False

            if self._threaded:
                time.sleep(self._timeout)

        if self._threaded:
            logger.info('bootloader thread complete')

Response: This method is run in a separate thread.


Instruction:
How would you explain what the following Python 3 function does
def instantiate(repo, validator_name=None, filename=None, rulesfiles=None):

    default_validators = repo.options.get('validator', {})

    validators = {}
    if validator_name is not None:
        # Handle the case validator is specified..
        if validator_name in default_validators:
            validators = {
                validator_name : default_validators[validator_name]
            }
        else:
            validators = {
                validator_name : {
                    'files': [],
                    'rules': {},
                    'rules-files': []
                }
            }
    else:
        validators = default_validators

    #=========================================
    # Insert the file names
    #=========================================
    if filename is not None:
        matching_files = repo.find_matching_files([filename])
        if len(matching_files) == 0:
            print("Filename could not be found", filename)
            raise Exception("Invalid filename pattern")
        for v in validators:
            validators[v]['files'] = matching_files
    else:
        # Instantiate the files from the patterns specified
        for v in validators:
            if 'files' not in validators[v]:
                validators[v]['files'] = []
            elif len(validators[v]['files']) > 0:
                matching_files = repo.find_matching_files(validators[v]['files'])
                validators[v]['files'] = matching_files

    #=========================================
    # Insert the rules files..
    #=========================================
    if rulesfiles is not None:
        # Command lines...
        matching_files = repo.find_matching_files([rulesfiles])
        if len(matching_files) == 0:
            print("Could not find matching rules files ({}) for {}".format(rulesfiles,v))
            raise Exception("Invalid rules")
        for v in validators:
            validators[v]['rules-files'] = matching_files
    else:
        # Instantiate the files from the patterns specified
        for v in validators:
            if 'rules-files' not in validators[v]:
                validators[v]['rules-files'] = []
            else:
                rulesfiles = validators[v]['rules-files']
                matching_files = repo.find_matching_files(rulesfiles)
                validators[v]['rules-files'] = matching_files

    return validators

Response: Instantiate the validation specification


Instruction:
Explain what the following Python 3 code does
def validate(repo,
             validator_name=None,
             filename=None,
             rulesfiles=None,
             args=[]):
    """
    Validate the content of the files for consistency. Validators can
    look as deeply as needed into the files. dgit treats them all as
    black boxes.

    Parameters
    ----------

    repo: Repository object
    validator_name: Name of validator, if any. If none, then all validators specified in dgit.json will be included.
    filename: Pattern that specifies files that must be processed by the validators selected. If none, then the default specification in dgit.json is used.
    rules: Pattern specifying the files that have rules that validators will use
    show: Print the validation results on the terminal

    Returns
    -------

    status: A list of dictionaries, each with target file processed, rules file applied, status of the validation and any error  message.
    """

    mgr = plugins_get_mgr()

    # Expand the specification. Now we have full file paths
    validator_specs = instantiate(repo, validator_name, filename, rulesfiles)

    # Run the validators with rules files...
    allresults = []
    for v in validator_specs:

        keys = mgr.search(what='validator',name=v)['validator']
        for k in keys:
            validator = mgr.get_by_key('validator', k)
            result = validator.evaluate(repo,
                                        validator_specs[v],
                                        args)
            allresults.extend(result)

    return allresults

Response: Validate the content of the files.


Instruction:
How would you implement a function in Python 3 that
checks if a URL exists and if the file system path exists...

Response:
def url_is_valid(self, url):
        """
        Check if a URL exists
        """
        # Check if the file system path exists...
        if url.startswith("file://"):
            url = url.replace("file://","")

        return os.path.exists(url)


Instruction:
Implement a function in Python 3 to
post to the metadata server

Response:
def post(self, repo):
        """
        Post to the metadata server

        Parameters
        ----------

        repo
        """

        datapackage = repo.package

        url = self.url
        token = self.token
        headers = {
            'Authorization': 'Token {}'.format(token),
            'Content-Type': 'application/json'
        }

        try:
            r = requests.post(url,
                              data = json.dumps(datapackage),
                              headers=headers)

            return r
        except Exception as e:
            #print(e)
            #traceback.print_exc()
            raise NetworkError()
        return ""


Instruction:
Can you implement a function in Python 3 that
imports and returns module class from path. to. module. Class argument

Response:
def get_module_class(class_path):
    """
    imports and returns module class from ``path.to.module.Class``
    argument
    """
    mod_name, cls_name = class_path.rsplit('.', 1)

    try:
        mod = import_module(mod_name)
    except ImportError as ex:
        raise EvoStreamException('Error importing module %s: '
                                 '"%s"' % (mod_name, ex))

    return getattr(mod, cls_name)


Instruction:
Can you create a Python 3 function that
finds all executable files in the current directory.

Response:
def find_executable_files():
    """
    Find max 5 executables that are responsible for this repo.
    """
    files = glob.glob("*") + glob.glob("*/*") + glob.glob('*/*/*')
    files = filter(lambda f: os.path.isfile(f), files)
    executable = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
    final = []
    for filename in files:
        if os.path.isfile(filename):
            st = os.stat(filename)
            mode = st.st_mode
            if mode & executable:
                final.append(filename)
                if len(final) > 5:
                    break
    return final