Instruction:
How would you implement a function in Python 3 that
clusters hits into a new cluster.

Response:
def _cluster_hits(hits, clusters, assigned_hit_array, cluster_hit_indices, column_cluster_distance, row_cluster_distance, frame_cluster_distance, min_hit_charge, max_hit_charge, ignore_same_hits, noisy_pixels, disabled_pixels):
    ''' Main precompiled function that loopes over the hits and clusters them
    '''
    total_hits = hits.shape[0]
    if total_hits == 0:
        return 0  # total clusters
    max_cluster_hits = cluster_hit_indices.shape[0]

    if total_hits != clusters.shape[0]:
        raise ValueError("hits and clusters must be the same size")

    if total_hits != assigned_hit_array.shape[0]:
        raise ValueError("hits and assigned_hit_array must be the same size")

    # Correction for charge weighting
    # Some chips have non-zero charge for a charge value of zero, charge needs to be corrected to calculate cluster center correctly
    if min_hit_charge == 0:
        charge_correction = 1
    else:
        charge_correction = 0

    # Temporary variables that are reset for each cluster or event
    start_event_hit_index = 0
    start_event_cluster_index = 0
    cluster_size = 0
    event_number = hits[0]['event_number']
    event_cluster_index = 0

    # Outer loop over all hits in the array (referred to as actual hit)
    for i in range(total_hits):

        # Check for new event and reset event variables
        if _new_event(hits[i]['event_number'], event_number):
            _finish_event(
                hits=hits,
                clusters=clusters,
                start_event_hit_index=start_event_hit_index,
                stop_event_hit_index=i,
                start_event_cluster_index=start_event_cluster_index,
                stop_event_cluster_index=start_event_cluster_index + event_cluster_index)

            start_event_hit_index = i
            start_event_cluster_index = start_event_cluster_index + event_cluster_index
            event_number = hits[i]['event_number']
            event_cluster_index = 0

        if assigned_hit_array[i] > 0:  # Hit was already assigned to a cluster in the inner loop, thus skip actual hit
            continue

        if not _hit_ok(
                hit=hits[i],
                min_hit_charge=min_hit_charge,
                max_hit_charge=max_hit_charge) or (disabled_pixels.shape[0] != 0 and _pixel_masked(hits[i], disabled_pixels)):
            _set_hit_invalid(hit=hits[i], cluster_id=-1)
            assigned_hit_array[i] = 1
            continue

        # Set/reset cluster variables for new cluster
        # Reset temp array with hit indices of actual cluster for the next cluster
        _set_1d_array(cluster_hit_indices, -1, cluster_size)
        cluster_hit_indices[0] = i
        assigned_hit_array[i] = 1
        cluster_size = 1  # actual cluster has one hit so far

        for j in cluster_hit_indices:  # Loop over all hits of the actual cluster; cluster_hit_indices is updated within the loop if new hit are found
            if j < 0:  # There are no more cluster hits found
                break

            for k in range(cluster_hit_indices[0] + 1, total_hits):
                # Stop event hits loop if new event is reached
                if _new_event(hits[k]['event_number'], event_number):
                    break

                # Hit is already assigned to a cluster, thus skip actual hit
                if assigned_hit_array[k] > 0:
                    continue

                if not _hit_ok(
                        hit=hits[k],
                        min_hit_charge=min_hit_charge,
                        max_hit_charge=max_hit_charge) or (disabled_pixels.shape[0] != 0 and _pixel_masked(hits[k], disabled_pixels)):
                    _set_hit_invalid(hit=hits[k], cluster_id=-1)
                    assigned_hit_array[k] = 1
                    continue

                # Check if event hit belongs to actual hit and thus to the actual cluster
                if _is_in_max_difference(hits[j]['column'], hits[k]['column'], column_cluster_distance) and _is_in_max_difference(hits[j]['row'], hits[k]['row'], row_cluster_distance) and _is_in_max_difference(hits[j]['frame'], hits[k]['frame'], frame_cluster_distance):
                    if not ignore_same_hits or hits[j]['column'] != hits[k]['column'] or hits[j]['row'] != hits[k]['row']:
                        cluster_size += 1
                        if cluster_size > max_cluster_hits:
                            raise IndexError('cluster_hit_indices is too small to contain all cluster hits')
                        cluster_hit_indices[cluster_size - 1] = k
                        assigned_hit_array[k] = 1

                    else:
                        _set_hit_invalid(hit=hits[k], cluster_id=-2)
                        assigned_hit_array[k] = 1

        # check for valid cluster and add it to the array
        if cluster_size == 1 and noisy_pixels.shape[0] != 0 and _pixel_masked(hits[cluster_hit_indices[0]], noisy_pixels):
            _set_hit_invalid(hit=hits[cluster_hit_indices[0]], cluster_id=-1)
        else:
            _finish_cluster(
                hits=hits,
                clusters=clusters,
                cluster_size=cluster_size,
                cluster_hit_indices=cluster_hit_indices,
                cluster_index=start_event_cluster_index + event_cluster_index,
                cluster_id=event_cluster_index,
                charge_correction=charge_correction,
                noisy_pixels=noisy_pixels,
                disabled_pixels=disabled_pixels)
            event_cluster_index += 1

    # Last event is assumed to be finished at the end of the hit array, thus add info
    _finish_event(
        hits=hits,
        clusters=clusters,
        start_event_hit_index=start_event_hit_index,
        stop_event_hit_index=total_hits,
        start_event_cluster_index=start_event_cluster_index,
        stop_event_cluster_index=start_event_cluster_index + event_cluster_index)

    total_clusters = start_event_cluster_index + event_cluster_index
    return total_clusters


Instruction:
Here you have a function in Python 3, explain what it does
def get_compute_sig(self) -> Signature:
        tret = []
        tparams = []
        for t in self.tret.components:
            if t in self.resolution and self.resolution[t] is not None:
                tret.append(self.resolution[t]().show_name())
            else:
                tret.append(t)
        if hasattr(self, 'tparams'):
            for p in self.tparams:
                tp = []
                for t in p.components:
                    if t in self.resolution and self.resolution[t] is not None:
                        tp.append(self.resolution[t]().show_name())
                    else:
                        tp.append(t)
                tparams.append(" ".join(tp))
            if self.variadic:
                if self._variadic_types is None:
                    raise ValueError("Can't compute the sig "
                                     + "with unresolved variadic argument"
                                     )
                for p in self._variadic_types:
                    tp = []
                    for t in p.components:
                        if (t in self.resolution
                            and self.resolution[t] is not None
                        ):
                            tp.append(self.resolution[t]().show_name())
                        else:
                            tp.append(t)
                    tparams.append(" ".join(tp))
        ret = Fun(self.name, " ".join(tret), tparams)
        # transform as-is into our internal Signature (Val, Var, whatever)
        ret.__class__ = self._sig.__class__
        return ret

Response: Compute a signature using resolution!!!


Instruction:
Implement a function in Python 3 to
set the parent of the object.

Response:
def set_parent(self, parent) -> object:
        """
        When we add a parent (from Symbol), don't forget to resolve.
        """
        ret = self
        if parent is not None:
            ret = self._sig.set_parent(parent)
            self.resolve()
        elif not hasattr(self, 'parent'):
            # only if parent didn't exist yet
            self.parent = None
        return ret


Instruction:
Can you generate the documentation for the following Python 3 function
def resolve(self):
        # collect types for resolution
        t2resolv = []
        if hasattr(self._sig, 'tret'):
            t2resolv.append(self._sig.tret)
        if hasattr(self._sig, 'tparams') and self._sig.tparams is not None:
            for p in self._sig.tparams:
                t2resolv.append(p)
        if self._translate_to is not None:
            t2resolv.append(self._translate_to.target)
        if self._variadic_types is not None:
            for t in self._variadic_types:
                t2resolv.append(t)
        for t in t2resolv:
            for c in t.components:
                if c not in self.resolution or self.resolution[c] is None:
                    # try to find what is c
                    parent = self.get_parent()
                    if parent is not None:
                        sc = parent.get_by_symbol_name(c)
                        if len(sc) == 1:
                            sc = list(sc.values())[0]
                            # unwrap EvalCtx around Type
                            if isinstance(sc, EvalCtx):
                                sc = sc._sig
                            rtyp = weakref.ref(sc)
                            self.resolution[c] = rtyp
                            continue
                    # unresolved
                    self.resolution[c] = None

Response: Resolves the type of the object.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_resolved_names(self, type_name: TypeName) -> list:
        if not isinstance(type_name, TypeName):
            raise Exception("Take a TypeName as parameter not a %s"
                            % type(type_name))
        rnames = []
        for name in type_name.components:
            if name not in self.resolution:
                raise Exception("Unknown type %s in a EvalCtx" % name)
            rname = self.resolution[name]
            if rname is not None:
                rname = rname().show_name()
            else:
                rname = name
            rnames.append(rname)
        return rnames

Response: Get the names of all resolved types of a type name.


Instruction:
Can you create a Python 3 function that
sets the resolved name for the type.

Response:
def set_resolved_name(self, ref: dict, type_name2solve: TypeName,
                          type_name_ref: TypeName):
        """
        Warning!!! Need to rethink it when global poly type
        """
        if self.resolution[type_name2solve.value] is None:
            self.resolution[type_name2solve.value] = ref[type_name_ref.value]


Instruction:
Write a Python 3 script for
returning an Fmt representation for pretty - printing the log entry.

Response:
def to_fmt(self) -> fmt.indentable:
        """
        Return an Fmt representation for pretty-printing
        """
        lsb = []
        if len(self._lsig) > 0:
            for s in self._lsig:
                lsb.append(s.to_fmt())
        block = fmt.block("(", ")", fmt.sep(', ', lsb))
        qual = "tuple"
        txt = fmt.sep("", [qual, block])
        return txt


Instruction:
Write a Python 3 function that can
return the unique internal name of the object.

Response:
def internal_name(self):
        """
        Return the unique internal name
        """
        unq = super().internal_name()
        if self.tret is not None:
            unq += "_" + self.tret
        return unq


Instruction:
Can you tell what is the following Python 3 function doing
def _delete_local(self, filename):

        if os.path.exists(filename):
            os.remove(filename)

Response: Deletes the specified file from the local filesystem.


Instruction:
Create a Python 3 function for
deleting the specified file from the given S3 bucket.

Response:
def _delete_s3(self, filename, bucket_name):
        """Deletes the specified file from the given S3 bucket."""

        conn = S3Connection(self.access_key_id, self.access_key_secret)
        bucket = conn.get_bucket(bucket_name)

        if type(filename).__name__ == 'Key':
            filename = '/' + filename.name

        path = self._get_s3_path(filename)
        k = Key(bucket)
        k.key = path

        try:
            bucket.delete_key(k)
        except S3ResponseError:
            pass


Instruction:
Given the following Python 3 function, write the documentation
def delete(self, filename, storage_type=None, bucket_name=None):

        if not (storage_type and bucket_name):
            self._delete_local(filename)
        else:
            if storage_type != 's3':
                raise ValueError('Storage type "%s" is invalid, the only supported storage type (apart from default local storage) is s3.' % storage_type)

            self._delete_s3(filename, bucket_name)

Response: Deletes the specified file either locally or from S3 depending on the file s storage type and bucket name.


Instruction:
Implement a Python 3 function for
saving the specified file to the local file system.

Response:
def _save_local(self, temp_file, filename, obj):
        """Saves the specified file to the local file system."""

        path = self._get_path(filename)
        if not os.path.exists(os.path.dirname(path)):
            os.makedirs(os.path.dirname(path), self.permission | 0o111)

        fd = open(path, 'wb')

        # Thanks to:
        # http://stackoverflow.com/a/3253276/2066849
        temp_file.seek(0)
        t = temp_file.read(1048576)
        while t:
            fd.write(t)
            t = temp_file.read(1048576)

        fd.close()

        if self.filesize_field:
            setattr(obj, self.filesize_field, os.path.getsize(path))

        return filename


Instruction:
Write a Python 3 script to
save the specified file to the configured S3 bucket.

Response:
def _save_s3(self, temp_file, filename, obj):
        """Saves the specified file to the configured S3 bucket."""

        conn = S3Connection(self.access_key_id, self.access_key_secret)
        bucket = conn.get_bucket(self.bucket_name)

        path = self._get_s3_path(filename)
        k = bucket.new_key(path)
        k.set_contents_from_string(temp_file.getvalue())
        k.set_acl(self.acl)

        if self.filesize_field:
            setattr(obj, self.filesize_field, k.size)

        return filename


Instruction:
Write a Python 3 function that can
save the specified file to either S3 or the local filesystem depending on the currently enabled storage type.

Response:
def save(self, temp_file, filename, obj):
        """Saves the specified file to either S3 or the local filesystem, depending on the currently enabled storage type."""

        if not (self.storage_type and self.bucket_name):
            ret = self._save_local(temp_file, filename, obj)
        else:
            if self.storage_type != 's3':
                raise ValueError('Storage type "%s" is invalid, the only supported storage type (apart from default local storage) is s3.' % self.storage_type)

            ret = self._save_s3(temp_file, filename, obj)

        if self.field_name:
            setattr(obj, self.field_name, ret)

        if self.storage_type == 's3':
            if self.storage_type_field:
                setattr(obj, self.storage_type_field, self.storage_type)
            if self.bucket_name_field:
                setattr(obj, self.bucket_name_field, self.bucket_name)
        else:
            if self.storage_type_field:
                setattr(obj, self.storage_type_field, '')
            if self.bucket_name_field:
                setattr(obj, self.bucket_name_field, '')

        return ret


Instruction:
Write a Python 3 function that can
find files by licking an S3 bucket s contents by prefix.

Response:
def _find_by_path_s3(self, path, bucket_name):
        """Finds files by licking an S3 bucket's contents by prefix."""

        conn = S3Connection(self.access_key_id, self.access_key_secret)
        bucket = conn.get_bucket(bucket_name)

        s3_path = self._get_s3_path(path)

        return bucket.list(prefix=s3_path)


Instruction:
Explain what the following Python 3 code does
def find_by_path(self, path, storage_type=None, bucket_name=None):

        if not (storage_type and bucket_name):
            return self._find_by_path_local(path)
        else:
            if storage_type != 's3':
                raise ValueError('Storage type "%s" is invalid, the only supported storage type (apart from default local storage) is s3.' % storage_type)

            return self._find_by_path_s3(path, bucket_name)

Response: Finds files at the specified path.


Instruction:
Write a Python 3 script for
building an enum statement from a sequence of parameters.

Response:
def enum(*sequential, **named):
    """
    Build an enum statement
    """
    #: build enums from parameter
    enums = dict(zip(sequential, range(len(sequential))), **named)
    enums['map'] = copy.copy(enums)
    #: build reverse mapping
    enums['rmap'] = {}
    for key, value in enums.items():
        if type(value) is int:
            enums['rmap'][value] = key
    return type('Enum', (), enums)


Instruction:
Can you tell what is the following Python 3 function doing
def checktypes(func):
    sig = inspect.signature(func)

    types = {}
    for param in sig.parameters.values():
        # Iterate through function's parameters and build the list of
        # arguments types
        param_type = param.annotation
        if param_type is param.empty or not inspect.isclass(param_type):
            # Missing annotation or not a type, skip it
            continue

        types[param.name] = param_type

        # If the argument has a type specified, let's check that its
        # default value (if present) conforms with the type.
        if (param.default is not param.empty and
                not isinstance(param.default, param_type)):
            raise ValueError(
                "{func}: wrong type of a default value for {arg!r}".format(
                    func=func.__qualname__, arg=param.name)
            )

    def check_type(sig, arg_name, arg_type, arg_value):
        # Internal function that encapsulates arguments type checking
        if not isinstance(arg_value, arg_type):
            raise ValueError("{func}: wrong type of {arg!r} argument, "
                             "{exp!r} expected, got {got!r}".
                             format(func=func.__qualname__, arg=arg_name,
                                    exp=arg_type.__name__,
                                    got=type(arg_value).__name__))

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        # Let's bind the arguments
        ba = sig.bind(*args, **kwargs)
        for arg_name, arg in ba.arguments.items():
            # And iterate through the bound arguments
            try:
                type_ = types[arg_name]
            except KeyError:
                continue
            else:
                # OK, we have a type for the argument, lets get the
                # corresponding parameter description from the signature object
                param = sig.parameters[arg_name]
                if param.kind == param.VAR_POSITIONAL:
                    # If this parameter is a variable-argument parameter,
                    # then we need to check each of its values
                    for value in arg:
                        check_type(sig, arg_name, type_, value)
                elif param.kind == param.VAR_KEYWORD:
                    # If this parameter is a variable-keyword-argument
                    # parameter:
                    for subname, value in arg.items():
                        check_type(sig, arg_name + ':' + subname, type_, value)
                else:
                    # And, finally, if this parameter a regular one:
                    check_type(sig, arg_name, type_, arg)

        result = func(*ba.args, **ba.kwargs)

        # The last bit - let's check that the result is correct
        return_type = sig.return_annotation
        if (return_type is not sig.empty and
                isinstance(return_type, type) and
                not isinstance(result, return_type)):

            raise ValueError(
                '{func}: wrong return type, {exp} expected, got {got}'.format(
                    func=func.__qualname__, exp=return_type.__name__,
                    got=type(result).__name__)
            )
        return result

    return wrapper

Response: Decorator to verify arguments and return types.


Instruction:
Make a summary of the following Python 3 code
def set_one(chainmap, thing_name, callobject):
    namespaces = reversed(thing_name.split("."))
    lstname = []
    for name in namespaces:
        lstname.insert(0, name)
        strname = '.'.join(lstname)
        chainmap[strname] = callobject

Response: Add a mapping with key thing_name for callobject in chainmap with namespace handling.


Instruction:
Write a Python 3 script to
attach a method to a class.

Response:
def add_method(cls):
    """Attach a method to a class."""
    def wrapper(f):
        #if hasattr(cls, f.__name__):
        #    raise AttributeError("{} already has a '{}' attribute".format(
        #        cls.__name__, f.__name__))
        setattr(cls, f.__name__, f)
        return f
    return wrapper


Instruction:
Create a Python 3 function for
attaching a method to a parsing class and register it as a parser hook.

Response:
def hook(cls, hookname=None, erase=False):
    """Attach a method to a parsing class and register it as a parser hook.

       The method is registered with its name unless hookname is provided.
    """
    if not hasattr(cls, '_hooks'):
        raise TypeError(
            "%s didn't seems to be a BasicParser subsclasse" % cls.__name__)
    class_hook_list = cls._hooks
    class_rule_list = cls._rules

    def wrapper(f):
        nonlocal hookname
        add_method(cls)(f)
        if hookname is None:
            hookname = f.__name__
        if not erase and (hookname in class_hook_list or hookname in class_rule_list):
            raise TypeError("%s is already define has rule or hook" % hookname)
        if '.' not in hookname:
            hookname = '.'.join([cls.__module__, cls.__name__, hookname])
        set_one(class_hook_list, hookname, f)
        return f
    return wrapper


Instruction:
Explain what the following Python 3 code does
def rule(cls, rulename=None, erase=False):
    if not hasattr(cls, '_rules'):
        raise TypeError(
            "%s didn't seems to be a BasicParser subsclasse" % cls.__name__)
    class_hook_list = cls._hooks
    class_rule_list = cls._rules

    def wrapper(f):
        nonlocal rulename
        add_method(cls)(f)
        if rulename is None:
            rulename = f.__name__
        if not erase and (rulename in class_hook_list or rulename in class_rule_list):
            raise TypeError("%s is already define has rule or hook" % rulename)
        if '.' not in rulename:
            rulename = cls.__module__ + '.' + cls.__name__ + '.' + rulename
        set_one(class_rule_list, rulename, f)
        return f
    return wrapper

Response: Attach a method to a parsing class and register it as a parser rule.


Instruction:
How would you explain what the following Python 3 function does
def directive(directname=None):
    global _directives
    class_dir_list = _directives

    def wrapper(f):
        nonlocal directname
        if directname is None:
            directname = f.__name__
        f.ns_name = directname
        set_one(class_dir_list, directname, f)
        return f
    return wrapper

Response: Attach a class to a parsing class and register it as a parser directive.


Instruction:
Here you have a function in Python 3, explain what it does
def decorator(directname=None):
    global _decorators
    class_deco_list = _decorators

    def wrapper(f):
        nonlocal directname
        if directname is None:
            directname = f.__name__
        f.ns_name = directname
        set_one(class_deco_list, directname, f)

    return wrapper

Response: A class decorator that registers a class as a parsing class.


Instruction:
Explain what the following Python 3 code does
def bind(self, dst: str, src: Node) -> bool:

    for m in self.rule_nodes.maps:
        for k, v in m.items():
            if k == dst:
                m[k] = src
                return True
    raise Exception('%s not found' % dst)

Response: Bind a node to another name.


Instruction:
Make a summary of the following Python 3 code
def read_eol(self) -> bool:
    if self.read_eof():
        return False
    self._stream.save_context()
    self.read_char('\r')
    if self.read_char('\n'):
        return self._stream.validate_context()
    return self._stream.restore_context()

Response: Return True if the parser can consume an EOL byte sequence.


Instruction:
Write a Python 3 script for
reading a hexadecimal number of a user s authors.

Response:
def read_hex_integer(self) -> bool:
    """
    read a hexadecimal number
    Read the following BNF rule else return False::

        readHexInteger = [
            [ '0'..'9' | 'a'..'f' | 'A'..'F' ]+
        ]
    """
    if self.read_eof():
        return False
    self._stream.save_context()
    c = self._stream.peek_char
    if c.isdigit() or ('a' <= c.lower() and c.lower() <= 'f'):
        self._stream.incpos()
        while not self.read_eof():
            c = self._stream.peek_char
            if not (c.isdigit() or ('a' <= c.lower() and c.lower() <= 'f')):
                break
            self._stream.incpos()
        return self._stream.validate_context()
    return self._stream.restore_context()


Instruction:
Given the following Python 3 function, write the documentation
def read_cstring(self) -> bool:
    self._stream.save_context()
    idx = self._stream.index
    if self.read_char("\"") and self.read_until("\"", "\\"):
        txt = self._stream[idx:self._stream.index]
        return self._stream.validate_context()
    return self._stream.restore_context()

Response: Read a double quoted string.


Instruction:
Here you have a function in Python 3, explain what it does
def push_rule_nodes(self) -> bool:
        if self.rule_nodes is None:
            self.rule_nodes = collections.ChainMap()
            self.tag_cache = collections.ChainMap()
            self.id_cache = collections.ChainMap()
        else:
            self.rule_nodes = self.rule_nodes.new_child()
            self.tag_cache = self.tag_cache.new_child()
            self.id_cache = self.id_cache.new_child()
        return True

Response: Push context variable to store rule nodes.


Instruction:
How would you explain what the following Python 3 function does
def pop_rule_nodes(self) -> bool:
        self.rule_nodes = self.rule_nodes.parents
        self.tag_cache = self.tag_cache.parents
        self.id_cache = self.id_cache.parents
        return True

Response: Pop the context variable that store rule nodes


Instruction:
Write a Python 3 script to
return the text value of the node

Response:
def value(self, n: Node) -> str:
        """Return the text value of the node"""
        id_n = id(n)
        idcache = self.id_cache
        if id_n not in idcache:
            return ""
        name = idcache[id_n]
        tag_cache = self.tag_cache
        if name not in tag_cache:
            raise Exception("Incoherent tag cache")
        tag = tag_cache[name]
        k = "%d:%d" % (tag._begin, tag._end)
        valcache = self._streams[-1].value_cache
        if k not in valcache:
            valcache[k] = str(tag)
        return valcache[k]


Instruction:
Can you implement a function in Python 3 that
pushes a new Stream into the parser.

Response:
def parsed_stream(self, content: str, name: str=None):
        """Push a new Stream into the parser.
        All subsequent called functions will parse this new stream,
        until the 'popStream' function is called.
        """
        self._streams.append(Stream(content, name))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def begin_tag(self, name: str) -> Node:
        # Check if we could attach tag cache to current rule_nodes scope
        self.tag_cache[name] = Tag(self._stream, self._stream.index)
        return True

Response: Save the current index under the given name.


Instruction:
Write a Python 3 script for
extracting the string between saved and current index.

Response:
def end_tag(self, name: str) -> Node:
        """Extract the string between saved and current index."""
        self.tag_cache[name].set_end(self._stream.index)
        return True


Instruction:
Write a Python 3 script for
merging internal rules set with the given rules

Response:
def set_rules(cls, rules: dict) -> bool:
        """
        Merge internal rules set with the given rules
        """
        cls._rules = cls._rules.new_child()
        for rule_name, rule_pt in rules.items():
            if '.' not in rule_name:
                rule_name = cls.__module__ \
                    + '.' + cls.__name__ \
                    + '.' + rule_name
            meta.set_one(cls._rules, rule_name, rule_pt)
        return True


Instruction:
Here you have a function in Python 3, explain what it does
def set_hooks(cls, hooks: dict) -> bool:
        cls._hooks = cls._hooks.new_child()
        for hook_name, hook_pt in hooks.items():
            if '.' not in hook_name:
                hook_name = cls.__module__ \
                    + '.' + cls.__name__ \
                    + '.' + hook_name
            meta.set_one(cls._hooks, hook_name, hook_pt)
        return True

Response: Merge internal hooks set with the given hooks.


Instruction:
Can you tell what is the following Python 3 function doing
def set_directives(cls, directives: dict) -> bool:
        meta._directives = meta._directives.new_child()
        for dir_name, dir_pt in directives.items():
            meta.set_one(meta._directives, dir_name, dir_pt)
            dir_pt.ns_name = dir_name
        return True

Response: Merge internal directives with the given directives.


Instruction:
Create a Python 3 function for
evaluating a rule by name.

Response:
def eval_rule(self, name: str) -> Node:
        """Evaluate a rule by name."""
        # context created by caller
        n = Node()
        id_n = id(n)
        self.rule_nodes['_'] = n
        self.id_cache[id_n] = '_'
        # TODO: other behavior for  empty rules?
        if name not in self.__class__._rules:
            self.diagnostic.notify(
                error.Severity.ERROR,
                "Unknown rule : %s" % name,
                error.LocationInfo.from_stream(self._stream, is_error=True)
            )
            raise self.diagnostic
        self._lastRule = name
        rule_to_eval = self.__class__._rules[name]
        # TODO: add packrat cache here, same rule - same pos == same res
        res = rule_to_eval(self)
        if res:
            res = self.rule_nodes['_']
        return res


Instruction:
Can you generate a brief explanation for the following Python 3 code
def eval_hook(self, name: str, ctx: list) -> Node:
        if name not in self.__class__._hooks:
            # TODO: don't always throw error, could have return True by default
            self.diagnostic.notify(
                error.Severity.ERROR,
                "Unknown hook : %s" % name,
                error.LocationInfo.from_stream(self._stream, is_error=True)
            )
            raise self.diagnostic
        self._lastRule = '#' + name
        res = self.__class__._hooks[name](self, *ctx)
        if type(res) is not bool:
            raise TypeError("Your hook %r didn't return a bool value" % name)
        return res

Response: Evaluate the hook by its name


Instruction:
Given the following Python 3 function, write the documentation
def peek_text(self, text: str) -> bool:
        start = self._stream.index
        stop = start + len(text)
        if stop > self._stream.eos_index:
            return False
        return self._stream[self._stream.index:stop] == text

Response: Same as readText but doesn t consume the stream.


Instruction:
Given the following Python 3 function, write the documentation
def one_char(self) -> bool:
        if self.read_eof():
            return False
        self._stream.incpos()
        return True

Response: Read one byte in stream.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def read_char(self, c: str) -> bool:
        if self.read_eof():
            return False
        self._stream.save_context()
        if c == self._stream.peek_char:
            self._stream.incpos()
            return self._stream.validate_context()
        return self._stream.restore_context()

Response: Consume the next character in the buffer and return True if it is valid False otherwise.


Instruction:
Make a summary of the following Python 3 code
def read_until(self, c: str, inhibitor='\\') -> bool:
        if self.read_eof():
            return False
        self._stream.save_context()
        while not self.read_eof():
            if self.peek_char(inhibitor):
                # Delete inhibitor and inhibited character
                self.one_char()
                self.one_char()
            if self.peek_char(c):
                self._stream.incpos()
                return self._stream.validate_context()
            self._stream.incpos()
        return self._stream.restore_context()

Response: Consume the stream until the specified character is read.


Instruction:
Write a Python 3 script to
consume all the stream. Same as EOF in BNF.

Response:
def read_until_eof(self) -> bool:
        """Consume all the stream. Same as EOF in BNF."""
        if self.read_eof():
            return True
        # TODO: read ALL
        self._stream.save_context()
        while not self.read_eof():
            self._stream.incpos()
        return self._stream.validate_context()


Instruction:
Create a Python 3 function for
reading a string of text from the stream.

Response:
def read_text(self, text: str) -> bool:
        """
        Consume a strlen(text) text at current position in the stream
        else return False.
        Same as "" in BNF
        ex : read_text("ls");.
        """
        if self.read_eof():
            return False
        self._stream.save_context()
        if self.peek_text(text):
            self._stream.incpos(len(text))
            return self._stream.validate_context()
        return self._stream.restore_context()


Instruction:
Can you tell what is the following Python 3 function doing
def read_range(self, begin: str, end: str) -> int:
        if self.read_eof():
            return False
        c = self._stream.peek_char
        if begin <= c <= end:
            self._stream.incpos()
            return True
        return False

Response: Read a range of the ISO - 8601 ISO - 8601 names.


Instruction:
Can you write a function in Python 3 where it
returns the unique internal name of the class.

Response:
def internal_name(self):
        """
        Return the unique internal name
        """
        unq = 'f_' + super().internal_name()
        if self.tparams is not None:
            unq += "_" + "_".join(self.tparams)
        if self.tret is not None:
            unq += "_" + self.tret
        return unq


Instruction:
Write a Python 3 function that can
set the hit fields for the clusterer.

Response:
def set_hit_fields(self, hit_fields):
        ''' Tell the clusterizer the meaning of the field names.

        The hit_fields parameter is a dict, e.g., {"new field name": "standard field name"}.

        If None default mapping is set.

        Example:
        --------
        Internally, the clusterizer uses the hit fields names "column"/"row". If the name of the hits fields are "x"/"y", call:
        set_hit_fields(hit_fields={'x': 'column',
                                   'y': 'row'})
        '''
        if not hit_fields:
            hit_fields_mapping_inverse = {}
            hit_fields_mapping = {}
        else:
            # Create also the inverse dictionary for faster lookup
            hit_fields_mapping_inverse = dict((k, v) for k, v in hit_fields.items())
            hit_fields_mapping = dict((v, k) for k, v in hit_fields.items())

        for old_name, new_name in self._default_hit_fields_mapping.items():
            if old_name not in hit_fields_mapping:
                hit_fields_mapping[old_name] = new_name
                hit_fields_mapping_inverse[new_name] = old_name

        self._hit_fields_mapping = hit_fields_mapping
        self._hit_fields_mapping_inverse = hit_fields_mapping_inverse


Instruction:
Write a Python 3 script to
set the cluster fields for the current class.

Response:
def set_cluster_fields(self, cluster_fields):
        ''' Tell the clusterizer the meaning of the field names.

        The cluster_fields parameter is a dict, e.g., {"new filed name": "standard field name"}.
        '''
        if not cluster_fields:
            cluster_fields_mapping_inverse = {}
            cluster_fields_mapping = {}
        else:
            # Create also the inverse dictionary for faster lookup
            cluster_fields_mapping_inverse = dict((k, v) for k, v in cluster_fields.items())
            cluster_fields_mapping = dict((v, k) for k, v in cluster_fields.items())

        for old_name, new_name in self._default_cluster_fields_mapping.items():
            if old_name not in cluster_fields_mapping:
                cluster_fields_mapping[old_name] = new_name
                cluster_fields_mapping_inverse[new_name] = old_name

        self._cluster_fields_mapping = cluster_fields_mapping
        self._cluster_fields_mapping_inverse = cluster_fields_mapping_inverse


Instruction:
Write a Python 3 function for
setting the data type of the clustered hits array.

Response:
def set_hit_dtype(self, hit_dtype):
        ''' Set the data type of the hits.

        Fields that are not mentioned here are NOT copied into the clustered hits array.
        Clusterizer has to know the hit data type to produce the clustered hit result with the same data types.

        Parameters:
        -----------
        hit_dtype : numpy.dtype or equivalent
            Defines the dtype of the hit array.

        Example:
        --------
        hit_dtype = [("column", np.uint16), ("row", np.uint16)], where
        "column", "row" is the field name of the input hit array.
        '''

        if not hit_dtype:
            hit_dtype = np.dtype([])
        else:
            hit_dtype = np.dtype(hit_dtype)
        cluster_hits_descr = hit_dtype.descr

        # Add default back to description
        for dtype_name, dtype in self._default_cluster_hits_descr:
            if self._hit_fields_mapping[dtype_name] not in hit_dtype.fields:
                cluster_hits_descr.append((dtype_name, dtype))
        self._cluster_hits_descr = cluster_hits_descr
        self._init_arrays(size=0)


Instruction:
Can you create a Python 3 function that
sets the data type of the cluster array.

Response:
def set_cluster_dtype(self, cluster_dtype):
        ''' Set the data type of the cluster.

        Parameters:
        -----------
        cluster_dtype : numpy.dtype or equivalent
            Defines the dtype of the cluster array.
        '''
        if not cluster_dtype:
            cluster_dtype = np.dtype([])
        else:
            cluster_dtype = np.dtype(cluster_dtype)
        cluster_descr = cluster_dtype.descr

        for dtype_name, dtype in self._default_cluster_descr:
            if self._cluster_fields_mapping[dtype_name] not in cluster_dtype.fields:
                cluster_descr.append((dtype_name, dtype))

        self._cluster_descr = cluster_descr
        self._init_arrays(size=0)


Instruction:
Here you have a function in Python 3, explain what it does
def add_cluster_field(self, description):
        ''' Adds a field or a list of fields to the cluster result array. Has to be defined as a numpy dtype entry, e.g.: ('parameter', '<i4') '''
        if isinstance(description, list):
            for item in description:
                if len(item) != 2:
                    raise TypeError("Description needs to be a list of 2-tuples of a string and a dtype.")
                self._cluster_descr.append(item)
        else:
            if len(description) != 2:
                raise TypeError("Description needs to be a 2-tuple of a string and a dtype.")
            self._cluster_descr.append(description)
        self._init_arrays(size=0)

Response: Adds a field or a list of fields to the cluster result array.


Instruction:
Can you tell what is the following Python 3 function doing
def set_end_of_cluster_function(self, function):
        ''' Adding function to module.
        This is maybe the only way to make the clusterizer to work with multiprocessing.
        '''
        self.cluster_functions._end_of_cluster_function = self._jitted(function)
        self._end_of_cluster_function = function

Response: Adds function to module.


Instruction:
Can you tell what is the following Python 3 function doing
def set_end_of_event_function(self, function):
        ''' Adding function to module.
        This is maybe the only way to make the clusterizer to work with multiprocessing.
        '''
        self.cluster_functions._end_of_event_function = self._jitted(function)
        self._end_of_event_function = function

Response: Adds function to module.


Instruction:
Make a summary of the following Python 3 code
def cluster_hits(self, hits, noisy_pixels=None, disabled_pixels=None):
        ''' Cluster given hit array.

        The noisy_pixels and disabled_pixels parameters are iterables of column/row index pairs, e.g. [[column_1, row_1], [column_2, row_2], ...].
        The noisy_pixels parameter allows for removing clusters that consist of a single noisy pixels. Clusters with 2 or more noisy pixels are not removed.
        The disabled_pixels parameter allows for ignoring pixels.
        '''
        # Jitting a second time to workaround different bahavior of the installation methods on different platforms (pip install vs. python setup.py).
        # In some circumstances, the Numba compiler can't compile functions that were pickled previously.
        self.cluster_functions._end_of_cluster_function = self._jitted(self._end_of_cluster_function)
        self.cluster_functions._end_of_event_function = self._jitted(self._end_of_event_function)

        n_hits = hits.shape[0]  # Set n_hits to new size

        if (n_hits < int(0.5 * self._cluster_hits.size)) or (n_hits > self._cluster_hits.size):
            self._init_arrays(size=int(1.1 * n_hits))  # oversize buffer slightly to reduce allocations
        else:
            self._assigned_hit_array.fill(0)  # The hit indices of the actual cluster, 0 means not assigned
            self._cluster_hit_indices.fill(-1)  # The hit indices of the actual cluster, -1 means not assigned

        self._clusters.dtype.names = self._unmap_cluster_field_names(self._clusters.dtype.names)  # Reset the data fields from previous renaming
        self._cluster_hits.dtype.names = self._unmap_hit_field_names(self._cluster_hits.dtype.names)  # Reset the data fields from previous renaming
        self._check_struct_compatibility(hits)

        # The hit info is extended by the cluster info; this is only possible by creating a new hit info array and copy data
        for field_name in hits.dtype.fields:
            if field_name in self._hit_fields_mapping_inverse:
                cluster_hits_field_name = self._hit_fields_mapping_inverse[field_name]
            else:
                cluster_hits_field_name = field_name
            if cluster_hits_field_name in self._cluster_hits.dtype.fields:
                self._cluster_hits[cluster_hits_field_name][:n_hits] = hits[field_name]

        noisy_pixels_array = np.array([]) if noisy_pixels is None else np.array(noisy_pixels)
        if noisy_pixels_array.shape[0] != 0:
            noisy_pixels_max_range = np.array([max(0, np.max(noisy_pixels_array[:, 0])), max(0, np.max(noisy_pixels_array[:, 1]))])
            noisy_pixels = np.zeros(noisy_pixels_max_range + 1, dtype=np.bool)
            noisy_pixels[noisy_pixels_array[:, 0], noisy_pixels_array[:, 1]] = 1
        else:
            noisy_pixels = np.zeros((0, 0), dtype=np.bool)

        disabled_pixels_array = np.array([]) if disabled_pixels is None else np.array(disabled_pixels)
        if disabled_pixels_array.shape[0] != 0:
            disabled_pixels_max_range = np.array([np.max(disabled_pixels_array[:, 0]), np.max(disabled_pixels_array[:, 1])])
            disabled_pixels = np.zeros(disabled_pixels_max_range + 1, dtype=np.bool)
            disabled_pixels[disabled_pixels_array[:, 0], disabled_pixels_array[:, 1]] = 1
        else:
            disabled_pixels = np.zeros((0, 0), dtype=np.bool)

#         col_dtype = self._cluster_hits.dtype.fields["column"][0]
#         row_dtype = self._cluster_hits.dtype.fields["row"][0]
#         mask_dtype = {"names": ["column", "row"],
#                       "formats": [col_dtype, row_dtype]}
#         noisy_pixels = np.recarray(noisy_pixels_array.shape[0], dtype=mask_dtype)
#         noisy_pixels[:] = [(item[0], item[1]) for item in noisy_pixels_array]
#         disabled_pixels = np.recarray(disabled_pixels_array.shape[0], dtype=mask_dtype)
#         disabled_pixels[:] = [(item[0], item[1]) for item in disabled_pixels_array]

        n_clusters = self.cluster_functions._cluster_hits(  # Set n_clusters to new size
            hits=self._cluster_hits[:n_hits],
            clusters=self._clusters[:n_hits],
            assigned_hit_array=self._assigned_hit_array[:n_hits],
            cluster_hit_indices=self._cluster_hit_indices[:n_hits],
            column_cluster_distance=self._column_cluster_distance,
            row_cluster_distance=self._row_cluster_distance,
            frame_cluster_distance=self._frame_cluster_distance,
            min_hit_charge=self._min_hit_charge,
            max_hit_charge=self._max_hit_charge,
            ignore_same_hits=self._ignore_same_hits,
            noisy_pixels=noisy_pixels,
            disabled_pixels=disabled_pixels)

        self._cluster_hits.dtype.names = self._map_hit_field_names(self._cluster_hits.dtype.names)  # Rename the data fields for the result
        self._clusters.dtype.names = self._map_cluster_field_names(self._clusters.dtype.names)  # Rename the data fields for the result

        return self._cluster_hits[:n_hits], self._clusters[:n_clusters]

Response: Cluster the hit array into a new array of hit indices and dtype.


Instruction:
Explain what the following Python 3 code does
def _check_struct_compatibility(self, hits):
        ''' Takes the hit array and checks if the important data fields have the same data type than the hit clustered array and that the field names are correct.'''
        for key, _ in self._cluster_hits_descr:
            if key in self._hit_fields_mapping_inverse:
                mapped_key = self._hit_fields_mapping_inverse[key]
            else:
                mapped_key = key
            # Only check hit fields that contain hit information
            if mapped_key in ['cluster_ID', 'is_seed', 'cluster_size', 'n_cluster']:
                continue
            if key not in hits.dtype.names:
                raise TypeError('Required hit field "%s" not found.' % key)
            if self._cluster_hits.dtype[mapped_key] != hits.dtype[key]:
                raise TypeError('The dtype for hit data field "%s" does not match. Got/expected: %s/%s.' % (key, hits.dtype[key], self._cluster_hits.dtype[mapped_key]))
        additional_hit_fields = set(hits.dtype.names) - set([key for key, val in self._cluster_hits_descr])
        if additional_hit_fields:
            logging.warning('Found additional hit fields: %s' % ", ".join(additional_hit_fields))

Response: Takes the hit array and checks if the important data fields have the same data type than the hit clustered array and that the field names are correct.


Instruction:
Write a Python 3 script to
create a tree. Complement LookAhead Neg Until

Response:
def add_mod(self, seq, mod):
    """Create a tree.{Complement, LookAhead, Neg, Until}"""
    modstr = self.value(mod)
    if modstr == '~':
        seq.parser_tree = parsing.Complement(seq.parser_tree)
    elif modstr == '!!':
        seq.parser_tree = parsing.LookAhead(seq.parser_tree)
    elif modstr == '!':
        seq.parser_tree = parsing.Neg(seq.parser_tree)
    elif modstr == '->':
        seq.parser_tree = parsing.Until(seq.parser_tree)
    return True


Instruction:
Explain what the following Python 3 code does
def add_ruleclause_name(self, ns_name, rid) -> bool:
    ns_name.parser_tree = parsing.Rule(self.value(rid))
    return True

Response: Add a ruleclause name to the namespace.


Instruction:
Explain what the following Python 3 code does
def add_rules(self, bnf, r) -> bool:
    bnf[r.rulename] = r.parser_tree
    return True

Response: Attach a parser tree to the dict of rules


Instruction:
Create a Python 3 function for
adding the rule name rn to the rule.

Response:
def add_rule(self, rule, rn, alts) -> bool:
    """Add the rule name"""
    rule.rulename = self.value(rn)
    rule.parser_tree = alts.parser_tree
    return True


Instruction:
Can you create a Python 3 function that
adds a sequence to the tree.

Response:
def add_sequences(self, sequences, cla) -> bool:
    """Create a tree.Seq"""
    if not hasattr(sequences, 'parser_tree'):
        # forward sublevel of sequence as is
        sequences.parser_tree = cla.parser_tree
    else:
        oldnode = sequences
        if isinstance(oldnode.parser_tree, parsing.Seq):
            oldpt = list(oldnode.parser_tree.ptlist)
        else:
            oldpt = [oldnode.parser_tree]
        oldpt.append(cla.parser_tree)
        sequences.parser_tree = parsing.Seq(*tuple(oldpt))
    return True


Instruction:
Can you write a function in Python 3 where it
adds an alternative to the list of alternatives.

Response:
def add_alt(self, alternatives, alt) -> bool:
    """Create a tree.Alt"""
    if not hasattr(alternatives, 'parser_tree'):
        # forward sublevel of alt as is
        if hasattr(alt, 'parser_tree'):
            alternatives.parser_tree = alt.parser_tree
        else:
            alternatives.parser_tree = alt
    else:
        oldnode = alternatives
        if isinstance(oldnode.parser_tree, parsing.Alt):
            oldpt = list(oldnode.parser_tree.ptlist)
        else:
            oldpt = [oldnode.parser_tree]
        oldpt.append(alt.parser_tree)
        alternatives.parser_tree = parsing.Alt(*tuple(oldpt))
    return True


Instruction:
Explain what the following Python 3 code does
def add_read_sqstring(self, sequence, s):
    v = self.value(s).strip("'")
    if len(v) > 1:
        sequence.parser_tree = parsing.Text(v)
        return True
    sequence.parser_tree = parsing.Char(v)
    return True

Response: Add a read_char or read_text primitive from simple quote string


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_range(self, sequence, begin, end):
    sequence.parser_tree = parsing.Range(self.value(begin).strip("'"),
                                         self.value(end).strip("'"))
    return True

Response: Add a read_range primitive


Instruction:
Create a Python 3 function for
adding a repeater to the previous sequence

Response:
def add_rpt(self, sequence, mod, pt):
    """Add a repeater to the previous sequence"""
    modstr = self.value(mod)
    if modstr == '!!':
        # cursor on the REPEATER
        self._stream.restore_context()
        # log the error
        self.diagnostic.notify(
            error.Severity.ERROR,
            "Cannot repeat a lookahead rule",
            error.LocationInfo.from_stream(self._stream, is_error=True)
        )
        raise self.diagnostic
    if modstr == '!':
        # cursor on the REPEATER
        self._stream.restore_context()
        # log the error
        self.diagnostic.notify(
            error.Severity.ERROR,
            "Cannot repeat a negated rule",
            error.LocationInfo.from_stream(self._stream, is_error=True)
        )
        raise self.diagnostic
    oldnode = sequence
    sequence.parser_tree = pt.functor(oldnode.parser_tree)
    return True


Instruction:
Write a Python 3 function that can
create a tree. Capture object for the cpt value.

Response:
def add_capture(self, sequence, cpt):
    """Create a tree.Capture"""
    cpt_value = self.value(cpt)
    sequence.parser_tree = parsing.Capture(cpt_value, sequence.parser_tree)
    return True


Instruction:
Can you implement a function in Python 3 that
creates a tree. Bind object for the cpt value.

Response:
def add_bind(self, sequence, cpt):
    """Create a tree.Bind"""
    cpt_value = self.value(cpt)
    sequence.parser_tree = parsing.Bind(cpt_value, sequence.parser_tree)
    return True


Instruction:
Explain what the following Python 3 code does
def add_hook(self, sequence, h):
    sequence.parser_tree = parsing.Hook(h.name, h.listparam)
    return True

Response: Create a tree. Hook


Instruction:
Can you create a Python 3 function that
parses a int in parameter list

Response:
def param_num(self, param, n):
    """Parse a int in parameter list"""
    param.pair = (int(self.value(n)), int)
    return True


Instruction:
Explain what the following Python 3 code does
def param_str(self, param, s):
    param.pair = (self.value(s).strip('"'), str)
    return True

Response: Parse a str in parameter list


Instruction:
Can you tell what is the following Python 3 function doing
def param_char(self, param, c):
    param.pair = (self.value(c).strip("'"), str)
    return True

Response: Parse a char in parameter list


Instruction:
Can you generate a brief explanation for the following Python 3 code
def param_id(self, param, i):
    param.pair = (self.value(i), parsing.Node)
    return True

Response: Parse a node name in parameter list


Instruction:
Implement a function in Python 3 to
parse a hook name

Response:
def hook_name(self, hook, n):
    """Parse a hook name"""
    hook.name = self.value(n)
    hook.listparam = []
    return True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def hook_param(self, hook, p):
    hook.listparam.append(p.pair)
    return True

Response: Parse a hook parameter


Instruction:
Can you generate the documentation for the following Python 3 function
def add_directive2(self, sequence, d, s):
    sequence.parser_tree = parsing.Directive2(
        d.name,
        d.listparam,
        s.parser_tree
    )
    return True

Response: Add a directive in the sequence


Instruction:
Make a summary of the following Python 3 code
def add_directive(self, sequence, d, s):
    if d.name in meta._directives:
        the_class = meta._directives[d.name]
        sequence.parser_tree = parsing.Directive(the_class(), d.listparam,
                                                 s.parser_tree)
    elif d.name in meta._decorators:
        the_class = meta._decorators[d.name]
        sequence.parser_tree = parsing.Decorator(the_class, d.listparam,
                                                 s.parser_tree)
    else:
        raise TypeError("Unkown directive or decorator %s" % d.name)
    return True

Response: Add a directive in the sequence


Instruction:
Implement a function in Python 3 to
parse the DSL and provide a dictionnaries of all resulting rules.

Response:
def get_rules(self) -> parsing.Node:
        """
        Parse the DSL and provide a dictionnaries of all resulting rules.
        Call by the MetaGrammar class.

        TODO: could be done in the rules property of parsing.BasicParser???
        """
        res = None
        try:
            res = self.eval_rule('bnf_dsl')
            if not res:
                # we fail to parse, but error is not set
                self.diagnostic.notify(
                    error.Severity.ERROR,
                    "Parse error in '%s' in EBNF bnf" % self._lastRule,
                    error.LocationInfo.from_maxstream(self._stream)
                )
                raise self.diagnostic
        except error.Diagnostic as d:
            d.notify(
                error.Severity.ERROR,
                "Parse error in '%s' in EBNF bnf" % self._lastRule
            )
            raise d
        return res


Instruction:
Make a summary of the following Python 3 code
def to_yml(self):
    pp = fmt.tab([])
    to_yml_item(self, pp.lsdata, "")
    return str(pp)

Response: Returns a string representation of the node as YML.


Instruction:
Can you implement a function in Python 3 that
consumes comments and whitespace characters.

Response:
def ignore_cxx(self) -> bool:
    """Consume comments and whitespace characters."""
    self._stream.save_context()
    while not self.read_eof():
        idxref = self._stream.index
        if self._stream.peek_char in " \t\v\f\r\n":
            while (not self.read_eof()
                    and self._stream.peek_char in " \t\v\f\r\n"):
                self._stream.incpos()
        if self.peek_text("//"):
            while not self.read_eof() and not self.peek_char("\n"):
                self._stream.incpos()
            if not self.read_char("\n") and self.read_eof():
                return self._stream.validate_context()
        if self.peek_text("/*"):
            while not self.read_eof() and not self.peek_text("*/"):
                self._stream.incpos()
            if not self.read_text("*/") and self.read_eof():
                return self._stream.restore_context()
        if idxref == self._stream.index:
            break
    return self._stream.validate_context()


Instruction:
Implement a function in Python 3 to
add a state to the internal state list

Response:
def add_state(self, s: State):
        """
        all state in the register have a uid
        """
        ids = id(s)
        uid = len(self.states)
        if ids not in self.states:
            self.states[ids] = (uid, s)


Instruction:
Implement a Python 3 function for
returning a. dot representation of the register.

Response:
def to_dot(self) -> str:
        """
        Provide a '.dot' representation of all State in the register.
        """
        txt = ""
        txt += "digraph S%d {\n" % id(self)
        if self.label is not None:
            txt += '\tlabel="%s";\n' % (self.label + '\l').replace('\n', '\l')
        txt += "\trankdir=LR;\n"
        #txt += '\tlabelloc="t";\n'
        txt += '\tgraph [labeljust=l, labelloc=t, nojustify=true];\n'
        txt += "\tesep=1;\n"
        txt += '\tranksep="equally";\n'
        txt += "\tnode [shape = circle];\n"
        txt += "\tsplines = ortho;\n"
        for s in self.states.values():
            txt += s[1].to_dot()
        txt += "}\n"
        return txt


Instruction:
Here you have a function in Python 3, explain what it does
def to_dot_file(self, fname: str):
        with open(fname, 'w') as f:
            f.write(self.to_dot())

Response: write a. dot file.


Instruction:
Implement a function in Python 3 to
write a. png file.

Response:
def to_png_file(self, fname: str):
        """
        write a '.png' file.
        """
        cmd = pipes.Template()
        cmd.append('dot -Tpng > %s' % fname, '-.')
        with cmd.open('pipefile', 'w') as f:
            f.write(self.to_dot())


Instruction:
Create a Python 3 function to
provide a useful representation of the register.

Response:
def to_fmt(self) -> str:
        """
        Provide a useful representation of the register.
        """
        infos = fmt.end(";\n", [])
        s = fmt.sep(', ', [])
        for ids in sorted(self.states.keys()):
            s.lsdata.append(str(ids))
        infos.lsdata.append(fmt.block('(', ')', [s]))
        infos.lsdata.append("events:" + repr(self.events))
        infos.lsdata.append(
            "named_events:" + repr(list(self.named_events.keys()))
        )
        infos.lsdata.append("uid_events:" + repr(list(self.uid_events.keys())))
        return infos


Instruction:
Can you generate a brief explanation for the following Python 3 code
def nextstate(self, newstate, treenode=None, user_data=None):
        if newstate is None:
            return self
        if isinstance(newstate, State) and id(newstate) != id(self):
            return newstate
        elif isinstance(newstate, StateEvent):
            self.state_register.named_events[newstate.name] = True
            return newstate.st
        elif isinstance(newstate, StatePrecond):
            return newstate.st
        elif isinstance(newstate, StateHook):
            # final API using PSL
            newstate.call(treenode, user_data)
            return newstate.st
        return self

Response: Return the next state in the sequence.


Instruction:
How would you explain what the following Python 3 function does
def checkValue(self, v) -> State:
        if self.wild_value:
            return self.nextstate(self.values['*'])
        elif str(v) in self.values:
            return self.nextstate(self.values[str(v)])
        return self

Response: check the value of the attribute


Instruction:
Given the following Python 3 function, write the documentation
def resetLivingState(self):
        # TODO: add some test to control number of instanciation of LivingState
        # clean all living state on S0
        must_delete = []
        l = len(self.ls)
        for idx, ls in zip(range(l), self.ls):
            # TODO: alive by default on False, change to True on the first match
            ids = id(ls[1].thestate())
            if ids == id(ls[0]) and (ls[1].have_finish or not ls[1].alive):
                must_delete.append(idx)
            elif ls[1].alive:
                ls[1].alive = False
        for delete in reversed(must_delete):
            self.ls.pop(delete)
        self.init_all()

Response: Reset all the living states on the S0 of each StateRegister


Instruction:
Can you generate the documentation for the following Python 3 function
def infer_type(self, init_scope: Scope=None, diagnostic=None):
        # create the first .infer_node
        if not hasattr(self, 'infer_node'):
            self.infer_node = InferNode(init_scope)
        elif init_scope is not None:
            # only change the root scope
            self.infer_node.scope_node = init_scope
        # get algo
        type_algo = self.type_algos()
        if diagnostic is None and hasattr(self, 'diagnostic'):
            diagnostic = self.diagnostic
        type_algo[0](type_algo[1], diagnostic)

Response: Do inference of the type of the current node.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def feedback(self, diagnostic=None):
        # get algo
        type_algo = self.type_algos()
        if diagnostic is None and hasattr(self, 'diagnostic'):
            diagnostic = self.diagnostic
        type_algo[2](diagnostic)

Response: Write infos into diagnostic object.


Instruction:
Write a Python 3 script to
infer type on the block.

Response:
def infer_block(self, body, diagnostic=None):
        """
        Infer type on block is to type each of is sub-element
        """
        # RootBlockStmt has his own .infer_node (created via infer_type)
        for e in body:
            e.infer_node = InferNode(parent=self.infer_node)
            e.infer_type(diagnostic=diagnostic)


Instruction:
Write a Python 3 function for
inferring the type of the subexpression expr.

Response:
def infer_subexpr(self, expr, diagnostic=None):
        """
        Infer type on the subexpr
        """
        expr.infer_node = InferNode(parent=self.infer_node)
        expr.infer_type(diagnostic=diagnostic)


Instruction:
Given the following Python 3 function, write the documentation
def infer_id(self, ident, diagnostic=None):
        # check if ID is declared
        #defined = self.type_node.get_by_symbol_name(ident)
        defined = self.infer_node.scope_node.get_by_symbol_name(ident)
        if len(defined) > 0:
            # set from matchings declarations
            #self.type_node.update(defined)
            self.infer_node.scope_node.update(defined)
        else:
            diagnostic.notify(
                Severity.ERROR,
                "%s never declared" % self.value,
                self.info
            )

Response: Infer type from an ID!


Instruction:
Can you write a function in Python 3 where it
infers type from an LITERAL!

Response:
def infer_literal(self, args, diagnostic=None):
        """
        Infer type from an LITERAL!
        Type of literal depend of language.
        We adopt a basic convention
        """
        literal, t = args
        #self.type_node.add(EvalCtx.from_sig(Val(literal, t)))
        self.infer_node.scope_node.add(EvalCtx.from_sig(Val(literal, t)))


Instruction:
Implement a function in Python 3 to
dump the node cache. For debug.

Response:
def dump_nodes(self):
    """
    Dump tag,rule,id and value cache. For debug.

    example::

        R = [
            #dump_nodes
        ]

    """
    print("DUMP NODE LOCAL INFOS")
    try:
        print("map Id->node name")
        for k, v in self.id_cache.items():
            print("[%d]=%s" % (k, v))
        print("map tag->capture infos")
        for k, v in self.tag_cache.items():
            print("[%s]=%s" % (k, v))
        print("map nodes->tag resolution")
        for k, v in self.rule_nodes.items():
            txt = "['%s']=%d" % (k, id(v))
            if k in self.tag_cache:
                tag = self.tag_cache[k]
                txt += " tag <%s>" % tag
                k = "%d:%d" % (tag._begin, tag._end)
                if k in self._stream.value_cache:
                    txt += " cache <%s>" % self._stream.value_cache[k]
            print(txt)
    except Exception as err:
        print("RECV Exception %s" % err)
    import sys
    sys.stdout.flush()
    return True


Instruction:
Make a summary of the following Python 3 code
def list_dataset_uris(cls, base_uri, config_path):
        uri_list = []

        parse_result = generous_parse_uri(base_uri)
        bucket_name = parse_result.netloc
        bucket = boto3.resource('s3').Bucket(bucket_name)

        for obj in bucket.objects.filter(Prefix='dtool').all():
            uuid = obj.key.split('-', 1)[1]
            uri = cls.generate_uri(None, uuid, base_uri)

            storage_broker = cls(uri, config_path)
            if storage_broker.has_admin_metadata():
                uri_list.append(uri)

        return uri_list

Response: Return list containing URIs with base URI.


Instruction:
Here you have a function in Python 3, explain what it does
def get_item_abspath(self, identifier):
        admin_metadata = self.get_admin_metadata()
        uuid = admin_metadata["uuid"]
        # Create directory for the specific dataset.
        dataset_cache_abspath = os.path.join(self._s3_cache_abspath, uuid)
        mkdir_parents(dataset_cache_abspath)

        bucket_fpath = self.data_key_prefix + identifier
        obj = self.s3resource.Object(self.bucket, bucket_fpath)
        relpath = obj.get()['Metadata']['handle']
        _, ext = os.path.splitext(relpath)

        local_item_abspath = os.path.join(
            dataset_cache_abspath,
            identifier + ext
        )
        if not os.path.isfile(local_item_abspath):

            tmp_local_item_abspath = local_item_abspath + ".tmp"
            self.s3resource.Bucket(self.bucket).download_file(
                bucket_fpath,
                tmp_local_item_abspath
            )
            os.rename(tmp_local_item_abspath, local_item_abspath)

        return local_item_abspath

Response: Return absolute path at which the item content can be accessed.


Instruction:
How would you explain what the following Python 3 function does
def list_overlay_names(self):

        bucket = self.s3resource.Bucket(self.bucket)

        overlay_names = []
        for obj in bucket.objects.filter(
            Prefix=self.overlays_key_prefix
        ).all():

            overlay_file = obj.key.rsplit('/', 1)[-1]
            overlay_name, ext = overlay_file.split('.')
            overlay_names.append(overlay_name)

        return overlay_names

Response: Return list of overlay names.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_item_metadata(self, handle, key, value):

        identifier = generate_identifier(handle)
        suffix = '{}.{}.json'.format(identifier, key)
        bucket_fpath = self.fragments_key_prefix + suffix

        self.s3resource.Object(self.bucket, bucket_fpath).put(
            Body=json.dumps(value)
        )

Response: Store the given key value pair for the item associated with handle.


Instruction:
Write a Python 3 function for
returning iterator over item handles.

Response:
def iter_item_handles(self):
        """Return iterator over item handles."""

        bucket = self.s3resource.Bucket(self.bucket)

        for obj in bucket.objects.filter(Prefix=self.data_key_prefix).all():
            relpath = obj.get()['Metadata']['handle']

            yield relpath


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_item_metadata(self, handle):

        bucket = self.s3resource.Bucket(self.bucket)

        metadata = {}

        identifier = generate_identifier(handle)
        prefix = self.fragments_key_prefix + '{}'.format(identifier)
        for obj in bucket.objects.filter(Prefix=prefix).all():
            metadata_key = obj.key.split('.')[-2]
            response = obj.get()
            value_as_string = response['Body'].read().decode('utf-8')
            value = json.loads(value_as_string)

            metadata[metadata_key] = value

        return metadata

Response: Get all the metadata associated with handle.


Instruction:
Can you create a Python 3 function that
generates code for a rule.

Response:
def parserrule_topython(parser: parsing.BasicParser,
                        rulename: str) -> ast.FunctionDef:
    """Generates code for a rule.

    def rulename(self):
        <code for the rule>
        return True
    """
    visitor = RuleVisitor()
    rule = parser._rules[rulename]
    fn_args = ast.arguments([ast.arg('self', None)], None, None, [], None,
                            None, [], [])
    body = visitor._clause(rule_topython(rule))
    body.append(ast.Return(ast.Name('True', ast.Load())))
    return ast.FunctionDef(rulename, fn_args, body, [], None)


Instruction:
Can you write a function in Python 3 where it
creates the appropriate scope exiting statement.

Response:
def __exit_scope(self) -> ast.stmt:
        """Create the appropriate scope exiting statement.

        The documentation only shows one level and always uses
        'return False' in examples.

        'raise AltFalse()' within a try.
        'break' within a loop.
        'return False' otherwise.
        """
        if self.in_optional:
            return ast.Pass()
        if self.in_try:
            return ast.Raise(
                ast.Call(ast.Name('AltFalse', ast.Load()), [], [], None, None),
                None)
        if self.in_loop:
            return ast.Break()
        return ast.Return(ast.Name('False', ast.Load()))


Instruction:
Write a Python 3 function that can
normalize a test expression into a statements list.

Response:
def _clause(self, pt: parsing.ParserTree) -> [ast.stmt]:
        """Normalize a test expression into a statements list.

        Statements list are returned as-is.
        Expression is packaged as:
        if not expr:
            return False
        """
        if isinstance(pt, list):
            return pt
        return [ast.If(ast.UnaryOp(ast.Not(), pt),
                       [self.__exit_scope()],
                       [])]


Instruction:
Create a Python 3 function for
generating python code calling the function.

Response:
def visit_Call(self, node: parsing.Call) -> ast.expr:
        """Generates python code calling the function.

        fn(*args)
        """
        return ast.Call(
            ast.Attribute(
                ast.Name('self', ast.Load),
                node.callObject.__name__,
                ast.Load()),
            [ast.Str(param) for param in node.params],
            [],
            None,
            None)


Instruction:
How would you explain what the following Python 3 function does
def visit_CallTrue(self, node: parsing.CallTrue) -> ast.expr:
        return ast.Lambda(
            ast.arguments([], None, None, [], None, None, [], []),
            ast.BoolOp(
                ast.Or(),
                [
                    self.visit_Call(node),
                    ast.Name('True', ast.Load())]))

Response: Generates python code calling the function and returning True.


Instruction:
Given the following Python 3 function, write the documentation
def visit_Hook(self, node: parsing.Hook) -> ast.expr:
        return ast.Call(
            ast.Attribute(
                ast.Name('self', ast.Load()), 'evalHook', ast.Load()),
            [
                ast.Str(node.name),
                ast.Subscript(
                    ast.Attribute(
                        ast.Name('self', ast.Load()), 'ruleNodes', ast.Load()),
                    ast.Index(ast.UnaryOp(ast.USub(), ast.Num(1))),
                    ast.Load())],
            [],
            None,
            None)

Response: Generates python code calling a hook.


Instruction:
Can you write a function in Python 3 where it
generates python code calling a rule. self. evalRule ( rulename

Response:
def visit_Rule(self, node: parsing.Rule) -> ast.expr:
        """Generates python code calling a rule.

        self.evalRule('rulename')
        """
        return ast.Call(
            ast.Attribute(ast.Name('self', ast.Load()),
                          'evalRule', ast.Load()),
            [ast.Str(node.name)], [], None, None)


Instruction:
Implement a function in Python 3 to
return a list of statements that can be used to capture text consumed by a clause.

Response:
def visit_Capture(self, node: parsing.Capture) -> [ast.stmt] or ast.expr:
        """Generates python code to capture text consumed by a clause.

        #If all clauses can be inlined
        self.beginTag('tagname') and clause and self.endTag('tagname')

        if not self.beginTag('tagname'):
            return False
        <code for the clause>
        if not self.endTag('tagname'):
            return False
        """
        begintag = ast.Attribute(
            ast.Name('self', ast.Load()), 'beginTag', ast.Load())
        endtag = ast.Attribute(
            ast.Name('self', ast.Load()), 'endTag', ast.Load())
        begin = ast.Call(begintag, [ast.Str(node.tagname)], [], None, None)
        end = ast.Call(endtag, [ast.Str(node.tagname)], [], None, None)
        result = [begin, self.visit(node.pt), end]
        for clause in result:
            if not isinstance(clause, ast.expr):
                break
        else:
            return ast.BoolOp(ast.And(), result)
        res = []
        for stmt in map(self._clause, result):
            res.extend(stmt)
        return res


Instruction:
Can you implement a function in Python 3 that
generates python code for a scope.

Response:
def visit_Scope(self, node: parsing.Capture) -> [ast.stmt] or ast.expr:
        """Generates python code for a scope.

        if not self.begin():
            return False
        res = self.pt()
        if not self.end():
            return False
        return res
        """
        return ast.Name('scope_not_implemented', ast.Load())
        raise NotImplementedError()


Instruction:
Given the following Python 3 function, write the documentation
def visit_Alt(self, node: parsing.Alt) -> [ast.stmt]:
        clauses = [self.visit(clause) for clause in node.ptlist]
        for clause in clauses:
            if not isinstance(clause, ast.expr):
                break
        else:
            return ast.BoolOp(ast.Or(), clauses)
        res = ast.Try([], [ast.ExceptHandler(
            ast.Name('AltTrue', ast.Load()), None, [ast.Pass()])], [], [])
        alt_true = [ast.Raise(ast.Call(
            ast.Name('AltTrue', ast.Load()), [], [], None, None), None)]
        alt_false = [ast.ExceptHandler(
            ast.Name('AltFalse', ast.Load()), None, [ast.Pass()])]
        self.in_try += 1
        for clause in node.ptlist:
            res.body.append(
                ast.Try(self._clause(self.visit(clause)) + alt_true,
                        alt_false, [], []))
        self.in_try -= 1
        res.body.append(self.__exit_scope())
        return [res]

Response: Generates python code for alternatives.


Instruction:
Can you generate the documentation for the following Python 3 function
def visit_Seq(self, node: parsing.Seq) -> [ast.stmt] or ast.expr:
        exprs, stmts = [], []
        for clause in node.ptlist:
            clause_ast = self.visit(clause)
            if isinstance(clause_ast, ast.expr):
                exprs.append(clause_ast)
            else:
                if exprs:
                    stmts.extend(self.combine_exprs_for_clauses(exprs))
                    exprs = []
                stmts.extend(self._clause(clause_ast))
        if not stmts:
            return ast.BoolOp(ast.And(), exprs)
        if exprs:
            stmts.extend(self.combine_exprs_for_clauses(exprs))
        return stmts

Response: Generates python code for the Seq node.


Instruction:
Explain what the following Python 3 code does
def visit_RepOptional(self, node: parsing.RepOptional) -> ([ast.stmt] or
                                                               ast.expr):
        """Generates python code for an optional clause.

        <code for the clause>
        """
        cl_ast = self.visit(node.pt)
        if isinstance(cl_ast, ast.expr):
            return ast.BoolOp(ast.Or(), [cl_ast, ast.Name('True', ast.Load())])
        self.in_optional += 1
        cl_ast = self.visit(node.pt)
        self.in_optional -= 1
        return cl_ast

Response: Returns the AST node for an optional clause.


Instruction:
Given the following Python 3 function, write the documentation
def visit_Rep0N(self, node: parsing.Rep0N) -> [ast.stmt]:
        cl_ast = self.visit(node.pt)
        if isinstance(cl_ast, ast.expr):
            return [ast.While(cl_ast, [ast.Pass()], [])]
        self.in_loop += 1
        clause = self._clause(self.visit(node.pt))
        self.in_loop -= 1
        return [ast.While(ast.Name('True', ast.Load()), clause, [])]

Response: Return an ast. stmt for repeated 0 or more times.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def visit_Rep1N(self, node: parsing.Rep0N) -> [ast.stmt]:
        clause = self.visit(node.pt)
        if isinstance(clause, ast.expr):
            return (self._clause(clause) + self.visit_Rep0N(node))
        self.in_loop += 1
        clause = self._clause(self.visit(node.pt))
        self.in_loop -= 1
        return self._clause(self.visit(node.pt)) + [
            ast.While(ast.Name('True', ast.Load()), clause, [])]

Response: Return the AST node s children as a list of statements.


Instruction:
Can you implement a function in Python 3 that
returns a list of all synthese deputes in a given month.

Response:
def synthese(self, month=None):
        """
        month format: YYYYMM
        """
        if month is None and self.legislature == '2012-2017':
            raise AssertionError('Global Synthesis on legislature does not work, see https://github.com/regardscitoyens/nosdeputes.fr/issues/69')

        if month is None:
            month = 'data'

        url = '%s/synthese/%s/%s' % (self.base_url, month, self.format)

        data = requests.get(url).json()
        return [depute[self.ptype] for depute in data[self.ptype_plural]]


Instruction:
Here you have a function in Python 3, explain what it does
def catend(dst: str, src: str, indent) -> str:
    res = dst
    txtsrc = src
    if not isinstance(src, str):
        txtsrc = str(src)
    for c in list(txtsrc):
        if len(res) > 0 and res[-1] == '\n':
            res += (indentable.char_indent * indentable.num_indent) * \
                   (indent - 1) + c
        else:
            res += c
    return res

Response: cat two strings but handle \ n for tabulation


Instruction:
Can you tell what is the following Python 3 function doing
def list_set_indent(lst: list, indent: int=1):
    for i in lst:
        if isinstance(i, indentable):
            i.set_indent(indent)
        if isinstance(i, list):
            list_set_indent(i, indent)

Response: recurs into list for indentation


Instruction:
Here you have a function in Python 3, explain what it does
def list_to_str(lst: list, content: str, indent: int=1):
    for i in lst:
        if isinstance(i, indentable):
            content = i.to_str(content, indent)
        elif isinstance(i, list):
            content = list_to_str(i, content, indent)
        elif isinstance(i, str):
            content = catend(content, i, indent)
    return content

Response: recurs into list for string computing


Instruction:
Given the following Python 3 function, write the documentation
def echo_nodes(self, *rest):
    txt = ""
    for thing in rest:
        if isinstance(thing, Node):
            txt += self.value(thing)
        else:
            txt += str(thing)
    print(txt)
    return True

Response:
Print nodes.

    example::

        R = [
            In : node #echo("coucou", 12, node)
        ]


Instruction:
Explain what the following Python 3 code does
def populate_from_sequence(seq: list, r: ref(Edge), sr: state.StateRegister):
    base_state = r
    # we need to detect the last state of the sequence
    idxlast = len(seq) - 1
    idx = 0
    for m in seq:
        # alternatives are represented by builtin list
        if isinstance(m, list):
            # so recursively connect all states of each alternative sequences.
            for item in m:
                populate_from_sequence(item, r, sr)
        elif isinstance(m, MatchExpr):
            # from the current state, have we a existing edge for this event?
            eX = r().get_next_edge(m)
            if eX is None:
                sX = None
                if idx != idxlast:
                    sX = state.State(sr)
                    sX.matchDefault(base_state().s)
                else:
                    # last state of sequence return to the base
                    sX = base_state().s
                eX = Edge(sX)
                r().next_edge[id(sX)] = eX
                m.attach(r().s, sX, sr)
            r = ref(eX)
        idx += 1

Response: function that connects each other one sequence of MatchExpr into the base state.


Instruction:
Make a summary of the following Python 3 code
def populate_state_register(all_seq: [list], sr: state.StateRegister) -> Edge:
    # Basic State
    s0 = state.State(sr)
    # loop on himself
    s0.matchDefault(s0)
    # this is default
    sr.set_default_state(s0)
    # use Edge to store connection
    e0 = Edge(s0)
    for seq in all_seq:
        r = ref(e0)
        # merge all sequences into one tree automata
        populate_from_sequence(seq, r, sr)
    # return edge for debug purpose
    return e0

Response:
function that creates a state for all instance
        of MatchExpr in the given list and connect each others to the state register.


Instruction:
Create a Python 3 function to
test if a node set with setint or setstr equal a certain value

Response:
def pred_eq(self, n, val):
    """
    Test if a node set with setint or setstr equal a certain value

    example::

        R = [
            __scope__:n
            ['a' #setint(n, 12) | 'b' #setint(n, 14)]
            C
            [#eq(n, 12) D]
        ]

    """
    v1 = n.value
    v2 = val
    if hasattr(val, 'value'):
        v2 = val.value
    if isinstance(v1, int) and not isinstance(v2, int):
        return v1 == int(v2)
    return v1 == v2


Instruction:
Explain what the following Python 3 code does
def from_string(bnf: str, entry=None, *optional_inherit) -> Grammar:
    inherit = [Grammar] + list(optional_inherit)
    scope = {'grammar': bnf, 'entry': entry}
    return build_grammar(tuple(inherit), scope)

Response: Create a Grammar from a string


Instruction:
Can you write a function in Python 3 where it
creates a Grammar from a file

Response:
def from_file(fn: str, entry=None, *optional_inherit) -> Grammar:
    """
    Create a Grammar from a file
    """
    import os.path
    if os.path.exists(fn):
        f = open(fn, 'r')
        bnf = f.read()
        f.close()
        inherit = [Grammar] + list(optional_inherit)
        scope = {'grammar': bnf, 'entry': entry, 'source': fn}
        return build_grammar(tuple(inherit), scope)
    raise Exception("File not Found!")


Instruction:
How would you explain what the following Python 3 function does
def parse(self, source: str=None, entry: str=None) -> parsing.Node:
        self.from_string = True
        if source is not None:
            self.parsed_stream(source)
        if entry is None:
            entry = self.entry
        if entry is None:
            raise ValueError("No entry rule name defined for {}".format(
                self.__class__.__name__))
        return self._do_parse(entry)

Response: Parse the source using the grammar


Instruction:
Write a Python 3 script to
parse a file using the grammar

Response:
def parse_file(self, filename: str, entry: str=None) -> parsing.Node:
        """Parse filename using the grammar"""
        self.from_string = False
        import os.path
        with open(filename, 'r') as f:
            self.parsed_stream(f.read(), os.path.abspath(filename))
        if entry is None:
            entry = self.entry
        if entry is None:
            raise ValueError("No entry rule name defined for {}".format(
                self.__class__.__name__))
        return self._do_parse(entry)


Instruction:
Explain what the following Python 3 code does
def set_node(self, dst, src):
    if not isinstance(src, Node):
        dst.value = src
    else:
        dst.set(src)
        idsrc = id(src)
        iddst = id(dst)
        if iddst not in self.id_cache:
            print("DST: %s" % repr(dst))
            print("RULE_NODES %s" % repr(self.rule_nodes))
            print("IDCACHE %s" % repr(self.id_cache))
        if idsrc in self.id_cache:
            k = self.id_cache[idsrc]
            k2 = self.id_cache[iddst]
            if k in self.rule_nodes:
                self.tag_cache[k2] = self.tag_cache[k]
    return True

Response: Basically copy one node to another.


Instruction:
How would you explain what the following Python 3 function does
def set_node_as_int(self, dst, src):
    dst.value = self.value(src)
    return True

Response: Set a node to a value captured from another node.


Instruction:
Create a Python 3 function to
get the value of subnode in the tree

Response:
def get_subnode(self, dst, ast, expr):
    """
        get the value of subnode

        example::

            R = [
                __scope__:big  getsomethingbig:>big
                #get(_, big, '.val') // copy big.val into _
            ]
    """
    dst.value = eval('ast' + expr)
    return True


Instruction:
Can you generate a brief explanation for the following Python 3 code
def default_serializer(o):
    defs = (
        ((datetime.date, datetime.time),
         lambda x: x.isoformat(), ),
        ((datetime.datetime, ),
         lambda x: dt2utc_timestamp(x), ),
    )
    for types, fun in defs:
        if isinstance(o, types):
            return fun(o)

Response: Default serializer for json.


Instruction:
Explain what the following Python 3 code does
def _get_depositions(user=None, type=None):
    from invenio.modules.workflows.models import BibWorkflowObject, Workflow
    from invenio.modules.deposit.models import InvalidDepositionType
    from flask import current_app
    from invenio.ext.sqlalchemy import db
    from invenio.modules.deposit.models import Deposition
    params = [
        Workflow.module_name == 'webdeposit',
    ]

    if user:
        params.append(BibWorkflowObject.id_user == user.get_id())
    else:
        params.append(BibWorkflowObject.id_user != 0)

    if type:
        params.append(Workflow.name == type.get_identifier())

    objects = BibWorkflowObject.query.join("workflow").options(
        db.contains_eager('workflow')).filter(*params)

    def _create_obj(o):
        try:
            obj = Deposition(o)
        except InvalidDepositionType as err:
            current_app.logger.exception(err)
            return None
        if type is None or obj.type == type:
            return obj
        return None

    def mapper_filter(objs):
        for o in objs:
            o = _create_obj(o)
            if o is not None:
                yield o

    return mapper_filter(objects)

Response: Get list of depositions for a user and type.


Instruction:
Write a Python 3 script for
getting all items in the cluster.

Response:
def get(query, from_date, limit=0, **kwargs):
    """Get deposits."""
    dep_generator = _get_depositions()
    total_depids = 1  # Count of depositions is hard to determine

    # If limit provided, serve only first n=limit items
    if limit > 0:
        dep_generator = islice(dep_generator, limit)
        total_depids = limit
    return total_depids, dep_generator


Instruction:
How would you explain what the following Python 3 function does
def dump(deposition, from_date, with_json=True, latest_only=False, **kwargs):
    # Serialize the __getstate__ and fall back to default serializer
    dep_json = json.dumps(deposition.__getstate__(),
                          default=default_serializer)
    dep_dict = json.loads(dep_json)
    dep_dict['_p'] = {}
    dep_dict['_p']['id'] = deposition.id
    dep_dict['_p']['created'] = dt2utc_timestamp(deposition.created)
    dep_dict['_p']['modified'] = dt2utc_timestamp(deposition.modified)
    dep_dict['_p']['user_id'] = deposition.user_id
    dep_dict['_p']['state'] = deposition.state
    dep_dict['_p']['has_sip'] = deposition.has_sip()
    dep_dict['_p']['submitted'] = deposition.submitted
    return dep_dict

Response: Dump the deposition object as dictionary.


Instruction:
Make a summary of the following Python 3 code
def _get_recids_invenio12(from_date):
    from invenio.dbquery import run_sql
    return (id[0] for id in run_sql(
        'select id_bibrec from '
        'bibrec_bibdoc as r join bibdoc as d on r.id_bibdoc=d.id '
        'where d.modification_date >=%s',
        (from_date, ), run_on_slave=True))

Response: Get BibDocs for Invenio 1.


Instruction:
Can you create a Python 3 function that
gets BibDocs for Invenio 2.

Response:
def _get_recids_invenio2(from_date):
    """Get BibDocs for Invenio 2."""
    from invenio.legacy.dbquery import run_sql
    return (id[0] for id in run_sql(
        'select id_bibrec from '
        'bibrec_bibdoc as r join bibdoc as d on r.id_bibdoc=d.id '
        'where d.modification_date >=%s',
        (from_date, ), run_on_slave=True))


Instruction:
Create a Python 3 function for
dumping all BibDoc files in a given BibDoc ID.

Response:
def dump_bibdoc(recid, from_date, **kwargs):
    """Dump all BibDoc metadata.

    :param docid: BibDoc ID
    :param from_date: Dump only BibDoc revisions newer than this date.

    :returns: List of version of the BibDoc formatted as a dict
    """
    BibRecDocs, BibDoc = _import_bibdoc()

    bibdocfile_dump = []

    date = datetime.datetime.strptime(from_date, '%Y-%m-%d %H:%M:%S')
    for bibdoc in BibRecDocs(recid).list_bibdocs():
        for version in bibdoc.list_versions():
            bibdoc_version = bibdoc.list_version_files(version)
            for f in bibdoc_version:
                if f.is_icon() or f.md < date:
                    # Don't care about icons
                    # Don't care about files not modified since from_date
                    continue
                bibdocfile_dump.append(dict(
                    bibdocid=f.get_bibdocid(),
                    checksum=f.get_checksum(),
                    comment=f.get_comment(),
                    copyright=(
                        f.get_copyright() if hasattr(f, 'get_copyright')
                        else None),
                    creation_date=datetime_toutc(f.cd).isoformat(),
                    description=f.get_description(),
                    encoding=f.encoding,
                    etag=f.etag,
                    flags=f.flags,
                    format=f.get_format(),
                    full_name=f.get_full_name(),
                    full_path=f.get_full_path(),
                    hidden=f.hidden,
                    license=(
                        f.get_license()if hasattr(f, 'get_license') else None),
                    modification_date=datetime_toutc(f.md).isoformat(),
                    name=f.get_name(),
                    mime=f.mime,
                    path=f.get_path(),
                    recid=f.get_recid(),
                    recids_doctype=f.recids_doctypes,
                    size=f.get_size(),
                    status=f.get_status(),
                    subformat=f.get_subformat(),
                    superformat=f.get_superformat(),
                    type=f.get_type(),
                    url=f.get_url(),
                    version=f.get_version(),
                ))

    return bibdocfile_dump


Instruction:
Implement a function in Python 3 to
get bibdocs to check.

Response:
def get_check():
    """Get bibdocs to check."""
    try:
        from invenio.dbquery import run_sql
    except ImportError:
        from invenio.legacy.dbquery import run_sql

    return (
        run_sql('select count(id) from bibdoc', run_on_slave=True)[0][0],
        [id[0] for id in run_sql('select id from bibdoc', run_on_slave=True)],
    )


Instruction:
Create a Python 3 function for
dumping the oauth2server tokens.

Response:
def dump(obj, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the oauth2server tokens."""
    return dict(id=obj.id,
                client_id=obj.client_id,
                user_id=obj.user_id,
                token_type=obj.token_type,
                access_token=obj.access_token,
                refresh_token=obj.refresh_token,
                expires=dt2iso_or_empty(obj.expires),
                _scopes=obj._scopes,
                is_personal=obj.is_personal,
                is_internal=obj.is_internal)


Instruction:
Create a Python 3 function to
dump the UserEXt object to a list of dictionaries.

Response:
def dump(u, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the UserEXt objects as a list of dictionaries.

    :param u: UserEXT to be dumped.
    :type u: `invenio_accounts.models.UserEXT [Invenio2.x]`
    :returns: User serialized to dictionary.
    :rtype: dict
    """
    return dict(id=u.id, method=u.method, id_user=u.id_user)


Instruction:
Implement a Python 3 function for
dumping the community featuring object as a dictionary.

Response:
def dump(fc, from_date, with_json=True, latest_only=False, **kwargs):
    """Dump the community object as dictionary.

    :param fc: Community featuring to be dumped.
    :type fc: `invenio_communities.models.FeaturedCommunity [Invenio2.x]`
    :returns: Community serialized to dictionary.
    :rtype: dict
    """
    return dict(id=fc.id,
                id_community=fc.id_community,
                start_date=fc.start_date.isoformat())


Instruction:
Implement a function in Python 3 to
get record ids for Invenio 1.

Response:
def _get_modified_recids_invenio12(from_date):
    """Get record ids for Invenio 1."""
    from invenio.search_engine import search_pattern
    from invenio.dbquery import run_sql
    return set((id[0] for id in run_sql(
        'select id from bibrec where modification_date >= %s',
        (from_date, ), run_on_slave=True))), search_pattern


Instruction:
How would you explain what the following Python 3 function does
def _get_modified_recids_invenio2(from_date):
    from invenio.legacy.search_engine import search_pattern
    from invenio.modules.records.models import Record

    date = datetime.datetime.strptime(from_date, '%Y-%m-%d %H:%M:%S')
    return set(
        (x[0]
         for x in Record.query.filter(Record.modification_date >= date).values(
             Record.id))), search_pattern

Response: Get record ids for Invenio 2.


Instruction:
Given the following Python 3 function, write the documentation
def _get_collection_restrictions(collection):
    try:
        from invenio.dbquery import run_sql
        from invenio.access_control_firerole import compile_role_definition
    except ImportError:
        from invenio.modules.access.firerole import compile_role_definition
        from invenio.legacy.dbquery import run_sql

    res = run_sql(
        'SELECT r.firerole_def_src, email '
        'FROM accROLE as r '
        'JOIN accROLE_accACTION_accARGUMENT ON r.id=id_accROLE '
        'JOIN accARGUMENT AS a ON a.id=id_accARGUMENT '
        'JOIN user_accROLE AS u ON r.id=u.id_accROLE '
        'JOIN user ON user.id=u.id_user '
        'WHERE a.keyword="collection" AND '
        'a.value=%s AND '
        'id_accACTION=(select id from accACTION where name="viewrestrcoll")',
        (collection, ), run_on_slave=True
    )
    fireroles = set()
    users = set()

    for f, u in res:
        fireroles.add(compile_role_definition(f))
        users.add(u)

    return {'fireroles': list(fireroles), 'users': users}

Response: Get all restrictions for a given collection users and fireroles.


Instruction:
Explain what the following Python 3 code does
def get_record_collections(recid):
    try:
        from invenio.search_engine import (
            get_all_collections_of_a_record,
            get_restricted_collections_for_recid)
    except ImportError:
        from invenio.legacy.search_engine import (
            get_all_collections_of_a_record,
            get_restricted_collections_for_recid)

    collections = {
        'all':
        get_all_collections_of_a_record(recid, recreate_cache_if_needed=False),
    }
    collections['restricted'] = dict(
        (coll, _get_collection_restrictions(coll))
        for coll in get_restricted_collections_for_recid(
                recid, recreate_cache_if_needed=False))

    return collections

Response: Get all collections the record belong to.


Instruction:
Here you have a function in Python 3, explain what it does
def dump_record_json(marcxml):
    try:
        from invenio.modules.records.api import Record
        d = Record.create(marcxml, 'marc')
        return d.dumps(clean=True)
    except ImportError:
        from invenio.bibfield import create_record
        d = create_record(marcxml, master_format='marc')
        return d.dumps()

Response: Dump JSON of record.