Instruction:
Create a Python 3 function to
convert dotted keys in a nested dictset to a nested dictset.

Response:
def convert_dotted(params):
        """ Convert dotted keys in :params: dictset to a nested dictset.

        E.g. {'settings.foo': 'bar'} -> {'settings': {'foo': 'bar'}}
        """
        if not isinstance(params, dictset):
            params = dictset(params)

        dotted_items = {k: v for k, v in params.items() if '.' in k}

        if dotted_items:
            dicts = [str2dict(key, val) for key, val in dotted_items.items()]
            dotted = six.functools.reduce(merge_dicts, dicts)
            params = params.subset(['-' + k for k in dotted_items.keys()])
            params.update(dict(dotted))

        return params


Instruction:
Can you generate a brief explanation for the following Python 3 code
def prepare_request_params(self, _query_params, _json_params):
        self._query_params = dictset(
            _query_params or self.request.params.mixed())
        self._json_params = dictset(_json_params)

        ctype = self.request.content_type
        if self.request.method in ['POST', 'PUT', 'PATCH']:
            if ctype == 'application/json':
                try:
                    self._json_params.update(self.request.json)
                except simplejson.JSONDecodeError:
                    log.error(
                        "Expecting JSON. Received: '{}'. "
                        "Request: {} {}".format(
                            self.request.body, self.request.method,
                            self.request.url))

            self._json_params = BaseView.convert_dotted(self._json_params)
            self._query_params = BaseView.convert_dotted(self._query_params)

        self._params = self._query_params.copy()
        self._params.update(self._json_params)

Response: Prepare query and update params.


Instruction:
Here you have a function in Python 3, explain what it does
def set_override_rendered(self):
        if '' in self.request.accept:
            self.request.override_renderer = self._default_renderer
        elif 'application/json' in self.request.accept:
            self.request.override_renderer = 'nefertari_json'
        elif 'text/plain' in self.request.accept:
            self.request.override_renderer = 'string'

Response: Set self. request. override_renderer if needed.


Instruction:
Implement a Python 3 function for
wrapping self. index method with ESAggregator.

Response:
def _setup_aggregation(self, aggregator=None):
        """ Wrap `self.index` method with ESAggregator.

        This makes `self.index` to first try to run aggregation and only
        on fail original method is run. Method is wrapped only if it is
        defined and `elasticsearch.enable_aggregations` setting is true.
        """
        from nefertari.elasticsearch import ES
        if aggregator is None:
            aggregator = ESAggregator
        aggregations_enabled = (
            ES.settings and ES.settings.asbool('enable_aggregations'))
        if not aggregations_enabled:
            log.debug('Elasticsearch aggregations are not enabled')
            return

        index = getattr(self, 'index', None)
        index_defined = index and index != self.not_allowed_action
        if index_defined:
            self.index = aggregator(self).wrap(self.index)


Instruction:
Write a Python 3 function for
querying ES collection and return results.

Response:
def get_collection_es(self):
        """ Query ES collection and return results.

        This is default implementation of querying ES collection with
        `self._query_params`. It must return found ES collection
        results for default response renderers to work properly.
        """
        from nefertari.elasticsearch import ES
        return ES(self.Model.__name__).get_collection(**self._query_params)


Instruction:
Can you generate the documentation for the following Python 3 function
def fill_null_values(self):
        if not self.Model:
            log.info("%s has no model defined" % self.__class__.__name__)
            return

        empty_values = self.Model.get_null_values()
        for field, value in empty_values.items():
            if field not in self._json_params:
                self._json_params[field] = value

Response: Fill missing model fields in JSON with null values.


Instruction:
Implement a function in Python 3 to
set public limits if auth is enabled and user is not authenticated.

Response:
def set_public_limits(self):
        """ Set public limits if auth is enabled and user is not
        authenticated.

        Also sets default limit for GET, HEAD requests.
        """
        if self.request.method.upper() in ['GET', 'HEAD']:
            self._query_params.process_int_param('_limit', 20)
        if self._auth_enabled and not getattr(self.request, 'user', None):
            wrappers.set_public_limits(self)


Instruction:
Can you create a Python 3 function that
converts object IDs from self. _json_params to objects if needed.

Response:
def convert_ids2objects(self):
        """ Convert object IDs from `self._json_params` to objects if needed.

        Only IDs that belong to relationship field of `self.Model`
        are converted.
        """
        if not self.Model:
            log.info("%s has no model defined" % self.__class__.__name__)
            return

        for field in self._json_params.keys():
            if not engine.is_relationship_field(field, self.Model):
                continue
            rel_model_cls = engine.get_relationship_cls(field, self.Model)
            self.id2obj(field, rel_model_cls)


Instruction:
How would you explain what the following Python 3 function does
def setup_default_wrappers(self):
        # Index
        self._after_calls['index'] = [
            wrappers.wrap_in_dict(self.request),
            wrappers.add_meta(self.request),
            wrappers.add_object_url(self.request),
        ]

        # Show
        self._after_calls['show'] = [
            wrappers.wrap_in_dict(self.request),
            wrappers.add_meta(self.request),
            wrappers.add_object_url(self.request),
        ]

        # Create
        self._after_calls['create'] = [
            wrappers.wrap_in_dict(self.request),
            wrappers.add_meta(self.request),
            wrappers.add_object_url(self.request),
        ]

        # Update
        self._after_calls['update'] = [
            wrappers.wrap_in_dict(self.request),
            wrappers.add_meta(self.request),
            wrappers.add_object_url(self.request),
        ]

        # Replace
        self._after_calls['replace'] = [
            wrappers.wrap_in_dict(self.request),
            wrappers.add_meta(self.request),
            wrappers.add_object_url(self.request),
        ]

        # Privacy wrappers
        if self._auth_enabled:
            for meth in ('index', 'show', 'create', 'update', 'replace'):
                self._after_calls[meth] += [
                    wrappers.apply_privacy(self.request),
                ]
            for meth in ('update', 'replace', 'update_many'):
                self._before_calls[meth] += [
                    wrappers.apply_request_privacy(
                        self.Model, self._json_params),
                ]

Response:
Setup defaulf wrappers.

        Wrappers are applied when view method does not return instance
        of Response. In this case nefertari renderers call wrappers and
        handle response generation.


Instruction:
Can you generate the documentation for the following Python 3 function
def migrate_codec(config_old, config_new):
    '''Migrate data from mongodict <= 0.2.1 to 0.3.0
    `config_old` and `config_new` should be dictionaries with the keys
    regarding to MongoDB server:
        - `host`
        - `port`
        - `database`
        - `collection`
    '''
    assert mongodict.__version__ in [(0, 3, 0), (0, 3, 1)]

    connection = pymongo.Connection(host=config_old['host'],
                                    port=config_old['port'])
    database = connection[config_old['database']]
    collection = database[config_old['collection']]
    new_dict = mongodict.MongoDict(**config_new) # uses pickle codec by default
    total_pairs = collection.count()
    start_time = time.time()
    for counter, pair in enumerate(collection.find(), start=1):
        key, value = pair['_id'], pair['value']
        new_dict[key] = value
        if counter % REPORT_INTERVAL == 0:
            print_report(counter, total_pairs, start_time)
    print_report(counter, total_pairs, start_time)
    print('')

Response:
Migrate data from mongodict < = 0. 2. 1 to 0. 3. 0
    config_old and config_new should be dictionaries with the keys
    and the values
    should be dicts with the keys
    regarding to MongoDB server.


Instruction:
How would you explain what the following Python 3 function does
def from_dict(cls, data, model):
        model_provided = model is not None
        result = {}
        for name, new_value in data.items():
            kwargs = {
                'name': name,
                'new_value': new_value,
            }
            if model_provided:
                kwargs['params'] = model.get_field_params(name)
            result[name] = cls(**kwargs)
        return result

Response: Generate a map of fieldName to clsInstance from dict.


Instruction:
Explain what the following Python 3 code does
def register(self):
        user, created = self.Model.create_account(
            self._json_params)

        if not created:
            raise JHTTPConflict('Looks like you already have an account.')

        self.request._user = user
        pk_field = user.pk_field()
        headers = remember(self.request, getattr(user, pk_field))
        return JHTTPOk('Registered', headers=headers)

Response: Register new user by POSTing all required data.


Instruction:
Given the following Python 3 function, write the documentation
def register(self):
        user, created = self.Model.create_account(self._json_params)
        if user.api_key is None:
            raise JHTTPBadRequest('Failed to generate ApiKey for user')

        if not created:
            raise JHTTPConflict('Looks like you already have an account.')

        self.request._user = user
        headers = remember(self.request, user.username)
        return JHTTPOk('Registered', headers=headers)

Response: Register a new user by POSTing all required data.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def claim_token(self, **params):
        self._json_params.update(params)
        success, self.user = self.Model.authenticate_by_password(
            self._json_params)

        if success:
            headers = remember(self.request, self.user.username)
            return JHTTPOk('Token claimed', headers=headers)
        if self.user:
            raise JHTTPUnauthorized('Wrong login or password')
        else:
            raise JHTTPNotFound('User not found')

Response: Claim current token by POSTing login and password.


Instruction:
Write a Python 3 script to
reset current token by POSTing login and password.

Response:
def reset_token(self, **params):
        """ Reset current token by POSTing 'login' and 'password'.

        User's `Authorization` header value is returned in `WWW-Authenticate`
        header.
        """
        response = self.claim_token(**params)
        if not self.user:
            return response

        self.user.api_key.reset_token()
        headers = remember(self.request, self.user.username)
        return JHTTPOk('Registered', headers=headers)


Instruction:
Implement a Python 3 function for
applying privacy to nested documents.

Response:
def _apply_nested_privacy(self, data):
        """ Apply privacy to nested documents.

        :param data: Dict of data to which privacy is already applied.
        """
        kw = {
            'is_admin': self.is_admin,
            'drop_hidden': self.drop_hidden,
        }
        for key, val in data.items():
            if is_document(val):
                data[key] = apply_privacy(self.request)(result=val, **kw)
            elif isinstance(val, list) and val and is_document(val[0]):
                data[key] = [apply_privacy(self.request)(result=doc, **kw)
                             for doc in val]
        return data


Instruction:
How would you explain what the following Python 3 function does
def _set_object_self(self, obj):
        from nefertari.elasticsearch import ES
        location = self.request.path_url
        route_kwargs = {}

        """ Check for parents """
        if self.request.matchdict:
            route_kwargs.update(self.request.matchdict)
        try:
            type_, obj_pk = obj['_type'], obj['_pk']
        except KeyError:
            return
        resource = (self.model_collections.get(type_) or
                    self.model_collections.get(ES.src2type(type_)))
        if resource is not None:
            route_kwargs.update({resource.id_name: obj_pk})
            location = self.request.route_url(
                resource.uid, **route_kwargs)
        obj.setdefault('_self', location)

Response: Add _self key value to obj.


Instruction:
Create a Python 3 function to
return the root resource.

Response:
def get_root_resource(config):
    """Returns the root resource."""
    app_package_name = get_app_package_name(config)
    return config.registry._root_resources.setdefault(
        app_package_name, Resource(config))


Instruction:
Here you have a function in Python 3, explain what it does
def add_resource_routes(config, view, member_name, collection_name, **kwargs):

    view = maybe_dotted(view)
    path_prefix = kwargs.pop('path_prefix', '')
    name_prefix = kwargs.pop('name_prefix', '')

    if config.route_prefix:
        name_prefix = "%s_%s" % (config.route_prefix, name_prefix)

    if collection_name:
        id_name = '/{%s}' % (kwargs.pop('id_name', None) or DEFAULT_ID_NAME)
    else:
        id_name = ''

    path = path_prefix.strip('/') + '/' + (collection_name or member_name)

    _factory = kwargs.pop('factory', None)
    # If factory is not set, than auth should be False
    _auth = kwargs.pop('auth', None) and _factory
    _traverse = (kwargs.pop('traverse', None) or id_name) if _factory else None

    action_route = {}
    added_routes = {}

    def add_route_and_view(config, action, route_name, path, request_method,
                           **route_kwargs):
        if route_name not in added_routes:
            config.add_route(
                route_name, path, factory=_factory,
                request_method=['GET', 'POST', 'PUT', 'PATCH', 'DELETE',
                                'OPTIONS'],
                **route_kwargs)
            added_routes[route_name] = path

        action_route[action] = route_name

        if _auth:
            permission = PERMISSIONS[action]
        else:
            permission = None
        config.add_view(view=view, attr=action, route_name=route_name,
                        request_method=request_method,
                        permission=permission,
                        **kwargs)
        config.commit()

    if collection_name == member_name:
        collection_name = collection_name + '_collection'

    if collection_name:
        add_route_and_view(
            config, 'index', name_prefix + collection_name, path,
            'GET')

        add_route_and_view(
            config, 'collection_options', name_prefix + collection_name, path,
            'OPTIONS')

    add_route_and_view(
        config, 'show', name_prefix + member_name, path + id_name,
        'GET', traverse=_traverse)

    add_route_and_view(
        config, 'item_options', name_prefix + member_name, path + id_name,
        'OPTIONS', traverse=_traverse)

    add_route_and_view(
        config, 'replace', name_prefix + member_name, path + id_name,
        'PUT', traverse=_traverse)

    add_route_and_view(
        config, 'update', name_prefix + member_name, path + id_name,
        'PATCH', traverse=_traverse)

    add_route_and_view(
        config, 'create', name_prefix + (collection_name or member_name), path,
        'POST')

    add_route_and_view(
        config, 'delete', name_prefix + member_name, path + id_name,
        'DELETE', traverse=_traverse)

    if collection_name:
        add_route_and_view(
            config, 'update_many',
            name_prefix + (collection_name or member_name),
            path, 'PUT', traverse=_traverse)

        add_route_and_view(
            config, 'update_many',
            name_prefix + (collection_name or member_name),
            path, 'PATCH', traverse=_traverse)

        add_route_and_view(
            config, 'delete_many',
            name_prefix + (collection_name or member_name),
            path, 'DELETE', traverse=_traverse)

    return action_route

Response: Add routes to the resource collection.


Instruction:
Here you have a function in Python 3, explain what it does
def get_default_view_path(resource):
    "Returns the dotted path to the default view class."

    parts = [a.member_name for a in resource.ancestors] +\
            [resource.collection_name or resource.member_name]

    if resource.prefix:
        parts.insert(-1, resource.prefix)

    view_file = '%s' % '_'.join(parts)
    view = '%s:%sView' % (view_file, snake2camel(view_file))

    app_package_name = get_app_package_name(resource.config)
    return '%s.views.%s' % (app_package_name, view)

Response: Returns the dotted path to the default view class.


Instruction:
Make a summary of the following Python 3 code
def get_ancestors(self):
        "Returns the list of ancestor resources."

        if self._ancestors:
            return self._ancestors

        if not self.parent:
            return []

        obj = self.resource_map.get(self.parent.uid)

        while obj and obj.member_name:
            self._ancestors.append(obj)
            obj = obj.parent

        self._ancestors.reverse()
        return self._ancestors

Response: Returns the list of ancestor resources.


Instruction:
Write a Python 3 function that can
add a new resource to the resource map.

Response:
def add(self, member_name, collection_name='', parent=None, uid='',
            **kwargs):
        """
        :param member_name: singular name of the resource. It should be the
            appropriate singular version of the resource given your locale
            and used with members of the collection.

        :param collection_name: plural name of the resource. It will be used
            to refer to the resource collection methods and should be a
            plural version of the ``member_name`` argument.
            Note: if collection_name is empty, it means resource is singular

        :param parent: parent resource name or object.

        :param uid: unique name for the resource

        :param kwargs:
            view: custom view to overwrite the default one.
            the rest of the keyward arguments are passed to
            add_resource_routes call.

        :return: ResourceMap object
        """
        # self is the parent resource on which this method is called.
        parent = (self.resource_map.get(parent) if type(parent)
                  is str else parent or self)

        prefix = kwargs.pop('prefix', '')

        uid = (uid or
               ':'.join(filter(bool, [parent.uid, prefix, member_name])))

        if uid in self.resource_map:
            raise ValueError('%s already exists in resource map' % uid)

        # Use id_name of parent for singular views to make url generation
        # easier
        id_name = kwargs.get('id_name', '')
        if not id_name and parent:
            id_name = parent.id_name

        new_resource = Resource(self.config, member_name=member_name,
                                collection_name=collection_name,
                                parent=parent, uid=uid,
                                id_name=id_name,
                                prefix=prefix)

        view = maybe_dotted(
            kwargs.pop('view', None) or get_default_view_path(new_resource))

        for name, val in kwargs.pop('view_args', {}).items():
            setattr(view, name, val)

        root_resource = self.config.get_root_resource()

        view.root_resource = root_resource
        new_resource.view = view
        path_segs = []
        kwargs['path_prefix'] = ''

        for res in new_resource.ancestors:
            if not res.is_singular:
                if res.id_name:
                    id_full = res.id_name
                else:
                    id_full = "%s_%s" % (res.member_name, DEFAULT_ID_NAME)

                path_segs.append('%s/{%s}' % (res.collection_name, id_full))
            else:
                path_segs.append(res.member_name)

        if path_segs:
            kwargs['path_prefix'] = '/'.join(path_segs)

        if prefix:
            kwargs['path_prefix'] += '/' + prefix

        name_segs = [a.member_name for a in new_resource.ancestors]
        name_segs.insert(1, prefix)
        name_segs = [seg for seg in name_segs if seg]
        if name_segs:
            kwargs['name_prefix'] = '_'.join(name_segs) + ':'

        new_resource.renderer = kwargs.setdefault(
            'renderer', view._default_renderer)

        kwargs.setdefault('auth', root_resource.auth)
        kwargs.setdefault('factory', root_resource.default_factory)
        _factory = maybe_dotted(kwargs['factory'])

        kwargs['auth'] = kwargs.get('auth', root_resource.auth)

        kwargs['http_cache'] = kwargs.get(
            'http_cache', root_resource.http_cache)

        new_resource.action_route_map = add_resource_routes(
            self.config, view, member_name, collection_name,
            **kwargs)

        self.resource_map[uid] = new_resource
        # add all route names for this resource as keys in the dict,
        # so its easy to find it in the view.
        self.resource_map.update(dict.fromkeys(
            list(new_resource.action_route_map.values()),
            new_resource))

        # Store resources in {modelName: resource} map if:
        #   * Its view has Model defined
        #   * It's not singular
        #   * Its parent is root or it's not already stored
        model = new_resource.view.Model
        is_collection = model is not None and not new_resource.is_singular
        if is_collection:
            is_needed = (model.__name__ not in self.model_collections or
                         new_resource.parent is root_resource)
            if is_needed:
                self.model_collections[model.__name__] = new_resource

        parent.children.append(new_resource)
        view._resource = new_resource
        view._factory = _factory

        return new_resource


Instruction:
Here you have a function in Python 3, explain what it does
def add_from_child(self, resource, **kwargs):

        new_resource = self.add(
            resource.member_name, resource.collection_name, **kwargs)
        for child in resource.children:
            new_resource.add_from_child(child, **kwargs)

Response: Add a resource with its all children resources to the current


Instruction:
Here you have a function in Python 3, explain what it does
def add(self, path):
        name_with_ext = os.path.split(path)[1]  # split directory and filename
        name = name_with_ext.split('.')[0]  # remove extension
        self.list.update({name: path})

Response: Adds the path of a data set to the list of available sets


Instruction:
How would you implement a function in Python 3 that
unpacks a data set to a Pandas DataFrame containing only the available datasets.

Response:
def unpack(self, name):
        """
        Unpacks a data set to a Pandas DataFrame

        Parameters
        ----------
        name : str
            call `.list` to see all availble datasets

        Returns
        -------
        pd.DataFrame
        """
        path = self.list[name]
        df = pd.read_pickle(path, compression='gzip')
        return df


Instruction:
Create a Python 3 function for
translating each sequence into six reading frames

Response:
def six_frame(genome, table, minimum = 10):
    """
    translate each sequence into six reading frames
    """
    for seq in parse_fasta(genome):
        dna = Seq(seq[1].upper().replace('U', 'T'), IUPAC.ambiguous_dna)
        counter = 0
        for sequence in ['f', dna], ['rc', dna.reverse_complement()]:
            direction, sequence = sequence
            for frame in range(0, 3):
                for prot in \
                            sequence[frame:].\
                            translate(table = table, to_stop = False).split('*'):
                    if len(prot) < minimum:
                        continue
                    counter += 1
                    header = '%s_%s table=%s frame=%s-%s %s' % \
                                (seq[0].split()[0], counter, table, frame+1, \
                                direction, ' '.join(seq[0].split()[1:]))
                    yield [header, prot]


Instruction:
Here you have a function in Python 3, explain what it does
def publish_processed_network_packets(
        name="not-set",
        task_queue=None,
        result_queue=None,
        need_response=False,
        shutdown_msg="SHUTDOWN"):

    """
    # Redis/RabbitMQ/SQS messaging endpoints for pub-sub
    routing_key = ev("PUBLISH_EXCHANGE",
                     "reporting.accounts")
    queue_name = ev("PUBLISH_QUEUE",
                    "reporting.accounts")
    auth_url = ev("PUB_BROKER_URL",
                  "redis://localhost:6379/15")
    serializer = "json"
    """

    # these keys need to be cycled to prevent
    # exploiting static keys
    filter_key = ev("IGNORE_KEY",
                    INCLUDED_IGNORE_KEY)

    forward_host = ev("FORWARD_HOST", "127.0.0.1")
    forward_port = int(ev("FORWARD_PORT", "80"))
    include_filter_key = ev("FILTER_KEY", "")
    if not include_filter_key and filter_key:
        include_filter_key = filter_key

    filter_keys = [filter_key]

    log.info(("START consumer={} "
              "forward={}:{} with "
              "key={} filters={}")
             .format(name,
                     forward_host,
                     forward_port,
                     include_filter_key,
                     filter_key))

    forward_skt = None

    not_done = True
    while not_done:

        if not forward_skt:
            forward_skt = connect_forwarder(
                            forward_host=forward_host,
                            forward_port=forward_port)

        next_task = task_queue.get()
        if next_task:

            if str(next_task) == shutdown_msg:
                # Poison pill for shutting down
                log.info(("{}: DONE CALLBACK "
                          "Exiting msg={}")
                         .format(name,
                                 next_task))
                task_queue.task_done()
                break
            # end of handling shutdown case

            try:
                log.debug(("{} parsing")
                          .format(name))

                source = next_task.source
                packet = next_task.payload

                if not packet:
                    log.error(("{} invalid task found "
                               "{} missing payload")
                              .format(name,
                                      next_task))
                    break

                log.debug(("{} found msg from src={}")
                          .format(name,
                                  source))

                network_data = parse_network_data(
                                  data_packet=packet,
                                  include_filter_key=include_filter_key,
                                  filter_keys=filter_keys)

                if network_data["status"] == VALID:
                    if network_data["data_type"] == TCP \
                        or network_data["data_type"] == UDP \
                            or network_data["data_type"] == ARP \
                            or network_data["data_type"] == ICMP:

                        log.info(("{} valid={} packet={} "
                                  "data={}")
                                 .format(name,
                                         network_data["id"],
                                         network_data["data_type"],
                                         network_data["target_data"]))

                        if not forward_skt:
                            forward_skt = connect_forwarder(
                                            forward_host=forward_host,
                                            forward_port=forward_port)

                        if forward_skt:
                            if network_data["stream"]:

                                sent = False
                                while not sent:
                                    try:
                                        log.info("sending={}".format(
                                            network_data["stream"]))
                                        send_msg(
                                            forward_skt,
                                            network_data["stream"]
                                            .encode("utf-8"))
                                        sent = True
                                    except Exception as e:
                                        sent = False
                                        time.sleep(0.5)
                                        try:
                                            forward_skt.close()
                                            forward_skt = None
                                        except Exception as w:
                                            forward_skt = None
                                        forward_skt = connect_forwarder(
                                            forward_host=forward_host,
                                            forward_port=forward_port)
                                # end of reconnecting

                                log.info("sent={}".format(
                                    network_data["stream"]))

                                if need_response:
                                    log.info("receiving")
                                    cdr_res = forward_skt.recv(1024)
                                    log.info(("cdr - res{}")
                                             .format(cdr_res))
                            else:
                                log.info(("{} EMPTY stream={} "
                                          "error={} status={}")
                                         .format(
                                            name,
                                            network_data["stream"],
                                            network_data["err"],
                                            network_data["status"]))
                    else:
                        log.info(("{} not_supported valid={} "
                                  "packet data_type={} status={}")
                                 .format(name,
                                         network_data["id"],
                                         network_data["data_type"],
                                         network_data["status"]))
                elif network_data["status"] == FILTERED:
                    log.info(("{} filtered={} status={}")
                             .format(name,
                                     network_data["filtered"],
                                     network_data["status"]))
                else:
                    if network_data["status"] == INVALID:
                        log.info(("{} invalid={} packet={} "
                                  "error={} status={}")
                                 .format(name,
                                         network_data["id"],
                                         network_data["data_type"],
                                         network_data["error"],
                                         network_data["status"]))
                    else:
                        log.info(("{} unknown={} packet={} "
                                  "error={} status={}")
                                 .format(name,
                                         network_data["id"],
                                         network_data["data_type"],
                                         network_data["error"],
                                         network_data["status"]))
                # end of if valid or not data
            except KeyboardInterrupt as k:
                log.info(("{} stopping")
                         .format(name))
                break
            except Exception as e:
                log.error(("{} failed packaging packet to forward "
                           "with ex={}")
                          .format(name,
                                  e))
                break
            # end of try/ex during payload processing
        # end of if found a next_task

        log.info(("Consumer: {} {}")
                 .format(name, next_task))
        task_queue.task_done()

        if need_response:
            answer = "processed: {}".format(next_task())
            result_queue.put(answer)
    # end of while

    if forward_skt:
        try:
            forward_skt.close()
            log.info("CLOSED connection")
            forward_skt = None
        except Exception:
            log.info("CLOSED connection")
    # end of cleaning up forwarding socket

    log.info("{} Done".format(name))

    return

Response: Publish all network packets and return a list of the messages that were processed.


Instruction:
Can you create a Python 3 function that
starts the packet processor and the packet consumers

Response:
def run_main(
        need_response=False,
        callback=None):
    """run_main

    start the packet consumers and the packet processors

    :param need_response: should send response back to publisher
    :param callback: handler method
    """

    stop_file = ev("STOP_FILE",
                   "/opt/stop_recording")

    num_workers = int(ev("NUM_WORKERS",
                         "1"))
    shutdown_msg = "SHUTDOWN"

    log.info("Start - {}".format(name))

    log.info("Creating multiprocessing queue")
    tasks = multiprocessing.JoinableQueue()
    queue_to_consume = multiprocessing.Queue()
    host = "localhost"

    # Start consumers
    log.info("Starting Consumers to process queued tasks")
    consumers = start_consumers_for_queue(
        num_workers=num_workers,
        tasks=tasks,
        queue_to_consume=queue_to_consume,
        shutdown_msg=shutdown_msg,
        consumer_class=WorkerToProcessPackets,
        callback=callback)

    log.info("creating socket")
    skt = create_layer_2_socket()
    log.info("socket created")

    not_done = True
    while not_done:

        if not skt:
            log.info("Failed to create layer 2 socket")
            log.info("Please make sure to run as root")
            not_done = False
            break

        try:
            if os.path.exists(stop_file):
                log.info(("Detected stop_file={}")
                         .format(stop_file))
                not_done = False
                break
            # stop if the file exists

            # Only works on linux
            packet = skt.recvfrom(65565)

            if os.path.exists(stop_file):
                log.info(("Detected stop_file={}")
                         .format(stop_file))
                not_done = False
                break
            # stop if the file was created during a wait loop

            tasks.put(NetworkPacketTask(source=host,
                                        payload=packet))

        except KeyboardInterrupt as k:
            log.info("Stopping")
            not_done = False
            break
        except Exception as e:
            log.error(("Failed reading socket with ex={}")
                      .format(e))
            not_done = False
            break
        # end of try/ex during socket receving

    # end of while processing network packets

    log.info(("Shutting down consumers={}")
             .format(len(consumers)))

    shutdown_consumers(num_workers=num_workers,
                       tasks=tasks)

    # Wait for all of the tasks to finish
    if need_response:
        log.info("Waiting for tasks to finish")
        tasks.join()

    log.info("Done waiting for tasks to finish")


Instruction:
Here you have a function in Python 3, explain what it does
def best_model(seq2hmm):
    for seq in seq2hmm:
        best = []
        for model in seq2hmm[seq]:
            best.append([model, sorted([i[-1] for i in seq2hmm[seq][model]], reverse = True)[0]])
        best_model = sorted(best, key = itemgetter(1), reverse = True)[0][0]
        seq2hmm[seq] = [best_model] + [seq2hmm[seq][best_model]]
    return seq2hmm

Response: determine the best model for each sequence in the sequence2hmm


Instruction:
Given the following Python 3 function, write the documentation
def check_gaps(matches, gap_threshold = 0):
    gaps = []
    prev = None
    for match in sorted(matches, key = itemgetter(0)):
        if prev is None:
            prev = match
            continue
        if match[0] - prev[1] >= gap_threshold:
            gaps.append([prev, match])
            prev = match
    return [[i[0][1], i[1][0]] for i in gaps]

Response: check for large gaps between alignment windows


Instruction:
Write a Python 3 script to
check if sequence has already hit the same part of the model

Response:
def check_overlap(current, hit, overlap = 200):
    """
    determine if sequence has already hit the same part of the model,
    indicating that this hit is for another 16S rRNA gene
    """
    for prev in current:
        p_coords = prev[2:4]
        coords = hit[2:4]
        if get_overlap(coords, p_coords) >= overlap:
            return True
    return False


Instruction:
Write a Python 3 function that can
check if the hits are in the order of the current sequence

Response:
def check_order(current, hit, overlap = 200):
    """
    determine if hits are sequential on model and on the
    same strand
        * if not, they should be split into different groups
    """
    prev_model = current[-1][2:4]
    prev_strand = current[-1][-2]
    hit_model = hit[2:4]
    hit_strand = hit[-2]
    # make sure they are on the same strand
    if prev_strand != hit_strand:
        return False
    # check for sequential hits on + strand
    if prev_strand == '+' and (prev_model[1] - hit_model[0] >= overlap):
        return False
    # check for sequential hits on - strand
    if prev_strand == '-' and (hit_model[1] - prev_model[0] >= overlap):
        return False
    else:
        return True


Instruction:
Given the following Python 3 function, write the documentation
def hit_groups(hits):
    groups = []
    current = False
    for hit in sorted(hits, key = itemgetter(0)):
        if current is False:
            current = [hit]
        elif check_overlap(current, hit) is True or check_order(current, hit) is False:
            groups.append(current)
            current = [hit]
        else:
            current.append(hit)
    groups.append(current)
    return groups

Response: Returns a list of groups that are hit by each sequence


Instruction:
Implement a function in Python 3 to
get info from ssu - cmsearch output

Response:
def get_info(line, bit_thresh):
    """
    get info from either ssu-cmsearch or cmsearch output
    """
    if len(line) >= 18: # output is from cmsearch
        id, model, bit, inc = line[0].split()[0], line[2], float(line[14]), line[16]
        sstart, send, strand = int(line[7]), int(line[8]), line[9]
        mstart, mend = int(line[5]), int(line[6])
    elif len(line) == 9: # output is from ssu-cmsearch
        if bit_thresh == 0:
            print('# ssu-cmsearch does not include a model-specific inclusion threshold, ', file=sys.stderr)
            print('# please specify a bit score threshold', file=sys.stderr)
            exit()
        id, model, bit = line[1].split()[0], line[0], float(line[6])
        inc = '!' # this is not a feature of ssu-cmsearch
        sstart, send = int(line[2]), int(line[3])
        mstart, mend = int(4), int(5)
        if send >= sstart:
            strand = '+'
        else:
            strand = '-'
    else:
        print('# unsupported hmm format:', file=sys.stderr)
        print('# provide tabular output from ssu-cmsearch and cmsearch supported', file=sys.stderr)
        exit()
    coords = [sstart, send]
    sstart, send = min(coords), max(coords)
    mcoords = [mstart, mend]
    mstart, mend = min(mcoords), max(mcoords)
    return id, model, bit, sstart, send, mstart, mend, strand, inc


Instruction:
Can you tell what is the following Python 3 function doing
def check_buffer(coords, length, buffer):
    s = min(coords[0], buffer)
    e = min(length - coords[1], buffer)
    return [s, e]

Response: check to see how much of the buffer is being used


Instruction:
How would you code a function in Python 3 to
convert a parser to a new type.

Response:
def convert_parser_to(parser, parser_or_type, metadata_props=None):
    """
    :return: a parser of type parser_or_type, initialized with the properties of parser. If parser_or_type
    is a type, an instance of it must contain a update method. The update method must also process
    the set of properties supported by MetadataParser for the conversion to have any affect.
    :param parser: the parser (or content or parser type) to convert to new_type
    :param parser_or_type: a parser (or content) or type of parser to return
    :see: get_metadata_parser(metadata_container) for more on how parser_or_type is treated
    """

    old_parser = parser if isinstance(parser, MetadataParser) else get_metadata_parser(parser)
    new_parser = get_metadata_parser(parser_or_type)

    for prop in (metadata_props or _supported_props):
        setattr(new_parser, prop, deepcopy(getattr(old_parser, prop, u'')))

    new_parser.update()

    return new_parser


Instruction:
Can you tell what is the following Python 3 function doing
def get_metadata_parser(metadata_container, **metadata_defaults):

    parser_type = None

    if isinstance(metadata_container, MetadataParser):
        parser_type = type(metadata_container)

    elif isinstance(metadata_container, type):
        parser_type = metadata_container
        metadata_container = metadata_container().update(**metadata_defaults)

    xml_root, xml_tree = get_parsed_content(metadata_container)

    # The get_parsed_content method ensures only these roots will be returned

    parser = None

    if parser_type is not None:
        parser = parser_type(xml_tree, **metadata_defaults)
    elif xml_root in ISO_ROOTS:
        parser = IsoParser(xml_tree, **metadata_defaults)
    else:
        has_arcgis_data = any(element_exists(xml_tree, e) for e in ARCGIS_NODES)

        if xml_root == FGDC_ROOT and not has_arcgis_data:
            parser = FgdcParser(xml_tree, **metadata_defaults)
        elif xml_root in ARCGIS_ROOTS:
            parser = ArcGISParser(xml_tree, **metadata_defaults)

    return parser

Response: Returns a new instance of a MetadataParser class corresponding to the standard represented by the given metadata_container.


Instruction:
Write a Python 3 script for
parsing any of the types of content and returns the XML root and XML Tree parsed by and compatible with element_utils.

Response:
def get_parsed_content(metadata_content):
    """
    Parses any of the following types of content:
    1. XML string or file object: parses XML content
    2. MetadataParser instance: deep copies xml_tree
    3. Dictionary with nested objects containing:
        - name (required): the name of the element tag
        - text: the text contained by element
        - tail: text immediately following the element
        - attributes: a Dictionary containing element attributes
        - children: a List of converted child elements

    :raises InvalidContent: if the XML is invalid or does not conform to a supported metadata standard
    :raises NoContent: If the content passed in is null or otherwise empty

    :return: the XML root along with an XML Tree parsed by and compatible with element_utils
    """

    _import_parsers()  # Prevents circular dependencies between modules

    xml_tree = None

    if metadata_content is None:
        raise NoContent('Metadata has no data')
    else:
        if isinstance(metadata_content, MetadataParser):
            xml_tree = deepcopy(metadata_content._xml_tree)
        elif isinstance(metadata_content, dict):
            xml_tree = get_element_tree(metadata_content)
        else:
            try:
                # Strip name spaces from file or XML content
                xml_tree = get_element_tree(metadata_content)
            except Exception:
                xml_tree = None  # Several exceptions possible, outcome is the same

    if xml_tree is None:
        raise InvalidContent(
            'Cannot instantiate a {parser_type} parser with invalid content to parse',
            parser_type=type(metadata_content).__name__
        )

    xml_root = get_element_name(xml_tree)

    if xml_root is None:
        raise NoContent('Metadata contains no data')
    elif xml_root not in VALID_ROOTS:
        content = type(metadata_content).__name__
        raise InvalidContent('Invalid root element for {content}: {xml_root}', content=content, xml_root=xml_root)

    return xml_root, xml_tree


Instruction:
Can you generate the documentation for the following Python 3 function
def _import_parsers():

    global ARCGIS_NODES
    global ARCGIS_ROOTS
    global ArcGISParser

    global FGDC_ROOT
    global FgdcParser

    global ISO_ROOTS
    global IsoParser

    global VALID_ROOTS

    if ARCGIS_NODES is None or ARCGIS_ROOTS is None or ArcGISParser is None:
        from gis_metadata.arcgis_metadata_parser import ARCGIS_NODES
        from gis_metadata.arcgis_metadata_parser import ARCGIS_ROOTS
        from gis_metadata.arcgis_metadata_parser import ArcGISParser

    if FGDC_ROOT is None or FgdcParser is None:
        from gis_metadata.fgdc_metadata_parser import FGDC_ROOT
        from gis_metadata.fgdc_metadata_parser import FgdcParser

    if ISO_ROOTS is None or IsoParser is None:
        from gis_metadata.iso_metadata_parser import ISO_ROOTS
        from gis_metadata.iso_metadata_parser import IsoParser

    if VALID_ROOTS is None:
        VALID_ROOTS = {FGDC_ROOT}.union(ARCGIS_ROOTS + ISO_ROOTS)

Response: Lazy imports to prevent circular dependencies between this module and utils


Instruction:
Can you create a Python 3 function that
initializes the data map for this object.

Response:
def _init_data_map(self):
        """ Default data map initialization: MUST be overridden in children """

        if self._data_map is None:
            self._data_map = {'_root': None}
            self._data_map.update({}.fromkeys(self._metadata_props))


Instruction:
Implement a Python 3 function for
getting the xml tree for the template.

Response:
def _get_template(self, root=None, **metadata_defaults):
        """ Iterate over items metadata_defaults {prop: val, ...} to populate template """

        if root is None:
            if self._data_map is None:
                self._init_data_map()

            root = self._xml_root = self._data_map['_root']

        template_tree = self._xml_tree = create_element_tree(root)

        for prop, val in iteritems(metadata_defaults):
            path = self._data_map.get(prop)
            if path and val:
                setattr(self, prop, val)
                update_property(template_tree, None, path, prop, val)

        return template_tree


Instruction:
Can you write a function in Python 3 where it
returns the xpath for a given property

Response:
def _get_xpath_for(self, prop):
        """ :return: the configured xpath for a given property """

        xpath = self._data_map.get(prop)
        return getattr(xpath, 'xpath', xpath)


Instruction:
Can you generate the documentation for the following Python 3 function
def _parse_complex(self, prop):

        xpath_root = None
        xpath_map = self._data_structures[prop]

        return parse_complex(self._xml_tree, xpath_root, xpath_map, prop)

Response: Default parsing operation for a complex struct


Instruction:
Write a Python 3 function for
defaulting parsing operation for lists of complex structs

Response:
def _parse_complex_list(self, prop):
        """ Default parsing operation for lists of complex structs """

        xpath_root = self._get_xroot_for(prop)
        xpath_map = self._data_structures[prop]

        return parse_complex_list(self._xml_tree, xpath_root, xpath_map, prop)


Instruction:
Can you write a function in Python 3 where it
creates and returns a Date Types data structure parsed from the metadata

Response:
def _parse_dates(self, prop=DATES):
        """ Creates and returns a Date Types data structure parsed from the metadata """

        return parse_dates(self._xml_tree, self._data_structures[prop])


Instruction:
How would you implement a function in Python 3 that
defaults update operation for a complex struct

Response:
def _update_complex(self, **update_props):
        """ Default update operation for a complex struct """

        prop = update_props['prop']
        xpath_root = self._get_xroot_for(prop)
        xpath_map = self._data_structures[prop]

        return update_complex(xpath_root=xpath_root, xpath_map=xpath_map, **update_props)


Instruction:
Can you tell what is the following Python 3 function doing
def _update_complex_list(self, **update_props):

        prop = update_props['prop']
        xpath_root = self._get_xroot_for(prop)
        xpath_map = self._data_structures[prop]

        return update_complex_list(xpath_root=xpath_root, xpath_map=xpath_map, **update_props)

Response: Default update operation for lists of complex structs


Instruction:
Make a summary of the following Python 3 code
def _update_dates(self, xpath_root=None, **update_props):

        tree_to_update = update_props['tree_to_update']
        prop = update_props['prop']
        values = (update_props['values'] or {}).get(DATE_VALUES) or u''
        xpaths = self._data_structures[prop]

        if not self.dates:
            date_xpaths = xpath_root
        elif self.dates[DATE_TYPE] != DATE_TYPE_RANGE:
            date_xpaths = xpaths.get(self.dates[DATE_TYPE], u'')
        else:
            date_xpaths = [
                xpaths[DATE_TYPE_RANGE_BEGIN],
                xpaths[DATE_TYPE_RANGE_END]
            ]

        if xpath_root:
            remove_element(tree_to_update, xpath_root)

        return update_property(tree_to_update, xpath_root, date_xpaths, prop, values)

Response: Update the dates property of the metadata.


Instruction:
Given the following Python 3 function, write the documentation
def write(self, use_template=False, out_file_or_path=None, encoding=DEFAULT_ENCODING):

        if not out_file_or_path:
            out_file_or_path = self.out_file_or_path

        if not out_file_or_path:
            # FileNotFoundError doesn't exist in Python 2
            raise IOError('Output file path has not been provided')

        write_element(self.update(use_template), out_file_or_path, encoding)

Response: Updates an XML tree with the instance properties and writes the content to a file.


Instruction:
Can you create a Python 3 function that
defaults validation for updated properties

Response:
def validate(self):
        """ Default validation for updated properties: MAY be overridden in children """

        validate_properties(self._data_map, self._metadata_props)

        for prop in self._data_map:
            validate_any(prop, getattr(self, prop), self._data_structures.get(prop))

        return self


Instruction:
Implement a function in Python 3 to
search order of the given regexps

Response:
def _search_regex(ops: dict, regex_pat: str):
    """
    Search order:
      * specified regexps
      * operators sorted from longer to shorter
    """
    custom_regexps = list(filter(None, [dic['regex'] for op, dic in ops.items() if 'regex' in dic]))
    op_names = [op for op, dic in ops.items() if 'regex' not in dic]
    regex = [regex_pat.format(_ops_regex(op_names))] if len(op_names) > 0 else []
    return re.compile('|'.join(custom_regexps + regex))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def spec(self, postf_un_ops: str) -> list:
        spec = [(l + op, {'pat': self.pat(pat),
                          'postf': self.postf(r, postf_un_ops),
                          'regex': None})
                for op, pat in self.styles.items()
                for l, r in self.brackets]
        spec[0][1]['regex'] = self.regex_pat.format(
            _ops_regex(l for l, r in self.brackets),
            _ops_regex(self.styles.keys())
        )
        return spec

Response: Return prefix unary operators list


Instruction:
Implement a Python 3 function for
returning prefix unary operators list. Sets only one regex for all items in the dict.

Response:
def spec(self) -> list:
        """Returns prefix unary operators list.
        Sets only one regex for all items in the dict."""
        spec = [item
                for op, pat in self.ops.items()
                for item in [('{' + op, {'pat': pat, 'postf': self.postf, 'regex': None}),
                             ('˱' + op, {'pat': pat, 'postf': self.postf, 'regex': None})]
                ]
        spec[0][1]['regex'] = self.regex_pat.format(_ops_regex(self.ops.keys()))
        return spec


Instruction:
Write a Python 3 function for
filling the internal state of the internal state with the postf_un_ops.

Response:
def fill(self, postf_un_ops: str):
        """
        Insert:
          * math styles
          * other styles
          * unary prefix operators without brackets
          * defaults
        """
        for op, dic in self.ops.items():
            if 'postf' not in dic:
                dic['postf'] = self.postf
        self.ops = OrderedDict(
            self.styles.spec(postf_un_ops) +
            self.other_styles.spec(postf_un_ops) +
            self.pref_un_greedy.spec() +
            list(self.ops.items())
        )
        for op, dic in self.ops.items():
            dic['postf'] = re.compile(dic['postf'])
        self.regex = _search_regex(self.ops, self.regex_pat)


Instruction:
Can you implement a function in Python 3 that
returns compiled regex and function for su_scripts.

Response:
def _su_scripts_regex(self):
        """
        :return:
            [compiled regex, function]
        """
        sups = re.escape(''.join([k for k in self.superscripts.keys()]))
        subs = re.escape(''.join([k for k in self.subscripts.keys()]))  # language=PythonRegExp
        su_regex = (r'\\([{su_}])|([{sub}]+|‹[{sub}]+›|˹[{sub}]+˺)' +
                    r'|([{sup}]+)(?=√)|([{sup}]+(?!√)|‹[{sup}]+›|˹[{sup}]+˺)').format(
            su_=subs + sups, sub=subs, sup=sups)
        su_regex = re.compile(su_regex)

        def su_replace(m):
            esc, sub, root_sup, sup = m.groups()
            if esc is not None:
                return esc
            elif sub is not None:
                return '_{' + ''.join([c if (c in ['‹', '›', '˹', '˺']) else self.subscripts[c] for c in sub]) + '}'
            elif root_sup is not None:
                return ''.join([self.superscripts[c] for c in root_sup])
            elif sup is not None:
                return '^{' + ''.join([c if (c in ['‹', '›', '˹', '˺']) else self.superscripts[c] for c in sup]) + '}'
            else:
                raise TypeError("Regex bug: this should never be reached")

        return [su_regex, su_replace]


Instruction:
Can you generate the documentation for the following Python 3 function
def _local_map(match, loc: str = 'lr') -> list:
        s = match.string
        map_ = [None] * (len(s) + 2)
        if loc == 'l' or loc == 'lr':
            balance = 0
            for i in reversed(range(0, match.start())):
                map_[i] = balance
                c, prev = s[i], (s[i - 1] if i > 0 else '')
                if (c == '}' or c == '˲') and prev != '\\':
                    balance += 1
                elif (c == '{' or c == '˱') and prev != '\\':
                    balance -= 1
                if balance < 0:
                    break
            map_[-1] = balance
        if loc == 'r' or loc == 'lr':
            balance = 0
            for i in range(match.end(), len(s)):
                map_[i] = balance
                c, prev = s[i], s[i - 1]
                if (c == '{' or c == '˱') and prev != '\\':
                    balance += 1
                elif (c == '}' or c == '˲') and prev != '\\':
                    balance -= 1
                if balance < 0:
                    break
            map_[len(s)] = balance
        return map_

Response: Returns a list of local map entries for loc.


Instruction:
Implement a Python 3 function for
replacing the operators in the string.

Response:
def _operators_replace(self, string: str) -> str:
        """
        Searches for first unary or binary operator (via self.op_regex
        that has only one group that contain operator)
        then replaces it (or escapes it if brackets do not match).
        Everything until:
          * space ' '
          * begin/end of the string
          * bracket from outer scope (like '{a/b}': term1=a term2=b)
        is considered a term (contents of matching brackets '{}' are
        ignored).

        Attributes
        ----------
        string: str
            string to replace
        """
        # noinspection PyShadowingNames
        def replace(string: str, start: int, end: int, substring: str) -> str:
            return string[0:start] + substring + string[end:len(string)]

        # noinspection PyShadowingNames
        def sub_pat(pat: Callable[[list], str] or str, terms: list) -> str:
            if isinstance(pat, str):
                return pat.format(*terms)
            else:
                return pat(terms)

        count = 0

        def check():
            nonlocal count
            count += 1
            if count > self.max_while:
                raise RuntimeError('Presumably while loop is stuck')

        # noinspection PyShadowingNames
        def null_replace(match) -> str:
            regex_terms = [gr for gr in match.groups() if gr is not None]
            op = regex_terms[0]
            terms = regex_terms[1:]
            return sub_pat(self.null_ops.ops[op]['pat'], terms)

        string = self.null_ops.regex.sub(null_replace, string)

        for ops, loc in [(self.pref_un_ops, 'r'), (self.postf_un_ops, 'l'),
                         (self.bin_centr_ops, 'lr')]:
            count = 0
            match = ops.regex.search(string)
            while match:
                check()
                regex_terms = [gr for gr in match.groups() if gr is not None]
                op = regex_terms[0]
                loc_map = self._local_map(match, loc)
                lmatch, rmatch = None, None
                if loc == 'l' or loc == 'lr':
                    for m in ops.ops[op]['pref'].finditer(string):
                        if m.end() <= match.start() and loc_map[m.end() - 1] == 0:
                            lmatch = m
                    if lmatch is None:
                        string = replace(string, match.start(), match.end(), match.group(0).replace(op, '\\' + op))
                        match = ops.regex.search(string)
                        continue
                    else:
                        term1 = string[lmatch.end():match.start()]
                if loc == 'r' or loc == 'lr':
                    for m in ops.ops[op]['postf'].finditer(string):
                        if m.start() >= match.end() and loc_map[m.start()] == 0:
                            rmatch = m
                            break
                    if rmatch is None:
                        string = replace(string, match.start(), match.end(), match.group(0).replace(op, '\\' + op))
                        match = ops.regex.search(string)
                        continue
                    else:
                        term2 = string[match.end():rmatch.start()]
                if loc == 'l':
                    # noinspection PyUnboundLocalVariable
                    terms = list(lmatch.groups()) + [term1] + regex_terms[1:]
                    start, end = lmatch.start(), match.end()
                elif loc == 'r':
                    # noinspection PyUnboundLocalVariable
                    terms = regex_terms[1:] + [term2] + list(rmatch.groups())
                    start, end = match.start(), rmatch.end()
                elif loc == 'lr':
                    terms = list(lmatch.groups()) + [term1] + regex_terms[1:] + [term2] + list(rmatch.groups())
                    start, end = lmatch.start(), rmatch.end()
                else:  # this never happen
                    terms = regex_terms[1:]
                    start, end = match.start(), match.end()

                string = replace(string, start, end, sub_pat(ops.ops[op]['pat'], terms))
                match = ops.regex.search(string)

        return string


Instruction:
Can you create a Python 3 function that
replaces LaTeX characters in src with new LaTeX characters.

Response:
def replace(self, src: str) -> str:
        """
        Extends LaTeX syntax via regex preprocess
        :param src: str
            LaTeX string
        :return: str
            New LaTeX string
        """
        if not self.readied:
            self.ready()

        # Brackets + simple pre replacements:
        src = self._dict_replace(self.simple_pre, src)

        # Superscripts and subscripts + pre regexps:
        for regex, replace in self.regex_pre:
            src = regex.sub(replace, src)

        # Unary and binary operators:
        src = self._operators_replace(src)

        # Loop regexps:
        src_prev = src
        for i in range(self.max_iter):
            for regex, replace in self.loop_regexps:
                src = regex.sub(replace, src)
            if src_prev == src:
                break
            else:
                src_prev = src

        # Post regexps:
        for regex, replace in self.regex_post:
            src = regex.sub(replace, src)

        # Simple post replacements:
        src = self._dict_replace(self.simple_post, src)

        # Escape characters:
        src = self.escapes_regex.sub(r'\1', src)

        return src


Instruction:
Can you generate the documentation for the following Python 3 function
def plot_gaps(plot, columns):
	from plot_window import window_plot_convolve as plot_window
#	plot_window([columns], len(columns)*.01, plot)
	plot_window([[100 - i for i in columns]], len(columns)*.01, plot)

Response: plot % of gaps at each position


Instruction:
How would you explain what the following Python 3 function does
def sample_group(sid, groups):
    for name in groups:
        if sid in groups[name].sids:
            return name

Response: Return the name of the first category that contains the given SampleID.


Instruction:
Create a Python 3 function to
combine multiple sets into a single set.

Response:
def combine_sets(*sets):
    """
    Combine multiple sets to create a single larger set.
    """
    combined = set()
    for s in sets:
        combined.update(s)
    return combined


Instruction:
Can you generate a brief explanation for the following Python 3 code
def unique_otuids(groups):
    uniques = {key: set() for key in groups}
    for i, group in enumerate(groups):
        to_combine = groups.values()[:i]+groups.values()[i+1:]
        combined = combine_sets(*to_combine)
        uniques[group] = groups[group].difference(combined)
    return uniques

Response: Get unique OTUIDs of each category.


Instruction:
Implement a Python 3 function for
getting all OTUIDs shared between all unique combinations of groups.

Response:
def shared_otuids(groups):
    """
    Get shared OTUIDs between all unique combinations of groups.

    :type groups: Dict
    :param groups: {Category name: OTUIDs in category}

    :return type: dict
    :return: Dict keyed on group combination and their shared OTUIDs as values.
    """
    for g in sorted(groups):
        print("Number of OTUs in {0}: {1}".format(g, len(groups[g].results["otuids"])))
    number_of_categories = len(groups)
    shared = defaultdict()
    for i in range(2, number_of_categories+1):
        for j in combinations(sorted(groups), i):
            combo_name = " & ".join(list(j))
            for grp in j:
                # initialize combo values
                shared[combo_name] = groups[j[0]].results["otuids"].copy()
                """iterate through all groups and keep updating combo OTUIDs with set
                intersection_update"""
                for grp in j[1:]:
                    shared[combo_name].intersection_update(groups[grp].results["otuids"])
    return shared


Instruction:
Write a Python 3 function that can
give a path, the method writes out one file for each group name in the uniques dictionary with the file name in the pattern PATH/prefix_group.txt with each file containing the unique OTUIDs found when comparing that group to all the other groups in uniques. :type path: str :param path: Output files will be saved in this PATH. :type prefix: str :param prefix: Prefix name added in front of output filename. :type uniques: dict :param uniques: Output from unique_otus() function.

Response:
def write_uniques(path, prefix, uniques):
    """
    Given a path, the method writes out one file for each group name in the
    uniques dictionary with the file name in the pattern

        PATH/prefix_group.txt

    with each file containing the unique OTUIDs found when comparing that group
    to all the other groups in uniques.

    :type path: str
    :param path: Output files will be saved in this PATH.

    :type prefix: str
    :param prefix: Prefix name added in front of output filename.

    :type uniques: dict
    :param uniques: Output from unique_otus() function.
    """
    for group in uniques:
        fp = osp.join(path, "{}_{}.txt".format(prefix, group))
        with open(fp, "w") as outf:
            outf.write("\n".join(uniques[group]))


Instruction:
How would you code a function in Python 3 to
parse the records in a FASTA - format file into a list of tuples.

Response:
def storeFASTA(fastaFNH):
    """
    Parse the records in a FASTA-format file by first reading the entire file into memory.

    :type source: path to FAST file or open file handle
    :param source: The data source from which to parse the FASTA records. Expects the
                   input to resolve to a collection that can be iterated through, such as
                   an open file handle.

    :rtype: tuple
    :return: FASTA records containing entries for id, description and data.
    """
    fasta = file_handle(fastaFNH).read()
    return [FASTARecord(rec[0].split()[0], rec[0].split(None, 1)[1], "".join(rec[1:]))
            for rec in (x.strip().split("\n") for x in fasta.split(">")[1:])]


Instruction:
Can you write a function in Python 3 where it
parses the records in a FASTA - format file keeping the file open and reading through .

Response:
def parseFASTA(fastaFNH):
    """
    Parse the records in a FASTA-format file keeping the file open, and reading through
    one line at a time.

    :type source: path to FAST file or open file handle
    :param source: The data source from which to parse the FASTA records.
                   Expects the input to resolve to a collection that can be iterated
                   through, such as an open file handle.

    :rtype: tuple
    :return: FASTA records containing entries for id, description and data.
    """
    recs = []
    seq = []
    seqID = ""
    descr = ""

    for line in file_handle(fastaFNH):
        line = line.strip()
        if line[0] == ";":
            continue
        if line[0] == ">":
            # conclude previous record
            if seq:
                recs.append(FASTARecord(seqID, descr, "".join(seq)))
                seq = []
            # start new record
            line = line[1:].split(None, 1)
            seqID, descr = line[0], line[1]
        else:
            seq.append(line)

    # catch last seq in file
    if seq:
        recs.append(FASTARecord(seqID, descr, "".join(seq)))
    return recs


Instruction:
How would you code a function in Python 3 to
read the contents of a QIIME mapping file and returns a dictionary keyed on SampleID BarcodeSequence LinkerPrimerSequence State Rat_Oral .

Response:
def parse_map_file(mapFNH):
    """
    Opens a QIIME mapping file and stores the contents in a dictionary keyed on SampleID
    (default) or a user-supplied one. The only required fields are SampleID,
    BarcodeSequence, LinkerPrimerSequence (in that order), and Description
    (which must be the final field).

    :type mapFNH: str
    :param mapFNH: Either the full path to the map file or an open file handle

    :rtype: tuple, dict
    :return: A tuple of header line for mapping file and a map associating each line of
             the mapping file with the appropriate sample ID (each value of the map also
             contains the sample ID). An OrderedDict is used for mapping so the returned
             map is guaranteed to have the same order as the input file.

    Example data:
    #SampleID BarcodeSequence LinkerPrimerSequence State   Description
    11.V13    ACGCTCGACA      GTTTGATCCTGGCTCAG    Disease Rat_Oral
    """
    m = OrderedDict()
    map_header = None

    with file_handle(mapFNH) as mapF:
        for line in mapF:
            if line.startswith("#SampleID"):
                map_header = line.strip().split("\t")
            if line.startswith("#") or not line:
                    continue
            line = line.strip().split("\t")
            m[line[0]] = line

    return map_header, m


Instruction:
Given the following Python 3 function, write the documentation
def get_vid_from_url(url):
        return match1(url, r'youtu\.be/([^?/]+)') or \
          match1(url, r'youtube\.com/embed/([^/?]+)') or \
          match1(url, r'youtube\.com/v/([^/?]+)') or \
          match1(url, r'youtube\.com/watch/([^/?]+)') or \
          parse_query_param(url, 'v') or \
          parse_query_param(parse_query_param(url, 'u'), 'v')

Response: Extracts video ID from URL.


Instruction:
Write a Python 3 script to
make Mimi from UIDs

Response:
def makeMimi(upid):
    """From http://cdn37.atwikiimg.com/sitescript/pub/dksitescript/FC2.site.js
    Also com.hps.util.fc2.FC2EncrptUtil.makeMimiLocal
    L110"""
    strSeed = "gGddgPfeaf_gzyr"
    prehash = upid + "_" + strSeed
    return md5(prehash.encode('utf-8')).hexdigest()


Instruction:
Can you generate the documentation for the following Python 3 function
def fc2video_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
    #'http://video.fc2.com/en/content/20151021bTVKnbEw'
    #'http://xiaojiadianvideo.asia/content/20151021bTVKnbEw'
    #'http://video.fc2.com/ja/content/20151021bTVKnbEw'
    #'http://video.fc2.com/tw/content/20151021bTVKnbEw'
    hostname = urlparse(url).hostname
    if not ('fc2.com' in hostname or 'xiaojiadianvideo.asia' in hostname):
        return False
    upid = match1(url, r'.+/content/(\w+)')

    fc2video_download_by_upid(upid, output_dir, merge, info_only)

Response: Download a video from the FC2 server.


Instruction:
Write a Python 3 script to
download Dailymotion videos by URL.

Response:
def dailymotion_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    """Downloads Dailymotion videos by URL.
    """

    html = get_content(rebuilt_url(url))
    info = json.loads(match1(html, r'qualities":({.+?}),"'))
    title = match1(html, r'"video_title"\s*:\s*"([^"]+)"') or \
            match1(html, r'"title"\s*:\s*"([^"]+)"')
    title = unicodize(title)

    for quality in ['1080','720','480','380','240','144','auto']:
        try:
            real_url = info[quality][1]["url"]
            if real_url:
                break
        except KeyError:
            pass

    mime, ext, size = url_info(real_url)

    print_info(site_info, title, mime, size)
    if not info_only:
        download_urls([real_url], title, ext, size, output_dir=output_dir, merge=merge)


Instruction:
How would you code a function in Python 3 to
download a single UCAS page.

Response:
def ucas_download_single(url, output_dir = '.', merge = False, info_only = False, **kwargs):
    '''video page'''
    html = get_content(url)
    # resourceID is UUID
    resourceID = re.findall( r'resourceID":"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', html)[0]
    assert resourceID != '', 'Cannot find resourceID!'

    title = match1(html, r'
(.+)
')
    url_lists = _ucas_get_url_lists_by_resourceID(resourceID)
    assert url_lists, 'Cannot find any URL of such class!'

    for k, part in enumerate(url_lists):
        part_title = title + '_' + str(k)
        print_info(site_info, part_title, 'flv', 0)
        if not info_only:
            download_urls(part, part_title, 'flv', total_size=None, output_dir=output_dir, merge=merge)


Instruction:
Can you write a function in Python 3 where it
downloads a playlist from the UCS.

Response:
def ucas_download_playlist(url, output_dir = '.', merge = False, info_only = False, **kwargs):
    '''course page'''
    html = get_content(url)

    parts = re.findall( r'(getplaytitle.do\?.+)"', html)
    assert parts, 'No part found!'

    for part_path in parts:
        ucas_download('http://v.ucas.ac.cn/course/' + part_path, output_dir=output_dir, merge=merge, info_only=info_only)


Instruction:
Implement a Python 3 function for
downloading a Sina video by its unique vid.

Response:
def sina_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
    """Downloads a Sina video by its unique vid.
    http://video.sina.com.cn/
    """
    xml = api_req(vid)
    urls, name, size = video_info(xml)
    if urls is None:
        log.wtf(name)
    title = name
    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls(urls, title, 'flv', size, output_dir = output_dir, merge = merge)


Instruction:
How would you explain what the following Python 3 function does
def sina_download_by_vkey(vkey, title=None, output_dir='.', merge=True, info_only=False):

    url = 'http://video.sina.com/v/flvideo/%s_0.flv' % vkey
    type, ext, size = url_info(url)

    print_info(site_info, title, 'flv', size)
    if not info_only:
        download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)

Response: Downloads a Sina video by its unique vkey.


Instruction:
Write a Python 3 function for
downloading Sina videos by URL.

Response:
def sina_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    """Downloads Sina videos by URL.
    """
    if 'news.sina.com.cn/zxt' in url:
        sina_zxt(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
        return

    vid = match1(url, r'vid=(\d+)')
    if vid is None:
        video_page = get_content(url)
        vid = hd_vid = match1(video_page, r'hd_vid\s*:\s*\'([^\']+)\'')
        if hd_vid == '0':
            vids = match1(video_page, r'[^\w]vid\s*:\s*\'([^\']+)\'').split('|')
            vid = vids[-1]

    if vid is None:
        vid = match1(video_page, r'vid:"?(\d+)"?')
    if vid:
        #title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
        sina_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
    else:
        vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
        if vkey is None:
            vid = match1(url, r'#(\d+)')
            sina_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
            return
        title = match1(video_page, r'title\s*:\s*"([^"]+)"')
        sina_download_by_vkey(vkey, title=title, output_dir=output_dir, merge=merge, info_only=info_only)


Instruction:
Can you generate the documentation for the following Python 3 function
def veoh_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
    '''Get item_id'''
    if re.match(r'http://www.veoh.com/watch/\w+', url):
        item_id = match1(url, r'http://www.veoh.com/watch/(\w+)')
    elif re.match(r'http://www.veoh.com/m/watch.php\?v=\.*', url):
        item_id = match1(url, r'http://www.veoh.com/m/watch.php\?v=(\w+)')
    else:
        raise NotImplementedError('Cannot find item ID')
    veoh_download_by_id(item_id, output_dir = '.', merge = False, info_only = info_only, **kwargs)

Response: Download a VEOH archive.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def veoh_download_by_id(item_id, output_dir = '.', merge = False, info_only = False, **kwargs):
    webpage_url = 'http://www.veoh.com/m/watch.php?v={item_id}&quality=1'.format(item_id = item_id)

    #grab download URL
    a = get_content(webpage_url, decoded=True)
    url = match1(a, r'<source src="(.*?)\"\W')

    #grab title
    title = match1(a, r'<meta property="og:title" content="([^"]*)"')

    type_, ext, size = url_info(url)
    print_info(site_info, title, type_, size)
    if not info_only:
        download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)

Response: Download a single item from Veoh.


Instruction:
Can you generate the documentation for the following Python 3 function
def download_by_id(self, vid = '', title = None, output_dir='.', merge=True, info_only=False,**kwargs):

        assert vid

        self.prepare(vid = vid, title = title, **kwargs)

        self.extract(**kwargs)

        self.download(output_dir = output_dir,
                    merge = merge,
                    info_only = info_only, **kwargs)

Response: This method will download the video with the specified ID from the BokeCC cloud and extract it into the specified directory.


Instruction:
Write a Python 3 function for
extracting video ID from live. qq. com.

Response:
def get_vid_from_url(self, url):
        """Extracts video ID from live.qq.com.
        """
        hit = re.search(r'live.qq.com/(\d+)', url)
        if hit is not None:
            return hit.group(1)
        hit = re.search(r'live.qq.com/directory/match/(\d+)', url)
        if hit is not None:
            return self.get_room_id_from_url(hit.group(1))
        html = get_content(url)
        room_id = match1(html, r'room_id\":(\d+)')
        if room_id is None:
            log.wtf('Unknown page {}'.format(url))
        return room_id


Instruction:
Write a Python 3 function that can
format text with color or other effects into ANSI escaped string.

Response:
def sprint(text, *colors):
    """Format text with color or other effects into ANSI escaped string."""
    return "\33[{}m{content}\33[{}m".format(";".join([str(color) for color in colors]), RESET, content=text) if IS_ANSI_TERMINAL and colors else text


Instruction:
Can you tell what is the following Python 3 function doing
def print_log(text, *colors):
    sys.stderr.write(sprint("{}: {}".format(script_name, text), *colors) + "\n")

Response: Print a log message to standard error.


Instruction:
Can you create a Python 3 function that
prints an error log message.

Response:
def e(message, exit_code=None):
    """Print an error log message."""
    print_log(message, YELLOW, BOLD)
    if exit_code is not None:
        sys.exit(exit_code)


Instruction:
How would you explain what the following Python 3 function does
def wtf(message, exit_code=1):
    print_log(message, RED, BOLD)
    if exit_code is not None:
        sys.exit(exit_code)

Response: What a Terrible Failure!


Instruction:
Write a Python 3 function for
downloading a single file from the weibo. com.

Response:
def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = False, **kwargs):
    '''Source: Android mobile'''
    page_url = 'http://video.weibo.com/show?fid=' + fid + '&type=mp4'

    mobile_page = get_content(page_url, headers=fake_headers_mobile)
    url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
    if url is None:
        wb_mp = re.search(r'<script src=([\'"])(.+?wb_mp\.js)\1>', mobile_page).group(2)
        return miaopai_download_by_wbmp(wb_mp, fid, output_dir=output_dir, merge=merge,
                                        info_only=info_only, total_size=None, **kwargs)
    title = match1(mobile_page, r'<title>((.|\n)+?)</title>')
    if not title:
        title = fid
    title = title.replace('\n', '_')
    ext, size = 'mp4', url_info(url)[2]
    print_info(site_info, title, ext, size)
    if not info_only:
        download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)


Instruction:
Write a Python 3 script for
downloading a vimeo file by channel.

Response:
def vimeo_download_by_channel(url, output_dir='.', merge=False, info_only=False, **kwargs):
    """str->None"""
    # https://vimeo.com/channels/464686
    channel_id = match1(url, r'http://vimeo.com/channels/(\w+)')
    vimeo_download_by_channel_id(channel_id, output_dir, merge, info_only, **kwargs)


Instruction:
Make a summary of the following Python 3 code
def vimeo_download_by_channel_id(channel_id, output_dir='.', merge=False, info_only=False, **kwargs):
    html = get_content('https://api.vimeo.com/channels/{channel_id}/videos?access_token={access_token}'.format(channel_id=channel_id, access_token=access_token))
    data = loads(html)
    id_list = []

    #print(data)
    for i in data['data']:
        id_list.append(match1(i['uri'], r'/videos/(\w+)'))

    for id in id_list:
        try:
            vimeo_download_by_id(id, None, output_dir, merge, info_only, **kwargs)
        except urllib.error.URLError as e:
            log.w('{} failed with {}'.format(id, e))

Response: Download all vimeo videos from a channel.


Instruction:
Write a Python 3 script to
download a Vimeo video by its id.

Response:
def vimeo_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
    '''
    try:
        # normal Vimeo video
        html = get_content('https://vimeo.com/' + id)
        cfg_patt = r'clip_page_config\s*=\s*(\{.+?\});'
        cfg = json.loads(match1(html, cfg_patt))
        video_page = get_content(cfg['player']['config_url'], headers=fake_headers)
        title = cfg['clip']['title']
        info = loads(video_page)
    except:
        # embedded player - referer may be required
        if 'referer' in kwargs:
            fake_headers['Referer'] = kwargs['referer']

        video_page = get_content('http://player.vimeo.com/video/%s' % id, headers=fake_headers)
        title = r1(r'<title>([^<]+)</title>', video_page)
        info = loads(match1(video_page, r'var t=(\{.+?\});'))

    streams = info['request']['files']['progressive']
    streams = sorted(streams, key=lambda i: i['height'])
    url = streams[-1]['url']

    type, ext, size = url_info(url, faker=True)

    print_info(site_info, title, type, size)
    if not info_only:
        download_urls([url], title, ext, size, output_dir, merge=merge, faker=True)
    '''
    site = VimeoExtractor()
    site.download_by_vid(id, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)


Instruction:
Can you generate the documentation for the following Python 3 function
def ckplayer_get_info_by_xml(ckinfo):
    e = ET.XML(ckinfo)
    video_dict = {'title': '',
                  #'duration': 0,
                  'links': [],
                  'size': 0,
                  'flashvars': '',}
    dictified = dictify(e)['ckplayer']
    if 'info' in dictified:
        if '_text' in dictified['info'][0]['title'][0]:  #title
            video_dict['title'] = dictified['info'][0]['title'][0]['_text'].strip()

    #if dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip():  #duration
        #video_dict['title'] = dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip()

    if '_text' in dictified['video'][0]['size'][0]:  #size exists for 1 piece
        video_dict['size'] = sum([int(i['size'][0]['_text']) for i in dictified['video']])

    if '_text' in dictified['video'][0]['file'][0]:  #link exist
        video_dict['links'] = [i['file'][0]['_text'].strip() for i in dictified['video']]

    if '_text' in dictified['flashvars'][0]:
        video_dict['flashvars'] = dictified['flashvars'][0]['_text'].strip()

    return video_dict

Response: Get CKPlayer API content by XML.


Instruction:
How would you implement a function in Python 3 that
gets video URL from video ID

Response:
def get_video_url_from_video_id(video_id):
    """Splicing URLs according to video ID to get video details"""
    # from js
    data = [""] * 256
    for index, _ in enumerate(data):
        t = index
        for i in range(8):
            t = -306674912 ^ unsigned_right_shitf(t, 1) if 1 & t else unsigned_right_shitf(t, 1)
        data[index] = t

    def tmp():
        rand_num = random.random()
        path = "/video/urls/v/1/toutiao/mp4/{video_id}?r={random_num}".format(video_id=video_id,
                                                                              random_num=str(rand_num)[2:])
        e = o = r = -1
        i, a = 0, len(path)
        while i < a:
            e = ord(path[i])
            i += 1
            if e < 128:
                r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ e)]
            else:
                if e < 2048:
                    r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (192 | e >> 6 & 31))]
                    r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & e))]
                else:
                    if 55296 <= e < 57344:
                        e = (1023 & e) + 64
                        i += 1
                        o = 1023 & t.url(i)
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (240 | e >> 8 & 7))]
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | e >> 2 & 63))]
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | o >> 6 & 15 | (3 & e) << 4))]
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & o))]
                    else:
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (224 | e >> 12 & 15))]
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | e >> 6 & 63))]
                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & e))]

        return "https://ib.365yg.com{path}&s={param}".format(path=path, param=unsigned_right_shitf(r ^ -1, 0))

    while 1:
        url = tmp()
        if url.split("=")[-1][0] != "-":  # 参数s不能为负数
            return url


Instruction:
How would you explain what the following Python 3 function does
def get_vid_from_url(url):
        vid = match1(url, 'https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
        if not vid:
            vid = match1(url, 'https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
        return vid

Response: Extracts video ID from URL.


Instruction:
Here you have a function in Python 3, explain what it does
def get_mgtv_real_url(url):
        content = loads(get_content(url))
        m3u_url = content['info']
        split = urlsplit(m3u_url)

        base_url = "{scheme}://{netloc}{path}/".format(scheme = split[0],
                                                      netloc = split[1],
                                                      path = dirname(split[2]))

        content = get_content(content['info'])  #get the REAL M3U url, maybe to be changed later?
        segment_list = []
        segments_size = 0
        for i in content.split():
            if not i.startswith('#'):  #not the best way, better we use the m3u8 package
                segment_list.append(base_url + i)
            # use ext-info for fast size calculate
            elif i.startswith('#EXT-MGTV-File-SIZE:'):
                segments_size += int(i[i.rfind(':')+1:])

        return m3u_url, segments_size, segment_list

Response: Get the REAL M3U url and size of the real files.


Instruction:
Can you write a function in Python 3 where it
gets branch and commit from HEAD of a git repo.

Response:
def get_head(repo_path):
    """Get (branch, commit) from HEAD of a git repo."""
    try:
        ref = open(os.path.join(repo_path, '.git', 'HEAD'), 'r').read().strip()[5:].split('/')
        branch = ref[-1]
        commit = open(os.path.join(repo_path, '.git', *ref), 'r').read().strip()[:7]
        return branch, commit
    except:
        return None


Instruction:
How would you explain what the following Python 3 function does
def legitimize(text, os=detect_os()):

    # POSIX systems
    text = text.translate({
        0: None,
        ord('/'): '-',
        ord('|'): '-',
    })

    # FIXME: do some filesystem detection
    if os == 'windows' or os == 'cygwin' or os == 'wsl':
        # Windows (non-POSIX namespace)
        text = text.translate({
            # Reserved in Windows VFAT and NTFS
            ord(':'): '-',
            ord('*'): '-',
            ord('?'): '-',
            ord('\\'): '-',
            ord('\"'): '\'',
            # Reserved in Windows VFAT
            ord('+'): '-',
            ord('<'): '-',
            ord('>'): '-',
            ord('['): '(',
            ord(']'): ')',
            ord('\t'): ' ',
        })
    else:
        # *nix
        if os == 'mac':
            # Mac OS HFS+
            text = text.translate({
                ord(':'): '-',
            })

        # Remove leading .
        if text.startswith("."):
            text = text[1:]

    text = text[:80] # Trim to 82 Unicode characters long
    return text

Response: Converts a string to a valid filename.


Instruction:
Can you tell what is the following Python 3 function doing
def get_terminal_size():
    try:
        import fcntl, termios, struct # fcntl module only available on Unix
        return struct.unpack('hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234'))
    except:
        return (40, 80)

Response: Get the width and height of the current terminal.


Instruction:
Create a Python 3 function for
downloading CBS videos by URL.

Response:
def cbs_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    """Downloads CBS videos by URL.
    """

    html = get_content(url)
    pid = match1(html, r'video\.settings\.pid\s*=\s*\'([^\']+)\'')
    title = match1(html, r'video\.settings\.title\s*=\s*\"([^\"]+)\"')

    theplatform_download_by_pid(pid, title, output_dir=output_dir, merge=merge, info_only=info_only)


Instruction:
Make a summary of the following Python 3 code
def download(self, **kwargs):
        if 'json_output' in kwargs and kwargs['json_output']:
            json_output.output(self)
        elif 'info_only' in kwargs and kwargs['info_only']:
            if 'stream_id' in kwargs and kwargs['stream_id']:
                # Display the stream
                stream_id = kwargs['stream_id']
                if 'index' not in kwargs:
                    self.p(stream_id)
                else:
                    self.p_i(stream_id)
            else:
                # Display all available streams
                if 'index' not in kwargs:
                    self.p([])
                else:
                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
                    self.p_i(stream_id)

        else:
            if 'stream_id' in kwargs and kwargs['stream_id']:
                # Download the stream
                stream_id = kwargs['stream_id']
            else:
                # Download stream with the best quality
                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']

            if 'index' not in kwargs:
                self.p(stream_id)
            else:
                self.p_i(stream_id)

            if stream_id in self.streams:
                urls = self.streams[stream_id]['src']
                ext = self.streams[stream_id]['container']
                total_size = self.streams[stream_id]['size']
            else:
                urls = self.dash_streams[stream_id]['src']
                ext = self.dash_streams[stream_id]['container']
                total_size = self.dash_streams[stream_id]['size']

            if not urls:
                log.wtf('[Failed] Cannot extract video source.')
            # For legacy main()

            #Here's the change!!
            download_url_ffmpeg(urls[0], self.title, 'mp4', output_dir=kwargs['output_dir'], merge=kwargs['merge'], stream=False)

            if not kwargs['caption']:
                print('Skipping captions.')
                return
            for lang in self.caption_tracks:
                filename = '%s.%s.srt' % (get_filename(self.title), lang)
                print('Saving %s ... ' % filename, end="", flush=True)
                srt = self.caption_tracks[lang]
                with open(os.path.join(kwargs['output_dir'], filename),
                          'w', encoding='utf-8') as x:
                    x.write(srt)
                print('Done.')

Response: Download the one available one - language object from the DASH and return the ID of the object.


Instruction:
Write a Python 3 function for
downloading Acfun video by vid.

Response:
def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
    """str, str, str, bool, bool ->None

    Download Acfun video by vid.

    Call Acfun API, decide which site to use, and pass the job to its
    extractor.
    """

    #first call the main parasing API
    info = json.loads(get_content('http://www.acfun.cn/video/getVideo.aspx?id=' + vid))

    sourceType = info['sourceType']

    #decide sourceId to know which extractor to use
    if 'sourceId' in info: sourceId = info['sourceId']
    # danmakuId = info['danmakuId']

    #call extractor decided by sourceId
    if sourceType == 'sina':
        sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
    elif sourceType == 'youku':
        youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
    elif sourceType == 'tudou':
        tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
    elif sourceType == 'qq':
        qq_download_by_vid(sourceId, title, True, output_dir=output_dir, merge=merge, info_only=info_only)
    elif sourceType == 'letv':
        letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
    elif sourceType == 'zhuzhan':
        #As in Jul.28.2016, Acfun is using embsig to anti hotlink so we need to pass this
#In Mar. 2017 there is a dedicated ``acfun_proxy'' in youku cloud player
#old code removed
        url = 'http://www.acfun.cn/v/ac' + vid
        yk_streams = youku_acfun_proxy(info['sourceId'], info['encode'], url)
        seq = ['mp4hd3', 'mp4hd2', 'mp4hd', 'flvhd']
        for t in seq:
            if yk_streams.get(t):
                preferred = yk_streams[t]
                break
#total_size in the json could be incorrect(F.I. 0)
        size = 0
        for url in preferred[0]:
            _, _, seg_size = url_info(url)
            size += seg_size
#fallback to flvhd is not quite possible
        if re.search(r'fid=[0-9A-Z\-]*.flv', preferred[0][0]):
            ext = 'flv'
        else:
            ext = 'mp4'
        print_info(site_info, title, ext, size)
        if not info_only:
            download_urls(preferred[0], title, ext, size, output_dir=output_dir, merge=merge)
    else:
        raise NotImplementedError(sourceType)

    if not info_only and not dry_run:
        if not kwargs['caption']:
            print('Skipping danmaku.')
            return
        try:
            title = get_filename(title)
            print('Downloading %s ...\n' % (title + '.cmt.json'))
            cmt = get_srt_json(vid)
            with open(os.path.join(output_dir, title + '.cmt.json'), 'w', encoding='utf-8') as x:
                x.write(cmt)
        except:
            pass


Instruction:
Write a Python 3 script to
download a set of files with FFMPEG and return the path to the output file.

Response:
def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.', stream=True):
    """str, str->True
    WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
    You can basicly download anything with this function
    but better leave it alone with
    """
    output = title + '.' + ext

    if not (output_dir == '.'):
        output = output_dir + '/' + output

    print('Downloading streaming content with FFmpeg, press q to stop recording...')
    if stream:
        ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
    else:
        ffmpeg_params = [FFMPEG] + ['-y', '-i']
    ffmpeg_params.append(files)  #not the same here!!!!

    if FFMPEG == 'avconv':  #who cares?
        ffmpeg_params += ['-c', 'copy', output]
    else:
        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']

    if params is not None:
        if len(params) > 0:
            for k, v in params:
                ffmpeg_params.append(k)
                ffmpeg_params.append(v)

    ffmpeg_params.append(output)

    print(' '.join(ffmpeg_params))

    try:
        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
        a.communicate()
    except KeyboardInterrupt:
        try:
            a.stdin.write('q'.encode('utf-8'))
        except:
            pass

    return True


Instruction:
Write a Python 3 function for
scanning through a string for substrings matched some patterns. Returns None if no match is found.

Response:
def match1(text, *patterns):
    """Scans through a string for substrings matched some patterns (first-subgroups only).

    Args:
        text: A string to be scanned.
        patterns: Arbitrary number of regex patterns.

    Returns:
        When only one pattern is given, returns a string (None if no match found).
        When more than one pattern are given, returns a list of strings ([] if no match found).
    """

    if len(patterns) == 1:
        pattern = patterns[0]
        match = re.search(pattern, text)
        if match:
            return match.group(1)
        else:
            return None
    else:
        ret = []
        for pattern in patterns:
            match = re.search(pattern, text)
            if match:
                ret.append(match.group(1))
        return ret


Instruction:
Implement a Python 3 function for
scanning through a string for substrings matched some patterns. Returns a list of the names of the substrings matched.

Response:
def matchall(text, patterns):
    """Scans through a string for substrings matched some patterns.

    Args:
        text: A string to be scanned.
        patterns: a list of regex pattern.

    Returns:
        a list if matched. empty if not.
    """

    ret = []
    for pattern in patterns:
        match = re.findall(pattern, text)
        ret += match

    return ret