Instruction:
Can you write a function in Python 3 where it
adds all endpoints middleware to a wsgi application.

Response:
def add_all(application, project_id, control_client,
            loader=service.Loaders.FROM_SERVICE_MANAGEMENT):
    """Adds all endpoints middleware to a wsgi application.

    Sets up application to use all default endpoints middleware.

    Example:

      >>> application = MyWsgiApp()  # an existing WSGI application
      >>>
      >>> # the name of the controlled service
      >>> service_name = 'my-service-name'
      >>>
      >>> # A GCP project  with service control enabled
      >>> project_id = 'my-project-id'
      >>>
      >>> # wrap the app for service control
      >>> from endpoints_management.control import wsgi
      >>> control_client = client.Loaders.DEFAULT.load(service_name)
      >>> control_client.start()
      >>> wrapped_app = add_all(application, project_id, control_client)
      >>>
      >>> # now use wrapped_app in place of app

    Args:
       application: the wrapped wsgi application
       project_id: the project_id thats providing service control support
       control_client: the service control client instance
       loader (:class:`endpoints_management.control.service.Loader`): loads the service
          instance that configures this instance's behaviour
    """
    return ConfigFetchWrapper(application, project_id, control_client, loader)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _create_authenticator(a_service):
    if not isinstance(a_service, sm_messages.Service):
        raise ValueError(u"service is None or not an instance of Service")

    authentication = a_service.authentication
    if not authentication:
        _logger.info(u"authentication is not configured in service, "
                     u"authentication checks will be disabled")
        return

    issuers_to_provider_ids = {}
    issuer_uri_configs = {}
    for provider in authentication.providers:
        issuer = provider.issuer
        jwks_uri = provider.jwksUri

        # Enable openID discovery if jwks_uri is unset
        open_id = jwks_uri is None
        issuer_uri_configs[issuer] = suppliers.IssuerUriConfig(open_id, jwks_uri)
        issuers_to_provider_ids[issuer] = provider.id

    key_uri_supplier = suppliers.KeyUriSupplier(issuer_uri_configs)
    jwks_supplier = suppliers.JwksSupplier(key_uri_supplier)
    authenticator = tokens.Authenticator(issuers_to_provider_ids, jwks_supplier)
    return authenticator

Response: Create an instance of google. auth. tokens. Authenticator.


Instruction:
Can you write a function in Python 3 where it
obtains a unique signature for an operation in a ReportRequest.

Response:
def _sign_operation(op):
    """Obtains a signature for an operation in a ReportRequest.

    Args:
       op (:class:`endpoints_management.gen.servicecontrol_v1_messages.Operation`): an
         operation used in a `ReportRequest`

    Returns:
       string: a unique signature for that operation
    """
    md5 = hashlib.md5()
    md5.update(op.consumerId.encode('utf-8'))
    md5.update(b'\x00')
    md5.update(op.operationName.encode('utf-8'))
    if op.labels:
        signing.add_dict_to_hash(md5, encoding.MessageToPyValue(op.labels))
    return md5.digest()


Instruction:
Implement a function in Python 3 to
make a LogEntry from this instance with the given log_name and the given datetime.

Response:
def _as_log_entry(self, name, now):
        """Makes a `LogEntry` from this instance for the given log_name.

        Args:
          rules (:class:`ReportingRules`): determines what labels, metrics and
            logs to include in the report request.
          now (:class:`datetime.DateTime`): the current time

        Return:
          a ``LogEntry`` generated from this instance with the given name
          and timestamp

        Raises:
          ValueError: if the fields in this instance are insufficient to
            to create a valid ``ServicecontrolServicesReportRequest``

        """
        # initialize the struct with fields that are always present
        d = {
            u'http_response_code': self.response_code,
            u'timestamp': time.mktime(now.timetuple())
        }

        # compute the severity
        severity = _SEVERITY.INFO
        if self.response_code >= 400:
            severity = _SEVERITY.ERROR
            d[u'error_cause'] = self.error_cause.name

        # add 'optional' fields to the struct
        if self.request_size > 0:
            d[u'request_size'] = self.request_size
        if self.response_size > 0:
            d[u'response_size'] = self.response_size
        if self.method:
            d[u'http_method'] = self.method
        if self.request_time:
            d[u'request_latency_in_ms'] = self.request_time.total_seconds() * 1000

        # add 'copyable' fields to the struct
        for key in self.COPYABLE_LOG_FIELDS:
            value = getattr(self, key, None)
            if value:
                d[key] = value

        return sc_messages.LogEntry(
            name=name,
            timestamp=timestamp.to_rfc3339(now),
            severity=severity,
            structPayload=_struct_payload_from(d))


Instruction:
Can you create a Python 3 function that
creates a ServicecontrolServicesReportRequest object from this instance and the provided rules.

Response:
def as_report_request(self, rules, timer=datetime.utcnow):
        """Makes a `ServicecontrolServicesReportRequest` from this instance

        Args:
          rules (:class:`ReportingRules`): determines what labels, metrics and
            logs to include in the report request.
          timer: a function that determines the current time

        Return:
          a ``ServicecontrolServicesReportRequest`` generated from this instance
          governed by the provided ``rules``

        Raises:
          ValueError: if the fields in this instance cannot be used to create
            a valid ``ServicecontrolServicesReportRequest``

        """
        if not self.service_name:
            raise ValueError(u'the service name must be set')
        op = super(Info, self).as_operation(timer=timer)

        # Populate metrics and labels if they can be associated with a
        # method/operation
        if op.operationId and op.operationName:
            labels = {}
            for known_label in rules.labels:
                known_label.do_labels_update(self, labels)

            # Forcibly add system label reporting here, as the base service
            # config does not specify it as a label.
            labels[_KNOWN_LABELS.SCC_PLATFORM.label_name] = (
                self.platform.friendly_string())
            labels[_KNOWN_LABELS.SCC_SERVICE_AGENT.label_name] = (
                SERVICE_AGENT)
            labels[_KNOWN_LABELS.SCC_USER_AGENT.label_name] = USER_AGENT

            if labels:
                op.labels = encoding.PyValueToMessage(
                    sc_messages.Operation.LabelsValue,
                    labels)
            for known_metric in rules.metrics:
                known_metric.do_operation_update(self, op)

        # Populate the log entries
        now = timer()
        op.logEntries = [self._as_log_entry(l, now) for l in rules.logs]

        return sc_messages.ServicecontrolServicesReportRequest(
            serviceName=self.service_name,
            reportRequest=sc_messages.ReportRequest(operations=[op]))


Instruction:
Create a Python 3 function for
flushing this instance s cache.

Response:
def flush(self):
        """Flushes this instance's cache.

        The driver of this instance should call this method every
        `flush_interval`.

        Returns:
          list[``ServicecontrolServicesReportRequest``]: corresponding to the
            pending cached operations

        """
        if self._cache is None:
            return _NO_RESULTS
        with self._cache as c:
            flushed_ops = [x.as_operation() for x in c.out_deque]
            c.out_deque.clear()
            reqs = []
            max_ops = self.MAX_OPERATION_COUNT
            for x in range(0, len(flushed_ops), max_ops):
                report_request = sc_messages.ReportRequest(
                    operations=flushed_ops[x:x + max_ops])
                reqs.append(
                    sc_messages.ServicecontrolServicesReportRequest(
                        serviceName=self.service_name,
                        reportRequest=report_request))

            return reqs


Instruction:
Create a Python 3 function for
adding a report request to the cache.

Response:
def report(self, req):
        """Adds a report request to the cache.

        Returns ``None`` if it could not be aggregated, and callers need to
        send the request to the server, otherwise it returns ``CACHED_OK``.

        Args:
           req (:class:`sc_messages.ReportRequest`): the request
             to be aggregated

        Result:
           ``None`` if the request as not cached, otherwise ``CACHED_OK``

        """
        if self._cache is None:
            return None  # no cache, send request now
        if not isinstance(req, sc_messages.ServicecontrolServicesReportRequest):
            raise ValueError(u'Invalid request')
        if req.serviceName != self.service_name:
            _logger.error(u'bad report(): service_name %s does not match ours %s',
                          req.serviceName, self.service_name)
            raise ValueError(u'Service name mismatch')
        report_req = req.reportRequest
        if report_req is None:
            _logger.error(u'bad report(): no report_request in %s', req)
            raise ValueError(u'Expected report_request not set')
        if _has_high_important_operation(report_req) or self._cache is None:
            return None
        ops_by_signature = _key_by_signature(report_req.operations,
                                             _sign_operation)

        # Concurrency:
        #
        # This holds a lock on the cache while updating it.  No i/o operations
        # are performed, so any waiting threads see minimal delays
        with self._cache as cache:
            for key, op in ops_by_signature.items():
                agg = cache.get(key)
                if agg is None:
                    cache[key] = operation.Aggregator(op, self._kinds)
                else:
                    agg.add(op)

        return self.CACHED_OK


Instruction:
Can you implement a function in Python 3 that
converts an AllocateQuotaResponse to a tuple.

Response:
def convert_response(allocate_quota_response, project_id):
    """Computes a http status code and message `AllocateQuotaResponse`

    The return value a tuple (code, message) where

    code: is the http status code
    message: is the message to return

    Args:
       allocate_quota_response (:class:`endpoints_management.gen.servicecontrol_v1_messages.AllocateQuotaResponse`):
         the response from calling an api

    Returns:
       tuple(code, message)
    """
    if not allocate_quota_response or not allocate_quota_response.allocateErrors:
        return _IS_OK

    # only allocate_quota the first error for now, as per ESP
    theError = allocate_quota_response.allocateErrors[0]
    error_tuple = _QUOTA_ERROR_CONVERSION.get(theError.code, _IS_UNKNOWN)
    if error_tuple[1].find(u'{') == -1:  # no replacements needed:
        return error_tuple

    updated_msg = error_tuple[1].format(project_id=project_id, detail=theError.description or u'')
    return error_tuple[0], updated_msg


Instruction:
Explain what the following Python 3 code does
def sign(allocate_quota_request):
    if not isinstance(allocate_quota_request, sc_messages.AllocateQuotaRequest):
        raise ValueError(u'Invalid request')
    op = allocate_quota_request.allocateOperation
    if op is None or op.methodName is None or op.consumerId is None:
        logging.error(u'Bad %s: not initialized => not signed', allocate_quota_request)
        raise ValueError(u'allocate_quota request must be initialized with an operation')
    md5 = hashlib.md5()
    md5.update(op.methodName.encode('utf-8'))
    md5.update(b'\x00')
    md5.update(op.consumerId.encode('utf-8'))
    if op.labels:
        signing.add_dict_to_hash(md5, encoding.MessageToPyValue(op.labels))
    for value_set in op.quotaMetrics:
        md5.update(b'\x00')
        md5.update(value_set.metricName.encode('utf-8'))
        for mv in value_set.metricValues:
            metric_value.update_hash(md5, mv)

    md5.update(b'\x00')
    return md5.digest()

Response: Gets a secure hash for an operation in a AllocateQuotaRequest.


Instruction:
Write a Python 3 function for
making a ServicecontrolServicesAllocateQuotaRequest from this instance.

Response:
def as_allocate_quota_request(self, timer=datetime.utcnow):
        """Makes a `ServicecontrolServicesAllocateQuotaRequest` from this instance

        Returns:
          a ``ServicecontrolServicesAllocateQuotaRequest``

        Raises:
          ValueError: if the fields in this instance are insufficient to
            to create a valid ``ServicecontrolServicesAllocateQuotaRequest``

        """
        if not self.service_name:
            raise ValueError(u'the service name must be set')
        if not self.operation_id:
            raise ValueError(u'the operation id must be set')
        if not self.operation_name:
            raise ValueError(u'the operation name must be set')
        op = super(Info, self).as_operation(timer=timer)
        labels = {}
        if self.client_ip:
            labels[_KNOWN_LABELS.SCC_CALLER_IP.label_name] = self.client_ip

        if self.referer:
            labels[_KNOWN_LABELS.SCC_REFERER.label_name] = self.referer

        qop = sc_messages.QuotaOperation(
            operationId=op.operationId,
            methodName=op.operationName,
            consumerId=op.consumerId,
            quotaMode=sc_messages.QuotaOperation.QuotaModeValueValuesEnum.BEST_EFFORT,
        )
        qop.labels = encoding.PyValueToMessage(
            sc_messages.QuotaOperation.LabelsValue, labels)

        quota_info = self.quota_info if self.quota_info else {}
        qop.quotaMetrics = [
            sc_messages.MetricValueSet(
                metricName=name, metricValues=[sc_messages.MetricValue(int64Value=cost)])
            for name, cost in quota_info.items()
        ]

        allocate_quota_request = sc_messages.AllocateQuotaRequest(allocateOperation=qop)
        if self.config_id:
            allocate_quota_request.serviceConfigId = self.config_id
        return sc_messages.ServicecontrolServicesAllocateQuotaRequest(
            serviceName=self.service_name,
            allocateQuotaRequest=allocate_quota_request)


Instruction:
How would you explain what the following Python 3 function does
def flush(self):
        if self._cache is None:
            return []
        with self._cache as c, self._out as out:
            c.expire()
            now = self._timer()
            for item in c.values():
                if (not self._in_flush_all) and (not self._should_expire(item)):
                    if (not item.is_in_flight) and item._op_aggregator is not None:
                        item.is_in_flight = True
                        item.last_refresh_timestamp = now
                        out.append(item.extract_request())  # pylint: disable=no-member
            flushed_items = list(out)
            out.clear()  # pylint: disable=no-member
            for req in flushed_items:
                assert isinstance(req, sc_messages.ServicecontrolServicesAllocateQuotaRequest)
            return flushed_items

Response: Flushes this instance s cache.


Instruction:
Can you implement a function in Python 3 that
clears this instance s cache.

Response:
def clear(self):
        """Clears this instance's cache."""
        if self._cache is not None:
            with self._cache as c, self._out as out:
                self.in_flush_all = True
                c.clear()
                out.clear()  # pylint: disable=no-member
                self.in_flush_all = False


Instruction:
Write a Python 3 script to
add the response from sending to req to this instance s cache.

Response:
def add_response(self, req, resp):
        """Adds the response from sending to `req` to this instance's cache.

        Args:
          req (`ServicecontrolServicesAllocateQuotaRequest`): the request
          resp (AllocateQuotaResponse): the response from sending the request
        """
        if self._cache is None:
            return
        signature = sign(req.allocateQuotaRequest)
        with self._cache as c:
            now = self._timer()
            item = c.get(signature)
            if item is None:
                c[signature] = CachedItem(
                    req, resp, self.service_name, now)
            else:
                # Update the cached item to reflect that it is updated
                item.last_check_time = now
                item.response = resp
                item.is_in_flight = False
                c[signature] = item


Instruction:
Given the following Python 3 function, write the documentation
def add_route(app_or_blueprint, fn, context=default_context):
    transmute_func = TransmuteFunction(
        fn,
        args_not_from_request=["request"]
    )
    handler = create_handler(transmute_func, context=context)
    get_swagger_spec(app_or_blueprint).add_func(transmute_func, context)
    for p in transmute_func.paths:
        sanic_path = _convert_to_sanic_path(p)
        app_or_blueprint.add_route(handler, sanic_path, methods=list(transmute_func.methods))

Response: A decorator that adds a transmute route to the application.


Instruction:
Can you generate the documentation for the following Python 3 function
def matches(self, desc):
        return (self.metric_name == desc.name and
                self.kind == desc.metricKind and
                self.value_type == desc.valueType)

Response: Determines if a given metric descriptor matches this enum instance instance


Instruction:
Create a Python 3 function for
updating an operation using the assigned update_op_func

Response:
def do_operation_update(self, info, an_op):
        """Updates an operation using the assigned update_op_func

        Args:
           info: (:class:`endpoints_management.control.report_request.Info`): the
              info instance to update
           an_op: (:class:`endpoints_management.control.report_request.Info`):
              the info instance to update

        Return:
           `True` if desc is supported, otherwise `False`

        """
        self.update_op_func(self.metric_name, info, an_op)


Instruction:
Can you create a Python 3 function that
determines if the given metric descriptor is supported.

Response:
def is_supported(cls, desc):
        """Determines if the given metric descriptor is supported.

        Args:
           desc (:class:`endpoints_management.gen.servicecontrol_v1_messages.MetricDescriptor`): the
             metric descriptor to test

        Return:
           `True` if desc is supported, otherwise `False`

        """
        for m in cls:
            if m.matches(desc):
                return True
        return False


Instruction:
Explain what the following Python 3 code does
def add_dict_to_hash(a_hash, a_dict):
    if a_dict is None:
        return
    for k, v in a_dict.items():
        a_hash.update(b'\x00' + k.encode('utf-8') + b'\x00' + v.encode('utf-8'))

Response: Adds a_dict to a_hash


Instruction:
Can you generate the documentation for the following Python 3 function
def add_swagger(app, json_route, html_route):
    app.add_route(create_swagger_json_handler(app), json_route, methods=["GET"])
    add_swagger_api_route(app, html_route, json_route)

Response: add a swagger. json route and a swagger. html route


Instruction:
Explain what the following Python 3 code does
def add_swagger_api_route(app, target_route, swagger_json_route):
    static_root = get_swagger_static_root()
    swagger_body = generate_swagger_html(
        STATIC_ROOT, swagger_json_route
    ).encode("utf-8")

    async def swagger_ui(request):
        return HTTPResponse(body_bytes=swagger_body, content_type="text/html")

    bp = Blueprint('swagger')
    bp.static(STATIC_ROOT, static_root)

    app.add_route(swagger_ui, target_route, methods=["GET"])
    app.blueprint(bp)

Response: Add a route to the sanic app that will serve the swagger json file.


Instruction:
Can you tell what is the following Python 3 function doing
def create_swagger_json_handler(app, **kwargs):

    spec = get_swagger_spec(app)
    _add_blueprint_specs(app, spec)
    spec_dict = spec.swagger_definition(**kwargs)
    encoded_spec = json.dumps(spec_dict).encode("UTF-8")

    async def swagger(request):
        return HTTPResponse(
            body_bytes=encoded_spec,
            headers={
                "Access-Control-Allow-Origin": "*"
            },
            content_type="application/json",
        )

    return swagger

Response: Create a handler that returns the swagger definition for an application.


Instruction:
Can you generate the documentation for the following Python 3 function
def compute_to_fields(self, to_fields):
        # for problem in trim_join, we must try to give the fields in a consistent order with others models...
        # see #26515 at  https://code.djangoproject.com/ticket/26515

        return OrderedDict(
            (k, (v if isinstance(v, CompositePart) else LocalFieldValue(v)))
            for k, v in (to_fields.items() if isinstance(to_fields, dict) else zip(to_fields, to_fields))
        )

Response: compute the to_fields parameterse to make it uniformly a dict of CompositePart objects


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_lookup(self, main_field, for_remote, alias):
        lookup_class = for_remote.get_lookup("exact")
        return lookup_class(for_remote.get_col(alias), self.value)

Response: create a fake field for the lookup capability


Instruction:
Explain what the following Python 3 code does
def is_valid(self):
        length = len(self.fqdn)
        if self.fqdn.endswith('.'):
            length -= 1
        if length > 253:
            return False
        return bool(self.FQDN_REGEX.match(self.fqdn))

Response: Returns True if the FQDN is valid.


Instruction:
Write a Python 3 script for
returning the FQDN as a string in relative form.

Response:
def relative(self):
        """
        The FQDN as a string in relative form
        """
        if not self.is_valid:
            raise ValueError('invalid FQDN `{0}`'.format(self.fqdn))

        if self.is_valid_absolute:
            return self.fqdn[:-1]

        return self.fqdn


Instruction:
Here you have a function in Python 3, explain what it does
def _available_cpu_count():
    try:
        return len(os.sched_getaffinity(0))
    except AttributeError:
        pass
    import re
    try:
        with open('/proc/self/status') as f:
            status = f.read()
        m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', status)
        if m:
            res = bin(int(m.group(1).replace(',', ''), 16)).count('1')
            if res > 0:
                return res
    except IOError:
        pass
    try:
        import multiprocessing
        return multiprocessing.cpu_count()
    except (ImportError, NotImplementedError):
        return 1

Response: Returns the number of available CPUs on this system


Instruction:
Can you generate a brief explanation for the following Python 3 code
def xopen(filename, mode='r', compresslevel=6, threads=None):
    if mode in ('r', 'w', 'a'):
        mode += 't'
    if mode not in ('rt', 'rb', 'wt', 'wb', 'at', 'ab'):
        raise ValueError("mode '{0}' not supported".format(mode))
    if not _PY3:
        mode = mode[0]
    filename = fspath(filename)
    if not isinstance(filename, basestring):
        raise ValueError("the filename must be a string")
    if compresslevel not in range(1, 10):
        raise ValueError("compresslevel must be between 1 and 9")

    if filename == '-':
        return _open_stdin_or_out(mode)
    elif filename.endswith('.bz2'):
        return _open_bz2(filename, mode)
    elif filename.endswith('.xz'):
        return _open_xz(filename, mode)
    elif filename.endswith('.gz'):
        return _open_gz(filename, mode, compresslevel, threads)
    else:
        # Python 2.6 and 2.7 have io.open, which we could use to make the returned
        # object consistent with the one returned in Python 3, but reading a file
        # with io.open() is 100 times slower (!) on Python 2.6, and still about
        # three times slower on Python 2.7 (tested with "for _ in io.open(path): pass")
        return open(filename, mode)

Response: A replacement for the open function that can also open files that have been compressed with gzip bzip2 or xz.


Instruction:
Write a Python 3 function that can
raise IOError if process is not running anymore and the process exit code is nonzero.

Response:
def _raise_if_error(self):
        """
        Raise IOError if process is not running anymore and the
        exit code is nonzero.
        """
        retcode = self.process.poll()
        if retcode is not None and retcode != 0:
            message = self._stderr.read().strip()
            raise IOError(message)


Instruction:
Can you tell what is the following Python 3 function doing
def run(self, trials=100, strategy=None, parameters=None):
        '''
        Run MAB test with T trials.

        Parameters
        ----------
        trials : int
            Number of trials to run.
        strategy : str
            Name of selected strategy.
        parameters : dict
            Parameters for selected strategy.

        Available strategies:
            - Epsilon-greedy ("eps_greedy")
            - Softmax ("softmax")
            - Upper confidence bound ("ucb")

        Returns
        -------
        None
        '''

        if trials < 1:
            raise Exception('MAB.run: Number of trials cannot be less than 1!')
        if not strategy:
            strategy = 'eps_greedy'
        else:
            if strategy not in self.strategies:
                raise Exception('MAB,run: Strategy name invalid. Choose from:'
                                ' {}'.format(', '.join(self.strategies)))

        # Run strategy
        for n in range(trials):
            self._run(strategy, parameters)

Response: Run MAB test with T trials.


Instruction:
Explain what the following Python 3 code does
def _run(self, strategy, parameters=None):
        '''
        Run single trial of MAB strategy.

        Parameters
        ----------
        strategy : function
        parameters : dict

        Returns
        -------
        None
        '''

        choice = self.run_strategy(strategy, parameters)
        self.choices.append(choice)
        payout = self.bandits.pull(choice)
        if payout is None:
            print('Trials exhausted. No more values for bandit', choice)
            return None
        else:
            self.wins[choice] += payout
        self.pulls[choice] += 1

Response: Run a single trial of MAB strategy.


Instruction:
Create a Python 3 function to
run the Bayesian Bandit algorithm which utilizes a beta distribution for exploration and exploitation. Parameters ---------- params : None For API consistency, this function can take a parameters argument, but it is ignored. Returns ------- int Index of chosen bandit

Response:
def bayesian(self, params=None):
        '''
        Run the Bayesian Bandit algorithm which utilizes a beta distribution
        for exploration and exploitation.

        Parameters
        ----------
        params : None
            For API consistency, this function can take a parameters argument,
            but it is ignored.

        Returns
        -------
        int
            Index of chosen bandit
        '''
        p_success_arms = [
            np.random.beta(self.wins[i] + 1, self.pulls[i] - self.wins[i] + 1)
            for i in range(len(self.wins))
            ]

        return np.array(p_success_arms).argmax()


Instruction:
How would you implement a function in Python 3 that
runs the epsilon - greedy strategy and update self. max_mean

Response:
def eps_greedy(self, params):
        '''
        Run the epsilon-greedy strategy and update self.max_mean()

        Parameters
        ----------
        Params : dict
            Epsilon

        Returns
        -------
        int
            Index of chosen bandit
        '''

        if params and type(params) == dict:
            eps = params.get('epsilon')
        else:
            eps = 0.1

        r = np.random.rand()

        if r < eps:
            return np.random.choice(list(set(range(len(self.wins))) -
                                    {self.max_mean()}))
        else:
            return self.max_mean()


Instruction:
How would you code a function in Python 3 to
run the softmax selection strategy.

Response:
def softmax(self, params):
        '''
        Run the softmax selection strategy.

        Parameters
        ----------
        Params : dict
            Tau

        Returns
        -------
        int
            Index of chosen bandit
        '''

        default_tau = 0.1

        if params and type(params) == dict:
            tau = params.get('tau')
            try:
                float(tau)
            except ValueError:
                'slots: softmax: Setting tau to default'
                tau = default_tau
        else:
            tau = default_tau

        # Handle cold start. Not all bandits tested yet.
        if True in (self.pulls < 3):
            return np.random.choice(range(len(self.pulls)))
        else:
            payouts = self.wins / (self.pulls + 0.1)
            norm = sum(np.exp(payouts/tau))

        ps = np.exp(payouts/tau)/norm

        # Randomly choose index based on CMF
        cmf = [sum(ps[:i+1]) for i in range(len(ps))]

        rand = np.random.rand()

        found = False
        found_i = None
        i = 0
        while not found:
            if rand < cmf[i]:
                found_i = i
                found = True
            else:
                i += 1

        return found_i


Instruction:
Here you have a function in Python 3, explain what it does
def ucb(self, params=None):
        '''
        Run the upper confidence bound MAB selection strategy.

        This is the UCB1 algorithm described in
        https://homes.di.unimi.it/~cesabian/Pubblicazioni/ml-02.pdf

        Parameters
        ----------
        params : None
            For API consistency, this function can take a parameters argument,
            but it is ignored.

        Returns
        -------
        int
            Index of chosen bandit
        '''

        # UCB = j_max(payout_j + sqrt(2ln(n_tot)/n_j))

        # Handle cold start. Not all bandits tested yet.
        if True in (self.pulls < 3):
            return np.random.choice(range(len(self.pulls)))
        else:
            n_tot = sum(self.pulls)
            payouts = self.wins / (self.pulls + 0.1)
            ubcs = payouts + np.sqrt(2*np.log(n_tot)/self.pulls)

            return np.argmax(ubcs)

Response: Run the upper confidence bound MAB selection strategy.


Instruction:
Write a Python 3 script to
return current best choice of bandit.

Response:
def best(self):
        '''
        Return current 'best' choice of bandit.

        Returns
        -------
        int
            Index of bandit
        '''

        if len(self.choices) < 1:
            print('slots: No trials run so far.')
            return None
        else:
            return np.argmax(self.wins/(self.pulls+0.1))


Instruction:
Implement a function in Python 3 to
calculate current estimate of average payout for each bandit.

Response:
def est_payouts(self):
        '''
        Calculate current estimate of average payout for each bandit.

        Returns
        -------
        array of floats or None
        '''

        if len(self.choices) < 1:
            print('slots: No trials run so far.')
            return None
        else:
            return self.wins/(self.pulls+0.1)


Instruction:
Can you tell what is the following Python 3 function doing
def regret(self):
        '''
        Calculate expected regret, where expected regret is
        maximum optimal reward - sum of collected rewards, i.e.

        expected regret = T*max_k(mean_k) - sum_(t=1-->T) (reward_t)

        Returns
        -------
        float
        '''

        return (sum(self.pulls)*np.max(np.nan_to_num(self.wins/self.pulls)) -
                sum(self.wins)) / sum(self.pulls)

Response: Calculate expected regret for the current set of records


Instruction:
How would you code a function in Python 3 to
determine if stopping criterion has been met.

Response:
def crit_met(self):
        '''
        Determine if stopping criterion has been met.

        Returns
        -------
        bool
        '''

        if True in (self.pulls < 3):
            return False
        else:
            return self.criteria[self.criterion](self.stop_value)


Instruction:
Can you create a Python 3 function that
determines if regret criterion has been met.

Response:
def regret_met(self, threshold=None):
        '''
        Determine if regret criterion has been met.

        Parameters
        ----------
        threshold : float

        Returns
        -------
        bool
        '''

        if not threshold:
            return self.regret() <= self.stop_value
        elif self.regret() <= threshold:
            return True
        else:
            return False


Instruction:
Create a Python 3 function to
update the bandits with the results of the previous live, online trial. Next run a the selection algorithm. If the stopping criteria is met, return the best arm estimate. Otherwise return the next arm to try. Parameters ---------- bandit : int Bandit index payout : float Payout value strategy : string Name of update strategy parameters : dict Parameters for update strategy function Returns ------- dict Format: {'new_trial': boolean, 'choice': int, 'best': int}

Response:
def online_trial(self, bandit=None, payout=None, strategy='eps_greedy',
                     parameters=None):
        '''
        Update the bandits with the results of the previous live, online trial.
            Next run a the selection algorithm. If the stopping criteria is
            met, return the best arm estimate. Otherwise return the next arm to
            try.

        Parameters
        ----------
        bandit : int
            Bandit index
        payout : float
            Payout value
        strategy : string
            Name of update strategy
        parameters : dict
            Parameters for update strategy function

        Returns
        -------
        dict
            Format: {'new_trial': boolean, 'choice': int, 'best': int}
        '''

        if bandit is not None and payout is not None:
            self.update(bandit=bandit, payout=payout)
        else:
            raise Exception('slots.online_trial: bandit and/or payout value'
                            ' missing.')

        if self.crit_met():
            return {'new_trial': False, 'choice': self.best(),
                    'best': self.best()}
        else:
            return {'new_trial': True,
                    'choice': self.run_strategy(strategy, parameters),
                    'best': self.best()}


Instruction:
Write a Python 3 script to
update trials and payouts for given bandit.

Response:
def update(self, bandit, payout):
        '''
        Update bandit trials and payouts for given bandit.

        Parameters
        ----------
        bandit : int
            Bandit index
        payout : float

        Returns
        -------
        None
        '''

        self.choices.append(bandit)
        self.pulls[bandit] += 1
        self.wins[bandit] += payout
        self.bandits.payouts[bandit] += payout


Instruction:
How would you code a function in Python 3 to
return the payout from a single pull of the bandit i s arm.

Response:
def pull(self, i):
        '''
        Return the payout from a single pull of the bandit i's arm.

        Parameters
        ----------
        i : int
            Index of bandit.

        Returns
        -------
        float or None
        '''

        if self.live:
            if len(self.payouts[i]) > 0:
                return self.payouts[i].pop()
            else:
                return None
        else:
            if np.random.rand() < self.probs[i]:
                return self.payouts[i]
            else:
                return 0.0


Instruction:
Can you generate the documentation for the following Python 3 function
def click_table_printer(headers, _filter, data):
    _filter = [h.lower() for h in _filter] + [h.upper() for h in _filter]
    headers = [h for h in headers if not _filter or h in _filter]
    # Maximum header width
    header_widths = [len(h) for h in headers]

    for row in data:
        for idx in range(len(headers)):
            # If a row contains an element which is wider update maximum width
            if header_widths[idx] < len(str(row[idx])):
                header_widths[idx] = len(str(row[idx]))
    # Prepare the format string with the maximum widths
    formatted_output_parts = ['{{:<{0}}}'.format(hw)
                              for hw in header_widths]
    formatted_output = '   '.join(formatted_output_parts)
    # Print the table with the headers capitalized
    click.echo(formatted_output.format(*[h.upper() for h in headers]))
    for row in data:
        click.echo(formatted_output.format(*row))

Response: Generate space separated output for click commands.


Instruction:
Can you create a Python 3 function that
calculates hash of directory.

Response:
def calculate_hash_of_dir(directory, file_list=None):
    """Calculate hash of directory."""
    md5_hash = md5()
    if not os.path.exists(directory):
        return -1

    try:
        for subdir, dirs, files in os.walk(directory):
            for _file in files:
                file_path = os.path.join(subdir, _file)
                if file_list is not None and file_path not in file_list:
                    continue
                try:
                    _file_object = open(file_path, 'rb')
                except Exception:
                    # You can't open the file for some reason
                    _file_object.close()
                    # We return -1 since we cannot ensure that the file that
                    # can not be read, will not change from one execution to
                    # another.
                    return -1
                while 1:
                    # Read file in little chunks
                    buf = _file_object.read(4096)
                    if not buf:
                        break
                    md5_hash.update(md5(buf).hexdigest().encode())
                _file_object.close()
    except Exception:
        return -1
    return md5_hash.hexdigest()


Instruction:
Can you tell what is the following Python 3 function doing
def calculate_job_input_hash(job_spec, workflow_json):
    if 'workflow_workspace' in job_spec:
        del job_spec['workflow_workspace']
    job_md5_buffer = md5()
    job_md5_buffer.update(json.dumps(job_spec).encode('utf-8'))
    job_md5_buffer.update(json.dumps(workflow_json).encode('utf-8'))
    return job_md5_buffer.hexdigest()

Response: Calculate md5 hash of job specification and workflow json.


Instruction:
Can you write a function in Python 3 where it
calculates access times of files in workspace.

Response:
def calculate_file_access_time(workflow_workspace):
    """Calculate access times of files in workspace."""
    access_times = {}
    for subdir, dirs, files in os.walk(workflow_workspace):
        for file in files:
            file_path = os.path.join(subdir, file)
            access_times[file_path] = os.stat(file_path).st_atime
    return access_times


Instruction:
Write a Python 3 function that can
copy generated and validated openapi specs to reana - commons module.

Response:
def copy_openapi_specs(output_path, component):
    """Copy generated and validated openapi specs to reana-commons module."""
    if component == 'reana-server':
        file = 'reana_server.json'
    elif component == 'reana-workflow-controller':
        file = 'reana_workflow_controller.json'
    elif component == 'reana-job-controller':
        file = 'reana_job_controller.json'
    if os.environ.get('REANA_SRCDIR'):
        reana_srcdir = os.environ.get('REANA_SRCDIR')
    else:
        reana_srcdir = os.path.join('..')
    try:
        reana_commons_specs_path = os.path.join(
            reana_srcdir,
            'reana-commons',
            'reana_commons',
            'openapi_specifications')
        if os.path.exists(reana_commons_specs_path):
            if os.path.isfile(output_path):
                shutil.copy(output_path,
                            os.path.join(reana_commons_specs_path,
                                         file))
                # copy openapi specs file as well to docs
                shutil.copy(output_path,
                            os.path.join('docs', 'openapi.json'))
    except Exception as e:
        click.echo('Something went wrong, could not copy openapi '
                   'specifications to reana-commons \n{0}'.format(e))


Instruction:
Write a Python 3 script for
giving the correct verb conjugation depending on status tense.

Response:
def get_workflow_status_change_verb(status):
    """Give the correct verb conjugation depending on status tense.

    :param status: String which represents the status the workflow changed to.
    """
    verb = ''
    if status.endswith('ing'):
        verb = 'is'
    elif status.endswith('ed'):
        verb = 'has been'
    else:
        raise ValueError('Unrecognised status {}'.format(status))

    return verb


Instruction:
Given the following Python 3 function, write the documentation
def build_progress_message(total=None,
                           running=None,
                           finished=None,
                           failed=None,
                           cached=None):
    """Build the progress message with correct formatting."""
    progress_message = {}
    if total:
        progress_message['total'] = total
    if running:
        progress_message['running'] = running
    if finished:
        progress_message['finished'] = finished
    if failed:
        progress_message['failed'] = failed
    if cached:
        progress_message['cached'] = cached
    return progress_message

Response: Build the progress message with correct formatting.


Instruction:
Make a summary of the following Python 3 code
def build_caching_info_message(job_spec,
                               job_id,
                               workflow_workspace,
                               workflow_json,
                               result_path):
    """Build the caching info message with correct formatting."""
    caching_info_message = {
        "job_spec": job_spec,
        "job_id": job_id,
        "workflow_workspace": workflow_workspace,
        "workflow_json": workflow_json,
        "result_path": result_path
    }
    return caching_info_message

Response: Build the caching info message with correct formatting.


Instruction:
How would you explain what the following Python 3 function does
def get_workspace_disk_usage(workspace, summarize=False):
    command = ['du', '-h']
    if summarize:
        command.append('-s')
    else:
        command.append('-a')
    command.append(workspace)
    disk_usage_info = subprocess.check_output(command).decode().split()
    # create pairs of (size, filename)
    filesize_pairs = list(zip(disk_usage_info[::2], disk_usage_info[1::2]))
    filesizes = []
    for filesize_pair in filesize_pairs:
        size, name = filesize_pair
        # trim workspace path in every file name
        filesizes.append({'name': name[len(workspace):],
                          'size': size})
    return filesizes

Response: Retrieve disk usage information of a workspace.


Instruction:
Create a Python 3 function for
creating CVMFS storage class.

Response:
def create_cvmfs_storage_class(cvmfs_volume):
    """Create CVMFS storage class."""
    from kubernetes.client.rest import ApiException
    from reana_commons.k8s.api_client import current_k8s_storagev1_api_client

    try:
        current_k8s_storagev1_api_client.\
            create_storage_class(
                render_cvmfs_sc(cvmfs_volume)
            )
    except ApiException as e:
        if e.status != 409:
            raise e


Instruction:
Given the following Python 3 function, write the documentation
def create_cvmfs_persistent_volume_claim(cvmfs_volume):
    from kubernetes.client.rest import ApiException
    from reana_commons.k8s.api_client import current_k8s_corev1_api_client

    try:
        current_k8s_corev1_api_client.\
            create_namespaced_persistent_volume_claim(
                "default",
                render_cvmfs_pvc(cvmfs_volume)
            )
    except ApiException as e:
        if e.status != 409:
            raise e

Response: Create CVMFS persistent volume claim.


Instruction:
Can you create a Python 3 function that
creates Kubernetes API client using config.

Response:
def create_api_client(api='BatchV1'):
    """Create Kubernetes API client using config.

    :param api: String which represents which Kubernetes API to spawn. By
        default BatchV1.
    :returns: Kubernetes python client object for a specific API i.e. BatchV1.
    """
    k8s_config.load_incluster_config()
    api_configuration = client.Configuration()
    api_configuration.verify_ssl = False
    if api == 'extensions/v1beta1':
        api_client = client.ExtensionsV1beta1Api()
    elif api == 'CoreV1':
        api_client = client.CoreV1Api()
    elif api == 'StorageV1':
        api_client = client.StorageV1Api()
    else:
        api_client = client.BatchV1Api()
    return api_client


Instruction:
Explain what the following Python 3 code does
def __error_callback(self, exception, interval):
        logging.error('Error while publishing {}'.format(
            exception))
        logging.info('Retry in %s seconds.', interval)

Response: Execute when there is an error while trying to send a message.


Instruction:
Explain what the following Python 3 code does
def _publish(self, msg):
        connection = self._connection.clone()
        publish = connection.ensure(self.producer, self.producer.publish,
                                    errback=self.__error_callback,
                                    max_retries=MQ_PRODUCER_MAX_RETRIES)
        publish(json.dumps(msg), exchange=self._exchange,
                routing_key=self._routing_key, declare=[self._queue])
        logging.debug('Publisher: message sent: %s', msg)

Response: Publish a message in the queue.


Instruction:
Implement a Python 3 function for
publishing the status of a workflow.

Response:
def publish_workflow_status(self, workflow_uuid, status,
                                logs='', message=None):
        """Publish workflow status using the configured.

        :param workflow_uudid: String which represents the workflow UUID.
        :param status: Integer which represents the status of the workflow,
            this is defined in the `reana-db` `Workflow` models.
        :param logs: String which represents the logs which the workflow
            has produced as output.
        :param message: Dictionary which includes additional information
            can be attached such as the overall progress of the workflow.
        """
        msg = {
            "workflow_uuid": workflow_uuid,
            "logs": logs,
            "status": status,
            "message": message
        }
        self._publish(msg)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def publish_workflow_submission(self,
                                    user_id,
                                    workflow_id_or_name,
                                    parameters):
        """Publish workflow submission parameters."""
        msg = {
            "user": user_id,
            "workflow_id_or_name": workflow_id_or_name,
            "parameters": parameters
        }
        self._publish(msg)

Response: Publish workflow submission parameters.


Instruction:
How would you implement a function in Python 3 that
validates and return a REANA Serial workflow specification.

Response:
def serial_load(workflow_file, specification, parameters=None, original=None):
    """Validate and return a expanded REANA Serial workflow specification.

    :param workflow_file: A specification file compliant with
        REANA Serial workflow specification.
    :returns: A dictionary which represents the valid Serial workflow with all
        parameters expanded.
    """
    parameters = parameters or {}

    if not specification:
        with open(workflow_file, 'r') as f:
            specification = json.loads(f.read())

    expanded_specification = _expand_parameters(specification,
                                                parameters,
                                                original)

    validate(specification, serial_workflow_schema)

    return expanded_specification


Instruction:
Create a Python 3 function to
expand parameters inside a Serial workflow specification.

Response:
def _expand_parameters(specification, parameters, original=None):
    """Expand parameters inside comands for Serial workflow specifications.

    :param specification: Full valid Serial workflow specification.
    :param parameters: Parameters to be extended on a Serial specification.
    :param original: Flag which, determins type of specifications to return.
    :returns: If 'original' parameter is set, a copy of the specification
        whithout expanded parametrers will be returned. If 'original' is not
        set, a copy of the specification with expanded parameters (all $varname
        and ${varname} will be expanded with their value). Otherwise an error
        will be thrown if the parameters can not be expanded.
    :raises: jsonschema.ValidationError
    """
    expanded_specification = deepcopy(specification)

    try:
        for step_num, step in enumerate(expanded_specification['steps']):
            current_step = expanded_specification['steps'][step_num]
            for command_num, command in enumerate(step['commands']):
                current_step['commands'][command_num] = \
                    Template(command).substitute(parameters)
        # if call is done from client, original==True and original
        # specifications withtout applied parameters are returned.
        if original:
            return specification
        else:
            return expanded_specification
    except KeyError as e:
        raise ValidationError('Workflow parameter(s) could not '
                              'be expanded. Please take a look '
                              'to {params}'.format(params=str(e)))


Instruction:
Can you tell what is the following Python 3 function doing
def reana_ready():
    from reana_commons.config import REANA_READY_CONDITIONS
    for module_name, condition_list in REANA_READY_CONDITIONS.items():
        for condition_name in condition_list:
            module = importlib.import_module(module_name)
            condition_func = getattr(module, condition_name)
            if not condition_func():
                return False
    return True

Response: Check if reana can start new workflows.


Instruction:
Can you write a function in Python 3 where it
checks k8s predefined conditions for the nodes.

Response:
def check_predefined_conditions():
    """Check k8s predefined conditions for the nodes."""
    try:
        node_info = current_k8s_corev1_api_client.list_node()
        for node in node_info.items:
            # check based on the predefined conditions about the
            # node status: MemoryPressure, OutOfDisk, KubeletReady
            #              DiskPressure, PIDPressure,
            for condition in node.status.conditions:
                if not condition.status:
                    return False
    except ApiException as e:
        log.error('Something went wrong while getting node information.')
        log.error(e)
        return False
    return True


Instruction:
Write a Python 3 function that can
check upper limit on running jobs.

Response:
def check_running_job_count():
    """Check upper limit on running jobs."""
    try:
        job_list = current_k8s_batchv1_api_client.\
            list_job_for_all_namespaces()
        if len(job_list.items) > K8S_MAXIMUM_CONCURRENT_JOBS:
            return False
    except ApiException as e:
        log.error('Something went wrong while getting running job list.')
        log.error(e)
        return False
    return True


Instruction:
How would you code a function in Python 3 to
get json specification from package data.

Response:
def _get_spec(self, spec_file):
        """Get json specification from package data."""
        spec_file_path = os.path.join(
            pkg_resources.
            resource_filename(
                'reana_commons',
                'openapi_specifications'),
            spec_file)

        with open(spec_file_path) as f:
            json_spec = json.load(f)
        return json_spec


Instruction:
How would you explain what the following Python 3 function does
def submit(self,
               workflow_uuid='',
               experiment='',
               image='',
               cmd='',
               prettified_cmd='',
               workflow_workspace='',
               job_name='',
               cvmfs_mounts='false'):
        """Submit a job to RJC API.

        :param name: Name of the job.
        :param experiment: Experiment the job belongs to.
        :param image: Identifier of the Docker image which will run the job.
        :param cmd: String which represents the command to execute. It can be
            modified by the workflow engine i.e. prepending ``cd /some/dir/``.
        :prettified_cmd: Original command submitted by the user.
        :workflow_workspace: Path to the workspace of the workflow.
        :cvmfs_mounts: String with CVMFS volumes to mount in job pods.
        :return: Returns a dict with the ``job_id``.
        """
        job_spec = {
            'experiment': experiment,
            'docker_img': image,
            'cmd': cmd,
            'prettified_cmd': prettified_cmd,
            'env_vars': {},
            'workflow_workspace': workflow_workspace,
            'job_name': job_name,
            'cvmfs_mounts': cvmfs_mounts,
            'workflow_uuid': workflow_uuid
        }

        response, http_response = self._client.jobs.create_job(job=job_spec).\
            result()
        if http_response.status_code == 400:
            raise HTTPBadRequest('Bad request to create a job. Error: {}'.
                                 format(http_response.data))
        elif http_response.status_code == 500:
            raise HTTPInternalServerError('Internal Server Error. Error: {}'.
                                          format(http_response.data))
        return response

Response: Submit a job to RJC API.


Instruction:
Make a summary of the following Python 3 code
def check_status(self, job_id):
        response, http_response = self._client.jobs.get_job(job_id=job_id).\
            result()
        if http_response.status_code == 404:
            raise HTTPNotFound('The given job ID was not found. Error: {}'.
                               format(http_response.data))
        return response

Response: Check status of a job.


Instruction:
How would you implement a function in Python 3 that
gets logs of a job.

Response:
def get_logs(self, job_id):
        """Get logs of a job."""
        response, http_response = self._client.jobs.get_logs(job_id=job_id).\
            result()
        if http_response.status_code == 404:
            raise HTTPNotFound('The given job ID was not found. Error: {}'.
                               format(http_response.data))
        return http_response.text


Instruction:
How would you implement a function in Python 3 that
checks if job result is in cache.

Response:
def check_if_cached(self, job_spec, step, workflow_workspace):
        """Check if job result is in cache."""
        response, http_response = self._client.job_cache.check_if_cached(
            job_spec=json.dumps(job_spec),
            workflow_json=json.dumps(step),
            workflow_workspace=workflow_workspace).\
            result()
        if http_response.status_code == 400:
            raise HTTPBadRequest('Bad request to check cache. Error: {}'.
                                 format(http_response.data))
        elif http_response.status_code == 500:
            raise HTTPInternalServerError('Internal Server Error. Error: {}'.
                                          format(http_response.data))
        return http_response


Instruction:
Can you write a function in Python 3 where it
gets the Kubernetes volumeMount and volume for a given job spec.

Response:
def get_shared_volume(workflow_workspace, shared_volume_root):
    """Get shared CephFS/hostPath volume to a given job spec.

    :param workflow_workspace: Absolute path to the job's workflow workspace.
    :param shared_volume_root: Root path in the underlying storage backend.
    :returns: Tuple consisting of the Kubernetes volumeMount and the volume.
    """
    workflow_workspace_relative_to_owner = workflow_workspace
    if os.path.isabs(workflow_workspace):
        workflow_workspace_relative_to_owner = \
            os.path.relpath(workflow_workspace, shared_volume_root)
    mount_path = os.path.join(shared_volume_root,
                              workflow_workspace_relative_to_owner)
    volume_mount = {
        "name": REANA_SHARED_VOLUME_NAME,
        "mountPath": mount_path,
        "subPath": workflow_workspace_relative_to_owner}

    if REANA_STORAGE_BACKEND == "CEPHFS":
        volume = get_k8s_cephfs_volume()
    else:
        volume = get_k8s_hostpath_volume(shared_volume_root)

    return volume_mount, volume


Instruction:
Explain what the following Python 3 code does
def run(self, name, cache_key,
            local_path, remote_path,
            local_options, remote_options, **kwargs):
        """
        The main work horse of the transfer task. Calls the transfer
        method with the local and remote storage backends as given
        with the parameters.

        :param name: name of the file to transfer
        :type name: str
        :param local_path: local storage class to transfer from
        :type local_path: str
        :param local_options: options of the local storage class
        :type local_options: dict
        :param remote_path: remote storage class to transfer to
        :type remote_path: str
        :param remote_options: options of the remote storage class
        :type remote_options: dict
        :param cache_key: cache key to set after a successful transfer
        :type cache_key: str
        :rtype: task result
        """
        local = import_attribute(local_path)(**local_options)
        remote = import_attribute(remote_path)(**remote_options)
        result = self.transfer(name, local, remote, **kwargs)

        if result is True:
            cache.set(cache_key, True)
            file_transferred.send(sender=self.__class__,
                                  name=name, local=local, remote=remote)
        elif result is False:
            args = [name, cache_key, local_path,
                    remote_path, local_options, remote_options]
            self.retry(args=args, kwargs=kwargs)
        else:
            raise ValueError("Task '%s' did not return True/False but %s" %
                             (self.__class__, result))
        return result

Response: This method is the main work horse of the transfer task.


Instruction:
Here you have a function in Python 3, explain what it does
def transfer(self, name, local, remote, **kwargs):
        try:
            remote.save(name, local.open(name))
            return True
        except Exception as e:
            logger.error("Unable to save '%s' to remote storage. "
                         "About to retry." % name)
            logger.exception(e)
            return False

Response: Transfers the file with the given name from the local storage backend to the remote storage backend.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_string(cfunc, *args):
    cstr = get_ctype("const char**", cfunc, *args)
    return backend.ffi.string(cstr).decode() if cstr else None

Response: Call a C function and return its return value as a Python string.


Instruction:
Write a Python 3 script for
calling a function that takes a pointer as its last argument and returns the C object that it contains after the function has finished.

Response:
def get_ctype(rtype, cfunc, *args):
    """ Call a C function that takes a pointer as its last argument and
        return the C object that it contains after the function has finished.

    :param rtype:   C data type is filled by the function
    :param cfunc:   C function to call
    :param args:    Arguments to call function with
    :return:        A pointer to the specified data type
    """
    val_p = backend.ffi.new(rtype)
    args = args + (val_p,)
    cfunc(*args)
    return val_p[0]


Instruction:
Implement a Python 3 function for
creating an indirect pointer to a GPhoto2 type call its matching constructor function and return the pointer to it.

Response:
def new_gp_object(typename):
    """ Create an indirect pointer to a GPhoto2 type, call its matching
        constructor function and return the pointer to it.

    :param typename:    Name of the type to create.
    :return:            A pointer to the specified data type.
    """
    obj_p = backend.ffi.new("{0}**".format(typename))
    backend.CONSTRUCTORS[typename](obj_p)
    return obj_p[0]


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_library_version():
    version_str = ffi.string(lib.gp_library_version(True)[0]).decode()
    return tuple(int(x) for x in version_str.split('.'))

Response: Get the version number of the underlying gphoto2 library.


Instruction:
Can you generate the documentation for the following Python 3 function
def list_cameras():
    ctx = lib.gp_context_new()
    camlist_p = new_gp_object("CameraList")
    port_list_p = new_gp_object("GPPortInfoList")
    lib.gp_port_info_list_load(port_list_p)
    abilities_list_p = new_gp_object("CameraAbilitiesList")
    lib.gp_abilities_list_load(abilities_list_p, ctx)
    lib.gp_abilities_list_detect(abilities_list_p, port_list_p,
                                 camlist_p, ctx)
    out = []
    for idx in range(lib.gp_list_count(camlist_p)):
        name = get_string(lib.gp_list_get_name, camlist_p, idx)
        value = get_string(lib.gp_list_get_value, camlist_p, idx)

        # Skip iteration if no matches
        matches = re.match(r"usb:(\d+),(\d+)", value)
        if not matches:
            continue

        bus_no, device_no = (int(x) for x in matches.groups())
        abilities = ffi.new("CameraAbilities*")
        ability_idx = lib.gp_abilities_list_lookup_model(
            abilities_list_p, name.encode())
        lib.gp_abilities_list_get_abilities(abilities_list_p, ability_idx,
                                            abilities)
        if abilities.device_type == lib.GP_DEVICE_STILL_CAMERA:
            out.append(Camera(bus_no, device_no, lazy=True,
                              _abilities=abilities))
    lib.gp_list_free(camlist_p)
    lib.gp_port_info_list_free(port_list_p)
    lib.gp_abilities_list_free(abilities_list_p)
    return out

Response: Return a list of all USB cameras that are supported by libgphoto2.


Instruction:
Can you generate the documentation for the following Python 3 function
def supported_cameras():
    ctx = lib.gp_context_new()
    abilities_list_p = new_gp_object("CameraAbilitiesList")
    lib.gp_abilities_list_load(abilities_list_p, ctx)
    abilities = ffi.new("CameraAbilities*")
    out = []
    for idx in range(lib.gp_abilities_list_count(abilities_list_p)):
        lib.gp_abilities_list_get_abilities(abilities_list_p, idx, abilities)
        if abilities.device_type == lib.GP_DEVICE_STILL_CAMERA:
            libname = os.path.basename(ffi.string(abilities.library)
                                       .decode())
            out.append((ffi.string(abilities.model).decode(), libname))
    lib.gp_abilities_list_free(abilities_list_p)
    key_func = lambda name, driver: driver
    out = sorted(out, key=key_func)
    return {k: tuple(x[0] for x in v)
            for k, v in itertools.groupby(out, key_func)}
    return out

Response:
List the names of all cameras supported by libgphoto2 grouped by the
    name of their driver.


Instruction:
Can you generate the documentation for the following Python 3 function
def path(self):
        if self.parent is None:
            return "/"
        else:
            return os.path.join(self.parent.path, self.name)

Response: Absolute path to the camera s filesystem.


Instruction:
Create a Python 3 function for
checking whether the directory exists on the camera.

Response:
def exists(self):
        """ Check whether the directory exists on the camera. """
        if self.name in ("", "/") and self.parent is None:
            return True
        else:
            return self in self.parent.directories


Instruction:
Can you write a function in Python 3 where it
gets a generator that yields all files in the directory.

Response:
def files(self):
        """ Get a generator that yields all files in the directory. """
        filelist_p = new_gp_object("CameraList")
        lib.gp_camera_folder_list_files(self._cam._cam, self.path.encode(),
                                        filelist_p, self._cam._ctx)
        for idx in range(lib.gp_list_count(filelist_p)):
            fname = get_string(lib.gp_list_get_name, filelist_p, idx)
            yield File(name=fname, directory=self, camera=self._cam)
        lib.gp_list_free(filelist_p)


Instruction:
Can you implement a function in Python 3 that
gets a generator that yields all subdirectories in the directory.

Response:
def directories(self):
        """ Get a generator that yields all subdirectories in the directory.
        """
        dirlist_p = new_gp_object("CameraList")
        lib.gp_camera_folder_list_folders(self._cam._cam, self.path.encode(),
                                          dirlist_p, self._cam._ctx)
        for idx in range(lib.gp_list_count(dirlist_p)):
            name = os.path.join(
                self.path, get_string(lib.gp_list_get_name, dirlist_p, idx))
            yield Directory(name=name, parent=self, camera=self._cam)
        lib.gp_list_free(dirlist_p)


Instruction:
Create a Python 3 function for
uploading a file to the camera s permanent storage.

Response:
def upload(self, local_path):
        """ Upload a file to the camera's permanent storage.

        :param local_path: Path to file to copy
        :type local_path:  str/unicode
        """
        camerafile_p = ffi.new("CameraFile**")
        with open(local_path, 'rb') as fp:
            lib.gp_file_new_from_fd(camerafile_p, fp.fileno())
            lib.gp_camera_folder_put_file(
                self._cam._cam, self.path.encode() + b"/",
                os.path.basename(local_path).encode(),
                backend.FILE_TYPES['normal'], camerafile_p[0],
                self._cam.ctx)


Instruction:
Can you generate the documentation for the following Python 3 function
def supported_operations(self):
        return tuple(op for op in backend.FILE_OPS if self._operations & op)

Response: All file operations supported by the camera.


Instruction:
Write a Python 3 function for
returning the image dimensions.

Response:
def dimensions(self):
        """ Dimensions of the image.

        :rtype: :py:class:`ImageDimensions`
        """
        return ImageDimensions(self._info.file.width, self._info.file.height)


Instruction:
Can you write a function in Python 3 where it
returns the permissions of the file.

Response:
def permissions(self):
        """ Permissions of the file.

        Can be "r-" (read-only), "-w" (write-only), "rw" (read-write)
        or "--" (no rights).

        :rtype: str
        """
        can_read = self._info.file.permissions & lib.GP_FILE_PERM_READ
        can_write = self._info.file.permissions & lib.GP_FILE_PERM_DELETE
        return "{0}{1}".format("r" if can_read else "-",
                               "w" if can_write else "-")


Instruction:
How would you implement a function in Python 3 that
saves the content of the current object to a local file.

Response:
def save(self, target_path, ftype='normal'):
        """ Save file content to a local file.

        :param target_path: Path to save remote file as.
        :type target_path:  str/unicode
        :param ftype:       Select 'view' on file.
        :type ftype:        str
        """
        camfile_p = ffi.new("CameraFile**")
        with open(target_path, 'wb') as fp:
            lib.gp_file_new_from_fd(camfile_p, fp.fileno())
            lib.gp_camera_file_get(
                self._cam._cam, self.directory.path.encode(),
                self.name.encode(), backend.FILE_TYPES[ftype], camfile_p[0],
                self._cam._ctx)


Instruction:
Create a Python 3 function to
get the content of the file as a bytestring.

Response:
def get_data(self, ftype='normal'):
        """ Get file content as a bytestring.

        :param ftype:       Select 'view' on file.
        :type ftype:        str
        :return:            File content
        :rtype:             bytes
        """
        camfile_p = ffi.new("CameraFile**")
        lib.gp_file_new(camfile_p)
        lib.gp_camera_file_get(
            self._cam._cam, self.directory.path.encode(), self.name.encode(),
            backend.FILE_TYPES[ftype], camfile_p[0], self._cam._ctx)
        data_p = ffi.new("char**")
        length_p = ffi.new("unsigned long*")
        lib.gp_file_get_data_and_size(camfile_p[0], data_p, length_p)
        byt = bytes(ffi.buffer(data_p[0], length_p[0]))
        # gphoto2 camera files MUST be freed.
        lib.gp_file_free(camfile_p[0])
        # just to be safe.
        del data_p, length_p, camfile_p
        return byt


Instruction:
How would you code a function in Python 3 to
get an iterator that yields the contents of the file.

Response:
def iter_data(self, chunk_size=2**16, ftype='normal'):
        """ Get an iterator that yields chunks of the file content.

        :param chunk_size:  Size of yielded chunks in bytes
        :type chunk_size:   int
        :param ftype:       Select 'view' on file.
        :type ftype:        str
        :return:            Iterator
        """
        self._check_type_supported(ftype)
        buf_p = ffi.new("char[{0}]".format(chunk_size))
        size_p = ffi.new("uint64_t*")
        offset_p = ffi.new("uint64_t*")
        for chunk_idx in range(int(math.ceil(self.size/chunk_size))):
            size_p[0] = chunk_size
            lib.gp_camera_file_read(
                self._cam._cam, self.directory.path.encode(),
                self.name.encode(), backend.FILE_TYPES[ftype], offset_p[0],
                buf_p, size_p, self._cam._ctx)
            yield ffi.buffer(buf_p, size_p[0])[:]


Instruction:
Here you have a function in Python 3, explain what it does
def remove(self):
        lib.gp_camera_file_delete(self._cam._cam, self.directory.path.encode(),
                                  self.name.encode(), self._cam._ctx)

Response: Remove file from device.


Instruction:
Can you write a function in Python 3 where it
sets the value of the option.

Response:
def set(self, value):
        """ Update value of the option.

        Only possible for options with :py:attr:`readonly` set to `False`.
        If :py:attr:`type` is `choice`, the value must be one of the
        :py:attr:`choices`.
        If :py:attr:`type` is `range`, the value must be in the range
        described by :py:attr:`range`.

        :param value:   Value to set
        """
        if self.readonly:
            raise ValueError("Option is read-only.")
        val_p = None
        if self.type == 'selection':
            if value not in self.choices:
                raise ValueError("Invalid choice (valid: {0})".format(
                                 repr(self.choices)))
            val_p = ffi.new("const char[]", value.encode())
        elif self.type == 'text':
            if not isinstance(value, basestring):
                raise ValueError("Value must be a string.")
            val_p = ffi.new("char**")
            val_p[0] = ffi.new("char[]", value.encode())
        elif self.type == 'range':
            if value < self.range.min or value > self.range.max:
                raise ValueError("Value exceeds valid range ({0}-{1}."
                                 .format(self.range.min, self.range.max))
            if value % self.range.step:
                raise ValueError("Value can only be changed in steps of {0}."
                                 .format(self.range.step))
            val_p = ffi.new("float*")
            val_p[0] = value
        elif self.type == 'toggle':
            if not isinstance(value, bool):
                raise ValueError("Value must be bool.")
            val_p = ffi.new("int*")
            val_p[0] = int(value)
        elif self.type == 'date':
            val_p = ffi.new("int*")
            val_p[0] = value
        lib.gp_widget_set_value(self._widget, val_p)
        lib.gp_camera_set_config(self._cam._cam, self._root, self._cam._ctx)
        self.value = value


Instruction:
Implement a function in Python 3 to
return a tuple of all operations supported by the camera.

Response:
def supported_operations(self):
        """ All operations supported by the camera. """
        return tuple(op for op in backend.CAM_OPS
                     if self._abilities.operations & op)


Instruction:
Write a Python 3 function for
returning the configuration parameters.

Response:
def config(self):
        """ Writeable configuration parameters.

        :rtype:     dict
        """
        config = self._get_config()
        return {section: {itm.name: itm for itm in config[section].values()
                          if not itm.readonly}
                for section in config
                if 'settings' in section or section == 'other'}


Instruction:
Explain what the following Python 3 code does
def status(self):
        config = self._get_config()
        is_hex = lambda name: (len(name) == 4 and
                               all(c in string.hexdigits for c in name))
        out = SimpleNamespace()
        for sect in config:
            for itm in config[sect].values():
                if (itm.readonly or sect == 'status') and not is_hex(itm.name):
                    setattr(out, itm.name, itm.value)
        return out

Response: Status information ( read - only.


Instruction:
Can you implement a function in Python 3 that
returns a list of all camera s storage information.

Response:
def storage_info(self):
        """ Information about the camera's storage. """
        info_p = ffi.new("CameraStorageInformation**")
        num_info_p = ffi.new("int*")
        lib.gp_camera_get_storageinfo(self._cam, info_p, num_info_p, self._ctx)
        infos = []
        for idx in range(num_info_p[0]):
            out = SimpleNamespace()
            struc = (info_p[0] + idx)
            fields = struc.fields
            if lib.GP_STORAGEINFO_BASE & fields:
                out.directory = next(
                    (d for d in self.list_all_directories()
                     if d.path == ffi.string(struc.basedir).decode()),
                    None)
            if lib.GP_STORAGEINFO_LABEL & fields:
                out.label = ffi.string(struc.label).decode()
            if lib.GP_STORAGEINFO_DESCRIPTION & fields:
                out.description = ffi.string(struc.description).decode()
            if lib.GP_STORAGEINFO_STORAGETYPE & fields:
                stype = struc.type
                if lib.GP_STORAGEINFO_ST_FIXED_ROM & stype:
                    out.type = 'fixed_rom'
                elif lib.GP_STORAGEINFO_ST_REMOVABLE_ROM & stype:
                    out.type = 'removable_rom'
                elif lib.GP_STORAGEINFO_ST_FIXED_RAM & stype:
                    out.type = 'fixed_ram'
                elif lib.GP_STORAGEINFO_ST_REMOVABLE_RAM & stype:
                    out.type = 'removable_ram'
                else:
                    out.type = 'unknown'
            if lib.GP_STORAGEINFO_ACCESS & fields:
                if lib.GP_STORAGEINFO_AC_READWRITE & struc.access:
                    out.access = 'read-write'
                elif lib.GP_STORAGEINFO_AC_READONLY & struc.access:
                    out.access = 'read-only'
                elif lib.GP_STORAGEINFO_AC_READONLY_WITH_DELETE & struc.access:
                    out.access = 'read-delete'
            if lib.GP_STORAGEINFO_MAXCAPACITY & fields:
                out.capacity = int(struc.capacitykbytes)
            if lib.GP_STORAGEINFO_FREESPACEKBYTES & fields:
                out.free_space = int(struc.freekbytes)
            if lib.GP_STORAGEINFO_FREESPACEIMAGES & fields:
                out.remaining_images = int(struc.freeimages)
            infos.append(out)
        return infos


Instruction:
Can you generate the documentation for the following Python 3 function
def list_all_files(self):
        def list_files_recursively(directory):
            f_gen = itertools.chain(
                directory.files,
                *tuple(list_files_recursively(d)
                       for d in directory.directories))
            for f in f_gen:
                yield f
        return list_files_recursively(self.filesystem)

Response:
Utility method that yields all files on the device s file
            systems.


Instruction:
Can you tell what is the following Python 3 function doing
def capture(self, to_camera_storage=False):
        target = self.config['settings']['capturetarget']
        if to_camera_storage and target.value != "Memory card":
            target.set("Memory card")
        elif not to_camera_storage and target.value != "Internal RAM":
            target.set("Internal RAM")
        lib.gp_camera_trigger_capture(self._cam, self._ctx)

        fobj = self._wait_for_event(event_type=lib.GP_EVENT_FILE_ADDED)
        if to_camera_storage:
            self._logger.info("File written to storage at {0}.".format(fobj))
            return fobj
        else:
            data = fobj.get_data()
            try:
                fobj.remove()
            except errors.CameraIOError:
                # That probably means the file is already gone from RAM,
                # so nothing to worry about.
                pass
            return data

Response:
Capture an image.

        Some cameras (mostly Canon and Nikon) support capturing to internal
        RAM. On these devices, you have to specify `to_camera_storage` if
        you want to save the images to the memory card. On devices that
        do not support saving to RAM, the only difference is that the file
        is automatically downloaded and deleted when set to `False`.

        :param to_camera_storage:   Save image to the camera's internal storage
        :type to_camera_storage:    bool
        :return:    A :py:class:`File` if `to_camera_storage` was `True`,
                    otherwise the captured image as a bytestring.
        :rtype:     :py:class:`File` or bytes


Instruction:
Implement a Python 3 function for
capturing a video. This always writes to the memory card, since internal RAM is likely to run out of space very quickly. Currently this only works with Nikon cameras. :param length: Length of the video to capture in seconds. :type length: int :return: Video file :rtype: :py:class:`File`

Response:
def capture_video(self, length):
        """ Capture a video.

        This always writes to the memory card, since internal RAM is likely
        to run out of space very quickly.

        Currently this only works with Nikon cameras.

        :param length:      Length of the video to capture in seconds.
        :type length:       int
        :return:            Video file
        :rtype:             :py:class:`File`
        """
        with self.capture_video_context() as ctx:
            time.sleep(length)
        return ctx.videofile


Instruction:
Can you implement a function in Python 3 that
gets a preview image from the camera s viewport.

Response:
def get_preview(self):
        """ Get a preview from the camera's viewport.

        This will usually be a JPEG image with the dimensions depending on
        the camera.  You will need to call the exit() method manually after
        you are done capturing a live preview.

        :return:    The preview image as a bytestring
        :rtype:     bytes
        """
        lib.gp_camera_capture_preview(self._cam, self.__camfile_p[0], self._ctx)
        lib.gp_file_get_data_and_size(self.__camfile_p[0], self.__data_p, self.__length_p)
        return ffi.buffer(self.__data_p[0], self.__length_p[0])[:]


Instruction:
How would you explain what the following Python 3 function does
def get_storage(self, name):
        cache_result = cache.get(self.get_cache_key(name))
        if cache_result:
            return self.remote
        elif cache_result is None and self.remote.exists(name):
            cache.set(self.get_cache_key(name), True)
            return self.remote
        else:
            return self.local

Response: Returns the storage instance responsible for the given file name.


Instruction:
Explain what the following Python 3 code does
def open(self, name, mode='rb'):
        return self.get_storage(name).open(name, mode)

Response: Retrieves the specified file from the storage.


Instruction:
How would you explain what the following Python 3 function does
def save(self, name, content, max_length=None):
        cache_key = self.get_cache_key(name)
        cache.set(cache_key, False)

        # Use a name that is available on both the local and remote storage
        # systems and save locally.
        name = self.get_available_name(name)
        try:
            name = self.local.save(name, content, max_length=max_length)
        except TypeError:
            # Django < 1.10
            name = self.local.save(name, content)

        # Pass on the cache key to prevent duplicate cache key creation,
        # we save the result in the storage to be able to test for it
        if not self.delayed:
            self.result = self.transfer(name, cache_key=cache_key)
        return name

Response: Saves the given content with the given name using the local storage.


Instruction:
Write a Python 3 function that can
transfer the file with the given name to the remote storage backend by queuing the task. :param name: file name :type name: str :param cache_key: the cache key to set after a successful task run :type cache_key: str :rtype: task result

Response:
def transfer(self, name, cache_key=None):
        """
        Transfers the file with the given name to the remote storage
        backend by queuing the task.

        :param name: file name
        :type name: str
        :param cache_key: the cache key to set after a successful task run
        :type cache_key: str
        :rtype: task result
        """
        if cache_key is None:
            cache_key = self.get_cache_key(name)
        return self.task.delay(name, cache_key,
                               self.local_path, self.remote_path,
                               self.local_options, self.remote_options)


Instruction:
Write a Python 3 script for
returning a filename that s free on both the local and remote storage structures and available for new content to be written to.

Response:
def get_available_name(self, name):
        """
        Returns a filename that's free on both the local and remote storage
        systems, and available for new content to be written to.

        :param name: file name
        :type name: str
        :rtype: str
        """
        local_available_name = self.local.get_available_name(name)
        remote_available_name = self.remote.get_available_name(name)

        if remote_available_name > local_available_name:
            return remote_available_name
        return local_available_name


Instruction:
Make a summary of the following Python 3 code
def generate_query_report(self, db_uri, parsed_query, db_name, collection_name):
        index_analysis = None
        recommendation = None
        namespace = parsed_query['ns']
        indexStatus = "unknown"

        index_cache_entry = self._ensure_index_cache(db_uri,
                                                     db_name,
                                                     collection_name)

        query_analysis = self._generate_query_analysis(parsed_query,
                                                       db_name,
                                                       collection_name)
        if ((query_analysis['analyzedFields'] != []) and
             query_analysis['supported']):
            index_analysis = self._generate_index_analysis(query_analysis,
                                                           index_cache_entry['indexes'])
            indexStatus = index_analysis['indexStatus']
            if index_analysis['indexStatus'] != 'full':
                recommendation = self._generate_recommendation(query_analysis,
                                                               db_name,
                                                               collection_name)
                # a temporary fix to suppress faulty parsing of $regexes.
                # if the recommendation cannot be re-parsed into yaml, we assume
                # it is invalid.
                if not validate_yaml(recommendation['index']):
                    recommendation = None
                    query_analysis['supported'] = False

        # QUERY REPORT
        return OrderedDict({
            'queryMask': parsed_query['queryMask'],
            'indexStatus': indexStatus,
            'parsed': parsed_query,
            'namespace': namespace,
            'queryAnalysis': query_analysis,
            'indexAnalysis': index_analysis,
            'recommendation': recommendation
        })

Response: Generates a comprehensive query report on the raw query


Instruction:
Given the following Python 3 function, write the documentation
def _ensure_index_cache(self, db_uri, db_name, collection_name):
        if not self._check_indexes or db_uri is None:
            return {'indexes': None}
        if db_name not in self.get_cache():
            self._internal_map[db_name] = {}
        if collection_name not in self._internal_map[db_name]:
            indexes = []
            try:
                if self._index_cache_connection is None:
                    self._index_cache_connection = pymongo.MongoClient(db_uri,
                                                                       document_class=OrderedDict,
                                                                       read_preference=pymongo.ReadPreference.PRIMARY_PREFERRED)

                db = self._index_cache_connection[db_name]
                indexes = db[collection_name].index_information()
            except:
                warning = 'Warning: unable to connect to ' + db_uri + "\n"
            else:
                internal_map_entry = {'indexes': indexes}
                self.get_cache()[db_name][collection_name] = internal_map_entry
        return self.get_cache()[db_name][collection_name]

Response: Adds a collections index entries to the cache if not present


Instruction:
Implement a function in Python 3 to
generate a Dex query analysis from a raw query object.

Response:
def _generate_query_analysis(self, parsed_query, db_name, collection_name):
        """Translates a raw query object into a Dex query analysis"""

        analyzed_fields = []
        field_count = 0
        supported = True
        sort_fields = []
        query_mask = None

        if 'command' in parsed_query and parsed_query['command'] not in SUPPORTED_COMMANDS:
            supported = False
        else:
            #if 'orderby' in parsed_query:
            sort_component = parsed_query['orderby'] if 'orderby' in parsed_query else []
            sort_seq = 0
            for key in sort_component:
                sort_field = {'fieldName': key,
                              'fieldType': SORT_TYPE,
                              'seq': sort_seq}
                sort_fields.append(key)
                analyzed_fields.append(sort_field)
                field_count += 1
                sort_seq += 1

            query_component = parsed_query['query'] if 'query' in parsed_query else {}
            for key in query_component:
                if key not in sort_fields:
                    field_type = UNSUPPORTED_TYPE
                    if ((key not in UNSUPPORTED_QUERY_OPERATORS) and
                            (key not in COMPOSITE_QUERY_OPERATORS)):
                        try:
                            if query_component[key] == {}:
                                raise
                            nested_field_list = query_component[key].keys()
                        except:
                            field_type = EQUIV_TYPE
                        else:
                            for nested_field in nested_field_list:
                                if ((nested_field in RANGE_QUERY_OPERATORS) and
                                    (nested_field not in UNSUPPORTED_QUERY_OPERATORS)):
                                    field_type = RANGE_TYPE
                                else:
                                    supported = False
                                    field_type = UNSUPPORTED_TYPE
                                    break

                    if field_type is UNSUPPORTED_TYPE:
                        supported = False

                    analyzed_field = {'fieldName': key,
                                      'fieldType': field_type}
                    analyzed_fields.append(analyzed_field)
                    field_count += 1

        query_mask = parsed_query['queryMask']

        # QUERY ANALYSIS
        return OrderedDict({
            'analyzedFields': analyzed_fields,
            'fieldCount': field_count,
            'supported': supported,
            'queryMask': query_mask
        })


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _generate_index_analysis(self, query_analysis, indexes):
        needs_recommendation = True
        full_indexes = []
        partial_indexes = []
        coverage = "unknown"

        if indexes is not None:
            for index_key in indexes.keys():
                index = indexes[index_key]
                index_report = self._generate_index_report(index,
                                                           query_analysis)
                if index_report['supported'] is True:
                    if index_report['coverage'] == 'full':
                        full_indexes.append(index_report)
                        if index_report['idealOrder']:
                            needs_recommendation = False
                    elif index_report['coverage'] == 'partial':
                        partial_indexes.append(index_report)

        if len(full_indexes) > 0:
            coverage = "full"
        elif (len(partial_indexes)) > 0:
            coverage = "partial"
        elif query_analysis['supported']:
            coverage = "none"

        # INDEX ANALYSIS
        return OrderedDict([('indexStatus', coverage),
                            ('fullIndexes', full_indexes),
                            ('partialIndexes', partial_indexes)])

Response:
Compares a query signature to the index cache to identify complete
            and partial indexes available to the query


Instruction:
Explain what the following Python 3 code does
def _generate_index_report(self, index, query_analysis):

        all_fields = []
        equiv_fields = []
        sort_fields = []
        range_fields = []

        for query_field in query_analysis['analyzedFields']:
            all_fields.append(query_field['fieldName'])
            if query_field['fieldType'] is EQUIV_TYPE:
                equiv_fields.append(query_field['fieldName'])
            elif query_field['fieldType'] is SORT_TYPE:
                sort_fields.append(query_field['fieldName'])
            elif query_field['fieldType'] is RANGE_TYPE:
                range_fields.append(query_field['fieldName'])

        max_equiv_seq = len(equiv_fields)
        max_sort_seq = max_equiv_seq + len(sort_fields)
        max_range_seq = max_sort_seq + len(range_fields)

        coverage = 'none'
        query_fields_covered = 0
        query_field_count = query_analysis['fieldCount']
        supported = True
        ideal_order = True
        for index_field in index['key']:
            field_name = index_field[0]

            if index_field[1] == '2d':
                supported = False
                break

            if field_name not in all_fields:
                break

            if query_fields_covered == 0:
                coverage = 'partial'

            if query_fields_covered < max_equiv_seq:
                if field_name not in equiv_fields:
                    ideal_order = False
            elif query_fields_covered < max_sort_seq:
                if field_name not in sort_fields:
                    ideal_order = False
            elif query_fields_covered < max_range_seq:
                if field_name not in range_fields:
                    ideal_order = False
            query_fields_covered += 1
        if query_fields_covered == query_field_count:
            coverage = 'full'

        # INDEX REPORT
        return OrderedDict({
            'coverage': coverage,
            'idealOrder': ideal_order,
            'queryFieldsCovered': query_fields_covered,
            'index': index,
            'supported': supported
        })

Response: Generates a report for an index based on the query analysis.


Instruction:
Given the following Python 3 function, write the documentation
def _generate_recommendation(self,
                                 query_analysis,
                                 db_name,
                                 collection_name):
        """Generates an ideal query recommendation"""
        index_rec = '{'
        for query_field in query_analysis['analyzedFields']:
            if query_field['fieldType'] is EQUIV_TYPE:
                if len(index_rec) is not 1:
                    index_rec += ', '
                index_rec += '"' + query_field['fieldName'] + '": 1'
        for query_field in query_analysis['analyzedFields']:
            if query_field['fieldType'] is SORT_TYPE:
                if len(index_rec) is not 1:
                    index_rec += ', '
                index_rec += '"' + query_field['fieldName'] + '": 1'
        for query_field in query_analysis['analyzedFields']:
            if query_field['fieldType'] is RANGE_TYPE:
                if len(index_rec) is not 1:
                    index_rec += ', '
                index_rec += '"' + query_field['fieldName'] + '": 1'
        index_rec += '}'

        # RECOMMENDATION
        return OrderedDict([('index',index_rec),
                            ('shellCommand', self.generate_shell_command(collection_name, index_rec))])

Response: Generates an ideal query recommendation


Instruction:
Can you generate the documentation for the following Python 3 function
def add_query_occurrence(self, report):

        initial_millis = int(report['parsed']['stats']['millis'])
        mask = report['queryMask']

        existing_report = self._get_existing_report(mask, report)

        if existing_report is not None:
            self._merge_report(existing_report, report)
        else:
            time = None
            if 'ts' in report['parsed']:
                time = report['parsed']['ts']
            self._reports.append(OrderedDict([
                ('namespace', report['namespace']),
                ('lastSeenDate', time),
                ('queryMask', mask),
                ('supported', report['queryAnalysis']['supported']),
                ('indexStatus', report['indexStatus']),
                ('recommendation', report['recommendation']),
                ('stats', OrderedDict([('count', 1),
                                       ('totalTimeMillis', initial_millis),
                                       ('avgTimeMillis', initial_millis)]))]))

Response: Adds a report to the aggregation


Instruction:
How would you explain what the following Python 3 function does
def get_reports(self):
        return sorted(self._reports,
                      key=lambda x: x['stats']['totalTimeMillis'],
                      reverse=True)

Response: Returns a minimized version of the aggregation


Instruction:
Can you write a function in Python 3 where it
returns the aggregated report that matches report

Response:
def _get_existing_report(self, mask, report):
        """Returns the aggregated report that matches report"""
        for existing_report in self._reports:
            if existing_report['namespace'] == report['namespace']:
                if mask == existing_report['queryMask']:
                    return existing_report
        return None


Instruction:
How would you implement a function in Python 3 that
merges a new report into the target report

Response:
def _merge_report(self, target, new):
        """Merges a new report into the target report"""
        time = None
        if 'ts' in new['parsed']:
            time = new['parsed']['ts']

        if (target.get('lastSeenDate', None) and
                time and
                    target['lastSeenDate'] < time):
            target['lastSeenDate'] = time

        query_millis = int(new['parsed']['stats']['millis'])
        target['stats']['totalTimeMillis'] += query_millis
        target['stats']['count'] += 1
        target['stats']['avgTimeMillis'] = target['stats']['totalTimeMillis'] / target['stats']['count']


Instruction:
Write a Python 3 function that can
parse input into a sequence of items.

Response:
def parse(self, input):
        """Passes input to each QueryLineHandler in use"""
        query = None
        for handler in self._line_handlers:
            try:
                query = handler.handle(input)
            except Exception as e:
                query = None
            finally:
                if query is not None:
                    return query
        return None


Instruction:
Can you write a function in Python 3 where it
analyzes a single query and generates a report of the results.

Response:
def generate_query_report(self, db_uri, query, db_name, collection_name):
        """Analyzes a single query"""
        return self._query_analyzer.generate_query_report(db_uri,
                                                          query,
                                                          db_name,
                                                          collection_name)