Instruction:
Can you generate the documentation for the following Python 3 function
def store_alias(self, seq_id, namespace, alias):

        if not self._writeable:
            raise RuntimeError("Cannot write -- opened read-only")

        log_pfx = "store({q},{n},{a})".format(n=namespace, a=alias, q=seq_id)
        try:
            c = self._db.execute("insert into seqalias (seq_id, namespace, alias) values (?, ?, ?)", (seq_id, namespace,
                                                                                alias))
            # success => new record
            return c.lastrowid
        except sqlite3.IntegrityError:
            pass

        # IntegrityError fall-through

        # existing record is guaranteed to exist uniquely; fetchone() should always succeed
        current_rec = self.find_aliases(namespace=namespace, alias=alias).fetchone()

        # if seq_id matches current record, it's a duplicate (seq_id, namespace, alias) tuple
        # and we return current record
        if current_rec["seq_id"] == seq_id:
            _logger.debug(log_pfx + ": duplicate record")
            return current_rec["seqalias_id"]

        # otherwise, we're reassigning; deprecate old record, then retry
        _logger.debug(log_pfx + ": collision; deprecating {s1}".format(s1=current_rec["seq_id"]))
        self._db.execute("update seqalias set is_current = 0 where seqalias_id = ?", [current_rec["seqalias_id"]])
        return self.store_alias(seq_id, namespace, alias)

Response: associate a namespaced alias with a sequence


Instruction:
Write a Python 3 function for
upgrading db using scripts for specified schema version

Response:
def _upgrade_db(self):
        """upgrade db using scripts for specified (current) schema version"""
        migration_path = "_data/migrations"
        sqlite3.connect(self._db_path).close()    # ensure that it exists
        db_url = "sqlite:///" + self._db_path
        backend = yoyo.get_backend(db_url)
        migration_dir = pkg_resources.resource_filename(__package__, migration_path)
        migrations = yoyo.read_migrations(migration_dir)
        assert len(migrations) > 0, "no migration scripts found -- wrong migraion path for " + __package__
        migrations_to_apply = backend.to_apply(migrations)
        backend.apply_migrations(migrations_to_apply)


Instruction:
How would you code a function in Python 3 to
add the names of the assembled sequences to the existing sequences

Response:
def add_assembly_names(opts):
    """add assembly names as aliases to existing sequences

    Specifically, associate aliases like GRCh37.p9:1 with existing
    refseq accessions

    ```
    [{'aliases': ['chr19'],
      'assembly_unit': 'Primary Assembly',
      'genbank_ac': 'CM000681.2',
      'length': 58617616,
      'name': '19',
      'refseq_ac': 'NC_000019.10',
      'relationship': '=',
      'sequence_role': 'assembled-molecule'}]
    ```

    For the above sample record, this function adds the following aliases:
      * genbank:CM000681.2
      * GRCh38:19
      * GRCh38:chr19
    to the sequence referred to by refseq:NC_000019.10.

    """
    seqrepo_dir = os.path.join(opts.root_directory, opts.instance_name)
    sr = SeqRepo(seqrepo_dir, writeable=True)

    assemblies = bioutils.assemblies.get_assemblies()
    if opts.reload_all:
        assemblies_to_load = sorted(assemblies)
    else:
        namespaces = [r["namespace"] for r in sr.aliases._db.execute("select distinct namespace from seqalias")]
        assemblies_to_load = sorted(k for k in assemblies if k not in namespaces)
    _logger.info("{} assemblies to load".format(len(assemblies_to_load)))

    ncbi_alias_map = {r["alias"]: r["seq_id"] for r in sr.aliases.find_aliases(namespace="NCBI", current_only=False)}

    for assy_name in tqdm.tqdm(assemblies_to_load, unit="assembly"):
        _logger.debug("loading " + assy_name)
        sequences = assemblies[assy_name]["sequences"]
        eq_sequences = [s for s in sequences if s["relationship"] in ("=", "<>")]
        if not eq_sequences:
            _logger.info("No '=' sequences to load for {an}; skipping".format(an=assy_name))
            continue

        # all assembled-molecules (1..22, X, Y, MT) have ncbi aliases in seqrepo
        not_in_seqrepo = [s["refseq_ac"] for s in eq_sequences if s["refseq_ac"] not in ncbi_alias_map]
        if not_in_seqrepo:
            _logger.warning("Assembly {an} references {n} accessions not in SeqRepo instance {opts.instance_name} (e.g., {acs})".format(
                an=assy_name, n=len(not_in_seqrepo), opts=opts, acs=", ".join(not_in_seqrepo[:5]+["..."]), seqrepo_dir=seqrepo_dir))
            if not opts.partial_load:
                _logger.warning("Skipping {an} (-p to enable partial loading)".format(an=assy_name))
                continue

        eq_sequences = [es for es in eq_sequences if es["refseq_ac"] in ncbi_alias_map]
        _logger.info("Loading {n} new accessions for assembly {an}".format(an=assy_name, n=len(eq_sequences)))

        for s in eq_sequences:
            seq_id = ncbi_alias_map[s["refseq_ac"]]
            aliases = [{"namespace": assy_name, "alias": a} for a in [s["name"]] + s["aliases"]]
            if "genbank_ac" in s and s["genbank_ac"]:
                aliases += [{"namespace": "genbank", "alias": s["genbank_ac"]}]
            for alias in aliases:
                sr.aliases.store_alias(seq_id=seq_id, **alias)
                _logger.debug("Added assembly alias {a[namespace]}:{a[alias]} for {seq_id}".format(a=alias, seq_id=seq_id))
        sr.commit()


Instruction:
Write a Python 3 script for
blocking until a job with the given job_id has updated its state.

Response:
def wait_for_job_update(self, job_id, timeout=None):
        """
        Blocks until a job given by job_id has updated its state (canceled, completed, progress updated, etc.)
        if timeout is not None, then this function raises iceqube.exceptions.TimeoutError.

        :param job_id: the job's job_id to monitor for changes.
        :param timeout: if None, wait forever for a job update. If given, wait until timeout seconds, and then raise
        iceqube.exceptions.TimeoutError.
        :return: the Job object corresponding to job_id.
        """
        # internally, we register an Event object for each entry in this function.
        # when self.notify_of_job_update() is called, we call Event.set() on all events
        # registered for that job, thereby releasing any threads waiting for that specific job.

        event = JOB_EVENT_MAPPING[job_id]
        event.clear()
        result = event.wait(timeout=timeout)
        job = self.get_job(job_id)

        if result:
            return job
        else:
            raise TimeoutError("Job {} has not received any updates.".format(job_id))


Instruction:
Can you implement a function in Python 3 that
imports a string that represents a module and function and return the same function.

Response:
def import_stringified_func(funcstring):
    """
    Import a string that represents a module and function, e.g. {module}.{funcname}.

    Given a function f, import_stringified_func(stringify_func(f)) will return the same function.
    :param funcstring: String to try to import
    :return: callable
    """
    assert isinstance(funcstring, str)

    modulestring, funcname = funcstring.rsplit('.', 1)

    mod = importlib.import_module(modulestring)

    func = getattr(mod, funcname)
    return func


Instruction:
Can you generate the documentation for the following Python 3 function
def main_loop(self, timeout=None):
        if self.trigger_event.wait(timeout):
            try:
                self.func()
            except Exception as e:
                self.logger.warning("Got an exception running {func}: {e}".format(func=self.func, e=str(e)))
            finally:
                self.trigger_event.clear()

Response: Main loop for the main event loop.


Instruction:
Explain what the following Python 3 code does
def set_sqlite_pragmas(self):

        def _pragmas_on_connect(dbapi_con, con_record):
            dbapi_con.execute("PRAGMA journal_mode = WAL;")

        event.listen(self.engine, "connect", _pragmas_on_connect)

Response: Sets the connection PRAGMAs for the sqlalchemy engine stored in self. engine.


Instruction:
Implement a Python 3 function for
adding the job given by j to the job queue.

Response:
def schedule_job(self, j):
        """
        Add the job given by j to the job queue.

        Note: Does not actually run the job.
        """
        job_id = uuid.uuid4().hex
        j.job_id = job_id

        session = self.sessionmaker()
        orm_job = ORMJob(
            id=job_id,
            state=j.state,
            app=self.app,
            namespace=self.namespace,
            obj=j)
        session.add(orm_job)
        try:
            session.commit()
        except Exception as e:
            logging.error(
                "Got an error running session.commit(): {}".format(e))

        return job_id


Instruction:
Write a Python 3 function for
marking the job as being canceled.

Response:
def mark_job_as_canceling(self, job_id):
        """
        Mark the job as requested for canceling. Does not actually try to cancel a running job.

        :param job_id: the job to be marked as canceling.
        :return: the job object
        """

        job, _ = self._update_job_state(job_id, State.CANCELING)
        return job


Instruction:
How would you code a function in Python 3 to
clear the queue and the job data.

Response:
def clear(self, job_id=None, force=False):
        """
        Clear the queue and the job data. If job_id is not given, clear out all
        jobs marked COMPLETED. If job_id is given, clear out the given job's
        data. This function won't do anything if the job's state is not COMPLETED or FAILED.
        :type job_id: NoneType or str
        :param job_id: the job_id to clear. If None, clear all jobs.
        :type force: bool
        :param force: If True, clear the job (or jobs), even if it hasn't completed or failed.
        """
        s = self.sessionmaker()
        q = self._ns_query(s)
        if job_id:
            q = q.filter_by(id=job_id)

        # filter only by the finished jobs, if we are not specified to force
        if not force:
            q = q.filter(
                or_(ORMJob.state == State.COMPLETED, ORMJob.state == State.FAILED))

        q.delete(synchronize_session=False)
        s.commit()
        s.close()


Instruction:
Write a Python 3 function for
updating the job s progress info.

Response:
def update_job_progress(self, job_id, progress, total_progress):
        """
        Update the job given by job_id's progress info.
        :type total_progress: int
        :type progress: int
        :type job_id: str
        :param job_id: The id of the job to update
        :param progress: The current progress achieved by the job
        :param total_progress: The total progress achievable by the job.
        :return: the job_id
        """

        session = self.sessionmaker()
        job, orm_job = self._update_job_state(
            job_id, state=State.RUNNING, session=session)

        # Note (aron): looks like SQLAlchemy doesn't automatically
        # save any pickletype fields even if we re-set (orm_job.obj = job) that
        # field. My hunch is that it's tracking the id of the object,
        # and if that doesn't change, then SQLAlchemy doesn't repickle the object
        # and save to the DB.
        # Our hack here is to just copy the job object, and then set thespecific
        # field we want to edit, in this case the job.state. That forces
        # SQLAlchemy to re-pickle the object, thus setting it to the correct state.

        job = copy(job)

        job.progress = progress
        job.total_progress = total_progress
        orm_job.obj = job

        session.add(orm_job)
        session.commit()
        session.close()
        return job_id


Instruction:
Can you generate the documentation for the following Python 3 function
def mark_job_as_failed(self, job_id, exception, traceback):
        session = self.sessionmaker()
        job, orm_job = self._update_job_state(
            job_id, State.FAILED, session=session)

        # Note (aron): looks like SQLAlchemy doesn't automatically
        # save any pickletype fields even if we re-set (orm_job.obj = job) that
        # field. My hunch is that it's tracking the id of the object,
        # and if that doesn't change, then SQLAlchemy doesn't repickle the object
        # and save to the DB.
        # Our hack here is to just copy the job object, and then set thespecific
        # field we want to edit, in this case the job.state. That forces
        # SQLAlchemy to re-pickle the object, thus setting it to the correct state.
        job = copy(job)

        job.exception = exception
        job.traceback = traceback
        orm_job.obj = job

        session.add(orm_job)
        session.commit()
        session.close()

Response: Mark the job as failed and record the traceback and exception.


Instruction:
Can you generate the documentation for the following Python 3 function
def _ns_query(self, session):
        return session.query(ORMJob).filter(ORMJob.app == self.app,
                                            ORMJob.namespace == self.namespace)

Response:
Returns a SQLAlchemy query that is already namespaced by the app and namespace given to this backend
        during initialization.


Instruction:
Create a Python 3 function for
starting up the message processor thread that continuously reads the messages sent to self. incoming_message_mailbox and starts or cancels jobs based on the message received.

Response:
def start_message_processing(self):
        """
        Starts up the message processor thread, that continuously reads
        messages sent to self.incoming_message_mailbox, and starts or cancels jobs based on the message received.
        Returns: the Thread object.

        """
        t = InfiniteLoopThread(self.process_messages, thread_name="MESSAGEPROCESSOR", wait_between_runs=0.5)
        t.start()
        return t


Instruction:
How would you implement a function in Python 3 that
reads from the incoming_message_mailbox and process the messages.

Response:
def process_messages(self):
        """
        Read from the incoming_message_mailbox and report to the storage backend
        based on the first message found there.
        Returns: None
        """
        try:
            msg = self.msgbackend.pop(self.incoming_message_mailbox)
            self.handle_incoming_message(msg)
        except queue.Empty:
            logger.debug("Worker message queue currently empty.")


Instruction:
Write a Python 3 script for
handling incoming messages from the queue.

Response:
def handle_incoming_message(self, msg):
        """
        Start or cancel a job, based on the msg.

        If msg.type == MessageType.START_JOB, then start the job given by msg.job.

        If msg.type == MessageType.CANCEL_JOB, then try to cancel the job given by msg.job.job_id.

        Args:
            msg (barbequeue.messaging.classes.Message):

        Returns: None

        """
        if msg.type == MessageType.START_JOB:
            job = msg.message['job']
            self.schedule_job(job)
        elif msg.type == MessageType.CANCEL_JOB:
            job_id = msg.message['job_id']
            self.cancel(job_id)


Instruction:
Can you tell what is the following Python 3 function doing
def _reraise_with_traceback(f):

    def wrap(*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except Exception as e:
            traceback_str = traceback.format_exc()
            e.traceback = traceback_str
            raise e

    return wrap

Response: A decorator that wraps a function with a traceback attribute.


Instruction:
Can you tell what is the following Python 3 function doing
def schedule_job(self, job):
        l = _reraise_with_traceback(job.get_lambda_to_execute())

        future = self.workers.submit(l, update_progress_func=self.update_progress, cancel_job_func=self._check_for_cancel)

        # assign the futures to a dict, mapping them to a job
        self.job_future_mapping[future] = job
        self.future_job_mapping[job.job_id] = future

        # callback for when the future is now!
        future.add_done_callback(self.handle_finished_future)
        # add the job to our cancel notifications data structure, with False at first
        self.cancel_notifications[job.job_id] = False

        return future

Response: schedule a job to the type of workers spawned by self. start_workers.


Instruction:
Can you tell what is the following Python 3 function doing
def cancel(self, job_id):
        future = self.future_job_mapping[job_id]
        is_future_cancelled = future.cancel()

        if is_future_cancelled:  # success!
            return True
        else:
            if future.running():
                # Already running, but let's mark the future as cancelled
                # anyway, to make sure that calling future.result() will raise an error.
                # Our cancelling callback will then check this variable to see its state,
                # and exit if it's cancelled.
                from concurrent.futures._base import CANCELLED
                future._state = CANCELLED
                return False
            else:  # probably finished already, too late to cancel!
                return False

Response: Cancels a job from the executor pool.


Instruction:
Write a Python 3 function for
checking if a job has been requested to be cancelled.

Response:
def _check_for_cancel(self, job_id, current_stage=""):
        """
        Check if a job has been requested to be cancelled. When called, the calling function can
        optionally give the stage it is currently in, so the user has information on where the job
        was before it was cancelled.

        :param job_id: The job_id to check
        :param current_stage: Where the job currently is

        :return: raises a CancelledError if we find out that we were cancelled.
        """

        future = self.future_job_mapping[job_id]
        is_cancelled = future._state in [CANCELLED, CANCELLED_AND_NOTIFIED]

        if is_cancelled:
            raise UserCancelledError(last_stage=current_stage)


Instruction:
How would you code a function in Python 3 to
start the scheduler thread.

Response:
def start_scheduler(self):
        """
        Start the scheduler thread. This thread reads the queue of jobs to be
        scheduled and sends them to the workers.
        Returns: None

        """
        t = InfiniteLoopThread(
            func=self.schedule_next_job,
            thread_name="SCHEDULER",
            wait_between_runs=0.5)
        t.start()
        return t


Instruction:
Implement a function in Python 3 to
start the worker message handler thread.

Response:
def start_worker_message_handler(self):
        """
        Start the worker message handler thread, that loops over messages from workers
        (job progress updates, failures and successes etc.) and then updates the job's status.
        Returns: None

        """
        t = InfiniteLoopThread(
            func=lambda: self.handle_worker_messages(timeout=2),
            thread_name="WORKERMESSAGEHANDLER",
            wait_between_runs=0.5)
        t.start()
        return t


Instruction:
Make a summary of the following Python 3 code
def shutdown(self, wait=True):
        self.scheduler_thread.stop()
        self.worker_message_handler_thread.stop()

        if wait:
            self.scheduler_thread.join()
            self.worker_message_handler_thread.join()

Response: Shut down the worker message handler and scheduler threads.


Instruction:
Write a Python 3 function that can
send a message to the workers to cancel the job with job_id.

Response:
def request_job_cancel(self, job_id):
        """
        Send a message to the workers to cancel the job with job_id. We then mark the job in the storage
        as being canceled.

        :param job_id: the job to cancel
        :return: None
        """
        msg = CancelMessage(job_id)
        self.messaging_backend.send(self.worker_mailbox, msg)
        self.storage_backend.mark_job_as_canceling(job_id)


Instruction:
Can you generate the documentation for the following Python 3 function
def schedule_next_job(self):
        next_job = self.storage_backend.get_next_scheduled_job()
        # TODO: don't loop over if workers are already all running

        if not next_job:
            logging.debug("No job to schedule right now.")
            return

        try:
            self.messaging_backend.send(self.worker_mailbox,
                                        Message(
                                            type=MessageType.START_JOB,
                                            message={'job': next_job}))
            self.storage_backend.mark_job_as_queued(next_job.job_id)
        except Full:
            logging.debug(
                "Worker queue full; skipping scheduling of job {} for now.".format(next_job.job_id)
            )
            return

Response: Schedules the next job in the queue.


Instruction:
Given the following Python 3 function, write the documentation
def handle_worker_messages(self, timeout):
        msgs = self.messaging_backend.popn(self.incoming_mailbox, n=20)

        for msg in msgs:
            self.handle_single_message(msg)

Response: Read messages that are placed in self. incoming_mailbox and update the job states corresponding to each message.


Instruction:
Make a summary of the following Python 3 code
def handle_single_message(self, msg):
        job_id = msg.message['job_id']
        actual_msg = msg.message
        if msg.type == MessageType.JOB_UPDATED:
            progress = actual_msg['progress']
            total_progress = actual_msg['total_progress']
            self.storage_backend.update_job_progress(job_id, progress,
                                                     total_progress)
        elif msg.type == MessageType.JOB_COMPLETED:
            self.storage_backend.complete_job(job_id)
        elif msg.type == MessageType.JOB_FAILED:
            exc = actual_msg['exception']
            trace = actual_msg['traceback']
            self.storage_backend.mark_job_as_failed(job_id, exc, trace)
        elif msg.type == MessageType.JOB_CANCELED:
            self.storage_backend.mark_job_as_canceled(job_id)
        else:
            self.logger.error("Unknown message type: {}".format(msg.type))

Response: Handle a single message and modify the job storage appropriately.


Instruction:
Write a Python 3 script for
returning a function that executes the function assigned to this job.

Response:
def get_lambda_to_execute(self):
        """
        return a function that executes the function assigned to this job.

        If job.track_progress is None (the default), the returned function accepts no argument
        and simply needs to be called. If job.track_progress is True, an update_progress function
        is passed in that can be used by the function to provide feedback progress back to the
        job scheduling system.

        :return: a function that executes the original function assigned to this job.
        """
        def y(update_progress_func, cancel_job_func):
            """
            Call the function stored in self.func, and passing in update_progress_func
            or cancel_job_func depending if self.track_progress or self.cancellable is defined,
            respectively.
            :param update_progress_func: The callback for when the job updates its progress.
            :param cancel_job_func: The function that the function has to call occasionally to see
            if the user wants to cancel the currently running job.
            :return: Any
            """

            func = import_stringified_func(self.func)
            extrafunckwargs = {}

            args, kwargs = copy.copy(self.args), copy.copy(self.kwargs)

            if self.track_progress:
                extrafunckwargs["update_progress"] = partial(update_progress_func, self.job_id)

            if self.cancellable:
                extrafunckwargs["check_for_cancel"] = partial(cancel_job_func, self.job_id)

            kwargs.update(extrafunckwargs)

            return func(*args, **kwargs)

        return y


Instruction:
Can you write a function in Python 3 where it
returns a float between 0 and 1 representing the current job s progress in its task.

Response:
def percentage_progress(self):
        """
        Returns a float between 0 and 1, representing the current job's progress in its task.
        If total_progress is not given or 0, just return self.progress.

        :return: float corresponding to the total percentage progress of the job.
        """

        if self.total_progress != 0:
            return float(self.progress) / self.total_progress
        else:
            return self.progress


Instruction:
How would you implement a function in Python 3 that
schedules a function func for execution.

Response:
def schedule(self, func, *args, **kwargs):
        """
        Schedules a function func for execution.

        One special parameter is track_progress. If passed in and not None, the func will be passed in a
        keyword parameter called update_progress:

        def update_progress(progress, total_progress, stage=""):

        The running function can call the update_progress function to notify interested parties of the function's
        current progress.

        Another special parameter is the "cancellable" keyword parameter. When passed in and not None, a special
        "check_for_cancel" parameter is passed in. When called, it raises an error when the user has requested a job
        to be cancelled.

        The caller can also pass in any pickleable object into the "extra_metadata" parameter. This data is stored
        within the job and can be retrieved when the job status is queried.

        All other parameters are directly passed to the function when it starts running.

        :type func: callable or str
        :param func: A callable object that will be scheduled for running.
        :return: a string representing the job_id.
        """

        # if the func is already a job object, just schedule that directly.
        if isinstance(func, Job):
            job = func
        # else, turn it into a job first.
        else:
            job = Job(func, *args, **kwargs)

        job.track_progress = kwargs.pop('track_progress', False)
        job.cancellable = kwargs.pop('cancellable', False)
        job.extra_metadata = kwargs.pop('extra_metadata', {})
        job_id = self.storage.schedule_job(job)
        return job_id


Instruction:
Write a Python 3 script for
waiting until the job with the given id has a new update.

Response:
def wait(self, job_id, timeout=None):
        """
        Wait until the job given by job_id has a new update.

        :param job_id: the id of the job to wait for.
        :param timeout: how long to wait for a job state change before timing out.
        :return: Job object corresponding to job_id
        """
        return self.storage.wait_for_job_update(job_id, timeout=timeout)


Instruction:
Explain what the following Python 3 code does
def wait_for_completion(self, job_id, timeout=None):
        while 1:
            job = self.wait(job_id, timeout=timeout)
            if job.state in [State.COMPLETED, State.FAILED, State.CANCELED]:
                return job
            else:
                continue

Response: Wait for the job given by job_id to change to COMPLETED or CANCELED.


Instruction:
How would you explain what the following Python 3 function does
def shutdown(self):
        self._storage.clear()
        self._scheduler.shutdown(wait=False)
        self._workers.shutdown(wait=False)

Response: Shutdown the client and all of its managed resources


Instruction:
Write a Python 3 function for
returning dict with info about nodes in cluster and current version { 'nodes': [ 'IP:port', 'IP:port', ], 'version': '1.4.4' }

Response:
def get_cluster_info(host, port, ignore_cluster_errors=False):
    """
    return dict with info about nodes in cluster and current version
    {
        'nodes': [
            'IP:port',
            'IP:port',
        ],
        'version': '1.4.4'
    }
    """
    client = Telnet(host, int(port))
    client.write(b'version\n')
    res = client.read_until(b'\r\n').strip()
    version_list = res.split(b' ')
    if len(version_list) not in [2, 3] or version_list[0] != b'VERSION':
        raise WrongProtocolData('version', res)
    version = version_list[1]
    if StrictVersion(smart_text(version)) >= StrictVersion('1.4.14'):
        cmd = b'config get cluster\n'
    else:
        cmd = b'get AmazonElastiCache:cluster\n'
    client.write(cmd)
    regex_index, match_object, res = client.expect([
        re.compile(b'\n\r\nEND\r\n'),
        re.compile(b'ERROR\r\n')
    ])
    client.close()

    if res == b'ERROR\r\n' and ignore_cluster_errors:
        return {
            'version': version,
            'nodes': [
                '{0}:{1}'.format(smart_text(host),
                               smart_text(port))
            ]
        }

    ls = list(filter(None, re.compile(br'\r?\n').split(res)))
    if len(ls) != 4:
        raise WrongProtocolData(cmd, res)

    try:
        version = int(ls[1])
    except ValueError:
        raise WrongProtocolData(cmd, res)
    nodes = []
    try:
        for node in ls[2].split(b' '):
            host, ip, port = node.split(b'|')
            nodes.append('{0}:{1}'.format(smart_text(ip or host),
                                        smart_text(port)))
    except ValueError:
        raise WrongProtocolData(cmd, res)
    return {
        'version': version,
        'nodes': nodes
    }


Instruction:
Given the following Python 3 function, write the documentation
def invalidate_cache_after_error(f):
    @wraps(f)
    def wrapper(self, *args, **kwds):
        try:
            return f(self, *args, **kwds)
        except Exception:
            self.clear_cluster_nodes_cache()
            raise
    return wrapper

Response: Decorator to invalidate internal cache after an error.


Instruction:
How would you code a function in Python 3 to
update connection params to maximize performance

Response:
def update_params(self, params):
        """
        update connection params to maximize performance
        """
        if not params.get('BINARY', True):
            raise Warning('To increase performance please use ElastiCache'
                          ' in binary mode')
        else:
            params['BINARY'] = True  # patch params, set binary mode
        if 'OPTIONS' not in params:
            # set special 'behaviors' pylibmc attributes
            params['OPTIONS'] = {
                'tcp_nodelay': True,
                'ketama': True
            }


Instruction:
Write a Python 3 function for
getting all nodes in cluster

Response:
def get_cluster_nodes(self):
        """
        return list with all nodes in cluster
        """
        if not hasattr(self, '_cluster_nodes_cache'):
            server, port = self._servers[0].split(':')
            try:
                self._cluster_nodes_cache = (
                    get_cluster_info(server, port,
                                     self._ignore_cluster_errors)['nodes'])
            except (socket.gaierror, socket.timeout) as err:
                raise Exception('Cannot connect to cluster {0} ({1})'.format(
                    self._servers[0], err
                ))
        return self._cluster_nodes_cache


Instruction:
Can you tell what is the following Python 3 function doing
def humanize_placeholders(msgid):
    return re.sub(
        r'%(?:\((\w+)\))?([sd])',
        lambda match: r'__{0}__'.format(
            match.group(1).lower() if match.group(1) else 'number' if match.group(2) == 'd' else 'item'),
        msgid)

Response: Convert the placeholders to the google translate service friendly form.


Instruction:
Given the following Python 3 function, write the documentation
def restore_placeholders(msgid, translation):
    placehoders = re.findall(r'(\s*)(%(?:\(\w+\))?[sd])(\s*)', msgid)
    return re.sub(
        r'(\s*)(__[\w]+?__)(\s*)',
        lambda matches: '{0}{1}{2}'.format(placehoders[0][0], placehoders[0][1], placehoders.pop(0)[2]),
        translation)

Response: Restore placeholders in the translated message.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_strings_to_translate(self, po):
        strings = []
        for index, entry in enumerate(po):
            if not self.need_translate(entry):
                continue
            strings.append(humanize_placeholders(entry.msgid))
            if entry.msgid_plural:
                strings.append(humanize_placeholders(entry.msgid_plural))
        return strings

Response: Return list of strings to translate from po file.


Instruction:
Can you write a function in Python 3 where it
updates the translations in the entries.

Response:
def update_translations(self, entries, translated_strings):
        """Update translations in entries.

        The order and number of translations should match to get_strings_to_translate() result.

        :param entries: list of entries to translate
        :type entries: collections.Iterable[polib.POEntry] | polib.POFile
        :param translated_strings: list of translations
        :type translated_strings: collections.Iterable[six.text_type]
        """
        translations = iter(translated_strings)
        for entry in entries:
            if not self.need_translate(entry):
                continue

            if entry.msgid_plural:
                # fill the first plural form with the entry.msgid translation
                translation = next(translations)
                translation = fix_translation(entry.msgid, translation)
                entry.msgstr_plural[0] = translation

                # fill the rest of plural forms with the entry.msgid_plural translation
                translation = next(translations)
                translation = fix_translation(entry.msgid_plural, translation)
                for k, v in entry.msgstr_plural.items():
                    if k != 0:
                        entry.msgstr_plural[k] = translation
            else:
                translation = next(translations)
                translation = fix_translation(entry.msgid, translation)
                entry.msgstr = translation

            # Set the 'fuzzy' flag on translation
            if self.set_fuzzy and 'fuzzy' not in entry.flags:
                entry.flags.append('fuzzy')


Instruction:
Can you tell what is the following Python 3 function doing
def load_ply(fileobj):
    def nextline():
        """Read next line, skip comments"""
        while True:
            line = fileobj.readline()
            assert line != ''  # eof
            if not line.startswith('comment'):
                return line.strip()

    assert nextline() == 'ply'
    assert nextline() == 'format ascii 1.0'
    line = nextline()
    assert line.startswith('element vertex')
    nverts = int(line.split()[2])
    # print 'nverts : ', nverts
    assert nextline() == 'property float x'
    assert nextline() == 'property float y'
    assert nextline() == 'property float z'
    line = nextline()

    assert line.startswith('element face')
    nfaces = int(line.split()[2])
    # print 'nfaces : ', nfaces
    assert nextline() == 'property list uchar int vertex_indices'
    line = nextline()
    has_texcoords = line == 'property list uchar float texcoord'
    if has_texcoords:
        assert nextline() == 'end_header'
    else:
        assert line == 'end_header'

    # Verts
    verts = np.zeros((nverts, 3))
    for i in range(nverts):
        vals = nextline().split()
        verts[i, :] = [float(v) for v in vals[:3]]
    # Faces
    faces = []
    faces_uv = []
    for i in range(nfaces):
        vals = nextline().split()
        assert int(vals[0]) == 3
        faces.append([int(v) for v in vals[1:4]])
        if has_texcoords:
            assert len(vals) == 11
            assert int(vals[4]) == 6
            faces_uv.append([(float(vals[5]), float(vals[6])),
                             (float(vals[7]), float(vals[8])),
                             (float(vals[9]), float(vals[10]))])
            # faces_uv.append([float(v) for v in vals[5:]])
        else:
            assert len(vals) == 4
    return verts, faces, faces_uv

Response: This function loads a PLY file into a new tree structure.


Instruction:
Explain what the following Python 3 code does
def read_ssh_config(path):
    with open(path, "r") as fh_:
        lines = fh_.read().splitlines()
    return SshConfig(lines)

Response: Read SSH config file and return parsed SshConfig


Instruction:
Write a Python 3 script for
changing key into correct casing if we know the parameter

Response:
def _remap_key(key):
    """ Change key into correct casing if we know the parameter """
    if key in KNOWN_PARAMS:
        return key
    if key.lower() in known_params:
        return KNOWN_PARAMS[known_params.index(key.lower())]
    return key


Instruction:
How would you code a function in Python 3 to
parse lines from ssh config file

Response:
def parse(self, lines):
        """Parse lines from ssh config file"""
        cur_entry = None
        for line in lines:
            kv_ = _key_value(line)
            if len(kv_) > 1:
                key, value = kv_
                if key.lower() == "host":
                    cur_entry = value
                    self.hosts_.add(value)
                self.lines_.append(ConfigLine(line=line, host=cur_entry, key=key, value=value))
            else:
                self.lines_.append(ConfigLine(line=line))


Instruction:
How would you explain what the following Python 3 function does
def host(self, host):
        if host in self.hosts_:
            vals = defaultdict(list)
            for k, value in [(x.key.lower(), x.value) for x in self.lines_
                             if x.host == host and x.key.lower() != "host"]:
                vals[k].append(value)
            flatten = lambda x: x[0] if len(x) == 1 else x
            return {k: flatten(v) for k, v in vals.items()}
        return {}

Response: Return the configuration of a specific host as a dictionary.


Instruction:
Explain what the following Python 3 code does
def set(self, host, **kwargs):
        self.__check_host_args(host, kwargs)

        def update_line(key, value):
            """Produce new config line"""
            return "  %s %s" % (key, value)

        for key, values in kwargs.items():
            if type(values) not in [list, tuple]:  # pylint: disable=unidiomatic-typecheck
                values = [values]

            lower_key = key.lower()
            update_idx = [idx for idx, x in enumerate(self.lines_)
                          if x.host == host and x.key.lower() == lower_key]
            extra_remove = []
            for idx in update_idx:
                if values:  # values available, update the line
                    value = values.pop()
                    self.lines_[idx].line = update_line(self.lines_[idx].key, value)
                    self.lines_[idx].value = value
                else:                # no more values available, remove the line
                    extra_remove.append(idx)

            for idx in reversed(sorted(extra_remove)):
                del self.lines_[idx]

            if values:
                mapped_key = _remap_key(key)
                max_idx = max([idx for idx, line in enumerate(self.lines_) if line.host == host])
                for value in values:
                    self.lines_.insert(max_idx + 1, ConfigLine(line=update_line(mapped_key, value),
                                                               host=host, key=mapped_key,
                                                               value=value))

Response: Set the configuration values for an existing host.


Instruction:
Explain what the following Python 3 code does
def unset(self, host, *args):
        self.__check_host_args(host, args)
        remove_idx = [idx for idx, x in enumerate(self.lines_)
                      if x.host == host and x.key.lower() in args]
        for idx in reversed(sorted(remove_idx)):
            del self.lines_[idx]

Response: Removes settings for a host.


Instruction:
Explain what the following Python 3 code does
def __check_host_args(self, host, keys):
        if host not in self.hosts_:
            raise ValueError("Host %s: not found" % host)

        if "host" in [x.lower() for x in keys]:
            raise ValueError("Cannot modify Host value")

Response: Checks the parameters of the host attribute


Instruction:
Can you tell what is the following Python 3 function doing
def rename(self, old_host, new_host):
        if new_host in self.hosts_:
            raise ValueError("Host %s: already exists." % new_host)
        for line in self.lines_:  # update lines
            if line.host == old_host:
                line.host = new_host
                if line.key.lower() == "host":
                    line.value = new_host
                    line.line = "Host %s" % new_host
        self.hosts_.remove(old_host)  # update host cache
        self.hosts_.add(new_host)

Response: Renames a host configuration.


Instruction:
How would you explain what the following Python 3 function does
def add(self, host, **kwargs):
        if host in self.hosts_:
            raise ValueError("Host %s: exists (use update)." % host)
        self.hosts_.add(host)
        self.lines_.append(ConfigLine(line="", host=None))
        self.lines_.append(ConfigLine(line="Host %s" % host, host=host, key="Host", value=host))
        for k, v in kwargs.items():
            if type(v) not in [list, tuple]:
                v = [v]
            mapped_k = _remap_key(k)
            for value in v:
                self.lines_.append(ConfigLine(line="  %s %s" % (mapped_k, str(value)), host=host, key=mapped_k, value=value))
        self.lines_.append(ConfigLine(line="", host=None))

Response: Add another host to the SSH configuration.


Instruction:
Given the following Python 3 function, write the documentation
def remove(self, host):
        if host not in self.hosts_:
            raise ValueError("Host %s: not found." % host)
        self.hosts_.remove(host)
        # remove lines, including comments inside the host lines
        host_lines = [ idx for idx, x in enumerate(self.lines_) if x.host == host ]
        remove_range = reversed(range(min(host_lines), max(host_lines) + 1))
        for idx in remove_range:
            del self.lines_[idx]

Response: Removes a host from the SSH configuration.


Instruction:
Create a Python 3 function to
write the config file to the specified path.

Response:
def write(self, path):
        """
        Writes ssh config file

        Parameters
        ----------
        path : The file to write to
        """
        with open(path, "w") as fh_:
            fh_.write(self.config())


Instruction:
Given the following Python 3 function, write the documentation
def orthogonal_vector(v):
    if v[1] != 0 or v[2] != 0:
        c = (1, 0, 0)
    else:
        c = (0, 1, 0)
    return np.cross(v, c)

Response: Return an arbitrary vector that is orthogonal to v


Instruction:
Can you generate a brief explanation for the following Python 3 code
def show_plane(orig, n, scale=1.0, **kwargs):
    b1 = orthogonal_vector(n)
    b1 /= la.norm(b1)
    b2 = np.cross(b1, n)
    b2 /= la.norm(b2)
    verts = [orig + scale*(-b1 - b2),
             orig + scale*(b1 - b2),
             orig + scale*(b1 + b2),
             orig + scale*(-b1 + b2)]
    faces = [(0, 1, 2), (0, 2, 3)]
    trimesh3d(np.array(verts), faces, **kwargs)

Response: Show a plane with the given origin and normal. scale give its size


Instruction:
Can you tell what is the following Python 3 function doing
def slice_triangle_plane(verts, tri, plane_orig, plane_norm):
    dists = [point_to_plane_dist(p, plane_orig, plane_norm)
             for p in verts[tri]]

    if np.sign(dists[0]) == np.sign(dists[1]) \
            and np.sign(dists[1]) == np.sign(dists[2]):
        # Triangle is on one side of the plane
        return []

    # Iterate through the edges, cutting the ones that intersect
    intersect_points = []
    for fi in range(3):
        v1 = verts[tri[fi]]
        d1 = dists[fi]
        v2 = verts[tri[(fi + 1) % 3]]
        d2 = dists[(fi + 1) % 3]

        if d1 * d2 < 0:
            # intersection factor (between 0 and 1)
            # here is a nice drawing :
            # https://ravehgonen.files.wordpress.com/2013/02/slide8.png
            s = d1 / (d1 - d2)
            vdir = v2 - v1
            intersect_points.append(v1 + vdir * s)
        elif np.fabs(d1) < 1e-5:
            # point on plane
            intersect_points.append(v1)

    return intersect_points

Response: Slice the internal list of the base class of the base class of the base class of the base class.


Instruction:
Write a Python 3 script to
return true if the given triangle is cut by the plane. This will return false if the given triangle is cut by the plane. This will return True if the given triangle is cut by the plane. This will return False if the given triangle lies on the plane.

Response:
def triangle_intersects_plane(mesh, tid, plane):
    """
    Returns true if the given triangle is cut by the plane. This will return
    false if a single vertex of the triangle lies on the plane
    """
    dists = [point_to_plane_dist(mesh.verts[vid], plane)
             for vid in mesh.tris[tid]]
    side = np.sign(dists)
    return not (side[0] == side[1] == side[2])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def compute_triangle_plane_intersections(mesh, tid, plane, dist_tol=1e-8):
    # TODO: Use a distance cache
    dists = {vid: point_to_plane_dist(mesh.verts[vid], plane)
             for vid in mesh.tris[tid]}
    # TODO: Use an edge intersection cache (we currently compute each edge
    # intersection twice : once for each tri)

    # This is to avoid registering the same vertex intersection twice
    # from two different edges
    vert_intersect = {vid: False for vid in dists.keys()}

    # Iterate through the edges, cutting the ones that intersect
    intersections = []
    for e in mesh.edges_for_triangle(tid):
        v1 = mesh.verts[e[0]]
        d1 = dists[e[0]]
        v2 = mesh.verts[e[1]]
        d2 = dists[e[1]]

        if np.fabs(d1) < dist_tol:
            # Avoid creating the vertex intersection twice
            if not vert_intersect[e[0]]:
                # point on plane
                intersections.append((INTERSECT_VERTEX, v1, e[0]))
                vert_intersect[e[0]] = True
        if np.fabs(d2) < dist_tol:
            if not vert_intersect[e[1]]:
                # point on plane
                intersections.append((INTERSECT_VERTEX, v2, e[1]))
                vert_intersect[e[1]] = True

        # If vertices are on opposite sides of the plane, we have an edge
        # intersection
        if d1 * d2 < 0:
            # Due to numerical accuracy, we could have both a vertex intersect
            # and an edge intersect on the same vertex, which is impossible
            if not vert_intersect[e[0]] and not vert_intersect[e[1]]:
                # intersection factor (between 0 and 1)
                # here is a nice drawing :
                # https://ravehgonen.files.wordpress.com/2013/02/slide8.png
                # keep in mind d1, d2 are *signed* distances (=> d1 - d2)
                s = d1 / (d1 - d2)
                vdir = v2 - v1
                ipos = v1 + vdir * s
                intersections.append((INTERSECT_EDGE, ipos, e))

    return intersections

Response: Compute the intersection between a triangle and a plane


Instruction:
Implement a Python 3 function for
returning the next triangle to visit given the intersection and T

Response:
def get_next_triangle(mesh, T, plane, intersection, dist_tol):
    """
    Returns the next triangle to visit given the intersection and
    the list of unvisited triangles (T)

    We look for a triangle that is cut by the plane (2 intersections) as
    opposed to one that only touch the plane (1 vertex intersection)
    """
    if intersection[0] == INTERSECT_EDGE:
        tris = mesh.triangles_for_edge(intersection[2])
    elif intersection[0] == INTERSECT_VERTEX:
        tris = mesh.triangles_for_vert(intersection[2])
    else:
        assert False, 'Invalid intersection[0] value : %d' % intersection[0]

    # Knowing where we come from is not enough. If an edge of the triangle
    # lies exactly on the plane, i.e. :
    #
    #   /t1\
    # -v1---v2-
    #   \t2/
    #
    # With v1, v2 being the vertices and t1, t2 being the triangles, then
    # if you just try to go to the next connected triangle that intersect,
    # you can visit v1 -> t1 -> v2 -> t2 -> v1 .
    # Therefore, we need to limit the new candidates to the set of unvisited
    # triangles and once we've visited a triangle and decided on a next one,
    # remove all the neighbors of the visited triangle so we don't come
    # back to it

    T = set(T)
    for tid in tris:
        if tid in T:
            intersections = compute_triangle_plane_intersections(
                    mesh, tid, plane, dist_tol)
            if len(intersections) == 2:
                T = T.difference(tris)
                return tid, intersections, T
    return None, [], T


Instruction:
How would you explain what the following Python 3 function does
def _walk_polyline(tid, intersect, T, mesh, plane, dist_tol):
    T = set(T)
    p = []
    # Loop until we have explored all the triangles for the current
    # polyline
    while True:
        p.append(intersect[1])

        tid, intersections, T = get_next_triangle(mesh, T, plane,
                                                  intersect, dist_tol)
        if tid is None:
            break

        # get_next_triangle returns triangles that our plane actually
        # intersects (as opposed to touching only a single vertex),
        # hence the assert
        assert len(intersections) == 2

        # Of the two returned intersections, one should have the
        # intersection point equal to p[-1]
        if la.norm(intersections[0][1] - p[-1]) < dist_tol:
            intersect = intersections[1]
        else:
            assert la.norm(intersections[1][1] - p[-1]) < dist_tol, \
                '%s not close to %s' % (str(p[-1]), str(intersections))
            intersect = intersections[0]

    return p, T

Response: Given an intersection walk through the mesh and compute the intersection with the cut plane and add those intersection to the polyline.


Instruction:
Can you create a Python 3 function that
returns a list of all triangles that are cross - sectioned by a given plane.

Response:
def cross_section_mesh(mesh, plane, dist_tol=1e-8):
    """
    Args:
        mesh: A geom.TriangleMesh instance
        plane: The cut plane : geom.Plane instance
        dist_tol: If two points are closer than dist_tol, they are considered
                  the same
    """
    # Set of all triangles
    T = set(range(len(mesh.tris)))
    # List of all cross-section polylines
    P = []

    while len(T) > 0:
        tid = T.pop()

        intersections = compute_triangle_plane_intersections(
                mesh, tid, plane, dist_tol)

        if len(intersections) == 2:
            for intersection in intersections:
                p, T = _walk_polyline(tid, intersection, T, mesh, plane,
                                      dist_tol)
                if len(p) > 1:
                    P.append(np.array(p))
    return P


Instruction:
Can you generate a brief explanation for the following Python 3 code
def cross_section(verts, tris, plane_orig, plane_normal, **kwargs):
    mesh = TriangleMesh(verts, tris)
    plane = Plane(plane_orig, plane_normal)
    return cross_section_mesh(mesh, plane, **kwargs)

Response: Compute the planar cross section of a mesh. This is a wrapper around the cross_section_mesh function that creates a polyline that is connected to the mesh.


Instruction:
Implement a Python 3 function for
computing spatial distance using pure numpy

Response:
def pdist_squareformed_numpy(a):
    """
    Compute spatial distance using pure numpy
    (similar to scipy.spatial.distance.cdist())

    Thanks to Divakar Roy (@droyed) at stackoverflow.com

    Note this needs at least np.float64 precision!

    Returns: dist
    """
    a = np.array(a, dtype=np.float64)
    a_sumrows = np.einsum('ij,ij->i', a, a)
    dist = a_sumrows[:, None] + a_sumrows - 2 * np.dot(a, a.T)
    np.fill_diagonal(dist, 0)
    return dist


Instruction:
Write a Python 3 function for
converting signed hexadecimal to floating value.

Response:
def signed_to_float(hex: str) -> float:
    """Convert signed hexadecimal to floating value."""
    if int(hex, 16) & 0x8000:
        return -(int(hex, 16) & 0x7FFF) / 10
    else:
        return int(hex, 16) / 10


Instruction:
Make a summary of the following Python 3 code
def decode_packet(packet: str) -> dict:
    node_id, _, protocol, attrs = packet.split(DELIM, 3)

    data = cast(Dict[str, Any], {
        'node': PacketHeader(node_id).name,
    })

    # make exception for version response
    data['protocol'] = UNKNOWN
    if '=' in protocol:
        attrs = protocol + DELIM + attrs

    # no attributes but instead the welcome banner
    elif 'RFLink Gateway' in protocol:
        data.update(parse_banner(protocol))

    elif protocol == 'PONG':
        data['ping'] = protocol.lower()

    # debug response
    elif protocol == 'DEBUG':
        data['protocol'] = protocol.lower()
        data['tm'] = packet[3:5]

    # failure response
    elif protocol == 'CMD UNKNOWN':
        data['response'] = 'command_unknown'
        data['ok'] = False

    # ok response
    elif protocol == 'OK':
        data['ok'] = True

    # its a regular packet
    else:
        data['protocol'] = protocol.lower()

    # convert key=value pairs where needed
    for attr in filter(None, attrs.strip(DELIM).split(DELIM)):
        key, value = attr.lower().split('=')
        if key in VALUE_TRANSLATION:
            value = VALUE_TRANSLATION.get(key)(value)
        name = PACKET_FIELDS.get(key, key)
        data[name] = value
        unit = UNITS.get(key, None)

        if unit:
            data[name + '_unit'] = unit

    # correct KaKu device address
    if data.get('protocol', '') == 'kaku' and len(data['id']) != 6:
        data['id'] = '0000' + data['id']

    return data

Response: Decode a packet into a dict of data structures.


Instruction:
How would you explain what the following Python 3 function does
def encode_packet(packet: dict) -> str:
    if packet['protocol'] == 'rfdebug':
        return '10;RFDEBUG=' + packet['command'] + ';'
    elif packet['protocol'] == 'rfudebug':
        return '10;RFDEBUG=' + packet['command'] + ';'
    else:
        return SWITCH_COMMAND_TEMPLATE.format(
            node=PacketHeader.master.value,
            **packet
        )

Response: Construct packet string from packet dictionary.


Instruction:
Explain what the following Python 3 code does
def serialize_packet_id(packet: dict) -> str:
    # translate protocol in something reversable
    protocol = protocol_translations[packet['protocol']]

    if protocol == UNKNOWN:
        protocol = 'rflink'

    return '_'.join(filter(None, [
        protocol,
        packet.get('id', None),
        packet.get('switch', None),
    ]))

Response: Serialize packet identifiers into one reversable string.


Instruction:
Explain what the following Python 3 code does
def deserialize_packet_id(packet_id: str) -> dict:
    if packet_id == 'rflink':
        return {'protocol': UNKNOWN}

    protocol, *id_switch = packet_id.split(PACKET_ID_SEP)
    assert len(id_switch) < 3

    packet_identifiers = {
        # lookup the reverse translation of the protocol in the translation
        # table, fallback to protocol. If this is a unserializable protocol
        # name, it has not been serialized before and is not in the
        # translate_protocols table this will result in an invalid command.
        'protocol': protocol_translations.get(protocol, protocol),
    }
    if id_switch:
        packet_identifiers['id'] = id_switch[0]
    if len(id_switch) > 1:
        packet_identifiers['switch'] = id_switch[1]

    return packet_identifiers

Response: r Deserialize a packet id into a dictionary of packet identifiers.


Instruction:
Implement a Python 3 function for
returning list of all events in the packet.

Response:
def packet_events(packet: dict) -> Generator:
    """Return list of all events in the packet.

    >>> x = list(packet_events({
    ...     'protocol': 'alecto v1',
    ...     'id': 'ec02',
    ...     'temperature': 1.0,
    ...     'temperature_unit': '°C',
    ...     'humidity': 10,
    ...     'humidity_unit': '%',
    ... }))
    >>> assert {
    ...     'id': 'alectov1_ec02_temp',
    ...     'sensor': 'temperature',
    ...     'value': 1.0,
    ...     'unit': '°C',
    ... } in x
    >>> assert {
    ...     'id': 'alectov1_ec02_hum',
    ...     'sensor': 'humidity',
    ...     'value': 10,
    ...     'unit': '%',
    ... } in x
    >>> y = list(packet_events({
    ...     'protocol': 'newkaku',
    ...     'id': '000001',
    ...     'switch': '01',
    ...     'command': 'on',
    ... }))
    >>> assert {'id': 'newkaku_000001_01', 'command': 'on'} in y
    """
    field_abbrev = {v: k for k, v in PACKET_FIELDS.items()}

    packet_id = serialize_packet_id(packet)
    events = {f: v for f, v in packet.items() if f in field_abbrev}
    if 'command' in events or 'version' in events:
        # switch events only have one event in each packet
        yield dict(id=packet_id, **events)
    else:
        if packet_id == 'debug':
            yield {
                'id': 'raw',
                'value': packet.get('pulses(usec)'),
                'tm': packet.get('tm'),
                'pulses': packet.get('pulses'),
            }
        else:
            # sensors can have multiple
            for sensor, value in events.items():
                unit = packet.get(sensor + '_unit', None)
                yield {
                    'id': packet_id + PACKET_ID_SEP + field_abbrev[sensor],
                    'sensor': sensor,
                    'value': value,
                    'unit': unit,
                }

            if packet_id != 'rflink':
                yield {
                   'id': packet_id + PACKET_ID_SEP + 'update_time',
                   'sensor': 'update_time',
                   'value': round(time.time()),
                   'unit': 's',
                }


Instruction:
Implement a Python 3 function for
decoding a single packet into a dict of data structures.

Response:
def decode_tx_packet(packet: str) -> dict:
    """Break packet down into primitives, and do basic interpretation.

    >>> decode_packet('10;Kaku;ID=41;SWITCH=1;CMD=ON;') == {
    ...     'node': 'gateway',
    ...     'protocol': 'kaku',
    ...     'id': '000041',
    ...     'switch': '1',
    ...     'command': 'on',
    ... }
    True
    """
    node_id, protocol, attrs = packet.split(DELIM, 2)

    data = cast(Dict[str, Any], {
        'node': PacketHeader(node_id).name,
    })

    data['protocol'] = protocol.lower()

    for i, attr in enumerate(filter(None, attrs.strip(DELIM).split(DELIM))):
        if i == 0:
            data['id'] = attr
        if i == 1:
            data['switch'] = attr
        if i == 2:
            data['command'] = attr

    # correct KaKu device address
    if data.get('protocol', '') == 'kaku' and len(data['id']) != 6:
        data['id'] = '0000' + data['id']

    return data


Instruction:
How would you implement a function in Python 3 that
parses argument and setup main program loop.

Response:
def main(argv=sys.argv[1:], loop=None):
    """Parse argument and setup main program loop."""
    args = docopt(__doc__, argv=argv,
                  version=pkg_resources.require('rflink')[0].version)

    level = logging.ERROR
    if args['-v']:
        level = logging.INFO
    if args['-v'] == 2:
        level = logging.DEBUG
    logging.basicConfig(level=level)

    if not loop:
        loop = asyncio.get_event_loop()

    host = args['--host']
    port = args['--port']
    baud = args['--baud']
    listenport = args['--listenport']

    proxy = RFLinkProxy(port=port, host=host, baud=baud, loop=loop)

    server_coro = asyncio.start_server(
        proxy.client_connected_callback,
        host="",
        port=listenport,
        loop=loop,
    )

    server = loop.run_until_complete(server_coro)
    addr = server.sockets[0].getsockname()
    log.info('Serving on %s', addr)

    conn_coro = proxy.connect()
    loop.run_until_complete(conn_coro)

    proxy.closing = False
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        proxy.closing = True

        # cleanup server
        server.close()
        loop.run_until_complete(server.wait_closed())

        # cleanup server connections
        writers = [i[1] for i in list(clients)]
        for writer in writers:
            writer.close()
            if sys.version_info >= (3, 7):
                loop.run_until_complete(writer.wait_closed())

        # cleanup RFLink connection
        proxy.transport.close()

    finally:
        loop.close()


Instruction:
Write a Python 3 script for
parsing a raw packet into a dict.

Response:
def handle_raw_packet(self, raw_packet):
        """Parse raw packet string into packet dict."""
        log.debug('got packet: %s', raw_packet)
        packet = None
        try:
            packet = decode_packet(raw_packet)
        except:
            log.exception('failed to parse packet: %s', packet)

        log.debug('decoded packet: %s', packet)

        if packet:
            if 'ok' in packet:
                # handle response packets internally
                log.debug('command response: %s', packet)
                self._last_ack = packet
                self._command_ack.set()
            elif self.raw_callback:
                self.raw_callback(raw_packet)
        else:
            log.warning('no valid packet')


Instruction:
Can you implement a function in Python 3 that
parses a raw packet into a dict.

Response:
def handle_raw_tx_packet(self, writer, raw_packet):
        """Parse raw packet string into packet dict."""
        peer = writer.get_extra_info('peername')
        log.debug(' %s:%s: processing data: %s', peer[0], peer[1], raw_packet)
        packet = None
        try:
            packet = decode_tx_packet(raw_packet)
        except:
            log.exception(' %s:%s: failed to parse packet: %s',
                          peer[0], peer[1], packet)

        log.debug(' %s:%s: decoded packet: %s', peer[0], peer[1], packet)
        if self.protocol and packet:
            if not ';PING;' in raw_packet:
                log.info(' %s:%s: forwarding packet %s to RFLink', peer[0], peer[1], raw_packet)
            else:
                log.debug(' %s:%s: forwarding packet %s to RFLink', peer[0], peer[1], raw_packet)
            yield from self.forward_packet(writer, packet, raw_packet)
        else:
            log.warning(' %s:%s: no valid packet %s', peer[0], peer[1], packet)


Instruction:
Can you write a function in Python 3 where it
forwards packet from client to RFLink.

Response:
def forward_packet(self, writer, packet, raw_packet):
        """Forward packet from client to RFLink."""
        peer = writer.get_extra_info('peername')
        log.debug(' %s:%s: forwarding data: %s', peer[0], peer[1], packet)
        if 'command' in packet:
            packet_id = serialize_packet_id(packet)
            command = packet['command']
            ack = yield from self.protocol.send_command_ack(
                packet_id, command)
            if ack:
                writer.write("20;00;OK;".encode() + CRLF)
            for _ in range(DEFAULT_SIGNAL_REPETITIONS-1):
                yield from self.protocol.send_command_ack(
                    packet_id, command)
        else:
            self.protocol.send_raw_packet(raw_packet)


Instruction:
Make a summary of the following Python 3 code
def client_connected_callback(self, reader, writer):
        peer = writer.get_extra_info('peername')
        clients.append((reader, writer, peer))
        log.info("Incoming connection from: %s:%s", peer[0], peer[1])
        try:
            while True:
                data = yield from reader.readline()
                if not data:
                    break
                try:
                    line = data.decode().strip()
                except UnicodeDecodeError:
                    line = '\x00'

                # Workaround for domoticz issue #2816
                if line[-1] != DELIM:
                    line = line + DELIM

                if valid_packet(line):
                    yield from self.handle_raw_tx_packet(writer, line)
                else:
                    log.warning(" %s:%s: dropping invalid data: '%s'", peer[0], peer[1], line)
                    pass
        except ConnectionResetError:
            pass
        except Exception as e:
            log.exception(e)

        log.info("Disconnected from: %s:%s", peer[0], peer[1])
        writer.close()
        clients.remove((reader, writer, peer))

Response: Handle a connection from a domoticz server.


Instruction:
Can you implement a function in Python 3 that
sends data to all connected clients.

Response:
def raw_callback(self, raw_packet):
        """Send data to all connected clients."""
        if not ';PONG;' in raw_packet:
            log.info('forwarding packet %s to clients', raw_packet)
        else:
            log.debug('forwarding packet %s to clients', raw_packet)
        writers = [i[1] for i in list(clients)]
        for writer in writers:
            writer.write(str(raw_packet).encode() + CRLF)


Instruction:
How would you explain what the following Python 3 function does
def reconnect(self, exc=None):
        # Reset protocol binding before starting reconnect
        self.protocol = None

        if not self.closing:
            log.warning('disconnected from Rflink, reconnecting')
            self.loop.create_task(self.connect())

Response: Schedule reconnect after connection has been unexpectedly lost.


Instruction:
Write a Python 3 function that can
set up connection and hook it into HA for reconnect/shutdown.

Response:
async def connect(self):
        """Set up connection and hook it into HA for reconnect/shutdown."""
        import serial

        log.info('Initiating Rflink connection')

        # Rflink create_rflink_connection decides based on the value of host
        # (string or None) if serial or tcp mode should be used

        # Setup protocol
        protocol = partial(
            ProxyProtocol,
            disconnect_callback=self.reconnect,
            raw_callback=self.raw_callback,
            loop=self.loop,
        )

        # Initiate serial/tcp connection to Rflink gateway
        if self.host:
            connection = self.loop.create_connection(protocol, self.host, self.port)
        else:
            connection = create_serial_connection(self.loop, protocol, self.port, self.baud)

        try:
            with async_timeout.timeout(CONNECTION_TIMEOUT,
                                       loop=self.loop):
                self.transport, self.protocol = await connection

        except (serial.serialutil.SerialException, ConnectionRefusedError,
                TimeoutError, OSError, asyncio.TimeoutError) as exc:
            reconnect_interval = DEFAULT_RECONNECT_INTERVAL
            log.error(
                "Error connecting to Rflink, reconnecting in %s",
                reconnect_interval)

            self.loop.call_later(reconnect_interval, self.reconnect, exc)
            return

        log.info('Connected to Rflink')


Instruction:
Explain what the following Python 3 code does
def create_rflink_connection(port=None, host=None, baud=57600, protocol=RflinkProtocol,
                             packet_callback=None, event_callback=None,
                             disconnect_callback=None, ignore=None, loop=None):
    """Create Rflink manager class, returns transport coroutine."""
    # use default protocol if not specified
    protocol = partial(
        protocol,
        loop=loop if loop else asyncio.get_event_loop(),
        packet_callback=packet_callback,
        event_callback=event_callback,
        disconnect_callback=disconnect_callback,
        ignore=ignore if ignore else [],
    )

    # setup serial connection if no transport specified
    if host:
        conn = loop.create_connection(protocol, host, port)
    else:
        baud = baud
        conn = create_serial_connection(loop, protocol, port, baud)

    return conn

Response: Create Rflink manager class, returns transport coroutine.


Instruction:
Can you write a function in Python 3 where it
adds incoming data to buffer.

Response:
def data_received(self, data):
        """Add incoming data to buffer."""
        data = data.decode()
        log.debug('received data: %s', data.strip())
        self.buffer += data
        self.handle_lines()


Instruction:
How would you explain what the following Python 3 function does
def handle_lines(self):
        while "\r\n" in self.buffer:
            line, self.buffer = self.buffer.split("\r\n", 1)
            if valid_packet(line):
                self.handle_raw_packet(line)
            else:
                log.warning('dropping invalid data: %s', line)

Response: Assemble incoming data into per - line packets.


Instruction:
How would you explain what the following Python 3 function does
def send_raw_packet(self, packet: str):
        data = packet + '\r\n'
        log.debug('writing data: %s', repr(data))
        self.transport.write(data.encode())

Response: Encode and put packet string onto write buffer.


Instruction:
Can you create a Python 3 function that
logs all data received from RFLink to file.

Response:
def log_all(self, file):
        """Log all data received from RFLink to file."""
        global rflink_log
        if file == None:
            rflink_log = None
        else:
            log.debug('logging to: %s', file)
            rflink_log = open(file, 'a')


Instruction:
Implement a function in Python 3 to
log when connection is lost.

Response:
def connection_lost(self, exc):
        """Log when connection is closed, if needed call callback."""
        if exc:
            log.exception('disconnected due to exception')
        else:
            log.info('disconnected because of close/abort.')
        if self.disconnect_callback:
            self.disconnect_callback(exc)


Instruction:
Make a summary of the following Python 3 code
def handle_raw_packet(self, raw_packet):
        log.debug('got packet: %s', raw_packet)
        if rflink_log:
            print(raw_packet, file=rflink_log)
            rflink_log.flush()
        packet = None
        try:
            packet = decode_packet(raw_packet)
        except:
            log.exception('failed to parse packet: %s', packet)

        log.debug('decoded packet: %s', packet)

        if packet:
            if 'ok' in packet:
                # handle response packets internally
                log.debug('command response: %s', packet)
                self._last_ack = packet
                self._command_ack.set()
            else:
                self.handle_packet(packet)
        else:
            log.warning('no valid packet')

Response: Parse raw packet into dict.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def handle_packet(self, packet):
        if self.packet_callback:
            # forward to callback
            self.packet_callback(packet)
        else:
            print('packet', packet)

Response: Process incoming packet dict and optionally call callback.


Instruction:
How would you implement a function in Python 3 that
sends a command to rflink gateway.

Response:
def send_command(self, device_id, action):
        """Send device command to rflink gateway."""
        command = deserialize_packet_id(device_id)
        command['command'] = action
        log.debug('sending command: %s', command)
        self.send_packet(command)


Instruction:
Create a Python 3 function for
sending command to device and wait for gateway to repond with acknowledgement.

Response:
def send_command_ack(self, device_id, action):
        """Send command, wait for gateway to repond with acknowledgment."""
        # serialize commands
        yield from self._ready_to_send.acquire()
        acknowledgement = None
        try:
            self._command_ack.clear()
            self.send_command(device_id, action)

            log.debug('waiting for acknowledgement')
            try:
                yield from asyncio.wait_for(self._command_ack.wait(),
                                            TIMEOUT.seconds, loop=self.loop)
                log.debug('packet acknowledged')
            except concurrent.futures._base.TimeoutError:
                acknowledgement = {'ok': False, 'message': 'timeout'}
                log.warning('acknowledge timeout')
            else:
                acknowledgement = self._last_ack.get('ok', False)
        finally:
            # allow next command
            self._ready_to_send.release()

        return acknowledgement


Instruction:
Can you tell what is the following Python 3 function doing
def _handle_packet(self, packet):
        events = packet_events(packet)

        for event in events:
            if self.ignore_event(event['id']):
                log.debug('ignoring event with id: %s', event)
                continue
            log.debug('got event: %s', event)
            if self.event_callback:
                self.event_callback(event)
            else:
                self.handle_event(event)

Response: Handle a single packet and call event callback or ignore event if event_callback is set to True.


Instruction:
Make a summary of the following Python 3 code
def handle_event(self, event):
        string = '{id:<32} '
        if 'command' in event:
            string += '{command}'
        elif 'version' in event:
            if 'hardware' in event:
                string += '{hardware} {firmware} '
            string += 'V{version} R{revision}'
        else:
            string += '{value}'
            if event.get('unit'):
                string += ' {unit}'

        print(string.format(**event))

Response: Default handling of incoming event ( print ).


Instruction:
Explain what the following Python 3 code does
def ignore_event(self, event_id):
        for ignore in self.ignore:
            if (ignore == event_id or
                    (ignore.endswith('*') and event_id.startswith(ignore[:-1]))):
                return True
        return False

Response: Verify event id against list of events to ignore.


Instruction:
Here you have a function in Python 3, explain what it does
def handle_event(self, event):
        if event.get('command'):
            if event['command'] == 'on':
                cmd = 'off'
            else:
                cmd = 'on'

            task = self.send_command_ack(event['id'], cmd)
            self.loop.create_task(task)

Response: Handle incoming packet from rflink gateway.


Instruction:
Create a Python 3 function to
handle incoming packet from rflink gateway.

Response:
def handle_event(self, packet):
        """Handle incoming packet from rflink gateway."""
        if packet.get('command'):
            task = self.send_command_ack(packet['id'], packet['command'])
            self.loop.create_task(task)


Instruction:
How would you implement a function in Python 3 that
parses argument and setup main program loop.

Response:
def main(argv=sys.argv[1:], loop=None):
    """Parse argument and setup main program loop."""
    args = docopt(__doc__, argv=argv,
                  version=pkg_resources.require('rflink')[0].version)

    level = logging.ERROR
    if args['-v']:
        level = logging.INFO
    if args['-v'] == 2:
        level = logging.DEBUG
    logging.basicConfig(level=level)

    if not loop:
        loop = asyncio.get_event_loop()

    if args['--ignore']:
        ignore = args['--ignore'].split(',')
    else:
        ignore = []

    command = next((c for c in ALL_COMMANDS if args[c] is True), None)

    if command:
        protocol = PROTOCOLS['command']
    else:
        protocol = PROTOCOLS[args['-m']]

    conn = create_rflink_connection(
        protocol=protocol,
        host=args['--host'],
        port=args['--port'],
        baud=args['--baud'],
        loop=loop,
        ignore=ignore,
    )

    transport, protocol = loop.run_until_complete(conn)

    try:
        if command:
            for _ in range(int(args['--repeat'])):
                loop.run_until_complete(
                    protocol.send_command_ack(
                        args['<id>'], command))
        else:
            loop.run_forever()
    except KeyboardInterrupt:
        # cleanup connection
        transport.close()
        loop.run_forever()
    finally:
        loop.close()


Instruction:
Write a Python 3 function that can
create a random initial population of floating point values.

Response:
def _initial_population_gsa(population_size, solution_size, lower_bounds,
                            upper_bounds):
    """Create a random initial population of floating point values.

    Args:
        population_size: an integer representing the number of solutions in the population.
        problem_size: the number of values in each solution.
        lower_bounds: a list, each value is a lower bound for the corresponding
                      part of the solution.
        upper_bounds: a list, each value is a upper bound for the corresponding
                      part of the solution.

    Returns:
        list; A list of random solutions.
    """
    if len(lower_bounds) != solution_size or len(upper_bounds) != solution_size:
        raise ValueError(
            "Lower and upper bounds much have a length equal to the problem size."
        )

    return common.make_population(population_size, common.random_real_solution,
                                  solution_size, lower_bounds, upper_bounds)


Instruction:
Make a summary of the following Python 3 code
def _new_population_gsa(population, fitnesses, velocities, lower_bounds,
                        upper_bounds, grav_initial, grav_reduction_rate,
                        iteration, max_iterations):
    """Generate a new population as given by GSA algorithm.

    In GSA paper, grav_initial is G_i
    """
    # Update the gravitational constant, and the best and worst of the population
    # Calculate the mass and acceleration for each solution
    # Update the velocity and position of each solution
    population_size = len(population)
    solution_size = len(population[0])

    # In GSA paper, grav is G
    grav = _next_grav_gsa(grav_initial, grav_reduction_rate, iteration,
                          max_iterations)
    masses = _get_masses(fitnesses)

    # Create bundled solution with position and mass for the K best calculation
    # Also store index to later check if two solutions are the same
    # Sorted by solution fitness (mass)
    solutions = [{
        'pos': pos,
        'mass': mass,
        'index': i
    } for i, (pos, mass) in enumerate(zip(population, masses))]
    solutions.sort(key=lambda x: x['mass'], reverse=True)

    # Get the force on each solution
    # Only the best K solutions apply force
    # K linearly decreases to 1
    num_best = int(population_size - (population_size - 1) *
                   (iteration / float(max_iterations)))
    forces = []
    for i in range(population_size):
        force_vectors = []
        for j in range(num_best):
            # If it is not the same solution
            if i != solutions[j]['index']:
                force_vectors.append(
                    _gsa_force(grav, masses[i], solutions[j]['mass'],
                               population[i], solutions[j]['pos']))
        forces.append(_gsa_total_force(force_vectors, solution_size))

    # Get the acceleration of each solution
    accelerations = []
    for i in range(population_size):
        accelerations.append(_gsa_acceleration(forces[i], masses[i]))

    # Update the velocity of each solution
    new_velocities = []
    for i in range(population_size):
        new_velocities.append(
            _gsa_update_velocity(velocities[i], accelerations[i]))

    # Create the new population
    new_population = []
    for i in range(population_size):
        new_position = _gsa_update_position(population[i], new_velocities[i])
        # Constrain to bounds
        new_position = list(
            numpy.clip(new_position, lower_bounds, upper_bounds))

        new_population.append(new_position)

    return new_population, new_velocities

Response: Generate a new population as given by GSA algorithm.


Instruction:
Given the following Python 3 function, write the documentation
def _next_grav_gsa(grav_initial, grav_reduction_rate, iteration,
                   max_iterations):
    """Calculate G as given by GSA algorithm.

    In GSA paper, grav is G
    """
    return grav_initial * math.exp(
        -grav_reduction_rate * iteration / float(max_iterations))

Response: Calculate G as given by GSA algorithm.


Instruction:
Can you tell what is the following Python 3 function doing
def _get_masses(fitnesses):
    # Obtain constants
    best_fitness = max(fitnesses)
    worst_fitness = min(fitnesses)
    fitness_range = best_fitness - worst_fitness

    # Calculate raw masses for each solution
    raw_masses = []
    for fitness in fitnesses:
        # Epsilon is added to prevent divide by zero errors
        raw_masses.append((fitness - worst_fitness) / (fitness_range + EPSILON)
                          + EPSILON)

    # Normalize to obtain final mass for each solution
    total_mass = sum(raw_masses)
    masses = []
    for mass in raw_masses:
        masses.append(mass / total_mass)

    return masses

Response: Convert fitnesses into masses as given by GSA algorithm.


Instruction:
Can you create a Python 3 function that
gives the force of solution i on solution j on solution i.

Response:
def _gsa_force(grav, mass_i, mass_j, position_i, position_j):
    """Gives the force of solution j on solution i.

    Variable name in GSA paper given in ()

    args:
        grav: The gravitational constant. (G)
        mass_i: The mass of solution i (derived from fitness). (M_i)
        mass_j: The mass of solution j (derived from fitness). (M_j)
        position_i: The position of solution i. (x_i)
        position_j: The position of solution j. (x_j)

    returns:
        numpy.array; The force vector of solution j on solution i.
    """

    position_diff = numpy.subtract(position_j, position_i)
    distance = numpy.linalg.norm(position_diff)

    # The first 3 terms give the magnitude of the force
    # The last term is a vector that provides the direction
    # Epsilon prevents divide by zero errors
    return grav * (mass_i * mass_j) / (distance + EPSILON) * position_diff


Instruction:
Here you have a function in Python 3, explain what it does
def _gsa_total_force(force_vectors, vector_length):
    if len(force_vectors) == 0:
        return [0.0] * vector_length
    # The GSA algorithm specifies that the total force in each dimension
    # is a random sum of the individual forces in that dimension.
    # For this reason we sum the dimensions individually instead of simply
    # using vec_a+vec_b
    total_force = [0.0] * vector_length
    for force_vec in force_vectors:
        for i in range(vector_length):
            total_force[i] += random.uniform(0.0, 1.0) * force_vec[i]
    return total_force

Response: Returns a randomly weighted sum of the force vectors.


Instruction:
Make a summary of the following Python 3 code
def _new_population_genalg(population,
                           fitnesses,
                           mutation_chance=0.02,
                           crossover_chance=0.7,
                           selection_function=gaoperators.tournament_selection,
                           crossover_function=gaoperators.one_point_crossover):
    """Perform all genetic algorithm operations on a population, and return a new population.

    population must have an even number of chromosomes.

    Args:
        population: A list of binary lists, ex. [[0,1,1,0], [1,0,1,0]]
        fitness: A list of fitnesses that correspond with chromosomes in the population,
                 ex. [1.2, 10.8]
        mutation_chance: the chance that a bit will be flipped during mutation
        crossover_chance: the chance that two parents will be crossed during crossover
        selection_function: A function that will select parents for crossover and mutation
        crossover_function: A function that will cross two parents

    Returns:
        list; A new population of chromosomes, that should be more fit.
    """
    # Selection
    # Create the population of parents that will be crossed and mutated.
    intermediate_population = selection_function(population, fitnesses)

    # Crossover
    new_population = _crossover(intermediate_population, crossover_chance,
                                crossover_function)

    # Mutation
    # Mutates chromosomes in place
    gaoperators.random_flip_mutate(new_population, mutation_chance)

    # Return new population
    return new_population

Response: Perform all genetic algorithm operations on a population and return a new population.


Instruction:
How would you explain what the following Python 3 function does
def _crossover(population, crossover_chance, crossover_operator):
    new_population = []
    for i in range(0, len(population), 2):  # For every other index
        # Take parents from every set of 2 in the population
        # Wrap index if out of range
        try:
            parents = (population[i], population[i + 1])
        except IndexError:
            parents = (population[i], population[0])

        # If crossover takes place
        if random.uniform(0.0, 1.0) <= crossover_chance:
            # Add children to the new population
            new_population.extend(crossover_operator(parents))
        else:
            new_population.extend(parents)

    return new_population

Response: Perform crossover on a population.


Instruction:
Implement a function in Python 3 to
make a list of random real numbers between lower and upper bounds.

Response:
def random_real_solution(solution_size, lower_bounds, upper_bounds):
    """Make a list of random real numbers between lower and upper bounds."""
    return [
        random.uniform(lower_bounds[i], upper_bounds[i])
        for i in range(solution_size)
    ]


Instruction:
Write a Python 3 script to
make a population with the supplied generator.

Response:
def make_population(population_size, solution_generator, *args, **kwargs):
    """Make a population with the supplied generator."""
    return [
        solution_generator(*args, **kwargs) for _ in range(population_size)
    ]


Instruction:
Can you write a function in Python 3 where it
creates a list of parents with tournament winners.

Response:
def tournament_selection(population,
                         fitnesses,
                         num_competitors=2,
                         diversity_weight=0.0):
    """Create a list of parents with tournament selection.

    Args:
        population: A list of solutions.
        fitnesses: A list of fitness values corresponding to solutions in population.
        num_competitors: Number of solutions to compare every round.
            Best solution among competitors is selected.
        diversity_weight: Weight of diversity metric.
            Determines how frequently diversity is used to select tournament winners.
            Note that fitness is given a weight of 1.0.
            diversity_weight == 1.0 gives equal weight to diversity and fitness.
    """
    # Optimization if diversity factor is disabled
    if diversity_weight <= 0.0:
        fitness_pop = zip(fitnesses,
                          population)  # Zip for easy fitness comparison

        # Get num_competitors random chromosomes, then add best to result,
        # by taking max fitness and getting chromosome from tuple.
        # Repeat until full.
        return [
            max(random.sample(fitness_pop, num_competitors))[1]
            for _ in range(len(population))
        ]
    else:
        indices = range(len(population))

        # Select tournament winners by either max fitness or diversity.
        # The metric to check is randomly selected, weighted by diversity_weight.
        # diversity_metric is calculated between the given solution,
        # and the list of all currently selected solutions.
        selected_solutions = []
        # Select as many solutions are there are in population
        for _ in range(len(population)):
            competitor_indices = random.sample(indices, num_competitors)

            # Select by either fitness or diversity,
            # Selected by weighted random selection
            # NOTE: We assume fitness has a weight of 1.0
            if random.uniform(0.0, 1.0) < (1.0 / (1.0 + diversity_weight)):
                # Fitness
                selected_solutions.append(
                    max(
                        zip([fitnesses[i] for i in competitor_indices],
                            [population[i] for i in competitor_indices]))[-1])
            else:
                # Diversity
                # Break ties by fitness
                selected_solutions.append(
                    max(
                        zip([
                            _diversity_metric(population[i], selected_solutions
                                              ) for i in competitor_indices
                        ], [fitnesses[i] for i in competitor_indices],
                            [population[i] for i in competitor_indices]))[-1])

        return selected_solutions


Instruction:
Make a summary of the following Python 3 code
def stochastic_selection(population, fitnesses):
    pop_size = len(population)
    probabilities = _fitnesses_to_probabilities(fitnesses)

    # Create selection list (for stochastic universal sampling)
    selection_list = []
    selection_spacing = 1.0 / pop_size
    selection_start = random.uniform(0.0, selection_spacing)
    for i in range(pop_size):
        selection_list.append(selection_start + selection_spacing * i)

    # Select intermediate population according to selection list
    intermediate_population = []
    for selection in selection_list:
        for (i, probability) in enumerate(probabilities):
            if probability >= selection:
                intermediate_population.append(population[i])
                break
    random.shuffle(intermediate_population)

    return intermediate_population

Response: Create a list of parents with stochastic universal sampling.


Instruction:
Write a Python 3 script to
create a list of parents with roulette selection.

Response:
def roulette_selection(population, fitnesses):
    """Create a list of parents with roulette selection."""
    probabilities = _fitnesses_to_probabilities(fitnesses)

    intermediate_population = []
    for _ in range(len(population)):
        # Choose a random individual
        selection = random.uniform(0.0, 1.0)
        # Iterate over probabilities list
        for i, probability in enumerate(probabilities):
            if probability >= selection:  # First probability that is greater
                intermediate_population.append(population[i])
                break

    return intermediate_population


Instruction:
How would you explain what the following Python 3 function does
def _rescale(vector):
    # Subtract min, making smallest value 0
    min_val = min(vector)
    vector = [v - min_val for v in vector]

    # Divide by max, making largest value 1
    max_val = float(max(vector))
    try:
        return [v / max_val for v in vector]
    except ZeroDivisionError:  # All values are the same
        return [1.0] * len(vector)

Response: Scale values in vector to the range [ 0 1 )


Instruction:
How would you explain what the following Python 3 function does
def _diversity_metric(solution, population):
    # Edge case for empty population
    # If there are no other solutions, the given solution has maximum diversity
    if population == []:
        return 1.0

    return (
        sum([_manhattan_distance(solution, other) for other in population])
        # Normalize (assuming each value in solution is in range [0.0, 1.0])
        # NOTE: len(solution) is maximum manhattan distance
        / (len(population) * len(solution)))

Response: Return diversity value for solution compared to given population.


Instruction:
Explain what the following Python 3 code does
def _manhattan_distance(vec_a, vec_b):
    if len(vec_a) != len(vec_b):
        raise ValueError('len(vec_a) must equal len(vec_b)')
    return sum(map(lambda a, b: abs(a - b), vec_a, vec_b))

Response: Return manhattan distance between two lists of numbers.


Instruction:
Explain what the following Python 3 code does
def _fitnesses_to_probabilities(fitnesses):
    # Do not allow negative fitness values
    min_fitness = min(fitnesses)
    if min_fitness < 0.0:
        # Make smallest fitness value 0
        fitnesses = map(lambda f: f - min_fitness, fitnesses)

    fitness_sum = sum(fitnesses)

    # Generate probabilities
    # Creates a list of increasing values.
    # The greater the gap between two values, the greater the probability.
    # Ex. [0.1, 0.23, 0.56, 1.0]
    prob_sum = 0.0
    probabilities = []
    for fitness in fitnesses:
        if fitness < 0:
            raise ValueError(
                "Fitness cannot be negative, fitness = {}.".format(fitness))
        prob_sum += (fitness / fitness_sum)
        probabilities.append(prob_sum)
    probabilities[-1] += 0.0001  # to compensate for rounding errors

    return probabilities

Response: Return a list of probabilities proportional to fitnesses.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def one_point_crossover(parents):
    # The point that the chromosomes will be crossed at (see Ex. above)
    crossover_point = random.randint(1, len(parents[0]) - 1)

    return (_one_parent_crossover(parents[0], parents[1], crossover_point),
            _one_parent_crossover(parents[1], parents[0], crossover_point))

Response: Perform one point crossover on two parent chromosomes.


Instruction:
Create a Python 3 function to
perform uniform crossover on two parent chromosomes.

Response:
def uniform_crossover(parents):
    """Perform uniform crossover on two parent chromosomes.

    Randomly take genes from one parent or the other.
    Ex. p1 = xxxxx, p2 = yyyyy, child = xyxxy
    """
    chromosome_length = len(parents[0])

    children = [[], []]

    for i in range(chromosome_length):
        selected_parent = random.randint(0, 1)

        # Take from the selected parent, and add it to child 1
        # Take from the other parent, and add it to child 2
        children[0].append(parents[selected_parent][i])
        children[1].append(parents[1 - selected_parent][i])

    return children


Instruction:
Can you implement a function in Python 3 that
mutates every chromosome in a population by randomly flipping bits.

Response:
def random_flip_mutate(population, mutation_chance):
    """Mutate every chromosome in a population, list is modified in place.

    Mutation occurs by randomly flipping bits (genes).
    """
    for chromosome in population:  # For every chromosome in the population
        for i in range(len(chromosome)):  # For every bit in the chromosome
            # If mutation takes place
            if random.uniform(0.0, 1.0) <= mutation_chance:
                chromosome[i] = 1 - chromosome[i]


Instruction:
Write a Python 3 script to
return dict mapping item to indices.

Response:
def _duplicates(list_):
    """Return dict mapping item -> indices."""
    item_indices = {}
    for i, item in enumerate(list_):
        try:
            item_indices[item].append(i)
        except KeyError:  # First time seen
            item_indices[item] = [i]
    return item_indices


Instruction:
Given the following Python 3 function, write the documentation
def _parse_parameter_locks(optimizer, meta_parameters, parameter_locks):
    # WARNING: meta_parameters is modified inline

    locked_values = {}
    if parameter_locks:
        for name in parameter_locks:
            # Store the current optimzier value
            # and remove from our dictionary of paramaters to optimize
            locked_values[name] = getattr(optimizer, name)
            meta_parameters.pop(name)

    return locked_values

Response: Synchronize meta_parameters and locked_values.


Instruction:
Write a Python 3 script to
determine size of binary encoding of parameters.

Response:
def _get_hyperparameter_solution_size(meta_parameters):
    """Determine size of binary encoding of parameters.

    Also adds binary size information for each parameter.
    """
    # WARNING: meta_parameters is modified inline

    solution_size = 0
    for _, parameters in meta_parameters.iteritems():
        if parameters['type'] == 'discrete':
            # Binary encoding of discrete values -> log_2 N
            num_values = len(parameters['values'])
            binary_size = helpers.binary_size(num_values)
        elif parameters['type'] == 'int':
            # Use enough bits to cover range from min to max
            # + 1 to include max in range
            int_range = parameters['max'] - parameters['min'] + 1
            binary_size = helpers.binary_size(int_range)
        elif parameters['type'] == 'float':
            # Use enough bits to provide fine steps between min and max
            float_range = parameters['max'] - parameters['min']
            # * 1000 provides 1000 values between each natural number
            binary_size = helpers.binary_size(float_range * 1000)
        else:
            raise ValueError('Parameter type "{}" does not match known values'.
                             format(parameters['type']))

        # Store binary size with parameters for use in decode function
        parameters['binary_size'] = binary_size

        solution_size += binary_size

    return solution_size