Instruction:
How would you explain what the following Python 3 function does
def reflect(self, name):
        # Return whatever the most local scope defines this as, or bubble all
        # the way to the top.
        result = None
        for scope in reversed(self.scopes):
            try:
                if isinstance(scope, type):
                    result = structured.reflect_static_member(scope, name)
                else:
                    result = structured.reflect_runtime_member(scope, name)

                if result is not None:
                    return result

            except (NotImplementedError, KeyError, AttributeError):
                continue

        return protocol.AnyType

Response: Reflect name starting with local scope all the way up to global.


Instruction:
Write a Python 3 script to
reflect name using ONLY runtime reflection.

Response:
def reflect_runtime_member(self, name):
        """Reflect 'name' using ONLY runtime reflection.

        You most likely want to use ScopeStack.reflect instead.

        Returns:
            Type of 'name', or protocol.AnyType.
        """
        for scope in reversed(self.scopes):
            try:
                return structured.reflect_runtime_member(scope, name)
            except (NotImplementedError, KeyError, AttributeError):
                continue

        return protocol.AnyType


Instruction:
Write a Python 3 function that can
reflect name using ONLY static reflection.

Response:
def reflect_static_member(cls, name):
        """Reflect 'name' using ONLY static reflection.

        You most likely want to use ScopeStack.reflect instead.

        Returns:
            Type of 'name', or protocol.AnyType.
        """
        for scope in reversed(cls.scopes):
            try:
                return structured.reflect_static_member(scope, name)
            except (NotImplementedError, KeyError, AttributeError):
                continue

        return protocol.AnyType


Instruction:
Make a summary of the following Python 3 code
def get_hostmap(profile):
    '''
    We abuse the profile combination to also derive a pilot-host map, which
    will tell us on what exact host each pilot has been running.  To do so, we
    check for the PMGR_ACTIVE advance event in agent_0.prof, and use the NTP
    sync info to associate a hostname.
    '''
    # FIXME: This should be replaced by proper hostname logging
    #        in `pilot.resource_details`.

    hostmap = dict()  # map pilot IDs to host names
    for entry in profile:
        if entry[ru.EVENT] == 'hostname':
            hostmap[entry[ru.UID]] = entry[ru.MSG]

    return hostmap

Response: This function returns a dict that associates pilot IDs to host names.


Instruction:
How would you explain what the following Python 3 function does
def run(self):

        try:

            # Set None objects local to each run
            self._wfp = None
            self._sync_thread = None
            self._terminate_sync = Event()
            self._resubmit_failed = False
            self._cur_attempt = 1

            if not self._workflow:
                self._logger.error('No workflow assigned currently, please check your script')
                raise MissingError(obj=self._uid, missing_attribute='workflow')

            if not self._resource_manager:
                self._logger.error('No resource manager assigned currently, please create and add a valid resource manager')
                raise MissingError(obj=self._uid, missing_attribute='resource_manager')

            self._prof.prof('amgr run started', uid=self._uid)

            # Setup rabbitmq stuff
            if not self._mqs_setup:

                self._report.info('Setting up RabbitMQ system')
                setup = self._setup_mqs()

                if not setup:
                    self._logger.error('RabbitMQ system not available')
                    raise EnTKError("RabbitMQ setup failed")

                self._mqs_setup = True

                self._report.ok('>>ok\n')

            # Create WFProcessor object
            self._prof.prof('creating wfp obj', uid=self._uid)
            self._wfp = WFprocessor(sid=self._sid,
                                    workflow=self._workflow,
                                    pending_queue=self._pending_queue,
                                    completed_queue=self._completed_queue,
                                    mq_hostname=self._mq_hostname,
                                    port=self._port,
                                    resubmit_failed=self._resubmit_failed)
            self._wfp._initialize_workflow()
            self._workflow = self._wfp.workflow

            # Submit resource request if not resource allocation done till now or
            # resubmit a new one if the old one has completed
            if self._resource_manager:
                res_alloc_state = self._resource_manager.get_resource_allocation_state()
                if (not res_alloc_state) or (res_alloc_state in self._resource_manager.get_completed_states()):

                    self._logger.info('Starting resource request submission')
                    self._prof.prof('init rreq submission', uid=self._uid)
                    self._resource_manager._submit_resource_request()
                    res_alloc_state = self._resource_manager.get_resource_allocation_state()
                    if res_alloc_state in self._resource_manager.get_completed_states():
                        raise EnTKError(msg="Cannot proceed. Resource allocation ended up in %s"%res_alloc_state)

            else:

                self._logger.exception('Cannot run without resource manager, please create and assign a resource manager')
                raise EnTKError(text='Missing resource manager')

            # Start synchronizer thread
            if not self._sync_thread:
                self._logger.info('Starting synchronizer thread')
                self._sync_thread = Thread(target=self._synchronizer, name='synchronizer-thread')
                self._prof.prof('starting synchronizer thread', uid=self._uid)
                self._sync_thread.start()

            # Start WFprocessor
            self._logger.info('Starting WFProcessor process from AppManager')
            self._wfp.start_processor()

            self._report.ok('All components created\n')

            # Create tmgr object only if it does not already exist
            if self._rts == 'radical.pilot':
                from radical.entk.execman.rp import TaskManager
            elif self._rts == 'mock':
                from radical.entk.execman.mock import TaskManager

            if not self._task_manager:
                self._prof.prof('creating tmgr obj', uid=self._uid)
                self._task_manager = TaskManager(sid=self._sid,
                                                 pending_queue=self._pending_queue,
                                                 completed_queue=self._completed_queue,
                                                 mq_hostname=self._mq_hostname,
                                                 rmgr=self._resource_manager,
                                                 port=self._port
                                                 )
                self._logger.info('Starting task manager process from AppManager')
                self._task_manager.start_manager()
                self._task_manager.start_heartbeat()

            active_pipe_count = len(self._workflow)
            finished_pipe_uids = []

            # We wait till all pipelines of the workflow are marked
            # complete
            while ((active_pipe_count > 0) and
                    (self._wfp.workflow_incomplete()) and
                    (self._resource_manager.get_resource_allocation_state() not
                     in self._resource_manager.get_completed_states())):

                if active_pipe_count > 0:

                    for pipe in self._workflow:

                        with pipe.lock:

                            if (pipe.completed) and (pipe.uid not in finished_pipe_uids):

                                self._logger.info('Pipe %s completed' % pipe.uid)
                                finished_pipe_uids.append(pipe.uid)
                                active_pipe_count -= 1
                                self._logger.info('Active pipes: %s' % active_pipe_count)

                if (not self._sync_thread.is_alive()) and (self._cur_attempt <= self._reattempts):

                    self._sync_thread = Thread(target=self._synchronizer,
                                               name='synchronizer-thread')
                    self._logger.info('Restarting synchronizer thread')
                    self._prof.prof('restarting synchronizer', uid=self._uid)
                    self._sync_thread.start()

                    self._cur_attempt += 1

                if (not self._wfp.check_processor()) and (self._cur_attempt <= self._reattempts):

                    """
                    If WFP dies, both child threads are also cleaned out.
                    We simply recreate the wfp object with a copy of the workflow
                    in the appmanager and start the processor.
                    """

                    self._prof.prof('recreating wfp obj', uid=self._uid)
                    self._wfp = WFProcessor(
                        sid=self._sid,
                        workflow=self._workflow,
                        pending_queue=self._pending_queue,
                        completed_queue=self._completed_queue,
                        mq_hostname=self._mq_hostname,
                        port=self._port,
                        resubmit_failed=self._resubmit_failed)

                    self._logger.info('Restarting WFProcessor process from AppManager')
                    self._wfp.start_processor()

                    self._cur_attempt += 1

                if (not self._task_manager.check_heartbeat()) and (self._cur_attempt <= self._reattempts):

                    """
                    If the tmgr process or heartbeat dies, we simply start a
                    new process using the start_manager method. We do not
                    need to create a new instance of the TaskManager object
                    itself. We stop and start a new instance of the
                    heartbeat thread as well.
                    """
                    self._prof.prof('restarting tmgr process and heartbeat', uid=self._uid)

                    self._logger.info('Terminating heartbeat thread')
                    self._task_manager.terminate_heartbeat()
                    self._logger.info('Terminating tmgr process')
                    self._task_manager.terminate_manager()
                    self._logger.info('Restarting task manager process')
                    self._task_manager.start_manager()
                    self._logger.info('Restarting heartbeat thread')
                    self._task_manager.start_heartbeat()

                    self._cur_attempt += 1

            self._prof.prof('start termination', uid=self._uid)

            # Terminate threads in following order: wfp, helper, synchronizer
            self._logger.info('Terminating WFprocessor')
            self._wfp.terminate_processor()

            self._logger.info('Terminating synchronizer thread')
            self._terminate_sync.set()
            self._sync_thread.join()
            self._logger.info('Synchronizer thread terminated')

            if self._autoterminate:
                self.resource_terminate()

            if self._write_workflow:
                write_workflow(self._workflow, self._sid)

            self._prof.prof('termination done', uid=self._uid)

        except KeyboardInterrupt:

            self._prof.prof('start termination', uid=self._uid)

            self._logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to cancel enqueuer thread gracefully...')

            # Terminate threads in following order: wfp, helper, synchronizer
            if self._wfp:
                self._logger.info('Terminating WFprocessor')
                self._wfp.terminate_processor()

            if self._task_manager:
                self._logger.info('Terminating task manager process')
                self._task_manager.terminate_manager()
                self._task_manager.terminate_heartbeat()

            if self._sync_thread:
                self._logger.info('Terminating synchronizer thread')
                self._terminate_sync.set()
                self._sync_thread.join()
                self._logger.info('Synchronizer thread terminated')

            if self._resource_manager:
                self._resource_manager._terminate_resource_request()

            self._prof.prof('termination done', uid=self._uid)

            raise KeyboardInterrupt

        except Exception, ex:

            self._prof.prof('start termination', uid=self._uid)

            self._logger.exception('Error in AppManager: %s' % ex)

            # Terminate threads in following order: wfp, helper, synchronizer
            if self._wfp:
                self._logger.info('Terminating WFprocessor')
                self._wfp.terminate_processor()

            if self._task_manager:
                self._logger.info('Terminating task manager process')
                self._task_manager.terminate_manager()
                self._task_manager.terminate_heartbeat()

            if self._sync_thread:
                self._logger.info('Terminating synchronizer thread')
                self._terminate_sync.set()
                self._sync_thread.join()
                self._logger.info('Synchronizer thread terminated')

            if self._resource_manager:
                self._resource_manager._terminate_resource_request()

            self._prof.prof('termination done', uid=self._uid)
            raise

Response: This method is executed by the application manager.


Instruction:
Make a summary of the following Python 3 code
def _synchronizer(self):

        try:

            self._prof.prof('synchronizer started', uid=self._uid)

            self._logger.info('synchronizer thread started')

            def task_update(msg, reply_to, corr_id, mq_channel):

                completed_task = Task()
                completed_task.from_dict(msg['object'])
                self._logger.info('Received %s with state %s' % (completed_task.uid, completed_task.state))

                found_task = False

                # Traverse the entire workflow to find the correct task
                for pipe in self._workflow:

                    if not pipe.completed:
                        if completed_task.parent_pipeline['uid'] == pipe.uid:

                            for stage in pipe.stages:

                                if completed_task.parent_stage['uid'] == stage.uid:

                                    for task in stage.tasks:

                                        if (completed_task.uid == task.uid)and(completed_task.state != task.state):

                                            task.state = str(completed_task.state)
                                            self._logger.debug('Found task %s with state %s' %
                                                               (task.uid, task.state))

                                            if completed_task.path:
                                                task.path = str(completed_task.path)

                                            mq_channel.basic_publish(exchange='',
                                                                     routing_key=reply_to,
                                                                     properties=pika.BasicProperties(
                                                                         correlation_id=corr_id),
                                                                     body='%s-ack' % task.uid)

                                            self._prof.prof('publishing sync ack for obj with state %s' %
                                                            msg['object']['state'],
                                                            uid=msg['object']['uid']
                                                            )

                                            mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)
                                            self._report.ok('Update: ')
                                            self._report.info('%s state: %s\n' % (task.luid, task.state))

                                            found_task = True

                                    if not found_task:

                                        # If there was a Stage update, but the Stage was not found in any of the Pipelines. This
                                        # means that this was a Stage that was added during runtime and the AppManager does not
                                        # know about it. The current solution is going to be: add it to the workflow object in the
                                        # AppManager via the synchronizer.

                                        self._prof.prof('Adap: adding new task')

                                        self._logger.info('Adding new task %s to parent stage: %s' % (completed_task.uid,
                                                                                stage.uid))

                                        stage.add_tasks(completed_task)
                                        mq_channel.basic_publish(exchange='',
                                                                 routing_key=reply_to,
                                                                 properties=pika.BasicProperties(
                                                                     correlation_id=corr_id),
                                                                 body='%s-ack' % completed_task.uid)

                                        self._prof.prof('Adap: added new task')

                                        self._prof.prof('publishing sync ack for obj with state %s' %
                                                        msg['object']['state'],
                                                        uid=msg['object']['uid']
                                                        )

                                        mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)
                                        self._report.ok('Update: ')
                                        self._report.info('%s state: %s\n' %
                                                          (completed_task.luid, completed_task.state))

            def stage_update(msg, reply_to, corr_id, mq_channel):

                completed_stage = Stage()
                completed_stage.from_dict(msg['object'])
                self._logger.info('Received %s with state %s' % (completed_stage.uid, completed_stage.state))

                found_stage = False

                # Traverse the entire workflow to find the correct stage
                for pipe in self._workflow:

                    if not pipe.completed:

                        if completed_stage.parent_pipeline['uid'] == pipe.uid:
                            self._logger.info('Found parent pipeline: %s' % pipe.uid)

                            for stage in pipe.stages:

                                if (completed_stage.uid == stage.uid)and(completed_stage.state != stage.state):

                                    self._logger.debug('Found stage %s' % stage.uid)

                                    stage.state = str(completed_stage.state)

                                    mq_channel.basic_publish(exchange='',
                                                             routing_key=reply_to,
                                                             properties=pika.BasicProperties(
                                                                 correlation_id=corr_id),
                                                             body='%s-ack' % stage.uid)

                                    self._prof.prof('publishing sync ack for obj with state %s' %
                                                    msg['object']['state'],
                                                    uid=msg['object']['uid']
                                                    )

                                    mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)
                                    self._report.ok('Update: ')
                                    self._report.info('%s state: %s\n' % (stage.luid, stage.state))

                                    found_stage = True

                            if not found_stage:

                                # If there was a Stage update, but the Stage was not found in any of the Pipelines. This
                                # means that this was a Stage that was added during runtime and the AppManager does not
                                # know about it. The current solution is going to be: add it to the workflow object in the
                                # AppManager via the synchronizer.

                                self._prof.prof('Adap: adding new stage', uid=self._uid)

                                self._logger.info('Adding new stage %s to parent pipeline: %s' % (completed_stage.uid,
                                                                                pipe.uid))

                                pipe.add_stages(completed_stage)
                                mq_channel.basic_publish(exchange='',
                                                         routing_key=reply_to,
                                                         properties=pika.BasicProperties(
                                                             correlation_id=corr_id),
                                                         body='%s-ack' % completed_stage.uid)

                                self._prof.prof('Adap: adding new stage', uid=self._uid)

                                self._prof.prof('publishing sync ack for obj with state %s' %
                                                msg['object']['state'],
                                                uid=msg['object']['uid']
                                                )

                                mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)

            def pipeline_update(msg, reply_to, corr_id, mq_channel):

                completed_pipeline = Pipeline()
                completed_pipeline.from_dict(msg['object'])

                self._logger.info('Received %s with state %s' % (completed_pipeline.uid, completed_pipeline.state))

                # Traverse the entire workflow to find the correct pipeline
                for pipe in self._workflow:

                    if not pipe.completed:

                        if (completed_pipeline.uid == pipe.uid)and(completed_pipeline.state != pipe.state):

                            pipe.state = str(completed_pipeline.state)

                            self._logger.info('Found pipeline %s, state %s, completed %s' % (pipe.uid,
                                                                                pipe.state,
                                                                                pipe.completed)
                                              )

                            # Reply with ack msg to the sender
                            mq_channel.basic_publish(exchange='',
                                                     routing_key=reply_to,
                                                     properties=pika.BasicProperties(
                                                         correlation_id=corr_id),
                                                     body='%s-ack' % pipe.uid)

                            self._prof.prof('publishing sync ack for obj with state %s' %
                                            msg['object']['state'],
                                            uid=msg['object']['uid']
                                            )

                            mq_channel.basic_ack(delivery_tag=method_frame.delivery_tag)

                            # Keep the assignment of the completed flag after sending the acknowledgment
                            # back. Otherwise the MainThread takes lock over the pipeline because of logging
                            # and profiling
                            if completed_pipeline.completed:
                                pipe._completed_flag.set()
                            self._report.ok('Update: ')
                            self._report.info('%s state: %s\n' % (pipe.luid, pipe.state))

            mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._mq_hostname, port=self._port))
            mq_channel = mq_connection.channel()

            last = time.time()

            while not self._terminate_sync.is_set():

                #-------------------------------------------------------------------------------------------------------
                # Messages between tmgr Main thread and synchronizer -- only Task objects

                method_frame, props, body = mq_channel.basic_get(queue='%s-tmgr-to-sync' % self._sid)

                """
                The message received is a JSON object with the following structure:

                msg = {
                        'type': 'Pipeline'/'Stage'/'Task',
                        'object': json/dict
                        }
                """

                if body:

                    msg = json.loads(body)

                    self._prof.prof('received obj with state %s for sync' %
                                    msg['object']['state'], uid=msg['object']['uid'])

                    self._logger.debug('received %s with state %s for sync' %
                                       (msg['object']['uid'], msg['object']['state']))

                    if msg['type'] == 'Task':
                        task_update(msg, '%s-sync-to-tmgr' % self._sid, props.correlation_id, mq_channel)

                #-------------------------------------------------------------------------------------------------------

                #-------------------------------------------------------------------------------------------------------
                # Messages between callback thread and synchronizer -- only Task objects

                method_frame, props, body = mq_channel.basic_get(queue='%s-cb-to-sync' % self._sid)

                """
                The message received is a JSON object with the following structure:

                msg = {
                        'type': 'Pipeline'/'Stage'/'Task',
                        'object': json/dict
                        }
                """

                if body:

                    msg = json.loads(body)

                    self._prof.prof('received obj with state %s for sync' %
                                    msg['object']['state'], uid=msg['object']['uid'])

                    self._logger.debug('received %s with state %s for sync' %
                                       (msg['object']['uid'], msg['object']['state']))

                    if msg['type'] == 'Task':
                        task_update(msg, '%s-sync-to-cb' % self._sid, props.correlation_id, mq_channel)

                #-------------------------------------------------------------------------------------------------------

                #-------------------------------------------------------------------------------------------------------
                # Messages between enqueue thread and synchronizer -- Task, Stage or Pipeline
                method_frame, props, body = mq_channel.basic_get(queue='%s-enq-to-sync' % self._sid)

                if body:

                    msg = json.loads(body)

                    self._prof.prof('received obj with state %s for sync' %
                                    msg['object']['state'], uid=msg['object']['uid'])

                    self._logger.debug('received %s with state %s for sync' %
                                       (msg['object']['uid'], msg['object']['state']))

                    if msg['type'] == 'Task':
                        task_update(msg, '%s-sync-to-enq' % self._sid, props.correlation_id, mq_channel)

                    elif msg['type'] == 'Stage':
                        stage_update(msg, '%s-sync-to-enq' % self._sid, props.correlation_id, mq_channel)

                    elif msg['type'] == 'Pipeline':
                        pipeline_update(msg, '%s-sync-to-enq' % self._sid, props.correlation_id, mq_channel)
                #-------------------------------------------------------------------------------------------------------

                #-------------------------------------------------------------------------------------------------------
                # Messages between dequeue thread and synchronizer -- Task, Stage or Pipeline
                method_frame, props, body = mq_channel.basic_get(queue='%s-deq-to-sync' % self._sid)

                if body:

                    msg = json.loads(body)

                    self._prof.prof('received obj with state %s for sync' %
                                    msg['object']['state'], uid=msg['object']['uid'])

                    self._logger.debug('received %s with state %s for sync' %
                                       (msg['object']['uid'], msg['object']['state']))

                    if msg['type'] == 'Task':
                        task_update(msg, '%s-sync-to-deq' % self._sid, props.correlation_id, mq_channel)

                    elif msg['type'] == 'Stage':
                        stage_update(msg, '%s-sync-to-deq' % self._sid, props.correlation_id, mq_channel)

                    elif msg['type'] == 'Pipeline':
                        pipeline_update(msg, '%s-sync-to-deq' % self._sid, props.correlation_id, mq_channel)
                #-------------------------------------------------------------------------------------------------------

                # Appease pika cos it thinks the connection is dead
                now = time.time()
                if now - last >= self._rmq_ping_interval:
                    mq_connection.process_data_events()
                    last = now

            self._prof.prof('terminating synchronizer', uid=self._uid)

        except KeyboardInterrupt:

            self._logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to terminate synchronizer thread gracefully...')

            raise KeyboardInterrupt

        except Exception, ex:

            self._logger.exception('Unknown error in synchronizer: %s. \n Terminating thread' % ex)
            raise

Response: Synchronize the master process with the master process.


Instruction:
Write a Python 3 script to
call only on a valid form this method will place the chosen metrics in the given catgory.

Response:
def categorize_metrics(self):
        """Called only on a valid form, this method will place the chosen
        metrics in the given catgory."""
        category = self.cleaned_data['category_name']
        metrics = self.cleaned_data['metrics']
        self.r.reset_category(category, metrics)


Instruction:
Can you tell what is the following Python 3 function doing
def _submit_resource_request(self):

        try:

            self._prof.prof('creating rreq', uid=self._uid)

            def _pilot_state_cb(pilot, state):
                self._logger.info('Pilot %s state: %s' % (pilot.uid, state))

                if state == rp.FAILED:
                    self._logger.error('Pilot has failed')

                elif state == rp.DONE:
                    self._logger.error('Pilot has completed')

            self._session = rp.Session(dburl=self._mlab_url, uid=self._sid)
            self._pmgr = rp.PilotManager(session=self._session)
            self._pmgr.register_callback(_pilot_state_cb)

            pd_init = {
                'resource': self._resource,
                'runtime': self._walltime,
                'cores': self._cpus,
                'project': self._project,
            }

            if self._gpus:
                pd_init['gpus'] = self._gpus

            if self._access_schema:
                pd_init['access_schema'] = self._access_schema

            if self._queue:
                pd_init['queue'] = self._queue

            if self._rts_config.get('sandbox_cleanup', None):
                pd_init['cleanup'] = True

            # Create Compute Pilot with validated resource description
            pdesc = rp.ComputePilotDescription(pd_init)

            self._prof.prof('rreq created', uid=self._uid)

            # Launch the pilot
            self._pilot = self._pmgr.submit_pilots(pdesc)

            self._prof.prof('rreq submitted', uid=self._uid)

            shared_staging_directives = list()
            for data in self._shared_data:
                temp = {
                    'source': data,
                    'target': 'pilot:///' + os.path.basename(data)
                }
                shared_staging_directives.append(temp)

            self._pilot.stage_in(shared_staging_directives)

            self._prof.prof('shared data staging initiated', uid=self._uid)
            self._logger.info('Resource request submission successful.. waiting for pilot to go Active')

            # Wait for pilot to go active
            self._pilot.wait([rp.PMGR_ACTIVE, rp.FAILED, rp.CANCELED])

            self._prof.prof('resource active', uid=self._uid)
            self._logger.info('Pilot is now active')

        except KeyboardInterrupt:

            if self._session:
                self._session.close()

            self._logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to exit callback thread gracefully...')
            raise KeyboardInterrupt

        except Exception, ex:
            self._logger.exception('Resource request submission failed')
            raise

Response: Submits a resource request to the RADICAL Pilot.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _terminate_resource_request(self):

        try:

            if self._pilot:

                self._prof.prof('canceling resource allocation', uid=self._uid)
                self._pilot.cancel()
                download_rp_profile = os.environ.get('RADICAL_PILOT_PROFILE', False)
                self._session.close(cleanup=self._rts_config.get('db_cleanup', False),
                                    download=download_rp_profile)
                self._prof.prof('resource allocation canceled', uid=self._uid)

        except KeyboardInterrupt:

            self._logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to exit callback thread gracefully...')
            raise KeyboardInterrupt

        except Exception, ex:
            self._logger.exception('Could not cancel resource request, error: %s' % ex)
            raise

Response: Cancel the resource allocation.


Instruction:
Write a Python 3 script for
requesting service locations .

Response:
def get_list(self, size=100, startIndex=0, searchText="", sortProperty="", sortOrder='ASC', status='Active,Pending'):
        """
        Request service locations

        Returns
        -------
        dict
        """

        url = urljoin(BASEURL, "sites", "list")

        params = {
            'api_key': self.token,
            'size': size,
            'startIndex': startIndex,
            'sortOrder': sortOrder,
            'status': status
        }

        if searchText:
            params['searchText'] = searchText

        if sortProperty:
            params['sortProperty'] = sortProperty

        r = requests.get(url, params)
        r.raise_for_status()
        return r.json()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def match(self, f, *args):
        try:
            match = f(self.tokenizer, *args)
        except StopIteration:
            # The grammar function might have tried to access more tokens than
            # are available. That's not really an error, it just means it didn't
            # match.
            return

        if match is None:
            return

        if not isinstance(match, grammar.TokenMatch):
            raise TypeError("Invalid grammar function %r returned %r."
                            % (f, match))

        self.matched = match
        return match

Response: Match grammar function f against next token and set self. matched.


Instruction:
Here you have a function in Python 3, explain what it does
def accept(self, f, *args):
        match = self.match(f, *args)
        if match is None:
            return

        self.tokenizer.skip(len(match.tokens))
        return match

Response: Like match but consume the token.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def reject(self, f, *args):
        match = self.match(f, *args)
        if match:
            token = self.peek(0)
            raise errors.EfilterParseError(
                query=self.tokenizer.source, token=token,
                message="Was not expecting a %s here." % token.name)

Response: Like match but throw a parse error if f matches.


Instruction:
Given the following Python 3 function, write the documentation
def expect(self, f, *args):
        match = self.accept(f, *args)
        if match:
            return match

        try:
            func_name = f.func_name
        except AttributeError:
            func_name = "<unnamed grammar function>"

        start, end = self.current_position()
        raise errors.EfilterParseError(
            query=self.tokenizer.source, start=start, end=end,
            message="Was expecting %s here." % (func_name))

Response: Like accept but throws a parse error if f doesn t match.


Instruction:
Write a Python 3 function that can
return a tuple of start end of the current token.

Response:
def current_position(self):
        """Return a tuple of (start, end)."""
        token = self.tokenizer.peek(0)
        if token:
            return token.start, token.end

        return self.tokenizer.position, self.tokenizer.position + 1


Instruction:
Here you have a function in Python 3, explain what it does
def ComplementEquivalence(*args, **kwargs):
    return ast.Complement(
        ast.Equivalence(*args, **kwargs), **kwargs)

Response: Change x!= y to not ( x == y.


Instruction:
Can you write a function in Python 3 where it
changes x not in y to not in y.

Response:
def ComplementMembership(*args, **kwargs):
    """Change (x not in y) to not(x in y)."""
    return ast.Complement(
        ast.Membership(*args, **kwargs), **kwargs)


Instruction:
Write a Python 3 script to
change x doesn t contain y to not x.

Response:
def ReverseComplementMembership(x, y, **kwargs):
    """Change (x doesn't contain y) to not(y in x)."""
    return ast.Complement(
        ast.Membership(y, x, **kwargs), **kwargs)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def __solve_for_repeated(expr, vars):
    var = solve(expr, vars).value
    if (var and isinstance(var, (tuple, list))
            and protocol.implements(var[0], structured.IStructured)):
        return repeated.meld(*var), False

    return var, repeated.isrepeating(var)

Response: Solve expr returning a value of a repeated value.


Instruction:
Write a Python 3 function that can
solve expr returning a scalar value.

Response:
def __solve_for_scalar(expr, vars):
    """Helper: solve 'expr' always returning a scalar (not IRepeated).

    If the output of 'expr' is a single value or a single RowTuple with a single
    column then return the value in that column. Otherwise raise.

    Arguments:
        expr: Expression to solve.
        vars: The scope.

    Returns:
        A scalar value (not an IRepeated).

    Raises:
        EfilterTypeError if it cannot get a scalar.
    """
    var = solve(expr, vars).value
    try:
        scalar = repeated.getvalue(var)
    except TypeError:
        raise errors.EfilterTypeError(
            root=expr, query=expr.source,
            message="Wasn't expecting more than one value here. Got %r."
            % (var,))

    if isinstance(scalar, row_tuple.RowTuple):
        try:
            return scalar.get_singleton()
        except ValueError:
            raise errors.EfilterTypeError(
                root=expr, query=expr.source,
                message="Was expecting a scalar value here. Got %r."
                % (scalar,))
    else:
        return scalar


Instruction:
Write a Python 3 script for
returning the value of the var named in the expression.

Response:
def solve_var(expr, vars):
    """Returns the value of the var named in the expression."""
    try:
        return Result(structured.resolve(vars, expr.value), ())
    except (KeyError, AttributeError) as e:
        # Raise a better exception for accessing a non-existent member.
        raise errors.EfilterKeyError(root=expr, key=expr.value, message=e,
                                     query=expr.source)
    except (TypeError, ValueError) as e:
        # Raise a better exception for what is probably a null pointer error.
        if vars.locals is None:
            raise errors.EfilterNoneError(
                root=expr, query=expr.source,
                message="Trying to access member %r of a null." % expr.value)
        else:
            raise errors.EfilterTypeError(
                root=expr, query=expr.source,
                message="%r (vars: %r)" % (e, vars))
    except NotImplementedError as e:
        raise errors.EfilterError(
            root=expr, query=expr.source,
            message="Trying to access member %r of an instance of %r." %
            (expr.value, type(vars)))


Instruction:
Explain what the following Python 3 code does
def solve_select(expr, vars):
    data, _ = __solve_for_repeated(expr.lhs, vars)
    key = solve(expr.rhs, vars).value

    try:
        results = [associative.select(d, key) for d in repeated.getvalues(data)]
    except (KeyError, AttributeError):
        # Raise a better exception for accessing a non-existent key.
        raise errors.EfilterKeyError(root=expr, key=key, query=expr.source)
    except (TypeError, ValueError):
        # Raise a better exception for what is probably a null pointer error.
        if vars.locals is None:
            raise errors.EfilterNoneError(
                root=expr, query=expr.source,
                message="Cannot select key %r from a null." % key)
        else:
            raise
    except NotImplementedError:
        raise errors.EfilterError(
            root=expr, query=expr.source,
            message="Cannot select keys from a non-associative value.")

    return Result(repeated.meld(*results), ())

Response: Use IAssociative. select to get key from the data ( lhs ).


Instruction:
Create a Python 3 function for
using IStructured. resolve to get member from the object ( rhs ) from the object ( lhs ).

Response:
def solve_resolve(expr, vars):
    """Use IStructured.resolve to get member (rhs) from the object (lhs).

    This operation supports both scalars and repeated values on the LHS -
    resolving from a repeated value implies a map-like operation and returns a
    new repeated values.
    """
    objs, _ = __solve_for_repeated(expr.lhs, vars)
    member = solve(expr.rhs, vars).value

    try:
        results = [structured.resolve(o, member)
                   for o in repeated.getvalues(objs)]
    except (KeyError, AttributeError):
        # Raise a better exception for the non-existent member.
        raise errors.EfilterKeyError(root=expr.rhs, key=member,
                                     query=expr.source)
    except (TypeError, ValueError):
        # Is this a null object error?
        if vars.locals is None:
            raise errors.EfilterNoneError(
                root=expr, query=expr.source,
                message="Cannot resolve member %r from a null." % member)
        else:
            raise
    except NotImplementedError:
        raise errors.EfilterError(
            root=expr, query=expr.source,
            message="Cannot resolve members from a non-structured value.")

    return Result(repeated.meld(*results), ())


Instruction:
Can you generate a brief explanation for the following Python 3 code
def solve_apply(expr, vars):
    func = __solve_for_scalar(expr.func, vars)
    args = []
    kwargs = {}
    for arg in expr.args:
        if isinstance(arg, ast.Pair):
            if not isinstance(arg.lhs, ast.Var):
                raise errors.EfilterError(
                    root=arg.lhs,
                    message="Invalid argument name.")

            kwargs[arg.key.value] = solve(arg.value, vars).value
        else:
            args.append(solve(arg, vars).value)

    result = applicative.apply(func, args, kwargs)

    return Result(result, ())

Response: Returns the result of applying a function to its arguments.


Instruction:
Write a Python 3 script to
solve the bind expression and return a RowTuple that is the result of evaluating the bind subtree.

Response:
def solve_bind(expr, vars):
    """Build a RowTuple from key/value pairs under the bind.

    The Bind subtree is arranged as follows:

    Bind
    | First KV Pair
    | | First Key Expression
    | | First Value Expression
    | Second KV Pair
    | | Second Key Expression
    | | Second Value Expression
    Etc...

    As we evaluate the subtree, each subsequent KV pair is evaluated with
    the all previous bingings already in scope. For example:

    bind(x: 5, y: x + 5)  # Will bind y = 10 because x is already available.
    """
    value_expressions = []
    keys = []
    for pair in expr.children:
        keys.append(solve(pair.key, vars).value)
        value_expressions.append(pair.value)

    result = row_tuple.RowTuple(ordered_columns=keys)
    intermediate_scope = scope.ScopeStack(vars, result)

    for idx, value_expression in enumerate(value_expressions):
        value = solve(value_expression, intermediate_scope).value
        # Update the intermediate bindings so as to make earlier bindings
        # already available to the next child-expression.
        result[keys[idx]] = value

    return Result(result, ())


Instruction:
Given the following Python 3 function, write the documentation
def solve_repeat(expr, vars):
    try:
        result = repeated.meld(*[solve(x, vars).value for x in expr.children])
        return Result(result, ())
    except TypeError:
        raise errors.EfilterTypeError(
            root=expr, query=expr.source,
            message="All values in a repeated value must be of the same type.")

Response: Build a repeated value from subexpressions.


Instruction:
Implement a Python 3 function for
building a tuple from subexpressions.

Response:
def solve_tuple(expr, vars):
    """Build a tuple from subexpressions."""
    result = tuple(solve(x, vars).value for x in expr.children)
    return Result(result, ())


Instruction:
Create a Python 3 function for
evaluating conditions and return the one that matches the given expression.

Response:
def solve_ifelse(expr, vars):
    """Evaluate conditions and return the one that matches."""
    for condition, result in expr.conditions():
        if boolean.asbool(solve(condition, vars).value):
            return solve(result, vars)

    return solve(expr.default(), vars)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def solve_map(expr, vars):
    lhs_values, _ = __solve_for_repeated(expr.lhs, vars)

    def lazy_map():
        try:
            for lhs_value in repeated.getvalues(lhs_values):
                yield solve(expr.rhs,
                            __nest_scope(expr.lhs, vars, lhs_value)).value
        except errors.EfilterNoneError as error:
            error.root = expr
            raise

    return Result(repeated.lazy(lazy_map), ())

Response: Solves the map - form by recursively calling its RHS with new vars.


Instruction:
How would you code a function in Python 3 to
solve a let - form by calling RHS with nested scope.

Response:
def solve_let(expr, vars):
    """Solves a let-form by calling RHS with nested scope."""
    lhs_value = solve(expr.lhs, vars).value
    if not isinstance(lhs_value, structured.IStructured):
        raise errors.EfilterTypeError(
            root=expr.lhs, query=expr.original,
            message="The LHS of 'let' must evaluate to an IStructured. Got %r."
            % (lhs_value,))

    return solve(expr.rhs, __nest_scope(expr.lhs, vars, lhs_value))


Instruction:
Implement a Python 3 function for
filtering values on the LHS by evaluating each value.

Response:
def solve_filter(expr, vars):
    """Filter values on the LHS by evaluating RHS with each value.

    Returns any LHS values for which RHS evaluates to a true value.
    """
    lhs_values, _ = __solve_for_repeated(expr.lhs, vars)

    def lazy_filter():
        for lhs_value in repeated.getvalues(lhs_values):
            if solve(expr.rhs, __nest_scope(expr.lhs, vars, lhs_value)).value:
                yield lhs_value

    return Result(repeated.lazy(lazy_filter), ())


Instruction:
Make a summary of the following Python 3 code
def solve_sort(expr, vars):
    lhs_values = repeated.getvalues(__solve_for_repeated(expr.lhs, vars)[0])

    sort_expression = expr.rhs

    def _key_func(x):
        return solve(sort_expression, __nest_scope(expr.lhs, vars, x)).value

    results = ordered.ordered(lhs_values, key_func=_key_func)

    return Result(repeated.meld(*results), ())

Response: Sort values on the LHS by the value they yield when passed to RHS.


Instruction:
Create a Python 3 function to
return True if RHS evaluates to a true value with each state of LHS.

Response:
def solve_each(expr, vars):
    """Return True if RHS evaluates to a true value with each state of LHS.

    If LHS evaluates to a normal IAssociative object then this is the same as
    a regular let-form, except the return value is always a boolean. If LHS
    evaluates to a repeared var (see efilter.protocols.repeated) of
    IAssociative objects then RHS will be evaluated with each state and True
    will be returned only if each result is true.
    """
    lhs_values, _ = __solve_for_repeated(expr.lhs, vars)

    for lhs_value in repeated.getvalues(lhs_values):
        result = solve(expr.rhs, __nest_scope(expr.lhs, vars, lhs_value))
        if not result.value:
            # Each is required to return an actual boolean.
            return result._replace(value=False)

    return Result(True, ())


Instruction:
Create a Python 3 function to
get cast LHS to RHS.

Response:
def solve_cast(expr, vars):
    """Get cast LHS to RHS."""
    lhs = solve(expr.lhs, vars).value
    t = solve(expr.rhs, vars).value

    if t is None:
        raise errors.EfilterTypeError(
            root=expr, query=expr.source,
            message="Cannot find type named %r." % expr.rhs.value)

    if not isinstance(t, type):
        raise errors.EfilterTypeError(
            root=expr.rhs, query=expr.source,
            message="%r is not a type and cannot be used with 'cast'." % (t,))

    try:
        cast_value = t(lhs)
    except TypeError:
        raise errors.EfilterTypeError(
            root=expr, query=expr.source,
            message="Invalid cast %s -> %s." % (type(lhs), t))

    return Result(cast_value, ())


Instruction:
Make a summary of the following Python 3 code
def solve_isinstance(expr, vars):
    lhs = solve(expr.lhs, vars)

    try:
        t = solve(expr.rhs, vars).value
    except errors.EfilterKeyError:
        t = None

    if t is None:
        raise errors.EfilterTypeError(
            root=expr.rhs, query=expr.source,
            message="Cannot find type named %r." % expr.rhs.value)

    if not isinstance(t, type):
        raise errors.EfilterTypeError(
            root=expr.rhs, query=expr.source,
            message="%r is not a type and cannot be used with 'isa'." % (t,))

    return Result(protocol.implements(lhs.value, t), ())

Response: Typecheck whether LHS is type on the RHS.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_version(mod_root):

    try:

        version_base   = None
        version_detail = None

        # get version from './VERSION'
        src_root = os.path.dirname(__file__)
        if  not src_root:
            src_root = '.'

        with open(src_root + '/VERSION', 'r') as f:
            version_base = f.readline().strip()

        # attempt to get version detail information from git
        # We only do that though if we are in a repo root dir,
        # ie. if 'git rev-parse --show-prefix' returns an empty string --
        # otherwise we get confused if the ve lives beneath another repository,
        # and the pip version used uses an install tmp dir in the ve space
        # instead of /tmp (which seems to happen with some pip/setuptools
        # versions).
        p = sp.Popen('cd %s ; '
                     'test -z `git rev-parse --show-prefix` || exit -1; '
                     'tag=`git describe --tags --always` 2>/dev/null ; '
                     'branch=`git branch | grep -e "^*" | cut -f 2- -d " "` 2>/dev/null ; '
                     'echo $tag@$branch' % src_root,
                     stdout=sp.PIPE, stderr=sp.STDOUT, shell=True)
        version_detail = str(p.communicate()[0].strip())
        version_detail = version_detail.replace('detached from ', 'detached-')

        # remove all non-alphanumeric (and then some) chars
        version_detail = re.sub('[/ ]+', '-', version_detail)
        version_detail = re.sub('[^a-zA-Z0-9_+@.-]+', '', version_detail)

        if  p.returncode   !=  0  or \
            version_detail == '@' or \
            'git-error' in version_detail or \
            'not-a-git-repo' in version_detail or \
            'not-found'      in version_detail or \
            'fatal'          in version_detail :
            version = version_base
        elif '@' not in version_base:
            version = '%s-%s' % (version_base, version_detail)
        else:
            version = version_base

        # make sure the version files exist for the runtime version inspection
        path = '%s/%s' % (src_root, mod_root)
        with open(path + "/VERSION", "w") as f:
            f.write(version + "\n")

        sdist_name = "%s-%s.tar.gz" % (name, version)
        sdist_name = sdist_name.replace('/', '-')
        sdist_name = sdist_name.replace('@', '-')
        sdist_name = sdist_name.replace('#', '-')
        sdist_name = sdist_name.replace('_', '-')

        if '--record'    in sys.argv or \
           'bdist_egg'   in sys.argv or \
           'bdist_wheel' in sys.argv    :
          # pip install stage 2 or easy_install stage 1
          #
          # pip install will untar the sdist in a tmp tree.  In that tmp
          # tree, we won't be able to derive git version tags -- so we pack the
          # formerly derived version as ./VERSION
            shutil.move("VERSION", "VERSION.bak")            # backup version
            shutil.copy("%s/VERSION" % path, "VERSION")      # use full version instead
            os.system  ("python setup.py sdist")             # build sdist
            shutil.copy('dist/%s' % sdist_name,
                        '%s/%s'   % (mod_root, sdist_name))  # copy into tree
            shutil.move("VERSION.bak", "VERSION")            # restore version

        with open(path + "/SDIST", "w") as f:
            f.write(sdist_name + "\n")

        return version_base, version_detail, sdist_name

    except Exception as e :
        raise RuntimeError('Could not extract/set version: %s' % e)

Response: This function will set the version of the mod_root and the version of the mod_root.


Instruction:
Write a Python 3 function for
creating distutil data_files structure from dir

Response:
def makeDataFiles(prefix, dir):
    """ Create distutils data_files structure from dir
    distutil will copy all file rooted under dir into prefix, excluding
    dir itself, just like 'ditto src dst' works, and unlike 'cp -r src
    dst, which copy src into dst'.
    Typical usage:
        # install the contents of 'wiki' under sys.prefix+'share/moin'
        data_files = makeDataFiles('share/moin', 'wiki')
    For this directory structure:
        root
            file1
            file2
            dir
                file
                subdir
                    file
    makeDataFiles('prefix', 'root')  will create this distutil data_files structure:
        [('prefix', ['file1', 'file2']),
         ('prefix/dir', ['file']),
         ('prefix/dir/subdir', ['file'])]
    """
    # Strip 'dir/' from of path before joining with prefix
    dir = dir.rstrip('/')
    strip = len(dir) + 1
    found = []
    os.path.walk(dir, visit, (prefix, strip, found))
    #print found[0]
    return found[0]


Instruction:
Explain what the following Python 3 code does
def visit((prefix, strip, found), dirname, names):
    files = []
    # Iterate over a copy of names, modify names
    for name in names[:]:
        path = os.path.join(dirname, name)
        # Ignore directories -  we will visit later
        if os.path.isdir(path):
            # Remove directories we don't want to visit later
            if isbad(name):
                names.remove(name)
            continue
        elif isgood(name):
            files.append(path)
    destination = os.path.join(prefix, dirname[strip:])
    found.append((destination, files))

Response: Visit directory and add distutil tuple for each file in names


Instruction:
Can you generate the documentation for the following Python 3 function
def _initialize_workflow(self):

        try:

            self._prof.prof('initializing workflow', uid=self._uid)

            for p in self._workflow:
                p._assign_uid(self._sid)

            self._prof.prof('workflow initialized', uid=self._uid)

        except Exception, ex:
            self._logger.exception(
                'Fatal error while initializing workflow: %s' % ex)
            raise

Response: Initialize the PST of the workflow with a uid and type checks.


Instruction:
Here you have a function in Python 3, explain what it does
def _enqueue(self, local_prof):

        try:

            local_prof.prof('enqueue-thread started', uid=self._uid)
            self._logger.info('enqueue-thread started')

            # Acquire a connection+channel to the rmq server
            mq_connection = pika.BlockingConnection(
                pika.ConnectionParameters(host=self._mq_hostname, port=self._port))
            mq_channel = mq_connection.channel()

            last = time.time()
            while not self._enqueue_thread_terminate.is_set():

                '''
                We iterate through all pipelines to collect tasks from
                stages that are pending scheduling. Once collected, these tasks
                will be communicated to the tmgr in bulk.
                '''

                workload = []
                scheduled_stages = []

                for pipe in self._workflow:

                    with pipe.lock:

                        if ((not pipe.completed) and (not pipe.state == states.SUSPENDED)):

                            # Test if the pipeline is already in the final state
                            if pipe.state in states.FINAL:
                                continue

                            elif pipe.state == states.INITIAL:

                                # Set state of pipeline to SCHEDULING if it is in INITIAL
                                transition(obj=pipe,
                                           obj_type='Pipeline',
                                           new_state=states.SCHEDULING,
                                           channel=mq_channel,
                                           queue='%s-enq-to-sync' % self._sid,
                                           profiler=local_prof,
                                           logger=self._logger)

                            executable_stage = pipe.stages[pipe.current_stage - 1]

                            if not executable_stage.uid:
                                executable_stage.parent_pipeline['uid'] = pipe.uid
                                executable_stage.parent_pipeline['name'] = pipe.name
                                executable_stage._assign_uid(self._sid)

                            if executable_stage.state in [states.INITIAL, states.SCHEDULED]:

                                if executable_stage.state == states.INITIAL:

                                    transition(obj=executable_stage,
                                               obj_type='Stage',
                                               new_state=states.SCHEDULING,
                                               channel=mq_channel,
                                               queue='%s-enq-to-sync' % self._sid,
                                               profiler=local_prof,
                                               logger=self._logger)

                                executable_tasks = executable_stage.tasks

                                for executable_task in executable_tasks:

                                    if (executable_task.state == states.INITIAL)or \
                                            ((executable_task.state == states.FAILED)and(self._resubmit_failed)):

                                        # Set state of Tasks in current Stage to SCHEDULING
                                        transition(obj=executable_task,
                                                   obj_type='Task',
                                                   new_state=states.SCHEDULING,
                                                   channel=mq_channel,
                                                   queue='%s-enq-to-sync' % self._sid,
                                                   profiler=local_prof,
                                                   logger=self._logger)

                                        # task_as_dict = json.dumps(executable_task.to_dict())
                                        workload.append(executable_task)

                                        if executable_stage not in scheduled_stages:
                                            scheduled_stages.append(
                                                executable_stage)

                if workload:

                    # Put the task on one of the pending_queues
                    workload_as_dict = list()
                    for task in workload:
                        workload_as_dict.append(task.to_dict())
                    workload_as_dict = json.dumps(workload_as_dict)

                    mq_channel.basic_publish(exchange='',
                                             routing_key=self._pending_queue[0],
                                             body=workload_as_dict
                                             # properties=pika.BasicProperties(
                                             # make message persistent
                                             # delivery_mode = 2)
                                             )

                    for task in workload:

                        # Set state of Tasks in current Stage to SCHEDULED
                        transition(obj=task,
                                   obj_type='Task',
                                   new_state=states.SCHEDULED,
                                   channel=mq_channel,
                                   queue='%s-enq-to-sync' % self._sid,
                                   profiler=local_prof,
                                   logger=self._logger)

                        self._logger.debug(
                            'Task %s published to pending queue' % task.uid)

                if scheduled_stages:
                    for executable_stage in scheduled_stages:

                        transition(obj=executable_stage,
                                   obj_type='Stage',
                                   new_state=states.SCHEDULED,
                                   channel=mq_channel,
                                   queue='%s-enq-to-sync' % self._sid,
                                   profiler=local_prof,
                                   logger=self._logger)

                # Appease pika cos it thinks the connection is dead
                now = time.time()
                if now - last >= self._rmq_ping_interval:
                    mq_connection.process_data_events()
                    last = now

            self._logger.info('Enqueue thread terminated')
            mq_connection.close()

            local_prof.prof('terminating enqueue-thread', uid=self._uid)

        except KeyboardInterrupt:

            self._logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to cancel enqueuer thread gracefully...')

            mq_connection.close()

            raise KeyboardInterrupt

        except Exception, ex:

            self._logger.exception('Error in enqueue-thread: %s' % ex)
            try:
                mq_connection.close()
            except Exception as ex:
                self._logger.warning('mq_connection not created, %s' % ex)

            raise

Response: This function is run in the enqueue thread. It will iterate through all pipelines and put them into the pending_q list.


Instruction:
Here you have a function in Python 3, explain what it does
def _wfp(self):

        try:

            local_prof = ru.Profiler(
                name='radical.entk.%s' % self._uid + '-proc', path=self._path)

            local_prof.prof('wfp process started', uid=self._uid)

            self._logger.info('WFprocessor started')

            # Process should run till terminate condtion is encountered
            while (not self._wfp_terminate.is_set()):

                try:

                    # Start dequeue thread
                    if (not self._dequeue_thread) or (not self._dequeue_thread.is_alive()):

                        local_prof.prof(
                            'creating dequeue-thread', uid=self._uid)
                        self._dequeue_thread = threading.Thread(
                            target=self._dequeue, args=(local_prof,), name='dequeue-thread')

                        self._logger.info('Starting dequeue-thread')
                        local_prof.prof(
                            'starting dequeue-thread', uid=self._uid)
                        self._dequeue_thread.start()

                    # Start enqueue thread
                    if (not self._enqueue_thread) or (not self._enqueue_thread.is_alive()):

                        local_prof.prof(
                            'creating enqueue-thread', uid=self._uid)
                        self._enqueue_thread = threading.Thread(
                            target=self._enqueue, args=(local_prof,), name='enqueue-thread')

                        self._logger.info('Starting enqueue-thread')
                        local_prof.prof(
                            'starting enqueue-thread', uid=self._uid)
                        self._enqueue_thread.start()

                except Exception, ex:
                    self._logger.exception('WFProcessor interrupted')
                    raise

            local_prof.prof('start termination', uid=self._uid)

            self._logger.info('Terminating enqueue-thread')
            self._enqueue_thread_terminate.set()
            self._enqueue_thread.join()
            self._logger.info('Terminating dequeue-thread')
            self._dequeue_thread_terminate.set()
            self._dequeue_thread.join()

            local_prof.prof('termination done', uid=self._uid)

            local_prof.prof('terminating wfp process', uid=self._uid)

            local_prof.close()

        except KeyboardInterrupt:

            self._logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to cancel wfprocessor process gracefully...')

            if self._enqueue_thread:

                if not self._enqueue_thread_terminate.is_set():
                    self._logger.info('Terminating enqueue-thread')
                    self._enqueue_thread_terminate.set()
                    self._enqueue_thread.join()

            if self._dequeue_thread:

                if not self._dequeue_thread_terminate.is_set():
                    self._logger.info('Terminating dequeue-thread')
                    self._dequeue_thread_terminate.set()
                    self._dequeue_thread.join()

            self._logger.info('WFprocessor process terminated')

            raise KeyboardInterrupt

        except Exception, ex:
            self._logger.exception(
                'Error in wfp process: %s. \n Closing enqueue, dequeue threads' % ex)

            if self._enqueue_thread:

                if not self._enqueue_thread_terminate.is_set():
                    self._logger.info('Terminating enqueue-thread')
                    self._enqueue_thread_terminate.set()
                    self._enqueue_thread.join()

            if self._dequeue_thread:

                if not self._dequeue_thread_terminate.is_set():
                    self._logger.info('Terminating dequeue-thread')
                    self._dequeue_thread_terminate.set()
                    self._dequeue_thread.join()

            self._logger.info('WFprocessor process terminated')

            self._logger.exception('%s failed with %s'%(self._uid, ex))
            raise EnTKError(ex)

Response: This function is used to create and spawn two threads for the slow version of the WF processor.


Instruction:
Make a summary of the following Python 3 code
def start_processor(self):

        if not self._wfp_process:

            try:

                self._prof.prof('creating wfp process', uid=self._uid)
                self._wfp_process = Process(
                    target=self._wfp, name='wfprocessor')

                self._enqueue_thread = None
                self._dequeue_thread = None
                self._enqueue_thread_terminate = threading.Event()
                self._dequeue_thread_terminate = threading.Event()

                self._wfp_terminate = Event()
                self._logger.info('Starting WFprocessor process')
                self._prof.prof('starting wfp process', uid=self._uid)
                self._wfp_process.start()

                return True

            except Exception, ex:

                self._logger.exception('WFprocessor not started')
                self.terminate_processor()
                raise

        else:
            self._logger.warn(
                'Wfp process already running, attempted to restart!')

Response: Start the WFprocessor process.


Instruction:
Explain what the following Python 3 code does
def terminate_processor(self):

        try:

            if self.check_processor():
                self._logger.debug(
                    'Attempting to end WFprocessor... event: %s' % self._wfp_terminate.is_set())
                self._wfp_terminate.set()
                self._wfp_process.join()
                self._wfp_process = None
                self._logger.debug('WFprocessor process terminated')
            else:
                self._logger.debug('WFprocessor process already terminated')

            self._prof.prof('wfp process terminated', uid=self._uid)

            self._prof.close()

        except Exception, ex:
            self._logger.exception('Could not terminate wfprocessor process')
            raise

Response: This method terminate the WFprocessor process.


Instruction:
Explain what the following Python 3 code does
def workflow_incomplete(self):

        try:
            for pipe in self._workflow:
                with pipe.lock:
                    if pipe.completed:
                        pass
                    else:
                        return True
            return False

        except Exception, ex:
            self._logger.exception(
                'Could not check if workflow is incomplete, error:%s' % ex)
            raise

Response: Checks if the workflow is incomplete.


Instruction:
Create a Python 3 function to
return the repeated value or first value if there s only one.

Response:
def meld(*values):
    """Return the repeated value, or the first value if there's only one.

    This is a convenience function, equivalent to calling
    getvalue(repeated(x)) to get x.

    This function skips over instances of None in values (None is not allowed
    in repeated variables).

    Examples:
        meld("foo", "bar") # => ListRepetition("foo", "bar")
        meld("foo", "foo") # => ListRepetition("foo", "foo")
        meld("foo", None) # => "foo"
        meld(None) # => None
    """
    values = [x for x in values if x is not None]
    if not values:
        return None

    result = repeated(*values)
    if isrepeating(result):
        return result

    return getvalue(result)


Instruction:
Can you tell what is the following Python 3 function doing
def getvalue(x):
    if isrepeating(x):
        raise TypeError(
            "Ambiguous call to getvalue for %r which has more than one value."
            % x)

    for value in getvalues(x):
        return value

Response: Return the single value of x.


Instruction:
Create a Python 3 function to
return the fully qualified unique ID of the current task.

Response:
def luid(self):
        """
        Unique ID of the current task (fully qualified).

        example:
            >>> task.luid
            pipe.0001.stage.0004.task.0234

        :getter: Returns the fully qualified uid of the current task
        :type: String
        """
        p_elem = self.parent_pipeline.get('name')
        if not p_elem:
            p_elem = self.parent_pipeline['uid']

        s_elem = self.parent_stage.get('name')
        if not s_elem:
            s_elem = self.parent_stage['uid']

        t_elem = self.name
        if not t_elem:
            t_elem = self.uid

        return '%s.%s.%s' % (p_elem, s_elem, t_elem)


Instruction:
Explain what the following Python 3 code does
def to_dict(self):

        task_desc_as_dict = {
            'uid': self._uid,
            'name': self._name,
            'state': self._state,
            'state_history': self._state_history,

            'pre_exec': self._pre_exec,
            'executable': self._executable,
            'arguments': self._arguments,
            'post_exec': self._post_exec,
            'cpu_reqs': self._cpu_reqs,
            'gpu_reqs': self._gpu_reqs,
            'lfs_per_process': self._lfs_per_process,

            'upload_input_data': self._upload_input_data,
            'copy_input_data': self._copy_input_data,
            'link_input_data': self._link_input_data,
            'move_input_data': self._move_input_data,
            'copy_output_data': self._copy_output_data,
            'move_output_data': self._move_output_data,
            'download_output_data': self._download_output_data,

            'stdout': self._stdout,
            'stderr': self._stderr,

            'exit_code': self._exit_code,
            'path': self._path,
            'tag': self._tag,

            'parent_stage': self._p_stage,
            'parent_pipeline': self._p_pipeline,
        }

        return task_desc_as_dict

Response: Convert the current Task into a dictionary.


Instruction:
Can you generate the documentation for the following Python 3 function
def from_dict(self, d):

        if 'uid' in d:
            if d['uid']:
                self._uid = d['uid']

        if 'name' in d:
            if d['name']:
                self._name = d['name']

        if 'state' in d:
            if isinstance(d['state'], str) or isinstance(d['state'], unicode):
                self._state = d['state']
            else:
                raise TypeError(entity='state', expected_type=str,
                                actual_type=type(d['state']))
        else:
            self._state = states.INITIAL

        if 'state_history' in d:
            if isinstance(d['state_history'], list):
                self._state_history = d['state_history']
            else:
                raise TypeError(entity='state_history', expected_type=list, actual_type=type(
                    d['state_history']))

        if 'pre_exec' in d:
            if isinstance(d['pre_exec'], list):
                self._pre_exec = d['pre_exec']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['pre_exec']))

        if 'executable' in d:
            if isinstance(d['executable'], str) or isinstance(d['executable'], unicode):
                self._executable = d['executable']
            else:
                raise TypeError(expected_type=str,
                                actual_type=type(d['executable']))

        if 'arguments' in d:
            if isinstance(d['arguments'], list):
                self._arguments = d['arguments']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['arguments']))

        if 'post_exec' in d:
            if isinstance(d['post_exec'], list):
                self._post_exec = d['post_exec']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['post_exec']))

        if 'cpu_reqs' in d:
            if isinstance(d['cpu_reqs'], dict):
                self._cpu_reqs = d['cpu_reqs']
            else:
                raise TypeError(expected_type=dict,
                                actual_type=type(d['cpu_reqs']))

        if 'gpu_reqs' in d:
            if isinstance(d['gpu_reqs'], dict):
                self._gpu_reqs = d['gpu_reqs']
            else:
                raise TypeError(expected_type=dict,
                                actual_type=type(d['gpu_reqs']))

        if 'lfs_per_process' in d:
            if d['lfs_per_process']:
                if isinstance(d['lfs_per_process'], int):
                    self._lfs_per_process = d['lfs_per_process']
                else:
                    raise TypeError(expected_type=int,
                                    actual_type=type(d['lfs_per_process']))

        if 'upload_input_data' in d:
            if isinstance(d['upload_input_data'], list):
                self._upload_input_data = d['upload_input_data']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['upload_input_data']))

        if 'copy_input_data' in d:
            if isinstance(d['copy_input_data'], list):
                self._copy_input_data = d['copy_input_data']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['copy_input_data']))

        if 'link_input_data' in d:
            if isinstance(d['link_input_data'], list):
                self._link_input_data = d['link_input_data']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['link_input_data']))

        if 'move_input_data' in d:
            if isinstance(d['move_input_data'], list):
                self._move_input_data = d['move_input_data']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['move_input_data']))

        if 'copy_output_data' in d:
            if isinstance(d['copy_output_data'], list):
                self._copy_output_data = d['copy_output_data']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['copy_output_data']))

        if 'move_output_data' in d:
            if isinstance(d['move_output_data'], list):
                self._move_output_data = d['move_output_data']
            else:
                raise TypeError(expected_type=list,
                                actual_type=type(d['move_output_data']))

        if 'download_output_data' in d:
            if isinstance(d['download_output_data'], list):
                self._download_output_data = d['download_output_data']
            else:
                raise TypeError(expected_type=list, actual_type=type(
                    d['download_output_data']))

        if 'stdout' in d:
            if d['stdout']:
                if isinstance(d['stdout'], str) or isinstance(d['stdout'], unicode):
                    self._stdout = d['stdout']
                else:
                    raise TypeError(expected_type=str, actual_type=type(d['stdout']))

        if 'stderr' in d:
            if d['stderr']:
                if isinstance(d['stderr'], str) or isinstance(d['stderr'], unicode):
                    self._stderr = d['stderr']
                else:
                    raise TypeError(expected_type=str, actual_type=type(d['stderr']))

        if 'exit_code' in d:
            if d['exit_code']:
                if isinstance(d['exit_code'], int):
                    self._exit_code = d['exit_code']
                else:
                    raise TypeError(
                        entity='exit_code', expected_type=int, actual_type=type(d['exit_code']))

        if 'path' in d:
            if d['path']:
                if isinstance(d['path'], str) or isinstance(d['path'], unicode):
                    self._path = d['path']
                else:
                    raise TypeError(entity='path', expected_type=str,
                                    actual_type=type(d['path']))

        if 'tag' in d:
            if d['tag']:
                if isinstance(d['tag'], str) or isinstance(d['tag'], unicode):
                    self._tag = str(d['tag'])
                else:
                    raise TypeError(expected_type=str,
                                    actual_type=type(d['tag']))

        if 'parent_stage' in d:
            if isinstance(d['parent_stage'], dict):
                self._p_stage = d['parent_stage']
            else:
                raise TypeError(
                    entity='parent_stage', expected_type=dict, actual_type=type(d['parent_stage']))

        if 'parent_pipeline' in d:
            if isinstance(d['parent_pipeline'], dict):
                self._p_pipeline = d['parent_pipeline']
            else:
                raise TypeError(entity='parent_pipeline', expected_type=dict, actual_type=type(
                    d['parent_pipeline']))

Response: Create a Task from a dictionary.


Instruction:
Can you implement a function in Python 3 that
assigns a uid to the current object based on the sid passed in.

Response:
def _assign_uid(self, sid):
        """
        Purpose: Assign a uid to the current object based on the sid passed
        """
        self._uid = ru.generate_id(
            'task.%(item_counter)04d', ru.ID_CUSTOM, namespace=sid)


Instruction:
Here you have a function in Python 3, explain what it does
def _validate(self):

        if self._state is not states.INITIAL:
            raise ValueError(obj=self._uid,
                             attribute='state',
                             expected_value=states.INITIAL,
                             actual_value=self._state)

        if not self._executable:
            raise MissingError(obj=self._uid,
                               missing_attribute='executable')

Response: Validate that the state of the current object is DESCRIBED and that the executable is specified for the current object.


Instruction:
Can you tell what is the following Python 3 function doing
def _process_tasks(self, task_queue, rmgr, logger, mq_hostname, port, local_prof, sid):

        '''
        **Purpose**: The new thread that gets spawned by the main tmgr process invokes this function. This
        function receives tasks from 'task_queue' and submits them to the RADICAL Pilot RTS.
        '''

        placeholder_dict = dict()

        def load_placeholder(task, rts_uid):

            parent_pipeline = str(task.parent_pipeline['name'])
            parent_stage = str(task.parent_stage['name'])

            if parent_pipeline not in placeholder_dict:
                placeholder_dict[parent_pipeline] = dict()

            if parent_stage not in placeholder_dict[parent_pipeline]:
                placeholder_dict[parent_pipeline][parent_stage] = dict()

            if None not in [parent_pipeline, parent_stage, task.name]:
                placeholder_dict[parent_pipeline][parent_stage][str(task.name)] = {'path': str(task.path),
                                                                                'rts_uid': rts_uid}

        def unit_state_cb(unit, state):

            try:

                logger.debug('Unit %s in state %s' % (unit.uid, unit.state))

                if unit.state in rp.FINAL:

                    # Acquire a connection+channel to the rmq server
                    mq_connection = pika.BlockingConnection(
                        pika.ConnectionParameters(host=mq_hostname, port=port))
                    mq_channel = mq_connection.channel()

                    task = None
                    task = create_task_from_cu(unit, local_prof)

                    transition(obj=task,
                               obj_type='Task',
                               new_state=states.COMPLETED,
                               channel=mq_channel,
                               queue='%s-cb-to-sync' % sid,
                               profiler=local_prof,
                               logger=logger)

                    load_placeholder(task, unit.uid)

                    task_as_dict = json.dumps(task.to_dict())

                    mq_channel.basic_publish(exchange='',
                                             routing_key='%s-completedq-1' % sid,
                                             body=task_as_dict
                                             # properties=pika.BasicProperties(
                                             # make message persistent
                                             #    delivery_mode = 2,
                                             # )
                                             )

                    logger.info('Pushed task %s with state %s to completed queue %s-completedq-1' % (task.uid, task.state,
                                                                                sid))

                    mq_connection.close()

            except KeyboardInterrupt:
                logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                                 'trying to exit callback thread gracefully...')

                raise KeyboardInterrupt

            except Exception, ex:
                logger.exception('Error in RP callback thread: %s' % ex)

        umgr = rp.UnitManager(session=rmgr._session)
        umgr.add_pilots(rmgr.pilot)
        umgr.register_callback(unit_state_cb)

        try:

            while not self._tmgr_terminate.is_set():

                body = None

                try:
                    body = task_queue.get(block=True, timeout=10)
                except Queue.Empty:
                    # Ignore empty exception, we don't always have new tasks to run
                    pass

                if body:

                    task_queue.task_done()

                    bulk_tasks = list()
                    bulk_cuds = list()

                    for task in body:
                        t = Task()
                        t.from_dict(task)
                        bulk_tasks.append(t)
                        bulk_cuds.append(create_cud_from_task(
                            t, placeholder_dict, local_prof))

                        mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=mq_hostname, port=port))
                        mq_channel = mq_connection.channel()

                        transition(obj=t,
                                   obj_type='Task',
                                   new_state=states.SUBMITTING,
                                   channel=mq_channel,
                                   queue='%s-tmgr-to-sync' % sid,
                                   profiler=local_prof,
                                   logger=logger)

                        mq_connection.close()

                    umgr.submit_units(bulk_cuds)

                    for task in bulk_tasks:

                        mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=mq_hostname, port=port))
                        mq_channel = mq_connection.channel()

                        transition(obj=task,
                                   obj_type='Task',
                                   new_state=states.SUBMITTED,
                                   channel=mq_channel,
                                   queue='%s-tmgr-to-sync' % sid,
                                   profiler=local_prof,
                                   logger=logger)

                        mq_connection.close()

        except KeyboardInterrupt as ex:
            logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                            'trying to cancel task processor gracefully...')

        except Exception as ex:
            logger.exception('%s failed with %s'%(self._uid, ex))
            raise EnTKError(ex)

Response: This function processes all tasks from the task_queue and creates a new thread and submits them to the RADICAL Pilot RTS.


Instruction:
Explain what the following Python 3 code does
def infer_type(expr, scope):
    # Do we know what the key even is?
    if isinstance(expr.key, ast.Literal):
        key = expr.key.value
    else:
        return protocol.AnyType

    container_type = infer_type(expr.value, scope)

    try:
        # Associative types are not subject to scoping rules so we can just
        # reflect using IAssociative.
        return associative.reflect(container_type, key) or protocol.AnyType
    except NotImplementedError:
        return protocol.AnyType

Response: Try to infer the type of x y.


Instruction:
Can you tell what is the following Python 3 function doing
def infer_type(expr, scope):
    # Do we know what the member is?
    if isinstance(expr.member, ast.Literal):
        member = expr.member.value
    else:
        return protocol.AnyType

    container_type = infer_type(expr.obj, scope)

    try:
        # We are not using lexical scope here on purpose - we want to see what
        # the type of the member is only on the container_type.
        return structured.reflect(container_type, member) or protocol.AnyType
    except NotImplementedError:
        return protocol.AnyType

Response: Try to infer the type of x. y if x is a known value.


Instruction:
Explain what the following Python 3 code does
def _tmgr(self, uid, rmgr, logger, mq_hostname, port, pending_queue, completed_queue):

        try:

            def heartbeat_response(mq_channel):

                try:

                    # Get request from heartbeat-req for heartbeat response
                    hb_method_frame, hb_props, hb_body = mq_channel.basic_get(
                        queue=self._hb_request_q)

                    if hb_body:

                        logger.info('Received heartbeat request')

                        mq_channel.basic_publish(exchange='',
                                                 routing_key=self._hb_response_q,
                                                 properties=pika.BasicProperties(
                                                     correlation_id=hb_props.correlation_id),
                                                 body='response')

                        logger.info('Sent heartbeat response')
                        mq_channel.basic_ack(
                            delivery_tag=hb_method_frame.delivery_tag)

                except Exception, ex:
                    logger.exception(
                        'Failed to respond to heartbeat request, error: %s' % ex)
                    raise

            local_prof = ru.Profiler(
                name='radical.entk.%s' % self._uid + '-proc', path=self._path)

            local_prof.prof('tmgr process started', uid=self._uid)
            logger.info('Task Manager process started')

            # Thread should run till terminate condtion is encountered
            mq_connection = pika.BlockingConnection(
                pika.ConnectionParameters(host=mq_hostname, port=port))
            mq_channel = mq_connection.channel()

            # Queue for communication between threads of this process
            task_queue = Queue.Queue()

            # Start second thread to receive tasks and push to RTS
            self._rts_runner = threading.Thread(target=self._process_tasks,
                                                args=(task_queue,
                                                      rmgr,
                                                      logger,
                                                      mq_hostname,
                                                      port,
                                                      local_prof,
                                                      self._sid))
            self._rts_runner.start()

            local_prof.prof('tmgr infrastructure setup done', uid=uid)

            last = time.time()
            while not self._tmgr_terminate.is_set():

                try:

                    method_frame, header_frame, body = mq_channel.basic_get(
                        queue=pending_queue[0])

                    if body:

                        body = json.loads(body)
                        task_queue.put(body)

                        mq_channel.basic_ack(
                            delivery_tag=method_frame.delivery_tag)

                    heartbeat_response(mq_channel)

                except Exception, ex:
                    logger.exception('Error in task execution: %s' % ex)
                    raise

        except KeyboardInterrupt:

            logger.exception('Execution interrupted by user (you probably hit Ctrl+C), ' +
                               'trying to cancel tmgr process gracefully...')

        except Exception, ex:

            logger.exception('%s failed with %s'%(self._uid, ex))
            raise EnTKError(ex)

        finally:

            local_prof.prof('terminating tmgr process', uid=uid)

            if self._rts_runner:
                self._rts_runner.join()

            mq_connection.close()
            local_prof.close()

Response: This method is used to run the App Manager process. It is called by the App Manager process. It is called by the App Manager process to process the Task from the pending_queue and the Task from the completed_queue.


Instruction:
Can you write a function in Python 3 where it
starts the task manager process.

Response:
def start_manager(self):
        """
        **Purpose**: Method to start the tmgr process. The tmgr function
        is not to be accessed directly. The function is started in a separate
        thread using this method.
        """

        if not self._tmgr_process:

            try:

                self._prof.prof('creating tmgr process', uid=self._uid)
                self._tmgr_terminate = Event()

                self._tmgr_process = Process(target=self._tmgr,
                                             name='task-manager',
                                             args=(
                                                 self._uid,
                                                 self._rmgr,
                                                 self._logger,
                                                 self._mq_hostname,
                                                 self._port,
                                                 self._pending_queue,
                                                 self._completed_queue)
                                             )

                self._logger.info('Starting task manager process')
                self._prof.prof('starting tmgr process', uid=self._uid)
                self._tmgr_process.start()

                return True

            except Exception, ex:

                self._logger.exception('Task manager not started, error: %s' % ex)
                self.terminate_manager()
                raise

        else:
            self._logger.warn(
                'tmgr process already running, but attempted to restart!')


Instruction:
Write a Python 3 script for
casing - insensitive keyword match.

Response:
def keyword(tokens, expected):
    """Case-insensitive keyword match."""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == "symbol" and token.value.lower() == expected:
        return TokenMatch(None, token.value, (token,))


Instruction:
How would you explain what the following Python 3 function does
def multi_keyword(tokens, keyword_parts):
    tokens = iter(tokens)
    matched_tokens = []
    limit = len(keyword_parts)

    for idx in six.moves.range(limit):
        try:
            token = next(tokens)
        except StopIteration:
            return

        if (not token or token.name != "symbol" or
                token.value.lower() != keyword_parts[idx]):
            return

        matched_tokens.append(token)

    return TokenMatch(None, token.value, matched_tokens)

Response: Match a case - insensitive keyword consisting of multiple tokens.


Instruction:
Implement a function in Python 3 to
match a prefix of an operator.

Response:
def prefix(tokens, operator_table):
    """Match a prefix of an operator."""
    operator, matched_tokens = operator_table.prefix.match(tokens)
    if operator:
        return TokenMatch(operator, None, matched_tokens)


Instruction:
Explain what the following Python 3 code does
def infix(tokens, operator_table):
    operator, matched_tokens = operator_table.infix.match(tokens)
    if operator:
        return TokenMatch(operator, None, matched_tokens)

Response: Match an infix of an operator.


Instruction:
How would you implement a function in Python 3 that
matches a suffix of an operator.

Response:
def suffix(tokens, operator_table):
    """Match a suffix of an operator."""
    operator, matched_tokens = operator_table.suffix.match(tokens)
    if operator:
        return TokenMatch(operator, None, matched_tokens)


Instruction:
Write a Python 3 function for
matching a token name ( type.

Response:
def token_name(tokens, expected):
    """Match a token name (type)."""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == expected:
        return TokenMatch(None, token.value, (token,))