Instruction:
Write a Python 3 script for
parsing the version data license name SPDX Identifier namespace and comment.

Response:
def parse_doc_fields(self, doc_term):
        """Parses the version, data license, name, SPDX Identifier, namespace,
        and comment."""
        try:
            self.builder.set_doc_spdx_id(self.doc, doc_term)
        except SPDXValueError:
            self.value_error('DOC_SPDX_ID_VALUE', doc_term)
        try:
            if doc_term.count('#', 0, len(doc_term)) <= 1:
                doc_namespace = doc_term.split('#')[0]
                self.builder.set_doc_namespace(self.doc, doc_namespace)
            else:
                self.value_error('DOC_NAMESPACE_VALUE', doc_term)
        except SPDXValueError:
            self.value_error('DOC_NAMESPACE_VALUE', doc_term)
        for _s, _p, o in self.graph.triples((doc_term, self.spdx_namespace['specVersion'], None)):
            try:
                self.builder.set_doc_version(self.doc, six.text_type(o))
            except SPDXValueError:
                self.value_error('DOC_VERS_VALUE', o)
            except CardinalityError:
                self.more_than_one_error('specVersion')
                break
        for _s, _p, o in self.graph.triples((doc_term, self.spdx_namespace['dataLicense'], None)):
            try:
                self.builder.set_doc_data_lic(self.doc, six.text_type(o))
            except SPDXValueError:
                self.value_error('DOC_D_LICS', o)
            except CardinalityError:
                self.more_than_one_error('dataLicense')
                break
        for _s, _p, o in self.graph.triples(
                (doc_term, self.spdx_namespace['name'], None)):
            try:
                self.builder.set_doc_name(self.doc, six.text_type(o))
            except CardinalityError:
                self.more_than_one_error('name')
                break
        for _s, _p, o in self.graph.triples((doc_term, RDFS.comment, None)):
            try:
                self.builder.set_doc_comment(self.doc, six.text_type(o))
            except CardinalityError:
                self.more_than_one_error('Document comment')
                break


Instruction:
Explain what the following Python 3 code does
def parse_ext_doc_ref(self, ext_doc_ref_term):
        for _s, _p, o in self.graph.triples(
                (ext_doc_ref_term,
                 self.spdx_namespace['externalDocumentId'],
                 None)):
            try:
                self.builder.set_ext_doc_id(self.doc, six.text_type(o))
            except SPDXValueError:
                self.value_error('EXT_DOC_REF_VALUE', 'External Document ID')
                break

        for _s, _p, o in self.graph.triples(
                (ext_doc_ref_term,
                 self.spdx_namespace['spdxDocument'],
                 None)):
            try:
                self.builder.set_spdx_doc_uri(self.doc, six.text_type(o))
            except SPDXValueError:
                self.value_error('EXT_DOC_REF_VALUE', 'SPDX Document URI')
                break

        for _s, _p, checksum in self.graph.triples(
                (ext_doc_ref_term, self.spdx_namespace['checksum'], None)):
            for _, _, value in self.graph.triples(
                    (checksum, self.spdx_namespace['checksumValue'], None)):
                try:
                    self.builder.set_chksum(self.doc, six.text_type(value))
                except SPDXValueError:
                    self.value_error('EXT_DOC_REF_VALUE', 'Checksum')
                    break

Response: Parses the external document reference term and sets the attributes of the external document object.


Instruction:
Create a Python 3 function for
validating the package fields.

Response:
def validate(self, messages):
        """
        Validate the package fields.
        Append user friendly error messages to the `messages` list.
        """
        messages = self.validate_checksum(messages)
        messages = self.validate_optional_str_fields(messages)
        messages = self.validate_mandatory_str_fields(messages)
        messages = self.validate_files(messages)
        messages = self.validate_mandatory_fields(messages)
        messages = self.validate_optional_fields(messages)

        return messages


Instruction:
Create a Python 3 function for
validating that the fields marked as optional and of type string in class class docstring are of type string in class class docstring.

Response:
def validate_optional_str_fields(self, messages):
        """Fields marked as optional and of type string in class
        docstring must be of a type that provides __str__ method.
        """
        FIELDS = [
            'file_name',
            'version',
            'homepage',
            'source_info',
            'summary',
            'description'
        ]
        messages = self.validate_str_fields(FIELDS, True, messages)

        return messages


Instruction:
Can you write a function in Python 3 where it
validates that all mandatory fields in class are of type string in class docstring.

Response:
def validate_mandatory_str_fields(self, messages):
        """Fields marked as Mandatory and of type string in class
        docstring must be of a type that provides __str__ method.
        """
        FIELDS = ['name', 'download_location', 'verif_code', 'cr_text']
        messages = self.validate_str_fields(FIELDS, False, messages)

        return messages


Instruction:
Here you have a function in Python 3, explain what it does
def validate_str_fields(self, fields, optional, messages):
        for field_str in fields:
            field = getattr(self, field_str)
            if field is not None:
                # FIXME: this does not make sense???
                attr = getattr(field, '__str__', None)
                if not callable(attr):
                    messages = messages + [
                        '{0} must provide __str__ method.'.format(field)
                    ]
                    # Continue checking.
            elif not optional:
                messages = messages + [
                    'Package {0} can not be None.'.format(field_str)
                ]

        return messages

Response: Helper for validate_mandatory_str_field and validate_optional_str_fields


Instruction:
Implement a function in Python 3 to
set the document data license.

Response:
def set_doc_data_lic(self, doc, res):
        """
        Set the document data license.
        Raise exceptions:
        - SPDXValueError if malformed value,
        - CardinalityError if already defined.
        """
        if not self.doc_data_lics_set:
            self.doc_data_lics_set = True
            # TODO: what is this split?
            res_parts = res.split('/')
            if len(res_parts) != 0:
                identifier = res_parts[-1]
                doc.data_license = document.License.from_identifier(identifier)
            else:
                raise SPDXValueError('Document::License')
        else:
            raise CardinalityError('Document::License')


Instruction:
Implement a function in Python 3 to
set the comment of the document. Raises CardinalityError if comment already set.

Response:
def set_doc_comment(self, doc, comment):
        """Sets document comment, Raises CardinalityError if
        comment already set.
        """
        if not self.doc_comment_set:
            self.doc_comment_set = True
            doc.comment = comment
        else:
            raise CardinalityError('Document::Comment')


Instruction:
Create a Python 3 function to
set the check sum of the external document reference if not already set.

Response:
def set_chksum(self, doc, chk_sum):
        """
        Sets the external document reference's check sum, if not already set.
        chk_sum - The checksum value in the form of a string.
        """
        if chk_sum:
            doc.ext_document_references[-1].check_sum = checksum.Algorithm(
                'SHA1', chk_sum)
        else:
            raise SPDXValueError('ExternalDocumentRef::Checksum')


Instruction:
Implement a Python 3 function for
setting the creation comment for the current object.

Response:
def set_creation_comment(self, doc, comment):
        """Sets creation comment, Raises CardinalityError if
        comment already set.
        Raises SPDXValueError if not free form text.
        """
        if not self.creation_comment_set:
            self.creation_comment_set = True
            doc.creation_info.comment = comment
            return True
        else:
            raise CardinalityError('CreationInfo::Comment')


Instruction:
Here you have a function in Python 3, explain what it does
def set_pkg_chk_sum(self, doc, chk_sum):
        self.assert_package_exists()
        if not self.package_chk_sum_set:
            self.package_chk_sum_set = True
            doc.package.check_sum = checksum.Algorithm('SHA1', chk_sum)
        else:
            raise CardinalityError('Package::CheckSum')

Response: Sets the package check sum.


Instruction:
Can you implement a function in Python 3 that
sets the package s source information if not already set.

Response:
def set_pkg_source_info(self, doc, text):
        """Sets the package's source information, if not already set.
        text - Free form text.
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_source_info_set:
            self.package_source_info_set = True
            doc.package.source_info = text
            return True
        else:
            raise CardinalityError('Package::SourceInfo')


Instruction:
Make a summary of the following Python 3 code
def set_pkg_verif_code(self, doc, code):
        self.assert_package_exists()
        if not self.package_verif_set:
            self.package_verif_set = True
            doc.package.verif_code = code
        else:
            raise CardinalityError('Package::VerificationCode')

Response: Sets the verification code for the given package.


Instruction:
How would you explain what the following Python 3 function does
def set_pkg_excl_file(self, doc, filename):
        self.assert_package_exists()
        doc.package.add_exc_file(filename)

Response: Sets the verification code excluded file. Raises OrderError if no package was defined.


Instruction:
Here you have a function in Python 3, explain what it does
def set_pkg_license_comment(self, doc, text):
        self.assert_package_exists()
        if not self.package_license_comment_set:
            self.package_license_comment_set = True
            doc.package.license_comment = text
            return True
        else:
            raise CardinalityError('Package::LicenseComment')

Response: Sets the package s license comment.


Instruction:
Given the following Python 3 function, write the documentation
def set_pkg_cr_text(self, doc, text):
        self.assert_package_exists()
        if not self.package_cr_text_set:
            self.package_cr_text_set = True
            doc.package.cr_text = text
        else:
            raise CardinalityError('Package::CopyrightText')

Response: Sets the package s license comment text.


Instruction:
How would you explain what the following Python 3 function does
def set_pkg_summary(self, doc, text):
        self.assert_package_exists()
        if not self.package_summary_set:
            self.package_summary_set = True
            doc.package.summary = text
        else:
            raise CardinalityError('Package::Summary')

Response: Set the package summary.


Instruction:
Can you implement a function in Python 3 that
sets the package s description.

Response:
def set_pkg_desc(self, doc, text):
        """Set's the package's description.
        Raises CardinalityError if description already set.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_desc_set:
            self.package_desc_set = True
            doc.package.description = text
        else:
            raise CardinalityError('Package::Description')


Instruction:
How would you implement a function in Python 3 that
sets the file check sum for the given document.

Response:
def set_file_chksum(self, doc, chk_sum):
        """Sets the file check sum, if not already set.
        chk_sum - A string
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        """
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_chksum_set:
                self.file_chksum_set = True
                self.file(doc).chk_sum = checksum.Algorithm('SHA1', chk_sum)
                return True
            else:
                raise CardinalityError('File::CheckSum')
        else:
            raise OrderError('File::CheckSum')


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_file_license_comment(self, doc, text):
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_license_comment_set:
                self.file_license_comment_set = True
                self.file(doc).license_comment = text
                return True
            else:
                raise CardinalityError('File::LicenseComment')
        else:
            raise OrderError('File::LicenseComment')

Response: Sets the license comment for the file.


Instruction:
Write a Python 3 script for
setting copyright text for a file in the order.

Response:
def set_file_copyright(self, doc, text):
        """Raises OrderError if no package or file defined.
        Raises CardinalityError if more than one.
        """
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_copytext_set:
                self.file_copytext_set = True
                self.file(doc).copyright = text
                return True
            else:
                raise CardinalityError('File::CopyRight')
        else:
            raise OrderError('File::CopyRight')


Instruction:
How would you explain what the following Python 3 function does
def set_file_comment(self, doc, text):
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_comment_set:
                self.file_comment_set = True
                self.file(doc).comment = text
                return True
            else:
                raise CardinalityError('File::Comment')
        else:
            raise OrderError('File::Comment')

Response: Sets the comment for the file.


Instruction:
How would you explain what the following Python 3 function does
def set_file_notice(self, doc, text):
        if self.has_package(doc) and self.has_file(doc):
            if not self.file_notice_set:
                self.file_notice_set = True
                self.file(doc).notice = tagvaluebuilders.str_from_text(text)
                return True
            else:
                raise CardinalityError('File::Notice')
        else:
            raise OrderError('File::Notice')

Response: Sets the notice field in the file object.


Instruction:
Make a summary of the following Python 3 code
def add_annotation_type(self, doc, annotation_type):
        if len(doc.annotations) != 0:
            if not self.annotation_type_set:
                if annotation_type.endswith('annotationType_other'):
                    self.annotation_type_set = True
                    doc.annotations[-1].annotation_type = 'OTHER'
                    return True
                elif annotation_type.endswith('annotationType_review'):
                    self.annotation_type_set = True
                    doc.annotations[-1].annotation_type = 'REVIEW'
                    return True
                else:
                    raise SPDXValueError('Annotation::AnnotationType')
            else:
                raise CardinalityError('Annotation::AnnotationType')
        else:
            raise OrderError('Annotation::AnnotationType')

Response: Sets the annotation type. Raises CardinalityError if no annotator defined before.


Instruction:
Explain what the following Python 3 code does
def validate(self, messages):
        messages = self.validate_version(messages)
        messages = self.validate_data_lics(messages)
        messages = self.validate_name(messages)
        messages = self.validate_spdx_id(messages)
        messages = self.validate_namespace(messages)
        messages = self.validate_ext_document_references(messages)
        messages = self.validate_creation_info(messages)
        messages = self.validate_package(messages)
        messages = self.validate_extracted_licenses(messages)
        messages = self.validate_reviews(messages)

        return messages

Response: Validate all fields of the document and update the list with user friendly error messages for display.


Instruction:
Here you have a function in Python 3, explain what it does
def include(f):
  '''
  includes the contents of a file on disk.
  takes a filename
  '''
  fl = open(f, 'r')
  data = fl.read()
  fl.close()
  return raw(data)

Response: includes the contents of a file on disk.


Instruction:
How would you implement a function in Python 3 that
pipes the output of a program

Response:
def system(cmd, data=None):
  '''
  pipes the output of a program
  '''
  import subprocess
  s = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
  out, err = s.communicate(data)
  return out.decode('utf8')


Instruction:
Make a summary of the following Python 3 code
def escape(data, quote=True):  # stoled from std lib cgi
  '''
  Escapes special characters into their html entities
  Replace special characters "&", "<" and ">" to HTML-safe sequences.
  If the optional flag quote is true, the quotation mark character (")
  is also translated.

  This is used to escape content that appears in the body of an HTML cocument
  '''
  data = data.replace("&", "&")  # Must be done first!
  data = data.replace("<", "<")
  data = data.replace(">", ">")
  if quote:
    data = data.replace('"', """)
  return data

Response: Escapes special characters into HTML - safe sequences.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def unescape(data):
  '''
  unescapes html entities. the opposite of escape.
  '''
  cc = re.compile(r'&(?:(?:#(\d+))|([^;]+));')

  result = []
  m = cc.search(data)
  while m:
    result.append(data[0:m.start()])
    d = m.group(1)
    if d:
      d = int(d)
      result.append(unichr(d))
    else:
      d = _unescape.get(m.group(2), ord('?'))
      result.append(unichr(d))

    data = data[m.end():]
    m = cc.search(data)

  result.append(data)
  return ''.join(result)

Response:
unescapes html entities. the opposite of escape.
 « unescapes html entities. the opposite of escape.
 »


Instruction:
Write a Python 3 script for
setting attributes on the current active tag context ethernet.

Response:
def attr(*args, **kwargs):
  '''
  Set attributes on the current active tag context
  '''
  ctx = dom_tag._with_contexts[_get_thread_context()]
  if ctx and ctx[-1]:
    dicts = args + (kwargs,)
    for d in dicts:
      for attr, value in d.items():
        ctx[-1].tag.set_attribute(*dom_tag.clean_pair(attr, value))
  else:
    raise ValueError('not in a tag context')


Instruction:
Can you generate the documentation for the following Python 3 function
def set_attribute(self, key, value):
    '''
    Add or update the value of an attribute.
    '''
    if isinstance(key, int):
      self.children[key] = value
    elif isinstance(key, basestring):
      self.attributes[key] = value
    else:
      raise TypeError('Only integer and string types are valid for assigning '
          'child tags and attributes, respectively.')

Response: Add or update the value of an attribute.


Instruction:
Implement a Python 3 function for
setting the document of the node.

Response:
def setdocument(self, doc):
    '''
    Creates a reference to the parent document to allow for partial-tree
    validation.
    '''
    # assume that a document is correct in the subtree
    if self.document != doc:
      self.document = doc
      for i in self.children:
        if not isinstance(i, dom_tag): return
        i.setdocument(doc)


Instruction:
Can you write a function in Python 3 where it
adds new child tags.

Response:
def add(self, *args):
    '''
    Add new child tags.
    '''
    for obj in args:
      if isinstance(obj, numbers.Number):
        # Convert to string so we fall into next if block
        obj = str(obj)

      if isinstance(obj, basestring):
        obj = escape(obj)
        self.children.append(obj)

      elif isinstance(obj, dom_tag):
        ctx = dom_tag._with_contexts[_get_thread_context()]
        if ctx and ctx[-1]:
          ctx[-1].used.add(obj)
        self.children.append(obj)
        obj.parent = self
        obj.setdocument(self.document)

      elif isinstance(obj, dict):
        for attr, value in obj.items():
          self.set_attribute(*dom_tag.clean_pair(attr, value))

      elif hasattr(obj, '__iter__'):
        for subobj in obj:
          self.add(subobj)

      else:  # wtf is it?
        raise ValueError('%r not a tag or string.' % obj)

    if len(args) == 1:
      return args[0]

    return args


Instruction:
Write a Python 3 function that can
clean the attribute name for shorthand and work arounds for limitations in Python s syntax

Response:
def clean_attribute(attribute):
    '''
    Normalize attribute names for shorthand and work arounds for limitations
    in Python's syntax
    '''

    # Shorthand
    attribute = {
      'cls': 'class',
      'className': 'class',
      'class_name': 'class',
      'fr': 'for',
      'html_for': 'for',
      'htmlFor': 'for',
    }.get(attribute, attribute)

    # Workaround for Python's reserved words
    if attribute[0] == '_':
      attribute = attribute[1:]

    # Workaround for dash
    if attribute in set(['http_equiv']) or attribute.startswith('data_'):
      attribute = attribute.replace('_', '-').lower()

    # Workaround for colon
    if attribute.split('_')[0] in ('xlink', 'xml', 'xmlns'):
      attribute = attribute.replace('_', ':', 1).lower()

    return attribute


Instruction:
Can you generate the documentation for the following Python 3 function
def clean_pair(cls, attribute, value):
    '''
    This will call `clean_attribute` on the attribute and also allows for the
    creation of boolean attributes.

    Ex. input(selected=True) is equivalent to input(selected="selected")
    '''
    attribute = cls.clean_attribute(attribute)

    # Check for boolean attributes
    # (i.e. selected=True becomes selected="selected")
    if value is True:
      value = attribute

    if value is False:
      value = "false"

    return (attribute, value)

Response: This method will call clean_attribute on the attribute and also check for boolean attributes.


Instruction:
Can you write a function in Python 3 where it
creates a new empty tag if not present and renders the DOCTYPE and tag tree.

Response:
def render(self, *args, **kwargs):
    '''
    Creates a <title> tag if not present and renders the DOCTYPE and tag tree.
    '''
    r = []

    #Validates the tag tree and adds the doctype if one was set
    if self.doctype:
      r.append(self.doctype)
      r.append('\n')
    r.append(super(document, self).render(*args, **kwargs))

    return u''.join(r)


Instruction:
How would you code a function in Python 3 to
get element with matching id value.

Response:
def getElementById(self, id):
    '''
    DOM API: Returns single element with matching id value.
    '''
    results = self.get(id=id)
    if len(results) > 1:
      raise ValueError('Multiple tags with id "%s".' % id)
    elif results:
      return results[0]
    else:
      return None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def getElementsByTagName(self, name):
    '''
    DOM API: Returns all tags that match name.
    '''
    if isinstance(name, basestring):
      return self.get(name.lower())
    else:
      return None

Response: getElementsByTagName - Returns all tags that match name.


Instruction:
Explain what the following Python 3 code does
def start(self):
        self.outgoing_q = zmq_pipes.TasksOutgoing(
            "127.0.0.1", self.interchange_port_range)
        self.incoming_q = zmq_pipes.ResultsIncoming(
            "127.0.0.1", self.interchange_port_range)

        self.is_alive = True

        self._queue_management_thread = None
        self._start_queue_management_thread()
        self._start_local_queue_process()

        logger.debug("Created management thread: {}"
                     .format(self._queue_management_thread))

        if self.provider:
            # debug_opts = "--debug" if self.worker_debug else ""
            l_cmd = self.launch_cmd.format(  # debug=debug_opts,
                task_url=self.worker_task_url,
                workers_per_node=self.workers_per_node,
                logdir="{}/{}".format(self.run_dir, self.label))
            self.launch_cmd = l_cmd
            logger.debug("Launch command: {}".format(self.launch_cmd))

            self._scaling_enabled = self.provider.scaling_enabled
            logger.debug(
                "Starting LowLatencyExecutor with provider:\n%s", self.provider)
            if hasattr(self.provider, 'init_blocks'):
                try:
                    for i in range(self.provider.init_blocks):
                        block = self.provider.submit(
                            self.launch_cmd, 1, self.workers_per_node)
                        logger.debug("Launched block {}:{}".format(i, block))
                        if not block:
                            raise(ScalingFailed(self.provider.label,
                                                "Attempts to provision nodes via provider has failed"))
                        self.blocks.extend([block])

                except Exception as e:
                    logger.error("Scaling out failed: {}".format(e))
                    raise e
        else:
            self._scaling_enabled = False
            logger.debug("Starting LowLatencyExecutor with no provider")

Response: Start the Interchange process and connect to it.


Instruction:
Create a Python 3 function for
starting the local queue process.

Response:
def _start_local_queue_process(self):
        """ TODO: docstring """

        comm_q = Queue(maxsize=10)
        self.queue_proc = Process(target=interchange.starter,
                                  args=(comm_q,),
                                  kwargs={"client_ports": (self.outgoing_q.port,
                                                           self.incoming_q.port),
                                          "worker_port": self.worker_port,
                                          "worker_port_range": self.worker_port_range
                                          # TODO: logdir and logging level
                                          })
        self.queue_proc.start()

        try:
            worker_port = comm_q.get(block=True, timeout=120)
            logger.debug(
                "Got worker port {} from interchange".format(worker_port))
        except queue.Empty:
            logger.error(
                "Interchange has not completed initialization in 120s. Aborting")
            raise Exception("Interchange failed to start")

        self.worker_task_url = "tcp://{}:{}".format(
            self.address, worker_port)


Instruction:
Make a summary of the following Python 3 code
def _start_queue_management_thread(self):
        if self._queue_management_thread is None:
            logger.debug("Starting queue management thread")
            self._queue_management_thread = threading.Thread(
                target=self._queue_management_worker)
            self._queue_management_thread.daemon = True
            self._queue_management_thread.start()
            logger.debug("Started queue management thread")

        else:
            logger.debug("Management thread already exists, returning")

Response: Start the queue management thread.


Instruction:
Here you have a function in Python 3, explain what it does
def _queue_management_worker(self):
        logger.debug("[MTHREAD] queue management worker starting")

        while True:
            task_id, buf = self.incoming_q.get()  # TODO: why does this hang?
            msg = deserialize_object(buf)[0]
            # TODO: handle exceptions
            task_fut = self.tasks[task_id]
            logger.debug("Got response for task id {}".format(task_id))

            if "result" in msg:
                task_fut.set_result(msg["result"])

            elif "exception" in msg:
                # TODO: handle exception
                pass
            elif 'exception' in msg:
                logger.warning("Task: {} has returned with an exception")
                try:
                    s, _ = deserialize_object(msg['exception'])
                    exception = ValueError("Remote exception description: {}".format(s))
                    task_fut.set_exception(exception)
                except Exception as e:
                    # TODO could be a proper wrapped exception?
                    task_fut.set_exception(
                        DeserializationError("Received exception, but handling also threw an exception: {}".format(e)))

            else:
                raise BadMessage(
                    "Message received is neither result nor exception")

            if not self.is_alive:
                break

        logger.info("[MTHREAD] queue management worker finished")

Response: This function is the main thread that handles the queue management.


Instruction:
Here you have a function in Python 3, explain what it does
def scale_in(self, blocks):
        to_kill = self.blocks[:blocks]
        if self.provider:
            r = self.provider.cancel(to_kill)
        return r

Response: Scale in the number of active blocks by specified amount.


Instruction:
Write a Python 3 function that can
create a JSON message to identify the worker to the interchange

Response:
def create_reg_message(self):
        """ Creates a registration message to identify the worker to the interchange
        """
        msg = {'parsl_v': PARSL_VERSION,
               'python_v': "{}.{}.{}".format(sys.version_info.major,
                                             sys.version_info.minor,
                                             sys.version_info.micro),
               'os': platform.system(),
               'hname': platform.node(),
               'dir': os.getcwd(),
        }
        b_msg = json.dumps(msg).encode('utf-8')
        return b_msg


Instruction:
Can you create a Python 3 function that
sends a heartbeat to the incoming task queue

Response:
def heartbeat(self):
        """ Send heartbeat to the incoming task queue
        """
        heartbeat = (HEARTBEAT_CODE).to_bytes(4, "little")
        r = self.task_incoming.send(heartbeat)
        logger.debug("Return from heartbeat : {}".format(r))


Instruction:
Write a Python 3 script to
receive a result from the workers and sends it out via 0mq

Response:
def recv_result_from_workers(self):
        """ Receives a results from the MPI worker pool and send it out via 0mq

        Returns:
        --------
            result: task result from the workers
        """
        info = MPI.Status()
        result = self.comm.recv(source=MPI.ANY_SOURCE, tag=RESULT_TAG, status=info)
        logger.debug("Received result from workers: {}".format(result))
        return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def recv_task_request_from_workers(self):
        info = MPI.Status()
        comm.recv(source=MPI.ANY_SOURCE, tag=TASK_REQUEST_TAG, status=info)
        worker_rank = info.Get_source()
        logger.info("Received task request from worker:{}".format(worker_rank))
        return worker_rank

Response:
Receives 1 task request from MPI comm

        Returns the worker_rank of the task request.


Instruction:
Write a Python 3 function for
pulling tasks from the incoming queue onto the internal task queue.

Response:
def pull_tasks(self, kill_event):
        """ Pulls tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK PULL THREAD] starting")
        poller = zmq.Poller()
        poller.register(self.task_incoming, zmq.POLLIN)

        # Send a registration message
        msg = self.create_reg_message()
        logger.debug("Sending registration message: {}".format(msg))
        self.task_incoming.send(msg)
        last_beat = time.time()
        last_interchange_contact = time.time()
        task_recv_counter = 0

        poll_timer = 1

        while not kill_event.is_set():
            time.sleep(LOOP_SLOWDOWN)
            ready_worker_count = self.ready_worker_queue.qsize()
            pending_task_count = self.pending_task_queue.qsize()

            logger.debug("[TASK_PULL_THREAD] ready workers:{}, pending tasks:{}".format(ready_worker_count,
                                                                                pending_task_count))

            if time.time() > last_beat + self.heartbeat_period:
                self.heartbeat()
                last_beat = time.time()

            if pending_task_count < self.max_queue_size and ready_worker_count > 0:
                logger.debug("[TASK_PULL_THREAD] Requesting tasks: {}".format(ready_worker_count))
                msg = ((ready_worker_count).to_bytes(4, "little"))
                self.task_incoming.send(msg)

            socks = dict(poller.poll(timeout=poll_timer))

            if self.task_incoming in socks and socks[self.task_incoming] == zmq.POLLIN:
                _, pkl_msg = self.task_incoming.recv_multipart()
                tasks = pickle.loads(pkl_msg)
                last_interchange_contact = time.time()

                if tasks == 'STOP':
                    logger.critical("[TASK_PULL_THREAD] Received stop request")
                    kill_event.set()
                    break

                elif tasks == HEARTBEAT_CODE:
                    logger.debug("Got heartbeat from interchange")

                else:
                    # Reset timer on receiving message
                    poll_timer = 1
                    task_recv_counter += len(tasks)
                    logger.debug("[TASK_PULL_THREAD] Got tasks: {} of {}".format([t['task_id'] for t in tasks],
                                                                                task_recv_counter))
                    for task in tasks:
                        self.pending_task_queue.put(task)
            else:
                logger.debug("[TASK_PULL_THREAD] No incoming tasks")
                # Limit poll duration to heartbeat_period
                # heartbeat_period is in s vs poll_timer in ms
                poll_timer = min(self.heartbeat_period * 1000, poll_timer * 2)

                # Only check if no messages were received.
                if time.time() > last_interchange_contact + self.heartbeat_threshold:
                    logger.critical("[TASK_PULL_THREAD] Missing contact with interchange beyond heartbeat_threshold")
                    kill_event.set()
                    logger.critical("[TASK_PULL_THREAD] Exiting")
                    break


Instruction:
Can you generate the documentation for the following Python 3 function
def push_results(self, kill_event):

        # We set this timeout so that the thread checks the kill_event and does not
        # block forever on the internal result queue
        timeout = 0.1
        # timer = time.time()
        logger.debug("[RESULT_PUSH_THREAD] Starting thread")

        while not kill_event.is_set():
            time.sleep(LOOP_SLOWDOWN)
            try:
                items = []
                while not self.pending_result_queue.empty():
                    r = self.pending_result_queue.get(block=True)
                    items.append(r)
                if items:
                    self.result_outgoing.send_multipart(items)

            except queue.Empty:
                logger.debug("[RESULT_PUSH_THREAD] No results to send in past {}seconds".format(timeout))

            except Exception as e:
                logger.exception("[RESULT_PUSH_THREAD] Got an exception : {}".format(e))

        logger.critical("[RESULT_PUSH_THREAD] Exiting")

Response: Sends out results to the result queue.


Instruction:
How would you code a function in Python 3 to
start the Manager process.

Response:
def start(self):
        """ Start the Manager process.

        The worker loops on this:

        1. If the last message sent was older than heartbeat period we send a heartbeat
        2.

        TODO: Move task receiving to a thread
        """

        self.comm.Barrier()
        logger.debug("Manager synced with workers")

        self._kill_event = threading.Event()
        self._task_puller_thread = threading.Thread(target=self.pull_tasks,
                                                    args=(self._kill_event,))
        self._result_pusher_thread = threading.Thread(target=self.push_results,
                                                      args=(self._kill_event,))
        self._task_puller_thread.start()
        self._result_pusher_thread.start()

        start = None

        result_counter = 0
        task_recv_counter = 0
        task_sent_counter = 0

        logger.info("Loop start")
        while not self._kill_event.is_set():
            time.sleep(LOOP_SLOWDOWN)

            # In this block we attempt to probe MPI for a set amount of time,
            # and if we have exhausted all available MPI events, we move on
            # to the next block. The timer and counter trigger balance
            # fairness and responsiveness.
            timer = time.time() + 0.05
            counter = min(10, comm.size)
            while time.time() < timer:
                info = MPI.Status()

                if counter > 10:
                    logger.debug("Hit max mpi events per round")
                    break

                if not self.comm.Iprobe(status=info):
                    logger.debug("Timer expired, processed {} mpi events".format(counter))
                    break
                else:
                    tag = info.Get_tag()
                    logger.info("Message with tag {} received".format(tag))

                    counter += 1
                    if tag == RESULT_TAG:
                        result = self.recv_result_from_workers()
                        self.pending_result_queue.put(result)
                        result_counter += 1

                    elif tag == TASK_REQUEST_TAG:
                        worker_rank = self.recv_task_request_from_workers()
                        self.ready_worker_queue.put(worker_rank)

                    else:
                        logger.error("Unknown tag {} - ignoring this message and continuing".format(tag))

            available_worker_cnt = self.ready_worker_queue.qsize()
            available_task_cnt = self.pending_task_queue.qsize()
            logger.debug("[MAIN] Ready workers: {} Ready tasks: {}".format(available_worker_cnt,
                                                                           available_task_cnt))
            this_round = min(available_worker_cnt, available_task_cnt)
            for i in range(this_round):
                worker_rank = self.ready_worker_queue.get()
                task = self.pending_task_queue.get()
                comm.send(task, dest=worker_rank, tag=worker_rank)
                task_sent_counter += 1
                logger.debug("Assigning worker:{} task:{}".format(worker_rank, task['task_id']))

            if not start:
                start = time.time()

            logger.debug("Tasks recvd:{} Tasks dispatched:{} Results recvd:{}".format(
                task_recv_counter, task_sent_counter, result_counter))
            # print("[{}] Received: {}".format(self.identity, msg))
            # time.sleep(random.randint(4,10)/10)

        self._task_puller_thread.join()
        self._result_pusher_thread.join()

        self.task_incoming.close()
        self.result_outgoing.close()
        self.context.term()

        delta = time.time() - start
        logger.info("mpi_worker_pool ran for {} seconds".format(delta))


Instruction:
Can you tell what is the following Python 3 function doing
def udp_messenger(domain_name, UDP_IP, UDP_PORT, sock_timeout, message):
    try:
        if message is None:
            raise ValueError("message was none")

        encoded_message = bytes(message, "utf-8")

        if encoded_message is None:
            raise ValueError("utf-8 encoding of message failed")

        if domain_name:
            try:
                UDP_IP = socket.gethostbyname(domain_name)
            except Exception:
                # (False, "Domain lookup failed, defaulting to {0}".format(UDP_IP))
                pass

        if UDP_IP is None:
            raise Exception("UDP_IP is None")

        if UDP_PORT is None:
            raise Exception("UDP_PORT is None")

        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
        sock.settimeout(sock_timeout)
        sock.sendto(bytes(message, "utf-8"), (UDP_IP, UDP_PORT))
        sock.close()

    except socket.timeout:
        logger.debug("Failed to send usage tracking data: socket timeout")
    except OSError as e:
        logger.debug("Failed to send usage tracking data: OSError: {}".format(e))
    except Exception as e:
        logger.debug("Failed to send usage tracking data: Exception: {}".format(e))

Response: Send a message to the UDP socket.


Instruction:
Explain what the following Python 3 code does
def check_tracking_enabled(self):
        track = True   # By default we track usage
        test = False  # By default we are not in testing mode

        testvar = str(os.environ.get("PARSL_TESTING", 'None')).lower()
        if testvar == 'true':
            test = True

        if not self.config.usage_tracking:
            track = False

        envvar = str(os.environ.get("PARSL_TRACKING", True)).lower()
        if envvar == "false":
            track = False

        return test, track

Response: Returns True if tracking is enabled False if tracking is disabled


Instruction:
Implement a function in Python 3 to
construct a json string that contains the start of the DFK.

Response:
def construct_start_message(self):
        """Collect preliminary run info at the start of the DFK.

        Returns :
              - Message dict dumped as json string, ready for UDP
        """
        uname = getpass.getuser().encode('latin1')
        hashed_username = hashlib.sha256(uname).hexdigest()[0:10]
        hname = socket.gethostname().encode('latin1')
        hashed_hostname = hashlib.sha256(hname).hexdigest()[0:10]
        message = {'uuid': self.uuid,
                   'uname': hashed_username,
                   'hname': hashed_hostname,
                   'test': self.test_mode,
                   'parsl_v': self.parsl_version,
                   'python_v': self.python_version,
                   'os': platform.system(),
                   'os_v': platform.release(),
                   'start': time.time()}

        return json.dumps(message)


Instruction:
Can you write a function in Python 3 where it
constructs the final run information at the time of DFK cleanup.

Response:
def construct_end_message(self):
        """Collect the final run information at the time of DFK cleanup.

        Returns:
             - Message dict dumped as json string, ready for UDP
        """
        app_count = self.dfk.task_count

        site_count = len([x for x in self.dfk.config.executors if x.managed])

        app_fails = len([t for t in self.dfk.tasks if
                         self.dfk.tasks[t]['status'] in FINAL_FAILURE_STATES])

        message = {'uuid': self.uuid,
                   'end': time.time(),
                   't_apps': app_count,
                   'sites': site_count,
                   'c_time': None,
                   'failed': app_fails,
                   'test': self.test_mode,
                   }

        return json.dumps(message)


Instruction:
Implement a Python 3 function for
sending a UDP message.

Response:
def send_UDP_message(self, message):
        """Send UDP message."""
        x = 0
        if self.tracking_enabled:
            try:
                proc = udp_messenger(self.domain_name, self.UDP_IP, self.UDP_PORT, self.sock_timeout, message)
                self.procs.append(proc)
            except Exception as e:
                logger.debug("Usage tracking failed: {}".format(e))
        else:
            x = -1

        return x


Instruction:
Can you tell what is the following Python 3 function doing
def send_message(self):
        start = time.time()
        message = None
        if not self.initialized:
            message = self.construct_start_message()
            self.initialized = True
        else:
            message = self.construct_end_message()

        self.send_UDP_message(message)
        end = time.time()

        return end - start

Response: Send a message over UDP.


Instruction:
Can you create a Python 3 function that
adds a stream log handler to the log file.

Response:
def set_file_logger(filename: str, name: str = 'parsl', level: int = logging.DEBUG, format_string: Optional[str] = None):
    """Add a stream log handler.

    Args:
        - filename (string): Name of the file to write logs to
        - name (string): Logger name
        - level (logging.LEVEL): Set the logging level.
        - format_string (string): Set the format string

    Returns:
       -  None
    """
    if format_string is None:
        format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s]  %(message)s"

    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    handler = logging.FileHandler(filename)
    handler.setLevel(level)
    formatter = logging.Formatter(format_string, datefmt='%Y-%m-%d %H:%M:%S')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    # see note in set_stream_logger for notes about logging
    # concurrent.futures
    futures_logger = logging.getLogger("concurrent.futures")
    futures_logger.addHandler(handler)


Instruction:
Can you tell what is the following Python 3 function doing
def start_file_logger(filename, name='database_manager', level=logging.DEBUG, format_string=None):
    if format_string is None:
        format_string = "%(asctime)s %(name)s:%(lineno)d [%(levelname)s]  %(message)s"

    global logger
    logger = logging.getLogger(name)
    logger.setLevel(level)
    handler = logging.FileHandler(filename)
    handler.setLevel(level)
    formatter = logging.Formatter(format_string, datefmt='%Y-%m-%d %H:%M:%S')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    return logger

Response: Start a logger for a file.


Instruction:
Can you write a function in Python 3 where it
starts the database manager process

Response:
def dbm_starter(priority_msgs, resource_msgs, *args, **kwargs):
    """Start the database manager process

    The DFK should start this function. The args, kwargs match that of the monitoring config

    """
    dbm = DatabaseManager(*args, **kwargs)
    dbm.start(priority_msgs, resource_msgs)


Instruction:
How would you code a function in Python 3 to
start the migration thread.

Response:
def start(self, priority_queue, resource_queue):

        self._kill_event = threading.Event()
        self._priority_queue_pull_thread = threading.Thread(target=self._migrate_logs_to_internal,
                                                            args=(
                                                                priority_queue, 'priority', self._kill_event,)
                                                            )
        self._priority_queue_pull_thread.start()

        self._resource_queue_pull_thread = threading.Thread(target=self._migrate_logs_to_internal,
                                                            args=(
                                                                resource_queue, 'resource', self._kill_event,)
                                                            )
        self._resource_queue_pull_thread.start()

        """
        maintain a set to track the tasks that are already INSERTED into database
        to prevent race condition that the first resource message (indicate 'running' state)
        arrives before the first task message.
        If race condition happens, add to left_messages and operate them later

        """
        inserted_tasks = set()
        left_messages = {}

        while (not self._kill_event.is_set() or
               self.pending_priority_queue.qsize() != 0 or self.pending_resource_queue.qsize() != 0 or
               priority_queue.qsize() != 0 or resource_queue.qsize() != 0):

            """
            WORKFLOW_INFO and TASK_INFO messages

            """
            self.logger.debug("""Checking STOP conditions: {}, {}, {}, {}, {}""".format(
                              self._kill_event.is_set(),
                              self.pending_priority_queue.qsize() != 0, self.pending_resource_queue.qsize() != 0,
                              priority_queue.qsize() != 0, resource_queue.qsize() != 0))

            # This is the list of first resource messages indicating that task starts running
            first_messages = []

            # Get a batch of priority messages
            messages = self._get_messages_in_batch(self.pending_priority_queue,
                                                   interval=self.batching_interval,
                                                   threshold=self.batching_threshold)
            if messages:
                self.logger.debug(
                    "Got {} messages from priority queue".format(len(messages)))
                update_messages, insert_messages, all_messages = [], [], []
                for msg_type, msg in messages:
                    if msg_type.value == MessageType.WORKFLOW_INFO.value:
                        if "python_version" in msg:   # workflow start message
                            self.logger.debug(
                                "Inserting workflow start info to WORKFLOW table")
                            self._insert(table=WORKFLOW, messages=[msg])
                        else:                         # workflow end message
                            self.logger.debug(
                                "Updating workflow end info to WORKFLOW table")
                            self._update(table=WORKFLOW,
                                         columns=['run_id', 'tasks_failed_count',
                                                  'tasks_completed_count', 'time_completed',
                                                  'workflow_duration'],
                                         messages=[msg])
                    else:                             # TASK_INFO message
                        all_messages.append(msg)
                        if msg['task_time_returned'] is not None:
                            update_messages.append(msg)
                        else:
                            inserted_tasks.add(msg['task_id'])
                            insert_messages.append(msg)

                            # check if there is an left_message for this task
                            if msg['task_id'] in left_messages:
                                first_messages.append(
                                    left_messages.pop(msg['task_id']))

                self.logger.debug(
                    "Updating and inserting TASK_INFO to all tables")
                self._update(table=WORKFLOW,
                             columns=['run_id', 'tasks_failed_count',
                                      'tasks_completed_count'],
                             messages=update_messages)

                if insert_messages:
                    self._insert(table=TASK, messages=insert_messages)
                    self.logger.debug(
                        "There are {} inserted task records".format(len(inserted_tasks)))
                if update_messages:
                    self._update(table=TASK,
                                 columns=['task_time_returned',
                                          'task_elapsed_time', 'run_id', 'task_id'],
                                 messages=update_messages)
                self._insert(table=STATUS, messages=all_messages)

            """
            RESOURCE_INFO messages

            """
            messages = self._get_messages_in_batch(self.pending_resource_queue,
                                                   interval=self.batching_interval,
                                                   threshold=self.batching_threshold)

            if messages or first_messages:
                self.logger.debug(
                    "Got {} messages from resource queue".format(len(messages)))
                self._insert(table=RESOURCE, messages=messages)
                for msg in messages:
                    if msg['first_msg']:
                        msg['task_status_name'] = States.running.name
                        msg['task_time_running'] = msg['timestamp']
                        if msg['task_id'] in inserted_tasks:
                            first_messages.append(msg)
                        else:
                            left_messages[msg['task_id']] = msg
                if first_messages:
                    self._insert(table=STATUS, messages=first_messages)
                    self._update(table=TASK,
                                 columns=['task_time_running',
                                          'run_id', 'task_id'],
                                 messages=first_messages)


Instruction:
Can you generate the documentation for the following Python 3 function
def _create_task_log_info(self, task_id, fail_mode=None):

        info_to_monitor = ['func_name', 'fn_hash', 'memoize', 'checkpoint', 'fail_count',
                           'fail_history', 'status', 'id', 'time_submitted', 'time_returned', 'executor']

        task_log_info = {"task_" + k: self.tasks[task_id][k] for k in info_to_monitor}
        task_log_info['run_id'] = self.run_id
        task_log_info['timestamp'] = datetime.datetime.now()
        task_log_info['task_status_name'] = self.tasks[task_id]['status'].name
        task_log_info['tasks_failed_count'] = self.tasks_failed_count
        task_log_info['tasks_completed_count'] = self.tasks_completed_count
        task_log_info['task_inputs'] = str(self.tasks[task_id]['kwargs'].get('inputs', None))
        task_log_info['task_outputs'] = str(self.tasks[task_id]['kwargs'].get('outputs', None))
        task_log_info['task_stdin'] = self.tasks[task_id]['kwargs'].get('stdin', None)
        task_log_info['task_stdout'] = self.tasks[task_id]['kwargs'].get('stdout', None)
        task_log_info['task_depends'] = None
        if self.tasks[task_id]['depends'] is not None:
            task_log_info['task_depends'] = ",".join([str(t._tid) for t in self.tasks[task_id]['depends']])
        task_log_info['task_elapsed_time'] = None
        if self.tasks[task_id]['time_returned'] is not None:
            task_log_info['task_elapsed_time'] = (self.tasks[task_id]['time_returned'] -
                                                  self.tasks[task_id]['time_submitted']).total_seconds()
        if fail_mode is not None:
            task_log_info['task_fail_mode'] = fail_mode
        return task_log_info

Response: Create the dictionary that will be included in the log.


Instruction:
Here you have a function in Python 3, explain what it does
def handle_app_update(self, task_id, future, memo_cbk=False):

        if not self.tasks[task_id]['app_fu'].done():
            logger.error("Internal consistency error: app_fu is not done for task {}".format(task_id))
        if not self.tasks[task_id]['app_fu'] == future:
            logger.error("Internal consistency error: callback future is not the app_fu in task structure, for task {}".format(task_id))

        if not memo_cbk:
            # Update the memoizer with the new result if this is not a
            # result from a memo lookup and the task has reached a terminal state.
            self.memoizer.update_memo(task_id, self.tasks[task_id], future)

            if self.checkpoint_mode == 'task_exit':
                self.checkpoint(tasks=[task_id])

        # Submit _*_stage_out tasks for output data futures that correspond with remote files
        if (self.tasks[task_id]['app_fu'] and
            self.tasks[task_id]['app_fu'].done() and
            self.tasks[task_id]['app_fu'].exception() is None and
            self.tasks[task_id]['executor'] != 'data_manager' and
            self.tasks[task_id]['func_name'] != '_ftp_stage_in' and
            self.tasks[task_id]['func_name'] != '_http_stage_in'):
            for dfu in self.tasks[task_id]['app_fu'].outputs:
                f = dfu.file_obj
                if isinstance(f, File) and f.is_remote():
                    self.data_manager.stage_out(f, self.tasks[task_id]['executor'])

        return

Response: This function is called by the app when an app future is in its final state. It will trigger the app processing such as checkpointing the files and stageout tasks.


Instruction:
Can you write a function in Python 3 where it
handles the actual submission of the task to the executor layer. If the app task has the executors attributes not set (default=='all') the task is launched on a randomly selected executor from the list of executors. This behavior could later be updated to support binding to executors based on user specified criteria. If the app task specifies a particular set of executors, it will be targeted at those specific executors. Args: task_id (uuid string) : A uuid string that uniquely identifies the task executable (callable) : A callable object args (list of positional args) kwargs (arbitrary keyword arguments) Returns: Future that tracks the execution of the submitted executable

Response:
def launch_task(self, task_id, executable, *args, **kwargs):
        """Handle the actual submission of the task to the executor layer.

        If the app task has the executors attributes not set (default=='all')
        the task is launched on a randomly selected executor from the
        list of executors. This behavior could later be updated to support
        binding to executors based on user specified criteria.

        If the app task specifies a particular set of executors, it will be
        targeted at those specific executors.

        Args:
            task_id (uuid string) : A uuid string that uniquely identifies the task
            executable (callable) : A callable object
            args (list of positional args)
            kwargs (arbitrary keyword arguments)

        Returns:
            Future that tracks the execution of the submitted executable
        """
        self.tasks[task_id]['time_submitted'] = datetime.datetime.now()

        hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])
        if hit:
            logger.info("Reusing cached result for task {}".format(task_id))
            return memo_fu

        executor_label = self.tasks[task_id]["executor"]
        try:
            executor = self.executors[executor_label]
        except Exception:
            logger.exception("Task {} requested invalid executor {}: config is\n{}".format(task_id, executor_label, self._config))

        if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:
            executable = self.monitoring.monitor_wrapper(executable, task_id,
                                                         self.monitoring.monitoring_hub_url,
                                                         self.run_id,
                                                         self.monitoring.resource_monitoring_interval)

        with self.submitter_lock:
            exec_fu = executor.submit(executable, *args, **kwargs)
        self.tasks[task_id]['status'] = States.launched
        if self.monitoring is not None:
            task_log_info = self._create_task_log_info(task_id, 'lazy')
            self.monitoring.send(MessageType.TASK_INFO, task_log_info)

        exec_fu.retries_left = self._config.retries - \
            self.tasks[task_id]['fail_count']
        logger.info("Task {} launched on executor {}".format(task_id, executor.label))
        return exec_fu


Instruction:
Write a Python 3 function that can
add input dependencies to the inputs list.

Response:
def _add_input_deps(self, executor, args, kwargs):
        """Look for inputs of the app that are remote files. Submit stage_in
        apps for such files and replace the file objects in the inputs list with
        corresponding DataFuture objects.

        Args:
            - executor (str) : executor where the app is going to be launched
            - args (List) : Positional args to app function
            - kwargs (Dict) : Kwargs to app function
        """

        # Return if the task is _*_stage_in
        if executor == 'data_manager':
            return args, kwargs

        inputs = kwargs.get('inputs', [])
        for idx, f in enumerate(inputs):
            if isinstance(f, File) and f.is_remote():
                inputs[idx] = self.data_manager.stage_in(f, executor)

        for kwarg, f in kwargs.items():
            if isinstance(f, File) and f.is_remote():
                kwargs[kwarg] = self.data_manager.stage_in(f, executor)

        newargs = list(args)
        for idx, f in enumerate(newargs):
            if isinstance(f, File) and f.is_remote():
                newargs[idx] = self.data_manager.stage_in(f, executor)

        return tuple(newargs), kwargs


Instruction:
Write a Python 3 function that can
count the number of unresolved futures on which a task depends.

Response:
def _gather_all_deps(self, args, kwargs):
        """Count the number of unresolved futures on which a task depends.

        Args:
            - args (List[args]) : The list of args list to the fn
            - kwargs (Dict{kwargs}) : The dict of all kwargs passed to the fn

        Returns:
            - count, [list of dependencies]

        """
        # Check the positional args
        depends = []
        count = 0
        for dep in args:
            if isinstance(dep, Future):
                if self.tasks[dep.tid]['status'] not in FINAL_STATES:
                    count += 1
                depends.extend([dep])

        # Check for explicit kwargs ex, fu_1=<fut>
        for key in kwargs:
            dep = kwargs[key]
            if isinstance(dep, Future):
                if self.tasks[dep.tid]['status'] not in FINAL_STATES:
                    count += 1
                depends.extend([dep])

        # Check for futures in inputs=[<fut>...]
        for dep in kwargs.get('inputs', []):
            if isinstance(dep, Future):
                if self.tasks[dep.tid]['status'] not in FINAL_STATES:
                    count += 1
                depends.extend([dep])

        return count, depends


Instruction:
Can you tell what is the following Python 3 function doing
def sanitize_and_wrap(self, task_id, args, kwargs):
        dep_failures = []

        # Replace item in args
        new_args = []
        for dep in args:
            if isinstance(dep, Future):
                try:
                    new_args.extend([dep.result()])
                except Exception as e:
                    if self.tasks[dep.tid]['status'] in FINAL_FAILURE_STATES:
                        dep_failures.extend([e])
            else:
                new_args.extend([dep])

        # Check for explicit kwargs ex, fu_1=<fut>
        for key in kwargs:
            dep = kwargs[key]
            if isinstance(dep, Future):
                try:
                    kwargs[key] = dep.result()
                except Exception as e:
                    if self.tasks[dep.tid]['status'] in FINAL_FAILURE_STATES:
                        dep_failures.extend([e])

        # Check for futures in inputs=[<fut>...]
        if 'inputs' in kwargs:
            new_inputs = []
            for dep in kwargs['inputs']:
                if isinstance(dep, Future):
                    try:
                        new_inputs.extend([dep.result()])
                    except Exception as e:
                        if self.tasks[dep.tid]['status'] in FINAL_FAILURE_STATES:
                            dep_failures.extend([e])

                else:
                    new_inputs.extend([dep])
            kwargs['inputs'] = new_inputs

        return new_args, kwargs, dep_failures

Response: This function should be called only when all the futures have been resolved.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwargs):

        if self.cleanup_called:
            raise ValueError("Cannot submit to a DFK that has been cleaned up")

        task_id = self.task_count
        self.task_count += 1
        if isinstance(executors, str) and executors.lower() == 'all':
            choices = list(e for e in self.executors if e != 'data_manager')
        elif isinstance(executors, list):
            choices = executors
        executor = random.choice(choices)

        # Transform remote input files to data futures
        args, kwargs = self._add_input_deps(executor, args, kwargs)

        task_def = {'depends': None,
                    'executor': executor,
                    'func': func,
                    'func_name': func.__name__,
                    'args': args,
                    'kwargs': kwargs,
                    'fn_hash': fn_hash,
                    'memoize': cache,
                    'callback': None,
                    'exec_fu': None,
                    'checkpoint': None,
                    'fail_count': 0,
                    'fail_history': [],
                    'env': None,
                    'status': States.unsched,
                    'id': task_id,
                    'time_submitted': None,
                    'time_returned': None,
                    'app_fu': None}

        if task_id in self.tasks:
            raise DuplicateTaskError(
                "internal consistency error: Task {0} already exists in task list".format(task_id))
        else:
            self.tasks[task_id] = task_def

        # Get the dep count and a list of dependencies for the task
        dep_cnt, depends = self._gather_all_deps(args, kwargs)
        self.tasks[task_id]['depends'] = depends

        # Extract stdout and stderr to pass to AppFuture:
        task_stdout = kwargs.get('stdout')
        task_stderr = kwargs.get('stderr')

        logger.info("Task {} submitted for App {}, waiting on tasks {}".format(task_id,
                                                                               task_def['func_name'],
                                                                               [fu.tid for fu in depends]))

        self.tasks[task_id]['task_launch_lock'] = threading.Lock()
        app_fu = AppFuture(tid=task_id,
                           stdout=task_stdout,
                           stderr=task_stderr)

        self.tasks[task_id]['app_fu'] = app_fu
        app_fu.add_done_callback(partial(self.handle_app_update, task_id))
        self.tasks[task_id]['status'] = States.pending
        logger.debug("Task {} set to pending state with AppFuture: {}".format(task_id, task_def['app_fu']))

        # at this point add callbacks to all dependencies to do a launch_if_ready
        # call whenever a dependency completes.

        # we need to be careful about the order of setting the state to pending,
        # adding the callbacks, and caling launch_if_ready explicitly once always below.

        # I think as long as we call launch_if_ready once after setting pending, then
        # we can add the callback dependencies at any point: if the callbacks all fire
        # before then, they won't cause a launch, but the one below will. if they fire
        # after we set it pending, then the last one will cause a launch, and the
        # explicit one won't.

        for d in depends:

            def callback_adapter(dep_fut):
                self.launch_if_ready(task_id)

            try:
                d.add_done_callback(callback_adapter)
            except Exception as e:
                logger.error("add_done_callback got an exception {} which will be ignored".format(e))

        self.launch_if_ready(task_id)

        return task_def['app_fu']

Response: Add a task to the dataflow system.


Instruction:
Write a Python 3 script for
waiting for all tasks in the task list to be completed.

Response:
def wait_for_current_tasks(self):
        """Waits for all tasks in the task list to be completed, by waiting for their
        AppFuture to be completed. This method will not necessarily wait for any tasks
        added after cleanup has started (such as data stageout?)
        """

        logger.info("Waiting for all remaining tasks to complete")
        for task_id in self.tasks:
            # .exception() is a less exception throwing way of
            # waiting for completion than .result()
            fut = self.tasks[task_id]['app_fu']
            if not fut.done():
                logger.debug("Waiting for task {} to complete".format(task_id))
                fut.exception()
        logger.info("All remaining tasks completed")


Instruction:
Here you have a function in Python 3, explain what it does
def cleanup(self):
        logger.info("DFK cleanup initiated")

        # this check won't detect two DFK cleanups happening from
        # different threads extremely close in time because of
        # non-atomic read/modify of self.cleanup_called
        if self.cleanup_called:
            raise Exception("attempt to clean up DFK when it has already been cleaned-up")
        self.cleanup_called = True

        self.log_task_states()

        # Checkpointing takes priority over the rest of the tasks
        # checkpoint if any valid checkpoint method is specified
        if self.checkpoint_mode is not None:
            self.checkpoint()

            if self._checkpoint_timer:
                logger.info("Stopping checkpoint timer")
                self._checkpoint_timer.close()

        # Send final stats
        self.usage_tracker.send_message()
        self.usage_tracker.close()

        logger.info("Terminating flow_control and strategy threads")
        self.flowcontrol.close()

        for executor in self.executors.values():
            if executor.managed:
                if executor.scaling_enabled:
                    job_ids = executor.provider.resources.keys()
                    executor.scale_in(len(job_ids))
                executor.shutdown()

        self.time_completed = datetime.datetime.now()

        if self.monitoring:
            self.monitoring.send(MessageType.WORKFLOW_INFO,
                                 {'tasks_failed_count': self.tasks_failed_count,
                                  'tasks_completed_count': self.tasks_completed_count,
                                  "time_began": self.time_began,
                                  'time_completed': self.time_completed,
                                  'workflow_duration': (self.time_completed - self.time_began).total_seconds(),
                                  'run_id': self.run_id, 'rundir': self.run_dir})

            self.monitoring.close()

        """
        if self.logging_server is not None:
            self.logging_server.terminate()
            self.logging_server.join()

        if self.web_app is not None:
            self.web_app.terminate()
            self.web_app.join()
        """
        logger.info("DFK cleanup complete")

Response: This function cleans up the data flow kernel and all of its executors and the tasks in the pool.


Instruction:
Write a Python 3 script to
load a checkpoint file into a memoized lookup table.

Response:
def _load_checkpoints(self, checkpointDirs):
        """Load a checkpoint file into a lookup table.

        The data being loaded from the pickle file mostly contains input
        attributes of the task: func, args, kwargs, env...
        To simplify the check of whether the exact task has been completed
        in the checkpoint, we hash these input params and use it as the key
        for the memoized lookup table.

        Args:
            - checkpointDirs (list) : List of filepaths to checkpoints
              Eg. ['runinfo/001', 'runinfo/002']

        Returns:
            - memoized_lookup_table (dict)
        """
        memo_lookup_table = {}

        for checkpoint_dir in checkpointDirs:
            logger.info("Loading checkpoints from {}".format(checkpoint_dir))
            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')
            try:
                with open(checkpoint_file, 'rb') as f:
                    while True:
                        try:
                            data = pickle.load(f)
                            # Copy and hash only the input attributes
                            memo_fu = Future()
                            if data['exception']:
                                memo_fu.set_exception(data['exception'])
                            else:
                                memo_fu.set_result(data['result'])
                            memo_lookup_table[data['hash']] = memo_fu

                        except EOFError:
                            # Done with the checkpoint file
                            break
            except FileNotFoundError:
                reason = "Checkpoint file was not found: {}".format(
                    checkpoint_file)
                logger.error(reason)
                raise BadCheckpoint(reason)
            except Exception:
                reason = "Failed to load checkpoint: {}".format(
                    checkpoint_file)
                logger.error(reason)
                raise BadCheckpoint(reason)

            logger.info("Completed loading checkpoint:{0} with {1} tasks".format(checkpoint_file,
                                                                                len(memo_lookup_table.keys())))
        return memo_lookup_table


Instruction:
Create a Python 3 function for
loading checkpoints from the checkpointDirs into a dictionary.

Response:
def load_checkpoints(self, checkpointDirs):
        """Load checkpoints from the checkpoint files into a dictionary.

        The results are used to pre-populate the memoizer's lookup_table

        Kwargs:
             - checkpointDirs (list) : List of run folder to use as checkpoints
               Eg. ['runinfo/001', 'runinfo/002']

        Returns:
             - dict containing, hashed -> future mappings
        """
        self.memo_lookup_table = None

        if not checkpointDirs:
            return {}

        if type(checkpointDirs) is not list:
            raise BadCheckpoint("checkpointDirs expects a list of checkpoints")

        return self._load_checkpoints(checkpointDirs)


Instruction:
Explain what the following Python 3 code does
def load(cls, config: Optional[Config] = None):
        if cls._dfk is not None:
            raise RuntimeError('Config has already been loaded')

        if config is None:
            cls._dfk = DataFlowKernel(Config())
        else:
            cls._dfk = DataFlowKernel(config)

        return cls._dfk

Response: Loads a DataFlowKernel object.


Instruction:
How would you implement a function in Python 3 that
starts the interchange process

Response:
def starter(comm_q, *args, **kwargs):
    """Start the interchange process

    The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__
    """
    # logger = multiprocessing.get_logger()
    ic = Interchange(*args, **kwargs)
    comm_q.put((ic.worker_task_port,
                ic.worker_result_port))
    ic.start()


Instruction:
How would you code a function in Python 3 to
get a batch of tasks from the internal pending_task_queue and returns them as a list of dicts.

Response:
def get_tasks(self, count):
        """ Obtains a batch of tasks from the internal pending_task_queue

        Parameters
        ----------
        count: int
            Count of tasks to get from the queue

        Returns
        -------
        List of upto count tasks. May return fewer than count down to an empty list
            eg. [{'task_id':<x>, 'buffer':<buf>} ... ]
        """
        tasks = []
        for i in range(0, count):
            try:
                x = self.pending_task_queue.get(block=False)
            except queue.Empty:
                break
            else:
                tasks.append(x)

        return tasks


Instruction:
Create a Python 3 function for
pulling tasks from the incoming tasks 0mq pipe onto the internal task queue and put them into the pending task queue.

Response:
def migrate_tasks_to_internal(self, kill_event):
        """Pull tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK_PULL_THREAD] Starting")
        task_counter = 0
        poller = zmq.Poller()
        poller.register(self.task_incoming, zmq.POLLIN)

        while not kill_event.is_set():
            try:
                msg = self.task_incoming.recv_pyobj()
            except zmq.Again:
                # We just timed out while attempting to receive
                logger.debug("[TASK_PULL_THREAD] {} tasks in internal queue".format(self.pending_task_queue.qsize()))
                continue

            if msg == 'STOP':
                kill_event.set()
                break
            else:
                self.pending_task_queue.put(msg)
                task_counter += 1
                logger.debug("[TASK_PULL_THREAD] Fetched task:{}".format(task_counter))


Instruction:
Given the following Python 3 function, write the documentation
def _command_server(self, kill_event):
        logger.debug("[COMMAND] Command Server Starting")
        while not kill_event.is_set():
            try:
                command_req = self.command_channel.recv_pyobj()
                logger.debug("[COMMAND] Received command request: {}".format(command_req))
                if command_req == "OUTSTANDING_C":
                    outstanding = self.pending_task_queue.qsize()
                    for manager in self._ready_manager_queue:
                        outstanding += len(self._ready_manager_queue[manager]['tasks'])
                    reply = outstanding

                elif command_req == "WORKERS":
                    num_workers = 0
                    for manager in self._ready_manager_queue:
                        num_workers += self._ready_manager_queue[manager]['worker_count']
                    reply = num_workers
                elif command_req == "MANAGERS":
                    reply = []
                    for manager in self._ready_manager_queue:
                        resp = {'manager': manager.decode('utf-8'),
                                'block_id': self._ready_manager_queue[manager]['block_id'],
                                'worker_count': self._ready_manager_queue[manager]['worker_count'],
                                'tasks': len(self._ready_manager_queue[manager]['tasks']),
                                'active': self._ready_manager_queue[manager]['active']}
                        reply.append(resp)

                elif command_req.startswith("HOLD_WORKER"):
                    cmd, s_manager = command_req.split(';')
                    manager = s_manager.encode('utf-8')
                    logger.info("[CMD] Received HOLD_WORKER for {}".format(manager))
                    if manager in self._ready_manager_queue:
                        self._ready_manager_queue[manager]['active'] = False
                        reply = True
                    else:
                        reply = False

                elif command_req == "SHUTDOWN":
                    logger.info("[CMD] Received SHUTDOWN command")
                    kill_event.set()
                    reply = True

                else:
                    reply = None

                logger.debug("[COMMAND] Reply: {}".format(reply))
                self.command_channel.send_pyobj(reply)

            except zmq.Again:
                logger.debug("[COMMAND] is alive")
                continue

Response: Command server to run async command


Instruction:
Explain what the following Python 3 code does
def start(self, poll_period=None):
        logger.info("Incoming ports bound")

        if poll_period is None:
            poll_period = self.poll_period

        start = time.time()
        count = 0

        self._kill_event = threading.Event()
        self._task_puller_thread = threading.Thread(target=self.migrate_tasks_to_internal,
                                                    args=(self._kill_event,))
        self._task_puller_thread.start()

        self._command_thread = threading.Thread(target=self._command_server,
                                                args=(self._kill_event,))
        self._command_thread.start()

        poller = zmq.Poller()
        # poller.register(self.task_incoming, zmq.POLLIN)
        poller.register(self.task_outgoing, zmq.POLLIN)
        poller.register(self.results_incoming, zmq.POLLIN)

        # These are managers which we should examine in an iteration
        # for scheduling a job (or maybe any other attention?).
        # Anything altering the state of the manager should add it
        # onto this list.
        interesting_managers = set()

        while not self._kill_event.is_set():
            self.socks = dict(poller.poll(timeout=poll_period))

            # Listen for requests for work
            if self.task_outgoing in self.socks and self.socks[self.task_outgoing] == zmq.POLLIN:
                logger.debug("[MAIN] starting task_outgoing section")
                message = self.task_outgoing.recv_multipart()
                manager = message[0]

                if manager not in self._ready_manager_queue:
                    reg_flag = False

                    try:
                        msg = json.loads(message[1].decode('utf-8'))
                        reg_flag = True
                    except Exception:
                        logger.warning("[MAIN] Got a non-json registration message from manager:{}".format(
                            manager))
                        logger.debug("[MAIN] Message :\n{}\n".format(message[0]))

                    # By default we set up to ignore bad nodes/registration messages.
                    self._ready_manager_queue[manager] = {'last': time.time(),
                                                          'free_capacity': 0,
                                                          'block_id': None,
                                                          'max_capacity': 0,
                                                          'active': True,
                                                          'tasks': []}
                    if reg_flag is True:
                        interesting_managers.add(manager)
                        logger.info("[MAIN] Adding manager: {} to ready queue".format(manager))
                        self._ready_manager_queue[manager].update(msg)
                        logger.info("[MAIN] Registration info for manager {}: {}".format(manager, msg))

                        if (msg['python_v'].rsplit(".", 1)[0] != self.current_platform['python_v'].rsplit(".", 1)[0] or
                            msg['parsl_v'] != self.current_platform['parsl_v']):
                            logger.warn("[MAIN] Manager {} has incompatible version info with the interchange".format(manager))

                            if self.suppress_failure is False:
                                logger.debug("Setting kill event")
                                self._kill_event.set()
                                e = ManagerLost(manager)
                                result_package = {'task_id': -1, 'exception': serialize_object(e)}
                                pkl_package = pickle.dumps(result_package)
                                self.results_outgoing.send(pkl_package)
                                logger.warning("[MAIN] Sent failure reports, unregistering manager")
                            else:
                                logger.debug("[MAIN] Suppressing shutdown due to version incompatibility")
                        else:
                            logger.info("[MAIN] Manager {} has compatible Parsl version {}".format(manager, msg['parsl_v']))
                            logger.info("[MAIN] Manager {} has compatible Python version {}".format(manager,
                                                                                msg['python_v'].rsplit(".", 1)[0]))
                    else:
                        # Registration has failed.
                        if self.suppress_failure is False:
                            self._kill_event.set()
                            e = BadRegistration(manager, critical=True)
                            result_package = {'task_id': -1, 'exception': serialize_object(e)}
                            pkl_package = pickle.dumps(result_package)
                            self.results_outgoing.send(pkl_package)
                        else:
                            logger.debug("[MAIN] Suppressing bad registration from manager:{}".format(
                                manager))

                else:
                    tasks_requested = int.from_bytes(message[1], "little")
                    self._ready_manager_queue[manager]['last'] = time.time()
                    if tasks_requested == HEARTBEAT_CODE:
                        logger.debug("[MAIN] Manager {} sent heartbeat".format(manager))
                        self.task_outgoing.send_multipart([manager, b'', PKL_HEARTBEAT_CODE])
                    else:
                        logger.debug("[MAIN] Manager {} requested {} tasks".format(manager, tasks_requested))
                        self._ready_manager_queue[manager]['free_capacity'] = tasks_requested
                        interesting_managers.add(manager)
                logger.debug("[MAIN] leaving task_outgoing section")

            # If we had received any requests, check if there are tasks that could be passed

            logger.debug("Managers count (total/interesting): {}/{}".format(len(self._ready_manager_queue),
                                                                            len(interesting_managers)))

            if interesting_managers and not self.pending_task_queue.empty():
                shuffled_managers = list(interesting_managers)
                random.shuffle(shuffled_managers)

                while shuffled_managers and not self.pending_task_queue.empty():  # cf. the if statement above...
                    manager = shuffled_managers.pop()
                    tasks_inflight = len(self._ready_manager_queue[manager]['tasks'])
                    real_capacity = min(self._ready_manager_queue[manager]['free_capacity'],
                                        self._ready_manager_queue[manager]['max_capacity'] - tasks_inflight)

                    if (real_capacity and self._ready_manager_queue[manager]['active']):
                        tasks = self.get_tasks(real_capacity)
                        if tasks:
                            self.task_outgoing.send_multipart([manager, b'', pickle.dumps(tasks)])
                            task_count = len(tasks)
                            count += task_count
                            tids = [t['task_id'] for t in tasks]
                            self._ready_manager_queue[manager]['free_capacity'] -= task_count
                            self._ready_manager_queue[manager]['tasks'].extend(tids)
                            logger.debug("[MAIN] Sent tasks: {} to manager {}".format(tids, manager))
                            if self._ready_manager_queue[manager]['free_capacity'] > 0:
                                logger.debug("[MAIN] Manager {} has free_capacity {}".format(manager, self._ready_manager_queue[manager]['free_capacity']))
                                # ... so keep it in the interesting_managers list
                            else:
                                logger.debug("[MAIN] Manager {} is now saturated".format(manager))
                                interesting_managers.remove(manager)
                    else:
                        interesting_managers.remove(manager)
                        # logger.debug("Nothing to send to manager {}".format(manager))
                logger.debug("[MAIN] leaving _ready_manager_queue section, with {} managers still interesting".format(len(interesting_managers)))
            else:
                logger.debug("[MAIN] either no interesting managers or no tasks, so skipping manager pass")
            # Receive any results and forward to client
            if self.results_incoming in self.socks and self.socks[self.results_incoming] == zmq.POLLIN:
                logger.debug("[MAIN] entering results_incoming section")
                manager, *b_messages = self.results_incoming.recv_multipart()
                if manager not in self._ready_manager_queue:
                    logger.warning("[MAIN] Received a result from a un-registered manager: {}".format(manager))
                else:
                    logger.debug("[MAIN] Got {} result items in batch".format(len(b_messages)))
                    for b_message in b_messages:
                        r = pickle.loads(b_message)
                        # logger.debug("[MAIN] Received result for task {} from {}".format(r['task_id'], manager))
                        self._ready_manager_queue[manager]['tasks'].remove(r['task_id'])
                    self.results_outgoing.send_multipart(b_messages)
                    logger.debug("[MAIN] Current tasks: {}".format(self._ready_manager_queue[manager]['tasks']))
                logger.debug("[MAIN] leaving results_incoming section")

            logger.debug("[MAIN] entering bad_managers section")
            bad_managers = [manager for manager in self._ready_manager_queue if
                            time.time() - self._ready_manager_queue[manager]['last'] > self.heartbeat_threshold]
            for manager in bad_managers:
                logger.debug("[MAIN] Last: {} Current: {}".format(self._ready_manager_queue[manager]['last'], time.time()))
                logger.warning("[MAIN] Too many heartbeats missed for manager {}".format(manager))

                for tid in self._ready_manager_queue[manager]['tasks']:
                    try:
                        raise ManagerLost(manager)
                    except Exception:
                        result_package = {'task_id': tid, 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}
                        pkl_package = pickle.dumps(result_package)
                        self.results_outgoing.send(pkl_package)
                        logger.warning("[MAIN] Sent failure reports, unregistering manager")
                self._ready_manager_queue.pop(manager, 'None')
            logger.debug("[MAIN] leaving bad_managers section")
            logger.debug("[MAIN] ending one main loop iteration")

        delta = time.time() - start
        logger.info("Processed {} tasks in {} seconds".format(count, delta))
        logger.warning("Exiting")

Response: Start the NeedNameQeueu thread.


Instruction:
Explain what the following Python 3 code does
def starter(comm_q, *args, **kwargs):
    # logger = multiprocessing.get_logger()
    ic = Interchange(*args, **kwargs)
    comm_q.put(ic.worker_port)
    ic.start()
    logger.debug("Port information sent back to client")

Response: Start the interchange process


Instruction:
Here you have a function in Python 3, explain what it does
def start(self):
        logger.info("Starting interchange")
        # last = time.time()

        while True:
            # active_flag = False
            socks = dict(self.poller.poll(1))

            if socks.get(self.task_incoming) == zmq.POLLIN:
                message = self.task_incoming.recv_multipart()
                logger.debug("Got new task from client")
                self.worker_messages.send_multipart(message)
                logger.debug("Sent task to worker")
                # active_flag = True
                # last = time.time()

            if socks.get(self.worker_messages) == zmq.POLLIN:
                message = self.worker_messages.recv_multipart()
                logger.debug("Got new result from worker")
                # self.result_outgoing.send_multipart(message)
                self.result_outgoing.send_multipart(message[1:])

                logger.debug("Sent result to client")

Response: Start the interchange loop.


Instruction:
Can you generate the documentation for the following Python 3 function
def execute_task(bufs):
    user_ns = locals()
    user_ns.update({'__builtins__': __builtins__})

    f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)

    # We might need to look into callability of the function from itself
    # since we change it's name in the new namespace
    prefix = "parsl_"
    fname = prefix + "f"
    argname = prefix + "args"
    kwargname = prefix + "kwargs"
    resultname = prefix + "result"

    user_ns.update({fname: f,
                    argname: args,
                    kwargname: kwargs,
                    resultname: resultname})

    code = "{0} = {1}(*{2}, **{3})".format(resultname, fname,
                                           argname, kwargname)
    try:
        # logger.debug("[RUNNER] Executing: {0}".format(code))
        exec(code, user_ns, user_ns)

    except Exception as e:
        logger.warning("Caught exception; will raise it: {}".format(e), exc_info=True)
        raise e

    else:
        # logger.debug("[RUNNER] Result: {0}".format(user_ns.get(resultname)))
        return user_ns.get(resultname)

Response: Deserialize the buffer and execute the task.


Instruction:
Write a Python 3 function that can
listen on the pending_result_queue and sends out results via 0mq

Response:
def push_results(self, kill_event):
        """ Listens on the pending_result_queue and sends out results via 0mq

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """

        logger.debug("[RESULT_PUSH_THREAD] Starting thread")

        push_poll_period = max(10, self.poll_period) / 1000    # push_poll_period must be atleast 10 ms
        logger.debug("[RESULT_PUSH_THREAD] push poll period: {}".format(push_poll_period))

        last_beat = time.time()
        items = []

        while not kill_event.is_set():

            try:
                r = self.pending_result_queue.get(block=True, timeout=push_poll_period)
                items.append(r)
            except queue.Empty:
                pass
            except Exception as e:
                logger.exception("[RESULT_PUSH_THREAD] Got an exception: {}".format(e))

            # If we have reached poll_period duration or timer has expired, we send results
            if len(items) >= self.max_queue_size or time.time() > last_beat + push_poll_period:
                last_beat = time.time()
                if items:
                    self.result_outgoing.send_multipart(items)
                    items = []

        logger.critical("[RESULT_PUSH_THREAD] Exiting")


Instruction:
Write a Python 3 script to
start the worker processes.

Response:
def start(self):
        """ Start the worker processes.

        TODO: Move task receiving to a thread
        """
        start = time.time()
        self._kill_event = threading.Event()

        self.procs = {}
        for worker_id in range(self.worker_count):
            p = multiprocessing.Process(target=worker, args=(worker_id,
                                                             self.uid,
                                                             self.pending_task_queue,
                                                             self.pending_result_queue,
                                                             self.ready_worker_queue,
                                                         ))
            p.start()
            self.procs[worker_id] = p

        logger.debug("Manager synced with workers")

        self._task_puller_thread = threading.Thread(target=self.pull_tasks,
                                                    args=(self._kill_event,))
        self._result_pusher_thread = threading.Thread(target=self.push_results,
                                                      args=(self._kill_event,))
        self._task_puller_thread.start()
        self._result_pusher_thread.start()

        logger.info("Loop start")

        # TODO : Add mechanism in this loop to stop the worker pool
        # This might need a multiprocessing event to signal back.
        self._kill_event.wait()
        logger.critical("[MAIN] Received kill event, terminating worker processes")

        self._task_puller_thread.join()
        self._result_pusher_thread.join()
        for proc_id in self.procs:
            self.procs[proc_id].terminate()
            logger.critical("Terminating worker {}:{}".format(self.procs[proc_id],
                                                              self.procs[proc_id].is_alive()))
            self.procs[proc_id].join()
            logger.debug("Worker:{} joined successfully".format(self.procs[proc_id]))

        self.task_incoming.close()
        self.result_outgoing.close()
        self.context.term()
        delta = time.time() - start
        logger.info("process_worker_pool ran for {} seconds".format(delta))
        return


Instruction:
Implement a function in Python 3 to
return the DataManager of the currently loaded DataFlowKernel.

Response:
def get_data_manager(cls):
        """Return the DataManager of the currently loaded DataFlowKernel.
        """
        from parsl.dataflow.dflow import DataFlowKernelLoader
        dfk = DataFlowKernelLoader.dfk()

        return dfk.executors['data_manager']


Instruction:
Can you create a Python 3 function that
shuts down the executor

Response:
def shutdown(self, block=False):
        """Shutdown the ThreadPool.

        Kwargs:
            - block (bool): To block for confirmations or not

        """
        x = self.executor.shutdown(wait=block)
        logger.debug("Done with executor shutdown")
        return x