Instruction:
How would you explain what the following Python 3 function does
def extract_execution_state(self, topology):
    execution_state = topology.execution_state

    executionState = {
        "cluster": execution_state.cluster,
        "environ": execution_state.environ,
        "role": execution_state.role,
        "jobname": topology.name,
        "submission_time": execution_state.submission_time,
        "submission_user": execution_state.submission_user,
        "release_username": execution_state.release_state.release_username,
        "release_tag": execution_state.release_state.release_tag,
        "release_version": execution_state.release_state.release_version,
        "has_physical_plan": None,
        "has_tmaster_location": None,
        "has_scheduler_location": None,
        "extra_links": [],
    }

    for extra_link in self.config.extra_links:
      link = extra_link.copy()
      link["url"] = self.config.get_formatted_url(executionState,
                                                  link[EXTRA_LINK_FORMATTER_KEY])
      executionState["extra_links"].append(link)
    return executionState

Response: Extracts the execution state of a topology from the Tracker.


Instruction:
Write a Python 3 function that can
extract the scheduler location from the given topology

Response:
def extract_scheduler_location(self, topology):
    """
    Returns the representation of scheduler location that will
    be returned from Tracker.
    """
    schedulerLocation = {
        "name": None,
        "http_endpoint": None,
        "job_page_link": None,
    }

    if topology.scheduler_location:
      schedulerLocation["name"] = topology.scheduler_location.topology_name
      schedulerLocation["http_endpoint"] = topology.scheduler_location.http_endpoint
      schedulerLocation["job_page_link"] = \
          topology.scheduler_location.job_page_link[0] \
          if len(topology.scheduler_location.job_page_link) > 0 else ""

    return schedulerLocation


Instruction:
Make a summary of the following Python 3 code
def extract_tmaster(self, topology):
    tmasterLocation = {
        "name": None,
        "id": None,
        "host": None,
        "controller_port": None,
        "master_port": None,
        "stats_port": None,
    }
    if topology.tmaster:
      tmasterLocation["name"] = topology.tmaster.topology_name
      tmasterLocation["id"] = topology.tmaster.topology_id
      tmasterLocation["host"] = topology.tmaster.host
      tmasterLocation["controller_port"] = topology.tmaster.controller_port
      tmasterLocation["master_port"] = topology.tmaster.master_port
      tmasterLocation["stats_port"] = topology.tmaster.stats_port

    return tmasterLocation

Response:
Returns the representation of tmaster that will
    be returned from Tracker.


Instruction:
Write a Python 3 script to
return the representation of the logical plan that will be returned from Tracker.

Response:
def extract_logical_plan(self, topology):
    """
    Returns the representation of logical plan that will
    be returned from Tracker.
    """
    logicalPlan = {
        "spouts": {},
        "bolts": {},
    }

    # Add spouts.
    for spout in topology.spouts():
      spoutName = spout.comp.name
      spoutType = "default"
      spoutSource = "NA"
      spoutVersion = "NA"
      spoutConfigs = spout.comp.config.kvs
      for kvs in spoutConfigs:
        if kvs.key == "spout.type":
          spoutType = javaobj.loads(kvs.serialized_value)
        elif kvs.key == "spout.source":
          spoutSource = javaobj.loads(kvs.serialized_value)
        elif kvs.key == "spout.version":
          spoutVersion = javaobj.loads(kvs.serialized_value)
      spoutPlan = {
          "config": convert_pb_kvs(spoutConfigs, include_non_primitives=False),
          "type": spoutType,
          "source": spoutSource,
          "version": spoutVersion,
          "outputs": []
      }
      for outputStream in list(spout.outputs):
        spoutPlan["outputs"].append({
            "stream_name": outputStream.stream.id
        })

      logicalPlan["spouts"][spoutName] = spoutPlan

    # Add bolts.
    for bolt in topology.bolts():
      boltName = bolt.comp.name
      boltPlan = {
          "config": convert_pb_kvs(bolt.comp.config.kvs, include_non_primitives=False),
          "outputs": [],
          "inputs": []
      }
      for outputStream in list(bolt.outputs):
        boltPlan["outputs"].append({
            "stream_name": outputStream.stream.id
        })
      for inputStream in list(bolt.inputs):
        boltPlan["inputs"].append({
            "stream_name": inputStream.stream.id,
            "component_name": inputStream.stream.component_name,
            "grouping": topology_pb2.Grouping.Name(inputStream.gtype)
        })

      logicalPlan["bolts"][boltName] = boltPlan

    return logicalPlan


Instruction:
How would you implement a function in Python 3 that
returns the representation of the physical plan that will be returned from Tracker.

Response:
def extract_physical_plan(self, topology):
    """
    Returns the representation of physical plan that will
    be returned from Tracker.
    """
    physicalPlan = {
        "instances": {},
        "instance_groups": {},
        "stmgrs": {},
        "spouts": {},
        "bolts": {},
        "config": {},
        "components": {}
    }

    if not topology.physical_plan:
      return physicalPlan

    spouts = topology.spouts()
    bolts = topology.bolts()
    stmgrs = None
    instances = None

    # Physical Plan
    stmgrs = list(topology.physical_plan.stmgrs)
    instances = list(topology.physical_plan.instances)

    # Configs
    if topology.physical_plan.topology.topology_config:
      physicalPlan["config"] = convert_pb_kvs(topology.physical_plan.topology.topology_config.kvs)

    for spout in spouts:
      spout_name = spout.comp.name
      physicalPlan["spouts"][spout_name] = []
      if spout_name not in physicalPlan["components"]:
        physicalPlan["components"][spout_name] = {
            "config": convert_pb_kvs(spout.comp.config.kvs)
        }
    for bolt in bolts:
      bolt_name = bolt.comp.name
      physicalPlan["bolts"][bolt_name] = []
      if bolt_name not in physicalPlan["components"]:
        physicalPlan["components"][bolt_name] = {
            "config": convert_pb_kvs(bolt.comp.config.kvs)
        }

    for stmgr in stmgrs:
      host = stmgr.host_name
      cwd = stmgr.cwd
      shell_port = stmgr.shell_port if stmgr.HasField("shell_port") else None
      physicalPlan["stmgrs"][stmgr.id] = {
          "id": stmgr.id,
          "host": host,
          "port": stmgr.data_port,
          "shell_port": shell_port,
          "cwd": cwd,
          "pid": stmgr.pid,
          "joburl": utils.make_shell_job_url(host, shell_port, cwd),
          "logfiles": utils.make_shell_logfiles_url(host, shell_port, cwd),
          "instance_ids": []
      }

    instance_groups = collections.OrderedDict()
    for instance in instances:
      instance_id = instance.instance_id
      stmgrId = instance.stmgr_id
      name = instance.info.component_name
      stmgrInfo = physicalPlan["stmgrs"][stmgrId]
      host = stmgrInfo["host"]
      cwd = stmgrInfo["cwd"]
      shell_port = stmgrInfo["shell_port"]

      # instance_id format container_<index>_component_1
      # group name is container_<index>
      group_name = instance_id.rsplit("_", 2)[0]
      igroup = instance_groups.get(group_name, list())
      igroup.append(instance_id)
      instance_groups[group_name] = igroup

      physicalPlan["instances"][instance_id] = {
          "id": instance_id,
          "name": name,
          "stmgrId": stmgrId,
          "logfile": utils.make_shell_logfiles_url(host, shell_port, cwd, instance.instance_id),
      }
      physicalPlan["stmgrs"][stmgrId]["instance_ids"].append(instance_id)
      if name in physicalPlan["spouts"]:
        physicalPlan["spouts"][name].append(instance_id)
      else:
        physicalPlan["bolts"][name].append(instance_id)

    physicalPlan["instance_groups"] = instance_groups

    return physicalPlan


Instruction:
Can you write a function in Python 3 where it
extracts the packing plan that will be returned from Tracker.

Response:
def extract_packing_plan(self, topology):
    """
    Returns the representation of packing plan that will
    be returned from Tracker.
    """
    packingPlan = {
        "id": "",
        "container_plans": []
    }

    if not topology.packing_plan:
      return packingPlan

    container_plans = topology.packing_plan.container_plans

    containers = []
    for container_plan in container_plans:
      instances = []
      for instance_plan in container_plan.instance_plans:
        instance_resources = {"cpu": instance_plan.resource.cpu,
                              "ram": instance_plan.resource.ram,
                              "disk": instance_plan.resource.disk}
        instance = {"component_name" : instance_plan.component_name,
                    "task_id" : instance_plan.task_id,
                    "component_index": instance_plan.component_index,
                    "instance_resources": instance_resources}
        instances.append(instance)
      required_resource = {"cpu": container_plan.requiredResource.cpu,
                           "ram": container_plan.requiredResource.ram,
                           "disk": container_plan.requiredResource.disk}
      scheduled_resource = {}
      if container_plan.scheduledResource:
        scheduled_resource = {"cpu": container_plan.scheduledResource.cpu,
                              "ram": container_plan.scheduledResource.ram,
                              "disk": container_plan.scheduledResource.disk}
      container = {"id": container_plan.id,
                   "instances": instances,
                   "required_resources": required_resource,
                   "scheduled_resources": scheduled_resource}
      containers.append(container)

    packingPlan["id"] = topology.packing_plan.id
    packingPlan["container_plans"] = containers
    return json.dumps(packingPlan)


Instruction:
Can you implement a function in Python 3 that
sets the topology info for the given topology.

Response:
def setTopologyInfo(self, topology):
    """
    Extracts info from the stored proto states and
    convert it into representation that is exposed using
    the API.
    This method is called on any change for the topology.
    For example, when a container moves and its host or some
    port changes. All the information is parsed all over
    again and cache is updated.
    """
    # Execution state is the most basic info.
    # If there is no execution state, just return
    # as the rest of the things don't matter.
    if not topology.execution_state:
      Log.info("No execution state found for: " + topology.name)
      return

    Log.info("Setting topology info for topology: " + topology.name)
    has_physical_plan = True
    if not topology.physical_plan:
      has_physical_plan = False

    Log.info("Setting topology info for topology: " + topology.name)
    has_packing_plan = True
    if not topology.packing_plan:
      has_packing_plan = False

    has_tmaster_location = True
    if not topology.tmaster:
      has_tmaster_location = False

    has_scheduler_location = True
    if not topology.scheduler_location:
      has_scheduler_location = False

    topologyInfo = {
        "name": topology.name,
        "id": topology.id,
        "logical_plan": None,
        "physical_plan": None,
        "packing_plan": None,
        "execution_state": None,
        "tmaster_location": None,
        "scheduler_location": None,
    }

    executionState = self.extract_execution_state(topology)
    executionState["has_physical_plan"] = has_physical_plan
    executionState["has_packing_plan"] = has_packing_plan
    executionState["has_tmaster_location"] = has_tmaster_location
    executionState["has_scheduler_location"] = has_scheduler_location
    executionState["status"] = topology.get_status()

    topologyInfo["metadata"] = self.extract_metadata(topology)
    topologyInfo["runtime_state"] = self.extract_runtime_state(topology)

    topologyInfo["execution_state"] = executionState
    topologyInfo["logical_plan"] = self.extract_logical_plan(topology)
    topologyInfo["physical_plan"] = self.extract_physical_plan(topology)
    topologyInfo["packing_plan"] = self.extract_packing_plan(topology)
    topologyInfo["tmaster_location"] = self.extract_tmaster(topology)
    topologyInfo["scheduler_location"] = self.extract_scheduler_location(topology)

    self.topologyInfos[(topology.name, topology.state_manager_name)] = topologyInfo


Instruction:
Can you tell what is the following Python 3 function doing
def getTopologyInfo(self, topologyName, cluster, role, environ):
    # Iterate over the values to filter the desired topology.
    for (topology_name, _), topologyInfo in self.topologyInfos.items():
      executionState = topologyInfo["execution_state"]
      if (topologyName == topology_name and
          cluster == executionState["cluster"] and
          environ == executionState["environ"]):
        # If role is specified, first try to match "role" field. If "role" field
        # does not exist, try to match "submission_user" field.
        if not role or executionState.get("role") == role:
          return topologyInfo
    if role is not None:
      Log.info("Could not find topology info for topology: %s," \
               "cluster: %s, role: %s, and environ: %s",
               topologyName, cluster, role, environ)
    else:
      Log.info("Could not find topology info for topology: %s," \
               "cluster: %s and environ: %s", topologyName, cluster, environ)
    raise Exception("No topology found")

Response: Returns the JSON representation of a topologyInfo object by its name cluster environ and role.


Instruction:
How would you explain what the following Python 3 function does
def validated_formatter(self, url_format):
    # We try to create a string by substituting all known
    # parameters. If an unknown parameter is present, an error
    # will be thrown
    valid_parameters = {
        "${CLUSTER}": "cluster",
        "${ENVIRON}": "environ",
        "${TOPOLOGY}": "topology",
        "${ROLE}": "role",
        "${USER}": "user",
    }
    dummy_formatted_url = url_format
    for key, value in valid_parameters.items():
      dummy_formatted_url = dummy_formatted_url.replace(key, value)

    # All $ signs must have been replaced
    if '$' in dummy_formatted_url:
      raise Exception("Invalid viz.url.format: %s" % (url_format))

    # No error is thrown, so the format is valid.
    return url_format

Response: validate visualization url format


Instruction:
Can you generate the documentation for the following Python 3 function
def emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID,
           direct_task=None, need_task_ids=False):
    """Emits a new tuple from this Spout

    It is compatible with StreamParse API.

    :type tup: list or tuple
    :param tup: the new output Tuple to send from this spout,
                should contain only serializable data.
    :type tup_id: str or object
    :param tup_id: the ID for the Tuple. Leave this blank for an unreliable emit.
                   (Same as messageId in Java)
    :type stream: str
    :param stream: the ID of the stream this Tuple should be emitted to.
                   Leave empty to emit to the default stream.
    :type direct_task: int
    :param direct_task: the task to send the Tuple to if performing a direct emit.
    :type need_task_ids: bool
    :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.
    """
    # first check whether this tuple is sane
    self.pplan_helper.check_output_schema(stream, tup)

    # get custom grouping target task ids; get empty list if not custom grouping
    custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)

    self.pplan_helper.context.invoke_hook_emit(tup, stream, None)

    data_tuple = tuple_pb2.HeronDataTuple()
    data_tuple.key = 0

    if direct_task is not None:
      if not isinstance(direct_task, int):
        raise TypeError("direct_task argument needs to be an integer, given: %s"
                        % str(type(direct_task)))
      # performing emit-direct
      data_tuple.dest_task_ids.append(direct_task)
    elif custom_target_task_ids is not None:
      # for custom grouping
      for task_id in custom_target_task_ids:
        data_tuple.dest_task_ids.append(task_id)

    if tup_id is not None:
      tuple_info = TupleHelper.make_root_tuple_info(stream, tup_id)
      if self.acking_enabled:
        # this message is rooted
        root = data_tuple.roots.add()
        root.taskid = self.pplan_helper.my_task_id
        root.key = tuple_info.key
        self.in_flight_tuples[tuple_info.key] = tuple_info
      else:
        self.immediate_acks.append(tuple_info)

    tuple_size_in_bytes = 0

    start_time = time.time()

    # Serialize
    for obj in tup:
      serialized = self.serializer.serialize(obj)
      data_tuple.values.append(serialized)
      tuple_size_in_bytes += len(serialized)

    serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS
    self.spout_metrics.serialize_data_tuple(stream, serialize_latency_ns)

    super(SpoutInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,
                                                tuple_size_in_bytes=tuple_size_in_bytes)
    self.total_tuples_emitted += 1
    self.spout_metrics.update_emit_count(stream)
    if need_task_ids:
      sent_task_ids = custom_target_task_ids or []
      if direct_task is not None:
        sent_task_ids.append(direct_task)
      return sent_task_ids

Response: Emits a new tuple from this Spout.


Instruction:
Here you have a function in Python 3, explain what it does
def _is_continue_to_work(self):
    if not self._is_topology_running():
      return False

    max_spout_pending = \
      self.pplan_helper.context.get_cluster_config().get(api_constants.TOPOLOGY_MAX_SPOUT_PENDING)

    if not self.acking_enabled and self.output_helper.is_out_queue_available():
      return True
    elif self.acking_enabled and self.output_helper.is_out_queue_available() and \
        len(self.in_flight_tuples) < max_spout_pending:
      return True
    elif self.acking_enabled and not self.in_stream.is_empty():
      return True
    else:
      return False

Response: Checks whether we need to continue to work based on the current state of the topology.


Instruction:
Write a Python 3 script for
creating parser for topology - name

Response:
def create_parser(subparsers):
  """ create parser """
  components_parser = subparsers.add_parser(
      'components',
      help='Display information of a topology\'s components',
      usage="%(prog)s cluster/[role]/[env] topology-name [options]",
      add_help=False)
  args.add_cluster_role_env(components_parser)
  args.add_topology_name(components_parser)
  args.add_spouts(components_parser)
  args.add_bolts(components_parser)
  args.add_verbose(components_parser)
  args.add_tracker_url(components_parser)
  args.add_config(components_parser)
  components_parser.set_defaults(subcommand='components')

  return subparsers


Instruction:
How would you explain what the following Python 3 function does
def to_table(components, topo_info):
  inputs, outputs = defaultdict(list), defaultdict(list)
  for ctype, component in components.items():
    if ctype == 'bolts':
      for component_name, component_info in component.items():
        for input_stream in component_info['inputs']:
          input_name = input_stream['component_name']
          inputs[component_name].append(input_name)
          outputs[input_name].append(component_name)
  info = []
  spouts_instance = topo_info['physical_plan']['spouts']
  bolts_instance = topo_info['physical_plan']['bolts']
  for ctype, component in components.items():
    # stages is an int so keep going
    if ctype == "stages":
      continue
    for component_name, component_info in component.items():
      row = [ctype[:-1], component_name]
      if ctype == 'spouts':
        row.append(len(spouts_instance[component_name]))
      else:
        row.append(len(bolts_instance[component_name]))
      row.append(','.join(inputs.get(component_name, ['-'])))
      row.append(','.join(outputs.get(component_name, ['-'])))
      info.append(row)
  header = ['type', 'name', 'parallelism', 'input', 'output']
  return info, header

Response: normalize raw logical plan info to table


Instruction:
Can you generate a brief explanation for the following Python 3 code
def filter_bolts(table, header):
  bolts_info = []
  for row in table:
    if row[0] == 'bolt':
      bolts_info.append(row)
  return bolts_info, header

Response: filter to keep bolts


Instruction:
Write a Python 3 script for
filtering to keep spouts

Response:
def filter_spouts(table, header):
  """ filter to keep spouts """
  spouts_info = []
  for row in table:
    if row[0] == 'spout':
      spouts_info.append(row)
  return spouts_info, header


Instruction:
Write a Python 3 function for
getting the topologies of the current node

Response:
def get_topologies(self, callback=None):
    """ get topologies """
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """Custom callback to get the topologies right now."""
        ret["result"] = data

    try:
      # Ensure the topology path exists. If a topology has never been deployed
      # then the path will not exist so create it and don't crash.
      # (fixme) add a watch instead of creating the path?
      self.client.ensure_path(self.get_topologies_path())

      self._get_topologies_with_watch(callback, isWatching)
    except NoNodeError:
      self.client.stop()
      path = self.get_topologies_path()
      raise_(StateException("Error required topology path '%s' not found" % (path),
                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])

    # The topologies are now populated with the data.
    return ret["result"]


Instruction:
Explain what the following Python 3 code does
def get_topology(self, topologyName, callback=None):
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """Custom callback to get the topologies right now."""
        ret["result"] = data

    self._get_topology_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]

Response: get the topology with the given name


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_topology_with_watch(self, topologyName, callback, isWatching):
    path = self.get_topology_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_topology(data, stats):
      """ watch topology """
      if data:
        topology = Topology()
        topology.ParseFromString(data)
        callback(topology)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

Response: Helper function to get topology with a callback.


Instruction:
How would you code a function in Python 3 to
delete a topology from the zookeeper.

Response:
def delete_topology(self, topologyName):
    """ delete topology """
    path = self.get_topology_path(topologyName)
    LOG.info("Removing topology: {0} from path: {1}".format(
        topologyName, path))
    try:
      self.client.delete(path)
      return True
    except NoNodeError:
      raise_(StateException("NoNodeError while deteling topology",
                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])
    except NotEmptyError:
      raise_(StateException("NotEmptyError while deleting topology",
                            StateException.EX_TYPE_NOT_EMPTY_ERROR), sys.exc_info()[2])
    except ZookeeperError:
      raise_(StateException("Zookeeper while deleting topology",
                            StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2])
    except Exception:
      # Just re raise the exception.
      raise


Instruction:
How would you explain what the following Python 3 function does
def get_packing_plan(self, topologyName, callback=None):
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """ Custom callback to get the topologies right now. """
        ret["result"] = data

    self._get_packing_plan_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]

Response: get packing plan for a given topology


Instruction:
How would you explain what the following Python 3 function does
def _get_packing_plan_with_watch(self, topologyName, callback, isWatching):
    path = self.get_packing_plan_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-argument,unused-variable
    @self.client.DataWatch(path)
    def watch_packing_plan(data, stats):
      """ watch the packing plan for updates """
      if data:
        packing_plan = PackingPlan()
        packing_plan.ParseFromString(data)
        callback(packing_plan)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

Response: Helper function to get packing_plan with a callback.


Instruction:
Given the following Python 3 function, write the documentation
def get_pplan(self, topologyName, callback=None):
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """
        Custom callback to get the topologies right now.
        """
        ret["result"] = data

    self._get_pplan_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]

Response: get physical plan for a given topologyName


Instruction:
Can you write a function in Python 3 where it
gets the execution state of a topology

Response:
def get_execution_state(self, topologyName, callback=None):
    """ get execution state """
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """
        Custom callback to get the topologies right now.
        """
        ret["result"] = data

    self._get_execution_state_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]


Instruction:
Can you tell what is the following Python 3 function doing
def _get_execution_state_with_watch(self, topologyName, callback, isWatching):
    path = self.get_execution_state_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_execution_state(data, stats):
      """ invoke callback to watch execute state """
      if data:
        executionState = ExecutionState()
        executionState.ParseFromString(data)
        callback(executionState)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

Response: Helper function to get execution state with a callback.


Instruction:
How would you explain what the following Python 3 function does
def get_tmaster(self, topologyName, callback=None):
    isWatching = False

    # Temp dict used to return result
    # if callback is not provided.
    ret = {
        "result": None
    }
    if callback:
      isWatching = True
    else:
      def callback(data):
        """
        Custom callback to get the topologies right now.
        """
        ret["result"] = data

    self._get_tmaster_with_watch(topologyName, callback, isWatching)

    # The topologies are now populated with the data.
    return ret["result"]

Response: get tmaster for a given topologyName


Instruction:
Given the following Python 3 function, write the documentation
def _get_scheduler_location_with_watch(self, topologyName, callback, isWatching):
    path = self.get_scheduler_location_path(topologyName)
    if isWatching:
      LOG.info("Adding data watch for path: " + path)

    # pylint: disable=unused-variable, unused-argument
    @self.client.DataWatch(path)
    def watch_scheduler_location(data, stats):
      """ invoke callback to watch scheduler location """
      if data:
        scheduler_location = SchedulerLocation()
        scheduler_location.ParseFromString(data)
        callback(scheduler_location)
      else:
        callback(None)

      # Returning False will result in no future watches
      # being triggered. If isWatching is True, then
      # the future watches will be triggered.
      return isWatching

Response: Helper function to get scheduler location with a callback.


Instruction:
Given the following Python 3 function, write the documentation
def readObject(self):
    try:
      _, res = self._read_and_exec_opcode(ident=0)

      position_bak = self.object_stream.tell()
      the_rest = self.object_stream.read()
      if len(the_rest):
        log_error("Warning!!!!: Stream still has %s bytes left.\
Enable debug mode of logging to see the hexdump." % len(the_rest))
        log_debug(self._create_hexdump(the_rest))
      else:
        log_debug("Java Object unmarshalled succesfully!")
      self.object_stream.seek(position_bak)

      return res
    except Exception:
      self._oops_dump_state()
      raise

Response: read object from the object stream


Instruction:
Can you generate the documentation for the following Python 3 function
def getMetricsTimeline(tmaster,
                       component_name,
                       metric_names,
                       instances,
                       start_time,
                       end_time,
                       callback=None):
  """
  Get the specified metrics for the given component name of this topology.
  Returns the following dict on success:
  {
    "timeline": {
      <metricname>: {
        <instance>: {
          <start_time> : <numeric value>,
          <start_time> : <numeric value>,
          ...
        }
        ...
      }, ...
    },
    "starttime": <numeric value>,
    "endtime": <numeric value>,
    "component": "..."
  }

  Returns the following dict on failure:
  {
    "message": "..."
  }
  """
  # Tmaster is the proto object and must have host and port for stats.
  if not tmaster or not tmaster.host or not tmaster.stats_port:
    raise Exception("No Tmaster found")

  host = tmaster.host
  port = tmaster.stats_port

  # Create the proto request object to get metrics.

  metricRequest = tmaster_pb2.MetricRequest()
  metricRequest.component_name = component_name

  # If no instances are give, metrics for all instances
  # are fetched by default.
  if len(instances) > 0:
    for instance in instances:
      metricRequest.instance_id.append(instance)

  for metricName in metric_names:
    metricRequest.metric.append(metricName)

  metricRequest.explicit_interval.start = start_time
  metricRequest.explicit_interval.end = end_time
  metricRequest.minutely = True

  # Serialize the metricRequest to send as a payload
  # with the HTTP request.
  metricRequestString = metricRequest.SerializeToString()

  # Form and send the http request.
  url = "http://{0}:{1}/stats".format(host, port)
  request = tornado.httpclient.HTTPRequest(url,
                                           body=metricRequestString,
                                           method='POST',
                                           request_timeout=5)

  Log.debug("Making HTTP call to fetch metrics")
  Log.debug("url: " + url)
  try:
    client = tornado.httpclient.AsyncHTTPClient()
    result = yield client.fetch(request)
    Log.debug("HTTP call complete.")
  except tornado.httpclient.HTTPError as e:
    raise Exception(str(e))

  # Check the response code - error if it is in 400s or 500s
  responseCode = result.code
  if responseCode >= 400:
    message = "Error in getting metrics from Tmaster, code: " + responseCode
    Log.error(message)
    raise Exception(message)

  # Parse the response from tmaster.
  metricResponse = tmaster_pb2.MetricResponse()
  metricResponse.ParseFromString(result.body)

  if metricResponse.status.status == common_pb2.NOTOK:
    if metricResponse.status.HasField("message"):
      Log.warn("Received response from Tmaster: %s", metricResponse.status.message)

  # Form the response.
  ret = {}
  ret["starttime"] = start_time
  ret["endtime"] = end_time
  ret["component"] = component_name
  ret["timeline"] = {}

  # Loop through all the metrics
  # One instance corresponds to one metric, which can have
  # multiple IndividualMetrics for each metricname requested.
  for metric in metricResponse.metric:
    instance = metric.instance_id

    # Loop through all individual metrics.
    for im in metric.metric:
      metricname = im.name
      if metricname not in ret["timeline"]:
        ret["timeline"][metricname] = {}
      if instance not in ret["timeline"][metricname]:
        ret["timeline"][metricname][instance] = {}

      # We get minutely metrics.
      # Interval-values correspond to the minutely mark for which
      # this metric value corresponds to.
      for interval_value in im.interval_values:
        ret["timeline"][metricname][instance][interval_value.interval.start] = interval_value.value

  raise tornado.gen.Return(ret)

Response: Get the specified metrics for the given component name and instances.


Instruction:
Can you generate the documentation for the following Python 3 function
def create_parser(subparsers):
  '''
  :param subparsers:
  :return:
  '''
  parser = subparsers.add_parser(
      'version',
      help='Print version of heron-cli',
      usage="%(prog)s [options] [cluster]",
      add_help=True)

  add_version_titles(parser)

  parser.add_argument(
      'cluster',
      nargs='?',
      type=str,
      default="",
      help='Name of the cluster')

  cli_args.add_service_url(parser)

  parser.set_defaults(subcommand='version')
  return parser

Response: Create a parser for heron - cli


Instruction:
Explain what the following Python 3 code does
def run(command, parser, cl_args, unknown_args):
  '''
  :param command:
  :param parser:
  :param args:
  :param unknown_args:
  :return:
  '''
  cluster = cl_args['cluster']

  # server mode
  if cluster:
    config_file = config.heron_rc_file()
    client_confs = dict()

    # Read the cluster definition, if not found
    client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args, config_file)

    if not client_confs[cluster]:
      Log.error('Neither service url nor %s cluster definition in %s file', cluster, config_file)
      return SimpleResult(Status.HeronError)

    # if cluster definition exists, but service_url is not set, it is an error
    if not 'service_url' in client_confs[cluster]:
      Log.error('No service url for %s cluster in %s', cluster, config_file)
      sys.exit(1)

    service_endpoint = cl_args['service_url']
    service_apiurl = service_endpoint + rest.ROUTE_SIGNATURES[command][1]
    service_method = rest.ROUTE_SIGNATURES[command][0]

    try:
      r = service_method(service_apiurl)
      if r.status_code != requests.codes.ok:
        Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
      sorted_items = sorted(r.json().items(), key=lambda tup: tup[0])
      for key, value in sorted_items:
        print("%s : %s" % (key, value))
    except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
      Log.error(err)
      return SimpleResult(Status.HeronError)
  else:
    config.print_build_info()

  return SimpleResult(Status.Ok)

Response: This function is used to run the command line interface.


Instruction:
Write a Python 3 script for
validating that the names of all state locations are unique.

Response:
def validate_state_locations(self):
    """
    Names of all state locations must be unique.
    """
    names = map(lambda loc: loc["name"], self.locations)
    assert len(names) == len(set(names)), "Names of state locations must be unique"


Instruction:
Implement a function in Python 3 to
add the arguments to the parser object.

Response:
def add_arguments(parser):
  '''
  :param parser:
  :return:
  '''
  parser.add_argument(
      '--tracker_url',
      metavar='(a url; path to tracker; default: "' + consts.DEFAULT_TRACKER_URL + '")',
      default=consts.DEFAULT_TRACKER_URL)

  parser.add_argument(
      '--address',
      metavar='(an string; address to listen; default: "' + consts.DEFAULT_ADDRESS + '")',
      default=consts.DEFAULT_ADDRESS)

  parser.add_argument(
      '--port',
      metavar='(an integer; port to listen; default: ' + str(consts.DEFAULT_PORT) + ')',
      type=int,
      default=consts.DEFAULT_PORT)

  parser.add_argument(
      '--base_url',
      metavar='(a string; the base url path if operating behind proxy; default: '
      + str(consts.DEFAULT_BASE_URL) + ')',
      default=consts.DEFAULT_BASE_URL)

  return parser


Instruction:
How would you explain what the following Python 3 function does
def get(self):
    '''
    :return:
    '''
    cluster = self.get_argument("cluster")
    environ = self.get_argument("environ")
    topology = self.get_argument("topology")
    component = self.get_argument("component", default=None)
    metricnames = self.get_arguments("metricname")
    instances = self.get_arguments("instance")
    interval = self.get_argument("interval", default=-1)
    time_range = (0, interval)
    compnames = [component] if component else (yield access.get_comps(cluster, environ, topology))

    # fetch the metrics
    futures = {}
    for comp in compnames:
      future = access.get_comp_metrics(
          cluster, environ, topology, comp, instances,
          metricnames, time_range)
      futures[comp] = future

    results = yield futures

    self.write(results[component] if component else results)

Response: Get the next available component


Instruction:
Write a Python 3 script for
yielding the next available component from the cluster.

Response:
def get(self):
    '''
    :return:
    '''
    cluster = self.get_argument("cluster")
    environ = self.get_argument("environ")
    topology = self.get_argument("topology")
    component = self.get_argument("component", default=None)
    metric = self.get_argument("metric")
    instances = self.get_argument("instance")
    start = self.get_argument("starttime")
    end = self.get_argument("endtime")
    maxquery = self.get_argument("max", default=False)
    timerange = (start, end)
    compnames = [component]

    # fetch the metrics
    futures = {}
    if metric == "backpressure":
      for comp in compnames:
        future = query_handler.fetch_backpressure(cluster, metric, topology, component,
                                                  instances, timerange, maxquery, environ)
        futures[comp] = future
    else:
      fetch = query_handler.fetch_max if maxquery else query_handler.fetch
      for comp in compnames:
        future = fetch(cluster, metric, topology, component,
                       instances, timerange, environ)
        futures[comp] = future

    results = yield futures
    self.write(results[component] if component else results)


Instruction:
Can you implement a function in Python 3 that
gets method returns a list of objects

Response:
def get(self, path):
    """ get method """
    if path is None:
      return {}

    if not utils.check_path(path):
      self.write("Only relative paths are allowed")
      self.set_status(403)
      self.finish()
      return

    offset = self.get_argument("offset", default=-1)
    length = self.get_argument("length", default=-1)
    if not os.path.isfile(path):
      return {}
    data = utils.read_chunk(path, offset=offset, length=length, escape_data=True)
    self.write(json.dumps(data))
    self.finish()


Instruction:
Write a Python 3 script to
implement Pulsar Spout s initialize method

Response:
def initialize(self, config, context):
    """Implements Pulsar Spout's initialize method"""
    self.logger.info("Initializing PulsarSpout with the following")
    self.logger.info("Component-specific config: \n%s" % str(config))
    self.logger.info("Context: \n%s" % str(context))

    self.emit_count = 0
    self.ack_count = 0
    self.fail_count = 0

    if not PulsarSpout.serviceUrl in config or not PulsarSpout.topicName in config:
      self.logger.fatal("Need to specify both serviceUrl and topicName")
    self.pulsar_cluster = str(config[PulsarSpout.serviceUrl])
    self.topic = str(config[PulsarSpout.topicName])
    mode = config[api_constants.TOPOLOGY_RELIABILITY_MODE]
    if mode == api_constants.TopologyReliabilityMode.ATLEAST_ONCE:
      self.acking_timeout = 1000 * int(config[api_constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS])
    else:
      self.acking_timeout = 30000
    if PulsarSpout.receiveTimeoutMs in config:
      self.receive_timeout_ms = config[PulsarSpout.receiveTimeoutMs]
    else:
      self.receive_timeout_ms = 10
    if PulsarSpout.deserializer in config:
      self.deserializer = config[PulsarSpout.deserializer]
      if not callable(self.deserializer):
        self.logger.fatal("Pulsar Message Deserializer needs to be callable")
    else:
      self.deserializer = self.default_deserializer

    # First generate the config
    self.logConfFileName = GenerateLogConfig(context)
    self.logger.info("Generated LogConf at %s" % self.logConfFileName)

    # We currently use the high level consumer API
    # For supporting effectively once, we will need to switch
    # to using lower level Reader API, when it becomes
    # available in python
    self.client = pulsar.Client(self.pulsar_cluster, log_conf_file_path=self.logConfFileName)
    self.logger.info("Setup Client with cluster %s" % self.pulsar_cluster)
    try:
      self.consumer = self.client.subscribe(self.topic, context.get_topology_name(),
                                            consumer_type=pulsar.ConsumerType.Failover,
                                            unacked_messages_timeout_ms=self.acking_timeout)
    except Exception as e:
      self.logger.fatal("Pulsar client subscription failed: %s" % str(e))

    self.logger.info("Subscribed to topic %s" % self.topic)


Instruction:
Create a Python 3 function for
getting all the values for the cluster and environ.

Response:
def get(self):
    """ get method """
    # Get all the values for parameter "cluster".
    clusters = self.get_arguments(constants.PARAM_CLUSTER)

    # Get all the values for parameter "environ".
    environs = self.get_arguments(constants.PARAM_ENVIRON)

    role = self.get_argument_role()

    ret = {}
    topologies = self.tracker.topologies
    for topology in topologies:
      cluster = topology.cluster
      environ = topology.environ
      if not cluster or not environ:
        continue

      # This cluster is not asked for.
      # Note that "if not clusters", then
      # we show for all the clusters.
      if clusters and cluster not in clusters:
        continue

      # This environ is not asked for.
      # Note that "if not environs", then
      # we show for all the environs.
      if environs and environ not in environs:
        continue

      if cluster not in ret:
        ret[cluster] = {}
      if environ not in ret[cluster]:
        ret[cluster][environ] = {}
      try:
        topology_info = self.tracker.getTopologyInfo(topology.name, cluster, role, environ)
        if topology_info and "execution_state" in topology_info:
          ret[cluster][environ][topology.name] = topology_info["execution_state"]
      except Exception:
        # Do nothing
        pass
    self.write_success_response(ret)


Instruction:
Can you implement a function in Python 3 that
fetches Instance jstack from heron - shell.

Response:
def getInstanceJstack(self, topology_info, instance_id):
    """
    Fetches Instance jstack from heron-shell.
    """
    pid_response = yield getInstancePid(topology_info, instance_id)
    try:
      http_client = tornado.httpclient.AsyncHTTPClient()
      pid_json = json.loads(pid_response)
      pid = pid_json['stdout'].strip()
      if pid == '':
        raise Exception('Failed to get pid')
      endpoint = utils.make_shell_endpoint(topology_info, instance_id)
      url = "%s/jstack/%s" % (endpoint, pid)
      response = yield http_client.fetch(url)
      Log.debug("HTTP call for url: %s", url)
      raise tornado.gen.Return(response.body)
    except tornado.httpclient.HTTPError as e:
      raise Exception(str(e))


Instruction:
Write a Python 3 script for
creating the parser for the update command

Response:
def create_parser(subparsers):
  """ Create the parse for the update command """
  parser = subparsers.add_parser(
      'update',
      help='Update a topology',
      usage="%(prog)s [options] cluster/[role]/[env] <topology-name> "
      + "[--component-parallelism <name:value>] "
      + "[--container-number value] "
      + "[--runtime-config [component:]<name:value>]",
      add_help=True)

  args.add_titles(parser)
  args.add_cluster_role_env(parser)
  args.add_topology(parser)

  args.add_config(parser)
  args.add_dry_run(parser)
  args.add_service_url(parser)
  args.add_verbose(parser)

  # Special parameters for update command
  def parallelism_type(value):
    pattern = re.compile(r"^[\w\.-]+:[\d]+$")
    if not pattern.match(value):
      raise argparse.ArgumentTypeError(
          "Invalid syntax for component parallelism (<component_name:value>): %s" % value)
    return value

  parser.add_argument(
      '--component-parallelism',
      action='append',
      type=parallelism_type,
      required=False,
      help='Component name and the new parallelism value '
      + 'colon-delimited: <component_name>:<parallelism>')

  def runtime_config_type(value):
    pattern = re.compile(r"^([\w\.-]+:){1,2}[\w\.-]+$")
    if not pattern.match(value):
      raise argparse.ArgumentTypeError(
          "Invalid syntax for runtime config ([component:]<name:value>): %s"
          % value)
    return value

  parser.add_argument(
      '--runtime-config',
      action='append',
      type=runtime_config_type,
      required=False,
      help='Runtime configurations for topology and components '
      + 'colon-delimited: [component:]<name>:<value>')

  def container_number_type(value):
    pattern = re.compile(r"^\d+$")
    if not pattern.match(value):
      raise argparse.ArgumentTypeError(
          "Invalid syntax for container number (value): %s"
          % value)
    return value

  parser.add_argument(
      '--container-number',
      action='append',
      type=container_number_type,
      required=False,
      help='Number of containers <value>')

  parser.set_defaults(subcommand='update')
  return parser


Instruction:
Given the following Python 3 function, write the documentation
def build_extra_args_dict(cl_args):
  # Check parameters
  component_parallelism = cl_args['component_parallelism']
  runtime_configs = cl_args['runtime_config']
  container_number = cl_args['container_number']
  # Users need to provide either (component-parallelism || container_number) or runtime-config
  if (component_parallelism and runtime_configs) or (container_number and runtime_configs):
    raise Exception(
        "(component-parallelism or container_num) and runtime-config " +
        "can't be updated at the same time")

  dict_extra_args = {}

  nothing_set = True
  if component_parallelism:
    dict_extra_args.update({'component_parallelism': component_parallelism})
    nothing_set = False

  if container_number:
    dict_extra_args.update({'container_number': container_number})
    nothing_set = False

  if runtime_configs:
    dict_extra_args.update({'runtime_config': runtime_configs})
    nothing_set = False

  if nothing_set:
    raise Exception(
        "Missing arguments --component-parallelism or --runtime-config or --container-number")

  if cl_args['dry_run']:
    dict_extra_args.update({'dry_run': True})
    if 'dry_run_format' in cl_args:
      dict_extra_args.update({'dry_run_format': cl_args["dry_run_format"]})

  return dict_extra_args

Response: Build a dictionary of extra arguments for the current component.


Instruction:
Can you tell what is the following Python 3 function doing
def convert_args_dict_to_list(dict_extra_args):
  list_extra_args = []
  if 'component_parallelism' in dict_extra_args:
    list_extra_args += ["--component_parallelism",
                        ','.join(dict_extra_args['component_parallelism'])]
  if 'runtime_config' in dict_extra_args:
    list_extra_args += ["--runtime_config",
                        ','.join(dict_extra_args['runtime_config'])]
  if 'container_number' in dict_extra_args:
    list_extra_args += ["--container_number",
                        ','.join(dict_extra_args['container_number'])]
  if 'dry_run' in dict_extra_args and dict_extra_args['dry_run']:
    list_extra_args += ['--dry_run']
  if 'dry_run_format' in dict_extra_args:
    list_extra_args += ['--dry_run_format', dict_extra_args['dry_run_format']]

  return list_extra_args

Response: converts dict_extra_args to list of extra args


Instruction:
Write a Python 3 function that can
run the update command

Response:
def run(command, parser, cl_args, unknown_args):
  """ run the update command """

  Log.debug("Update Args: %s", cl_args)

  # Build jar list
  extra_lib_jars = jars.packing_jars()
  action = "update topology%s" % (' in dry-run mode' if cl_args["dry_run"] else '')

  # Build extra args
  dict_extra_args = {}
  try:
    dict_extra_args = build_extra_args_dict(cl_args)
  except Exception as err:
    return SimpleResult(Status.InvocationError, err.message)

  # Execute
  if cl_args['deploy_mode'] == config.SERVER_MODE:
    return cli_helper.run_server(command, cl_args, action, dict_extra_args)
  else:
    # Convert extra argument to commandline format and then execute
    list_extra_args = convert_args_dict_to_list(dict_extra_args)
    return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars)


Instruction:
Here you have a function in Python 3, explain what it does
def getInstancePid(topology_info, instance_id):
  try:
    http_client = tornado.httpclient.AsyncHTTPClient()
    endpoint = utils.make_shell_endpoint(topology_info, instance_id)
    url = "%s/pid/%s" % (endpoint, instance_id)
    Log.debug("HTTP call for url: %s", url)
    response = yield http_client.fetch(url)
    raise tornado.gen.Return(response.body)
  except tornado.httpclient.HTTPError as e:
    raise Exception(str(e))

Response: Retrieves Instance pid from heron - shell.


Instruction:
How would you implement a function in Python 3 that
gets method returns a generator that yields the instance PID of the current instance

Response:
def get(self):
    """ get method """
    try:
      cluster = self.get_argument_cluster()
      role = self.get_argument_role()
      environ = self.get_argument_environ()
      topology_name = self.get_argument_topology()
      instance = self.get_argument_instance()
      topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)
      result = yield getInstancePid(topology_info, instance)
      self.write_success_response(result)
    except Exception as e:
      Log.debug(traceback.format_exc())
      self.write_error_response(e)


Instruction:
Write a Python 3 function that can
check if a given gtype is sane

Response:
def is_grouping_sane(cls, gtype):
    """Checks if a given gtype is sane"""
    if gtype == cls.SHUFFLE or gtype == cls.ALL or gtype == cls.LOWEST or gtype == cls.NONE:
      return True
    elif isinstance(gtype, cls.FIELDS):
      return gtype.gtype == topology_pb2.Grouping.Value("FIELDS") and \
             gtype.fields is not None
    elif isinstance(gtype, cls.CUSTOM):
      return gtype.gtype == topology_pb2.Grouping.Value("CUSTOM") and \
             gtype.python_serialized is not None
    else:
      #pylint: disable=fixme
      #TODO: DIRECT are not supported yet
      return False


Instruction:
Can you tell what is the following Python 3 function doing
def fields(cls, *fields):
    if len(fields) == 1 and isinstance(fields[0], list):
      fields = fields[0]
    else:
      fields = list(fields)

    for i in fields:
      if not isinstance(i, str):
        raise TypeError("Non-string cannot be specified in fields")

    if not fields:
      raise ValueError("List cannot be empty for fields grouping")

    return cls.FIELDS(gtype=topology_pb2.Grouping.Value("FIELDS"),
                      fields=fields)

Response: Create a new FIELDS object with the given list of fields.


Instruction:
How would you explain what the following Python 3 function does
def custom_serialized(cls, serialized, is_java=True):
    if not isinstance(serialized, bytes):
      raise TypeError("Argument to custom_serialized() must be "
                      "a serialized Python class as bytes, given: %s" % str(serialized))
    if not is_java:
      return cls.CUSTOM(gtype=topology_pb2.Grouping.Value("CUSTOM"),
                        python_serialized=serialized)
    else:
      raise NotImplementedError("Custom grouping implemented in Java for Python topology"
                                "is not yet supported.")

Response: Custom grouping from a given serialized string


Instruction:
Can you tell what is the following Python 3 function doing
def register_metrics(self, metrics_collector, interval):
    for field, metrics in self.metrics.items():
      metrics_collector.register_metric(field, metrics, interval)

Response: Registers its metrics to a given metrics collector with a given interval


Instruction:
Here you have a function in Python 3, explain what it does
def update_count(self, name, incr_by=1, key=None):
    if name not in self.metrics:
      Log.error("In update_count(): %s is not registered in the metric", name)

    if key is None and isinstance(self.metrics[name], CountMetric):
      self.metrics[name].incr(incr_by)
    elif key is not None and isinstance(self.metrics[name], MultiCountMetric):
      self.metrics[name].incr(key, incr_by)
    else:
      Log.error("In update_count(): %s is registered but not supported with this method", name)

Response: Updates the value of a count metric or MultiCountMetric.


Instruction:
Create a Python 3 function to
update the value of a reduced metric or MultiReducedMetric.

Response:
def update_reduced_metric(self, name, value, key=None):
    """Update the value of ReducedMetric or MultiReducedMetric

    :type name: str
    :param name: name of the registered metric to be updated.
    :param value: specifies a value to be reduced.
    :type key: str or None
    :param key: specifies a key for MultiReducedMetric. Needs to be `None` for updating
                ReducedMetric.
    """
    if name not in self.metrics:
      Log.error("In update_reduced_metric(): %s is not registered in the metric", name)

    if key is None and isinstance(self.metrics[name], ReducedMetric):
      self.metrics[name].update(value)
    elif key is not None and isinstance(self.metrics[name], MultiReducedMetric):
      self.metrics[name].update(key, value)
    else:
      Log.error("In update_count(): %s is registered but not supported with this method", name)


Instruction:
Create a Python 3 function to
update the received packet count and size metrics.

Response:
def update_received_packet(self, received_pkt_size_bytes):
    """Update received packet metrics"""
    self.update_count(self.RECEIVED_PKT_COUNT)
    self.update_count(self.RECEIVED_PKT_SIZE, incr_by=received_pkt_size_bytes)


Instruction:
Here you have a function in Python 3, explain what it does
def update_sent_packet(self, sent_pkt_size_bytes):
    self.update_count(self.SENT_PKT_COUNT)
    self.update_count(self.SENT_PKT_SIZE, incr_by=sent_pkt_size_bytes)

Response: Update sent packet metrics


Instruction:
Here you have a function in Python 3, explain what it does
def register_metrics(self, context):
    sys_config = system_config.get_sys_config()
    interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])
    collector = context.get_metrics_collector()
    super(ComponentMetrics, self).register_metrics(collector, interval)

Response: Registers metrics to context


Instruction:
Given the following Python 3 function, write the documentation
def serialize_data_tuple(self, stream_id, latency_in_ns):
    self.update_count(self.TUPLE_SERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)

Response: Apply update to serialization metrics


Instruction:
Make a summary of the following Python 3 code
def _init_multi_count_metrics(self, pplan_helper):
    to_init = [self.metrics[i] for i in self.to_multi_init
               if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]
    for out_stream in pplan_helper.get_my_spout().outputs:
      stream_id = out_stream.stream.id
      for metric in to_init:
        metric.add_key(stream_id)

Response: Initializes the default values for a necessary set of MultiCountMetrics


Instruction:
Here you have a function in Python 3, explain what it does
def next_tuple(self, latency_in_ns):
    self.update_reduced_metric(self.NEXT_TUPLE_LATENCY, latency_in_ns)
    self.update_count(self.NEXT_TUPLE_COUNT)

Response: Apply updates to the next tuple metric


Instruction:
Write a Python 3 script to
apply updates to the ack metrics

Response:
def acked_tuple(self, stream_id, complete_latency_ns):
    """Apply updates to the ack metrics"""
    self.update_count(self.ACK_COUNT, key=stream_id)
    self.update_reduced_metric(self.COMPLETE_LATENCY, complete_latency_ns, key=stream_id)


Instruction:
How would you code a function in Python 3 to
apply updates to the fail metrics for a given stream

Response:
def failed_tuple(self, stream_id, fail_latency_ns):
    """Apply updates to the fail metrics"""
    self.update_count(self.FAIL_COUNT, key=stream_id)
    self.update_reduced_metric(self.FAIL_LATENCY, fail_latency_ns, key=stream_id)


Instruction:
Write a Python 3 function that can
initialize the default values for a necessary set of MultiCountMetrics

Response:
def _init_multi_count_metrics(self, pplan_helper):
    """Initializes the default values for a necessary set of MultiCountMetrics"""
    # inputs
    to_in_init = [self.metrics[i] for i in self.inputs_init
                  if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]
    for in_stream in pplan_helper.get_my_bolt().inputs:
      stream_id = in_stream.stream.id
      global_stream_id = in_stream.stream.component_name + "/" + stream_id
      for metric in to_in_init:
        metric.add_key(stream_id)
        metric.add_key(global_stream_id)
    # outputs
    to_out_init = [self.metrics[i] for i in self.outputs_init
                   if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]
    for out_stream in pplan_helper.get_my_bolt().outputs:
      stream_id = out_stream.stream.id
      for metric in to_out_init:
        metric.add_key(stream_id)


Instruction:
Can you write a function in Python 3 where it
applies updates to the execute metrics

Response:
def execute_tuple(self, stream_id, source_component, latency_in_ns):
    """Apply updates to the execute metrics"""
    self.update_count(self.EXEC_COUNT, key=stream_id)
    self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, stream_id)
    self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=stream_id)

    global_stream_id = source_component + "/" + stream_id
    self.update_count(self.EXEC_COUNT, key=global_stream_id)
    self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, global_stream_id)
    self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=global_stream_id)


Instruction:
How would you code a function in Python 3 to
apply updates to the deserialization metrics

Response:
def deserialize_data_tuple(self, stream_id, source_component, latency_in_ns):
    """Apply updates to the deserialization metrics"""
    self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)
    global_stream_id = source_component + "/" + stream_id
    self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns,
                      key=global_stream_id)


Instruction:
Can you implement a function in Python 3 that
applies updates to the ack metrics

Response:
def acked_tuple(self, stream_id, source_component, latency_in_ns):
    """Apply updates to the ack metrics"""
    self.update_count(self.ACK_COUNT, key=stream_id)
    self.update_reduced_metric(self.PROCESS_LATENCY, latency_in_ns, stream_id)
    global_stream_id = source_component + '/' + stream_id
    self.update_count(self.ACK_COUNT, key=global_stream_id)
    self.update_reduced_metric(self.PROCESS_LATENCY, latency_in_ns, global_stream_id)


Instruction:
How would you explain what the following Python 3 function does
def failed_tuple(self, stream_id, source_component, latency_in_ns):
    self.update_count(self.FAIL_COUNT, key=stream_id)
    self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, stream_id)
    global_stream_id = source_component + '/' + stream_id
    self.update_count(self.FAIL_COUNT, key=global_stream_id)
    self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, global_stream_id)

Response: Apply updates to the fail metrics


Instruction:
Here you have a function in Python 3, explain what it does
def register_metric(self, name, metric, time_bucket_in_sec):
    if name in self.metrics_map:
      raise RuntimeError("Another metric has already been registered with name: %s" % name)

    Log.debug("Register metric: %s, with interval: %s", name, str(time_bucket_in_sec))
    self.metrics_map[name] = metric

    if time_bucket_in_sec in self.time_bucket_in_sec_to_metrics_name:
      self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec].append(name)
    else:
      self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec] = [name]
      self._register_timer_task(time_bucket_in_sec)

Response: Registers a metric with the metrics manager


Instruction:
Make a summary of the following Python 3 code
def poll(self):
    try:
      # non-blocking
      ret = self._buffer.get(block=False)
      if self._producer_callback is not None:
        self._producer_callback()
      return ret
    except Queue.Empty:
      Log.debug("%s: Empty in poll()" % str(self))
      raise Queue.Empty

Response: Poll from the buffer


Instruction:
Create a Python 3 function for
offering to the buffer and return True if the buffer is full.

Response:
def offer(self, item):
    """Offer to the buffer

    It is a non-blocking operation, and when the buffer is full, it raises Queue.Full exception
    """
    try:
      # non-blocking
      self._buffer.put(item, block=False)
      if self._consumer_callback is not None:
        self._consumer_callback()
      return True
    except Queue.Full:
      Log.debug("%s: Full in offer()" % str(self))
      raise Queue.Full


Instruction:
Make a summary of the following Python 3 code
def parse(version):
    match = _REGEX.match(version)
    if match is None:
        raise ValueError('%s is not valid SemVer string' % version)

    verinfo = match.groupdict()

    verinfo['major'] = int(verinfo['major'])
    verinfo['minor'] = int(verinfo['minor'])
    verinfo['patch'] = int(verinfo['patch'])

    return verinfo

Response: Parse a SemVer string into major minor patch pre - release build parts.


Instruction:
Implement a Python 3 function for
creating a parser for the command.

Response:
def create_parser(subparsers, action, help_arg):
  '''
  :param subparsers:
  :param action:
  :param help_arg:
  :return:
  '''
  parser = subparsers.add_parser(
      action,
      help=help_arg,
      usage="%(prog)s [options] cluster/[role]/[env] <topology-name>",
      add_help=True)

  args.add_titles(parser)
  args.add_cluster_role_env(parser)
  args.add_topology(parser)

  args.add_config(parser)
  args.add_service_url(parser)
  args.add_verbose(parser)

  parser.set_defaults(subcommand=action)
  return parser


Instruction:
Can you tell what is the following Python 3 function doing
def run_server(command, cl_args, action, extra_args=dict()):
  '''
  helper function to take action on topologies using REST API
  :param command:
  :param cl_args:
  :param action:        description of action taken
  :return:
  '''
  topology_name = cl_args['topology-name']

  service_endpoint = cl_args['service_url']
  apiroute = rest.ROUTE_SIGNATURES[command][1] % (
      cl_args['cluster'],
      cl_args['role'],
      cl_args['environ'],
      topology_name
  )
  service_apiurl = service_endpoint + apiroute
  service_method = rest.ROUTE_SIGNATURES[command][0]

  # convert the dictionary to a list of tuples
  data = flatten_args(extra_args)

  err_msg = "Failed to %s: %s" % (action, topology_name)
  succ_msg = "Successfully %s: %s" % (action, topology_name)

  try:
    r = service_method(service_apiurl, data=data)
    s = Status.Ok if r.status_code == requests.codes.ok else Status.HeronError
    if r.status_code != requests.codes.ok:
      Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
  except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
    Log.error(err)
    return SimpleResult(Status.HeronError, err_msg, succ_msg)

  return SimpleResult(s, err_msg, succ_msg)

Response:
helper function to take action on topologies using REST API
 athorian


Instruction:
Make a summary of the following Python 3 code
def run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):
  '''
  helper function to take action on topologies
  :param command:
  :param cl_args:
  :param action:        description of action taken
  :return:
  '''
  topology_name = cl_args['topology-name']

  new_args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--submit_user", cl_args['submit_user'],
      "--heron_home", config.get_heron_dir(),
      "--config_path", cl_args['config_path'],
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", config.get_heron_release_file(),
      "--topology_name", topology_name,
      "--command", command,
  ]
  new_args += extra_args
  lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())
  lib_jars += extra_lib_jars

  if Log.getEffectiveLevel() == logging.DEBUG:
    new_args.append("--verbose")

  # invoke the runtime manager to kill the topology
  result = execute.heron_class(
      'org.apache.heron.scheduler.RuntimeManagerMain',
      lib_jars,
      extra_jars=[],
      args=new_args
  )

  err_msg = "Failed to %s: %s" % (action, topology_name)
  succ_msg = "Successfully %s: %s" % (action, topology_name)
  result.add_context(err_msg, succ_msg)
  return result

Response:
helper function to take action on topologies
 helper function to take action on topologies
  helper function to take action on topologies
 «direct»


Instruction:
How would you explain what the following Python 3 function does
def get_all_zk_state_managers(conf):
  state_managers = []
  state_locations = conf.get_state_locations_of_type("zookeeper")
  for location in state_locations:
    name = location['name']
    hostport = location['hostport']
    hostportlist = []
    for hostportpair in hostport.split(','):
      host = None
      port = None
      if ':' in hostport:
        hostandport = hostportpair.split(':')
        if len(hostandport) == 2:
          host = hostandport[0]
          port = int(hostandport[1])
      if not host or not port:
        raise Exception("Hostport for %s must be of the format 'host:port'." % (name))
      hostportlist.append((host, port))
    tunnelhost = location['tunnelhost']
    rootpath = location['rootpath']
    LOG.info("Connecting to zk hostports: " + str(hostportlist) + " rootpath: " + rootpath)
    state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost)
    state_managers.append(state_manager)

  return state_managers

Response:
Creates all the zookeeper state_managers and returns
 athe


Instruction:
How would you implement a function in Python 3 that
returns all the file state managers.

Response:
def get_all_file_state_managers(conf):
  """
  Returns all the file state_managers.
  """
  state_managers = []
  state_locations = conf.get_state_locations_of_type("file")
  for location in state_locations:
    name = location['name']
    rootpath = os.path.expanduser(location['rootpath'])
    LOG.info("Connecting to file state with rootpath: " + rootpath)
    state_manager = FileStateManager(name, rootpath)
    state_managers.append(state_manager)

  return state_managers


Instruction:
Can you create a Python 3 function that
increments the value of a given key by to_add.

Response:
def incr(self, key, to_add=1):
    """Increments the value of a given key by ``to_add``"""
    if key not in self.value:
      self.value[key] = CountMetric()
    self.value[key].incr(to_add)


Instruction:
Explain what the following Python 3 code does
def update(self, key, value):
    if key not in self.value:
      self.value[key] = ReducedMetric(self.reducer)

    self.value[key].update(value)

Response: Updates a value of a given key and apply reduction


Instruction:
Write a Python 3 script for
adding a new key to this metric

Response:
def add_key(self, key):
    """Adds a new key to this metric"""
    if key not in self.value:
      self.value[key] = ReducedMetric(self.reducer)


Instruction:
Can you generate the documentation for the following Python 3 function
def add_data_tuple(self, stream_id, new_data_tuple, tuple_size_in_bytes):
    if (self.current_data_tuple_set is None) or \
        (self.current_data_tuple_set.stream.id != stream_id) or \
        (len(self.current_data_tuple_set.tuples) >= self.data_tuple_set_capacity) or \
        (self.current_data_tuple_size_in_bytes >= self.max_data_tuple_size_in_bytes):
      self._init_new_data_tuple(stream_id)

    added_tuple = self.current_data_tuple_set.tuples.add()
    added_tuple.CopyFrom(new_data_tuple)

    self.current_data_tuple_size_in_bytes += tuple_size_in_bytes
    self.total_data_emitted_in_bytes += tuple_size_in_bytes

Response: Add a new data tuple to the currently buffered set of tuples


Instruction:
Make a summary of the following Python 3 code
def add_control_tuple(self, new_control_tuple, tuple_size_in_bytes, is_ack):
    if self.current_control_tuple_set is None:
      self._init_new_control_tuple()
    elif is_ack and (len(self.current_control_tuple_set.fails) > 0 or
                     len(self.current_control_tuple_set.acks) >= self.control_tuple_set_capacity):
      self._init_new_control_tuple()
    elif not is_ack and \
        (len(self.current_control_tuple_set.acks) > 0 or
         len(self.current_control_tuple_set.fails) >= self.control_tuple_set_capacity):
      self._init_new_control_tuple()

    if is_ack:
      added_tuple = self.current_control_tuple_set.acks.add()
    else:
      added_tuple = self.current_control_tuple_set.fails.add()

    added_tuple.CopyFrom(new_control_tuple)

    self.total_data_emitted_in_bytes += tuple_size_in_bytes

Response: Adds a new control tuple to the currently buffered set of tuples


Instruction:
Create a Python 3 function to
add the checkpoint state message to be sent back the stmgr

Response:
def add_ckpt_state(self, ckpt_id, ckpt_state):
    """Add the checkpoint state message to be sent back the stmgr

    :param ckpt_id: The id of the checkpoint
    :ckpt_state: The checkpoint state
    """
    # first flush any buffered tuples
    self._flush_remaining()
    msg = ckptmgr_pb2.StoreInstanceStateCheckpoint()
    istate = ckptmgr_pb2.InstanceStateCheckpoint()
    istate.checkpoint_id = ckpt_id
    istate.state = ckpt_state
    msg.state.CopyFrom(istate)
    self._push_tuple_to_stream(msg)


Instruction:
Explain what the following Python 3 code does
def create_parser(subparsers):
  '''
  :param subparsers:
  :return:
  '''
  parser = subparsers.add_parser(
      'config',
      help='Config properties for a cluster',
      usage="%(prog)s [cluster]",
      add_help=True)

  parser.add_argument(
      'cluster',
      help='Cluster to configure'
  )

  ex_subparsers = parser.add_subparsers(
      title="Commands",
      description=None)

  # add config list parser
  list_parser = ex_subparsers.add_parser(
      'list',
      help='List config properties for a cluster',
      usage="%(prog)s",
      add_help=True)
  list_parser.set_defaults(configcommand='list')

  # add config set parser
  set_parser = ex_subparsers.add_parser(
      'set',
      help='Set a cluster config property',
      usage="%(prog)s [property] [value]",
      add_help=True)

  set_parser.add_argument(
      'property',
      help='Config property to set'
  )

  set_parser.add_argument(
      'value',
      help='Value of config property'
  )
  set_parser.set_defaults(configcommand='set')

  # add config unset parser
  unset_parser = ex_subparsers.add_parser(
      'unset',
      help='Unset a cluster config property',
      usage="%(prog)s [property]",
      add_help=True)

  unset_parser.add_argument(
      'property',
      help='Config property to unset'
  )
  unset_parser.set_defaults(configcommand='unset')

  parser.set_defaults(subcommand='config')
  return parser

Response: Create a parser for the cluster config command


Instruction:
How would you code a function in Python 3 to
run command with no arguments

Response:
def run(command, parser, cl_args, unknown_args):
  '''
  :param command:
  :param parser:
  :param args:
  :param unknown_args:
  :return:
  '''
  configcommand = cl_args.get('configcommand', None)
  if configcommand == 'set':
    return _set(cl_args)
  elif configcommand == 'unset':
    return _unset(cl_args)
  else:
    return _list(cl_args)


Instruction:
How would you code a function in Python 3 to
check whether the java classpath contains valid path entries.

Response:
def valid_java_classpath(classpath):
  '''
  Given a java classpath, check whether the path entries are valid or not
  '''
  paths = classpath.split(':')
  for path_entry in paths:
    if not valid_path(path_entry.strip()):
      return False
  return True


Instruction:
Create a Python 3 function to
add an edge between two nodes.

Response:
def add_edge(self, U, V):
    '''
    :param U:
    :param V:
    :return:
    '''
    if not U in self.edges:
      self.edges[U] = set()
    if not V in self.edges:
      self.edges[V] = set()
    if not V in self.edges[U]:
      self.edges[U].add(V)


Instruction:
Write a Python 3 function for
returning the maximum distance between any vertex and U in the connected component containing U.

Response:
def bfs_depth(self, U):
    '''
    Returns the maximum distance between any vertex and U in the connected
    component containing U
    :param U:
    :return:
    '''
    bfs_queue = [[U, 0]]  # Stores the vertices whose BFS hadn't been completed.
    visited = set()
    max_depth = 0
    while bfs_queue:
      [V, depth] = bfs_queue.pop()
      if max_depth < depth:
        max_depth = depth
      visited.add(V)
      adj_set = self.edges[V]
      for W in adj_set:
        if W not in visited:
          bfs_queue.append([W, depth + 1])
    return max_depth


Instruction:
Implement a function in Python 3 to
return the maximum distance between any vertex and U in the connected component containing U.

Response:
def diameter(self):
    '''
    Returns the maximum distance between any vertex and U in the connected
    component containing U
    :return:
    '''
    diameter = 0
    for U in self.edges:
      depth = self.bfs_depth(U)
      if depth > diameter:
        diameter = depth
    return diameter


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_deps_list(abs_path_to_pex):
  pex = zipfile.ZipFile(abs_path_to_pex, mode='r')
  deps = list(set([re.match(egg_regex, i).group(1) for i in pex.namelist()
                   if re.match(egg_regex, i) is not None]))
  return deps

Response:
Get a list of paths to included dependencies in the specified pex file

 Get a list of paths to included dependencies in the specified pex file


Instruction:
Make a summary of the following Python 3 code
def load_pex(path_to_pex, include_deps=True):
  abs_path_to_pex = os.path.abspath(path_to_pex)
  Log.debug("Add a pex to the path: %s" % abs_path_to_pex)
  if abs_path_to_pex not in sys.path:
    sys.path.insert(0, os.path.dirname(abs_path_to_pex))

  # add dependencies to path
  if include_deps:
    for dep in _get_deps_list(abs_path_to_pex):
      to_join = os.path.join(os.path.dirname(abs_path_to_pex), dep)
      if to_join not in sys.path:
        Log.debug("Add a new dependency to the path: %s" % dep)
        sys.path.insert(0, to_join)

  Log.debug("Python path: %s" % str(sys.path))

Response: Loads a pex file and its dependencies to the current python path


Instruction:
Explain what the following Python 3 code does
def resolve_heron_suffix_issue(abs_pex_path, class_path):
  # import top-level package named `heron` of a given pex file
  importer = zipimport.zipimporter(abs_pex_path)
  importer.load_module("heron")

  # remove 'heron' and the classname
  to_load_lst = class_path.split('.')[1:-1]
  loaded = ['heron']
  loaded_mod = None
  for to_load in to_load_lst:
    sub_importer = zipimport.zipimporter(os.path.join(abs_pex_path, '/'.join(loaded)))
    loaded_mod = sub_importer.load_module(to_load)
    loaded.append(to_load)

  return loaded_mod

Response: Resolves duplicate package suffix problems with Heron instance pex file and class path.


Instruction:
Can you write a function in Python 3 where it
imports and loads a class from a given pex file path and python class name.

Response:
def import_and_get_class(path_to_pex, python_class_name):
  """Imports and load a class from a given pex file path and python class name

  For example, if you want to get a class called `Sample` in
  /some-path/sample.pex/heron/examples/src/python/sample.py,
  ``path_to_pex`` needs to be ``/some-path/sample.pex``, and
  ``python_class_name`` needs to be ``heron.examples.src.python.sample.Sample``
  """
  abs_path_to_pex = os.path.abspath(path_to_pex)

  Log.debug("Add a pex to the path: %s" % abs_path_to_pex)
  Log.debug("In import_and_get_class with cls_name: %s" % python_class_name)
  split = python_class_name.split('.')
  from_path = '.'.join(split[:-1])
  import_name = python_class_name.split('.')[-1]

  Log.debug("From path: %s, import name: %s" % (from_path, import_name))

  # Resolve duplicate package suffix problem (heron.), if the top level package name is heron
  if python_class_name.startswith("heron."):
    try:
      mod = resolve_heron_suffix_issue(abs_path_to_pex, python_class_name)
      return getattr(mod, import_name)
    except:
      Log.error("Could not resolve class %s with special handling" % python_class_name)

  mod = __import__(from_path, fromlist=[import_name], level=-1)
  Log.debug("Imported module: %s" % str(mod))
  return getattr(mod, import_name)


Instruction:
Here you have a function in Python 3, explain what it does
def new_source(self, source):

    source_streamlet = None
    if callable(source):
      source_streamlet = SupplierStreamlet(source)
    elif isinstance(source, Generator):
      source_streamlet = GeneratorStreamlet(source)
    else:
      raise RuntimeError("Builder's new source has to be either a Generator or a function")

    self._sources.append(source_streamlet)
    return source_streamlet

Response: Adds a new source to the computation DAG


Instruction:
Here you have a function in Python 3, explain what it does
def build(self, bldr):
    stage_names = sets.Set()
    for source in self._sources:
      source._build(bldr, stage_names)
    for source in self._sources:
      if not source._all_built():
        raise RuntimeError("Topology cannot be fully built! Are all sources added?")

Response: Builds the topology and returns the builder


Instruction:
Can you generate the documentation for the following Python 3 function
def load_state_manager_locations(cluster, state_manager_config_file='heron-conf/statemgr.yaml',
                                 overrides={}):
  """ Reads configs to determine which state manager to use and converts them to state manager
  locations. Handles a subset of config wildcard substitution supported in the substitute method in
  org.apache.heron.spi.common.Misc.java"""
  with open(state_manager_config_file, 'r') as stream:
    config = yaml.load(stream)

  home_dir = os.path.expanduser("~")
  wildcards = {
      "~" : home_dir,
      "${HOME}" : home_dir,
      "${CLUSTER}" : cluster,
  }
  if os.getenv('JAVA_HOME'):
    wildcards["${JAVA_HOME}"] = os.getenv('JAVA_HOME')

  config = __replace(config, wildcards, state_manager_config_file)

  # merge with overrides
  if overrides:
    config.update(overrides)

  # need to convert from the format in statemgr.yaml to the format that the python state managers
  # takes. first, set reasonable defaults to local
  state_manager_location = {
      'type': 'file',
      'name': 'local',
      'tunnelhost': '127.0.0.1',
      'rootpath': '~/.herondata/repository/state/local',
  }

  # then map the statemgr.yaml config keys to the python state manager location
  key_mappings = {
      'heron.statemgr.connection.string': 'hostport',
      'heron.statemgr.tunnel.host': 'tunnelhost',
      'heron.statemgr.root.path': 'rootpath',
  }
  for config_key in key_mappings:
    if config_key in config:
      state_manager_location[key_mappings[config_key]] = config[config_key]

  state_manager_class = config['heron.class.state.manager']
  if state_manager_class == 'org.apache.heron.statemgr.zookeeper.curator.CuratorStateManager':
    state_manager_location['type'] = 'zookeeper'
    state_manager_location['name'] = 'zk'

  return [state_manager_location]

Response: Reads the state manager configuration file and converts it to state manager locations.


Instruction:
Given the following Python 3 function, write the documentation
def __replace(config, wildcards, config_file):
  for config_key in config:
    config_value = config[config_key]
    original_value = config_value
    if isinstance(config_value, str):
      for token in wildcards:
        if wildcards[token]:
          config_value = config_value.replace(token, wildcards[token])
      found = re.findall(r'\${[A-Z_]+}', config_value)
      if found:
        raise ValueError("%s=%s in file %s contains unsupported or unset wildcard tokens: %s" %
                         (config_key, original_value, config_file, ", ".join(found)))
      config[config_key] = config_value
  return config

Response: Replace all the values in config with the values in wildcards


Instruction:
Can you implement a function in Python 3 that
creates a map of command names and handlers that are used by the command line tool.

Response:
def get_command_handlers():
  '''
  Create a map of command names and handlers
  '''
  return {
      'activate': activate,
      'config': hconfig,
      'deactivate': deactivate,
      'help': cli_help,
      'kill': kill,
      'restart': restart,
      'submit': submit,
      'update': update,
      'version': version
  }


Instruction:
Explain what the following Python 3 code does
def create_parser(command_handlers):
  '''
  Main parser
  :return:
  '''
  parser = argparse.ArgumentParser(
      prog='heron',
      epilog=HELP_EPILOG,
      formatter_class=config.SubcommandHelpFormatter,
      add_help=True)

  subparsers = parser.add_subparsers(
      title="Available commands",
      metavar='<command> <options>')

  command_list = sorted(command_handlers.items())
  for command in command_list:
    command[1].create_parser(subparsers)

  return parser

Response: Create a parser for the availabe command list.


Instruction:
Create a Python 3 function to
run the command .

Response:
def run(handlers, command, parser, command_args, unknown_args):
  '''
  Run the command
  :param command:
  :param parser:
  :param command_args:
  :param unknown_args:
  :return:
  '''

  if command in handlers:
    return handlers[command].run(command, parser, command_args, unknown_args)
  else:
    err_context = 'Unknown subcommand: %s' % command
    return result.SimpleResult(result.Status.InvocationError, err_context)


Instruction:
Write a Python 3 script to
remove all the files in the list of files

Response:
def cleanup(files):
  '''
  :param files:
  :return:
  '''
  for cur_file in files:
    if os.path.isdir(cur_file):
      shutil.rmtree(cur_file)
    else:
      shutil.rmtree(os.path.dirname(cur_file))


Instruction:
Write a Python 3 script for
checking the server deployment mode for the given cluster

Response:
def server_deployment_mode(command, parser, cluster, cl_args):
  '''
  check the server deployment mode for the given cluster
  if it is valid return the valid set of args
  :param cluster:
  :param cl_args:
  :return:
  '''
  # Read the cluster definition, if not found
  client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args)

  if not client_confs[cluster]:
    return dict()

  # tell the user which definition that we are using
  if not cl_args.get('service_url', None):
    Log.debug("Using cluster definition from file %s" \
        % cliconfig.get_cluster_config_file(cluster))
  else:
    Log.debug("Using cluster service url %s" % cl_args['service_url'])

  # if cluster definition exists, but service_url is not set, it is an error
  if not 'service_url' in client_confs[cluster]:
    config_file = cliconfig.get_cluster_config_file(cluster)
    Log.error('No service url for %s cluster in %s', cluster, config_file)
    sys.exit(1)

  # get overrides
  if 'config_property' in cl_args:
    pass

  try:
    cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])
    config.server_mode_cluster_role_env(cluster_role_env, client_confs)
    cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
  except Exception as ex:
    Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
    sys.exit(1)

  new_cl_args = dict()
  new_cl_args['cluster'] = cluster_tuple[0]
  new_cl_args['role'] = cluster_tuple[1]
  new_cl_args['environ'] = cluster_tuple[2]
  new_cl_args['service_url'] = client_confs[cluster]['service_url'].rstrip('/')
  new_cl_args['deploy_mode'] = config.SERVER_MODE

  cl_args.update(new_cl_args)
  return cl_args


Instruction:
Make a summary of the following Python 3 code
def direct_deployment_mode(command, parser, cluster, cl_args):
  '''
  check the direct deployment mode for the given cluster
  if it is valid return the valid set of args
  :param command:
  :param parser:
  :param cluster:
  :param cl_args:
  :return:
  '''

  cluster = cl_args['cluster']
  try:
    config_path = cl_args['config_path']
    override_config_file = config.parse_override_config_and_write_file(cl_args['config_property'])
  except KeyError:
    # if some of the arguments are not found, print error and exit
    subparser = config.get_subparser(parser, command)
    print(subparser.format_help())
    return dict()

  # check if the cluster config directory exists
  if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):
    Log.error("Cluster config directory \'%s\' does not exist", config_path)
    return dict()

  config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
  if not os.path.isdir(config_path):
    Log.error("Cluster config directory \'%s\' does not exist", config_path)
    return dict()

  Log.info("Using cluster definition in %s" % config_path)

  try:
    cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])
    config.direct_mode_cluster_role_env(cluster_role_env, config_path)
    cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)
  except Exception as ex:
    Log.error("Argument cluster/[role]/[env] is not correct: %s", str(ex))
    return dict()

  new_cl_args = dict()
  new_cl_args['cluster'] = cluster_tuple[0]
  new_cl_args['role'] = cluster_tuple[1]
  new_cl_args['environ'] = cluster_tuple[2]
  new_cl_args['config_path'] = config_path
  new_cl_args['override_config_file'] = override_config_file
  new_cl_args['deploy_mode'] = config.DIRECT_MODE

  cl_args.update(new_cl_args)
  return cl_args

Response: check if direct deployment mode for the given cluster


Instruction:
Can you generate a brief explanation for the following Python 3 code
def extract_common_args(command, parser, cl_args):
  '''
  Extract all the common args for all commands
  :param command:
  :param parser:
  :param cl_args:
  :return:
  '''
  try:
    cluster_role_env = cl_args.pop('cluster/[role]/[env]')
  except KeyError:
    try:
      cluster_role_env = cl_args.pop('cluster')  # for version command
    except KeyError:
      # if some of the arguments are not found, print error and exit
      subparser = config.get_subparser(parser, command)
      print(subparser.format_help())
      return dict()

  new_cl_args = dict()
  cluster_tuple = config.get_cluster_role_env(cluster_role_env)
  new_cl_args['cluster'] = cluster_tuple[0]
  new_cl_args['role'] = cluster_tuple[1]
  new_cl_args['environ'] = cluster_tuple[2]
  new_cl_args['submit_user'] = getpass.getuser()

  cl_args.update(new_cl_args)
  return cl_args

Response: Extract all the common args for all commands


Instruction:
Write a Python 3 function that can
execute the command line and return the exit code.

Response:
def execute(handlers, local_commands):
  '''
  Run the command
  :return:
  '''
  # verify if the environment variables are correctly set
  check_environment()

  # create the argument parser
  parser = create_parser(handlers)

  # if no argument is provided, print help and exit
  if len(sys.argv[1:]) == 0:
    parser.print_help()
    return 0

  # insert the boolean values for some of the options
  sys.argv = config.insert_bool_values(sys.argv)

  try:
    # parse the args
    args, unknown_args = parser.parse_known_args()
  except ValueError as ex:
    Log.error("Error while parsing arguments: %s", str(ex))
    Log.debug(traceback.format_exc())
    sys.exit(1)

  command_line_args = vars(args)

  # set log level
  log.set_logging_level(command_line_args)
  Log.debug("Input Command Line Args: %s", command_line_args)

# command to be execute
  command = command_line_args['subcommand']
  is_local_command = command in local_commands

  if command == 'version':
    results = run(handlers, command, parser, command_line_args, unknown_args)
    return 0 if result.is_successful(results) else 1

  if not is_local_command:
    log.set_logging_level(command_line_args)
    Log.debug("Input Command Line Args: %s", command_line_args)

    # determine the mode of deployment
    command_line_args = extract_common_args(command, parser, command_line_args)
    command_line_args = deployment_mode(command, parser, command_line_args)

    # bail out if args are empty
    if not command_line_args:
      return 1

    # register dirs cleanup function during exit
    if command_line_args['deploy_mode'] == config.DIRECT_MODE and command != "version":
      cleaned_up_files.append(command_line_args['override_config_file'])
      atexit.register(cleanup, cleaned_up_files)

  # print the input parameters, if verbose is enabled
  Log.debug("Processed Command Line Args: %s", command_line_args)

  start = time.time()
  results = run(handlers, command, parser, command_line_args, unknown_args)
  if not is_local_command:
    result.render(results)
  end = time.time()

  if not is_local_command:
    sys.stdout.flush()
    Log.debug('Elapsed time: %.3fs.', (end - start))

  return 0 if result.is_successful(results) else 1


Instruction:
Create a Python 3 function to
get method returns a sequence of metrics

Response:
def get(self):
    """ get method """
    try:
      cluster = self.get_argument_cluster()
      role = self.get_argument_role()
      environ = self.get_argument_environ()
      topology_name = self.get_argument_topology()
      component = self.get_argument_component()
      metric_names = self.get_required_arguments_metricnames()

      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(
          cluster, role, environ, topology_name)

      interval = int(self.get_argument(constants.PARAM_INTERVAL, default=-1))
      instances = self.get_arguments(constants.PARAM_INSTANCE)

      metrics = yield tornado.gen.Task(
          self.getComponentMetrics,
          topology.tmaster, component, metric_names, instances, interval)

      self.write_success_response(metrics)
    except Exception as e:
      Log.debug(traceback.format_exc())
      self.write_error_response(e)


Instruction:
How would you code a function in Python 3 to
get the specified metrics for the given component name of this Tmaster.

Response:
def getComponentMetrics(self,
                          tmaster,
                          componentName,
                          metricNames,
                          instances,
                          interval,
                          callback=None):
    """
    Get the specified metrics for the given component name of this topology.
    Returns the following dict on success:
    {
      "metrics": {
        <metricname>: {
          <instance>: <numeric value>,
          <instance>: <numeric value>,
          ...
        }, ...
      },
      "interval": <numeric value>,
      "component": "..."
    }

    Raises exception on failure.
    """
    if not tmaster or not tmaster.host or not tmaster.stats_port:
      raise Exception("No Tmaster found")

    host = tmaster.host
    port = tmaster.stats_port

    metricRequest = tmaster_pb2.MetricRequest()
    metricRequest.component_name = componentName
    if len(instances) > 0:
      for instance in instances:
        metricRequest.instance_id.append(instance)
    for metricName in metricNames:
      metricRequest.metric.append(metricName)
    metricRequest.interval = interval

    # Serialize the metricRequest to send as a payload
    # with the HTTP request.
    metricRequestString = metricRequest.SerializeToString()

    url = "http://{0}:{1}/stats".format(host, port)
    request = tornado.httpclient.HTTPRequest(url,
                                             body=metricRequestString,
                                             method='POST',
                                             request_timeout=5)

    Log.debug("Making HTTP call to fetch metrics")
    Log.debug("url: " + url)
    try:
      client = tornado.httpclient.AsyncHTTPClient()
      result = yield client.fetch(request)
      Log.debug("HTTP call complete.")
    except tornado.httpclient.HTTPError as e:
      raise Exception(str(e))

    # Check the response code - error if it is in 400s or 500s
    responseCode = result.code
    if responseCode >= 400:
      message = "Error in getting metrics from Tmaster, code: " + responseCode
      Log.error(message)
      raise Exception(message)

    # Parse the response from tmaster.
    metricResponse = tmaster_pb2.MetricResponse()
    metricResponse.ParseFromString(result.body)

    if metricResponse.status.status == common_pb2.NOTOK:
      if metricResponse.status.HasField("message"):
        Log.warn("Received response from Tmaster: %s", metricResponse.status.message)

    # Form the response.
    ret = {}
    ret["interval"] = metricResponse.interval
    ret["component"] = componentName
    ret["metrics"] = {}
    for metric in metricResponse.metric:
      instance = metric.instance_id
      for im in metric.metric:
        metricname = im.name
        value = im.value
        if metricname not in ret["metrics"]:
          ret["metrics"][metricname] = {}
        ret["metrics"][metricname][instance] = value

    raise tornado.gen.Return(ret)


Instruction:
How would you explain what the following Python 3 function does
def initialize(self, config, context):
    if SlidingWindowBolt.WINDOW_DURATION_SECS in config:
      self.window_duration = int(config[SlidingWindowBolt.WINDOW_DURATION_SECS])
    else:
      self.logger.fatal("Window Duration has to be specified in the config")
    if SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS in config:
      self.slide_interval = int(config[SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS])
    else:
      self.slide_interval = self.window_duration
    if self.slide_interval > self.window_duration:
      self.logger.fatal("Slide Interval should be <= Window Duration")

    # By modifying the config, we are able to setup the tick timer
    config[api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS] = str(self.slide_interval)
    self.current_tuples = deque()
    if hasattr(self, 'saved_state'):
      if 'tuples' in self.saved_state:
        self.current_tuples = self.saved_state['tuples']

Response: Initialize the current window duration and slide interval


Instruction:
Write a Python 3 function for
processing a single tuple of input We add the tuple pair into our current_tuples and then expire the elemnents .

Response:
def process(self, tup):
    """Process a single tuple of input

    We add the (time, tuple) pair into our current_tuples. And then look for expiring
    elemnents
    """
    curtime = int(time.time())
    self.current_tuples.append((tup, curtime))
    self._expire(curtime)


Instruction:
Implement a Python 3 function for
initializing the window duration and slide interval

Response:
def initialize(self, config, context):
    """We initialize the window duration and slide interval
    """
    if TumblingWindowBolt.WINDOW_DURATION_SECS in config:
      self.window_duration = int(config[TumblingWindowBolt.WINDOW_DURATION_SECS])
    else:
      self.logger.fatal("Window Duration has to be specified in the config")

    # By modifying the config, we are able to setup the tick timer
    config[api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS] = str(self.window_duration)
    self.current_tuples = deque()
    if hasattr(self, 'saved_state'):
      if 'tuples' in self.saved_state:
        self.current_tuples = self.saved_state['tuples']