Instruction:
Can you generate a brief explanation for the following Python 3 code
def fetch_backpressure(self, cluster, metric, topology, component, instance, \
    timerange, is_max, environ=None):
    '''
    :param cluster:
    :param metric:
    :param topology:
    :param component:
    :param instance:
    :param timerange:
    :param isMax:
    :param environ:
    :return:
    '''
    instances = yield get_instances(cluster, environ, topology)
    if component != "*":
      filtered_inst = [instance for instance in instances if instance.split("_")[2] == component]
    else:
      filtered_inst = instances

    futures_dict = {}
    for inst in filtered_inst:
      query = queries.get(metric).format(inst)
      futures_dict[inst] = get_metrics(cluster, environ, topology, timerange, query)

    res = yield futures_dict

    if not is_max:
      timelines = []
      for key in res:
        result = res[key]
        # Replacing stream manager instance name with component instance name
        if len(result["timeline"]) > 0:
          result["timeline"][0]["instance"] = key
        timelines.extend(result["timeline"])
      result = self.get_metric_response(timerange, timelines, is_max)
    else:
      data = self.compute_max(res.values())
      result = self.get_metric_response(timerange, data, is_max)

    raise tornado.gen.Return(result)

Response: Get backpressure metrics for a given component.


Instruction:
How would you explain what the following Python 3 function does
def compute_max(self, multi_ts):
    '''
    :param multi_ts:
    :return:
    '''
    if len(multi_ts) > 0 and len(multi_ts[0]["timeline"]) > 0:
      keys = multi_ts[0]["timeline"][0]["data"].keys()
      timelines = ([res["timeline"][0]["data"][key] for key in keys] for res in multi_ts)
      values = (max(v) for v in zip(*timelines))
      return dict(zip(keys, values))
    return {}

Response: Compute the maximum value of a multi - Timeseries


Instruction:
Can you write a function in Python 3 where it
returns a dictionary of the response from the metric service

Response:
def get_metric_response(self, timerange, data, isMax):
    '''
    :param timerange:
    :param data:
    :param isMax:
    :return:
    '''
    if isMax:
      return dict(
          status="success",
          starttime=timerange[0],
          endtime=timerange[1],
          result=dict(timeline=[dict(data=data)])
      )

    return dict(
        status="success",
        starttime=timerange[0],
        endtime=timerange[1],
        result=dict(timeline=data)
    )


Instruction:
How would you explain what the following Python 3 function does
def get_query(self, metric, component, instance):
    '''
    :param metric:
    :param component:
    :param instance:
    :return:
    '''
    q = queries.get(metric)
    return q.format(component, instance)

Response: Get query string for a given metric component and instance.


Instruction:
Implement a Python 3 function for
normalizing raw result to table

Response:
def to_table(result):
  ''' normalize raw result to table '''
  max_count = 20
  table, count = [], 0
  for role, envs_topos in result.items():
    for env, topos in envs_topos.items():
      for topo in topos:
        count += 1
        if count > max_count:
          continue
        else:
          table.append([role, env, topo])
  header = ['role', 'env', 'topology']
  rest_count = 0 if count <= max_count else count - max_count
  return table, header, rest_count


Instruction:
Can you implement a function in Python 3 that
shows topologies in a cluster

Response:
def show_cluster(cl_args, cluster):
  ''' print topologies information to stdout '''
  try:
    result = tracker_access.get_cluster_topologies(cluster)
    if not result:
      Log.error('No topologies in cluster \'%s\'' % cluster)
      return False
    result = result[cluster]
  except Exception:
    Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
    return False
  table, header, rest_count = to_table(result)
  print('Topologies running in cluster \'%s\'' % cluster)
  if rest_count:
    print('  with %d more...' % rest_count)
  print(tabulate(table, headers=header))
  return True


Instruction:
Make a summary of the following Python 3 code
def show_cluster_role(cl_args, cluster, role):
  ''' print topologies information to stdout '''
  try:
    result = tracker_access.get_cluster_role_topologies(cluster, role)
    if not result:
      Log.error('Unknown cluster/role \'%s\'' % '/'.join([cluster, role]))
      return False
    result = result[cluster]
  except Exception:
    Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
    return False
  table, header, rest_count = to_table(result)
  print('Topologies running in cluster \'%s\' submitted by \'%s\':' % (cluster, role))
  if rest_count:
    print('  with %d more...' % rest_count)
  print(tabulate(table, headers=header))
  return True

Response: show topologies of a given cluster and role


Instruction:
Explain what the following Python 3 code does
def show_cluster_role_env(cl_args, cluster, role, env):
  ''' print topologies information to stdout '''
  try:
    result = tracker_access.get_cluster_role_env_topologies(cluster, role, env)
    if not result:
      Log.error('Unknown cluster/role/env \'%s\'' % '/'.join([cluster, role, env]))
      return False
    result = result[cluster]
  except Exception:
    Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
    return False
  table, header, rest_count = to_table(result)
  print('Topologies running in cluster \'%s\', submitted by \'%s\', and\
 under environment \'%s\':' % (cluster, role, env))
  if rest_count:
    print('  with %d more...' % rest_count)
  print(tabulate(table, headers=header))
  return True

Response: show topologies running in a cluster and under a specific environment


Instruction:
Implement a function in Python 3 to
execute a Heron class given the args and the jars needed for class path .

Response:
def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):
  '''
  Execute a heron class given the args and the jars needed for class path
  :param class_name:
  :param lib_jars:
  :param extra_jars:
  :param args:
  :param java_defines:
  :return:
  '''
  # default optional params to empty list if not provided
  if extra_jars is None:
    extra_jars = []
  if args is None:
    args = []
  if java_defines is None:
    java_defines = []

  # Format all java -D options that need to be passed while running
  # the class locally.
  java_opts = ['-D' + opt for opt in java_defines]

  # Construct the command line for the sub process to run
  # Because of the way Python execute works,
  # the java opts must be passed as part of the list
  all_args = [config.get_java_path(), "-client", "-Xmx1g"] + \
             java_opts + \
             ["-cp", config.get_classpath(extra_jars + lib_jars)]

  all_args += [class_name] + list(args)

  # set heron_config environment variable
  heron_env = os.environ.copy()
  heron_env['HERON_OPTIONS'] = opts.get_heron_config()

  # print the verbose message
  Log.debug("Invoking class using command: ``%s''", ' '.join(all_args))
  Log.debug("Heron options: {%s}", str(heron_env["HERON_OPTIONS"]))

  # invoke the command with subprocess and print error message, if any
  process = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, bufsize=1)
  # stdout message has the information Java program sends back
  # stderr message has extra information, such as debugging message
  return ProcessResult(process)


Instruction:
Given the following Python 3 function, write the documentation
def get(self, cluster, environ, topology, comp_name):
    '''
    :param cluster:
    :param environ:
    :param topology:
    :param comp_name:
    :return:
    '''
    start_time = time.time()
    comp_names = []
    if comp_name == "All":
      lplan = yield access.get_logical_plan(cluster, environ, topology)
      if not lplan:
        self.write(dict())
        return

      if not 'spouts' in lplan or not 'bolts' in lplan:
        self.write(dict())
        return
      comp_names = lplan['spouts'].keys()
      comp_names.extend(lplan['bolts'].keys())
    else:
      comp_names = [comp_name]
    exception_infos = dict()
    for comp_name in comp_names:
      exception_infos[comp_name] = yield access.get_component_exceptionsummary(
          cluster, environ, topology, comp_name)

    # Combine exceptions from multiple component
    aggregate_exceptions = dict()
    for comp_name, exception_logs in exception_infos.items():
      for exception_log in exception_logs:
        class_name = exception_log['class_name']
        if class_name != '':
          if not class_name in aggregate_exceptions:
            aggregate_exceptions[class_name] = 0
          aggregate_exceptions[class_name] += int(exception_log['count'])
    # Put the exception value in a table
    aggregate_exceptions_table = []
    for key in aggregate_exceptions:
      aggregate_exceptions_table.append([key, str(aggregate_exceptions[key])])
    result = dict(
        status="success",
        executiontime=time.time() - start_time,
        result=aggregate_exceptions_table)
    self.write(result)

Response: Get the exception summary for a given component.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get(self):
    '''
    :return:
    '''
    # get all the topologies from heron nest
    topologies = yield access.get_topologies_states()

    result = dict()

    # now convert some of the fields to be displayable
    for cluster, cluster_value in topologies.items():
      result[cluster] = dict()
      for environ, environ_value in cluster_value.items():
        result[cluster][environ] = dict()
        for topology, topology_value in environ_value.items():
          if "jobname" not in topology_value or topology_value["jobname"] is None:
            continue

          if "submission_time" in topology_value:
            topology_value["submission_time"] = topology_value["submission_time"]
          else:
            topology_value["submission_time"] = '-'

          result[cluster][environ][topology] = topology_value

    self.write(result)

Response: get all the topologies from heron nest


Instruction:
Can you create a Python 3 function that
gets a new node from the cluster.

Response:
def get(self, cluster, environ, topology):
    '''
    :param cluster:
    :param environ:
    :param topology:
    :return:
    '''

    start_time = time.time()
    lplan = yield access.get_logical_plan(cluster, environ, topology)

    # construct the result
    result = dict(
        status="success",
        message="",
        version=common.VERSION,
        executiontime=time.time() - start_time,
        result=lplan
    )

    self.write(result)


Instruction:
Here you have a function in Python 3, explain what it does
def get(self, cluster, environ, topology):
    '''
    :param cluster:
    :param environ:
    :param topology:
    :return:
    '''

    start_time = time.time()
    pplan = yield access.get_physical_plan(cluster, environ, topology)

    result_map = dict(
        status="success",
        message="",
        version=common.VERSION,
        executiontime=time.time() - start_time,
        result=pplan
    )

    self.write(result_map)

Response: Get a new cluster with physical plan.


Instruction:
How would you implement a function in Python 3 that
gets a single component from the cluster.

Response:
def get(self, cluster, environ, topology, component):
    '''
    :param cluster:
    :param environ:
    :param topology:
    :param component:
    :return:
    '''
    start_time = time.time()
    futures = yield access.get_component_exceptions(cluster, environ, topology, component)
    result_map = dict(
        status='success',
        executiontime=time.time() - start_time,
        result=futures)
    self.write(json.dumps(result_map))


Instruction:
How would you implement a function in Python 3 that
gets the current cluster s physical plan and instance.

Response:
def get(self, cluster, environ, topology, instance):
    '''
    :param cluster:
    :param environ:
    :param topology:
    :param instance:
    :return:
    '''
    pplan = yield access.get_physical_plan(cluster, environ, topology)
    host = pplan['stmgrs'][pplan['instances'][instance]['stmgrId']]['host']
    result = json.loads((yield access.get_instance_pid(
        cluster, environ, topology, instance)))
    self.write('

$%s>: %s

%s
' % (
        host,
        tornado.escape.xhtml_escape(result['command']),
        tornado.escape.xhtml_escape(result['stdout'])))


Instruction:
Given the following Python 3 function, write the documentation
def renderProcessStdErr(self, stderr_line):
    retcode = self.process.poll()
    if retcode is not None and status_type(retcode) == Status.InvocationError:
      self._do_log(Log.error, stderr_line)
    else:
      self._do_print(sys.stderr, stderr_line)

Response: render stderr of shelled - out process


Instruction:
Explain what the following Python 3 code does
def renderProcessStdOut(self, stdout):
    # since we render stdout line based on Java process return code,
    # ``status'' has to be already set
    assert self.status is not None
    # remove pending newline
    if self.status == Status.Ok:
      self._do_log(Log.info, stdout)
    elif self.status == Status.HeronError:
      # remove last newline since logging will append newline
      self._do_log(Log.error, stdout)
    # No need to prefix [INFO] here. We want to display dry-run response in a clean way
    elif self.status == Status.DryRun:
      self._do_print(sys.stdout, stdout)
    elif self.status == Status.InvocationError:
      self._do_print(sys.stdout, stdout)
    else:
      raise RuntimeError(
          "Unknown status type of value %d. Expected value: %s" % \
          (self.status.value, list(Status)))

Response: render process standard output of process


Instruction:
Write a Python 3 script to
return true if the host is reachable.

Response:
def is_host_port_reachable(self):
    """
    Returns true if the host is reachable. In some cases, it may not be reachable a tunnel
    must be used.
    """
    for hostport in self.hostportlist:
      try:
        socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS)
        return True
      except:
        LOG.info("StateManager %s Unable to connect to host: %s port %i"
                 % (self.name, hostport[0], hostport[1]))
        continue
    return False


Instruction:
Can you write a function in Python 3 where it
picks an unused port.

Response:
def pick_unused_port(self):
    """ Pick an unused port. There is a slight chance that this wont work. """
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('127.0.0.1', 0))
    _, port = s.getsockname()
    s.close()
    return port


Instruction:
Implement a Python 3 function for
establishing an ssh tunnel for each local host and port.

Response:
def establish_ssh_tunnel(self):
    """
    Establish an ssh tunnel for each local host and port
    that can be used to communicate with the state host.
    """
    localportlist = []
    for (host, port) in self.hostportlist:
      localport = self.pick_unused_port()
      self.tunnel.append(subprocess.Popen(
          ('ssh', self.tunnelhost, '-NL127.0.0.1:%d:%s:%d' % (localport, host, port))))
      localportlist.append(('127.0.0.1', localport))
    return localportlist


Instruction:
How would you code a function in Python 3 to
remove the topology from the Zookeeper.

Response:
def delete_topology_from_zk(self, topologyName):
    """
    Removes the topology entry from:
    1. topologies list,
    2. pplan,
    3. execution_state, and
    """
    self.delete_pplan(topologyName)
    self.delete_execution_state(topologyName)
    self.delete_topology(topologyName)


Instruction:
Can you generate the documentation for the following Python 3 function
def monitor(self):

    def trigger_watches_based_on_files(watchers, path, directory, ProtoClass):
      """
      For all the topologies in the watchers, check if the data
      in directory has changed. Trigger the callback if it has.
      """
      for topology, callbacks in watchers.items():
        file_path = os.path.join(path, topology)
        data = ""
        if os.path.exists(file_path):
          with open(os.path.join(path, topology)) as f:
            data = f.read()
        if topology not in directory or data != directory[topology]:
          proto_object = ProtoClass()
          proto_object.ParseFromString(data)
          for callback in callbacks:
            callback(proto_object)
          directory[topology] = data

    while not self.monitoring_thread_stop_signal:
      topologies_path = self.get_topologies_path()

      topologies = []
      if os.path.isdir(topologies_path):
        topologies = list(filter(
            lambda f: os.path.isfile(os.path.join(topologies_path, f)),
            os.listdir(topologies_path)))
      if set(topologies) != set(self.topologies_directory):
        for callback in self.topologies_watchers:
          callback(topologies)
      self.topologies_directory = topologies

      trigger_watches_based_on_files(
          self.topology_watchers, topologies_path, self.topologies_directory, Topology)

      # Get the directory name for execution state
      execution_state_path = os.path.dirname(self.get_execution_state_path(""))
      trigger_watches_based_on_files(
          self.execution_state_watchers, execution_state_path,
          self.execution_state_directory, ExecutionState)

      # Get the directory name for packing_plan
      packing_plan_path = os.path.dirname(self.get_packing_plan_path(""))
      trigger_watches_based_on_files(
          self.packing_plan_watchers, packing_plan_path, self.packing_plan_directory, PackingPlan)

      # Get the directory name for pplan
      pplan_path = os.path.dirname(self.get_pplan_path(""))
      trigger_watches_based_on_files(
          self.pplan_watchers, pplan_path,
          self.pplan_directory, PhysicalPlan)

      # Get the directory name for tmaster
      tmaster_path = os.path.dirname(self.get_tmaster_path(""))
      trigger_watches_based_on_files(
          self.tmaster_watchers, tmaster_path,
          self.tmaster_directory, TMasterLocation)

      # Get the directory name for scheduler location
      scheduler_location_path = os.path.dirname(self.get_scheduler_location_path(""))
      trigger_watches_based_on_files(
          self.scheduler_location_watchers, scheduler_location_path,
          self.scheduler_location_directory, SchedulerLocation)

      # Sleep for some time
      self.event.wait(timeout=5)

Response: Monitor the rootpath and call the callback functions for each topology in the directory.


Instruction:
Can you write a function in Python 3 where it
gets topology from file

Response:
def get_topology(self, topologyName, callback=None):
    """get topology"""
    if callback:
      self.topology_watchers[topologyName].append(callback)
    else:
      topology_path = self.get_topology_path(topologyName)
      with open(topology_path) as f:
        data = f.read()
        topology = Topology()
        topology.ParseFromString(data)
        return topology


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_pplan(self, topologyName, callback=None):
    if callback:
      self.pplan_watchers[topologyName].append(callback)
    else:
      pplan_path = self.get_pplan_path(topologyName)
      with open(pplan_path) as f:
        data = f.read()
        pplan = PhysicalPlan()
        pplan.ParseFromString(data)
        return pplan

Response: Get a physical plan of a topology


Instruction:
Create a Python 3 function for
getting the execution state of a topology

Response:
def get_execution_state(self, topologyName, callback=None):
    """
    Get execution state
    """
    if callback:
      self.execution_state_watchers[topologyName].append(callback)
    else:
      execution_state_path = self.get_execution_state_path(topologyName)
      with open(execution_state_path) as f:
        data = f.read()
        executionState = ExecutionState()
        executionState.ParseFromString(data)
        return executionState


Instruction:
Write a Python 3 script for
getting the TMasterLocation object for a given topology name.

Response:
def get_tmaster(self, topologyName, callback=None):
    """
    Get tmaster
    """
    if callback:
      self.tmaster_watchers[topologyName].append(callback)
    else:
      tmaster_path = self.get_tmaster_path(topologyName)
      with open(tmaster_path) as f:
        data = f.read()
        tmaster = TMasterLocation()
        tmaster.ParseFromString(data)
        return tmaster


Instruction:
Can you generate the documentation for the following Python 3 function
def create_socket_options():
  sys_config = system_config.get_sys_config()
  opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,
              const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,
              const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,
              const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,
              const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,
              const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]

  Log.debug("In create_socket_options()")
  try:
    value_lst = [int(sys_config[opt]) for opt in opt_list]
    sock_opt = SocketOptions(*value_lst)
    return sock_opt
  except ValueError as e:
    # couldn't convert to int
    raise ValueError("Invalid value in sys_config: %s" % str(e))
  except KeyError as e:
    # option key was not found
    raise KeyError("Incomplete sys_config: %s" % str(e))

Response: Creates SocketOptions object from a given sys_config dict


Instruction:
Implement a function in Python 3 to
take a class __dict__ and returns a HeronComponentSpec entries

Response:
def class_dict_to_specs(mcs, class_dict):
    """Takes a class `__dict__` and returns `HeronComponentSpec` entries"""
    specs = {}

    for name, spec in class_dict.items():
      if isinstance(spec, HeronComponentSpec):
        # Use the variable name as the specification name.
        if spec.name is None:
          spec.name = name
        if spec.name in specs:
          raise ValueError("Duplicate component name: %s" % spec.name)
        else:
          specs[spec.name] = spec
    return specs


Instruction:
Can you create a Python 3 function that
takes a class dict and returns a map containing topology - wide configuration.

Response:
def class_dict_to_topo_config(mcs, class_dict):
    """
    Takes a class `__dict__` and returns a map containing topology-wide
    configuration.

    The returned dictionary is a sanitized `dict` of type `<str ->
    (str|object)>`.

    This classmethod firsts insert default topology configuration and then
    overrides it with a given topology-wide configuration. Note that this
    configuration will be overriden by a component-specific configuration at
    runtime.
    """
    topo_config = {}

    # add defaults
    topo_config.update(mcs.DEFAULT_TOPOLOGY_CONFIG)

    for name, custom_config in class_dict.items():
      if name == 'config' and isinstance(custom_config, dict):
        sanitized_dict = mcs._sanitize_config(custom_config)
        topo_config.update(sanitized_dict)

    return topo_config


Instruction:
Write a Python 3 script to
initialize a topology protobuf

Response:
def init_topology(mcs, classname, class_dict):
    """Initializes a topology protobuf"""
    if classname == 'Topology':
      # Base class can't initialize protobuf
      return
    heron_options = TopologyType.get_heron_options_from_env()
    initial_state = heron_options.get("cmdline.topology.initial.state", "RUNNING")
    tmp_directory = heron_options.get("cmdline.topologydefn.tmpdirectory")
    if tmp_directory is None:
      raise RuntimeError("Topology definition temp directory not specified")

    topology_name = heron_options.get("cmdline.topology.name", classname)
    topology_id = topology_name + str(uuid.uuid4())

    # create protobuf
    topology = topology_pb2.Topology()
    topology.id = topology_id
    topology.name = topology_name
    topology.state = topology_pb2.TopologyState.Value(initial_state)
    topology.topology_config.CopyFrom(TopologyType.get_topology_config_protobuf(class_dict))

    TopologyType.add_bolts_and_spouts(topology, class_dict)

    class_dict['topology_name'] = topology_name
    class_dict['topology_id'] = topology_id
    class_dict['protobuf_topology'] = topology
    class_dict['topologydefn_tmpdir'] = tmp_directory
    class_dict['heron_runtime_options'] = heron_options


Instruction:
Explain what the following Python 3 code does
def get_heron_options_from_env():
    heron_options_raw = os.environ.get("HERON_OPTIONS")
    if heron_options_raw is None:
      raise RuntimeError("HERON_OPTIONS environment variable not found")

    options = {}
    for option_line in heron_options_raw.replace("%%%%", " ").split(','):
      key, sep, value = option_line.partition("=")
      if sep:
        options[key] = value
      else:
        raise ValueError("Invalid HERON_OPTIONS part %r" % option_line)
    return options

Response: Retrieves heron options from the HERON_OPTIONS environment variable.


Instruction:
Write a Python 3 script for
adding a set of HeronComponentSpecs to the topology.

Response:
def add_spec(self, *specs):
    """Add specs to the topology

    :type specs: HeronComponentSpec
    :param specs: specs to add to the topology
    """
    for spec in specs:
      if not isinstance(spec, HeronComponentSpec):
        raise TypeError("Argument to add_spec needs to be HeronComponentSpec, given: %s"
                        % str(spec))
      if spec.name is None:
        raise ValueError("TopologyBuilder cannot take a spec without name")
      if spec.name == "config":
        raise ValueError("config is a reserved name")
      if spec.name in self._specs:
        raise ValueError("Attempting to add duplicate spec name: %r %r" % (spec.name, spec))

      self._specs[spec.name] = spec


Instruction:
Can you create a Python 3 function that
adds a spout to the topology

Response:
def add_spout(self, name, spout_cls, par, config=None, optional_outputs=None):
    """Add a spout to the topology"""
    spout_spec = spout_cls.spec(name=name, par=par, config=config,
                                optional_outputs=optional_outputs)
    self.add_spec(spout_spec)
    return spout_spec


Instruction:
Explain what the following Python 3 code does
def add_bolt(self, name, bolt_cls, par, inputs, config=None, optional_outputs=None):
    bolt_spec = bolt_cls.spec(name=name, par=par, inputs=inputs, config=config,
                              optional_outputs=optional_outputs)
    self.add_spec(bolt_spec)
    return bolt_spec

Response: Add a bolt to the topology


Instruction:
Can you generate the documentation for the following Python 3 function
def set_config(self, config):
    if not isinstance(config, dict):
      raise TypeError("Argument to set_config needs to be dict, given: %s" % str(config))
    self._topology_config = config

Response: Set the topology - wide configuration to the topology - wide configuration.


Instruction:
Create a Python 3 function to
build the topology and submits to the destination

Response:
def build_and_submit(self):
    """Builds the topology and submits to the destination"""
    class_dict = self._construct_topo_class_dict()
    topo_cls = TopologyType(self.topology_name, (Topology,), class_dict)
    topo_cls.write()


Instruction:
Write a Python 3 script to
fetch the given url and convert the response to json.

Response:
def fetch_url_as_json(fetch_url, default_value=None):
  '''
  Fetch the given url and convert the response to json.
  :param fetch_url: URL to fetch
  :param default_value: value to return in case of failure
  :return:
  '''
  # assign empty dict for optional param
  if default_value is None:
    default_value = dict()

  Log.debug("fetching url %s", fetch_url)
  ret = default_value

  # time the duration of the fetch
  start = time.time()

  # fetch the URL asynchronously
  http_response = yield tornado.httpclient.AsyncHTTPClient().fetch(fetch_url)

  # handle http errors, and return if any
  if http_response.error:
    Log.error("Unable to get response from %s. Error %s", fetch_url, http_response.error)
    raise tornado.gen.Return(ret)

  # load response and handle return errors, if any
  response = json.loads(http_response.body)
  if not 'result' in response:
    Log.error("Empty response from %s", fetch_url)
    raise tornado.gen.Return(ret)

  # get the response and execution time on server side
  ret = response['result']
  execution = 1000 * response['executiontime']

  # calculate the time
  end = time.time()
  duration = 1000 * (end - start)

  Log.debug("TIME: url fetch took %.2f ms server time %s", execution, fetch_url)
  Log.debug("TIME: url fetch took %.2f ms round trip  %s", duration, fetch_url)

  # convert future to value
  raise tornado.gen.Return(ret)


Instruction:
Can you implement a function in Python 3 that
maps from query parameter to query name

Response:
def queries_map():
  """map from query parameter to query name"""
  qs = _all_metric_queries()
  return dict(zip(qs[0], qs[1]) + zip(qs[2], qs[3]))


Instruction:
Given the following Python 3 function, write the documentation
def get_clusters():
  instance = tornado.ioloop.IOLoop.instance()
  # pylint: disable=unnecessary-lambda
  try:
    return instance.run_sync(lambda: API.get_clusters())
  except Exception:
    Log.debug(traceback.format_exc())
    raise

Response: Synced API call to get all cluster names


Instruction:
Can you generate the documentation for the following Python 3 function
def get_topology_info(*args):
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_topology_info(*args))
  except Exception:
    Log.debug(traceback.format_exc())
    raise

Response: Synced API call to get topology information


Instruction:
Can you tell what is the following Python 3 function doing
def configure(level=logging.INFO, logfile=None):

  # Remove all the existing StreamHandlers to avoid duplicate
  for handler in Log.handlers:
    if isinstance(handler, logging.StreamHandler):
      Log.handlers.remove(handler)

  Log.setLevel(level)

  # if logfile is specified, FileHandler is used
  if logfile is not None:
    log_format = "[%(asctime)s] [%(levelname)s]: %(message)s"
    formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
    file_handler = logging.FileHandler(logfile)
    file_handler.setFormatter(formatter)
    Log.addHandler(file_handler)
  # otherwise, use StreamHandler to output to stream (stdout, stderr...)
  else:
    log_format = "[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s"
    # pylint: disable=redefined-variable-type
    formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    Log.addHandler(stream_handler)

Response: Configure logger which dumps log on terminal


Instruction:
Can you generate a brief explanation for the following Python 3 code
def init_rotating_logger(level, logfile, max_files, max_bytes):
  logging.basicConfig()

  root_logger = logging.getLogger()
  log_format = "[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s"

  root_logger.setLevel(level)
  handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)
  handler.setFormatter(logging.Formatter(fmt=log_format, datefmt=date_format))
  root_logger.addHandler(handler)

  for handler in root_logger.handlers:
    root_logger.debug("Associated handlers - " + str(handler))
    if isinstance(handler, logging.StreamHandler):
      root_logger.debug("Removing StreamHandler: " + str(handler))
      root_logger.handlers.remove(handler)

Response: Initializes a rotating logger with the given level logfile and max_bytes.


Instruction:
Can you generate the documentation for the following Python 3 function
def set_logging_level(cl_args):
  if 'verbose' in cl_args and cl_args['verbose']:
    configure(logging.DEBUG)
  else:
    configure(logging.INFO)

Response: simply set verbose level based on command - line args


Instruction:
Can you tell what is the following Python 3 function doing
def _get_spout(self):
    spout = topology_pb2.Spout()
    spout.comp.CopyFrom(self._get_base_component())

    # Add output streams
    self._add_out_streams(spout)
    return spout

Response: Returns a Spout protobuf message


Instruction:
Here you have a function in Python 3, explain what it does
def _get_bolt(self):
    bolt = topology_pb2.Bolt()
    bolt.comp.CopyFrom(self._get_base_component())

    # Add streams
    self._add_in_streams(bolt)
    self._add_out_streams(bolt)
    return bolt

Response: Returns Bolt protobuf message


Instruction:
Implement a Python 3 function for
returning Component protobuf message

Response:
def _get_base_component(self):
    """Returns Component protobuf message"""
    comp = topology_pb2.Component()
    comp.name = self.name
    comp.spec = topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")
    comp.class_name = self.python_class_path
    comp.config.CopyFrom(self._get_comp_config())
    return comp


Instruction:
Create a Python 3 function to
return the component - specific Config protobuf message that is used by the component.

Response:
def _get_comp_config(self):
    """Returns component-specific Config protobuf message

    It first adds ``topology.component.parallelism``, and is overriden by
    a user-defined component-specific configuration, specified by spec().
    """
    proto_config = topology_pb2.Config()

    # first add parallelism
    key = proto_config.kvs.add()
    key.key = TOPOLOGY_COMPONENT_PARALLELISM
    key.value = str(self.parallelism)
    key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")

    # iterate through self.custom_config
    if self.custom_config is not None:
      sanitized = self._sanitize_config(self.custom_config)
      for key, value in sanitized.items():
        if isinstance(value, str):
          kvs = proto_config.kvs.add()
          kvs.key = key
          kvs.value = value
          kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
        else:
          # need to serialize
          kvs = proto_config.kvs.add()
          kvs.key = key
          kvs.serialized_value = default_serializer.serialize(value)
          kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")

    return proto_config


Instruction:
Here you have a function in Python 3, explain what it does
def _sanitize_config(custom_config):
    if not isinstance(custom_config, dict):
      raise TypeError("Component-specific configuration must be given as a dict type, given: %s"
                      % str(type(custom_config)))
    sanitized = {}
    for key, value in custom_config.items():
      if not isinstance(key, str):
        raise TypeError("Key for component-specific configuration must be string, given: %s:%s"
                        % (str(type(key)), str(key)))

      if isinstance(value, bool):
        sanitized[key] = "true" if value else "false"
      elif isinstance(value, (str, int, float)):
        sanitized[key] = str(value)
      else:
        sanitized[key] = value

    return sanitized

Response: Checks whether a given dictionary is sane and returns a sanitized dict that is suitable for use in a protobuf message.


Instruction:
Write a Python 3 function for
adding inputs to a given protobuf Bolt message

Response:
def _add_in_streams(self, bolt):
    """Adds inputs to a given protobuf Bolt message"""
    if self.inputs is None:
      return
    # sanitize inputs and get a map <GlobalStreamId -> Grouping>
    input_dict = self._sanitize_inputs()

    for global_streamid, gtype in input_dict.items():
      in_stream = bolt.inputs.add()
      in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,
                                                    global_streamid.stream_id))
      if isinstance(gtype, Grouping.FIELDS):
        # it's a field grouping
        in_stream.gtype = gtype.gtype
        in_stream.grouping_fields.CopyFrom(self._get_stream_schema(gtype.fields))
      elif isinstance(gtype, Grouping.CUSTOM):
        # it's a custom grouping
        in_stream.gtype = gtype.gtype
        in_stream.custom_grouping_object = gtype.python_serialized
        in_stream.type = topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT")
      else:
        in_stream.gtype = gtype


Instruction:
Make a summary of the following Python 3 code
def _sanitize_inputs(self):
    ret = {}
    if self.inputs is None:
      return

    if isinstance(self.inputs, dict):
      # inputs are dictionary, must be either <HeronComponentSpec -> Grouping> or
      # <GlobalStreamId -> Grouping>
      for key, grouping in self.inputs.items():
        if not Grouping.is_grouping_sane(grouping):
          raise ValueError('A given grouping is not supported')
        if isinstance(key, HeronComponentSpec):
          # use default streamid
          if key.name is None:
            # should not happen as TopologyType metaclass sets name attribute
            # before calling this method
            raise RuntimeError("In _sanitize_inputs(): HeronComponentSpec doesn't have a name")
          global_streamid = GlobalStreamId(key.name, Stream.DEFAULT_STREAM_ID)
          ret[global_streamid] = grouping
        elif isinstance(key, GlobalStreamId):
          ret[key] = grouping
        else:
          raise ValueError("%s is not supported as a key to inputs" % str(key))
    elif isinstance(self.inputs, (list, tuple)):
      # inputs are lists, must be either a list of HeronComponentSpec or GlobalStreamId
      # will use SHUFFLE grouping
      for input_obj in self.inputs:
        if isinstance(input_obj, HeronComponentSpec):
          if input_obj.name is None:
            # should not happen as TopologyType metaclass sets name attribute
            # before calling this method
            raise RuntimeError("In _sanitize_inputs(): HeronComponentSpec doesn't have a name")
          global_streamid = GlobalStreamId(input_obj.name, Stream.DEFAULT_STREAM_ID)
          ret[global_streamid] = Grouping.SHUFFLE
        elif isinstance(input_obj, GlobalStreamId):
          ret[input_obj] = Grouping.SHUFFLE
        else:
          raise ValueError("%s is not supported as an input" % str(input_obj))
    else:
      raise TypeError("Inputs must be a list, dict, or None, given: %s" % str(self.inputs))

    return ret

Response: Sanitize the inputs of the HeronComponentSpec and return a map of GlobalStreamId - > Grouping


Instruction:
Given the following Python 3 function, write the documentation
def _add_out_streams(self, spbl):
    if self.outputs is None:
      return

    # sanitize outputs and get a map <stream_id -> out fields>
    output_map = self._sanitize_outputs()

    for stream_id, out_fields in output_map.items():
      out_stream = spbl.outputs.add()
      out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))
      out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))

Response: Adds outputs to a given protobuf Bolt or Spout message


Instruction:
Explain what the following Python 3 code does
def _sanitize_outputs(self):
    ret = {}
    if self.outputs is None:
      return

    if not isinstance(self.outputs, (list, tuple)):
      raise TypeError("Argument to outputs must be either list or tuple, given: %s"
                      % str(type(self.outputs)))

    for output in self.outputs:
      if not isinstance(output, (str, Stream)):
        raise TypeError("Outputs must be a list of strings or Streams, given: %s" % str(output))

      if isinstance(output, str):
        # it's a default stream
        if Stream.DEFAULT_STREAM_ID not in ret:
          ret[Stream.DEFAULT_STREAM_ID] = list()
        ret[Stream.DEFAULT_STREAM_ID].append(output)
      else:
        # output is a Stream object
        if output.stream_id == Stream.DEFAULT_STREAM_ID and Stream.DEFAULT_STREAM_ID in ret:
          # some default stream fields are already in there
          ret[Stream.DEFAULT_STREAM_ID].extend(output.fields)
        else:
          ret[output.stream_id] = output.fields
    return ret

Response: Sanitize the outputs and returns a map <stream_id > = list of output fields


Instruction:
Write a Python 3 script for
returning a set of output stream ids registered for this component

Response:
def get_out_streamids(self):
    """Returns a set of output stream ids registered for this component"""
    if self.outputs is None:
      return set()

    if not isinstance(self.outputs, (list, tuple)):
      raise TypeError("Argument to outputs must be either list or tuple, given: %s"
                      % str(type(self.outputs)))
    ret_lst = []
    for output in self.outputs:
      if not isinstance(output, (str, Stream)):
        raise TypeError("Outputs must be a list of strings or Streams, given: %s" % str(output))
      ret_lst.append(Stream.DEFAULT_STREAM_ID if isinstance(output, str) else output.stream_id)
    return set(ret_lst)


Instruction:
Make a summary of the following Python 3 code
def _get_stream_id(comp_name, stream_id):
    proto_stream_id = topology_pb2.StreamId()
    proto_stream_id.id = stream_id
    proto_stream_id.component_name = comp_name
    return proto_stream_id

Response: Returns a StreamId protobuf message


Instruction:
Write a Python 3 script for
returning a StreamSchema protobuf message for the given fields.

Response:
def _get_stream_schema(fields):
    """Returns a StreamSchema protobuf message"""
    stream_schema = topology_pb2.StreamSchema()
    for field in fields:
      key = stream_schema.keys.add()
      key.key = field
      key.type = topology_pb2.Type.Value("OBJECT")

    return stream_schema


Instruction:
Given the following Python 3 function, write the documentation
def component_id(self):
    if isinstance(self._component_id, HeronComponentSpec):
      if self._component_id.name is None:
        # HeronComponentSpec instance's name attribute might not be available until
        # TopologyType metaclass finally sets it. This statement is to support __eq__(),
        # __hash__() and __str__() methods with safety, as raising Exception is not
        # appropriate this case.
        return "<No name available for HeronComponentSpec yet, uuid: %s>" % self._component_id.uuid
      return self._component_id.name
    elif isinstance(self._component_id, str):
      return self._component_id
    else:
      raise ValueError("Component Id for this GlobalStreamId is not properly set: <%s:%s>"
                       % (str(type(self._component_id)), str(self._component_id)))

Response: Returns the component_id of this GlobalStreamId


Instruction:
Create a Python 3 function for
writing an error message to the output buffer.

Response:
def write_error(self, status_code, **kwargs):
    '''
    :param status_code:
    :param kwargs:
    :return:
    '''
    if "exc_info" in kwargs:
      exc_info = kwargs["exc_info"]
      error = exc_info[1]

      errormessage = "%s: %s" % (status_code, error)
      self.render("error.html", errormessage=errormessage)
    else:
      errormessage = "%s" % (status_code)
      self.render("error.html", errormessage=errormessage)


Instruction:
Can you generate the documentation for the following Python 3 function
def register_metric(self, name, metric, time_bucket_in_sec):
    collector = self.get_metrics_collector()
    collector.register_metric(name, metric, time_bucket_in_sec)

Response: Registers a new metric to this context


Instruction:
Can you generate the documentation for the following Python 3 function
def get_sources(self, component_id):
    # this is necessary because protobuf message is not hashable
    StreamId = namedtuple('StreamId', 'id, component_name')
    if component_id in self.inputs:
      ret = {}
      for istream in self.inputs.get(component_id):
        key = StreamId(id=istream.stream.id, component_name=istream.stream.component_name)
        ret[key] = istream.gtype
      return ret
    else:
      return None

Response: Returns the declared inputs to specified component_id


Instruction:
Can you tell what is the following Python 3 function doing
def get_component_tasks(self, component_id):
    ret = []
    for task_id, comp_id in self.task_to_component_map.items():
      if comp_id == component_id:
        ret.append(task_id)
    return ret

Response: Returns the list of task ids allocated for the given component id


Instruction:
Given the following Python 3 function, write the documentation
def add_task_hook(self, task_hook):
    if not isinstance(task_hook, ITaskHook):
      raise TypeError("In add_task_hook(): attempt to add non ITaskHook instance, given: %s"
                      % str(type(task_hook)))
    self.task_hooks.append(task_hook)

Response: Registers a specified task hook to this context


Instruction:
Create a Python 3 function for
returning this context s metrics collector

Response:
def get_metrics_collector(self):
    """Returns this context's metrics collector"""
    if self.metrics_collector is None or not isinstance(self.metrics_collector, MetricsCollector):
      raise RuntimeError("Metrics collector is not registered in this context")
    return self.metrics_collector


Instruction:
Can you tell what is the following Python 3 function doing
def invoke_hook_prepare(self):
    for task_hook in self.task_hooks:
      task_hook.prepare(self.get_cluster_config(), self)

Response: invoke task hooks for after the spout s initialize method


Instruction:
How would you explain what the following Python 3 function does
def invoke_hook_emit(self, values, stream_id, out_tasks):
    if len(self.task_hooks) > 0:
      emit_info = EmitInfo(values=values, stream_id=stream_id,
                           task_id=self.get_task_id(), out_tasks=out_tasks)
      for task_hook in self.task_hooks:
        task_hook.emit(emit_info)

Response: invoke task hooks for every time a tuple is emitted in spout or bolt


Instruction:
How would you explain what the following Python 3 function does
def invoke_hook_spout_ack(self, message_id, complete_latency_ns):
    if len(self.task_hooks) > 0:
      spout_ack_info = SpoutAckInfo(message_id=message_id,
                                    spout_task_id=self.get_task_id(),
                                    complete_latency_ms=complete_latency_ns *
                                    system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.spout_ack(spout_ack_info)

Response: invoke task hooks for every time spout acks a tuple


Instruction:
Explain what the following Python 3 code does
def invoke_hook_spout_fail(self, message_id, fail_latency_ns):
    if len(self.task_hooks) > 0:
      spout_fail_info = SpoutFailInfo(message_id=message_id,
                                      spout_task_id=self.get_task_id(),
                                      fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.spout_fail(spout_fail_info)

Response: invoke task hooks for every time spout fails a tuple


Instruction:
Given the following Python 3 function, write the documentation
def invoke_hook_bolt_execute(self, heron_tuple, execute_latency_ns):
    if len(self.task_hooks) > 0:
      bolt_execute_info = \
        BoltExecuteInfo(heron_tuple=heron_tuple,
                        executing_task_id=self.get_task_id(),
                        execute_latency_ms=execute_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_execute(bolt_execute_info)

Response: invoke task hooks for every time bolt processes a tuple


Instruction:
Write a Python 3 script for
invoking task hooks for every time bolt acks a tuple

Response:
def invoke_hook_bolt_ack(self, heron_tuple, process_latency_ns):
    """invoke task hooks for every time bolt acks a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is acked
    :type process_latency_ns: float
    :param process_latency_ns: process latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_ack_info = BoltAckInfo(heron_tuple=heron_tuple,
                                  acking_task_id=self.get_task_id(),
                                  process_latency_ms=process_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_ack(bolt_ack_info)


Instruction:
Implement a Python 3 function for
invoking task hooks for every time bolt fails a tuple

Response:
def invoke_hook_bolt_fail(self, heron_tuple, fail_latency_ns):
    """invoke task hooks for every time bolt fails a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is failed
    :type fail_latency_ns: float
    :param fail_latency_ns: fail latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_fail_info = BoltFailInfo(heron_tuple=heron_tuple,
                                    failing_task_id=self.get_task_id(),
                                    fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_fail(bolt_fail_info)


Instruction:
How would you code a function in Python 3 to
create a subparser for the submit command

Response:
def create_parser(subparsers):
  '''
  Create a subparser for the submit command
  :param subparsers:
  :return:
  '''
  parser = subparsers.add_parser(
      'submit',
      help='Submit a topology',
      usage="%(prog)s [options] cluster/[role]/[env] " + \
            "topology-file-name topology-class-name [topology-args]",
      add_help=True
  )

  cli_args.add_titles(parser)
  cli_args.add_cluster_role_env(parser)
  cli_args.add_topology_file(parser)
  cli_args.add_topology_class(parser)
  cli_args.add_config(parser)
  cli_args.add_deactive_deploy(parser)
  cli_args.add_dry_run(parser)
  cli_args.add_extra_launch_classpath(parser)
  cli_args.add_release_yaml_file(parser)
  cli_args.add_service_url(parser)
  cli_args.add_system_property(parser)
  cli_args.add_verbose(parser)

  parser.set_defaults(subcommand='submit')
  return parser


Instruction:
Write a Python 3 script to
launch a topology given a topology jar its definition file and configurations This function is used to launch a topology given a topology jar its definition file and configurations This function is used to launch a topology given a topology jar and its configuration file and the configuration file and the configuration file.

Response:
def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param tmp_dir:
  :param topology_file:
  :param topology_defn_file:
  :param topology_name:
  :return:
  '''
  # get the normalized path for topology.tar.gz
  topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))

  # get the release yaml file
  release_yaml_file = cl_args['release_yaml_file']

  # create a tar package with the cluster configuration and generated config files
  config_path = cl_args['config_path']
  tar_pkg_files = [topology_file, topology_defn_file]
  generated_config_files = [release_yaml_file, cl_args['override_config_file']]

  config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)

  # pass the args to submitter main
  args = [
      "--cluster", cl_args['cluster'],
      "--role", cl_args['role'],
      "--environment", cl_args['environ'],
      "--submit_user", cl_args['submit_user'],
      "--heron_home", config.get_heron_dir(),
      "--config_path", config_path,
      "--override_config_file", cl_args['override_config_file'],
      "--release_file", release_yaml_file,
      "--topology_package", topology_pkg_path,
      "--topology_defn", topology_defn_file,
      "--topology_bin", os.path.basename(topology_file)   # pex/cpp file if pex/cpp specified
  ]

  if Log.getEffectiveLevel() == logging.DEBUG:
    args.append("--verbose")

  if cl_args["dry_run"]:
    args.append("--dry_run")
    if "dry_run_format" in cl_args:
      args += ["--dry_run_format", cl_args["dry_run_format"]]

  lib_jars = config.get_heron_libs(
      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()
  )
  extra_jars = cl_args['extra_launch_classpath'].split(':')

  # invoke the submitter to submit and launch the topology
  main_class = 'org.apache.heron.scheduler.SubmitterMain'
  res = execute.heron_class(
      class_name=main_class,
      lib_jars=lib_jars,
      extra_jars=extra_jars,
      args=args,
      java_defines=[])

  err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
  succ_ctxt = "Successfully launched topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))

  res.add_context(err_ctxt, succ_ctxt)
  return res


Instruction:
Can you tell what is the following Python 3 function doing
def launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name):
  '''
  Launch a topology given topology jar, its definition file and configurations
  :param cl_args:
  :param topology_file:
  :param topology_defn_file:
  :param topology_name:
  :return:
  '''
  service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1]
  service_method = rest.ROUTE_SIGNATURES['submit'][0]
  data = dict(
      name=topology_name,
      cluster=cl_args['cluster'],
      role=cl_args['role'],
      environment=cl_args['environ'],
      user=cl_args['submit_user'],
  )

  Log.info("" + str(cl_args))
  overrides = dict()
  if 'config_property' in cl_args:
    overrides = config.parse_override_config(cl_args['config_property'])

  if overrides:
    data.update(overrides)

  if cl_args['dry_run']:
    data["dry_run"] = True

  files = dict(
      definition=open(topology_defn_file, 'rb'),
      topology=open(topology_file, 'rb'),
  )

  err_ctxt = "Failed to launch topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))
  succ_ctxt = "Successfully launched topology '%s' %s" % (topology_name, launch_mode_msg(cl_args))

  try:
    r = service_method(service_apiurl, data=data, files=files)
    ok = r.status_code is requests.codes.ok
    created = r.status_code is requests.codes.created
    s = Status.Ok if created or ok else Status.HeronError
    if s is Status.HeronError:
      Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
    elif ok:
      # this case happens when we request a dry_run
      print(r.json().get("response"))
  except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
    Log.error(err)
    return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)
  return SimpleResult(s, err_ctxt, succ_ctxt)

Response:
Launch a topology given a topology jar its definition file and configurations
  Launch a topology given a topology jar its definition file and configurations


Instruction:
Can you generate a brief explanation for the following Python 3 code
def launch_topologies(cl_args, topology_file, tmp_dir):
  '''
  Launch topologies
  :param cl_args:
  :param topology_file:
  :param tmp_dir:
  :return: list(Responses)
  '''
  # the submitter would have written the .defn file to the tmp_dir
  defn_files = glob.glob(tmp_dir + '/*.defn')

  if len(defn_files) == 0:
    return SimpleResult(Status.HeronError, "No topologies found under %s" % tmp_dir)

  results = []
  for defn_file in defn_files:
    # load the topology definition from the file
    topology_defn = topology_pb2.Topology()
    try:
      handle = open(defn_file, "rb")
      topology_defn.ParseFromString(handle.read())
      handle.close()
    except Exception as e:
      err_context = "Cannot load topology definition '%s': %s" % (defn_file, e)
      return SimpleResult(Status.HeronError, err_context)

    # launch the topology
    Log.info("Launching topology: \'%s\'%s", topology_defn.name, launch_mode_msg(cl_args))

    # check if we have to do server or direct based deployment
    if cl_args['deploy_mode'] == config.SERVER_MODE:
      res = launch_topology_server(
          cl_args, topology_file, defn_file, topology_defn.name)
    else:
      res = launch_a_topology(
          cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)
    results.append(res)

  return results

Response: Launch all topologies in the topology_file under the tmp_dir.


Instruction:
Write a Python 3 script for
submitting a topology jar file to the Zookeeper.

Response:
def submit_fatjar(cl_args, unknown_args, tmp_dir):
  '''
  We use the packer to make a package for the jar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.

  This will run the jar file with the topology_class_name. The submitter
  inside will write out the topology defn file to a location that
  we specify. Then we write the topology defn file to a well known
  location. We then write to appropriate places in zookeeper
  and launch the scheduler jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']

  main_class = cl_args['topology-class-name']

  res = execute.heron_class(
      class_name=main_class,
      lib_jars=config.get_heron_libs(jars.topology_jars()),
      extra_jars=[topology_file],
      args=tuple(unknown_args),
      java_defines=cl_args['topology_main_jvm_property'])

  result.render(res)

  if not result.is_successful(res):
    err_context = ("Failed to create topology definition " \
      "file when executing class '%s' of file '%s'") % (main_class, topology_file)
    res.add_context(err_context)
    return res

  results = launch_topologies(cl_args, topology_file, tmp_dir)

  return results


Instruction:
Explain what the following Python 3 code does
def submit_tar(cl_args, unknown_args, tmp_dir):
  '''
  Extract and execute the java files inside the tar and then add topology
  definition file created by running submitTopology

  We use the packer to make a package for the tar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
  This will run the jar file with the topology class name.

  The submitter inside will write out the topology defn file to a location
  that we specify. Then we write the topology defn file to a well known
  packer location. We then write to appropriate places in zookeeper
  and launch the aurora jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']
  java_defines = cl_args['topology_main_jvm_property']
  main_class = cl_args['topology-class-name']
  res = execute.heron_tar(
      main_class,
      topology_file,
      tuple(unknown_args),
      tmp_dir,
      java_defines)

  result.render(res)

  if not result.is_successful(res):
    err_context = ("Failed to create topology definition " \
      "file when executing class '%s' of file '%s'") % (main_class, topology_file)
    res.add_context(err_context)
    return res

  return launch_topologies(cl_args, topology_file, tmp_dir)

Response: Submit the topology file to the Zookeeper


Instruction:
How would you code a function in Python 3 to
submit the topology to the scheduler

Response:
def run(command, parser, cl_args, unknown_args):
  '''
  Submits the topology to the scheduler
    * Depending on the topology file name extension, we treat the file as a
      fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).
    * We upload the topology file to the packer, update zookeeper and launch
      scheduler jobs representing that topology
    * You can see your topology in Heron UI
  :param command:
  :param parser:
  :param cl_args:
  :param unknown_args:
  :return:
  '''
  Log.debug("Submit Args %s", cl_args)

  # get the topology file name
  topology_file = cl_args['topology-file-name']

  if urlparse.urlparse(topology_file).scheme:
    cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])
    topology_file = cl_args['topology-file-name']
    Log.debug("download uri to local file: %s", topology_file)

  # check to see if the topology file exists
  if not os.path.isfile(topology_file):
    err_context = "Topology file '%s' does not exist" % topology_file
    return SimpleResult(Status.InvocationError, err_context)

  # check if it is a valid file type
  jar_type = topology_file.endswith(".jar")
  tar_type = topology_file.endswith(".tar") or topology_file.endswith(".tar.gz")
  pex_type = topology_file.endswith(".pex")
  cpp_type = topology_file.endswith(".dylib") or topology_file.endswith(".so")
  if not (jar_type or tar_type or pex_type or cpp_type):
    _, ext_name = os.path.splitext(topology_file)
    err_context = "Unknown file type '%s'. Please use .tar "\
                  "or .tar.gz or .jar or .pex or .dylib or .so file"\
                  % ext_name
    return SimpleResult(Status.InvocationError, err_context)

  # check if extra launch classpath is provided and if it is validate
  if cl_args['extra_launch_classpath']:
    valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath'])
    if not valid_classpath:
      err_context = "One of jar or directory in extra launch classpath does not exist: %s" % \
        cl_args['extra_launch_classpath']
      return SimpleResult(Status.InvocationError, err_context)

  # create a temporary directory for topology definition file
  tmp_dir = tempfile.mkdtemp()
  opts.cleaned_up_files.append(tmp_dir)

  # if topology needs to be launched in deactivated state, do it so
  if cl_args['deploy_deactivated']:
    initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)
  else:
    initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)

  # set the tmp dir and deactivated state in global options
  opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)
  opts.set_config('cmdline.topology.initial.state', initial_state)
  opts.set_config('cmdline.topology.role', cl_args['role'])
  opts.set_config('cmdline.topology.environment', cl_args['environ'])

  # Use CLI release yaml file if the release_yaml_file config is empty
  if not cl_args['release_yaml_file']:
    cl_args['release_yaml_file'] = config.get_heron_release_file()

  # check the extension of the file name to see if it is tar/jar file.
  if jar_type:
    return submit_fatjar(cl_args, unknown_args, tmp_dir)
  elif tar_type:
    return submit_tar(cl_args, unknown_args, tmp_dir)
  elif cpp_type:
    return submit_cpp(cl_args, unknown_args, tmp_dir)
  else:
    return submit_pex(cl_args, unknown_args, tmp_dir)


Instruction:
Write a Python 3 function for
getting method returns file data

Response:
def get(self):
    """ get method """
    try:
      cluster = self.get_argument_cluster()
      role = self.get_argument_role()
      environ = self.get_argument_environ()
      topology_name = self.get_argument_topology()
      container = self.get_argument(constants.PARAM_CONTAINER)
      path = self.get_argument(constants.PARAM_PATH)
      offset = self.get_argument_offset()
      length = self.get_argument_length()
      topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)

      stmgr_id = "stmgr-" + container
      stmgr = topology_info["physical_plan"]["stmgrs"][stmgr_id]
      host = stmgr["host"]
      shell_port = stmgr["shell_port"]
      file_data_url = "http://%s:%d/filedata/%s?offset=%s&length=%s" % \
        (host, shell_port, path, offset, length)

      http_client = tornado.httpclient.AsyncHTTPClient()
      response = yield http_client.fetch(file_data_url)
      self.write_success_response(json.loads(response.body))
      self.finish()
    except Exception as e:
      Log.debug(traceback.format_exc())
      self.write_error_response(e)


Instruction:
Implement a Python 3 function for
implementing TextFile Generator s setup method

Response:
def setup(self, context):
    """Implements TextFile Generator's setup method"""
    myindex = context.get_partition_index()
    self._files_to_consume = self._files[myindex::context.get_num_partitions()]
    self.logger.info("TextFileSpout files to consume %s" % self._files_to_consume)
    self._lines_to_consume = self._get_next_lines()
    self._emit_count = 0


Instruction:
Create a Python 3 function for
adding config to the parser

Response:
def add_config(parser):
  """ add config """
  # the default config path
  default_config_path = config.get_heron_conf_dir()

  parser.add_argument(
      '--config-path',
      metavar='(a string; path to cluster config; default: "' + default_config_path + '")',
      default=os.path.join(config.get_heron_dir(), default_config_path))

  return parser


Instruction:
Given the following Python 3 function, write the documentation
def add_verbose(parser):
  parser.add_argument(
      '--verbose',
      metavar='(a boolean; default: "false")',
      type=bool,
      default=False)
  return parser

Response: add optional verbose argument


Instruction:
How would you explain what the following Python 3 function does
def add_tracker_url(parser):
  parser.add_argument(
      '--tracker_url',
      metavar='(tracker url; default: "' + DEFAULT_TRACKER_URL + '")',
      type=str, default=DEFAULT_TRACKER_URL)
  return parser

Response: add optional tracker_url argument


Instruction:
Can you generate the documentation for the following Python 3 function
def hex_escape(bin_str):
  printable = string.ascii_letters + string.digits + string.punctuation + ' '
  return ''.join(ch if ch in printable else r'0x{0:02x}'.format(ord(ch)) for ch in bin_str)

Response: Hex encode a binary string


Instruction:
How would you code a function in Python 3 to
make the http endpoint for the heron shell

Response:
def make_shell_endpoint(topologyInfo, instance_id):
  """
  Makes the http endpoint for the heron shell
  if shell port is present, otherwise returns None.
  """
  # Format: container_<id>_<instance_id>
  pplan = topologyInfo["physical_plan"]
  stmgrId = pplan["instances"][instance_id]["stmgrId"]
  host = pplan["stmgrs"][stmgrId]["host"]
  shell_port = pplan["stmgrs"][stmgrId]["shell_port"]
  return "http://%s:%d" % (host, shell_port)


Instruction:
Can you tell what is the following Python 3 function doing
def make_shell_logfiles_url(host, shell_port, _, instance_id=None):
  if not shell_port:
    return None
  if not instance_id:
    return "http://%s:%d/browse/log-files" % (host, shell_port)
  else:
    return "http://%s:%d/file/log-files/%s.log.0" % (host, shell_port, instance_id)

Response: Make the url for log - files in heron - shell.


Instruction:
Implement a function in Python 3 to
make the url for log - file data in heron - shell.

Response:
def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length):
  """
  Make the url for log-file data in heron-shell
  from the info stored in stmgr.
  """
  return "http://%s:%d/filedata/log-files/%s.log.0?offset=%s&length=%s" % \
    (host, shell_port, instance_id, offset, length)


Instruction:
Can you tell what is the following Python 3 function doing
def get_heron_tracker_dir():
  path = "/".join(os.path.realpath(__file__).split('/')[:-8])
  return normalized_class_path(path)

Response: This will extract heron tracker directory from. pex file.


Instruction:
Given the following Python 3 function, write the documentation
def parse_config_file(config_file):
  expanded_config_file_path = os.path.expanduser(config_file)
  if not os.path.lexists(expanded_config_file_path):
    return None

  configs = {}
  # Read the configuration file
  with open(expanded_config_file_path, 'r') as f:
    configs = yaml.load(f)

  return configs

Response:
This will parse the config file for the tracker
  × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 × the tracker
 ×


Instruction:
Given the following Python 3 function, write the documentation
def _handle_register_response(self, response):
    if response.status.status != common_pb2.StatusCode.Value("OK"):
      raise RuntimeError("Stream Manager returned a not OK response for register")
    Log.info("We registered ourselves to the Stream Manager")

    self.is_registered = True
    if response.HasField("pplan"):
      Log.info("Handling assignment message from response")
      self._handle_assignment_message(response.pplan)
    else:
      Log.debug("Received a register response with no pplan")

Response: Called when a RegisterInstanceResponse arrives


Instruction:
Explain what the following Python 3 code does
def _handle_assignment_message(self, pplan):
    Log.debug("In handle_assignment_message() of STStmgrClient, Physical Plan: \n%s", str(pplan))
    self.heron_instance_cls.handle_assignment_msg(pplan)

Response: Called when a new NewInstanceAssignmentMessage arrives


Instruction:
Explain what the following Python 3 code does
def decode_packet(packet):
    if not packet.is_complete:
      raise RuntimeError("In decode_packet(): Packet corrupted")

    data = packet.data

    len_typename = HeronProtocol.unpack_int(data[:4])
    data = data[4:]

    typename = data[:len_typename]
    data = data[len_typename:]

    reqid = REQID.unpack(data[:REQID.REQID_SIZE])
    data = data[REQID.REQID_SIZE:]

    len_msg = HeronProtocol.unpack_int(data[:4])
    data = data[4:]

    serialized_msg = data[:len_msg]

    return typename, reqid, serialized_msg

Response: Decodes an IncomingPacket object and returns a tuple containing the type name reqid and serialized message.


Instruction:
Write a Python 3 function for
creating a new Outgoing Packet from a given REQID and message.

Response:
def create_packet(reqid, message):
    """Creates Outgoing Packet from a given reqid and message

    :param reqid: REQID object
    :param message: protocol buffer object
    """
    assert message.IsInitialized()
    packet = ''

    # calculate the totla size of the packet incl. header
    typename = message.DESCRIPTOR.full_name

    datasize = HeronProtocol.get_size_to_pack_string(typename) + \
               REQID.REQID_SIZE + HeronProtocol.get_size_to_pack_message(message)

    # first write out how much data is there as the header
    packet += HeronProtocol.pack_int(datasize)

    # next write the type string
    packet += HeronProtocol.pack_int(len(typename))
    packet += typename

    # reqid
    packet += reqid.pack()

    # add the proto
    packet += HeronProtocol.pack_int(message.ByteSize())
    packet += message.SerializeToString()
    return OutgoingPacket(packet)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def send(self, dispatcher):
    if self.sent_complete:
      return

    sent = dispatcher.send(self.to_send)
    self.to_send = self.to_send[sent:]

Response: Sends this outgoing packet to dispatcher s socket


Instruction:
Implement a function in Python 3 to
create an IncomingPacket object from header and data

Response:
def create_packet(header, data):
    """Creates an IncomingPacket object from header and data

    This method is for testing purposes
    """
    packet = IncomingPacket()
    packet.header = header
    packet.data = data

    if len(header) == HeronProtocol.HEADER_SIZE:
      packet.is_header_read = True
      if len(data) == packet.get_datasize():
        packet.is_complete = True

    return packet


Instruction:
Here you have a function in Python 3, explain what it does
def read(self, dispatcher):
    try:
      if not self.is_header_read:
        # try reading header
        to_read = HeronProtocol.HEADER_SIZE - len(self.header)
        self.header += dispatcher.recv(to_read)
        if len(self.header) == HeronProtocol.HEADER_SIZE:
          self.is_header_read = True
        else:
          Log.debug("Header read incomplete; read %d bytes of header" % len(self.header))
          return

      if self.is_header_read and not self.is_complete:
        # try reading data
        to_read = self.get_datasize() - len(self.data)
        self.data += dispatcher.recv(to_read)
        if len(self.data) == self.get_datasize():
          self.is_complete = True
    except socket.error as e:
      if e.errno == socket.errno.EAGAIN or e.errno == socket.errno.EWOULDBLOCK:
        # Try again later -> call continue_read later
        Log.debug("Try again error")
      else:
        # Fatal error
        Log.debug("Fatal error when reading IncomingPacket")
        raise RuntimeError("Fatal error occured in IncomingPacket.read()")

Response: Reads incoming data from asyncore. dispatcher


Instruction:
Can you generate a brief explanation for the following Python 3 code
def generate():
    data_bytes = bytearray(random.getrandbits(8) for i in range(REQID.REQID_SIZE))
    return REQID(data_bytes)

Response: Generates a random REQID for a request


Instruction:
Implement a Python 3 function for
creating a parser for the command

Response:
def create_parser(subparsers):
  '''
  :param subparsers:
  :return:
  '''
  parser = subparsers.add_parser(
      'restart',
      help='Restart a topology',
      usage="%(prog)s [options] cluster/[role]/[env] <topology-name> [container-id]",
      add_help=True)

  args.add_titles(parser)
  args.add_cluster_role_env(parser)
  args.add_topology(parser)

  parser.add_argument(
      'container-id',
      nargs='?',
      type=int,
      default=-1,
      help='Identifier of the container to be restarted')

  args.add_config(parser)
  args.add_service_url(parser)
  args.add_verbose(parser)

  parser.set_defaults(subcommand='restart')
  return parser


Instruction:
Here you have a function in Python 3, explain what it does
def get_heron_config():
  '''
  Get config opts from the global variable
  :return:
  '''
  opt_list = []
  for (key, value) in config_opts.items():
    opt_list.append('%s=%s' % (key, value))

  all_opts = (','.join(opt_list)).replace(' ', '%%%%')
  return all_opts

Response: Get config opts from the global variable


Instruction:
Here you have a function in Python 3, explain what it does
def yaml_config_reader(config_path):
  if not config_path.endswith(".yaml"):
    raise ValueError("Config file not yaml")

  with open(config_path, 'r') as f:
    config = yaml.load(f)

  return config

Response: Reads yaml config file and returns auto - typed config_dict


Instruction:
Create a Python 3 function to
call when a new HeronTupleSet2 arrives in the stream

Response:
def handle_new_tuple_set_2(self, hts2):
    """Called when new HeronTupleSet2 arrives
       Convert(Assemble) HeronTupleSet2(raw byte array) to HeronTupleSet
       See more at GitHub PR #1421
    :param tuple_msg_set: HeronTupleSet2 type
    """
    if self.my_pplan_helper is None or self.my_instance is None:
      Log.error("Got tuple set when no instance assigned yet")
    else:
      hts = tuple_pb2.HeronTupleSet()
      if hts2.HasField('control'):
        hts.control.CopyFrom(hts2.control)
      else:
        hdts = tuple_pb2.HeronDataTupleSet()
        hdts.stream.CopyFrom(hts2.data.stream)
        try:
          for trunk in hts2.data.tuples:
            added_tuple = hdts.tuples.add()
            added_tuple.ParseFromString(trunk)
        except Exception:
          Log.exception('Fail to deserialize HeronDataTuple')
        hts.data.CopyFrom(hdts)
      self.in_stream.offer(hts)
      if self.my_pplan_helper.is_topology_running():
        self.my_instance.py_class.process_incoming_tuples()


Instruction:
How would you code a function in Python 3 to
call when we get InitiateStatefulCheckpoint message

Response:
def handle_initiate_stateful_checkpoint(self, ckptmsg):
    """Called when we get InitiateStatefulCheckpoint message
    :param ckptmsg: InitiateStatefulCheckpoint type
    """
    self.in_stream.offer(ckptmsg)
    if self.my_pplan_helper.is_topology_running():
      self.my_instance.py_class.process_incoming_tuples()


Instruction:
Can you create a Python 3 function that
calls when we receive StartInstanceStatefulProcessing message

Response:
def handle_start_stateful_processing(self, start_msg):
    """Called when we receive StartInstanceStatefulProcessing message
    :param start_msg: StartInstanceStatefulProcessing type
    """
    Log.info("Received start stateful processing for %s" % start_msg.checkpoint_id)
    self.is_stateful_started = True
    self.start_instance_if_possible()


Instruction:
Here you have a function in Python 3, explain what it does
def handle_restore_instance_state(self, restore_msg):
    Log.info("Restoring instance state to checkpoint %s" % restore_msg.state.checkpoint_id)
    # Stop the instance
    if self.is_stateful_started:
      self.my_instance.py_class.stop()
      self.my_instance.py_class.clear_collector()
      self.is_stateful_started = False

    # Clear all buffers
    self.in_stream.clear()
    self.out_stream.clear()

    # Deser the state
    if self.stateful_state is not None:
      self.stateful_state.clear()
    if restore_msg.state.state is not None and restore_msg.state.state:
      try:
        self.stateful_state = self.serializer.deserialize(restore_msg.state.state)
      except Exception as e:
        raise RuntimeError("Could not serialize state during restore " + str(e))
    else:
      Log.info("The restore request does not have an actual state")
    if self.stateful_state is None:
      self.stateful_state = HashMapState()

    Log.info("Instance restore state deserialized")

    # Send the response back
    resp = ckptmgr_pb2.RestoreInstanceStateResponse()
    resp.status.status = common_pb2.StatusCode.Value("OK")
    resp.checkpoint_id = restore_msg.state.checkpoint_id
    self._stmgr_client.send_message(resp)

Response: Called when we receive a RestoreInstanceStateRequest message


Instruction:
Implement a function in Python 3 to
send buffered messages in out_stream to the Stream Manager

Response:
def send_buffered_messages(self):
    """Send messages in out_stream to the Stream Manager"""
    while not self.out_stream.is_empty() and self._stmgr_client.is_registered:
      tuple_set = self.out_stream.poll()
      if isinstance(tuple_set, tuple_pb2.HeronTupleSet):
        tuple_set.src_task_id = self.my_pplan_helper.my_task_id
        self.gateway_metrics.update_sent_packet(tuple_set.ByteSize())
      self._stmgr_client.send_message(tuple_set)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _handle_state_change_msg(self, new_helper):
    assert self.my_pplan_helper is not None
    assert self.my_instance is not None and self.my_instance.py_class is not None

    if self.my_pplan_helper.get_topology_state() != new_helper.get_topology_state():
      # handle state change
      # update the pplan_helper
      self.my_pplan_helper = new_helper
      if new_helper.is_topology_running():
        if not self.is_instance_started:
          self.start_instance_if_possible()
        self.my_instance.py_class.invoke_activate()
      elif new_helper.is_topology_paused():
        self.my_instance.py_class.invoke_deactivate()
      else:
        raise RuntimeError("Unexpected TopologyState update: %s" % new_helper.get_topology_state())
    else:
      Log.info("Topology state remains the same.")

Response: Called when state change is commanded by stream manager


Instruction:
Write a Python 3 script to
call when a NewInstanceAssignmentMessage arrives Tells this instance to become either spout or bolt.

Response:
def handle_assignment_msg(self, pplan):
    """Called when new NewInstanceAssignmentMessage arrives

    Tells this instance to become either spout/bolt.

    :param pplan: PhysicalPlan proto
    """

    new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id,
                                    self.topo_pex_file_abs_path)
    if self.my_pplan_helper is not None and \
      (self.my_pplan_helper.my_component_name != new_helper.my_component_name or
       self.my_pplan_helper.my_task_id != new_helper.my_task_id):
      raise RuntimeError("Our Assignment has changed. We will die to pick it.")

    new_helper.set_topology_context(self.metrics_collector)

    if self.my_pplan_helper is None:
      Log.info("Received a new Physical Plan")
      Log.info("Push the new pplan_helper to Heron Instance")
      self._handle_assignment_msg(new_helper)
    else:
      Log.info("Received a new Physical Plan with the same assignment -- State Change")
      Log.info("Old state: %s, new state: %s.",
               self.my_pplan_helper.get_topology_state(), new_helper.get_topology_state())
      self._handle_state_change_msg(new_helper)


Instruction:
Can you create a Python 3 function that
checks if a given stream_id and tuple matches with the output schema.

Response:
def check_output_schema(self, stream_id, tup):
    """Checks if a given stream_id and tuple matches with the output schema

    :type stream_id: str
    :param stream_id: stream id into which tuple is sent
    :type tup: list
    :param tup: tuple that is going to be sent
    """
    # do some checking to make sure that the number of fields match what's expected
    size = self._output_schema.get(stream_id, None)
    if size is None:
      raise RuntimeError("%s emitting to stream %s but was not declared in output fields"
                         % (self.my_component_name, stream_id))
    elif size != len(tup):
      raise RuntimeError("Number of fields emitted in stream %s does not match what's expected. "
                         "Expected: %s, Observed: %s" % (stream_id, size, len(tup)))


Instruction:
Make a summary of the following Python 3 code
def get_topology_config(self):
    if self.pplan.topology.HasField("topology_config"):
      return self._get_dict_from_config(self.pplan.topology.topology_config)
    else:
      return {}

Response: Returns the topology config


Instruction:
Can you tell what is the following Python 3 function doing
def set_topology_context(self, metrics_collector):
    Log.debug("Setting topology context")
    cluster_config = self.get_topology_config()
    cluster_config.update(self._get_dict_from_config(self.my_component.config))
    task_to_component_map = self._get_task_to_comp_map()
    self.context = TopologyContextImpl(cluster_config, self.pplan.topology, task_to_component_map,
                                       self.my_task_id, metrics_collector,
                                       self.topology_pex_abs_path)

Response: Sets a new topology context


Instruction:
Can you tell what is the following Python 3 function doing
def _get_dict_from_config(topology_config):
    config = {}
    for kv in topology_config.kvs:
      if kv.HasField("value"):
        assert kv.type == topology_pb2.ConfigValueType.Value("STRING_VALUE")
        # value is string
        if PhysicalPlanHelper._is_number(kv.value):
          config[kv.key] = PhysicalPlanHelper._get_number(kv.value)
        elif kv.value.lower() in ("true", "false"):
          config[kv.key] = True if kv.value.lower() == "true" else False
        else:
          config[kv.key] = kv.value
      elif kv.HasField("serialized_value") and \
        kv.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"):
        # deserialize that
        config[kv.key] = default_serializer.deserialize(kv.serialized_value)
      else:
        assert kv.HasField("type")
        Log.error("Unsupported config <key:value> found: %s, with type: %s"
                  % (str(kv), str(kv.type)))
        continue

    return config

Response: Converts Config protobuf message to python dictionary


Instruction:
Write a Python 3 function that can
check whether there are any streams that consume any of my streams using custom grouping

Response:
def _setup_custom_grouping(self, topology):
    """Checks whether there are any bolts that consume any of my streams using custom grouping"""
    for i in range(len(topology.bolts)):
      for in_stream in topology.bolts[i].inputs:
        if in_stream.stream.component_name == self.my_component_name and \
          in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"):
          # this bolt takes my output in custom grouping manner
          if in_stream.type == topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT"):
            custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object)
            if isinstance(custom_grouping_obj, str):
              pex_loader.load_pex(self.topology_pex_abs_path)
              grouping_cls = \
                pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)
              custom_grouping_obj = grouping_cls()
            assert isinstance(custom_grouping_obj, ICustomGrouping)
            self.custom_grouper.add(in_stream.stream.id,
                                    self._get_taskids_for_component(topology.bolts[i].comp.name),
                                    custom_grouping_obj,
                                    self.my_component_name)

          elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value("JAVA_OBJECT"):
            raise NotImplementedError("Java-serialized custom grouping is not yet supported "
                                      "for python topology")
          else:
            raise ValueError("Unrecognized custom grouping type found: %s" % str(in_stream.type))


Instruction:
Write a Python 3 function that can
add the target component to the list of tasks emitted by the given stream.

Response:
def add(self, stream_id, task_ids, grouping, source_comp_name):
    """Adds the target component

    :type stream_id: str
    :param stream_id: stream id into which tuples are emitted
    :type task_ids: list of str
    :param task_ids: list of task ids to which tuples are emitted
    :type grouping: ICustomStreamGrouping object
    :param grouping: custom grouping to use
    :type source_comp_name: str
    :param source_comp_name: source component name
    """
    if stream_id not in self.targets:
      self.targets[stream_id] = []
    self.targets[stream_id].append(Target(task_ids, grouping, source_comp_name))


Instruction:
How would you explain what the following Python 3 function does
def prepare(self, context):
    for stream_id, targets in self.targets.items():
      for target in targets:
        target.prepare(context, stream_id)

Response: Prepares the custom grouping for this component


Instruction:
Can you generate a brief explanation for the following Python 3 code
def choose_tasks(self, stream_id, values):
    if stream_id not in self.targets:
      return []

    ret = []
    for target in self.targets[stream_id]:
      ret.extend(target.choose_tasks(values))
    return ret

Response: Choose tasks for a given stream_id and values and Returns a list of target tasks


Instruction:
Can you generate a brief explanation for the following Python 3 code
def prepare(self, context, stream_id):
    self.grouping.prepare(context, self.source_comp_name, stream_id, self.task_ids)

Response: Invoke prepare method of this custom grouping


Instruction:
Can you tell what is the following Python 3 function doing
def choose_tasks(self, values):
    ret = self.grouping.choose_tasks(values)
    if not isinstance(ret, list):
      raise TypeError("Returned object after custom grouping's choose_tasks() "
                      "needs to be a list, given: %s" % str(type(ret)))
    else:
      for i in ret:
        if not isinstance(i, int):
          raise TypeError("Returned object after custom grouping's choose_tasks() "
                          "contained non-integer: %s" % str(i))
        if i not in self.task_ids:
          raise ValueError("Returned object after custom grouping's choose_tasks() contained "
                           "a task id that is not registered: %d" % i)
      return ret

Response: Invoke the custom grouping s choose_tasks method of this custom grouping.


Instruction:
Write a Python 3 script for
adding the configuration file to the given parser.

Response:
def add_config(parser):
  '''
  :param parser:
  :return:
  '''
  # the default config path
  default_config_path = config.get_heron_conf_dir()

  parser.add_argument(
      '--config-path',
      default=os.path.join(config.get_heron_dir(), default_config_path),
      help='Path to cluster configuration files')

  parser.add_argument(
      '--config-property',
      metavar='PROPERTY=VALUE',
      action='append',
      default=[],
      help='Configuration properties that overrides default options')
  return parser


Instruction:
Can you tell what is the following Python 3 function doing
def add_dry_run(parser):
  '''
  :param parser:
  :return:
  '''
  default_format = 'table'
  resp_formats = ['raw', 'table', 'colored_table', 'json']
  available_options = ', '.join(['%s' % opt for opt in resp_formats])

  def dry_run_resp_format(value):
    if value not in resp_formats:
      raise argparse.ArgumentTypeError(
          'Invalid dry-run response format: %s. Available formats: %s'
          % (value, available_options))
    return value

  parser.add_argument(
      '--dry-run',
      default=False,
      action='store_true',
      help='Enable dry-run mode. Information about '
           'the command will print but no action will be taken on the topology')

  parser.add_argument(
      '--dry-run-format',
      metavar='DRY_RUN_FORMAT',
      default='colored_table' if sys.stdout.isatty() else 'table',
      type=dry_run_resp_format,
      help='The format of the dry-run output ([%s], default=%s). '
           'Ignored when dry-run mode is not enabled' % ('|'.join(resp_formats), default_format))

  return parser

Response: Adds a dry - run command line argument to the given parser.


Instruction:
Implement a function in Python 3 to
read the cluster definition for server mode Read the cluster definition for server mode

Response:
def read_server_mode_cluster_definition(cluster, cl_args):
  '''
  Read the cluster definition for server mode
  :param cluster:
  :param cl_args:
  :param config_file:
  :return:
  '''

  client_confs = dict()
  client_confs[cluster] = cliconfig.cluster_config(cluster)

  # now check if the service-url from command line is set, if so override it
  if cl_args.get('service_url', None):
    client_confs[cluster]['service_url'] = cl_args['service_url']

  # the return value of yaml.load can be None if conf_file is an empty file
  # or there is no service-url in command line, if needed.

  return client_confs


Instruction:
How would you implement a function in Python 3 that
checks the cluster definition for direct mode Check the cluster definition for direct mode

Response:
def check_direct_mode_cluster_definition(cluster, config_path):
  '''
  Check the cluster definition for direct mode
  :param cluster:
  :param config_path:
  :return:
  '''
  config_path = config.get_heron_cluster_conf_dir(cluster, config_path)
  if not os.path.isdir(config_path):
    return False
  return True


Instruction:
Make a summary of the following Python 3 code
def format_mode(sres):
  mode = sres.st_mode

  root = (mode & 0o700) >> 6
  group = (mode & 0o070) >> 3
  user = (mode & 0o7)

  def stat_type(md):
    ''' stat type'''
    if stat.S_ISDIR(md):
      return 'd'
    elif stat.S_ISSOCK(md):
      return 's'
    else:
      return '-'

  def triple(md):
    ''' triple '''
    return '%c%c%c' % (
        'r' if md & 0b100 else '-',
        'w' if md & 0b010 else '-',
        'x' if md & 0b001 else '-')

  return ''.join([stat_type(mode), triple(root), triple(group), triple(user)])

Response: Format a mode value in the directory list.


Instruction:
Here you have a function in Python 3, explain what it does
def format_mtime(mtime):
  now = datetime.now()
  dt = datetime.fromtimestamp(mtime)
  return '%s %2d %5s' % (
      dt.strftime('%b'), dt.day,
      dt.year if dt.year != now.year else dt.strftime('%H:%M'))

Response: Formats the mtime of a file to be displayed in directory listing.


Instruction:
Can you generate the documentation for the following Python 3 function
def format_prefix(filename, sres):
  try:
    pwent = pwd.getpwuid(sres.st_uid)
    user = pwent.pw_name
  except KeyError:
    user = sres.st_uid

  try:
    grent = grp.getgrgid(sres.st_gid)
    group = grent.gr_name
  except KeyError:
    group = sres.st_gid

  return '%s %3d %10s %10s %10d %s' % (
      format_mode(sres),
      sres.st_nlink,
      user,
      group,
      sres.st_size,
      format_mtime(sres.st_mtime),
  )

Response: Format a filename in the directory listing.


Instruction:
Implement a Python 3 function for
returning the list of files and directories in a path.

Response:
def get_listing(path):
  """
  Returns the list of files and directories in a path.
  Prepents a ".." (parent directory link) if path is not current dir.
  """
  if path != ".":
    listing = sorted(['..'] + os.listdir(path))
  else:
    listing = sorted(os.listdir(path))
  return listing


Instruction:
Write a Python 3 script for
reading a chunk of a file from an offset upto the length.

Response:
def read_chunk(filename, offset=-1, length=-1, escape_data=False):
  """
  Read a chunk of a file from an offset upto the length.
  """
  try:
    length = int(length)
    offset = int(offset)
  except ValueError:
    return {}

  if not os.path.isfile(filename):
    return {}

  try:
    fstat = os.stat(filename)
  except Exception:
    return {}

  if offset == -1:
    offset = fstat.st_size

  if length == -1:
    length = fstat.st_size - offset

  with open(filename, "r") as fp:
    fp.seek(offset)
    try:
      data = fp.read(length)
    except IOError:
      return {}

  if data:
    data = _escape_data(data) if escape_data else data
    return dict(offset=offset, length=len(data), data=data)

  return dict(offset=offset, length=0)


Instruction:
Create a Python 3 function for
piping output of prev_proc into to_cmd. Returns piped process.

Response:
def pipe(prev_proc, to_cmd):
  """
  Pipes output of prev_proc into to_cmd.
  Returns piped process
  """
  stdin = None if prev_proc is None else prev_proc.stdout
  process = subprocess.Popen(to_cmd,
                             stdout=subprocess.PIPE,
                             stdin=stdin)
  if prev_proc is not None:
    prev_proc.stdout.close() # Allow prev_proc to receive a SIGPIPE
  return process


Instruction:
Can you tell what is the following Python 3 function doing
def str_cmd(cmd, cwd, env):
  process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, cwd=cwd, env=env)
  stdout_builder, stderr_builder = proc.async_stdout_stderr_builder(process)
  process.wait()
  stdout, stderr = stdout_builder.result(), stderr_builder.result()
  return {'command': ' '.join(cmd), 'stderr': stderr, 'stdout': stdout}

Response: Runs the command and returns its stdout and stderr.


Instruction:
Make a summary of the following Python 3 code
def chain(cmd_list):
  command = ' | '.join(map(lambda x: ' '.join(x), cmd_list))
  chained_proc = functools.reduce(pipe, [None] + cmd_list)
  stdout_builder = proc.async_stdout_builder(chained_proc)
  chained_proc.wait()
  return {
      'command': command,
      'stdout': stdout_builder.result()
  }

Response: Chain commands into one command and return the final output.


Instruction:
Given the following Python 3 function, write the documentation
def create_parser(subparsers):
  metrics_parser = subparsers.add_parser(
      'metrics',
      help='Display info of a topology\'s metrics',
      usage="%(prog)s cluster/[role]/[env] topology-name [options]",
      add_help=False)
  args.add_cluster_role_env(metrics_parser)
  args.add_topology_name(metrics_parser)
  args.add_verbose(metrics_parser)
  args.add_tracker_url(metrics_parser)
  args.add_config(metrics_parser)
  args.add_component_name(metrics_parser)
  metrics_parser.set_defaults(subcommand='metrics')

  containers_parser = subparsers.add_parser(
      'containers',
      help='Display info of a topology\'s containers metrics',
      usage="%(prog)s cluster/[role]/[env] topology-name [options]",
      add_help=False)
  args.add_cluster_role_env(containers_parser)
  args.add_topology_name(containers_parser)
  args.add_verbose(containers_parser)
  args.add_tracker_url(containers_parser)
  args.add_config(containers_parser)
  args.add_container_id(containers_parser)
  containers_parser.set_defaults(subcommand='containers')

  return subparsers

Response: create parser for topology - name


Instruction:
Implement a Python 3 function for
normalizing raw metrics API result to table

Response:
def to_table(metrics):
  """ normalize raw metrics API result to table """
  all_queries = tracker_access.metric_queries()
  m = tracker_access.queries_map()
  names = metrics.values()[0].keys()
  stats = []
  for n in names:
    info = [n]
    for field in all_queries:
      try:
        info.append(str(metrics[field][n]))
      except KeyError:
        pass
    stats.append(info)
  header = ['container id'] + [m[k] for k in all_queries if k in metrics.keys()]
  return stats, header


Instruction:
Can you implement a function in Python 3 that
defines the options for the network connection.

Response:
def define_options(address, port, tracker_url, base_url):
  '''
  :param address:
  :param port:
  :param tracker_url:
  :return:
  '''
  define("address", default=address)
  define("port", default=port)
  define("tracker_url", default=tracker_url)
  define("base_url", default=base_url)