Instruction:
How would you explain what the following Python 3 function does
def get(self, path):

    logging.debug("request to download: %s", path)
    # If the file is large, we want to abandon downloading
    # if user cancels the requests.
    # pylint: disable=attribute-defined-outside-init
    self.connection_closed = False

    self.set_header("Content-Disposition", "attachment")
    if not utils.check_path(path):
      self.write("Only relative paths are allowed")
      self.set_status(403)
      self.finish()
      return

    if path is None or not os.path.isfile(path):
      self.write("File %s  not found" % path)
      self.set_status(404)
      self.finish()
      return

    length = int(4 * 1024 * 1024)
    offset = int(0)
    while True:
      data = utils.read_chunk(path, offset=offset, length=length, escape_data=False)
      if self.connection_closed or 'data' not in data or len(data['data']) < length:
        break
      offset += length
      self.write(data['data'])
      self.flush()

    if 'data' in data:
      self.write(data['data'])
    self.finish()

Response: get method for the base class


Instruction:
Write a Python 3 function that can
get summary of stream managers registration summary

Response:
def getStmgrsRegSummary(self, tmaster, callback=None):
    """
    Get summary of stream managers registration summary
    """
    if not tmaster or not tmaster.host or not tmaster.stats_port:
      return
    reg_request = tmaster_pb2.StmgrsRegistrationSummaryRequest()
    request_str = reg_request.SerializeToString()
    port = str(tmaster.stats_port)
    host = tmaster.host
    url = "http://{0}:{1}/stmgrsregistrationsummary".format(host, port)
    request = tornado.httpclient.HTTPRequest(url,
                                             body=request_str,
                                             method='POST',
                                             request_timeout=5)
    Log.debug('Making HTTP call to fetch stmgrsregistrationsummary url: %s', url)
    try:
      client = tornado.httpclient.AsyncHTTPClient()
      result = yield client.fetch(request)
      Log.debug("HTTP call complete.")
    except tornado.httpclient.HTTPError as e:
      raise Exception(str(e))
    # Check the response code - error if it is in 400s or 500s
    responseCode = result.code
    if responseCode >= 400:
      message = "Error in getting exceptions from Tmaster, code: " + responseCode
      Log.error(message)
      raise tornado.gen.Return({
          "message": message
      })
    # Parse the response from tmaster.
    reg_response = tmaster_pb2.StmgrsRegistrationSummaryResponse()
    reg_response.ParseFromString(result.body)
    # Send response
    ret = {}
    for stmgr in reg_response.registered_stmgrs:
      ret[stmgr] = True
    for stmgr in reg_response.absent_stmgrs:
      ret[stmgr] = False
    raise tornado.gen.Return(ret)


Instruction:
Can you generate the documentation for the following Python 3 function
def get(self):
    try:
      cluster = self.get_argument_cluster()
      role = self.get_argument_role()
      environ = self.get_argument_environ()
      topology_name = self.get_argument_topology()
      topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)
      runtime_state = topology_info["runtime_state"]
      runtime_state["topology_version"] = topology_info["metadata"]["release_version"]
      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(
          cluster, role, environ, topology_name)
      reg_summary = yield tornado.gen.Task(self.getStmgrsRegSummary, topology.tmaster)
      for stmgr, reg in reg_summary.items():
        runtime_state["stmgrs"].setdefault(stmgr, {})["is_registered"] = reg
      self.write_success_response(runtime_state)
    except Exception as e:
      Log.debug(traceback.format_exc())
      self.write_error_response(e)

Response: get method returns a generator that yields all the services that are registered


Instruction:
Implement a function in Python 3 to
write content to a file and then rename it to target path

Response:
def atomic_write_file(path, content):
  """
  file.write(...) is not atomic.
  We write to a tmp file and then rename to target path since rename is atomic.
  We do this to avoid the content of file is dirty read/partially read by others.
  """
  # Write to a randomly tmp file
  tmp_file = get_tmp_filename()
  with open(tmp_file, 'w') as f:
    f.write(content)
    # make sure that all data is on disk
    f.flush()
    os.fsync(f.fileno())

  # Rename the tmp file
  os.rename(tmp_file, path)


Instruction:
Create a Python 3 function for
setting up log, process and signal handlers

Response:
def setup(executor):
  """Set up log, process and signal handlers"""
  # pylint: disable=unused-argument
  def signal_handler(signal_to_handle, frame):
    # We would do nothing here but just exit
    # Just catch the SIGTERM and then cleanup(), registered with atexit, would invoke
    Log.info('signal_handler invoked with signal %s', signal_to_handle)
    executor.stop_state_manager_watches()
    sys.exit(signal_to_handle)

  def cleanup():
    """Handler to trigger when receiving the SIGTERM signal
    Do cleanup inside this method, including:
    1. Terminate all children processes
    """
    Log.info('Executor terminated; exiting all process in executor.')

    # Kill child processes first and wait for log collection to finish
    for pid in executor.processes_to_monitor.keys():
      os.kill(pid, signal.SIGTERM)
    time.sleep(5)

    # We would not wait or check whether process spawned dead or not
    os.killpg(0, signal.SIGTERM)

  # Redirect stdout and stderr to files in append mode
  # The filename format is heron-executor-<container_id>.stdxxx
  shardid = executor.shard
  log.configure(logfile='heron-executor-%s.stdout' % shardid)

  pid = os.getpid()
  sid = os.getsid(pid)

  # POSIX prohibits the change of the process group ID of a session leader
  if pid <> sid:
    Log.info('Set up process group; executor becomes leader')
    os.setpgrp() # create new process group, become its leader

  Log.info('Register the SIGTERM signal handler')
  signal.signal(signal.SIGTERM, signal_handler)

  Log.info('Register the atexit clean up')
  atexit.register(cleanup)


Instruction:
Explain what the following Python 3 code does
def main():
  # Since Heron on YARN runs as headless users, pex compiled
  # binaries should be exploded into the container working
  # directory. In order to do this, we need to set the
  # PEX_ROOT shell environment before forking the processes
  shell_env = os.environ.copy()
  shell_env["PEX_ROOT"] = os.path.join(os.path.abspath('.'), ".pex")

  # Instantiate the executor, bind it to signal handlers and launch it
  executor = HeronExecutor(sys.argv, shell_env)
  executor.initialize()

  start(executor)

Response: Register exit handlers initialize the executor and run it.


Instruction:
Implement a Python 3 function for
initializing from parsed arguments

Response:
def init_from_parsed_args(self, parsed_args):
    """ initialize from parsed arguments """
    self.shard = parsed_args.shard
    self.topology_name = parsed_args.topology_name
    self.topology_id = parsed_args.topology_id
    self.topology_defn_file = parsed_args.topology_defn_file
    self.state_manager_connection = parsed_args.state_manager_connection
    self.state_manager_root = parsed_args.state_manager_root
    self.state_manager_config_file = parsed_args.state_manager_config_file
    self.tmaster_binary = parsed_args.tmaster_binary
    self.stmgr_binary = parsed_args.stmgr_binary
    self.metrics_manager_classpath = parsed_args.metrics_manager_classpath
    self.metricscache_manager_classpath = parsed_args.metricscache_manager_classpath
    # '=' can be parsed in a wrong way by some schedulers (aurora) hence it needs to be escaped.
    # It is escaped in two different ways. '(61)' is the new escaping. '=' was
    # the original replacement but it is not friendly to bash and is causing issues. The original
    # escaping is still left there for reference and backward compatibility purposes (to be
    # removed after no topology needs it)
    self.instance_jvm_opts =\
        base64.b64decode(parsed_args.instance_jvm_opts.lstrip('"').
                         rstrip('"').replace('(61)', '=').replace('=', '='))
    self.classpath = parsed_args.classpath
    # Needed for Docker environments since the hostname of a docker container is the container's
    # id within docker, rather than the host's hostname. NOTE: this 'HOST' env variable is not
    # guaranteed to be set in all Docker executor environments (outside of Marathon)
    if is_docker_environment():
      self.master_host = os.environ.get('HOST') if 'HOST' in os.environ else socket.gethostname()
    else:
      self.master_host = socket.gethostname()
    self.master_port = parsed_args.master_port
    self.tmaster_controller_port = parsed_args.tmaster_controller_port
    self.tmaster_stats_port = parsed_args.tmaster_stats_port
    self.heron_internals_config_file = parsed_args.heron_internals_config_file
    self.override_config_file = parsed_args.override_config_file
    self.component_ram_map =\
        map(lambda x: {x.split(':')[0]:
                           int(x.split(':')[1])}, parsed_args.component_ram_map.split(','))
    self.component_ram_map =\
        functools.reduce(lambda x, y: dict(x.items() + y.items()), self.component_ram_map)

    # component_jvm_opts_in_base64 itself is a base64-encoding-json-map, which is appended with
    # " at the start and end. It also escapes "=" to "&equals" due to aurora limitation
    # And the json is a map from base64-encoding-component-name to base64-encoding-jvm-options
    self.component_jvm_opts = {}
    # First we need to decode the base64 string back to a json map string.
    # '=' can be parsed in a wrong way by some schedulers (aurora) hence it needs to be escaped.
    # It is escaped in two different ways. '(61)' is the new escaping. '=' was
    # the original replacement but it is not friendly to bash and is causing issues. The original
    # escaping is still left there for reference and backward compatibility purposes (to be
    # removed after no topology needs it)
    component_jvm_opts_in_json =\
        base64.b64decode(parsed_args.component_jvm_opts.
                         lstrip('"').rstrip('"').replace('(61)', '=').replace('=', '='))
    if component_jvm_opts_in_json != "":
      for (k, v) in json.loads(component_jvm_opts_in_json).items():
        # In json, the component name and JVM options are still in base64 encoding
        self.component_jvm_opts[base64.b64decode(k)] = base64.b64decode(v)

    self.pkg_type = parsed_args.pkg_type
    self.topology_binary_file = parsed_args.topology_binary_file
    self.heron_java_home = parsed_args.heron_java_home
    self.shell_port = parsed_args.shell_port
    self.heron_shell_binary = parsed_args.heron_shell_binary
    self.metrics_manager_port = parsed_args.metrics_manager_port
    self.metricscache_manager_master_port = parsed_args.metricscache_manager_master_port
    self.metricscache_manager_stats_port = parsed_args.metricscache_manager_stats_port
    self.cluster = parsed_args.cluster
    self.role = parsed_args.role
    self.environment = parsed_args.environment
    self.instance_classpath = parsed_args.instance_classpath
    self.metrics_sinks_config_file = parsed_args.metrics_sinks_config_file
    self.scheduler_classpath = parsed_args.scheduler_classpath
    self.scheduler_port = parsed_args.scheduler_port
    self.python_instance_binary = parsed_args.python_instance_binary
    self.cpp_instance_binary = parsed_args.cpp_instance_binary

    self.is_stateful_topology = (parsed_args.is_stateful.lower() == 'true')
    self.checkpoint_manager_classpath = parsed_args.checkpoint_manager_classpath
    self.checkpoint_manager_port = parsed_args.checkpoint_manager_port
    self.checkpoint_manager_ram = parsed_args.checkpoint_manager_ram
    self.stateful_config_file = parsed_args.stateful_config_file
    self.metricscache_manager_mode = parsed_args.metricscache_manager_mode \
        if parsed_args.metricscache_manager_mode else "disabled"
    self.health_manager_mode = parsed_args.health_manager_mode
    self.health_manager_classpath = '%s:%s'\
        % (self.scheduler_classpath, parsed_args.health_manager_classpath)
    self.jvm_remote_debugger_ports = \
      parsed_args.jvm_remote_debugger_ports.split(",") \
        if parsed_args.jvm_remote_debugger_ports else None


Instruction:
Can you tell what is the following Python 3 function doing
def parse_args(args):
    Log.info("Input args: %r" % args)

    parser = argparse.ArgumentParser()

    parser.add_argument("--shard", type=int, required=True)
    parser.add_argument("--topology-name", required=True)
    parser.add_argument("--topology-id", required=True)
    parser.add_argument("--topology-defn-file", required=True)
    parser.add_argument("--state-manager-connection", required=True)
    parser.add_argument("--state-manager-root", required=True)
    parser.add_argument("--state-manager-config-file", required=True)
    parser.add_argument("--tmaster-binary", required=True)
    parser.add_argument("--stmgr-binary", required=True)
    parser.add_argument("--metrics-manager-classpath", required=True)
    parser.add_argument("--instance-jvm-opts", required=True)
    parser.add_argument("--classpath", required=True)
    parser.add_argument("--master-port", required=True)
    parser.add_argument("--tmaster-controller-port", required=True)
    parser.add_argument("--tmaster-stats-port", required=True)
    parser.add_argument("--heron-internals-config-file", required=True)
    parser.add_argument("--override-config-file", required=True)
    parser.add_argument("--component-ram-map", required=True)
    parser.add_argument("--component-jvm-opts", required=True)
    parser.add_argument("--pkg-type", required=True)
    parser.add_argument("--topology-binary-file", required=True)
    parser.add_argument("--heron-java-home", required=True)
    parser.add_argument("--shell-port", required=True)
    parser.add_argument("--heron-shell-binary", required=True)
    parser.add_argument("--metrics-manager-port", required=True)
    parser.add_argument("--cluster", required=True)
    parser.add_argument("--role", required=True)
    parser.add_argument("--environment", required=True)
    parser.add_argument("--instance-classpath", required=True)
    parser.add_argument("--metrics-sinks-config-file", required=True)
    parser.add_argument("--scheduler-classpath", required=True)
    parser.add_argument("--scheduler-port", required=True)
    parser.add_argument("--python-instance-binary", required=True)
    parser.add_argument("--cpp-instance-binary", required=True)
    parser.add_argument("--metricscache-manager-classpath", required=True)
    parser.add_argument("--metricscache-manager-master-port", required=True)
    parser.add_argument("--metricscache-manager-stats-port", required=True)
    parser.add_argument("--metricscache-manager-mode", required=False)
    parser.add_argument("--is-stateful", required=True)
    parser.add_argument("--checkpoint-manager-classpath", required=True)
    parser.add_argument("--checkpoint-manager-port", required=True)
    parser.add_argument("--checkpoint-manager-ram", type=long, required=True)
    parser.add_argument("--stateful-config-file", required=True)
    parser.add_argument("--health-manager-mode", required=True)
    parser.add_argument("--health-manager-classpath", required=True)
    parser.add_argument("--jvm-remote-debugger-ports", required=False,
                        help="ports to be used by a remote debugger for JVM instances")

    parsed_args, unknown_args = parser.parse_known_args(args[1:])

    if unknown_args:
      Log.error('Unknown argument: %s' % unknown_args[0])
      parser.print_help()
      sys.exit(1)

    return parsed_args

Response: Uses argparse to collect positional args


Instruction:
Explain what the following Python 3 code does
def initialize(self):
    create_folders = Command('mkdir -p %s' % self.log_dir, self.shell_env)
    self.run_command_or_exit(create_folders)

    chmod_logs_dir = Command('chmod a+rx . && chmod a+x %s' % self.log_dir, self.shell_env)
    self.run_command_or_exit(chmod_logs_dir)

    chmod_x_binaries = [self.tmaster_binary, self.stmgr_binary, self.heron_shell_binary]

    for binary in chmod_x_binaries:
      stat_result = os.stat(binary)[stat.ST_MODE]
      if not stat_result & stat.S_IXOTH:
        chmod_binary = Command('chmod +x %s' % binary, self.shell_env)
        self.run_command_or_exit(chmod_binary)

    # Log itself pid
    log_pid_for_process(get_heron_executor_process_name(self.shard), os.getpid())

Response:
Initialize the environment. Done with a method call outside of the constructor for 2 reasons:
    1. Unit tests probably won't want/need to do this
    2. We don't initialize the logger (also something unit tests don't want) until after the
    constructor


Instruction:
Write a Python 3 script to
get the command to start the metrics manager processes

Response:
def _get_metricsmgr_cmd(self, metricsManagerId, sink_config_file, port):
    ''' get the command to start the metrics manager processes '''
    metricsmgr_main_class = 'org.apache.heron.metricsmgr.MetricsManager'

    metricsmgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),
                      # We could not rely on the default -Xmx setting, which could be very big,
                      # for instance, the default -Xmx in Twitter mesos machine is around 18GB
                      '-Xmx1024M',
                      '-XX:+PrintCommandLineFlags',
                      '-verbosegc',
                      '-XX:+PrintGCDetails',
                      '-XX:+PrintGCTimeStamps',
                      '-XX:+PrintGCDateStamps',
                      '-XX:+PrintGCCause',
                      '-XX:+UseGCLogFileRotation',
                      '-XX:NumberOfGCLogFiles=5',
                      '-XX:GCLogFileSize=100M',
                      '-XX:+PrintPromotionFailure',
                      '-XX:+PrintTenuringDistribution',
                      '-XX:+PrintHeapAtGC',
                      '-XX:+HeapDumpOnOutOfMemoryError',
                      '-XX:+UseConcMarkSweepGC',
                      '-XX:+PrintCommandLineFlags',
                      '-Xloggc:log-files/gc.metricsmgr.log',
                      '-Djava.net.preferIPv4Stack=true',
                      '-cp',
                      self.metrics_manager_classpath,
                      metricsmgr_main_class,
                      '--id=' + metricsManagerId,
                      '--port=' + str(port),
                      '--topology=' + self.topology_name,
                      '--cluster=' + self.cluster,
                      '--role=' + self.role,
                      '--environment=' + self.environment,
                      '--topology-id=' + self.topology_id,
                      '--system-config-file=' + self.heron_internals_config_file,
                      '--override-config-file=' + self.override_config_file,
                      '--sink-config-file=' + sink_config_file]

    return Command(metricsmgr_cmd, self.shell_env)


Instruction:
Create a Python 3 function to
get the command to start the metrics manager processes

Response:
def _get_metrics_cache_cmd(self):
    ''' get the command to start the metrics manager processes '''
    metricscachemgr_main_class = 'org.apache.heron.metricscachemgr.MetricsCacheManager'

    metricscachemgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),
                           # We could not rely on the default -Xmx setting, which could be very big,
                           # for instance, the default -Xmx in Twitter mesos machine is around 18GB
                           '-Xmx1024M',
                           '-XX:+PrintCommandLineFlags',
                           '-verbosegc',
                           '-XX:+PrintGCDetails',
                           '-XX:+PrintGCTimeStamps',
                           '-XX:+PrintGCDateStamps',
                           '-XX:+PrintGCCause',
                           '-XX:+UseGCLogFileRotation',
                           '-XX:NumberOfGCLogFiles=5',
                           '-XX:GCLogFileSize=100M',
                           '-XX:+PrintPromotionFailure',
                           '-XX:+PrintTenuringDistribution',
                           '-XX:+PrintHeapAtGC',
                           '-XX:+HeapDumpOnOutOfMemoryError',
                           '-XX:+UseConcMarkSweepGC',
                           '-XX:+PrintCommandLineFlags',
                           '-Xloggc:log-files/gc.metricscache.log',
                           '-Djava.net.preferIPv4Stack=true',
                           '-cp',
                           self.metricscache_manager_classpath,
                           metricscachemgr_main_class,
                           "--metricscache_id", 'metricscache-0',
                           "--master_port", self.metricscache_manager_master_port,
                           "--stats_port", self.metricscache_manager_stats_port,
                           "--topology_name", self.topology_name,
                           "--topology_id", self.topology_id,
                           "--system_config_file", self.heron_internals_config_file,
                           "--override_config_file", self.override_config_file,
                           "--sink_config_file", self.metrics_sinks_config_file,
                           "--cluster", self.cluster,
                           "--role", self.role,
                           "--environment", self.environment]

    return Command(metricscachemgr_cmd, self.shell_env)


Instruction:
Can you tell what is the following Python 3 function doing
def _get_healthmgr_cmd(self):
    ''' get the command to start the topology health manager processes '''
    healthmgr_main_class = 'org.apache.heron.healthmgr.HealthManager'

    healthmgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),
                     # We could not rely on the default -Xmx setting, which could be very big,
                     # for instance, the default -Xmx in Twitter mesos machine is around 18GB
                     '-Xmx1024M',
                     '-XX:+PrintCommandLineFlags',
                     '-verbosegc',
                     '-XX:+PrintGCDetails',
                     '-XX:+PrintGCTimeStamps',
                     '-XX:+PrintGCDateStamps',
                     '-XX:+PrintGCCause',
                     '-XX:+UseGCLogFileRotation',
                     '-XX:NumberOfGCLogFiles=5',
                     '-XX:GCLogFileSize=100M',
                     '-XX:+PrintPromotionFailure',
                     '-XX:+PrintTenuringDistribution',
                     '-XX:+PrintHeapAtGC',
                     '-XX:+HeapDumpOnOutOfMemoryError',
                     '-XX:+UseConcMarkSweepGC',
                     '-XX:+PrintCommandLineFlags',
                     '-Xloggc:log-files/gc.healthmgr.log',
                     '-Djava.net.preferIPv4Stack=true',
                     '-cp', self.health_manager_classpath,
                     healthmgr_main_class,
                     "--cluster", self.cluster,
                     "--role", self.role,
                     "--environment", self.environment,
                     "--topology_name", self.topology_name,
                     "--metricsmgr_port", self.metrics_manager_port]

    return Command(healthmgr_cmd, self.shell_env)

Response: get the command to start the topology health manager processes


Instruction:
Can you tell what is the following Python 3 function doing
def _get_tmaster_processes(self):
    ''' get the command to start the tmaster processes '''
    retval = {}
    tmaster_cmd_lst = [
        self.tmaster_binary,
        '--topology_name=%s' % self.topology_name,
        '--topology_id=%s' % self.topology_id,
        '--zkhostportlist=%s' % self.state_manager_connection,
        '--zkroot=%s' % self.state_manager_root,
        '--myhost=%s' % self.master_host,
        '--master_port=%s' % str(self.master_port),
        '--controller_port=%s' % str(self.tmaster_controller_port),
        '--stats_port=%s' % str(self.tmaster_stats_port),
        '--config_file=%s' % self.heron_internals_config_file,
        '--override_config_file=%s' % self.override_config_file,
        '--metrics_sinks_yaml=%s' % self.metrics_sinks_config_file,
        '--metricsmgr_port=%s' % str(self.metrics_manager_port),
        '--ckptmgr_port=%s' % str(self.checkpoint_manager_port)]

    tmaster_env = self.shell_env.copy() if self.shell_env is not None else {}
    tmaster_cmd = Command(tmaster_cmd_lst, tmaster_env)
    if os.environ.get('ENABLE_HEAPCHECK') is not None:
      tmaster_cmd.env.update({
          'LD_PRELOAD': "/usr/lib/libtcmalloc.so",
          'HEAPCHECK': "normal"
      })

    retval["heron-tmaster"] = tmaster_cmd

    if self.metricscache_manager_mode.lower() != "disabled":
      retval["heron-metricscache"] = self._get_metrics_cache_cmd()

    if self.health_manager_mode.lower() != "disabled":
      retval["heron-healthmgr"] = self._get_healthmgr_cmd()

    retval[self.metricsmgr_ids[0]] = self._get_metricsmgr_cmd(
        self.metricsmgr_ids[0],
        self.metrics_sinks_config_file,
        self.metrics_manager_port)

    if self.is_stateful_topology:
      retval.update(self._get_ckptmgr_process())

    return retval

Response: get the command to start the tmaster processes


Instruction:
Explain what the following Python 3 code does
def _get_ckptmgr_process(self):
    ''' Get the command to start the checkpoint manager process'''

    ckptmgr_main_class = 'org.apache.heron.ckptmgr.CheckpointManager'

    ckptmgr_ram_mb = self.checkpoint_manager_ram / (1024 * 1024)
    ckptmgr_cmd = [os.path.join(self.heron_java_home, "bin/java"),
                   '-Xms%dM' % ckptmgr_ram_mb,
                   '-Xmx%dM' % ckptmgr_ram_mb,
                   '-XX:+PrintCommandLineFlags',
                   '-verbosegc',
                   '-XX:+PrintGCDetails',
                   '-XX:+PrintGCTimeStamps',
                   '-XX:+PrintGCDateStamps',
                   '-XX:+PrintGCCause',
                   '-XX:+UseGCLogFileRotation',
                   '-XX:NumberOfGCLogFiles=5',
                   '-XX:GCLogFileSize=100M',
                   '-XX:+PrintPromotionFailure',
                   '-XX:+PrintTenuringDistribution',
                   '-XX:+PrintHeapAtGC',
                   '-XX:+HeapDumpOnOutOfMemoryError',
                   '-XX:+UseConcMarkSweepGC',
                   '-XX:+UseConcMarkSweepGC',
                   '-Xloggc:log-files/gc.ckptmgr.log',
                   '-Djava.net.preferIPv4Stack=true',
                   '-cp',
                   self.checkpoint_manager_classpath,
                   ckptmgr_main_class,
                   '-t' + self.topology_name,
                   '-i' + self.topology_id,
                   '-c' + self.ckptmgr_ids[self.shard],
                   '-p' + self.checkpoint_manager_port,
                   '-f' + self.stateful_config_file,
                   '-o' + self.override_config_file,
                   '-g' + self.heron_internals_config_file]
    retval = {}
    retval[self.ckptmgr_ids[self.shard]] = Command(ckptmgr_cmd, self.shell_env)

    return retval

Response: Get the command to start the checkpoint manager process


Instruction:
Make a summary of the following Python 3 code
def _get_instance_plans(self, packing_plan, container_id):
    this_container_plan = None
    for container_plan in packing_plan.container_plans:
      if container_plan.id == container_id:
        this_container_plan = container_plan

    # When the executor runs in newly added container by `heron update`,
    # there is no plan for this container. In this situation,
    # return None to bypass instance processes.
    if this_container_plan is None:
      return None
    return this_container_plan.instance_plans

Response: Get the instance_plans for the given packing_plan.


Instruction:
Can you generate the documentation for the following Python 3 function
def _get_heron_support_processes(self):
    retval = {}

    retval[self.heron_shell_ids[self.shard]] = Command([
        '%s' % self.heron_shell_binary,
        '--port=%s' % self.shell_port,
        '--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir, self.shard),
        '--secret=%s' % self.topology_id], self.shell_env)

    return retval

Response: Get a map from all daemon services name to the command to start them


Instruction:
Implement a function in Python 3 to
wait for the termination of a process and log its stdout & stderr.

Response:
def _wait_process_std_out_err(self, name, process):
    ''' Wait for the termination of a process and log its stdout & stderr '''
    proc.stream_process_stdout(process, stdout_log_fn(name))
    process.wait()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _start_processes(self, commands):
    Log.info("Start processes")
    processes_to_monitor = {}
    # First start all the processes
    for (name, command) in commands.items():
      p = self._run_process(name, command)
      processes_to_monitor[p.pid] = ProcessInfo(p, name, command)

      # Log down the pid file
      log_pid_for_process(name, p.pid)

    with self.process_lock:
      self.processes_to_monitor.update(processes_to_monitor)

Response: Start all the processes and add them to the processes_to_monitor dict


Instruction:
Can you generate the documentation for the following Python 3 function
def start_process_monitor(self):
    # Now wait for any child to die
    Log.info("Start process monitor")
    while True:
      if len(self.processes_to_monitor) > 0:
        (pid, status) = os.wait()

        with self.process_lock:
          if pid in self.processes_to_monitor.keys():
            old_process_info = self.processes_to_monitor[pid]
            name = old_process_info.name
            command = old_process_info.command
            Log.info("%s (pid=%s) exited with status %d. command=%s" % (name, pid, status, command))
            # Log the stdout & stderr of the failed process
            self._wait_process_std_out_err(name, old_process_info.process)

            # Just make it world readable
            if os.path.isfile("core.%d" % pid):
              os.system("chmod a+r core.%d" % pid)
            if old_process_info.attempts >= self.max_runs:
              Log.info("%s exited too many times" % name)
              sys.exit(1)
            time.sleep(self.interval_between_runs)
            p = self._run_process(name, command)
            del self.processes_to_monitor[pid]
            self.processes_to_monitor[p.pid] =\
              ProcessInfo(p, name, command, old_process_info.attempts + 1)

            # Log down the pid file
            log_pid_for_process(name, p.pid)

Response: Start the process monitor.


Instruction:
How would you explain what the following Python 3 function does
def get_commands_to_run(self):
    # During shutdown the watch might get triggered with the empty packing plan
    if len(self.packing_plan.container_plans) == 0:
      return {}
    if self._get_instance_plans(self.packing_plan, self.shard) is None and self.shard != 0:
      retval = {}
      retval['heron-shell'] = Command([
          '%s' % self.heron_shell_binary,
          '--port=%s' % self.shell_port,
          '--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir, self.shard),
          '--secret=%s' % self.topology_id], self.shell_env)
      return retval

    if self.shard == 0:
      commands = self._get_tmaster_processes()
    else:
      self._untar_if_needed()
      commands = self._get_streaming_processes()

    # Attach daemon processes
    commands.update(self._get_heron_support_processes())
    return commands

Response: Prepare either TMaster or Streaming commands according to shard.


Instruction:
Implement a function in Python 3 to
return a 3 - tuple of dicts that contains the current command and the updated command that has changed.

Response:
def get_command_changes(self, current_commands, updated_commands):
    """
    Compares the current command with updated command to return a 3-tuple of dicts,
    keyed by command name: commands_to_kill, commands_to_keep and commands_to_start.
    """
    commands_to_kill = {}
    commands_to_keep = {}
    commands_to_start = {}

    # if the current command has a matching command in the updated commands we keep it
    # otherwise we kill it
    for current_name, current_command in current_commands.items():
      # We don't restart tmaster since it watches the packing plan and updates itself. The stream
      # manager is restarted just to reset state, but we could update it to do so without a restart
      if current_name in updated_commands.keys() and \
        current_command == updated_commands[current_name] and \
        not current_name.startswith('stmgr-'):
        commands_to_keep[current_name] = current_command
      else:
        commands_to_kill[current_name] = current_command

    # updated commands not in the keep list need to be started
    for updated_name, updated_command in updated_commands.items():
      if updated_name not in commands_to_keep.keys():
        commands_to_start[updated_name] = updated_command

    return commands_to_kill, commands_to_keep, commands_to_start


Instruction:
Write a Python 3 function for
launching the processes required by the processes in the process_lock.

Response:
def launch(self):
    ''' Determines the commands to be run and compares them with the existing running commands.
    Then starts new ones required and kills old ones no longer required.
    '''
    with self.process_lock:
      current_commands = dict(map((lambda process: (process.name, process.command)),
                                  self.processes_to_monitor.values()))
      updated_commands = self.get_commands_to_run()

      # get the commands to kill, keep and start
      commands_to_kill, commands_to_keep, commands_to_start = \
          self.get_command_changes(current_commands, updated_commands)

      Log.info("current commands: %s" % sorted(current_commands.keys()))
      Log.info("new commands    : %s" % sorted(updated_commands.keys()))
      Log.info("commands_to_kill: %s" % sorted(commands_to_kill.keys()))
      Log.info("commands_to_keep: %s" % sorted(commands_to_keep.keys()))
      Log.info("commands_to_start: %s" % sorted(commands_to_start.keys()))

      self._kill_processes(commands_to_kill)
      self._start_processes(commands_to_start)
      Log.info("Launch complete - processes killed=%s kept=%s started=%s monitored=%s" %
               (len(commands_to_kill), len(commands_to_keep),
                len(commands_to_start), len(self.processes_to_monitor)))


Instruction:
Here you have a function in Python 3, explain what it does
def start_state_manager_watches(self):
    Log.info("Start state manager watches")
    statemgr_config = StateMgrConfig()
    statemgr_config.set_state_locations(configloader.load_state_manager_locations(
        self.cluster, state_manager_config_file=self.state_manager_config_file,
        overrides={"heron.statemgr.connection.string": self.state_manager_connection}))
    try:
      self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
      for state_manager in self.state_managers:
        state_manager.start()
    except Exception as ex:
      Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex)
      traceback.print_exc()
      sys.exit(1)

    # pylint: disable=unused-argument
    def on_packing_plan_watch(state_manager, new_packing_plan):
      Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" %
                (self.shard, str(self.packing_plan), str(new_packing_plan)))

      if self.packing_plan != new_packing_plan:
        Log.info("PackingPlan change detected on shard %s, relaunching effected processes."
                 % self.shard)
        self.update_packing_plan(new_packing_plan)

        Log.info("Updating executor processes")
        self.launch()
      else:
        Log.info(
            "State watch triggered for PackingPlan update but plan not changed so not relaunching.")

    for state_manager in self.state_managers:
      # The callback function with the bound
      # state_manager as first variable.
      onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager)
      state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)
      Log.info("Registered state watch for packing plan changes with state manager %s." %
               str(state_manager))

Response: Start watching for state managers and update processes as needed.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run(self, name, config, builder):
    if not isinstance(name, str):
      raise RuntimeError("Name has to be a string type")
    if not isinstance(config, Config):
      raise RuntimeError("config has to be a Config type")
    if not isinstance(builder, Builder):
      raise RuntimeError("builder has to be a Builder type")
    bldr = TopologyBuilder(name=name)
    builder.build(bldr)
    bldr.set_config(config._api_config)
    bldr.build_and_submit()

Response: Builds the topology and submits it


Instruction:
Create a Python 3 function to
force every module in modList to be placed into main

Response:
def _modules_to_main(modList):
  """Force every module in modList to be placed into main"""
  if not modList:
    return

  main = sys.modules['__main__']
  for modname in modList:
    if isinstance(modname, str):
      try:
        mod = __import__(modname)
      except Exception:
        sys.stderr.write(
            'warning: could not import %s\n.  '
            'Your function may unexpectedly error due to this import failing;'
            'A version mismatch is likely.  Specific error was:\n' % modname)
        print_exec(sys.stderr)
      else:
        setattr(main, mod.__name__, mod)


Instruction:
Can you implement a function in Python 3 that
fills in the rest of the function data into the skeleton function object that were created via _make_skel_func.

Response:
def _fill_function(func, globalsn, defaults, dictn, module):
  """ Fills in the rest of function data into the skeleton function object
    that were created via _make_skel_func().
     """
  func.__globals__.update(globalsn)
  func.__defaults__ = defaults
  func.__dict__ = dictn
  func.__module__ = module

  return func


Instruction:
Make a summary of the following Python 3 code
def _make_skel_func(code, closures, base_globals=None):
  closure = _reconstruct_closure(closures) if closures else None

  if base_globals is None:
    base_globals = {}
  base_globals['__builtins__'] = __builtins__

  return types.FunctionType(code, base_globals, None, None, closure)

Response: Creates a skeleton function object that contains just the provided code and the correct number of cells in func_closure.


Instruction:
Can you tell what is the following Python 3 function doing
def _load_class(cls, d):
  for k, v in d.items():
    if isinstance(k, tuple):
      typ, k = k
      if typ == 'property':
        v = property(*v)
      elif typ == 'staticmethod':
        v = staticmethod(v) # pylint: disable=redefined-variable-type
      elif typ == 'classmethod':
        v = classmethod(v)
    setattr(cls, k, v)
  return cls

Response: Load additional properties into class cls.


Instruction:
Create a Python 3 function to
save a module as an import

Response:
def save_module(self, obj):
    """
    Save a module as an import
    """
    self.modules.add(obj)
    self.save_reduce(subimport, (obj.__name__,), obj=obj)


Instruction:
Create a Python 3 function for
saving a function object to the pickle file.

Response:
def save_function(self, obj, name=None):
    """ Registered with the dispatch to handle all function types.
    Determines what kind of function obj is (e.g. lambda, defined at
    interactive prompt, etc) and handles the pickling appropriately.
    """
    write = self.write

    if name is None:
      name = obj.__name__
    try:
      # whichmodule() could fail, see
      # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
      modname = pickle.whichmodule(obj, name)
    except Exception:
      modname = None
    # print('which gives %s %s %s' % (modname, obj, name))
    try:
      themodule = sys.modules[modname]
    except KeyError:
      # eval'd items such as namedtuple give invalid items for their function __module__
      modname = '__main__'

    if modname == '__main__':
      themodule = None

    if themodule:
      self.modules.add(themodule)
      if getattr(themodule, name, None) is obj:
        return self.save_global(obj, name)

    # if func is lambda, def'ed at prompt, is in main, or is nested, then
    # we'll pickle the actual function object rather than simply saving a
    # reference (as is done in default pickler), via save_function_tuple.
    if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule is None:
      #print("save global", islambda(obj), obj.__code__.co_filename, modname, themodule)
      self.save_function_tuple(obj)
      return
    else:
      # func is nested
      klass = getattr(themodule, name, None)
      if klass is None or klass is not obj:
        self.save_function_tuple(obj)
        return

    if obj.__dict__:
      # essentially save_reduce, but workaround needed to avoid recursion
      self.save(_restore_attr)
      write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n')
      self.memoize(obj)
      self.save(obj.__dict__)
      write(pickle.TUPLE + pickle.REDUCE)
    else:
      write(pickle.GLOBAL + modname + '\n' + name + '\n')
      self.memoize(obj)


Instruction:
Can you implement a function in Python 3 that
saves a tuple of functions to disk.

Response:
def save_function_tuple(self, func):
    """  Pickles an actual func object.
    A func comprises: code, globals, defaults, closure, and dict.  We
    extract and save these, injecting reducing functions at certain points
    to recreate the func object.  Keep in mind that some of these pieces
    can contain a ref to the func itself.  Thus, a naive save on these
    pieces could trigger an infinite loop of save's.  To get around that,
    we first create a skeleton func object using just the code (this is
    safe, since this won't contain a ref to the func), and memoize it as
    soon as it's created.  The other stuff can then be filled in later.
    """
    save = self.save
    write = self.write

    code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data(func)

    save(_fill_function)  # skeleton function updater
    write(pickle.MARK)    # beginning of tuple that _fill_function expects

    # create a skeleton function object and memoize it
    save(_make_skel_func)
    save((code, closure, base_globals))
    write(pickle.REDUCE)
    self.memoize(func)

    # save the rest of the func data needed by _fill_function
    save(f_globals)
    save(defaults)
    save(dct)
    save(func.__module__)
    write(pickle.TUPLE)
    write(pickle.REDUCE)


Instruction:
Can you write a function in Python 3 where it
saves a file to the pickle file.

Response:
def save_file(self, obj): # pylint: disable=too-many-branches
    """Save a file"""
    try:
      import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute
    except ImportError:
      import io as pystringIO # pylint: disable=reimported

    if not hasattr(obj, 'name') or  not hasattr(obj, 'mode'):
      raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
    if obj is sys.stdout:
      return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
    if obj is sys.stderr:
      return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
    if obj is sys.stdin:
      raise pickle.PicklingError("Cannot pickle standard input")
    if  hasattr(obj, 'isatty') and obj.isatty():
      raise pickle.PicklingError("Cannot pickle files that map to tty objects")
    if 'r' not in obj.mode:
      raise pickle.PicklingError("Cannot pickle files that are not opened for reading")
    name = obj.name
    try:
      fsize = os.stat(name).st_size
    except OSError:
      raise pickle.PicklingError("Cannot pickle file %s as it cannot be stat" % name)

    if obj.closed:
      #create an empty closed string io
      retval = pystringIO.StringIO("")
      retval.close()
    elif not fsize: #empty file
      retval = pystringIO.StringIO("")
      try:
        tmpfile = file(name)
        tst = tmpfile.read(1)
      except IOError:
        raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
      tmpfile.close()
      if tst != '':
        raise pickle.PicklingError(
            "Cannot pickle file %s as it does not appear to map to a physical, real file" % name)
    else:
      try:
        tmpfile = file(name)
        contents = tmpfile.read()
        tmpfile.close()
      except IOError:
        raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
      retval = pystringIO.StringIO(contents)
      curloc = obj.tell()
      retval.seek(curloc)

    retval.name = name
    self.save(retval)
    self.memoize(obj)


Instruction:
Explain what the following Python 3 code does
def tail(filename, n):
  size = os.path.getsize(filename)
  with open(filename, "rb") as f:
   fm = mmap.mmap(f.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
   try:
      for i in xrange(size - 1, -1, -1):
          if fm[i] == '\n':
             n -= 1
             if n == -1:
                break
      return fm[i + 1 if i else 0:].splitlines()
   finally:
        fm.close()

Response: Returns the last n lines from the filename. No exception handling.


Instruction:
How would you code a function in Python 3 to
return a serializer for a given context

Response:
def get_serializer(context):
    """Returns a serializer for a given context"""
    cluster_config = context.get_cluster_config()
    serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None)
    if serializer_clsname is None:
      return PythonSerializer()
    else:
      try:
        topo_pex_path = context.get_topology_pex_path()
        pex_loader.load_pex(topo_pex_path)
        serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname)
        serializer = serializer_cls()
        return serializer
      except Exception as e:
        raise RuntimeError("Error with loading custom serializer class: %s, with error message: %s"
                           % (serializer_clsname, str(e)))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _run_once(self):
    try:
      self.do_wait()
      self._execute_wakeup_tasks()
      self._trigger_timers()
    except Exception as e:
      Log.error("Error occured during _run_once(): " + str(e))
      Log.error(traceback.format_exc())
      self.should_exit = True

Response: Run once in the loop.


Instruction:
Implement a function in Python 3 to
register a task in the timer list

Response:
def register_timer_task_in_sec(self, task, second):
    """Registers a new timer task

    :param task: function to be run at a specified second from now
    :param second: how many seconds to wait before the timer is triggered
    """
    # Python time is in float
    second_in_float = float(second)
    expiration = time.time() + second_in_float
    heappush(self.timer_tasks, (expiration, task))


Instruction:
Write a Python 3 script to
get the next timeout interval from now This should be used from do_wait.

Response:
def _get_next_timeout_interval(self):
    """Get the next timeout from now

    This should be used from do_wait().
    :returns (float) next_timeout, or 10.0 if there are no timer events
    """
    if len(self.timer_tasks) == 0:
      return sys.maxsize
    else:
      next_timeout_interval = self.timer_tasks[0][0] - time.time()
      return next_timeout_interval


Instruction:
Write a Python 3 function that can
execute wakeup tasks in order to avoid concurrent calls to loop

Response:
def _execute_wakeup_tasks(self):
    """Executes wakeup tasks, should only be called from loop()"""
    # Check the length of wakeup tasks first to avoid concurrent issues
    size = len(self.wakeup_tasks)
    for i in range(size):
      self.wakeup_tasks[i]()


Instruction:
Write a Python 3 script for
posting processing of the process.

Response:
def post(self):
    """ post method """
    def status_finish(ret):
      self.set_status(ret)
      self.finish()

    def kill_parent():
      status_finish(200)
      logger.info("Killing parent executor")
      os.killpg(os.getppid(), signal.SIGTERM)

    logger = logging.getLogger(__file__)
    logger.info("Received 'Killing process' request")
    data = dict(urlparse.parse_qsl(self.request.body))

    # check shared secret
    sharedSecret = data.get('secret')
    if sharedSecret != options.secret:
      status_finish(403)
      return

    instanceId = data.get('instance_id_to_restart')
    if instanceId:
      filepath = instanceId + '.pid'
      if os.path.isfile(filepath): # instance_id found
        if instanceId.startswith('heron-executor-'): # kill heron-executor
          kill_parent()
        else: # kill other normal instance
          fh = open(filepath)
          firstLine = int(fh.readline())
          fh.close()
          logger.info("Killing process " + instanceId + " " + str(firstLine))
          os.kill(firstLine, signal.SIGTERM)
          status_finish(200)
      else: # instance_id not found
        logger.info(filepath + " not found")
        status_finish(422)
    else: # instance_id not given, which means kill the container
      kill_parent()


Instruction:
Write a Python 3 function that can
execute query with tmaster

Response:
def execute_query(self, tmaster, query_string, start, end):
    """ execute query """
    if not tmaster:
      raise Exception("No tmaster found")
    self.tmaster = tmaster
    root = self.parse_query_string(query_string)
    metrics = yield root.execute(self.tracker, self.tmaster, start, end)
    raise tornado.gen.Return(metrics)


Instruction:
Given the following Python 3 function, write the documentation
def find_closing_braces(self, query):
    if query[0] != '(':
      raise Exception("Trying to find closing braces for no opening braces")
    num_open_braces = 0
    for i in range(len(query)):
      c = query[i]
      if c == '(':
        num_open_braces += 1
      elif c == ')':
        num_open_braces -= 1
      if num_open_braces == 0:
        return i
    raise Exception("No closing braces found")

Response: Find the index of the closing braces for the opening braces in the query string.


Instruction:
How would you code a function in Python 3 to
return a parse tree for the query string.

Response:
def parse_query_string(self, query):
    """Returns a parse tree for the query, each of the node is a
    subclass of Operator. This is both a lexical as well as syntax analyzer step."""
    if not query:
      return None
    # Just braces do not matter
    if query[0] == '(':
      index = self.find_closing_braces(query)
      # This must be the last index, since this was an NOP starting brace
      if index != len(query) - 1:
        raise Exception("Invalid syntax")
      else:
        return self.parse_query_string(query[1:-1])
    start_index = query.find("(")
    # There must be a ( in the query
    if start_index < 0:
      # Otherwise it must be a constant
      try:
        constant = float(query)
        return constant
      except ValueError:
        raise Exception("Invalid syntax")
    token = query[:start_index]
    if token not in self.operators:
      raise Exception("Invalid token: " + token)

    # Get sub components
    rest_of_the_query = query[start_index:]
    braces_end_index = self.find_closing_braces(rest_of_the_query)
    if braces_end_index != len(rest_of_the_query) - 1:
      raise Exception("Invalid syntax")
    parts = self.get_sub_parts(rest_of_the_query[1:-1])

    # parts are simple strings in this case
    if token == "TS":
      # This will raise exception if parts are not syntactically correct
      return self.operators[token](parts)

    children = []
    for part in parts:
      children.append(self.parse_query_string(part))

    # Make a node for the current token
    node = self.operators[token](children)
    return node


Instruction:
Can you write a function in Python 3 where it
gets method executes the metrics query and writes the result to the client

Response:
def get(self):
    """ get method """
    try:
      cluster = self.get_argument_cluster()

      role = self.get_argument_role()
      environ = self.get_argument_environ()
      topology_name = self.get_argument_topology()
      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(
          cluster, role, environ, topology_name)

      start_time = self.get_argument_starttime()
      end_time = self.get_argument_endtime()
      self.validateInterval(start_time, end_time)

      query = self.get_argument_query()
      metrics = yield tornado.gen.Task(self.executeMetricsQuery,
                                       topology.tmaster, query, int(start_time), int(end_time))
      self.write_success_response(metrics)
    except Exception as e:
      Log.debug(traceback.format_exc())
      self.write_error_response(e)


Instruction:
Write a Python 3 function for
executing a query on the given tmaster and return the results as a list of dicts.

Response:
def executeMetricsQuery(self, tmaster, queryString, start_time, end_time, callback=None):
    """
    Get the specified metrics for the given query in this topology.
    Returns the following dict on success:
    {
      "timeline": [{
        "instance": <instance>,
        "data": {
          <start_time> : <numeric value>,
          <start_time> : <numeric value>,
          ...
        }
      }, {
        ...
      }, ...
      "starttime": <numeric value>,
      "endtime": <numeric value>,
    },

    Returns the following dict on failure:
    {
      "message": "..."
    }
    """

    query = Query(self.tracker)
    metrics = yield query.execute_query(tmaster, queryString, start_time, end_time)

    # Parse the response
    ret = {}
    ret["starttime"] = start_time
    ret["endtime"] = end_time
    ret["timeline"] = []

    for metric in metrics:
      tl = {
          "data": metric.timeline
      }
      if metric.instance:
        tl["instance"] = metric.instance
      ret["timeline"].append(tl)

    raise tornado.gen.Return(ret)


Instruction:
Write a Python 3 script for
creating a parser for the command.

Response:
def create_parser(subparsers):
  '''
  :param subparsers:
  :return:
  '''
  parser = subparsers.add_parser(
      'help',
      help='Prints help for commands',
      add_help=True)

  # pylint: disable=protected-access
  parser._positionals.title = "Required arguments"
  parser._optionals.title = "Optional arguments"

  parser.add_argument(
      'help-command',
      nargs='?',
      default='help',
      help='Provide help for a command')

  parser.set_defaults(subcommand='help')
  return parser


Instruction:
Explain what the following Python 3 code does
def emit(self, tup, stream=Stream.DEFAULT_STREAM_ID,
           anchors=None, direct_task=None, need_task_ids=False):
    """Emits a new tuple from this Bolt

    It is compatible with StreamParse API.

    :type tup: list or tuple
    :param tup: the new output Tuple to send from this bolt,
                should only contain only serializable data.
    :type stream: str
    :param stream: the ID of the stream to emit this Tuple to.
                   Leave empty to emit to the default stream.
    :type anchors: list
    :param anchors: a list of HeronTuples to which the emitted Tuples should be anchored.
    :type direct_task: int
    :param direct_task: the task to send the Tupel to if performing a direct emit.
    :type need_task_ids: bool
    :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.
    """
    # first check whether this tuple is sane
    self.pplan_helper.check_output_schema(stream, tup)

    # get custom grouping target task ids; get empty list if not custom grouping
    custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)

    self.pplan_helper.context.invoke_hook_emit(tup, stream, None)

    data_tuple = tuple_pb2.HeronDataTuple()
    data_tuple.key = 0

    if direct_task is not None:
      if not isinstance(direct_task, int):
        raise TypeError("direct_task argument needs to be an integer, given: %s"
                        % str(type(direct_task)))
      # performing emit-direct
      data_tuple.dest_task_ids.append(direct_task)
    elif custom_target_task_ids is not None:
      for task_id in custom_target_task_ids:
        # for custom grouping
        data_tuple.dest_task_ids.append(task_id)

    # Set the anchors for a tuple
    if anchors is not None:
      merged_roots = set()
      for tup in [t for t in anchors if isinstance(t, HeronTuple) and t.roots is not None]:
        merged_roots.update(tup.roots)
      for rt in merged_roots:
        to_add = data_tuple.roots.add()
        to_add.CopyFrom(rt)

    tuple_size_in_bytes = 0
    start_time = time.time()

    # Serialize
    for obj in tup:
      serialized = self.serializer.serialize(obj)
      data_tuple.values.append(serialized)
      tuple_size_in_bytes += len(serialized)
    serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS
    self.bolt_metrics.serialize_data_tuple(stream, serialize_latency_ns)

    super(BoltInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,
                                               tuple_size_in_bytes=tuple_size_in_bytes)

    self.bolt_metrics.update_emit_count(stream)
    if need_task_ids:
      sent_task_ids = custom_target_task_ids or []
      if direct_task is not None:
        sent_task_ids.append(direct_task)
      return sent_task_ids

Response: Emits a new tuple from this Bolt


Instruction:
Create a Python 3 function to
call when tuple was buffered into in_stream

Response:
def process_incoming_tuples(self):
    """Should be called when tuple was buffered into in_stream

    This method is equivalent to ``addBoltTasks()`` but
    is designed for event-driven single-thread bolt.
    """
    # back-pressure
    if self.output_helper.is_out_queue_available():
      self._read_tuples_and_execute()
      self.output_helper.send_out_tuples()
    else:
      # update outqueue full count
      self.bolt_metrics.update_out_queue_full_count()


Instruction:
Write a Python 3 script for
indicating that processing of a tuple has succeeded

Response:
def ack(self, tup):
    """Indicate that processing of a Tuple has succeeded

    It is compatible with StreamParse API.
    """
    if not isinstance(tup, HeronTuple):
      Log.error("Only HeronTuple type is supported in ack()")
      return

    if self.acking_enabled:
      ack_tuple = tuple_pb2.AckTuple()
      ack_tuple.ackedtuple = int(tup.id)

      tuple_size_in_bytes = 0
      for rt in tup.roots:
        to_add = ack_tuple.roots.add()
        to_add.CopyFrom(rt)
        tuple_size_in_bytes += rt.ByteSize()
      super(BoltInstance, self).admit_control_tuple(ack_tuple, tuple_size_in_bytes, True)

    process_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS
    self.pplan_helper.context.invoke_hook_bolt_ack(tup, process_latency_ns)
    self.bolt_metrics.acked_tuple(tup.stream, tup.component, process_latency_ns)


Instruction:
Create a Python 3 function to
indicate that processing of a tuple has failed

Response:
def fail(self, tup):
    """Indicate that processing of a Tuple has failed

    It is compatible with StreamParse API.
    """
    if not isinstance(tup, HeronTuple):
      Log.error("Only HeronTuple type is supported in fail()")
      return

    if self.acking_enabled:
      fail_tuple = tuple_pb2.AckTuple()
      fail_tuple.ackedtuple = int(tup.id)

      tuple_size_in_bytes = 0
      for rt in tup.roots:
        to_add = fail_tuple.roots.add()
        to_add.CopyFrom(rt)
        tuple_size_in_bytes += rt.ByteSize()
      super(BoltInstance, self).admit_control_tuple(fail_tuple, tuple_size_in_bytes, False)

    fail_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS
    self.pplan_helper.context.invoke_hook_bolt_fail(tup, fail_latency_ns)
    self.bolt_metrics.failed_tuple(tup.stream, tup.component, fail_latency_ns)


Instruction:
Can you generate the documentation for the following Python 3 function
def execute(handlers):
  '''
  Run the command
  :return:
  '''
  # verify if the environment variables are correctly set
  check_environment()

  # create the argument parser
  parser = create_parser(handlers)

  # if no argument is provided, print help and exit
  if len(sys.argv[1:]) == 0:
    parser.print_help()
    return 0

  # insert the boolean values for some of the options
  sys.argv = config.insert_bool_values(sys.argv)

  try:
    # parse the args
    args, unknown_args = parser.parse_known_args()
  except ValueError as ex:
    Log.error("Error while parsing arguments: %s", str(ex))
    Log.debug(traceback.format_exc())
    sys.exit(1)

  command_line_args = vars(args)

  # set log level
  log.set_logging_level(command_line_args)
  Log.debug("Input Command Line Args: %s", command_line_args)

  # command to be execute
  command = command_line_args['subcommand']

  # print the input parameters, if verbose is enabled
  Log.debug("Processed Command Line Args: %s", command_line_args)

  results = run(handlers, command, parser, command_line_args, unknown_args)

  return 0 if result.is_successful(results) else 1

Response:
Runs the command
 Execute the command


Instruction:
Write a Python 3 function that can
create a subparser for the standalone command

Response:
def create_parser(subparsers):
  '''
  Create a subparser for the standalone command
  :param subparsers:
  :return:
  '''
  parser = subparsers.add_parser(
      'standalone',
      help='Start a standalone Heron cluster',
      add_help=True
  )

  cli_args.add_titles(parser)

  parser_action = parser.add_subparsers()

  parser_cluster = parser_action.add_parser(
      Action.CLUSTER,
      help='Start or stop cluster',
      add_help=True,
      formatter_class=argparse.RawTextHelpFormatter,
  )
  parser_cluster.set_defaults(action=Action.CLUSTER)

  parser_set = parser_action.add_parser(
      Action.SET,
      help='Set configurations for standalone cluster e.g. master or slave nodes',
      add_help=True,
      formatter_class=argparse.RawTextHelpFormatter
  )
  parser_set.set_defaults(action=Action.SET)

  parser_template = parser_action.add_parser(
      Action.TEMPLATE,
      help='Template Heron configurations based on cluster roles',
      add_help=True,
      formatter_class=argparse.RawTextHelpFormatter
  )
  parser_template.set_defaults(action=Action.TEMPLATE)

  parser_cluster.add_argument(
      TYPE,
      type=str,
      choices={Cluster.START, Cluster.STOP},
      help= \
"""
Choices supports the following:
  start     - Start standalone Heron cluster
  stop      - Stop standalone Heron cluster
"""
  )

  parser_template.add_argument(
      TYPE,
      type=str,
      choices={"configs"},
  )

  parser_get = parser_action.add_parser(
      Action.GET,
      help='Get attributes about the standalone cluster',
      add_help=True,
      formatter_class=argparse.RawTextHelpFormatter
  )
  parser_get.set_defaults(action=Action.GET)

  parser_get.add_argument(
      TYPE,
      type=str,
      choices={Get.SERVICE_URL, Get.HERON_TRACKER_URL, Get.HERON_UI_URL},
      help= \
      """
      Choices supports the following:
        service-url         - Get the service url for standalone cluster
        heron-tracker-url   - Get the url for the heron tracker in standalone cluster
        heron-ui-url        - Get the url for the heron ui standalone cluster
      """
  )

  parser_info = parser_action.add_parser(
      Action.INFO,
      help='Get general information about the standalone cluster',
      add_help=True,
      formatter_class=argparse.RawTextHelpFormatter
  )
  parser_info.set_defaults(action=Action.INFO)

  add_additional_args([parser_set, parser_cluster, parser_template, parser_get, parser_info])
  parser.set_defaults(subcommand='standalone')
  return parser


Instruction:
Write a Python 3 function that can
template slave config file

Response:
def template_slave_hcl(cl_args, masters):
  '''
  Template slave config file
  '''
  slave_config_template = "%s/standalone/templates/slave.template.hcl" % cl_args["config_path"]
  slave_config_actual = "%s/standalone/resources/slave.hcl" % cl_args["config_path"]
  masters_in_quotes = ['"%s"' % master for master in masters]
  template_file(slave_config_template, slave_config_actual,
                {"<nomad_masters:master_port>": ", ".join(masters_in_quotes)})


Instruction:
Can you write a function in Python 3 where it
templates the apiserver. hcl file for the given master and zookeepers.

Response:
def template_apiserver_hcl(cl_args, masters, zookeepers):
  """
  template apiserver.hcl
  """
  single_master = masters[0]
  apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \
                              % cl_args["config_path"]
  apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args["config_path"]

  replacements = {
      "<heron_apiserver_hostname>": '"%s"' % get_hostname(single_master, cl_args),
      "<heron_apiserver_executable>": '"%s/heron-apiserver"'
                                      % config.get_heron_bin_dir()
                                      if is_self(single_master)
                                      else '"%s/.heron/bin/heron-apiserver"'
                                      % get_remote_home(single_master, cl_args),
      "<zookeeper_host:zookeeper_port>": ",".join(
          ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
      "<scheduler_uri>": "http://%s:4646" % single_master
  }

  template_file(apiserver_config_template, apiserver_config_actual, replacements)


Instruction:
Here you have a function in Python 3, explain what it does
def print_cluster_info(cl_args):
  '''
  get cluster info for standalone cluster
  '''
  parsed_roles = read_and_parse_roles(cl_args)
  masters = list(parsed_roles[Role.MASTERS])
  slaves = list(parsed_roles[Role.SLAVES])
  zookeepers = list(parsed_roles[Role.ZOOKEEPERS])
  cluster = list(parsed_roles[Role.CLUSTER])

  # OrderedDicts are used here so that the key order can be
  # specified directly
  info = OrderedDict()
  info['numNodes'] = len(cluster)
  info['nodes'] = cluster
  roles = OrderedDict()
  roles['masters'] = masters
  roles['slaves'] = slaves
  roles['zookeepers'] = zookeepers
  urls = OrderedDict()
  urls['serviceUrl'] = get_service_url(cl_args)
  urls['heronUi'] = get_heron_ui_url(cl_args)
  urls['heronTracker'] = get_heron_tracker_url(cl_args)
  info['roles'] = roles
  info['urls'] = urls

  print json.dumps(info, indent=2)

Response:
get cluster info for standalone cluster
 → print info about the cluster


Instruction:
How would you explain what the following Python 3 function does
def add_additional_args(parsers):
  '''
  add additional parameters to parser
  '''
  for parser in parsers:
    cli_args.add_verbose(parser)
    cli_args.add_config(parser)
    parser.add_argument(
        '--heron-dir',
        default=config.get_heron_dir(),
        help='Path to Heron home directory')

Response: add additional parameters to parser


Instruction:
Can you create a Python 3 function that
starts a Heron standalone cluster.

Response:
def start_cluster(cl_args):
  '''
  Start a Heron standalone cluster
  '''
  roles = read_and_parse_roles(cl_args)
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]
  zookeepers = roles[Role.ZOOKEEPERS]
  Log.info("Roles:")
  Log.info(" - Master Servers: %s" % list(masters))
  Log.info(" - Slave Servers: %s" % list(slaves))
  Log.info(" - Zookeeper Servers: %s" % list(zookeepers))
  if not masters:
    Log.error("No master servers specified!")
    sys.exit(-1)
  if not slaves:
    Log.error("No slave servers specified!")
    sys.exit(-1)
  if not zookeepers:
    Log.error("No zookeeper servers specified!")
    sys.exit(-1)
  # make sure configs are templated
  update_config_files(cl_args)

  dist_nodes = list(masters.union(slaves))
  # if just local deployment
  if not (len(dist_nodes) == 1 and is_self(dist_nodes[0])):
    distribute_package(roles, cl_args)
  start_master_nodes(masters, cl_args)
  start_slave_nodes(slaves, cl_args)
  start_api_server(masters, cl_args)
  start_heron_tools(masters, cl_args)
  Log.info("Heron standalone cluster complete!")


Instruction:
Here you have a function in Python 3, explain what it does
def start_heron_tools(masters, cl_args):
  '''
  Start Heron tracker and UI
  '''
  single_master = list(masters)[0]
  wait_for_master_to_start(single_master)

  cmd = "%s run %s >> /tmp/heron_tools_start.log 2>&1 &" \
        % (get_nomad_path(cl_args), get_heron_tools_job_file(cl_args))
  Log.info("Starting Heron Tools on %s" % single_master)

  if not is_self(single_master):
    cmd = ssh_remote_execute(cmd, single_master, cl_args)
  Log.debug(cmd)
  pid = subprocess.Popen(cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  return_code = pid.wait()
  output = pid.communicate()
  Log.debug("return code: %s output: %s" % (return_code, output))
  if return_code != 0:
    Log.error("Failed to start Heron Tools on %s with error:\n%s" % (single_master, output[1]))
    sys.exit(-1)

  wait_for_job_to_start(single_master, "heron-tools")
  Log.info("Done starting Heron Tools")

Response: Start Heron Tools on a single master.


Instruction:
How would you code a function in Python 3 to
distribute Heron packages to all nodes

Response:
def distribute_package(roles, cl_args):
  '''
  distribute Heron packages to all nodes
  '''
  Log.info("Distributing heron package to nodes (this might take a while)...")
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]

  tar_file = tempfile.NamedTemporaryFile(suffix=".tmp").name
  Log.debug("TAR file %s to %s" % (cl_args["heron_dir"], tar_file))
  make_tarfile(tar_file, cl_args["heron_dir"])
  dist_nodes = masters.union(slaves)

  scp_package(tar_file, dist_nodes, cl_args)


Instruction:
Write a Python 3 function that can
wait for a nomad master to start

Response:
def wait_for_master_to_start(single_master):
  '''
  Wait for a nomad master to start
  '''
  i = 0
  while True:
    try:
      r = requests.get("http://%s:4646/v1/status/leader" % single_master)
      if r.status_code == 200:
        break
    except:
      Log.debug(sys.exc_info()[0])
      Log.info("Waiting for cluster to come up... %s" % i)
      time.sleep(1)
      if i > 10:
        Log.error("Failed to start Nomad Cluster!")
        sys.exit(-1)
    i = i + 1


Instruction:
Write a Python 3 script to
wait for a Nomad job to start

Response:
def wait_for_job_to_start(single_master, job):
  '''
  Wait for a Nomad job to start
  '''
  i = 0
  while True:
    try:
      r = requests.get("http://%s:4646/v1/job/%s" % (single_master, job))
      if r.status_code == 200 and r.json()["Status"] == "running":
        break
      else:
        raise RuntimeError()
    except:
      Log.debug(sys.exc_info()[0])
      Log.info("Waiting for %s to come up... %s" % (job, i))
      time.sleep(1)
      if i > 20:
        Log.error("Failed to start Nomad Cluster!")
        sys.exit(-1)
    i = i + 1


Instruction:
Can you generate a brief explanation for the following Python 3 code
def scp_package(package_file, destinations, cl_args):
  '''
  scp and extract package
  '''
  pids = []
  for dest in destinations:
    if is_self(dest):
      continue
    Log.info("Server: %s" % dest)
    file_path = "/tmp/heron.tar.gz"
    dest_file_path = "%s:%s" % (dest, file_path)

    remote_cmd = "rm -rf ~/.heron && mkdir ~/.heron " \
                 "&& tar -xzvf %s -C ~/.heron --strip-components 1" % (file_path)
    cmd = '%s && %s' \
          % (scp_cmd(package_file, dest_file_path, cl_args),
             ssh_remote_execute(remote_cmd, dest, cl_args))
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": dest})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to scp package to %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done distributing packages")

Response:
scp and extract package
 °


Instruction:
How would you explain what the following Python 3 function does
def make_tarfile(output_filename, source_dir):
  '''
  Tar a directory
  '''
  with tarfile.open(output_filename, "w:gz") as tar:
    tar.add(source_dir, arcname=os.path.basename(source_dir))

Response: Create a tar file from a source directory.


Instruction:
Given the following Python 3 function, write the documentation
def read_and_parse_roles(cl_args):
  '''
  read config files to get roles
  '''
  roles = dict()

  with open(get_inventory_file(cl_args), 'r') as stream:
    try:
      roles = yaml.load(stream)
    except yaml.YAMLError as exc:
      Log.error("Error parsing inventory file: %s" % exc)
      sys.exit(-1)

  if Role.ZOOKEEPERS not in roles or not roles[Role.ZOOKEEPERS]:
    Log.error("Zookeeper servers node defined!")
    sys.exit(-1)

  if Role.CLUSTER not in roles or not roles[Role.CLUSTER]:
    Log.error("Heron cluster nodes defined!")
    sys.exit(-1)

  # Set roles
  roles[Role.MASTERS] = set([roles[Role.CLUSTER][0]])
  roles[Role.SLAVES] = set(roles[Role.CLUSTER])
  roles[Role.ZOOKEEPERS] = set(roles[Role.ZOOKEEPERS])
  roles[Role.CLUSTER] = set(roles[Role.CLUSTER])

  return roles

Response:
read config files to get roles
 – parse them and return a dict


Instruction:
Write a Python 3 script for
getting remote home directory of remote host

Response:
def get_remote_home(host, cl_args):
  '''
  get home directory of remote host
  '''
  cmd = "echo ~"
  if not is_self(host):
    cmd = ssh_remote_execute(cmd, host, cl_args)
  pid = subprocess.Popen(cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  return_code = pid.wait()
  output = pid.communicate()

  if return_code != 0:
    Log.error("Failed to get home path for remote host %s with output:\n%s" % (host, output))
    sys.exit(-1)
  return output[0].strip("\n")


Instruction:
Implement a function in Python 3 to
get host name of remote host

Response:
def get_hostname(ip_addr, cl_args):
  '''
  get host name of remote host
  '''
  if is_self(ip_addr):
    return get_self_hostname()
  cmd = "hostname"
  ssh_cmd = ssh_remote_execute(cmd, ip_addr, cl_args)
  pid = subprocess.Popen(ssh_cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  return_code = pid.wait()
  output = pid.communicate()

  if return_code != 0:
    Log.error("Failed to get hostname for remote host %s with output:\n%s" % (ip_addr, output))
    sys.exit(-1)
  return output[0].strip("\n")


Instruction:
How would you code a function in Python 3 to
check if this host is this addr check if this host is self addr check if this host is self addr athor

Response:
def is_self(addr):
  '''
  check if this host is this addr
  '''
  ips = []
  for i in netifaces.interfaces():
    entry = netifaces.ifaddresses(i)
    if netifaces.AF_INET in entry:
      for ipv4 in entry[netifaces.AF_INET]:
        if "addr" in ipv4:
          ips.append(ipv4["addr"])
  return addr in ips or addr == get_self_hostname()


Instruction:
Here you have a function in Python 3, explain what it does
def log(self, message, level=None):
    if level is None:
      _log_level = logging.INFO
    else:
      if level == "trace" or level == "debug":
        _log_level = logging.DEBUG
      elif level == "info":
        _log_level = logging.INFO
      elif level == "warn":
        _log_level = logging.WARNING
      elif level == "error":
        _log_level = logging.ERROR
      else:
        raise ValueError("%s is not supported as logging level" % str(level))

    self.logger.log(_log_level, message)

Response: Log a message to the log file.


Instruction:
Here you have a function in Python 3, explain what it does
def load_py_instance(self, is_spout):
    try:
      if is_spout:
        spout_proto = self.pplan_helper.get_my_spout()
        py_classpath = spout_proto.comp.class_name
        self.logger.info("Loading Spout from: %s", py_classpath)
      else:
        bolt_proto = self.pplan_helper.get_my_bolt()
        py_classpath = bolt_proto.comp.class_name
        self.logger.info("Loading Bolt from: %s", py_classpath)

      pex_loader.load_pex(self.pplan_helper.topology_pex_abs_path)
      spbl_class = pex_loader.import_and_get_class(self.pplan_helper.topology_pex_abs_path,
                                                   py_classpath)
    except Exception as e:
      spbl = "spout" if is_spout else "bolt"
      self.logger.error(traceback.format_exc())
      raise RuntimeError("Error when loading a %s from pex: %s" % (spbl, str(e)))
    return spbl_class

Response: Loads a user defined component from the pex file.


Instruction:
Can you create a Python 3 function that
gets all the values for the cluster and environ.

Response:
def get(self):
    """ get method """
    # Get all the values for parameter "cluster".
    clusters = self.get_arguments(constants.PARAM_CLUSTER)
    # Get all the values for parameter "environ".
    environs = self.get_arguments(constants.PARAM_ENVIRON)
    # Get role
    role = self.get_argument_role()

    ret = {}
    topologies = self.tracker.topologies
    for topology in topologies:
      cluster = topology.cluster
      environ = topology.environ
      execution_state = topology.execution_state

      if not cluster or not execution_state or not environ:
        continue

      topo_role = execution_state.role
      if not topo_role:
        continue

      # This cluster is not asked for.
      # Note that "if not clusters", then
      # we show for all the clusters.
      if clusters and cluster not in clusters:
        continue

      # This environ is not asked for.
      # Note that "if not environs", then
      # we show for all the environs.
      if environs and environ not in environs:
        continue

      # This role is not asked for.
      # Note that "if not role", then
      # we show for all the roles.
      if role and role != topo_role:
        continue

      if cluster not in ret:
        ret[cluster] = {}
      if topo_role not in ret[cluster]:
        ret[cluster][topo_role] = {}
      if environ not in ret[cluster][topo_role]:
        ret[cluster][topo_role][environ] = []
      ret[cluster][topo_role][environ].append(topology.name)
    self.write_success_response(ret)


Instruction:
Write a Python 3 function for
resolving all symbolic references that src points to.

Response:
def dereference_symlinks(src):
    """
    Resolve all symbolic references that `src` points to.  Note that this
    is different than `os.path.realpath` as path components leading up to
    the final location may still be symbolic links.
    """
    while os.path.islink(src):
        src = os.path.join(os.path.dirname(src), os.readlink(src))

    return src


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get(self):
    clusters = [statemgr.name for statemgr in self.tracker.state_managers]

    self.write_success_response(clusters)

Response: get method returns list of all available state managers


Instruction:
Implement a Python 3 function for
getting the list of topologies given a cluster submitted by a given role under a given environment

Response:
def get_cluster_role_env_topologies(cluster, role, env):
  '''
  Get the list of topologies given a cluster submitted by a given role under a given environment
  :param cluster:
  :param role:
  :param env:
  :return:
  '''
  return _get_topologies(cluster, role=role, env=env)


Instruction:
Write a Python 3 function that can
get the execution state of a topology in a cluster.

Response:
def get_execution_state(cluster, environ, topology, role=None):
  '''
  Get the execution state of a topology in a cluster
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(cluster=cluster, environ=environ, topology=topology)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(create_url(EXECUTION_STATE_URL_FMT), params)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
Can you implement a function in Python 3 that
gets the logical plan state of a topology in a cluster.

Response:
def get_logical_plan(cluster, environ, topology, role=None):
  '''
  Get the logical plan state of a topology in a cluster
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(cluster=cluster, environ=environ, topology=topology)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(LOGICALPLAN_URL_FMT), params)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
Write a Python 3 script to
get the list of component names for the topology from Heron Nest

Response:
def get_comps(cluster, environ, topology, role=None):
  '''
  Get the list of component names for the topology from Heron Nest
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(cluster=cluster, environ=environ, topology=topology)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(LOGICALPLAN_URL_FMT), params)
  lplan = yield fetch_url_as_json(request_url)
  comps = lplan['spouts'].keys() + lplan['bolts'].keys()
  raise tornado.gen.Return(comps)


Instruction:
Implement a function in Python 3 to
get the list of instances for the topology from Heron Nest

Response:
def get_instances(cluster, environ, topology, role=None):
  '''
  Get the list of instances for the topology from Heron Nest
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(cluster=cluster, environ=environ, topology=topology)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(PHYSICALPLAN_URL_FMT), params)
  pplan = yield fetch_url_as_json(request_url)
  instances = pplan['instances'].keys()
  raise tornado.gen.Return(instances)


Instruction:
How would you explain what the following Python 3 function does
def get_physical_plan(cluster, environ, topology, role=None):
  '''
  Get the physical plan state of a topology in a cluster from tracker
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(cluster=cluster, environ=environ, topology=topology)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(PHYSICALPLAN_URL_FMT), params)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get the physical plan state of a topology in a cluster from tracker


Instruction:
Write a Python 3 script for
getting the scheduler location of a topology in a cluster from tracker

Response:
def get_scheduler_location(cluster, environ, topology, role=None):
  '''
  Get the scheduler location of a topology in a cluster from tracker
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(cluster=cluster, environ=environ, topology=topology)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(SCHEDULER_LOCATION_URL_FMT), params)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
How would you implement a function in Python 3 that
gets summary of exception for a component.

Response:
def get_component_exceptionsummary(cluster, environ, topology, component, role=None):
  '''
  Get summary of exception for a component
  :param cluster:
  :param environ:
  :param topology:
  :param component:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      component=component)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(EXCEPTION_SUMMARY_URL_FMT), params)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
Can you generate the documentation for the following Python 3 function
def get_component_exceptions(cluster, environ, topology, component, role=None):
  '''
  Get exceptions for 'component' for 'topology'
  :param cluster:
  :param environ:
  :param topology:
  :param component:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      component=component)
  if role is not None:
    params['role'] = role
  request_url = tornado.httputil.url_concat(
      create_url(EXCEPTIONS_URL_FMT), params)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get exceptions for component for cluster.


Instruction:
Can you generate the documentation for the following Python 3 function
def get_comp_instance_metrics(cluster, environ, topology, component,
                              metrics, instances, time_range, role=None):
  '''
  Get the metrics for some instances of a topology from tracker
  :param cluster:
  :param environ:
  :param topology:
  :param component:
  :param metrics:     dict of display name to cuckoo name
  :param instances:
  :param time_range:  2-tuple consisting of start and end of range
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      component=component)
  if role is not None:
    params['role'] = role

  # form the fetch url
  request_url = tornado.httputil.url_concat(
      create_url(METRICS_URL_FMT), params)

  # convert a single instance to a list, if needed
  all_instances = instances if isinstance(instances, list) else [instances]

  # append each metric to the url
  for _, metric_name in metrics.items():
    request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name[0]))

  # append each instance to the url
  for i in all_instances:
    request_url = tornado.httputil.url_concat(request_url, dict(instance=i))

  # append the time interval to the url
  request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[1]))

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get the metrics for some instances of a topology from tracker


Instruction:
Can you create a Python 3 function that
gets the metrics for all the instances of a topology from Heron Nest

Response:
def get_comp_metrics(cluster, environ, topology, component,
                     instances, metricnames, time_range, role=None):
  '''
  Get the metrics for all the instances of a topology from Heron Nest
  :param cluster:
  :param environ:
  :param topology:
  :param component:
  :param instances:
  :param metricnames:   dict of display name to cuckoo name
  :param time_range:    2-tuple consisting of start and end of range
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      component=component)
  if role is not None:
    params['role'] = role

  # form the url
  request_url = tornado.httputil.url_concat(
      create_url(METRICS_URL_FMT), params)

  # append each metric to the url
  for metric_name in metricnames:
    request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))

  # append each instance to the url
  for instance in instances:
    request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))

  # append the time interval to the url
  request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[1]))

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
Make a summary of the following Python 3 code
def get_metrics(cluster, environment, topology, timerange, query, role=None):
  '''
  Get the metrics for a topology from tracker
  :param cluster:
  :param environment:
  :param topology:
  :param timerange:
  :param query:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environment,
      topology=topology,
      starttime=timerange[0],
      endtime=timerange[1],
      query=query)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(
      create_url(METRICS_QUERY_URL_FMT), params
  )

  logging.info("get_metrics %s", request_url)
  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get the metrics for a topology from tracker


Instruction:
Write a Python 3 script to
get the minute - by - minute metrics for all instances of a topology from tracker This function returns the minute - by - minute metrics for all instances of a topology from tracker

Response:
def get_comp_metrics_timeline(cluster, environ, topology, component,
                              instances, metricnames, time_range, role=None):
  '''
  Get the minute-by-minute metrics for all instances of a topology from tracker
  :param cluster:
  :param environ:
  :param topology:
  :param component:
  :param instances:
  :param metricnames:   dict of display name to cuckoo name
  :param time_range:    2-tuple consisting of start and end of range
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      component=component)

  if role is not None:
    params['role'] = role

  # form the url
  request_url = tornado.httputil.url_concat(create_url(METRICS_TIMELINE_URL_FMT), params)

  if role is not None:
    request_url = tornado.httputil.url_concat(request_url, dict(role=role))

  # append each metric to the url
  for metric_name in metricnames:
    request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))

  # append each instance to the url
  for instance in instances:
    request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))

  # append the time interval to the url
  request_url = tornado.httputil.url_concat(
      request_url, dict(starttime=time_range[0], endtime=time_range[1]))

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
Here you have a function in Python 3, explain what it does
def get_topology_info(cluster, environ, topology, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(create_url(INFO_URL_FMT), params)

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get topology info from the server.


Instruction:
Can you tell what is the following Python 3 function doing
def get_instance_pid(cluster, environ, topology, instance, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param instance:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      instance=instance)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(create_url(PID_URL_FMT), params)

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get the PID of the instance.


Instruction:
Make a summary of the following Python 3 code
def get_instance_jstack(cluster, environ, topology, instance, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param instance:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      instance=instance)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(
      create_url(JSTACK_URL_FMT), params)

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get the JStack for a single instance.


Instruction:
Can you write a function in Python 3 where it
gets the memory histogram of an instance.

Response:
def get_instance_mem_histogram(cluster, environ, topology, instance, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param instance:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      instance=instance)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(
      create_url(HISTOGRAM_URL_FMT), params)

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def run_instance_jmap(cluster, environ, topology, instance, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param instance:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      instance=instance)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(
      create_url(JMAP_URL_FMT), params)

  if role is not None:
    request_url = tornado.httputil.url_concat(request_url, dict(role=role))

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Runs a JMAP request for the specified instance.


Instruction:
Implement a Python 3 function for
returning the URL to download the container file.

Response:
def get_container_file_download_url(cluster, environ, topology, container,
                                    path, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param container:
  :param path:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      container=container,
      path=path)
  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(
      create_url(FILE_DOWNLOAD_URL_FMT), params)

  if role is not None:
    request_url = tornado.httputil.url_concat(request_url, dict(role=role))
  return request_url


Instruction:
How would you code a function in Python 3 to
get container file data from a container.

Response:
def get_container_file_data(cluster, environ, topology, container,
                            path, offset, length, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param container:
  :param path:
  :param offset:
  :param length:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      container=container,
      path=path,
      offset=offset,
      length=length)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(
      create_url(FILE_DATA_URL_FMT), params)

  if role is not None:
    request_url = tornado.httputil.url_concat(request_url, dict(role=role))

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))


Instruction:
How would you explain what the following Python 3 function does
def get_filestats(cluster, environ, topology, container, path, role=None):
  '''
  :param cluster:
  :param environ:
  :param topology:
  :param container:
  :param path:
  :param role:
  :return:
  '''
  params = dict(
      cluster=cluster,
      environ=environ,
      topology=topology,
      container=container,
      path=path)

  if role is not None:
    params['role'] = role

  request_url = tornado.httputil.url_concat(create_url(FILESTATS_URL_FMT), params)

  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))

Response: Get all the files in a cluster.


Instruction:
Create a Python 3 function to
get metrics for a given instance.

Response:
def fetch(self, cluster, metric, topology, component, instance, timerange, environ=None):
    '''
    :param cluster:
    :param metric:
    :param topology:
    :param component:
    :param instance:
    :param timerange:
    :param environ:
    :return:
    '''
    components = [component] if component != "*" else (yield get_comps(cluster, environ, topology))

    futures = []
    for comp in components:
      query = self.get_query(metric, comp, instance)
      future = get_metrics(cluster, environ, topology, timerange, query)
      futures.append(future)

    results = yield futures

    timelines = []
    for result in results:
      timelines.extend(result["timeline"])

    result = self.get_metric_response(timerange, timelines, False)

    raise tornado.gen.Return(result)


Instruction:
Write a Python 3 script to
get the max metric for each component in the cluster.

Response:
def fetch_max(self, cluster, metric, topology, component, instance, timerange, environ=None):
    '''
    :param cluster:
    :param metric:
    :param topology:
    :param component:
    :param instance:
    :param timerange:
    :param environ:
    :return:
    '''
    components = [component] if component != "*" else (yield get_comps(cluster, environ, topology))

    result = {}
    futures = []
    for comp in components:
      query = self.get_query(metric, comp, instance)
      max_query = "MAX(%s)" % query
      future = get_metrics(cluster, environ, topology, timerange, max_query)
      futures.append(future)

    results = yield futures

    data = self.compute_max(results)

    result = self.get_metric_response(timerange, data, True)

    raise tornado.gen.Return(result)