Instruction:
Write a Python 3 script to
notify all barriers affected by a slot.

Response:
def notify_barriers(self,
                      slot_key,
                      cursor,
                      use_barrier_indexes,
                      max_to_notify=_MAX_BARRIERS_TO_NOTIFY):
    """Searches for barriers affected by a slot and triggers completed ones.

    Args:
      slot_key: db.Key or stringified key of the _SlotRecord that was filled.
      cursor: Stringified Datastore cursor where the notification query
        should pick up.
      use_barrier_indexes: When True, use _BarrierIndex records to determine
        which _Barriers to trigger by having this _SlotRecord filled. When
        False, use the old method that queries for _BarrierRecords by
        the blocking_slots parameter.
      max_to_notify: Used for testing.

    Raises:
      PipelineStatusError: If any of the barriers are in a bad state.
    """
    if not isinstance(slot_key, db.Key):
      slot_key = db.Key(slot_key)
    logging.debug('Notifying slot %r', slot_key)

    if use_barrier_indexes:
      # Please see models.py:_BarrierIndex to understand how _BarrierIndex
      # entities relate to _BarrierRecord entities.
      query = (
          _BarrierIndex.all(cursor=cursor, keys_only=True)
          .ancestor(slot_key))
      barrier_index_list = query.fetch(max_to_notify)
      barrier_key_list = [
          _BarrierIndex.to_barrier_key(key) for key in barrier_index_list]

      # If there are task and pipeline kickoff retries it's possible for a
      # _BarrierIndex to exist for a _BarrierRecord that was not successfully
      # written. It's safe to ignore this because the original task that wrote
      # the _BarrierIndex and _BarrierRecord would not have made progress to
      # kick off a real pipeline or child pipeline unless all of the writes for
      # these dependent entities went through. We assume that the instigator
      # retried from scratch and somehwere there exists a good _BarrierIndex and
      # corresponding _BarrierRecord that tries to accomplish the same thing.
      barriers = db.get(barrier_key_list)
      results = []
      for barrier_key, barrier in zip(barrier_key_list, barriers):
        if barrier is None:
          logging.debug('Ignoring that Barrier "%r" is missing, '
                        'relies on Slot "%r"', barrier_key, slot_key)
        else:
          results.append(barrier)
    else:
      # TODO(user): Delete this backwards compatible codepath and
      # make use_barrier_indexes the assumed default in all cases.
      query = (
          _BarrierRecord.all(cursor=cursor)
          .filter('blocking_slots =', slot_key))
      results = query.fetch(max_to_notify)

    # Fetch all blocking _SlotRecords for any potentially triggered barriers.
    blocking_slot_keys = []
    for barrier in results:
      blocking_slot_keys.extend(barrier.blocking_slots)

    blocking_slot_dict = {}
    for slot_record in db.get(blocking_slot_keys):
      if slot_record is None:
        continue
      blocking_slot_dict[slot_record.key()] = slot_record

    task_list = []
    updated_barriers = []
    for barrier in results:
      ready_slots = []
      for blocking_slot_key in barrier.blocking_slots:
        slot_record = blocking_slot_dict.get(blocking_slot_key)
        if slot_record is None:
          raise UnexpectedPipelineError(
              'Barrier "%r" relies on Slot "%r" which is missing.' %
              (barrier.key(), blocking_slot_key))
        if slot_record.status == _SlotRecord.FILLED:
          ready_slots.append(blocking_slot_key)

      # When all of the blocking_slots have been filled, consider the barrier
      # ready to trigger. We'll trigger it regardless of the current
      # _BarrierRecord status, since there could be task queue failures at any
      # point in this flow; this rolls forward the state and de-dupes using
      # the task name tombstones.
      pending_slots = set(barrier.blocking_slots) - set(ready_slots)
      if not pending_slots:
        if barrier.status != _BarrierRecord.FIRED:
          barrier.status = _BarrierRecord.FIRED
          barrier.trigger_time = self._gettime()
          updated_barriers.append(barrier)

        purpose = barrier.key().name()
        if purpose == _BarrierRecord.START:
          path = self.pipeline_handler_path
          countdown = None
        else:
          path = self.finalized_handler_path
          # NOTE: Wait one second before finalization to prevent
          # contention on the _PipelineRecord entity.
          countdown = 1
        pipeline_key = _BarrierRecord.target.get_value_for_datastore(barrier)
        pipeline_record = db.get(pipeline_key)
        logging.debug('Firing barrier %r', barrier.key())
        task_list.append(taskqueue.Task(
            url=path,
            countdown=countdown,
            name='ae-barrier-fire-%s-%s' % (pipeline_key.name(), purpose),
            params=dict(pipeline_key=pipeline_key, purpose=purpose),
            headers={'X-Ae-Pipeline-Key': pipeline_key},
            target=pipeline_record.params['target']))
      else:
        logging.debug('Not firing barrier %r, Waiting for slots: %r',
                      barrier.key(), pending_slots)

    # Blindly overwrite _BarrierRecords that have an updated status. This is
    # acceptable because by this point all finalization barriers for
    # generator children should have already had their final outputs assigned.
    if updated_barriers:
      db.put(updated_barriers)

    # Task continuation with sequence number to prevent fork-bombs.
    if len(results) == max_to_notify:
      the_match = re.match('(.*)-ae-barrier-notify-([0-9]+)', self.task_name)
      if the_match:
        prefix = the_match.group(1)
        end = int(the_match.group(2)) + 1
      else:
        prefix = self.task_name
        end = 0
      task_list.append(taskqueue.Task(
          name='%s-ae-barrier-notify-%d' % (prefix, end),
          url=self.barrier_handler_path,
          params=dict(
              slot_key=slot_key,
              cursor=query.cursor(),
              use_barrier_indexes=use_barrier_indexes)))

    if task_list:
      try:
        taskqueue.Queue(self.queue_name).add(task_list)
      except (taskqueue.TombstonedTaskError, taskqueue.TaskAlreadyExistsError):
        pass


Instruction:
Can you generate a brief explanation for the following Python 3 code
def begin_abort(self, root_pipeline_key, abort_message):
    def txn():
      pipeline_record = db.get(root_pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to abort root pipeline ID "%s" but it does not exist.',
            root_pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status == _PipelineRecord.ABORTED:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; already in state: %s',
            root_pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()
      if pipeline_record.abort_requested:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; abort signal already sent.',
            root_pipeline_key.name())
        raise db.Rollback()

      pipeline_record.abort_requested = True
      pipeline_record.abort_message = abort_message
      pipeline_record.put()

      task = taskqueue.Task(
          url=self.fanout_abort_handler_path,
          params=dict(root_pipeline_key=root_pipeline_key))
      task.add(queue_name=self.queue_name, transactional=True)
      return True

    return db.run_in_transaction(txn)

Response: Starts the abort process for a root pipeline and all its children.


Instruction:
How would you implement a function in Python 3 that
sends the abort signal to all child pipelines that are in the root pipeline.

Response:
def continue_abort(self,
                     root_pipeline_key,
                     cursor=None,
                     max_to_notify=_MAX_ABORTS_TO_BEGIN):
    """Sends the abort signal to all children for a root pipeline.

    Args:
      root_pipeline_key: db.Key of the root pipeline to abort.
      cursor: The query cursor for enumerating _PipelineRecords when inserting
        tasks to cause child pipelines to terminate.
      max_to_notify: Used for testing.
    """
    if not isinstance(root_pipeline_key, db.Key):
      root_pipeline_key = db.Key(root_pipeline_key)
    # NOTE: The results of this query may include _PipelineRecord instances
    # that are not actually "reachable", meaning you cannot get to them by
    # starting at the root pipeline and following "fanned_out" onward. This
    # is acceptable because even these defunct _PipelineRecords will properly
    # set their status to ABORTED when the signal comes, regardless of any
    # other status they may have had.
    #
    # The only gotcha here is if a Pipeline's finalize method somehow modifies
    # its inputs (like deleting an input file). In the case there are
    # unreachable child pipelines, it will appear as if two finalize methods
    # have been called instead of just one. The saving grace here is that
    # finalize must be idempotent, so this *should* be harmless.
    query = (
        _PipelineRecord.all(cursor=cursor)
        .filter('root_pipeline =', root_pipeline_key))
    results = query.fetch(max_to_notify)

    task_list = []
    for pipeline_record in results:
      if pipeline_record.status not in (
          _PipelineRecord.RUN, _PipelineRecord.WAITING):
        continue

      pipeline_key = pipeline_record.key()
      task_list.append(taskqueue.Task(
          name='%s-%s-abort' % (self.task_name, pipeline_key.name()),
          url=self.abort_handler_path,
          params=dict(pipeline_key=pipeline_key, purpose=_BarrierRecord.ABORT),
          headers={'X-Ae-Pipeline-Key': pipeline_key}))

    # Task continuation with sequence number to prevent fork-bombs.
    if len(results) == max_to_notify:
      the_match = re.match('(.*)-([0-9]+)', self.task_name)
      if the_match:
        prefix = the_match.group(1)
        end = int(the_match.group(2)) + 1
      else:
        prefix = self.task_name
        end = 0
      task_list.append(taskqueue.Task(
          name='%s-%d' % (prefix, end),
          url=self.fanout_abort_handler_path,
          params=dict(root_pipeline_key=root_pipeline_key,
                      cursor=query.cursor())))

    if task_list:
      try:
        taskqueue.Queue(self.queue_name).add(task_list)
      except (taskqueue.TombstonedTaskError, taskqueue.TaskAlreadyExistsError):
        pass


Instruction:
Explain what the following Python 3 code does
def start(self, pipeline, return_task=True, countdown=None, eta=None):
    # Adjust all pipeline output keys for this Pipeline to be children of
    # the _PipelineRecord, that way we can write them all and submit in a
    # single transaction.
    for name, slot in pipeline.outputs._output_dict.iteritems():
      slot.key = db.Key.from_path(
          *slot.key.to_path(), **dict(parent=pipeline._pipeline_key))

    _, output_slots, params_text, params_blob = _generate_args(
        pipeline, pipeline.outputs, self.queue_name, self.base_path)

    @db.transactional(propagation=db.INDEPENDENT)
    def txn():
      pipeline_record = db.get(pipeline._pipeline_key)
      if pipeline_record is not None:
        raise PipelineExistsError(
            'Pipeline with idempotence key "%s" already exists; params=%s' %
            (pipeline._pipeline_key.name(),
             _short_repr(pipeline_record.params)))

      entities_to_put = []
      for name, slot in pipeline.outputs._output_dict.iteritems():
        entities_to_put.append(_SlotRecord(
            key=slot.key,
            root_pipeline=pipeline._pipeline_key))

      entities_to_put.append(_PipelineRecord(
          key=pipeline._pipeline_key,
          root_pipeline=pipeline._pipeline_key,
          is_root_pipeline=True,
          # Bug in DB means we need to use the storage name here,
          # not the local property name.
          params=params_text,
          params_blob=params_blob,
          start_time=self._gettime(),
          class_path=pipeline._class_path,
          max_attempts=pipeline.max_attempts))

      entities_to_put.extend(_PipelineContext._create_barrier_entities(
          pipeline._pipeline_key,
          pipeline._pipeline_key,
          _BarrierRecord.FINALIZE,
          output_slots))

      db.put(entities_to_put)

      task = taskqueue.Task(
          url=self.pipeline_handler_path,
          params=dict(pipeline_key=pipeline._pipeline_key),
          headers={'X-Ae-Pipeline-Key': pipeline._pipeline_key},
          target=pipeline.target,
          countdown=countdown,
          eta=eta)
      if return_task:
        return task
      task.add(queue_name=self.queue_name, transactional=True)

    task = txn()
    # Immediately mark the output slots as existing so they can be filled
    # by asynchronous pipelines or used in test mode.
    for output_slot in pipeline.outputs._output_dict.itervalues():
      output_slot._exists = True
    return task

Response: Starts a new pipeline.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def evaluate(self, pipeline_key, purpose=None, attempt=0):
    After._thread_init()
    InOrder._thread_init()
    InOrder._local._activated = False

    if not isinstance(pipeline_key, db.Key):
      pipeline_key = db.Key(pipeline_key)
    pipeline_record = db.get(pipeline_key)
    if pipeline_record is None:
      logging.error('Pipeline ID "%s" does not exist.', pipeline_key.name())
      return
    if pipeline_record.status not in (
        _PipelineRecord.WAITING, _PipelineRecord.RUN):

      # If we're attempting to abort an already aborted pipeline,
      # we silently advance. #50
      if (pipeline_record.status == _PipelineRecord.ABORTED and
            purpose == _BarrierRecord.ABORT):
        return

      logging.error('Pipeline ID "%s" in bad state for purpose "%s": "%s"',
                    pipeline_key.name(), purpose or _BarrierRecord.START,
                    pipeline_record.status)
      return

    params = pipeline_record.params
    root_pipeline_key = \
        _PipelineRecord.root_pipeline.get_value_for_datastore(pipeline_record)
    default_slot_key = db.Key(params['output_slots']['default'])

    default_slot_record, root_pipeline_record = db.get([
        default_slot_key, root_pipeline_key])
    if default_slot_record is None:
      logging.error('Pipeline ID "%s" default slot "%s" does not exist.',
                    pipeline_key.name(), default_slot_key)
      return
    if root_pipeline_record is None:
      logging.error('Pipeline ID "%s" root pipeline ID "%s" is missing.',
                    pipeline_key.name(), root_pipeline_key.name())
      return

    # Always finalize if we're aborting so pipelines have a chance to cleanup
    # before they terminate. Pipelines must access 'was_aborted' to find
    # out how their finalization should work.
    abort_signal = (
        purpose == _BarrierRecord.ABORT or
        root_pipeline_record.abort_requested == True)
    finalize_signal = (
        (default_slot_record.status == _SlotRecord.FILLED and
         purpose == _BarrierRecord.FINALIZE) or abort_signal)

    try:
      pipeline_func_class = mr_util.for_name(pipeline_record.class_path)
    except ImportError, e:
      # This means something is wrong with the deployed code. Rely on the
      # taskqueue system to do retries.
      retry_message = '%s: %s' % (e.__class__.__name__, str(e))
      logging.exception(
          'Could not locate %s#%s. %s',
          pipeline_record.class_path, pipeline_key.name(), retry_message)
      raise

    try:
      pipeline_func = pipeline_func_class.from_id(
          pipeline_key.name(),
          resolve_outputs=finalize_signal,
          _pipeline_record=pipeline_record)
    except SlotNotFilledError, e:
      logging.exception(
          'Could not resolve arguments for %s#%s. Most likely this means there '
          'is a bug in the Pipeline runtime or some intermediate data has been '
          'deleted from the Datastore. Giving up.',
          pipeline_record.class_path, pipeline_key.name())
      self.transition_aborted(pipeline_key)
      return
    except Exception, e:
      retry_message = '%s: %s' % (e.__class__.__name__, str(e))
      logging.exception(
          'Instantiating %s#%s raised exception. %s',
          pipeline_record.class_path, pipeline_key.name(), retry_message)
      self.transition_retry(pipeline_key, retry_message)
      if pipeline_record.params['task_retry']:
        raise
      else:
        return
    else:
      pipeline_generator = mr_util.is_generator_function(
          pipeline_func_class.run)
      caller_output = pipeline_func.outputs

    if (abort_signal and pipeline_func.async and
        pipeline_record.status == _PipelineRecord.RUN
        and not pipeline_func.try_cancel()):
      logging.warning(
          'Could not cancel and abort mid-flight async pipeline: %r#%s',
          pipeline_func, pipeline_key.name())
      return

    if finalize_signal:
      try:
        pipeline_func._finalized_internal(
              self, pipeline_key, root_pipeline_key,
              caller_output, abort_signal)
      except Exception, e:
        # This means something is wrong with the deployed finalization code.
        # Rely on the taskqueue system to do retries.
        retry_message = '%s: %s' % (e.__class__.__name__, str(e))
        logging.exception('Finalizing %r#%s raised exception. %s',
                          pipeline_func, pipeline_key.name(), retry_message)
        raise
      else:
        if not abort_signal:
          self.transition_complete(pipeline_key)
          return

    if abort_signal:
      logging.debug('Marking as aborted %s#%s', pipeline_func,
                    pipeline_key.name())
      self.transition_aborted(pipeline_key)
      return

    if pipeline_record.current_attempt != attempt:
      logging.error(
          'Received evaluation task for pipeline ID "%s" attempt %d but '
          'current pending attempt is %d', pipeline_key.name(), attempt,
          pipeline_record.current_attempt)
      return

    if pipeline_record.current_attempt >= pipeline_record.max_attempts:
      logging.error(
          'Received evaluation task for pipeline ID "%s" on attempt %d '
          'but that exceeds max attempts %d', pipeline_key.name(), attempt,
          pipeline_record.max_attempts)
      return

    if pipeline_record.next_retry_time is not None:
      retry_time = pipeline_record.next_retry_time - _RETRY_WIGGLE_TIMEDELTA
      if self._gettime() <= retry_time:
        detail_message = (
            'Received evaluation task for pipeline ID "%s" on attempt %d, '
            'which will not be ready until: %s' % (pipeline_key.name(),
            pipeline_record.current_attempt, pipeline_record.next_retry_time))
        logging.warning(detail_message)
        raise UnexpectedPipelineError(detail_message)

    if pipeline_record.status == _PipelineRecord.RUN and pipeline_generator:
      if (default_slot_record.status == _SlotRecord.WAITING and
          not pipeline_record.fanned_out):
        # This properly handles the yield-less generator case when the
        # RUN state transition worked properly but outputting to the default
        # slot failed.
        self.fill_slot(pipeline_key, caller_output.default, None)
      return

    if (pipeline_record.status == _PipelineRecord.WAITING and
        pipeline_func.async):
      self.transition_run(pipeline_key)

    try:
      result = pipeline_func._run_internal(
          self, pipeline_key, root_pipeline_key, caller_output)
    except Exception, e:
      if self.handle_run_exception(pipeline_key, pipeline_func, e):
        raise
      else:
        return

    if pipeline_func.async:
      return

    if not pipeline_generator:
      # Catch any exceptions that are thrown when the pipeline's return
      # value is being serialized. This ensures that serialization errors
      # will cause normal abort/retry behavior.
      try:
        self.fill_slot(pipeline_key, caller_output.default, result)
      except Exception, e:
        retry_message = 'Bad return value. %s: %s' % (
            e.__class__.__name__, str(e))
        logging.exception(
            'Generator %r#%s caused exception while serializing return '
            'value %r. %s', pipeline_func, pipeline_key.name(), result,
            retry_message)
        self.transition_retry(pipeline_key, retry_message)
        if pipeline_func.task_retry:
          raise
        else:
          return

      expected_outputs = set(caller_output._output_dict.iterkeys())
      found_outputs = self.session_filled_output_names
      if expected_outputs != found_outputs:
        exception = SlotNotFilledError(
            'Outputs %r for pipeline ID "%s" were never filled by "%s".' % (
            expected_outputs - found_outputs,
            pipeline_key.name(), pipeline_func._class_path))
        if self.handle_run_exception(pipeline_key, pipeline_func, exception):
          raise exception
      return

    pipeline_iter = result
    next_value = None
    last_sub_stage = None
    sub_stage = None
    sub_stage_dict = {}
    sub_stage_ordering = []

    while True:
      try:
        yielded = pipeline_iter.send(next_value)
      except StopIteration:
        break
      except Exception, e:
        if self.handle_run_exception(pipeline_key, pipeline_func, e):
          raise
        else:
          return

      if isinstance(yielded, Pipeline):
        if yielded in sub_stage_dict:
          raise UnexpectedPipelineError(
              'Already yielded pipeline object %r with pipeline ID %s' %
              (yielded, yielded.pipeline_id))

        last_sub_stage = yielded
        next_value = PipelineFuture(yielded.output_names)
        next_value._after_all_pipelines.update(After._local._after_all_futures)
        next_value._after_all_pipelines.update(InOrder._local._in_order_futures)
        sub_stage_dict[yielded] = next_value
        sub_stage_ordering.append(yielded)
        InOrder._add_future(next_value)

        # To aid local testing, the task_retry flag (which instructs the
        # evaluator to raise all exceptions back up to the task queue) is
        # inherited by all children from the root down.
        yielded.task_retry = pipeline_func.task_retry
      else:
        raise UnexpectedPipelineError(
            'Yielded a disallowed value: %r' % yielded)

    if last_sub_stage:
      # Final yielded stage inherits outputs from calling pipeline that were not
      # already filled during the generator's execution.
      inherited_outputs = params['output_slots']
      for slot_name in self.session_filled_output_names:
        del inherited_outputs[slot_name]
      sub_stage_dict[last_sub_stage]._inherit_outputs(
          pipeline_record.class_path, inherited_outputs)
    else:
      # Here the generator has yielded nothing, and thus acts as a synchronous
      # function. We can skip the rest of the generator steps completely and
      # fill the default output slot to cause finalizing.
      expected_outputs = set(caller_output._output_dict.iterkeys())
      expected_outputs.remove('default')
      found_outputs = self.session_filled_output_names
      if expected_outputs != found_outputs:
        exception = SlotNotFilledError(
            'Outputs %r for pipeline ID "%s" were never filled by "%s".' % (
            expected_outputs - found_outputs,
            pipeline_key.name(), pipeline_func._class_path))
        if self.handle_run_exception(pipeline_key, pipeline_func, exception):
          raise exception
      else:
        self.fill_slot(pipeline_key, caller_output.default, None)
        self.transition_run(pipeline_key)
      return

    # Allocate any SlotRecords that do not yet exist.
    entities_to_put = []
    for future in sub_stage_dict.itervalues():
      for slot in future._output_dict.itervalues():
        if not slot._exists:
          entities_to_put.append(_SlotRecord(
              key=slot.key, root_pipeline=root_pipeline_key))

    # Allocate PipelineRecords and BarrierRecords for generator-run Pipelines.
    pipelines_to_run = set()
    all_children_keys = []
    all_output_slots = set()
    for sub_stage in sub_stage_ordering:
      future = sub_stage_dict[sub_stage]

      # Catch any exceptions that are thrown when the pipeline's parameters
      # are being serialized. This ensures that serialization errors will
      # cause normal retry/abort behavior.
      try:
        dependent_slots, output_slots, params_text, params_blob = \
            _generate_args(sub_stage, future, self.queue_name, self.base_path)
      except Exception, e:
        retry_message = 'Bad child arguments. %s: %s' % (
            e.__class__.__name__, str(e))
        logging.exception(
            'Generator %r#%s caused exception while serializing args for '
            'child pipeline %r. %s', pipeline_func, pipeline_key.name(),
            sub_stage, retry_message)
        self.transition_retry(pipeline_key, retry_message)
        if pipeline_func.task_retry:
          raise
        else:
          return

      child_pipeline_key = db.Key.from_path(
          _PipelineRecord.kind(), uuid.uuid4().hex)
      all_output_slots.update(output_slots)
      all_children_keys.append(child_pipeline_key)

      child_pipeline = _PipelineRecord(
          key=child_pipeline_key,
          root_pipeline=root_pipeline_key,
          # Bug in DB means we need to use the storage name here,
          # not the local property name.
          params=params_text,
          params_blob=params_blob,
          class_path=sub_stage._class_path,
          max_attempts=sub_stage.max_attempts)
      entities_to_put.append(child_pipeline)

      if not dependent_slots:
        # This child pipeline will run immediately.
        pipelines_to_run.add(child_pipeline_key)
        child_pipeline.start_time = self._gettime()
      else:
        entities_to_put.extend(_PipelineContext._create_barrier_entities(
            root_pipeline_key,
            child_pipeline_key,
            _BarrierRecord.START,
            dependent_slots))

      entities_to_put.extend(_PipelineContext._create_barrier_entities(
          root_pipeline_key,
          child_pipeline_key,
          _BarrierRecord.FINALIZE,
          output_slots))

    # This generator pipeline's finalization barrier must include all of the
    # outputs of any child pipelines that it runs. This ensures the finalized
    # calls will not happen until all child pipelines have completed.
    #
    # The transition_run() call below will update the FINALIZE _BarrierRecord
    # for this generator pipeline to include all of these child outputs in
    # its list of blocking_slots. That update is done transactionally to
    # make sure the _BarrierRecord only lists the slots that matter.
    #
    # However, the notify_barriers() method doesn't find _BarrierRecords
    # through the blocking_slots field. It finds them through _BarrierIndexes
    # entities. Thus, before we update the FINALIZE _BarrierRecord in
    # transition_run(), we need to write _BarrierIndexes for all child outputs.
    barrier_entities = _PipelineContext._create_barrier_entities(
        root_pipeline_key,
        pipeline_key,
        _BarrierRecord.FINALIZE,
        all_output_slots)
    # Ignore the first element which is the _BarrierRecord. That entity must
    # have already been created and put in the datastore for the parent
    # pipeline before this code generated child pipelines.
    barrier_indexes = barrier_entities[1:]
    entities_to_put.extend(barrier_indexes)

    db.put(entities_to_put)

    self.transition_run(pipeline_key,
                        blocking_slot_keys=all_output_slots,
                        fanned_out_pipelines=all_children_keys,
                        pipelines_to_run=pipelines_to_run)

Response: Evaluates the given Pipeline and enqueues sub - stages for execution.


Instruction:
Here you have a function in Python 3, explain what it does
def _create_barrier_entities(root_pipeline_key,
                               child_pipeline_key,
                               purpose,
                               blocking_slot_keys):
    """Creates all of the entities required for a _BarrierRecord.

    Args:
      root_pipeline_key: The root pipeline this is part of.
      child_pipeline_key: The pipeline this barrier is for.
      purpose: _BarrierRecord.START or _BarrierRecord.FINALIZE.
      blocking_slot_keys: Set of db.Keys corresponding to _SlotRecords that
        this barrier should wait on before firing.

    Returns:
      List of entities, starting with the _BarrierRecord entity, followed by
      _BarrierIndexes used for firing when _SlotRecords are filled in the same
      order as the blocking_slot_keys list provided. All of these entities
      should be put in the Datastore to ensure the barrier fires properly.
    """
    result = []

    blocking_slot_keys = list(blocking_slot_keys)

    barrier = _BarrierRecord(
        parent=child_pipeline_key,
        key_name=purpose,
        target=child_pipeline_key,
        root_pipeline=root_pipeline_key,
        blocking_slots=blocking_slot_keys)

    result.append(barrier)

    for slot_key in blocking_slot_keys:
      barrier_index_path = []
      barrier_index_path.extend(slot_key.to_path())
      barrier_index_path.extend(child_pipeline_key.to_path())
      barrier_index_path.extend([_BarrierIndex.kind(), purpose])
      barrier_index_key = db.Key.from_path(*barrier_index_path)
      barrier_index = _BarrierIndex(
          key=barrier_index_key,
          root_pipeline=root_pipeline_key)
      result.append(barrier_index)

    return result

Response: Creates all of the entities required for a _BarrierRecord.


Instruction:
How would you code a function in Python 3 to
handle an exception raised by a Pipeline s user code.

Response:
def handle_run_exception(self, pipeline_key, pipeline_func, e):
    """Handles an exception raised by a Pipeline's user code.

    Args:
      pipeline_key: The pipeline that raised the error.
      pipeline_func: The class path name of the Pipeline that was running.
      e: The exception that was raised.

    Returns:
      True if the exception should be re-raised up through the calling stack
      by the caller of this method.
    """
    if isinstance(e, Retry):
      retry_message = str(e)
      logging.warning('User forced retry for pipeline ID "%s" of %r: %s',
                      pipeline_key.name(), pipeline_func, retry_message)
      self.transition_retry(pipeline_key, retry_message)
    elif isinstance(e, Abort):
      abort_message = str(e)
      logging.warning('User forced abort for pipeline ID "%s" of %r: %s',
                      pipeline_key.name(), pipeline_func, abort_message)
      pipeline_func.abort(abort_message)
    else:
      retry_message = '%s: %s' % (e.__class__.__name__, str(e))
      logging.exception('Generator %r#%s raised exception. %s',
                        pipeline_func, pipeline_key.name(), retry_message)
      self.transition_retry(pipeline_key, retry_message)

    return pipeline_func.task_retry


Instruction:
Can you tell what is the following Python 3 function doing
def transition_run(self,
                     pipeline_key,
                     blocking_slot_keys=None,
                     fanned_out_pipelines=None,
                     pipelines_to_run=None):
    """Marks an asynchronous or generator pipeline as running.

    Does nothing if the pipeline is no longer in a runnable state.

    Args:
      pipeline_key: The db.Key of the _PipelineRecord to update.
      blocking_slot_keys: List of db.Key instances that this pipeline's
        finalization barrier should wait on in addition to the existing one.
        This is used to update the barrier to include all child outputs. When
        None, the barrier will not be updated.
      fanned_out_pipelines: List of db.Key instances of _PipelineRecords that
        were fanned out by this generator pipeline. This is distinct from the
        'pipelines_to_run' list because not all of the pipelines listed here
        will be immediately ready to execute. When None, then this generator
        yielded no children.
      pipelines_to_run: List of db.Key instances of _PipelineRecords that should
        be kicked off (fan-out) transactionally as part of this transition.
        When None, no child pipelines will run. All db.Keys in this list must
        also be present in the fanned_out_pipelines list.

    Raises:
      UnexpectedPipelineError if blocking_slot_keys was not empty and the
      _BarrierRecord has gone missing.
    """
    def txn():
      pipeline_record = db.get(pipeline_key)
      if pipeline_record is None:
        logging.warning('Pipeline ID "%s" cannot be marked as run. '
                        'Does not exist.', pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status != _PipelineRecord.WAITING:
        logging.warning('Pipeline ID "%s" in bad state to be marked as run: %s',
                        pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()

      pipeline_record.status = _PipelineRecord.RUN

      if fanned_out_pipelines:
        # NOTE: We must model the pipeline relationship in a top-down manner,
        # meaning each pipeline must point forward to the pipelines that it
        # fanned out to. The reason is race conditions. If evaluate()
        # dies early, it may create many unused _PipelineRecord and _SlotRecord
        # instances that never progress. The only way we know which of these
        # are valid is by traversing the graph from the root, where the
        # fanned_out property refers to those pipelines that were run using a
        # transactional task.
        child_pipeline_list = list(fanned_out_pipelines)
        pipeline_record.fanned_out = child_pipeline_list

        if pipelines_to_run:
          child_indexes = [
              child_pipeline_list.index(p) for p in pipelines_to_run]
          child_indexes.sort()
          task = taskqueue.Task(
              url=self.fanout_handler_path,
              params=dict(parent_key=str(pipeline_key),
                          child_indexes=child_indexes))
          task.add(queue_name=self.queue_name, transactional=True)

      pipeline_record.put()

      if blocking_slot_keys:
        # NOTE: Always update a generator pipeline's finalization barrier to
        # include all of the outputs of any pipelines that it runs, to ensure
        # that finalized calls will not happen until all child pipelines have
        # completed. This must happen transactionally with the enqueue of
        # the fan-out kickoff task above to ensure the child output slots and
        # the barrier blocking slots are the same.
        barrier_key = db.Key.from_path(
            _BarrierRecord.kind(), _BarrierRecord.FINALIZE,
            parent=pipeline_key)
        finalize_barrier = db.get(barrier_key)
        if finalize_barrier is None:
          raise UnexpectedPipelineError(
              'Pipeline ID "%s" cannot update finalize barrier. '
              'Does not exist.' % pipeline_key.name())
        else:
          finalize_barrier.blocking_slots = list(
              blocking_slot_keys.union(set(finalize_barrier.blocking_slots)))
          finalize_barrier.put()

    db.run_in_transaction(txn)

Response: Transition a generator pipeline to run.


Instruction:
How would you implement a function in Python 3 that
marks the given pipeline as complete.

Response:
def transition_complete(self, pipeline_key):
    """Marks the given pipeline as complete.

    Does nothing if the pipeline is no longer in a state that can be completed.

    Args:
      pipeline_key: db.Key of the _PipelineRecord that has completed.
    """
    def txn():
      pipeline_record = db.get(pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to mark pipeline ID "%s" as complete but it does not exist.',
            pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status not in (
          _PipelineRecord.WAITING, _PipelineRecord.RUN):
        logging.warning(
            'Tried to mark pipeline ID "%s" as complete, found bad state: %s',
            pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()

      pipeline_record.status = _PipelineRecord.DONE
      pipeline_record.finalized_time = self._gettime()
      pipeline_record.put()

    db.run_in_transaction(txn)


Instruction:
Given the following Python 3 function, write the documentation
def transition_retry(self, pipeline_key, retry_message):
    def txn():
      pipeline_record = db.get(pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to retry pipeline ID "%s" but it does not exist.',
            pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status not in (
          _PipelineRecord.WAITING, _PipelineRecord.RUN):
        logging.warning(
            'Tried to retry pipeline ID "%s", found bad state: %s',
            pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()

      params = pipeline_record.params
      offset_seconds = (
          params['backoff_seconds'] *
          (params['backoff_factor'] ** pipeline_record.current_attempt))
      pipeline_record.next_retry_time = (
          self._gettime() + datetime.timedelta(seconds=offset_seconds))
      pipeline_record.current_attempt += 1
      pipeline_record.retry_message = retry_message
      pipeline_record.status = _PipelineRecord.WAITING

      if pipeline_record.current_attempt >= pipeline_record.max_attempts:
        root_pipeline_key = (
            _PipelineRecord.root_pipeline.get_value_for_datastore(
                pipeline_record))
        logging.warning(
            'Giving up on pipeline ID "%s" after %d attempt(s); causing abort '
            'all the way to the root pipeline ID "%s"', pipeline_key.name(),
            pipeline_record.current_attempt, root_pipeline_key.name())
        # NOTE: We do *not* set the status to aborted here to ensure that
        # this pipeline will be finalized before it has been marked as aborted.
        pipeline_record.abort_message = (
            'Aborting after %d attempts' % pipeline_record.current_attempt)
        task = taskqueue.Task(
            url=self.fanout_abort_handler_path,
            params=dict(root_pipeline_key=root_pipeline_key))
        task.add(queue_name=self.queue_name, transactional=True)
      else:
        task = taskqueue.Task(
            url=self.pipeline_handler_path,
            eta=pipeline_record.next_retry_time,
            params=dict(pipeline_key=pipeline_key,
                        purpose=_BarrierRecord.START,
                        attempt=pipeline_record.current_attempt),
            headers={'X-Ae-Pipeline-Key': pipeline_key},
            target=pipeline_record.params['target'])
        task.add(queue_name=self.queue_name, transactional=True)

      pipeline_record.put()

    db.run_in_transaction(txn)

Response: Transition the given pipeline to requiring another retry.


Instruction:
How would you code a function in Python 3 to
run the callback for the pipeline specified in the request.

Response:
def run_callback(self):
    """Runs the callback for the pipeline specified in the request.

    Raises:
      _CallbackTaskError if something was wrong with the request parameters.
    """
    pipeline_id = self.request.get('pipeline_id')
    if not pipeline_id:
      raise _CallbackTaskError('"pipeline_id" parameter missing.')

    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), pipeline_id)
    pipeline_record = db.get(pipeline_key)
    if pipeline_record is None:
      raise _CallbackTaskError(
          'Pipeline ID "%s" for callback does not exist.' % pipeline_id)

    params = pipeline_record.params
    real_class_path = params['class_path']
    try:
      pipeline_func_class = mr_util.for_name(real_class_path)
    except ImportError, e:
      raise _CallbackTaskError(
          'Cannot load class named "%s" for pipeline ID "%s".'
          % (real_class_path, pipeline_id))

    if 'HTTP_X_APPENGINE_TASKNAME' not in self.request.environ:
      if pipeline_func_class.public_callbacks:
        pass
      elif pipeline_func_class.admin_callbacks:
        if not users.is_current_user_admin():
          raise _CallbackTaskError(
              'Unauthorized callback for admin-only pipeline ID "%s"'
              % pipeline_id)
      else:
        raise _CallbackTaskError(
            'External callback for internal-only pipeline ID "%s"'
            % pipeline_id)

    kwargs = {}
    for key in self.request.arguments():
      if key != 'pipeline_id':
        kwargs[str(key)] = self.request.get(key)

    def perform_callback():
      stage = pipeline_func_class.from_id(pipeline_id)
      if stage is None:
        raise _CallbackTaskError(
            'Pipeline ID "%s" deleted during callback' % pipeline_id)
      return stage._callback_internal(kwargs)

    # callback_xg_transaction is a 3-valued setting (None=no trans,
    # False=1-eg-trans, True=xg-trans)
    if pipeline_func_class._callback_xg_transaction is not None:
      transaction_options = db.create_transaction_options(
          xg=pipeline_func_class._callback_xg_transaction)
      callback_result = db.run_in_transaction_options(transaction_options,
                                                      perform_callback)
    else:
      callback_result = perform_callback()

    if callback_result is not None:
      status_code, content_type, content = callback_result
      self.response.set_status(status_code)
      self.response.headers['Content-Type'] = content_type
      self.response.out.write(content)


Instruction:
Implement a function in Python 3 to
find the google_appengine directory and fixes Python imports to use it.

Response:
def _fix_path():
  """Finds the google_appengine directory and fixes Python imports to use it."""
  import os
  import sys
  all_paths = os.environ.get('PYTHONPATH').split(os.pathsep)
  for path_dir in all_paths:
    dev_appserver_path = os.path.join(path_dir, 'dev_appserver.py')
    if os.path.exists(dev_appserver_path):
      logging.debug('Found appengine SDK on path!')
      google_appengine = os.path.dirname(os.path.realpath(dev_appserver_path))
      sys.path.append(google_appengine)
      # Use the next import will fix up sys.path even further to bring in
      # any dependent lib directories that the SDK needs.
      dev_appserver = __import__('dev_appserver')
      sys.path.extend(dev_appserver.EXTRA_PATHS)
      return


Instruction:
Can you generate the documentation for the following Python 3 function
def params(self):
    if hasattr(self, '_params_decoded'):
      return self._params_decoded

    if self.params_blob is not None:
      value_encoded = self.params_blob.open().read()
    else:
      value_encoded = self.params_text

    value = json.loads(value_encoded, cls=util.JsonDecoder)
    if isinstance(value, dict):
      kwargs = value.get('kwargs')
      if kwargs:
        adjusted_kwargs = {}
        for arg_key, arg_value in kwargs.iteritems():
          # Python only allows non-unicode strings as keyword arguments.
          adjusted_kwargs[str(arg_key)] = arg_value
        value['kwargs'] = adjusted_kwargs

    self._params_decoded = value
    return self._params_decoded

Response: Returns the dictionary of parameters for this Pipeline.


Instruction:
How would you code a function in Python 3 to
return the value of this Slot.

Response:
def value(self):
    """Returns the value of this Slot."""
    if hasattr(self, '_value_decoded'):
      return self._value_decoded

    if self.value_blob is not None:
      encoded_value = self.value_blob.open().read()
    else:
      encoded_value = self.value_text

    self._value_decoded = json.loads(encoded_value, cls=util.JsonDecoder)
    return self._value_decoded


Instruction:
Here you have a function in Python 3, explain what it does
def to_barrier_key(cls, barrier_index_key):
    barrier_index_path = barrier_index_key.to_path()

    # Pick out the items from the _BarrierIndex key path that we need to
    # construct the _BarrierRecord key path.
    (pipeline_kind, dependent_pipeline_id,
     unused_kind, purpose) = barrier_index_path[-4:]

    barrier_record_path = (
        pipeline_kind, dependent_pipeline_id,
        _BarrierRecord.kind(), purpose)

    return db.Key.from_path(*barrier_record_path)

Response: Converts a _BarrierIndex key to a _BarrierRecord key.


Instruction:
How would you explain what the following Python 3 function does
def partial_page(self, page, used_labels):
        # Check the page number is valid.
        if page <= self.page_count:
            raise ValueError("Page {0:d} has already started, cannot mark used labels now.".format(page))

        # Add these to any existing labels marked as used.
        used = self._used.get(page, set())
        for row, column in used_labels:
            # Check the index is valid.
            if row < 1 or row > self.specs.rows:
                raise IndexError("Invalid row number: {0:d}.".format(row))
            if column < 1 or column > self.specs.columns:
                raise IndexError("Invalid column number: {0:d}.".format(column))

            # Add it.
            used.add((int(row), int(column)))

        # Save the details.
        self._used[page] = used

Response: Allows a page to be marked as already partially used so you can print on the remaining labels.


Instruction:
Make a summary of the following Python 3 code
def _next_unused_label(self):
        self._next_label()

        # This label may be missing.
        if self.page_count in self._used:
            # Keep try while the label is missing.
            missing = self._used.get(self.page_count, set())
            while tuple(self._position) in missing:
                # Throw the missing information away now we have used it. This
                # allows the _shade_remaining_missing method to work.
                missing.discard(tuple(self._position))

                # Shade the missing label if desired.
                if self.shade_missing:
                    self._shade_missing_label()

                # Try our luck with the next label.
                self._next_label()
                missing = self._used.get(self.page_count, set())

        # Increment the count now we have found a suitable position.
        self.label_count += 1

Response: This method moves to the next unused label. Not intended for external use.


Instruction:
Can you tell what is the following Python 3 function doing
def _calculate_edges(self):
        # Calculate the left edge of the label.
        left = self.specs.left_margin
        left += (self.specs.label_width * (self._position[1] - 1))
        if self.specs.column_gap:
            left += (self.specs.column_gap * (self._position[1] - 1))
        left *= mm

        # And the bottom.
        bottom = self.specs.sheet_height - self.specs.top_margin
        bottom -= (self.specs.label_height * self._position[0])
        if self.specs.row_gap:
            bottom -= (self.specs.row_gap * (self._position[0] - 1))
        bottom *= mm

        # Done.
        return float(left), float(bottom)

Response: Calculate the edges of the current label. Not intended for external use.


Instruction:
How would you explain what the following Python 3 function does
def _shade_remaining_missing(self):
        # Sanity check.
        if not self.shade_missing:
            return

        # Run through each missing label left in the current page and shade it.
        missing = self._used.get(self.page_count, set())
        for position in missing:
            self._position = position
            self._shade_missing_label()

Response:
Helper method to shade any missing labels left on the current
        page.


Instruction:
Given the following Python 3 function, write the documentation
def _draw_label(self, obj, count):
        # Start a drawing for the whole label.
        label = Drawing(float(self._lw), float(self._lh))
        label.add(self._clip_label)

        # And one for the available area (i.e., after padding).
        available = Drawing(float(self._dw), float(self._dh))
        available.add(self._clip_drawing)

        # Call the drawing function.
        self.drawing_callable(available, float(self._dw), float(self._dh), obj)

        # Render the contents on the label.
        available.shift(float(self._lp), float(self._bp))
        label.add(available)

        # Draw the border if requested.
        if self.border:
            label.add(self._border)

        # Add however many copies we need to.
        for i in range(count):
            # Find the next available label.
            self._next_unused_label()

            # Have we been told to skip this page?
            if self.pages_to_draw and self.page_count not in self.pages_to_draw:
                continue

            # Add the label to the page. ReportLab stores the added drawing by
            # reference so we have to copy it N times.
            thislabel = copy(label)
            thislabel.shift(*self._calculate_edges())
            self._current_page.add(thislabel)

Response: Draw the label on the current page.


Instruction:
Here you have a function in Python 3, explain what it does
def add_labels(self, objects, count=1):
        # If we can convert it to an int, do so and use the itertools.repeat()
        # method to create an infinite iterator from it. Otherwise, assume it
        # is an iterable or sequence.
        try:
            count = int(count)
        except TypeError:
            pass
        else:
            count = repeat(count)

        # If it is not an iterable (e.g., a list or range object),
        # create an iterator over it.
        if not hasattr(count, 'next') and not hasattr(count, '__next__'):
            count = iter(count)

        # Go through the objects.
        for obj in objects:
            # Check we have a count for this one.
            try:
                thiscount = next(count)
            except StopIteration:
                break

            # Draw it.
            self._draw_label(obj, thiscount)

Response: Adds multiple labels to the sheet.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def save(self, filelike):
        # Shade any remaining missing labels if desired.
        self._shade_remaining_missing()

        # Create a canvas.
        canvas = Canvas(filelike, pagesize=self._pagesize)

        # Render each created page onto the canvas.
        for page in self._pages:
            renderPDF.draw(page, canvas, 0, 0)
            canvas.showPage()

        # Done.
        canvas.save()

Response: Save the file as a PDF.


Instruction:
Make a summary of the following Python 3 code
def preview(self, page, filelike, format='png', dpi=72, background_colour=0xFFFFFF):
        # Check the page number.
        if page < 1 or page > self.page_count:
            raise ValueError("Invalid page number; should be between 1 and {0:d}.".format(self.page_count))

        # Shade any remaining missing labels if desired.
        self._shade_remaining_missing()

        # Rendering to an image (as opposed to a PDF) requires any background
        # to have an integer width and height if it is a ReportLab Image
        # object. Drawing objects are exempt from this.
        oldw, oldh = None, None
        if isinstance(self._bgimage, Image):
            oldw, oldh = self._bgimage.width, self._bgimage.height
            self._bgimage.width = int(oldw) + 1
            self._bgimage.height = int(oldh) + 1

        # Let ReportLab do the heavy lifting.
        renderPM.drawToFile(self._pages[page-1], filelike, format, dpi, background_colour)

        # Restore the size of the background image if we changed it.
        if oldw:
            self._bgimage.width = oldw
            self._bgimage.height = oldh

Response: Render a preview image of a page.


Instruction:
Implement a function in Python 3 to
render a preview image of a page as a string.

Response:
def preview_string(self, page, format='png', dpi=72, background_colour=0xFFFFFF):
        """Render a preview image of a page as a string.

        Parameters
        ----------
        page: positive integer
            Which page to render. Must be in the range [1, page_count]
        format: string
            The image format to use for the preview. ReportLab uses the Python
            Imaging Library (PIL) internally, so any PIL format should be
            supported.
        dpi: positive real
            The dots-per-inch to use when rendering.
        background_colour: Hex colour specification
            What color background to use.

        Notes
        -----
        If you are creating this sheet for a preview only, you can pass the
        pages_to_draw parameter to the constructor to avoid the drawing function
        being called for all the labels on pages you'll never look at. If you
        preview a page you did not tell the sheet to draw, you will get a blank
        image.

        Raises
        ------
        ValueError:
            If the page number is not valid.

        """
        # Check the page number.
        if page < 1 or page > self.page_count:
            raise ValueError("Invalid page number; should be between 1 and {0:d}.".format(self.page_count))

        # Shade any remaining missing labels if desired.
        self._shade_remaining_missing()

        # Rendering to an image (as opposed to a PDF) requires any background
        # to have an integer width and height if it is a ReportLab Image
        # object. Drawing objects are exempt from this.
        oldw, oldh = None, None
        if isinstance(self._bgimage, Image):
            oldw, oldh = self._bgimage.width, self._bgimage.height
            self._bgimage.width = int(oldw) + 1
            self._bgimage.height = int(oldh) + 1

        # Let ReportLab do the heavy lifting.
        s = renderPM.drawToString(self._pages[page-1], format, dpi, background_colour)

        # Restore the size of the background image if we changed it.
        if oldw:
            self._bgimage.width = oldw
            self._bgimage.height = oldh

        # Done.
        return s


Instruction:
Write a Python 3 script to
calculate the size of the current object and returns the size of the object.

Response:
def _calculate(self):
        """Checks the dimensions of the sheet are valid and consistent.

        NB: this is called internally when needed; there should be no need for
        user code to call it.

        """
        # Check the dimensions are larger than zero.
        for dimension in ('_sheet_width', '_sheet_height', '_columns', '_rows', '_label_width', '_label_height'):
            if getattr(self, dimension) <= 0:
                name = dimension.replace('_', ' ').strip().capitalize()
                raise InvalidDimension("{0:s} must be greater than zero.".format(name))

        # Check margins / gaps are not smaller than zero if given.
        # At the same time, force the values to decimals.
        for margin in ('_left_margin', '_column_gap', '_right_margin', '_top_margin', '_row_gap', '_bottom_margin',
                       '_left_padding', '_right_padding', '_top_padding', '_bottom_padding'):
            val = getattr(self, margin)
            if val is not None:
                if margin in self._autoset:
                    val = None
                else:
                    val = Decimal(val)
                    if val < 0:
                        name = margin.replace('_', ' ').strip().capitalize()
                        raise InvalidDimension("{0:s} cannot be less than zero.".format(name))
                setattr(self, margin, val)
            else:
                self._autoset.add(margin)

        # Check the corner radius.
        if self._corner_radius < 0:
            raise InvalidDimension("Corner radius cannot be less than zero.")
        if self._corner_radius > (self._label_width / 2):
            raise InvalidDimension("Corner radius cannot be more than half the label width.")
        if self._corner_radius > (self._label_height / 2):
            raise InvalidDimension("Corner radius cannot be more than half the label height.")

        # If there is no padding, we don't need the padding radius.
        if (self._left_padding + self._right_padding + self._top_padding + self._bottom_padding) == 0:
            if self._padding_radius != 0:
                raise InvalidDimension("Padding radius must be zero if there is no padding.")
        else:
            if (self._left_padding + self._right_padding) >= self._label_width:
                raise InvalidDimension("Sum of horizontal padding must be less than the label width.")
            if (self._top_padding + self._bottom_padding) >= self._label_height:
                raise InvalidDimension("Sum of vertical padding must be less than the label height.")
            if self._padding_radius < 0:
                raise InvalidDimension("Padding radius cannot be less than zero.")

        # Calculate the amount of spare space.
        hspace = self._sheet_width - (self._label_width * self._columns)
        vspace = self._sheet_height - (self._label_height * self._rows)

        # Cannot fit.
        if hspace < 0:
            raise InvalidDimension("Labels are too wide to fit on the sheet.")
        if vspace < 0:
            raise InvalidDimension("Labels are too tall to fit on the sheet.")

        # Process the horizontal margins / gaps.
        hcount = 1 + self._columns
        if self._left_margin is not None:
            hspace -= self._left_margin
            if hspace < 0:
                raise InvalidDimension("Left margin is too wide for the labels to fit on the sheet.")
            hcount -= 1
        if self._column_gap is not None:
            hspace -= ((self._columns - 1) * self._column_gap)
            if hspace < 0:
                raise InvalidDimension("Column gap is too wide for the labels to fit on the sheet.")
            hcount -= (self._columns - 1)
        if self._right_margin is not None:
            hspace -= self._right_margin
            if hspace < 0.01 and hspace > -0.01:
                self._right_margin += hspace
                hspace = 0
            if hspace < 0:
                raise InvalidDimension("Right margin is too wide for the labels to fit on the sheet.")
            hcount -= 1

        # Process the vertical margins / gaps.
        vcount = 1 + self._rows
        if self._top_margin is not None:
            vspace -= self._top_margin
            if vspace < 0:
                raise InvalidDimension("Top margin is too tall for the labels to fit on the sheet.")
            vcount -= 1
        if self._row_gap is not None:
            vspace -= ((self._rows - 1) * self._row_gap)
            if vspace < 0:
                raise InvalidDimension("Row gap is too tall for the labels to fit on the sheet.")
            vcount -= (self._rows - 1)
        if self._bottom_margin is not None:
            vspace -= self._bottom_margin
            if vspace < 0.01 and vspace > -0.01:
                self._bottom_margin += vspace
                vspace = 0
            if vspace < 0:
                raise InvalidDimension("Bottom margin is too tall for the labels to fit on the sheet.")
            vcount -= 1

        # If all the margins are specified, they must use up all available space.
        if hcount == 0 and hspace != 0:
            raise InvalidDimension("Not all width used by manually specified margins/gaps; {}mm left.".format(hspace))
        if vcount == 0 and vspace != 0:
            raise InvalidDimension("Not all height used by manually specified margins/gaps; {}mm left.".format(vspace))

        # Split any extra horizontal space and allocate it.
        if hcount:
            auto_margin = hspace / hcount
            for margin in ('_left_margin', '_column_gap', '_right_margin'):
                if getattr(self, margin) is None:
                    setattr(self, margin, auto_margin)

        # And allocate any extra vertical space.
        if vcount:
            auto_margin = vspace / vcount
            for margin in ('_top_margin', '_row_gap', '_bottom_margin'):
                if getattr(self, margin) is None:
                    setattr(self, margin, auto_margin)


Instruction:
Write a Python 3 script to
return the bounding boxes of the labels on a page.

Response:
def bounding_boxes(self, mode='fraction', output='dict'):
        """Get the bounding boxes of the labels on a page.

        Parameters
        ----------
        mode: 'fraction', 'actual'
            If 'fraction', the bounding boxes are expressed as a fraction of the
            height and width of the sheet. If 'actual', they are the actual
            position of the labels in millimetres from the top-left of the
            sheet.
        output: 'dict', 'json'
            If 'dict', a dictionary with label identifier tuples (row, column)
            as keys and a dictionary with 'left', 'right', 'top', and 'bottom'
            entries as the values.
            If 'json', a JSON encoded string which represents a dictionary with
            keys of the string format 'rowxcolumn' and each value being a
            bounding box dictionary with 'left', 'right', 'top', and 'bottom'
            entries.

        Returns
        -------
        The bounding boxes in the format set by the output parameter.

        """
        boxes = {}

        # Check the parameters.
        if mode not in ('fraction', 'actual'):
            raise ValueError("Unknown mode {0}.".format(mode))
        if output not in ('dict', 'json'):
            raise ValueError("Unknown output {0}.".format(output))

        # Iterate over the rows.
        for row in range(1, self.rows + 1):
            # Top and bottom of all labels in the row.
            top = self.top_margin + ((row - 1) * (self.label_height + self.row_gap))
            bottom = top + self.label_height

            # Now iterate over all columns in this row.
            for column in range(1, self.columns + 1):
                # Left and right position of this column.
                left = self.left_margin + ((column - 1) * (self.label_width + self.column_gap))
                right = left + self.label_width

                # Output in the appropriate mode format.
                if mode == 'fraction':
                    box = {
                        'top': top / self.sheet_height,
                        'bottom': bottom / self.sheet_height,
                        'left': left / self.sheet_width,
                        'right': right / self.sheet_width,
                    }
                elif mode == 'actual':
                    box = {'top': top, 'bottom': bottom, 'left': left, 'right': right}

                # Add to the collection.
                if output == 'json':
                    boxes['{0:d}x{1:d}'.format(row, column)] = box
                    box['top'] = float(box['top'])
                    box['bottom'] = float(box['bottom'])
                    box['left'] = float(box['left'])
                    box['right'] = float(box['right'])
                else:
                    boxes[(row, column)] = box

        # Done.
        if output == 'json':
            return json.dumps(boxes)
        return boxes


Instruction:
Here you have a function in Python 3, explain what it does
def templatesCollector(text, open, close):
    others = []
    spans = [i for i in findBalanced(text, open, close)]
    spanscopy = copy(spans)
    for i in range(len(spans)):
        start, end = spans[i]

        o = text[start:end]
        ol = o.lower()
        if 'vaata|' in ol or 'wikitable' in ol:
            spanscopy.remove(spans[i])
            continue
        others.append(o)
    text = dropSpans(spanscopy, text)

    return text, others

Response: returns the text and the other templates that are in place


Instruction:
Here you have a function in Python 3, explain what it does
def assert_legal_arguments(kwargs):
    seen_layers = set()
    for k, v in kwargs.items():
        if k not in LEGAL_ARGUMENTS:
            raise ValueError('Illegal argument <{0}>!'.format(k))
        if k in AESTHETICS:
            if v in seen_layers:
                raise ValueError('Layer <{0}> mapped for more than a single aesthetic!'.format(v))
            seen_layers.add(v)
        if k in VALUES:
            if not isinstance(v, six.string_types) and not isinstance(v, list):
                raise ValueError('Value <{0}> must be either string or list'.format(k))
            if isinstance(v, list):
                if len(v) == 0:
                    raise ValueError('Rules cannot be empty list')
                for rule_matcher, rule_value in v:
                    if not isinstance(rule_matcher, six.string_types) or not isinstance(rule_value, six.string_types):
                        raise ValueError('Rule tuple elements must be strings')

Response: Assert that PrettyPrinter arguments are correct.


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_arguments(kwargs):
    aesthetics = {}
    values = {}
    for aes in AESTHETICS:
        if aes in kwargs:
            aesthetics[aes] = kwargs[aes]
            val_name = AES_VALUE_MAP[aes]
            # map the user-provided CSS value or use the default
            values[aes] = kwargs.get(val_name, DEFAULT_VALUE_MAP[aes])
    return aesthetics, values

Response: Function that parses the keyword arguments to get aesthetics and values from the PrettyPrinter output.


Instruction:
How would you implement a function in Python 3 that
returns ------- str The CSS.

Response:
def css(self):
        """Returns
        -------
        str
            The CSS.
        """
        css_list = [DEFAULT_MARK_CSS]
        for aes in self.aesthetics:
            css_list.extend(get_mark_css(aes, self.values[aes]))
        #print('\n'.join(css_list))
        return '\n'.join(css_list)


Instruction:
Write a Python 3 script for
rendering the HTML. Parameters ---------- add_header: boolean (default: False) If True, add HTML5 header and footer. Returns ------- str The rendered HTML.

Response:
def render(self, text, add_header=False):
        """Render the HTML.

        Parameters
        ----------
        add_header: boolean (default: False)
            If True, add HTML5 header and footer.

        Returns
        -------
        str
            The rendered HTML.
        """

        html = mark_text(text, self.aesthetics, self.rules)
        html = html.replace('\n', '<br/>')
        if add_header:
            html = '\n'.join([HEADER, self.css, MIDDLE, html, FOOTER])
        #print('\n'.join((HEADER, self.css, MIDDLE, html, FOOTER)))
        return html


Instruction:
Can you generate the documentation for the following Python 3 function
def train(self, nerdocs, mode_filename):

        trainer = pycrfsuite.Trainer(algorithm=self.algorithm,
                                     params={'c2': self.c2},
                                     verbose=self.verbose)

        for doc in nerdocs:
            for snt in doc.sentences:
                xseq = [t.feature_list() for t in snt]
                yseq = [t.label for t in snt]
                trainer.append(xseq, yseq)

        trainer.train(mode_filename)

Response: Train a CRF model using a list of nerdocs.


Instruction:
Make a summary of the following Python 3 code
def tag(self, nerdoc):

        labels = []
        for snt in nerdoc.sentences:
            xseq = [t.feature_list() for t in snt]
            yseq = self.tagger.tag(xseq)
            labels.append(yseq)
        return labels

Response: Tag the given document with the tagger.


Instruction:
Given the following Python 3 function, write the documentation
def balancedSlicer(text, openDelim='[', closeDelim=']'):
    openbr = 0
    cur = 0
    for char in text:
        cur +=1
        if char == openDelim:
            openbr += 1
        if char == closeDelim:
            openbr -= 1
        if openbr == 0:
            break
    return text[:cur], cur

Response: Assuming that the text contains a properly balanced expression using


Instruction:
Write a Python 3 function that can
convert a Wikipedia article to Text object.

Response:
def json_2_text(inp, out, verbose = False):
    """Convert a Wikipedia article to Text object.
    Concatenates the sections in wikipedia file and rearranges other information so it
    can be interpreted as a Text object.

    Links and other elements with start and end positions are annotated
    as layers.

    Parameters
    ----------
    inp: directory of parsed et.wikipedia articles in json format

    out: output directory of .txt files

    verbose: if True, prints every article title and total count of converted files
             if False prints every 50th count
    Returns
    -------
    estnltk.text.Text
        The Text object.
    """

    for root, dirs, filenames in os.walk(inp):
        for f in filenames:
            log = codecs.open(os.path.join(root, f), 'r')
            j_obj = json.load(log)
            j_obj = json_format(j_obj)

            #not needed, cause the json_format takes care of the right structuring
            #text = Text(j_obj)

            textWriter(j_obj, out, verbose)


Instruction:
Here you have a function in Python 3, explain what it does
def concatenate_matches(a, b, text, name):
    match = Match(a.start, b.end, text[a.start:b.end], name)
    for k, v in a.matches.items():
        match.matches[k] = v
    for k, v in b.matches.items():
        match.matches[k] = v
    if a.name is not None:
        aa = copy(a)
        del aa[MATCHES]
        match.matches[a.name] = aa
    if b.name is not None:
        bb = copy(b)
        del bb[MATCHES]
        match.matches[b.name] = bb
    return match

Response:
Concatenate matches a and b.
    All submatches will be copied to result.


Instruction:
Explain what the following Python 3 code does
def regex_from_markers(markers):
    return re.compile('|'.join([re.escape(c) for c in markers]))

Response: Given a string of characters construct a regular expression that matches them.


Instruction:
Explain what the following Python 3 code does
def convert(word):
    if six.PY2:
        if isinstance(word, unicode):
            return word.encode('utf-8')
        else:
            return word.decode('utf-8').encode('utf-8') # make sure it is real utf8, otherwise complain
    else: # ==> Py3
        if isinstance(word, bytes):
            return word.decode('utf-8') # bytes must be in utf8
        return word

Response:
This method converts given word to UTF - 8 encoding and bytes type for the
       SWIG wrapper.


Instruction:
Can you generate the documentation for the following Python 3 function
def postprocess_result(morphresult, trim_phonetic, trim_compound):
    word, analysis = morphresult
    return {
        'text': deconvert(word),
        'analysis': [postprocess_analysis(a, trim_phonetic, trim_compound) for a in analysis]
    }

Response: Postprocess vabamorf wrapper output.


Instruction:
Can you tell what is the following Python 3 function doing
def trim_phonetics(root):
    global phonetic_markers
    global phonetic_regex
    if root in phonetic_markers:
        return root
    else:
        return phonetic_regex.sub('', root)

Response: Function that trims phonetic markup from the root.


Instruction:
Can you write a function in Python 3 where it
gets the root form without markers.

Response:
def get_root(root, phonetic, compound):
    """Get the root form without markers.

    Parameters
    ----------
    root: str
        The word root form.
    phonetic: boolean
        If True, add phonetic information to the root forms.
    compound: boolean
        if True, add compound word markers to root forms.
    """
    global compound_regex
    if not phonetic:
        root = trim_phonetics(root)
    if not compound:
        root = trim_compounds(root)
    return root


Instruction:
Given the following Python 3 function, write the documentation
def get_group_tokens(root):
    global all_markers
    if root in all_markers or root in ['-', '_']: # special case
        return [[root]]
    groups = []
    for group in root.split('-'):
        toks = [trim_phonetics(trim_compounds(tok)) for tok in group.split('_')]
        groups.append(toks)
    return groups

Response: Function to extract tokens in hyphenated groups ( saunameheks - tallimeheks.


Instruction:
Can you write a function in Python 3 where it
synthesizes a single word based on given morphological attributes.

Response:
def synthesize(lemma, form, partofspeech='', hint='', guess=True, phonetic=False):
    """Synthesize a single word based on given morphological attributes.

    Note that spellchecker does not respect pre-tokenized words and concatenates
    token sequences such as "New York".

    Parameters
    ----------
    lemma: str
        The lemma of the word(s) to be synthesized.
    form: str
        The form of the word(s) to be synthesized.
    partofspeech: str
        Part-of-speech.
    hint: str
        Hint.
    guess: boolean (default: True)
        Use heuristics when synthesizing unknown words.
    phonetic: boolean (default: False)
        Add phonetic markup to synthesized words.

    Returns
    -------
    list
        List of synthesized words.
    """
    return Vabamorf.instance().synthesize(lemma, form, partofspeech, hint, guess, phonetic)


Instruction:
Explain what the following Python 3 code does
def instance():
        if not hasattr(Vabamorf, 'pid') or Vabamorf.pid != os.getpid():
            Vabamorf.pid = os.getpid()
            Vabamorf.morf = Vabamorf()
        return Vabamorf.morf

Response: Return an instance of PyVabamorf.


Instruction:
Write a Python 3 function for
performing morphological analysis and disambiguation of given text.

Response:
def analyze(self, words, **kwargs):
        """Perform morphological analysis and disambiguation of given text.

        Parameters
        ----------
        words: list of str or str
            Either a list of pretokenized words or a string. In case of a string, it will be splitted using
            default behaviour of string.split() function.
        disambiguate: boolean (default: True)
            Disambiguate the output and remove incosistent analysis.
        guess: boolean (default: True)
            Use guessing in case of unknown words
        propername: boolean (default: True)
            Perform additional analysis of proper names.
        compound: boolean (default: True)
            Add compound word markers to root forms.
        phonetic: boolean (default: False)
            Add phonetic information to root forms.

        Returns
        -------
        list of (list of dict)
            List of analysis for each word in input.
        """
        # if input is a string, then tokenize it
        if isinstance(words, six.string_types):
            words = words.split()

        # convert words to native strings
        words = [convert(w) for w in words]

        morfresults = self._morf.analyze(
            vm.StringVector(words),
            kwargs.get('disambiguate', True),
            kwargs.get('guess', True),
            True, # phonetic and compound information
            kwargs.get('propername', True))
        trim_phonetic = kwargs.get('phonetic', False)
        trim_compound = kwargs.get('compound', True)

        return [postprocess_result(mr, trim_phonetic, trim_compound) for mr in morfresults]


Instruction:
Can you generate the documentation for the following Python 3 function
def disambiguate(self, words):
        words = vm.SentenceAnalysis([as_wordanalysis(w) for w in words])
        disambiguated = self._morf.disambiguate(words)
        return [postprocess_result(mr, False, True) for mr in disambiguated]

Response: Disambiguate previously analyzed words.


Instruction:
Can you generate the documentation for the following Python 3 function
def fix_spelling(self, words, join=True, joinstring=' '):

        fixed_words = []
        for word in self.spellcheck(words, suggestions=True):
            if word['spelling']:
                fixed_words.append(word['text'])
            else:
                suggestions = word['suggestions']
                if len(suggestions) > 0:
                    fixed_words.append(suggestions[0])
                else:
                    fixed_words.append(word['text'])
        if join:
            return joinstring.join(fixed_words)
        else:
            return fixed_words

Response: Simple function for quickly correcting misspelled words.


Instruction:
Write a Python 3 function for
synthesizing a single word based on given morphological attributes.

Response:
def synthesize(self, lemma, form, partofspeech='', hint='', guess=True, phonetic=False):
        """Synthesize a single word based on given morphological attributes.

        Note that spellchecker does not respect pre-tokenized words and concatenates
        token sequences such as "New York".

        Parameters
        ----------
        lemma: str
            The lemma of the word(s) to be synthesized.
        form: str
            The form of the word(s) to be synthesized.
        partofspeech: str
            Part-of-speech.
        hint: str
            Hint.
        guess: boolean (default: True)
            Use heuristics when synthesizing unknown words.
        phonetic: boolean (default: False)
            Add phonetic markup to synthesized words.

        Returns
        -------
        list
            List of synthesized words.
        """
        words = self._morf.synthesize(
            convert(lemma.strip()),
            convert(form.strip()),
            convert(partofspeech.strip()),
            convert(hint.strip()),
            guess,
            phonetic
        )
        return [deconvert(w) for w in words]


Instruction:
Here you have a function in Python 3, explain what it does
def prepare_sentence(self, sentence):
        # depending on how the morphological analysis was added, there may be
        # phonetic markup. Remove it, if it exists.
        for word in sentence:
            for analysis in word[ANALYSIS]:
                analysis[ROOT] = analysis[ROOT].replace('~', '')
                analysis[ROOT] = re.sub('[?<\]]([aioueöäõü])', '\\1', analysis[ROOT])
        return json.dumps({WORDS: sentence})

Response: Prepare the sentence for segment detection.


Instruction:
How would you code a function in Python 3 to
add clause indexes to already annotated sentence.

Response:
def annotate_indices(self, sentence):
        """Add clause indexes to already annotated sentence."""
        max_index = 0
        max_depth = 1
        stack_of_indexes = [ max_index ]
        for token in sentence:
            if CLAUSE_ANNOT not in token:
                token[CLAUSE_IDX] = stack_of_indexes[-1]
            else:
                # Alustavad märgendused
                for annotation in token[CLAUSE_ANNOT]:
                    if annotation == "KIILU_ALGUS":
                        # Liigume sügavamale, alustame järgmist kiilu
                        max_index += 1
                        stack_of_indexes.append(max_index)
                        if (len(stack_of_indexes) > max_depth):
                            max_depth = len(stack_of_indexes)
                token[CLAUSE_IDX] = stack_of_indexes[-1]
                # Lõpetavad märgendused
                for annotation in token[CLAUSE_ANNOT]:
                    if annotation == "KINDEL_PIIR":
                        # Liigume edasi samal tasandil, alustame järgmist osalauset
                        max_index += 1
                        stack_of_indexes[-1] = max_index
                    elif annotation == "KIILU_LOPP":
                        # Taandume sügavusest, sulgeme ühe kiilu
                        stack_of_indexes.pop()
        return sentence


Instruction:
Implement a function in Python 3 to
function that renames and restructures clause information.

Response:
def rename_annotations(self, sentence):
        """Function that renames and restructures clause information."""
        annotations = []
        for token in sentence:
            data = {CLAUSE_IDX: token[CLAUSE_IDX]}
            if CLAUSE_ANNOT in token:
                if 'KINDEL_PIIR' in token[CLAUSE_ANNOT]:
                    data[CLAUSE_ANNOTATION] = CLAUSE_BOUNDARY
                elif 'KIILU_ALGUS' in token[CLAUSE_ANNOT]:
                    data[CLAUSE_ANNOTATION] = EMBEDDED_CLAUSE_START
                elif 'KIILU_LOPP' in token[CLAUSE_ANNOT]:
                    data[CLAUSE_ANNOTATION] = EMBEDDED_CLAUSE_END
            annotations.append(data)
        return annotations


Instruction:
Create a Python 3 function to
split the text into smaller texts.

Response:
def split_Text( text, file_name, verbose = True ):
    ''' Tokenizes the *text* (from *file_name*) into sentences, and if the number of
        sentences exceeds *max_sentences*, splits the text into smaller texts.

        Returns a list containing the original text (if no splitting was required),
        or a list containing results of the splitting (smaller texts);
    '''
    if verbose:
       print('  processing '+file_name+' ... ', end="" )
    # Tokenize text into sentences
    start = timer()
    text = text.tokenize_sentences()
    all_sentences = len(text[SENTENCES])
    end = timer()
    if verbose:
       print('  (tok time: '+format_time( end-start )+')', end="" )
    if all_sentences > max_sentences:
       # Acquire spans of length *max_sentences* from the text
       start = timer()
       i = 0
       spans = []
       len_total = 0
       while i < all_sentences:
            startSent = text[SENTENCES][i]
            endSent   = text[SENTENCES][min(i+(max_sentences-1), all_sentences-1)]
            span = (startSent[START], endSent[END])
            len_total += (span[1]-span[0])
            spans.append(span)
            i += max_sentences
       # Divide the text into spans
       text_spans = text.texts_from_spans(spans)
       assert len(text.text) >= len_total, '(!) Total spans_len must be =< than text_len: '+str(len_total)+'/'+str(len(text.text))
       new_texts  = []
       for i, small_text in enumerate( text_spans ):
           newText = Text( small_text )
           for key in text.keys():
               if key != TEXT and key != SENTENCES and key != PARAGRAPHS:
                  newText[key] = text[key]
           newText['_text_split_id']     = i
           newText['_text_split_origin'] = str(spans[i])  #  Convert it to string; Otherwise, split_by(*) may mistakenly consider
                                                          #  it a layer and may run into error while trying to split it;
           newText['_text_split_file']   = file_name
           #print( json.dumps(newText) )
           new_texts.append( newText )
       end = timer()
       if verbose:
           print('  (split time: '+format_time( end-start )+')', end="" )
           print('  (sents: '+str(all_sentences)+', new_texts:'+str(len(new_texts))+')', end="")
           print()
       return new_texts
    else:
       if verbose:
           print('  (sents: '+str(all_sentences)+', no_split)', end=" \n")
       return [text]


Instruction:
Explain what the following Python 3 code does
def write_Text_into_file( text, old_file_name, out_dir, suffix='__split', verbose=True ):
    ''' Based on *old_file_name*, *suffix* and *out_dir*, constructs a new file name and
        writes *text* (in the ascii normalised JSON format) into the new file.
    '''
    name = os.path.basename( old_file_name )
    if '.' in name:
       new_name = re.sub('\.([^.]+)$', suffix+'.\\1', name)
    else:
       new_name = name + suffix
    new_path = os.path.join( out_dir, new_name )
    start = timer()
    #write_document( text, new_path )  # <--- this leaves indent=2 - takes too much extra space ...
    o_f = codecs.open( new_path, mode='wb', encoding='ascii' )
    o_f.write( json.dumps( text ) )
    o_f.close()
    end = timer()
    timestamp = format_time( end-start )
    if verbose:
       print('  ==>  '+new_path+' (file writing time: '+timestamp+')' )

Response: Writes text into a new file.


Instruction:
Make a summary of the following Python 3 code
def parse_tei_corpora(root, prefix='', suffix='.xml', target=['artikkel'], encoding=None):
    documents = []
    for fnm in get_filenames(root, prefix, suffix):
        path = os.path.join(root, fnm)
        docs = parse_tei_corpus(path, target, encoding)
        for doc in docs:
            doc[FILE] = fnm
        documents.extend(docs)
    return documents

Response: Parse TEI style XML files and return a list of Text objects.


Instruction:
Write a Python 3 function that can
parse TEI style XML file into a list of text documents.

Response:
def parse_tei_corpus(path, target=['artikkel'], encoding=None):
    """Parse documents from a TEI style XML file.

    Parameters
    ----------
    path: str
        The path of the XML file.
    target: list of str
        List of <div> types, that are considered documents in the XML files (default: ["artikkel"]).
    encoding: str
        Encoding to be used for decoding the content of the XML file. If not specified (default),
        then no separate decoding step is applied.

    Returns
    -------
    list of esnltk.text.Text
    """
    with open(path, 'rb') as f:
        html_doc = f.read()
    if encoding:
        html_doc = html_doc.decode( encoding )
    soup = BeautifulSoup(html_doc, 'html5lib')
    title = soup.find_all('title')[0].string

    documents = []
    for div1 in soup.find_all('div1'):
        documents.extend(parse_div(div1, dict(), target))
    return tokenize_documents(documents)


Instruction:
Given the following Python 3 function, write the documentation
def parse_div(soup, metadata, target):
    documents = []
    div_type = soup.get('type', None)
    div_title = list(soup.children)[0].string.strip()

    if div_type in target:
        div_authors = soup.find_all('author')
        document = {
            'type': div_type,
            'title': div_title,
            'paragraphs': parse_paragraphs(soup)
        }
        # add author, if it exists
        if len(div_authors) > 0:
            div_author = div_authors[0].text.strip()
            document['author'] = div_author
        # add collected metadata
        for k, v in metadata.items():
            document[k] = v
        documents.append(document)
    else:
        metadata[div_type] = div_title

        # recurse subdivs
        subdiv_name = get_subdiv(soup.name)
        subdivs = []
        if subdiv_name is not None:
            subdivs = soup.find_all(subdiv_name)
        if len(subdivs) > 0:
            for subdiv in subdivs:
                documents.extend(parse_div(subdiv, deepcopy(metadata), target))
    return documents

Response: Parses a div from the XML file and returns a list of dicts.


Instruction:
Write a Python 3 script for
parsing sentences and paragraphs in the section.

Response:
def parse_paragraphs(soup):
    """Parse sentences and paragraphs in the section.

    Parameters
    ----------
    soup: bs4.BeautifulSoup
        The parsed XML data.

    Returns
    -------
    list of (list of str)
        List of paragraphs given as list of sentences.
    """
    paragraphs = []
    for para in soup.find_all('p'):
        sentences = []
        for sent in para.find_all('s'):
            sentence = sent.text.strip()
            if len(sentence) > 0:
                sentences.append(sentence)
        if len(sentences) > 0:
            paragraphs.append({'sentences': sentences})
    return paragraphs


Instruction:
Implement a function in Python 3 to
convert the imported documents to Text instances.

Response:
def tokenize_documents(docs):
    """Convert the imported documents to :py:class:'~estnltk.text.Text' instances."""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts


Instruction:
Can you write a function in Python 3 where it
functions for training the default NER model.

Response:
def train_default_model():
    """Function for training the default NER model.

    NB! It overwrites the default model, so do not use it unless
    you know what are you doing.

    The training data is in file estnltk/corpora/estner.json.bz2 .
    The resulting model will be saved to estnltk/estner/models/default.bin
    """
    docs = read_json_corpus(DEFAULT_NER_DATASET)
    trainer = NerTrainer(default_nersettings)
    trainer.train(docs, DEFAULT_NER_MODEL_DIR)


Instruction:
Make a summary of the following Python 3 code
def process_line(self, line):
        assert isinstance(line, str)
        try:
            self._process.stdin.write(as_binary(line))
            self._process.stdin.write(as_binary('\n'))
            self._process.stdin.flush()
            result = as_unicode(self._process.stdout.readline())
            if result == '':
                stderr = as_unicode(self._process.stderr.read())
                raise Exception('EOF encountered while reading stream. Stderr is {0}.'.format(stderr))
            return result
        except Exception:
            self._process.terminate()
            raise

Response: Process a line of data and return the line.


Instruction:
Write a Python 3 function that can
return the offsets in the WordNet file for every synset index.

Response:
def _get_synset_offsets(synset_idxes):
    """Returs pointer offset in the WordNet file for every synset index.

    Notes
    -----
    Internal function. Do not call directly.
    Preserves order -- for [x,y,z] returns [offset(x),offset(y),offset(z)].

    Parameters
    ----------
    synset_idxes : list of ints
      Lists synset IDs, which need offset.

    Returns
    -------
    list of ints
      Lists pointer offsets in Wordnet file.

    """
    offsets = {}
    current_seeked_offset_idx = 0

    ordered_synset_idxes = sorted(synset_idxes)

    with codecs.open(_SOI,'rb', 'utf-8') as fin:
        for line in fin:
            split_line = line.split(':')

            while current_seeked_offset_idx < len(ordered_synset_idxes) and split_line[0] == str(ordered_synset_idxes[current_seeked_offset_idx]):
                # Looping on single line entries in case synset_indexes contains duplicates.
                offsets[synset_idxes[current_seeked_offset_idx]] = int(split_line[1])
                current_seeked_offset_idx += 1
            if current_seeked_offset_idx >= len(synset_idxes):
                break
    return [offsets[synset_idx] for synset_idx in synset_idxes]


Instruction:
Write a Python 3 function for
giving a list of offsets in the WordNet file parses each synset object for every offset and stores it in the global dictionary.

Response:
def _get_synsets(synset_offsets):
    """Given synset offsets in the WordNet file, parses synset object for every offset.

    Notes
    -----
    Internal function. Do not call directly.
    Stores every parsed synset into global synset dictionary under two keys:
      synset's name lemma.pos.sense_no and synset's id (unique integer).

    Parameters
    ----------
    synset_offsets : list of ints
      Lists pointer offsets from which synset objects will be parsed.

    Returns
    -------
    list of Synsets
      Lists synset objects which synset_offsets point to.

    """
    global parser
    if parser is None:
        parser = Parser(_WN_FILE)
    synsets = []

    for offset in synset_offsets:
        raw_synset = parser.parse_synset(offset)
        synset = Synset(raw_synset)

        SYNSETS_DICT[_get_key_from_raw_synset(raw_synset)] = synset
        SYNSETS_DICT[synset.id] = synset

        synsets.append(synset)

    return synsets


Instruction:
Can you tell what is the following Python 3 function doing
def _get_key_from_raw_synset(raw_synset):
    pos = raw_synset.pos
    literal = raw_synset.variants[0].literal
    sense = "%02d"%raw_synset.variants[0].sense
    return '.'.join([literal,pos,sense])

Response: Derives synset key from the provided eurown. py Synset object.


Instruction:
Can you tell what is the following Python 3 function doing
def synset(synset_key):

    if synset_key in SYNSETS_DICT:
        return SYNSETS_DICT[synset_key]

    def _get_synset_idx(synset_key):
        """Returns synset index for the provided key.

        Note
        ----
          Internal function. Do not call directly.

        """
        with codecs.open(_SENSE_FILE,'rb', 'utf-8') as fin:
            for line in fin:
                split_line = line.split(':')
                if split_line[0] == synset_key:
                    return int(split_line[1].strip())
        return None

    synset_idx = _get_synset_idx(synset_key)

    if synset_idx == None:
        return None

    synset_offset = _get_synset_offsets([synset_idx])
    synset = _get_synsets(synset_offset)

    return synset[0]

Response: Returns a synset object with the provided key.


Instruction:
Can you tell what is the following Python 3 function doing
def synsets(lemma,pos=None):

    def _get_synset_idxes(lemma,pos):
        line_prefix_regexp = "%s:%s:(.*)"%(lemma,pos if pos else "\w+")
        line_prefix = re.compile(line_prefix_regexp)

        idxes = []

        with codecs.open(_LIT_POS_FILE,'rb', 'utf-8') as fin:
            for line in fin:
                result = line_prefix.match(line)
                if result:
                    res_indices = [int(x) for x in result.group(1).split(' ')]
                    idxes.extend(res_indices)

        LEM_POS_2_SS_IDX[lemma][pos].extend(idxes)
        return sorted(idxes)

    synset_idxes = None

    if lemma in LEM_POS_2_SS_IDX:
        if pos in LEM_POS_2_SS_IDX[lemma]:
            synset_idxes = LEM_POS_2_SS_IDX[lemma][pos]
        else:
            synset_idxes = [idx for pos in LEM_POS_2_SS_IDX[lemma] for idx in LEM_POS_2_SS_IDX[lemma][pos]]
    if not synset_idxes:
        synset_idxes = _get_synset_idxes(lemma,pos)

    if len(synset_idxes) == 0:
        return []

    stored_synsets = [SYNSETS_DICT[synset_idxes[i]] for i in range(len(synset_idxes)) if synset_idxes[i] in SYNSETS_DICT]
    unstored_synset_idxes = [synset_idxes[i] for i in range(len(synset_idxes)) if synset_idxes[i] not in SYNSETS_DICT]

    synset_offsets = _get_synset_offsets(unstored_synset_idxes)
    synsets = _get_synsets(synset_offsets)

    return stored_synsets + synsets

Response: Returns all synsets which have the given lemma and fixed pos.


Instruction:
Write a Python 3 script for
returning all the synsets which have the provided pos.

Response:
def all_synsets(pos=None):
    """Return all the synsets which have the provided pos.

    Notes
    -----
    Returns thousands or tens of thousands of synsets - first time will take significant time.
    Useful for initializing synsets as each returned synset is also stored in a global dictionary for fast retrieval the next time.

    Parameters
    ----------
    pos : str
      Part-of-speech of the sought synsets. Sensible alternatives are wn.ADJ, wn.ADV, wn.VERB, wn.NOUN and `*`.
      If pos == `*`, all the synsets are retrieved and initialized for fast retrieval the next time.

    Returns
    -------
    list of Synsets
      Lists the Synsets which have `pos` as part-of-speech.
      Empty list, if `pos` not in [wn.ADJ, wn.ADV, wn.VERB, wn.NOUN, `*`].

    """
    def _get_unique_synset_idxes(pos):
        idxes = []

        with codecs.open(_LIT_POS_FILE,'rb', 'utf-8') as fin:
            if pos == None:
                for line in fin:
                  split_line = line.strip().split(':')
                  idxes.extend([int(x) for x in split_line[2].split()])
            else:
                for line in fin:
                    split_line = line.strip().split(':')
                    if split_line[1] == pos:
                        idxes.extend([int(x) for x in split_line[2].split()])
        idxes = list(set(idxes))
        idxes.sort()
        return idxes

    if pos in LOADED_POS:
        return [SYNSETS_DICT[idx] for lemma in LEM_POS_2_SS_IDX for idx in LEM_POS_2_SS_IDX[lemma][pos]]
    else:
        synset_idxes = _get_unique_synset_idxes(pos)

        if len(synset_idxes) == 0:
            return []

        stored_synsets = [SYNSETS_DICT[synset_idxes[i]] for i in range(len(synset_idxes)) if synset_idxes[i] in SYNSETS_DICT]
        unstored_synset_idxes = [synset_idxes[i] for i in range(len(synset_idxes)) if synset_idxes[i] not in SYNSETS_DICT]

        synset_offsets = _get_synset_offsets(unstored_synset_idxes)
        synsets = _get_synsets(synset_offsets)

        for synset in synsets:
            for variant in synset.get_variants():
                LEM_POS_2_SS_IDX[variant.literal][synset.pos].append(synset.id)

        LOADED_POS.add(pos)

        return stored_synsets + synsets


Instruction:
How would you code a function in Python 3 to
return the Lemma object with the given key.

Response:
def lemma(lemma_key):
    """Returns the Lemma object with the given key.

    Parameters
    ----------
    lemma_key : str
      Key of the returned lemma.

    Returns
    -------
    Lemma
      Lemma matching the `lemma_key`.

    """
    if lemma_key in LEMMAS_DICT:
        return LEMMAS_DICT[lemma_key]
    split_lemma_key = lemma_key.split('.')

    synset_key = '.'.join(split_lemma_key[:3])
    lemma_literal = split_lemma_key[3]

    lemma_obj = Lemma(synset_key,lemma_literal)
    LEMMAS_DICT[lemma_key] = lemma_obj
    return lemma_obj


Instruction:
Write a Python 3 function that can
return all the Lemmas of which name is lemma and which have pos as part of speech.

Response:
def lemmas(lemma,pos=None):
    """Returns all the Lemma objects of which name is `lemma` and which have `pos` as part
    of speech.

    Parameters
    ----------
    lemma : str
      Literal of the sought Lemma objects.
    pos : str, optional
      Part of speech of the sought Lemma objects. If None, matches any part of speech.
      Defaults to None

    Returns
    -------
    list of Lemmas
      Lists all the matched Lemmas.
    """
    lemma = lemma.lower()
    return [lemma_obj
        for synset in synsets(lemma,pos)
        for lemma_obj in synset.lemmas()
        if lemma_obj.name.lower() == lemma]