Instruction:
Can you implement a function in Python 3 that
passes the transfer details to aspera and receive back a populated with access token

Response:
def _create_transfer_spec(self, call_args):
        ''' pass the transfer details to aspera and receive back a
            populated transfer spec complete with access token '''
        _paths = []
        for _file_pair in call_args.file_pair_list:
            _path = OrderedDict()
            if call_args.direction == enumAsperaDirection.SEND:
                _action = "upload_setup"
                _path['source'] = _file_pair.fileobj
                _path['destination'] = _file_pair.key
            else:
                _action = "download_setup"
                _path['source'] = _file_pair.key
                _path['destination'] = _file_pair.fileobj
            _paths.append(_path)

        # Add credentials before the transfer spec is requested.
        delegated_token = self._delegated_token_manager.get_token()
        _response = self._fetch_transfer_spec(_action, delegated_token, call_args.bucket, _paths)

        tspec_dict = json.loads(_response.content)['transfer_specs'][0]['transfer_spec']

        tspec_dict["destination_root"] = "/"

        if (call_args.transfer_config):
            tspec_dict.update(call_args.transfer_config.dict)
            if call_args.transfer_config.is_multi_session_all:
                tspec_dict['multi_session'] = 0
                _remote_host = tspec_dict['remote_host'].split('.')
                # now we append '-all' to the remote host
                _remote_host[0] += "-all"
                tspec_dict['remote_host'] = ".".join(_remote_host)
                logger.info("New remote_host(%s)" % tspec_dict['remote_host'])

        call_args.transfer_spec = json.dumps(tspec_dict)

        return True


Instruction:
Write a Python 3 function that can
upload a directory using Aspera

Response:
def upload_directory(self, directory, bucket, key, transfer_config=None, subscribers=None):
        ''' upload a directory using Aspera '''
        check_io_access(directory, os.R_OK)
        return self._queue_task(bucket, [FilePair(key, directory)], transfer_config,
                                subscribers, enumAsperaDirection.SEND)


Instruction:
Here you have a function in Python 3, explain what it does
def download_directory(self, bucket, key, directory, transfer_config=None, subscribers=None):
        ''' download a directory using Aspera '''
        check_io_access(directory, os.W_OK)
        return self._queue_task(bucket, [FilePair(key, directory)], transfer_config,
                                subscribers, enumAsperaDirection.RECEIVE)

Response: download a directory using Aspera


Instruction:
Can you write a function in Python 3 where it
uploads a file using Aspera

Response:
def upload(self, fileobj, bucket, key, transfer_config=None, subscribers=None):
        ''' upload a file using Aspera '''
        check_io_access(fileobj, os.R_OK, True)
        return self._queue_task(bucket, [FilePair(key, fileobj)], transfer_config,
                                subscribers, enumAsperaDirection.SEND)


Instruction:
How would you explain what the following Python 3 function does
def download(self, bucket, key, fileobj, transfer_config=None, subscribers=None):
        ''' download a file using Aspera '''
        check_io_access(os.path.dirname(fileobj), os.W_OK)
        return self._queue_task(bucket, [FilePair(key, fileobj)], transfer_config,
                                subscribers, enumAsperaDirection.RECEIVE)

Response: download a file using Aspera


Instruction:
Write a Python 3 function that can
set the log details for the current aspera process

Response:
def set_log_details(aspera_log_path=None,
                        sdk_log_level=logging.NOTSET):
        ''' set the aspera log path - used by th Ascp process
            set the internal aspera sdk activity - for debug purposes '''
        if aspera_log_path:
            check_io_access(aspera_log_path, os.W_OK)
            AsperaTransferCoordinator.set_log_location(aspera_log_path)

        if sdk_log_level != logging.NOTSET:
            if logger:
                if not len(logger.handlers):
                    handler = logging.StreamHandler()
                    _fmt = '%(asctime)s %(levelname)s %(message)s'
                    handler.setFormatter(logging.Formatter(_fmt))
                    logger.addHandler(handler)
                    logger.setLevel(sdk_log_level)


Instruction:
How would you code a function in Python 3 to
validate the user arguments

Response:
def _validate_args(self, args):
        ''' validate the user arguments '''
        assert(args.bucket)

        if args.subscribers:
            for _subscriber in args.subscribers:
                assert(isinstance(_subscriber, AsperaBaseSubscriber))

        if (args.transfer_config):
            assert(isinstance(args.transfer_config, AsperaConfig))

            # number of sessions requested cant be greater than max ascps
            if args.transfer_config.multi_session > self._config.ascp_max_concurrent:
                raise ValueError("Max sessions is %d" % self._config.ascp_max_concurrent)

        for _pair in args.file_pair_list:
            if not _pair.key or not _pair.fileobj:
                raise ValueError("Invalid file pair")


Instruction:
Given the following Python 3 function, write the documentation
def _shutdown(self, cancel, cancel_msg, exc_type=CancelledError):
        ''' Internal shutdown used by 'shutdown' method  above '''
        if cancel:
            # Cancel all in-flight transfers if requested, before waiting
            # for them to complete.
            self._coordinator_controller.cancel(cancel_msg, exc_type)
        try:
            # Wait until there are no more in-progress transfers. This is
            # wrapped in a try statement because this can be interrupted
            # with a KeyboardInterrupt that needs to be caught.
            self._coordinator_controller.wait()
        except KeyboardInterrupt:
            # If not errors were raised in the try block, the cancel should
            # have no coordinators it needs to run cancel on. If there was
            # an error raised in the try statement we want to cancel all of
            # the inflight transfers before shutting down to speed that
            # process up.
            self._coordinator_controller.cancel('KeyboardInterrupt()')
            raise
        finally:
            self._coordinator_controller.cleanup()

Response: Internal shutdown used by shutdown method above.


Instruction:
Explain what the following Python 3 code does
def cleanup(self):
        ''' Stop backgroud thread and cleanup resources '''
        self._processing_stop = True
        self._wakeup_processing_thread()
        self._processing_stopped_event.wait(3)

Response: Stop backgroud thread and cleanup resources


Instruction:
Make a summary of the following Python 3 code
def tracked_coordinator_count(self, count_ascps=False):
        ''' count the number of cooridnators currently being processed
        or count the number of ascps currently being used '''
        with self._lock:
            _count = 0
            if count_ascps:
                for _coordinator in self._tracked_transfer_coordinators:
                    _count += _coordinator.session_count
            else:
                _count = len(self._tracked_transfer_coordinators)
            return _count

Response: count the number of cooridnators currently being processed and the number of ascps currently being used


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _queue_task(self, args):
        ''' add transfer to waiting queue if possible
            then notify the background thread to process it '''
        if self._cancel_called:
            raise AsperaTransferQueueError("Cancel already called")
        elif self._wait_called:
            raise AsperaTransferQueueError("Cant queue items during wait")
        elif self.waiting_coordinator_count() >= self._config.max_submission_queue_size:
            raise AsperaTransferQueueError("Max queued items reached")
        else:
            _coordinator = AsperaTransferCoordinator(args)
            _components = {'meta': TransferMeta(args, transfer_id=args.transfer_id),
                           'coordinator': _coordinator}

            _transfer_future = AsperaTransferFuture(**_components)
            _coordinator.add_subscribers(args.subscribers, future=_transfer_future)
            _coordinator.add_done_callback(self.remove_aspera_coordinator,
                                           transfer_coordinator=_coordinator)
            self.append_waiting_queue(_coordinator)

            if not self._processing_thread:
                self._processing_thread = threading.Thread(target=self._process_waiting_queue)
                self._processing_thread.daemon = True
                self._processing_thread.start()

            self._wakeup_processing_thread()

        return _transfer_future

Response: add a transfer to waiting queue if possible then notify the background thread to process it


Instruction:
Write a Python 3 script for
removing entry from waiting waiting or remove item from processig queue and add to processed quque notify background thread as it may be able to process watiign requests

Response:
def remove_aspera_coordinator(self, transfer_coordinator):
        ''' remove entry from the waiting waiting
            or remove item from processig queue and add to processed quque
            notify background thread as it may be able to process watiign requests
        '''
        # usually called on processing completion - but can be called for a cancel
        if self._in_waiting_queue(transfer_coordinator):
            logger.info("Remove from waiting queue count=%d" % self.waiting_coordinator_count())
            with self._lockw:
                self._waiting_transfer_coordinators.remove(transfer_coordinator)
        else:
            logger.info("Remove from processing queue count=%d" % self.tracked_coordinator_count())
            try:
                self.remove_transfer_coordinator(transfer_coordinator)
                self.append_processed_queue(transfer_coordinator)
            except Exception:
                pass

            self._wakeup_processing_thread()


Instruction:
Write a Python 3 function for
appending item to waiting queue

Response:
def append_waiting_queue(self, transfer_coordinator):
        ''' append item to waiting queue '''
        logger.debug("Add to waiting queue count=%d" % self.waiting_coordinator_count())
        with self._lockw:
            self._waiting_transfer_coordinators.append(transfer_coordinator)


Instruction:
Can you generate the documentation for the following Python 3 function
def free_processed_queue(self):
        ''' call the Aspera sdk to freeup resources '''
        with self._lock:
            if len(self._processed_coordinators) > 0:
                for _coordinator in self._processed_coordinators:
                    _coordinator.free_resources()
                self._processed_coordinators = []

Response: call the Aspera sdk to freeup resources


Instruction:
How would you code a function in Python 3 to
have either of the stop processing flags been set

Response:
def is_stop(self):
        ''' has either of the stop processing flags been set '''
        if len(self._processed_coordinators) > 0:
            self.free_processed_queue()
        return self._cancel_called or self._processing_stop


Instruction:
How would you explain what the following Python 3 function does
def _process_waiting_queue(self):
        ''' thread to processes the waiting queue
            fetches transfer spec
            then calls start transfer
            ensures that max ascp is not exceeded '''
        logger.info("Queue processing thread started")
        while not self.is_stop():
            self._processing_event.wait(3)
            self._processing_event.clear()
            if self.is_stop():
                break

            while self.waiting_coordinator_count() > 0:
                if self.is_stop():
                    break
                _used_slots = self.tracked_coordinator_count(True)
                _free_slots = self._config.ascp_max_concurrent - _used_slots
                if _free_slots <= 0:
                    break

                with self._lockw:
                    # check are there enough free slots
                    _req_slots = self._waiting_transfer_coordinators[0].session_count
                    if _req_slots > _free_slots:
                        break
                    _coordinator = self._waiting_transfer_coordinators.popleft()
                    self.add_transfer_coordinator(_coordinator)

                if not _coordinator.set_transfer_spec():
                    self.remove_aspera_coordinator(_coordinator)
                else:
                    logger.info("ASCP process queue - Max(%d) InUse(%d) Free(%d) New(%d)" %
                                (self._config.ascp_max_concurrent,
                                 _used_slots,
                                 _free_slots,
                                 _req_slots))
                    _coordinator.start_transfer()

        logger.info("Queue processing thread stopped")
        self._processing_stopped_event.set()

Response:
thread to process the waiting queue
            fetches transfer spec
            then calls start transfer


Instruction:
Write a Python 3 script to
remove all entries from waiting queue or cancell all in waiting queue

Response:
def clear_waiting_coordinators(self, cancel=False):
        ''' remove all entries from waiting queue or cancell all in waiting queue '''
        with self._lockw:
            if cancel:
                for _coordinator in self._waiting_transfer_coordinators:
                    _coordinator.notify_cancelled("Clear Waiting Queue", False)
            self._waiting_transfer_coordinators.clear()


Instruction:
Create a Python 3 function to
cancel all queue items - then attempt to cancel all in progress items - then attempt to cancel all in progress items - then clear waiting coordinator list

Response:
def cancel(self, *args, **kwargs):
        """ Cancel all queue items - then attempt to cancel all in progress items """
        self._cancel_called = True
        self.clear_waiting_coordinators(cancel=True)
        super(AsperaTransferCoordinatorController, self).cancel(*args, **kwargs)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def wait(self):
        self._wait_called = True
        while self.tracked_coordinator_count() > 0 or \
                self.waiting_coordinator_count() > 0:
            time.sleep(1)
            super(AsperaTransferCoordinatorController, self).wait()
        self._wait_called = False

Response: Wait until all in progress and queued items are processed.


Instruction:
Explain what the following Python 3 code does
def execute(self, input_data):
        ''' Execute the ViewZip worker '''

        # Just a small check to make sure we haven't been called on the wrong file type
        if (input_data['meta']['type_tag'] != 'zip'):
            return {'error': self.__class__.__name__+': called on '+input_data['meta']['type_tag']}

        view = {}
        view['payload_md5s'] = input_data['unzip']['payload_md5s']
        view['yara_sigs'] = input_data['yara_sigs']['matches'].keys()
        view.update(input_data['meta'])

        # Okay this view is going to also give the meta data about the payloads
        view['payload_meta'] = [self.workbench.work_request('meta', md5) for md5 in input_data['unzip']['payload_md5s']]
        return view

Response: Execute the ViewZip worker


Instruction:
Make a summary of the following Python 3 code
def set_exception(self, exception):
        if not self.is_done():
            raise TransferNotDoneError(
                'set_exception can only be called once the transfer is '
                'complete.')
        self._coordinator.set_exception(exception, override=True)

Response: Sets the exception on the future.


Instruction:
Create a Python 3 function to
pass the transfer spec to the Aspera sdk and start the transfer

Response:
def start_transfer(self):
        ''' pass the transfer spec to the Aspera sdk and start the transfer '''
        try:
            if not self.is_done():
                faspmanager2.startTransfer(self.get_transfer_id(),
                                           None,
                                           self.get_transfer_spec(),
                                           self)
        except Exception as ex:
            self.notify_exception(ex)


Instruction:
How would you code a function in Python 3 to
check whether a transfer is currently running

Response:
def is_running(self, is_stopped):
        ''' check whether a transfer is currently running '''
        if is_stopped and self.is_stopped():
            return False

        return faspmanager2.isRunning(self.get_transfer_id())


Instruction:
Explain what the following Python 3 code does
def is_stopped(self, is_stopping=True):
        ''' check whether a transfer is stopped or is being stopped '''
        if is_stopping:
            return self._is_stopped or self._is_stopping
        return self._is_stopped

Response: check whether a transfer is being stopped


Instruction:
Can you write a function in Python 3 where it
modifies an in progress eg pause or resume

Response:
def _modify_transfer(self, option, value=0):
        ''' call Apsera sdk modify an in progress eg pause/resume
            allowed values defined in enumAsperaModifyTransfer class '''
        _ret = False
        try:
            if self.is_running(True):
                logger.info("ModifyTransfer called %d = %d" % (option, value))
                _ret = faspmanager2.modifyTransfer(self.get_transfer_id(), option, value)
                logger.info("ModifyTransfer returned %s" % _ret)
        except Exception as ex:
            self.notify_exception(ex)

        return _ret


Instruction:
Explain what the following Python 3 code does
def stop(self, free_resource=False):
        ''' send a stop transfer request to the Aspera sdk, can  be done for:
            cancel - stop an in progress transfer
            free_resource - request to the Aspera sdk free resouces related to trasnfer_id
        '''
        if not self.is_stopped():
            self._is_stopping = True
            try:
                if free_resource or self.is_running(False):
                    if not free_resource:
                        logger.info("StopTransfer called - %s" % self.get_transfer_id())
                    self._is_stopped = faspmanager2.stopTransfer(self.get_transfer_id())
                    if not free_resource:
                        logger.info("StopTransfer returned %s - %s" % (
                                    self._is_stopped, self.get_transfer_id()))
            except Exception as ex:
                self.notify_exception(ex)

            self._is_stopping = False

        return self.is_stopped(False)

Response: send a stop transfer request to the Aspera sdk


Instruction:
Here you have a function in Python 3, explain what it does
def free_resources(self):
        ''' call stop to free up resources '''
        if not self.is_stopped():
            logger.info("Freeing resources: %s" % self.get_transfer_id())
            self.stop(True)

Response: call stop to free up resources


Instruction:
Can you write a function in Python 3 where it
searches message to find and extract a named value

Response:
def extract_message_value(self, name):
        ''' search message to find and extract a named value '''
        name += ":"
        assert(self._message)
        _start = self._message.find(name)
        if _start >= 0:
            _start += len(name) + 1
            _end = self._message.find("\n", _start)
            _value = self._message[_start:_end]
            return _value.strip()

        return None


Instruction:
Make a summary of the following Python 3 code
def _set_status(self, status, ex=None):
        ''' set session status - eg failed, success --
            valid values contained in enumAsperaControllerStatus class '''
        self._status = status
        logger.debug("Set status(%s) for %s" % (self._status, self.session_id))
        self.set_done()
        if ex:
            self._exception = ex

Response: set status of the aspera controller


Instruction:
Implement a Python 3 function for
setting the number of bytes transferred - return True if it has changed return False

Response:
def set_bytes_transferred(self, bytes_transferred):
        ''' set the number of bytes transferred - if it has changed return True '''
        _changed = False
        if bytes_transferred:
            _changed = (self._bytes_transferred != int(bytes_transferred))
            if _changed:
                self._bytes_transferred = int(bytes_transferred)
                logger.debug("(%s) BytesTransferred: %d" % (
                             self.session_id, self._bytes_transferred))
            if AsperaSession.PROGRESS_MSGS_SEND_ALL:
                return True
        return _changed


Instruction:
Create a Python 3 function to
set the exception message and set the status to failed

Response:
def set_exception(self, exception):
        ''' set the exception message and set the status to failed '''
        logger.error("%s : %s" % (exception.__class__.__name__, str(exception)))
        self._set_status(enumAsperaControllerStatus.FAILED, exception)


Instruction:
Given the following Python 3 function, write the documentation
def wait(self):
        ''' wait for the done event to be set - no timeout'''
        self._done_event.wait(MAXINT)
        return self._status, self._exception

Response: wait for the status and exception to be set


Instruction:
How would you code a function in Python 3 to
cancel the TransferFuture :param msg: The message to attach to the cancellation :param exc_type: The type of exception to set for the cancellation

Response:
def cancel(self, msg='', exc_type=CancelledError):
        """Cancels the TransferFuture
           :param msg: The message to attach to the cancellation
           :param exc_type: The type of exception to set for the cancellation
        """
        _ret = False
        if not self.is_done():
            self.notify_cancelled(msg, True)
            _ret = True

        return _ret


Instruction:
Write a Python 3 script for
updating the session count based on the type of session

Response:
def _update_session_count(self, type=0, actutal_session_count=0):
        ''' update the session/ascp count
            0 : set the number of sessions being used to 1 or number specified in transfer config
            -1: decrement the session count by one
             1: set the session count to param value
        '''
        if type == 0:  # init
            _count = 0
            if self._args.transfer_config:
                _count = self._args.transfer_config.multi_session
            self._session_count = _count if _count > 0 else 1
        elif type == -1:  # decrement
            self._session_count -= 1
        elif type == 1:  # set from number of actual session objects
            self._session_count = actutal_session_count


Instruction:
Write a Python 3 script for
waiting until the TransferFuture is done and returns the result. If the TransferFuture failed it will raise an exception associated to the exception associated to the exception. If the TransferFuture failed it will raise an exception associated to the exception.

Response:
def result(self, raise_exception=True):
        """Waits until TransferFuture is done and returns the result

        If the TransferFuture succeeded, it will return the result. If the
        TransferFuture failed, it will raise the exception associated to the
        failure.
        """
        _status = None
        _exception = None
        self._done_event.wait(MAXINT)  # first wait for session global
        if self.is_failed():  # global exception set
            _exception = self._exception
            _status = enumAsperaControllerStatus.FAILED
        else:
            for _session in self._sessions.values():
                _status_tmp, _exception_tmp = _session.wait()
                if _exception_tmp and not _exception:
                    _exception = _exception_tmp
                    _status = _status_tmp

        # Once done waiting, raise an exception if present or return the final status
        if _exception and raise_exception:
            raise _exception

        return _status


Instruction:
How would you explain what the following Python 3 function does
def notify_init(self):
        ''' run the queed callback  for just the first session only '''
        _session_count = len(self._sessions)
        self._update_session_count(1, _session_count)
        if _session_count == 1:
            self._run_queued_callbacks()

Response: run the queed callback for just the first session only


Instruction:
Create a Python 3 function to
notify all the active session that all other users have done

Response:
def notify_done(self, error=False, run_done_callbacks=True):
        ''' if error clear all sessions otherwise check to see if all other sessions are complete
            then run the done callbacks
        '''
        if error:
            for _session in self._sessions.values():
                _session.set_done()
            self._session_count = 0
        else:
            self._update_session_count(-1)
            for _session in self._sessions.values():
                if not _session.is_done():
                    return

        if run_done_callbacks:
            self._run_done_callbacks()
        self._done_event.set()


Instruction:
Explain what the following Python 3 code does
def notify_progress(self):
        ''' only call the progress callback if total has changed
            or PROGRESS_MSGS_SEND_ALL is set '''
        _total = 0
        for _session in self._sessions.values():
            _total += _session.bytes_transferred

        if AsperaSession.PROGRESS_MSGS_SEND_ALL:
            self._run_progress_callbacks(_total)
        else:
            # dont call progress callback unless total has changed
            if self._total_bytes_transferred != _total:
                self._total_bytes_transferred = _total
                self._run_progress_callbacks(_total)

Response:
only call the progress callback if total has changed
            or PROGRESS_MSGS_SEND_ALL is set


Instruction:
Can you implement a function in Python 3 that
sets the exception message stop transfer if running and set the done event

Response:
def notify_exception(self, exception, run_done_callbacks=True):
        ''' set the exception message, stop transfer if running and set the done event '''
        logger.error("%s : %s" % (exception.__class__.__name__, str(exception)))
        self._exception = exception
        if self.is_running(True):
            # wait for a short 5 seconds for it to finish
            for _cnt in range(0, 5):
                if not self._cancel():
                    time.sleep(1)
                else:
                    break

        self.notify_done(error=True, run_done_callbacks=run_done_callbacks)


Instruction:
Here you have a function in Python 3, explain what it does
def is_success(self):
        ''' check all sessions to see if they have completed successfully  '''
        for _session in self._sessions.values():
            if not _session.is_success():
                return False
        return True

Response: check all sessions to see if they have completed successfully


Instruction:
Can you implement a function in Python 3 that
runs the function to set the transfer spec on error set associated exception

Response:
def set_transfer_spec(self):
        ''' run the function to set the transfer spec on error set associated exception '''
        _ret = False
        try:
            self._args.transfer_spec_func(self._args)
            _ret = True
        except Exception as ex:
            self.notify_exception(AsperaTransferSpecError(ex), False)
        return _ret


Instruction:
Given the following Python 3 function, write the documentation
def _add_subscribers_for_type(self, callback_type, subscribers, callbacks, **kwargs):
        ''' add a done/queued/progress callback to the appropriate list '''
        for subscriber in subscribers:
            callback_name = 'on_' + callback_type
            if hasattr(subscriber, callback_name):
                _function = functools.partial(getattr(subscriber, callback_name), **kwargs)
                callbacks.append(_function)

Response: add a done or queued progress callback to the appropriate list of callbacks for a specific type of event


Instruction:
How would you code a function in Python 3 to
add a done callback to be invoked when transfer is complete

Response:
def add_done_callback(self, function, **kwargs):
        """Add a done callback to be invoked when transfer is complete """
        with self._callbacks_lock:
            _function = functools.partial(function, **kwargs)
            self._done_callbacks.append(_function)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_subscribers(self, subscribers, **kwargs):
        if subscribers:
            with self._callbacks_lock:
                self._add_subscribers_for_type(
                    'done', subscribers, self._done_callbacks, **kwargs)
                self._add_subscribers_for_type(
                    'queued', subscribers, self._queued_callbacks, **kwargs)
                self._add_subscribers_for_type(
                    'progress', subscribers, self._progress_callbacks, **kwargs)

Response: Add a callbacks to be invoked during transfer


Instruction:
Given the following Python 3 function, write the documentation
def _run_queued_callbacks(self):
        ''' run the init/quued calback when the trasnfer is initiated on apsera '''
        for callback in self._queued_callbacks:
            try:
                callback()
            except Exception as ex:
                logger.error("Exception: %s" % str(ex))

Response: run the init or quued calback when the trasnfer is initiated on apsera


Instruction:
Can you tell what is the following Python 3 function doing
def _run_progress_callbacks(self, bytes_transferred):
        ''' pass the number of bytes process to progress callbacks '''
        if bytes_transferred:
            for callback in self._progress_callbacks:
                try:
                    callback(bytes_transferred=bytes_transferred)
                except Exception as ex:
                    logger.error("Exception: %s" % str(ex))

Response: pass the number of bytes process to progress callbacks


Instruction:
How would you implement a function in Python 3 that
runs the callbacks and remove the callbacks from the internal list so they do not get run again.

Response:
def _run_done_callbacks(self):
        ''' Run the callbacks and remove the callbacks from the internal
            List so they do not get run again if done is notified more than once.
        '''
        with self._callbacks_lock:
            for callback in self._done_callbacks:
                try:
                    callback()
                # We do not want a callback interrupting the process, especially
                # in the failure cleanups. So log and catch, the excpetion.
                except Exception as ex:
                    logger.error("Exception: %s" % str(ex))
                    logger.error("Exception raised in %s." % callback, exc_info=True)

            self._done_callbacks = []


Instruction:
Implement a Python 3 function for
getting the first node with the given role

Response:
def get_node(self, role: str, default=None) -> BioCNode:
        """
        Get the first node with role

        Args:
            role: role
            default: node returned instead of raising StopIteration

        Returns:
            the first node with role
        """
        return next((node for node in self.nodes if node.role == role), default)


Instruction:
Can you generate the documentation for the following Python 3 function
def get_sentence(self, offset: int) -> BioCSentence or None:
        for sentence in self.sentences:
            if sentence.offset == offset:
                return sentence
        return None

Response: Gets the BioCSentence object with the specified offset


Instruction:
Can you write a function in Python 3 where it
gets a passage with the specified offset

Response:
def get_passage(self, offset: int) -> BioCPassage or None:
        """
        Gets passage

        Args:
            offset: passage offset

        Return:
            the passage with specified offset
        """
        for passage in self.passages:
            if passage.offset == offset:
                return passage
        return None


Instruction:
Here you have a function in Python 3, explain what it does
def of(cls, *passages: BioCPassage):
        if len(passages) <= 0:
            raise ValueError("There has to be at least one passage.")
        c = BioCDocument()
        for passage in passages:
            if passage is None:
                raise ValueError('Passage is None')
            c.add_passage(passage)
        return c

Response: Returns a collection of passages.


Instruction:
Write a Python 3 function for
returning a collection of documents.

Response:
def of(cls, *documents: BioCDocument):
        """
        Returns a collection documents
        """
        if len(documents) <= 0:
            raise ValueError("There has to be at least one document.")
        c = BioCCollection()
        for document in documents:
            if document is None:
                raise ValueError('Document is None')
            c.add_document(document)
        return c


Instruction:
Explain what the following Python 3 code does
def add_it(workbench, file_list, labels):
    md5s = []
    for filename in file_list:
        if filename != '.DS_Store':
            with open(filename, 'rb') as pe_file:
                base_name = os.path.basename(filename)
                md5 = workbench.store_sample(pe_file.read(), base_name, 'exe')
                workbench.add_node(md5, md5[:6], labels)
                md5s.append(md5)
    return md5s

Response: Add the given file_list to workbench as samples also add them as nodes.


Instruction:
How would you explain what the following Python 3 function does
def jaccard_sims(feature_list):

    sim_info_list = []
    for feature_info in feature_list:
        md5_source = feature_info['md5']
        features_source = feature_info['features']
        for feature_info in feature_list:
            md5_target = feature_info['md5']
            features_target = feature_info['features']
            if md5_source == md5_target:
                continue
            sim = jaccard_sim(features_source, features_target)
            if sim > .5:
                sim_info_list.append({'source': md5_source, 'target': md5_target, 'sim': sim})

    return sim_info_list

Response: Compute Jaccard similarities between all the observations in the feature list.


Instruction:
Make a summary of the following Python 3 code
def jaccard_sim(features1, features2):
    set1 = set(features1)
    set2 = set(features2)
    try:
        return len(set1.intersection(set2))/float(max(len(set1), len(set2)))
    except ZeroDivisionError:
        return 0

Response: Compute similarity between two sets using Jaccard similarity.


Instruction:
Write a Python 3 script for
padding a text until the given length.

Response:
def pad_char(text: str, width: int, char: str = '\n') -> str:
    """Pads a text until length width."""
    dis = width - len(text)
    if dis < 0:
        raise ValueError
    if dis > 0:
        text += char * dis
    return text


Instruction:
Write a Python 3 function for
returning the text of the most recent occurrence of a single object in the document with its offset in the document.

Response:
def get_text(obj) -> Tuple[int, str]:
    """
    Return text with its offset in the document

    Args:
        obj: BioCDocument, BioCPassage, or BioCSentence

    Returns:
        offset, text
    """
    from bioc.bioc import BioCDocument, BioCPassage, BioCSentence

    if isinstance(obj, BioCSentence):
        return obj.offset, obj.text
    if isinstance(obj, BioCPassage):
        if obj.text:
            return obj.offset, obj.text
        text = ''
        for sentence in obj.sentences:
            try:
                text = pad_char(text, sentence.offset - obj.offset, ' ')
                assert sentence.text, f'BioC sentence has no text: {sentence.offset}'
                text += sentence.text
            except ValueError:
                raise ValueError(f'Overlapping sentences {sentence.offset}')
        return obj.offset, text
    if isinstance(obj, BioCDocument):
        text = ''
        for passage in obj.passages:
            try:
                text = pad_char(text, passage.offset)
                text += get_text(passage)[1]
            except ValueError:
                raise ValueError(f'{obj.id}: overlapping passages {passage.offset}')
        return 0, text
    raise TypeError(f'Object of type {obj.__class__.__name__} must be BioCCollection, '
                    f'BioCDocument, BioCPassage, or BioCSentence')


Instruction:
Can you generate the documentation for the following Python 3 function
def shorten_text(text: str):
    if len(text) <= 40:
        text = text
    else:
        text = text[:17] + ' ... ' + text[-17:]
    return repr(text)

Response: Return a short repr of text if it is longer than 40


Instruction:
Explain what the following Python 3 code does
def execute(self, input_data):
        ''' This worker computes meta data for any file type. '''
        raw_bytes = input_data['sample']['raw_bytes']
        self.meta['md5'] = hashlib.md5(raw_bytes).hexdigest()
        self.meta['tags'] = input_data['tags']['tags']
        self.meta['type_tag'] = input_data['sample']['type_tag']
        with magic.Magic() as mag:
            self.meta['file_type'] = mag.id_buffer(raw_bytes[:1024])
        with magic.Magic(flags=magic.MAGIC_MIME_TYPE) as mag:
            self.meta['mime_type'] = mag.id_buffer(raw_bytes[:1024])
        with magic.Magic(flags=magic.MAGIC_MIME_ENCODING) as mag:
            try:
                self.meta['encoding'] = mag.id_buffer(raw_bytes[:1024])
            except magic.MagicError:
                self.meta['encoding'] = 'unknown'
        self.meta['file_size'] = len(raw_bytes)
        self.meta['filename'] = input_data['sample']['filename']
        self.meta['import_time'] = input_data['sample']['import_time']
        self.meta['customer'] = input_data['sample']['customer']
        self.meta['length'] = input_data['sample']['length']

        return self.meta

Response: This worker computes meta data for any file type.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def execute(self, input_data):
        ''' Execute the Strings worker '''
        raw_bytes = input_data['sample']['raw_bytes']
        strings = self.find_strings.findall(raw_bytes)
        return {'string_list': strings}

Response: Execute the Strings worker


Instruction:
Given the following Python 3 function, write the documentation
def execute(self, input_data):
        ''' Execute the PEIndicators worker '''
        raw_bytes = input_data['sample']['raw_bytes']

        # Analyze the output of pefile for any anomalous conditions.
        # Have the PE File module process the file
        try:
            self.pefile_handle = pefile.PE(data=raw_bytes, fast_load=False)
        except (AttributeError, pefile.PEFormatError), error:
            return {'error': str(error), 'indicator_list': [{'Error': 'PE module failed!'}]}

        indicators = []
        indicators += [{'description': warn, 'severity': 2, 'category': 'PE_WARN'}
                       for warn in self.pefile_handle.get_warnings()]

        # Automatically invoke any method of this class that starts with 'check'
        check_methods = self._get_check_methods()
        for check_method in check_methods:
            hit_data = check_method()
            if hit_data:
                indicators.append(hit_data)

        return {'indicator_list': indicators}

Response: Execute the PEIndicators worker


Instruction:
Create a Python 3 function for
checking for a checksum that doesn t match the generated checksum

Response:
def check_checksum_mismatch(self):
        ''' Checking for a checksum that doesn't match the generated checksum '''
        if self.pefile_handle.OPTIONAL_HEADER:
            if self.pefile_handle.OPTIONAL_HEADER.CheckSum != self.pefile_handle.generate_checksum():
                return {'description': 'Reported Checksum does not match actual checksum',
                        'severity': 2, 'category': 'MALFORMED'}
        return None


Instruction:
Implement a function in Python 3 to
check if a non - standard section name is present in the PE file.

Response:
def check_nonstandard_section_name(self):
        ''' Checking for an non-standard section name '''
        std_sections = ['.text', '.bss', '.rdata', '.data', '.rsrc', '.edata', '.idata',
                        '.pdata', '.debug', '.reloc', '.stab', '.stabstr', '.tls',
                        '.crt', '.gnu_deb', '.eh_fram', '.exptbl', '.rodata']
        for i in range(200):
            std_sections.append('/'+str(i))
        non_std_sections = []
        for section in self.pefile_handle.sections:
            name = convert_to_ascii_null_term(section.Name).lower()
            if (name not in std_sections):
                non_std_sections.append(name)
        if non_std_sections:
            return{'description': 'Section(s) with a non-standard name, tamper indication',
                   'severity': 3, 'category': 'MALFORMED', 'attributes': non_std_sections}

        return None


Instruction:
How would you implement a function in Python 3 that
checks if the reported image size matches the actual image size

Response:
def check_image_size_incorrect(self):
        ''' Checking if the reported image size matches the actual image size '''
        last_virtual_address = 0
        last_virtual_size = 0

        section_alignment = self.pefile_handle.OPTIONAL_HEADER.SectionAlignment
        total_image_size = self.pefile_handle.OPTIONAL_HEADER.SizeOfImage

        for section in self.pefile_handle.sections:
            if section.VirtualAddress > last_virtual_address:
                last_virtual_address = section.VirtualAddress
                last_virtual_size = section.Misc_VirtualSize

        # Just pad the size to be equal to the alignment and check for mismatch
        last_virtual_size += section_alignment - (last_virtual_size % section_alignment)
        if (last_virtual_address + last_virtual_size) != total_image_size:
            return {'description': 'Image size does not match reported size',
                    'severity': 3, 'category': 'MALFORMED'}

        return None


Instruction:
Can you tell what is the following Python 3 function doing
def check_section_unaligned(self):
        ''' Checking if any of the sections are unaligned '''
        file_alignment = self.pefile_handle.OPTIONAL_HEADER.FileAlignment
        unaligned_sections = []
        for section in self.pefile_handle.sections:
            if section.PointerToRawData % file_alignment:
                unaligned_sections.append(section.Name)

        # If we had any unaligned sections, return them
        if unaligned_sections:
            return {'description': 'Unaligned section, tamper indication',
                    'severity': 3, 'category': 'MALFORMED', 'attributes': unaligned_sections}
        return None

Response: Checks if any of the sections are unaligned


Instruction:
Can you tell what is the following Python 3 function doing
def check_section_oversized(self):
        ''' Checking if any of the sections go past the total size of the image '''
        total_image_size = self.pefile_handle.OPTIONAL_HEADER.SizeOfImage

        for section in self.pefile_handle.sections:
            if section.PointerToRawData + section.SizeOfRawData > total_image_size:
                return {'description': 'Oversized section, storing addition data within the PE',
                        'severity': 3, 'category': 'MALFORMED', 'attributes': section.Name}

        return None

Response: Checks if any of the sections go past the total size of the image


Instruction:
Write a Python 3 function for
searching for all symbols that match a pattern in the list of matches.

Response:
def _search_for_import_symbols(self, matches):
        ''' Just encapsulating a search that takes place fairly often '''

        # Sanity check
        if not hasattr(self.pefile_handle, 'DIRECTORY_ENTRY_IMPORT'):
            return []

        # Find symbols that match
        pattern = '|'.join(re.escape(match) for match in matches)
        exp = re.compile(pattern)
        symbol_list = []
        for module in self.pefile_handle.DIRECTORY_ENTRY_IMPORT:
            for symbol in module.imports:
                if (symbol.name):
                    symbol_list.append(symbol.name.lower())
        symbol_matches = []
        for symbol in symbol_list:
            if exp.search(symbol):
                symbol_matches.append(symbol)
        return symbol_matches


Instruction:
Given the following Python 3 function, write the documentation
def _search_for_export_symbols(self, matches):
        ''' Just encapsulating a search that takes place fairly often '''
        pattern = '|'.join(re.escape(match) for match in matches)
        exp = re.compile(pattern)
        symbol_list = []
        try:
            for symbol in self.pefile_handle.DIRECTORY_ENTRY_EXPORT.symbols:
                if symbol.name:
                    symbol_list.append(symbol.name.lower())
            symbol_matches = []
            for symbol in symbol_list:
                if exp.search(symbol):
                    symbol_matches.append(symbol)
            return symbol_matches
        except AttributeError:
            return []

Response: This method uses a regular expression to search for export symbols.


Instruction:
Write a Python 3 function that can
return an iterator over the annotations of a given document or a collection of documents.

Response:
def annotations(obj: BioCCollection or BioCDocument or BioCPassage or BioCSentence,
                docid: str = None, level: int = PASSAGE) -> Generator[BioCAnnotation, None, None]:
    """
    Get all annotations in document id.

    Args:
        obj: BioCCollection, BioCDocument, BioCPassage, or BioCSentence
        docid: document id. If None, all documents
        level: one of DOCUMENT, PASSAGE, SENTENCE

    Yields:
        one annotation
    """
    if isinstance(obj, BioCCollection):
        for document in filter(lambda d: docid is None or docid == d.id, obj.documents):
            yield from annotations(document, level=level)
    elif isinstance(obj, BioCDocument):
        if level == DOCUMENT:
            yield from obj.annotations
        elif level in (PASSAGE, SENTENCE):
            for passage in obj.passages:
                yield from annotations(passage, level=level)
        else:
            raise ValueError('level must be DOCUMENT, PASSAGE, or SENTENCE')
    elif isinstance(obj, BioCPassage):
        if level == PASSAGE:
            yield from obj.annotations
        elif level == SENTENCE:
            for sentence in obj.sentences:
                yield from annotations(sentence, level=level)
        else:
            raise ValueError('level must be PASSAGE or SENTENCE')
    elif isinstance(obj, BioCSentence):
        if level == SENTENCE:
            yield from obj.annotations
        else:
            raise ValueError('level must be SENTENCE')
    else:
        raise TypeError(f'Object of type {obj.__class__.__name__} must be BioCCollection, '
                        f'BioCDocument, BioCPassage, or BioCSentence')


Instruction:
Can you create a Python 3 function that
returns an iterator over all sentences in a document id.

Response:
def sentences(obj: BioCCollection or BioCDocument or BioCPassage) \
        -> Generator[BioCSentence, None, None]:
    """
    Get all sentences in document id.

    Args:
        obj: BioCCollection, BioCDocument, or BioCPassage

    Yields:
        one sentence
    """
    if isinstance(obj, BioCCollection):
        for document in obj.documents:
            yield from sentences(document)
    elif isinstance(obj, BioCDocument):
        for passage in obj.passages:
            yield from sentences(passage)
    elif isinstance(obj, BioCPassage):
        yield from obj.sentences
    else:
        raise TypeError(f'Object of type {obj.__class__.__name__} must be BioCCollection, '
                        f'BioCDocument, BioCPassage, or BioCSentence')


Instruction:
Explain what the following Python 3 code does
def execute(self, input_data):
        ''' This worker classifies PEFiles as Evil or AOK  (TOY not a real classifier at this point)'''

        # In general you'd do something different with these two outputs
        # for this toy example will just smash them in a big string
        pefile_output = input_data['pe_features']
        indicators = input_data['pe_indicators']
        all_input = str(pefile_output) + str(indicators)

        flag = 'Reported Checksum does not match actual checksum'
        if flag in all_input:
            self.output['classification'] = 'Toy/Fake Classifier says Evil!'

        return self.output

Response: This worker classifies PEFiles as Evil or AOK


Instruction:
How would you implement a function in Python 3 that
serializes a collection as a BioC formatted stream to fp.

Response:
def dump(collection: BioCCollection, fp, pretty_print: bool = True):
    """
    Serialize ``collection`` as a BioC formatted stream to ``fp``.

    Args:
        collection: the BioC collection
        fp: a ``.write()``-supporting file-like object
        pretty_print: enables formatted XML
    """
    fp.write(dumps(collection, pretty_print))


Instruction:
How would you code a function in Python 3 to
serialize a BioC collection to a BioC formatted str.

Response:
def dumps(collection: BioCCollection, pretty_print: bool = True) -> str:
    """
    Serialize ``collection`` to a BioC formatted ``str``.

    Args:
        collection: the BioC collection
        pretty_print: enables formatted XML

    Returns:
        a BioC formatted ``str``
    """
    doc = etree.ElementTree(BioCXMLEncoder().encode(collection))
    s = etree.tostring(doc, pretty_print=pretty_print, encoding=collection.encoding,
                       standalone=collection.standalone)
    return s.decode(collection.encoding)


Instruction:
Can you tell what is the following Python 3 function doing
def encode_location(location: BioCLocation):
    return etree.Element('location',
                         {'offset': str(location.offset), 'length': str(location.length)})

Response: Encode a single location.


Instruction:
How would you code a function in Python 3 to
encode a single relation.

Response:
def encode_relation(relation: BioCRelation):
    """Encode a single relation."""
    tree = etree.Element('relation', {'id': relation.id})
    encode_infons(tree, relation.infons)
    for node in relation.nodes:
        tree.append(encode_node(node))
    return tree


Instruction:
Here you have a function in Python 3, explain what it does
def encode_annotation(annotation):
    tree = etree.Element('annotation', {'id': annotation.id})
    encode_infons(tree, annotation.infons)
    for location in annotation.locations:
        tree.append(encode_location(location))
    etree.SubElement(tree, 'text').text = annotation.text
    return tree

Response: Encode a single annotation.


Instruction:
How would you implement a function in Python 3 that
encodes a single sentence.

Response:
def encode_sentence(sentence):
    """Encode a single sentence."""
    tree = etree.Element('sentence')
    encode_infons(tree, sentence.infons)
    etree.SubElement(tree, 'offset').text = str(sentence.offset)
    if sentence.text:
        etree.SubElement(tree, 'text').text = sentence.text
    for ann in sentence.annotations:
        tree.append(encode_annotation(ann))
    for rel in sentence.relations:
        tree.append(encode_relation(rel))
    return tree


Instruction:
Write a Python 3 function that can
encode a single passage.

Response:
def encode_passage(passage):
    """Encode a single passage."""
    tree = etree.Element('passage')
    encode_infons(tree, passage.infons)
    etree.SubElement(tree, 'offset').text = str(passage.offset)
    if passage.text:
        etree.SubElement(tree, 'text').text = passage.text
    for sen in passage.sentences:
        tree.append(encode_sentence(sen))
    for ann in passage.annotations:
        tree.append(encode_annotation(ann))
    for rel in passage.relations:
        tree.append(encode_relation(rel))
    return tree


Instruction:
How would you explain what the following Python 3 function does
def encode_document(document):
    tree = etree.Element('document')
    etree.SubElement(tree, 'id').text = document.id
    encode_infons(tree, document.infons)
    for passage in document.passages:
        tree.append(encode_passage(passage))
    for ann in document.annotations:
        tree.append(encode_annotation(ann))
    for rel in document.relations:
        tree.append(encode_relation(rel))
    return tree

Response: Encode a single document.


Instruction:
Can you implement a function in Python 3 that
encodes a single collection.

Response:
def encode_collection(collection):
    """Encode a single collection."""
    tree = etree.Element('collection')
    etree.SubElement(tree, 'source').text = collection.source
    etree.SubElement(tree, 'date').text = collection.date
    etree.SubElement(tree, 'key').text = collection.key
    encode_infons(tree, collection.infons)
    for doc in collection.documents:
        tree.append(encode_document(doc))
    return tree


Instruction:
Explain what the following Python 3 code does
def default(self, obj):
        if isinstance(obj, BioCDocument):
            return encode_document(obj)
        if isinstance(obj, BioCCollection):
            return encode_collection(obj)
        raise TypeError(f'Object of type {obj.__class__.__name__} is not BioC XML serializable')

Response: Implement this method in a subclass such that it returns a tree for o.


Instruction:
Make a summary of the following Python 3 code
def write_collection_info(self, collection: BioCCollection):
        elem = etree.Element('source')
        elem.text = collection.source
        self.__writer.send(elem)

        elem = etree.Element('date')
        elem.text = collection.date
        self.__writer.send(elem)

        elem = etree.Element('key')
        elem.text = collection.key
        self.__writer.send(elem)

        for k, v in collection.infons.items():
            elem = etree.Element('infon', {'key': str(k)})
            elem.text = str(v)
            self.__writer.send(elem)

Response: Writes the collection information to the underlying XML file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def write_document(self, document: BioCDocument):
        tree = self.encoder.encode(document)
        self.__writer.send(tree)

Response: Encode and write a single document.


Instruction:
Make a summary of the following Python 3 code
def run():

    # Grab server args
    args = client_helper.grab_server_args()

    # Start up workbench connection
    workbench = zerorpc.Client(timeout=300, heartbeat=60)
    workbench.connect('tcp://'+args['server']+':'+args['port'])

    all_set = workbench.generate_sample_set()
    results = workbench.set_work_request('view_customer', all_set)
    for customer in results:
        print customer['customer']

Response: This client generates customer reports on all the samples in workbench.


Instruction:
Can you create a Python 3 function that
checks if a file or folder exists and has a given IO access

Response:
def check_io_access(ioobj, access, is_file=False):
    ''' check if a file/folder exists and has a given IO access '''
    if ((is_file and not os.path.isfile(ioobj)) or
        (not is_file and not os.path.isdir(ioobj)) or
            not os.access(ioobj, access)):
            _objtype = "File" if is_file else "Directory"
            raise IOError("Error accessing %s: %s" % (_objtype, ioobj))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def validate(collection, onerror: Callable[[str, List], None] = None):
    BioCValidator(onerror).validate(collection)

Response: Validate BioC data structure.


Instruction:
Can you generate the documentation for the following Python 3 function
def validate_doc(self, document: BioCDocument):
        annotations = []
        annotations.extend(document.annotations)
        annotations.extend(document.relations)
        for passage in document.passages:
            annotations.extend(passage.annotations)
            annotations.extend(passage.relations)
            for sentence in passage.sentences:
                annotations.extend(sentence.annotations)
                annotations.extend(sentence.relations)

        self.current_docid = document.id
        self.traceback.append(document)

        text = self.__get_doc_text(document)
        self.__validate_ann(document.annotations, text, 0)
        self.__validate_rel(annotations, document.relations, f'document {document.id}')

        for passage in document.passages:
            self.traceback.append(passage)

            text = self.__get_passage_text(passage)
            self.__validate_ann(passage.annotations, text, passage.offset)
            self.__validate_rel(annotations, passage.relations,
                                f'document {document.id} --> passage {passage.offset}')

            for sentence in passage.sentences:
                self.traceback.append(sentence)
                self.__validate_ann(sentence.annotations, sentence.text, sentence.offset)
                self.__validate_rel(annotations, sentence.relations,
                                    f'document {document.id} --> sentence {sentence.offset}')
                self.traceback.pop()
            self.traceback.pop()
        self.traceback.pop()

Response: Validate a single document.


Instruction:
Implement a function in Python 3 to
validate a single collection.

Response:
def validate(self, collection: BioCCollection):
        """Validate a single collection."""
        for document in collection.documents:
            self.validate_doc(document)


Instruction:
How would you explain what the following Python 3 function does
def execute(self, input_data):
        ''' Execute method '''

        # Spin up the rekall adapter
        adapter = RekallAdapter()
        adapter.set_plugin_name(self.plugin_name)

        # Create a temporary directory and run this plugin from there
        with self.goto_temp_directory():

            # Run the procdump plugin
            rekall_output = adapter.execute(input_data)

            # Process the output data
            for line in rekall_output:

                if line['type'] == 'm':  # Meta
                    self.output['meta'] = line['data']
                elif line['type'] == 't': # New Table Headers (column names)
                    self.column_map = {item['cname']: item['name'] if 'name' in item else item['cname'] for item in line['data']}
                elif line['type'] == 'r': # Row

                    # Add the row to our current table
                    row = RekallAdapter.process_row(line['data'], self.column_map)
                    self.output['tables'][self.current_table_name].append(row)

            # Scrape any extracted files
            print 'mem_procdump: Scraping dumped files...'
            for output_file in glob.glob('*'):

                # Store the output into workbench, put md5s in the 'dumped_files' field
                output_name = os.path.basename(output_file)
                output_name = output_name.replace('executable.', '')
                with open(output_file, 'rb') as dumped_file:
                    raw_bytes = dumped_file.read()
                    md5 = self.c.store_sample(raw_bytes, output_name, 'exe')

                    # Remove some columns from meta data
                    meta = self.c.work_request('meta', md5)['meta']
                    del meta['customer']
                    del meta['encoding']
                    del meta['import_time']
                    del meta['mime_type']
                    self.output['tables'][self.current_table_name].append(meta)

            # All done
            return self.output

Response: Execute the procdump plugin and store the output into the output field of the current table.


Instruction:
Explain what the following Python 3 code does
def convert_to_utf8(string):
    ''' Convert string to UTF8 '''
    if (isinstance(string, unicode)):
        return string.encode('utf-8')
    try:
        u = unicode(string, 'utf-8')
    except TypeError:
        return str(string)
    utf8 = u.encode('utf-8')
    return utf8

Response: Convert string to UTF8


Instruction:
Can you generate the documentation for the following Python 3 function
def execute(self, input_data):

        ''' Process the input bytes with pefile '''
        raw_bytes = input_data['sample']['raw_bytes']

        # Have the PE File module process the file
        pefile_handle, error_str = self.open_using_pefile('unknown', raw_bytes)
        if not pefile_handle:
            return {'error': error_str, 'dense_features': [], 'sparse_features': []}

        # Now extract the various features using pefile
        dense_features, sparse_features = self.extract_features_using_pefile(pefile_handle)

        # Okay set my response
        return {'dense_features': dense_features, 'sparse_features': sparse_features, 'tags': input_data['tags']['tags']}

Response: Process the input bytes with pefile and extract the features


Instruction:
Write a Python 3 function that can
open the PE File using the Python PEFile module.

Response:
def open_using_pefile(input_name, input_bytes):
        ''' Open the PE File using the Python pefile module. '''
        try:
            pef = pefile.PE(data=input_bytes, fast_load=False)
        except (AttributeError, pefile.PEFormatError), error:
            print 'warning: pe_fail (with exception from pefile module) on file: %s' % input_name
            error_str = '(Exception):, %s' % (str(error))
            return None, error_str

        # Now test to see if the features are there/extractable if not return FAIL flag
        if pef.PE_TYPE is None or pef.OPTIONAL_HEADER is None or len(pef.OPTIONAL_HEADER.DATA_DIRECTORY) < 7:
            print 'warning: pe_fail on file: %s' % input_name
            error_str = 'warning: pe_fail on file: %s' % input_name
            return None, error_str

        # Success
        return pef, None


Instruction:
How would you explain what the following Python 3 function does
def extract_features_using_pefile(self, pef):
        ''' Process the PE File using the Python pefile module. '''

        # Store all extracted features into feature lists
        extracted_dense = {}
        extracted_sparse = {}

        # Now slog through the info and extract the features
        feature_not_found_flag = -99
        feature_default_value = 0
        self._warnings = []

        # Set all the dense features and sparse features to 'feature not found'
        # value and then check later to see if it was found
        for feature in self._dense_feature_list:
            extracted_dense[feature] = feature_not_found_flag
        for feature in self._sparse_feature_list:
            extracted_sparse[feature] = feature_not_found_flag

        # Check to make sure all the section names are standard
        std_sections = ['.text', '.bss', '.rdata', '.data', '.rsrc', '.edata', '.idata',
                        '.pdata', '.debug', '.reloc', '.stab', '.stabstr', '.tls',
                        '.crt', '.gnu_deb', '.eh_fram', '.exptbl', '.rodata']
        for i in range(200):
            std_sections.append('/'+str(i))
        std_section_names = 1
        extracted_sparse['section_names'] = []
        for section in pef.sections:
            name = convert_to_ascii_null_term(section.Name).lower()
            extracted_sparse['section_names'].append(name)
            if name not in std_sections:
                std_section_names = 0

        extracted_dense['std_section_names'] = std_section_names
        extracted_dense['debug_size'] = pef.OPTIONAL_HEADER.DATA_DIRECTORY[6].Size
        extracted_dense['major_version'] = pef.OPTIONAL_HEADER.MajorImageVersion
        extracted_dense['minor_version'] = pef.OPTIONAL_HEADER.MinorImageVersion
        extracted_dense['iat_rva']	 = pef.OPTIONAL_HEADER.DATA_DIRECTORY[1].VirtualAddress
        extracted_dense['export_size'] = pef.OPTIONAL_HEADER.DATA_DIRECTORY[0].Size
        extracted_dense['check_sum'] = pef.OPTIONAL_HEADER.CheckSum
        try:
            extracted_dense['generated_check_sum'] = pef.generate_checksum()
        except ValueError:
            extracted_dense['generated_check_sum'] = 0
        if len(pef.sections) > 0:
            extracted_dense['virtual_address'] = pef.sections[0].VirtualAddress
            extracted_dense['virtual_size'] = pef.sections[0].Misc_VirtualSize
        extracted_dense['number_of_sections'] = pef.FILE_HEADER.NumberOfSections
        extracted_dense['compile_date'] = pef.FILE_HEADER.TimeDateStamp
        extracted_dense['number_of_rva_and_sizes'] = pef.OPTIONAL_HEADER.NumberOfRvaAndSizes
        extracted_dense['total_size_pe'] = len(pef.__data__)

        # Number of import and exports
        if hasattr(pef, 'DIRECTORY_ENTRY_IMPORT'):
            extracted_dense['number_of_imports'] = len(pef.DIRECTORY_ENTRY_IMPORT)
            num_imported_symbols = 0
            for module in pef.DIRECTORY_ENTRY_IMPORT:
                num_imported_symbols += len(module.imports)
            extracted_dense['number_of_import_symbols'] = num_imported_symbols

        if hasattr(pef, 'DIRECTORY_ENTRY_BOUND_IMPORT'):
            extracted_dense['number_of_bound_imports'] = len(pef.DIRECTORY_ENTRY_BOUND_IMPORT)
            num_imported_symbols = 0
            for module in pef.DIRECTORY_ENTRY_BOUND_IMPORT:
                num_imported_symbols += len(module.entries)
            extracted_dense['number_of_bound_import_symbols'] = num_imported_symbols

        if hasattr(pef, 'DIRECTORY_ENTRY_EXPORT'):
            try:
                extracted_dense['number_of_export_symbols'] = len(pef.DIRECTORY_ENTRY_EXPORT.symbols)
                symbol_set = set()
                for symbol in pef.DIRECTORY_ENTRY_EXPORT.symbols:
                    symbol_info = 'unknown'
                    if not symbol.name:
                        symbol_info = 'ordinal=' + str(symbol.ordinal)
                    else:
                        symbol_info = 'name=' + symbol.name
                    symbol_set.add(convert_to_utf8('%s' % (symbol_info)).lower())

                # Now convert set to list and add to features
                extracted_sparse['ExportedSymbols'] = list(symbol_set)

            except AttributeError:
                extracted_sparse['ExportedSymbols'] = ['AttributeError']

        # Specific Import info (Note this will be a sparse field woo hoo!)
        if hasattr(pef, 'DIRECTORY_ENTRY_IMPORT'):
            symbol_set = set()
            for module in pef.DIRECTORY_ENTRY_IMPORT:
                for symbol in module.imports:
                    symbol_info = 'unknown'
                    if symbol.import_by_ordinal is True:
                        symbol_info = 'ordinal=' + str(symbol.ordinal)
                    else:
                        symbol_info = 'name=' + symbol.name
                        # symbol_info['hint'] = symbol.hint
                    if symbol.bound:
                        symbol_info += ' bound=' + str(symbol.bound)

                    symbol_set.add(convert_to_utf8('%s:%s' % (module.dll, symbol_info)).lower())

            # Now convert set to list and add to features
            extracted_sparse['imported_symbols'] = list(symbol_set)

        # Do we have a second section
        if len(pef.sections) >= 2:
            extracted_dense['virtual_size_2'] = pef.sections[1].Misc_VirtualSize

        extracted_dense['size_image'] = pef.OPTIONAL_HEADER.SizeOfImage
        extracted_dense['size_code'] = pef.OPTIONAL_HEADER.SizeOfCode
        extracted_dense['size_initdata'] = pef.OPTIONAL_HEADER.SizeOfInitializedData
        extracted_dense['size_uninit'] = pef.OPTIONAL_HEADER.SizeOfUninitializedData
        extracted_dense['pe_majorlink'] = pef.OPTIONAL_HEADER.MajorLinkerVersion
        extracted_dense['pe_minorlink'] = pef.OPTIONAL_HEADER.MinorLinkerVersion
        extracted_dense['pe_driver'] = 1 if pef.is_driver() else 0
        extracted_dense['pe_exe'] = 1 if pef.is_exe() else 0
        extracted_dense['pe_dll'] = 1 if pef.is_dll() else 0
        extracted_dense['pe_i386'] = 1
        if pef.FILE_HEADER.Machine != 0x014c:
            extracted_dense['pe_i386'] = 0
        extracted_dense['pe_char'] = pef.FILE_HEADER.Characteristics

        # Data directory features!!
        datadirs = {
            0: 'IMAGE_DIRECTORY_ENTRY_EXPORT', 1: 'IMAGE_DIRECTORY_ENTRY_IMPORT',
            2: 'IMAGE_DIRECTORY_ENTRY_RESOURCE', 5: 'IMAGE_DIRECTORY_ENTRY_BASERELOC',
            12: 'IMAGE_DIRECTORY_ENTRY_IAT'}
        for idx, datadir in datadirs.items():
            datadir = pefile.DIRECTORY_ENTRY[idx]
            if len(pef.OPTIONAL_HEADER.DATA_DIRECTORY) <= idx:
                continue

            directory = pef.OPTIONAL_HEADER.DATA_DIRECTORY[idx]
            extracted_dense['datadir_%s_size' % datadir] = directory.Size

        # Section features
        section_flags = ['IMAGE_SCN_MEM_EXECUTE', 'IMAGE_SCN_CNT_CODE', 'IMAGE_SCN_MEM_WRITE', 'IMAGE_SCN_MEM_READ']
        rawexecsize = 0
        vaexecsize = 0
        for sec in pef.sections:
            if not sec:
                continue

            for char in section_flags:
                # does the section have one of our attribs?
                if hasattr(sec, char):
                    rawexecsize += sec.SizeOfRawData
                    vaexecsize += sec.Misc_VirtualSize
                    break

            # Take out any weird characters in section names
            secname = convert_to_ascii_null_term(sec.Name).lower()
            secname = secname.replace('.', '')
            if secname in std_sections:
                extracted_dense['sec_entropy_%s' % secname] = sec.get_entropy()
                extracted_dense['sec_rawptr_%s' % secname] = sec.PointerToRawData
                extracted_dense['sec_rawsize_%s' % secname] = sec.SizeOfRawData
                extracted_dense['sec_vasize_%s' % secname] = sec.Misc_VirtualSize

        extracted_dense['sec_va_execsize'] = vaexecsize
        extracted_dense['sec_raw_execsize'] = rawexecsize

        # Imphash (implemented in pefile 1.2.10-139 or later)
        try:
            extracted_sparse['imp_hash'] = pef.get_imphash()
        except AttributeError:
            extracted_sparse['imp_hash'] = 'Not found: Install pefile 1.2.10-139 or later'

        # Register if there were any pe warnings
        warnings = pef.get_warnings()
        if warnings:
            extracted_dense['pe_warnings'] = 1
            extracted_sparse['pe_warning_strings'] = warnings
        else:
            extracted_dense['pe_warnings'] = 0

        # Issue a warning if the feature isn't found
        for feature in self._dense_feature_list:
            if extracted_dense[feature] == feature_not_found_flag:
                extracted_dense[feature] = feature_default_value
                if (self._verbose):
                    print 'info: Feature: %s not found! Setting to %d' % (feature, feature_default_value)

        # Issue a warning if the feature isn't found
        for feature in self._sparse_feature_list:
            if extracted_sparse[feature] == feature_not_found_flag:
                extracted_sparse[feature] = []  # For sparse data probably best default
                if (self._verbose):
                    print 'info: Feature: %s not found! Setting to %d' % (feature, feature_default_value)

        # Set the features for the class var
        self._dense_features = extracted_dense
        self._sparse_features = extracted_sparse

        return self.get_dense_features(), self.get_sparse_features()

Response: Process the PE File using the Python PEFile module.


Instruction:
Make a summary of the following Python 3 code
def _parse_bro_header(self, logfile):

        # Skip until you find the #fields line
        _line = next(logfile)
        while (not _line.startswith('#fields')):
            _line = next(logfile)

        # Read in the field names
        _field_names = _line.strip().split(self.delimiter)[1:]

        # Read in the types
        _line = next(logfile)
        _field_types = _line.strip().split(self.delimiter)[1:]

        # Return the header info
        return _field_names, _field_types

Response: This method tries to parse the Bro log header section.


Instruction:
Can you tell what is the following Python 3 function doing
def _cast_dict(self, data_dict):
        for key, value in data_dict.iteritems():
            data_dict[key] = self._cast_value(value)

        # Fixme: resp_body_data can be very large so removing it for now
        if 'resp_body_data' in data_dict:
            del data_dict['resp_body_data']

        return data_dict

Response:
Internal method that makes sure any dictionary elements
        are properly cast into the correct types.


Instruction:
How would you explain what the following Python 3 function does
def run():

    # Grab server args
    args = client_helper.grab_server_args()

    # Start up workbench connection
    workbench = zerorpc.Client(timeout=300, heartbeat=60)
    workbench.connect('tcp://'+args['server']+':'+args['port'])

    # Test out zip data
    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/zip')
    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
    for filename in file_list:
        with open(filename,'rb') as f:
            base_name = os.path.basename(filename)
            md5 = workbench.store_sample(f.read(), base_name, 'zip')
            results = workbench.work_request('view', md5)
            print 'Filename: %s ' % (base_name)
            pprint.pprint(results)

            # The unzip worker gives you a list of md5s back
            # Run meta on all the unzipped files.
            results = workbench.work_request('unzip', md5)
            print '\n*** Filename: %s ***' % (base_name)
            for child_md5 in results['unzip']['payload_md5s']:
                pprint.pprint(workbench.work_request('meta', child_md5))

Response: This client shows workbench extacting files from a zip file.


Instruction:
How would you implement a function in Python 3 that
executes the VTQuery worker

Response:
def execute(self, input_data):
        ''' Execute the VTQuery worker '''
        md5 = input_data['meta']['md5']
        response = requests.get('http://www.virustotal.com/vtapi/v2/file/report',
                                params={'apikey':self.apikey,'resource':md5, 'allinfo':1})

        # Make sure we got a json blob back
        try:
            vt_output = response.json()
        except ValueError:
            return {'vt_error': 'VirusTotal Query Error, no valid response... past per min quota?'}

        # Just pull some of the fields
        output = {field:vt_output[field] for field in vt_output.keys() if field not in self.exclude}

        # Check for not-found
        not_found = False if output else True

        # Add in file_type
        output['file_type'] = input_data['meta']['file_type']

        # Toss back a not found
        if not_found:
            output['not_found'] = True
            return output

        # Organize the scans fields
        scan_results = collections.Counter()
        for scan in vt_output['scans'].values():
            if 'result' in scan:
                if scan['result']:
                    scan_results[scan['result']] += 1
        output['scan_results'] = scan_results.most_common(5)
        return output


Instruction:
Implement a function in Python 3 to
grab the peid_userdb. txt file from local disk

Response:
def get_peid_db():
    ''' Grab the peid_userdb.txt file from local disk '''

    # Try to find the yara rules directory relative to the worker
    my_dir = os.path.dirname(os.path.realpath(__file__))
    db_path = os.path.join(my_dir, 'peid_userdb.txt')
    if not os.path.exists(db_path):
        raise RuntimeError('peid could not find peid_userdb.txt under: %s' % db_path)

    # Okay load up signature
    signatures = peutils.SignatureDatabase(data = open(db_path, 'rb').read())
    return signatures


Instruction:
Given the following Python 3 function, write the documentation
def peid_features(self, pefile_handle):
        ''' Get features from PEid signature database'''
        peid_match = self.peid_sigs.match(pefile_handle)
        return peid_match if peid_match else []

Response: Get features from PEid signature database


Instruction:
Make a summary of the following Python 3 code
def run():

    global WORKBENCH

    # Grab grab_server_argsrver args
    args = client_helper.grab_server_args()

    # Start up workbench connection
    WORKBENCH = zerorpc.Client(timeout=300, heartbeat=60)
    WORKBENCH.connect('tcp://'+args['server']+':'+args['port'])

    data_path = os.path.join(
        os.path.dirname(os.path.realpath(__file__)), '../data/pcap')
    file_list = [os.path.join(data_path, child) for child in \
                os.listdir(data_path)]
    results = []
    for filename in file_list:

        # Skip OS generated files
        if '.DS_Store' in filename: continue

        # Process the pcap file
        with open(filename,'rb') as f:
            md5 = WORKBENCH.store_sample(f.read(), filename, 'pcap')
            result = WORKBENCH.work_request('view_pcap', md5)
            result.update(WORKBENCH.work_request('meta', result['view_pcap']['md5']))
            result['filename'] = result['meta']['filename'].split('/')[-1]
            results.append(result)

    return results

Response: This client pulls PCAP files for building report.


Instruction:
How would you explain what the following Python 3 function does
def show_files(md5):
    '''Renders template with `view` of the md5.'''
    if not WORKBENCH:
        return flask.redirect('/')

    md5_view = WORKBENCH.work_request('view', md5)
    return flask.render_template('templates/md5_view.html', md5_view=md5_view['view'], md5=md5)

Response: Renders template with view of the md5.


Instruction:
How would you implement a function in Python 3 that
renders template with stream_sample of the md5.

Response:
def show_md5_view(md5):
    '''Renders template with `stream_sample` of the md5.'''

    if not WORKBENCH:
        return flask.redirect('/')

    md5_view = WORKBENCH.stream_sample(md5)
    return flask.render_template('templates/md5_view.html', md5_view=list(md5_view), md5=md5)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_node(self, node_id, name, labels):
        node = self.graph_db.get_or_create_indexed_node('Node', 'node_id', node_id, {'node_id': node_id, 'name': name})
        try:
            node.add_labels(*labels)
        except NotImplementedError:
            pass

Response: Add the node with name and labels.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def add_rel(self, source_node_id, target_node_id, rel):

        # Add the relationship
        n1_ref = self.graph_db.get_indexed_node('Node', 'node_id', source_node_id)
        n2_ref = self.graph_db.get_indexed_node('Node', 'node_id', target_node_id)

        # Sanity check
        if not n1_ref or not n2_ref:
            print 'Cannot add relationship between unfound nodes: %s --> %s' % (source_node_id, target_node_id)
            return
        path = neo4j.Path(n1_ref, rel, n2_ref)
        path.get_or_create(self.graph_db)

Response: Add a relationship between nodes.


Instruction:
Make a summary of the following Python 3 code
def add_node(self, node_id, name, labels):
        print 'NeoDB Stub getting called...'
        print '%s %s %s %s' % (self, node_id, name, labels)

Response: Add a node to the database.


Instruction:
How would you explain what the following Python 3 function does
def add_rel(self, source_node_id, target_node_id, rel):
        print 'NeoDB Stub getting called...'
        print '%s %s %s %s' % (self, source_node_id, target_node_id, rel)

Response: Add a relationship between two nodes.


Instruction:
How would you explain what the following Python 3 function does
def tail_file(filename):
    ''' Tail a file using pygtail. Note: this could probably be improved '''
    with make_temp_file() as offset_file:
        while True:
            for line in pygtail.Pygtail(filename, offset_file=offset_file):
                yield line
            time.sleep(1.0)

Response: Yields the lines of a file in a generator.


Instruction:
Write a Python 3 script for
executing the ViewPE worker

Response:
def execute(self, input_data):
        ''' Execute the ViewPE worker '''

        # Just a small check to make sure we haven't been called on the wrong file type
        if (input_data['meta']['type_tag'] != 'exe'):
            return {'error': self.__class__.__name__+': called on '+input_data['meta']['type_tag']}

        view = {}
        view['indicators'] = list(set([item['category'] for item in input_data['pe_indicators']['indicator_list']]))
        view['peid_matches'] = input_data['pe_peid']['match_list']
        view['yara_sigs'] = input_data['yara_sigs']['matches'].keys()
        view['classification'] = input_data['pe_classifier']['classification']
        view['disass'] = self.safe_get(input_data, ['pe_disass', 'decode'])[:15]
        view.update(input_data['meta'])

        return view


Instruction:
Here you have a function in Python 3, explain what it does
def safe_get(data, key_list):
        ''' Safely access dictionary keys when plugin may have failed '''
        for key in key_list:
            data = data.get(key, {})
        return data if data else 'plugin_failed'

Response: Safely access dictionary keys when plugin may have failed


Instruction:
Implement a Python 3 function for
beginning capturing PCAPs and sending them to workbench

Response:
def execute(self):
        ''' Begin capturing PCAPs and sending them to workbench '''

        # Create a temporary directory
        self.temp_dir = tempfile.mkdtemp()
        os.chdir(self.temp_dir)

        # Spin up the directory watcher
        DirWatcher(self.temp_dir, self.file_created)

        # Spin up tcpdump
        self.subprocess_manager(self.tcpdump_cmd)


Instruction:
Implement a function in Python 3 to
store a file into workbench

Response:
def store_file(self, filename):
        ''' Store a file into workbench '''

        # Spin up workbench
        self.workbench = zerorpc.Client(timeout=300, heartbeat=60)
        self.workbench.connect("tcp://127.0.0.1:4242")

        # Open the file and send it to workbench
        storage_name = "streaming_pcap" + str(self.pcap_index)
        print filename, storage_name
        with open(filename,'rb') as f:
            self.workbench.store_sample(f.read(), storage_name, 'pcap')
        self.pcap_index += 1

        # Close workbench client
        self.workbench.close()


Instruction:
How would you code a function in Python 3 to
encode a single document.

Response:
def encode_document(obj):
    """Encode a single document."""
    warnings.warn("deprecated. Please use bioc.biocxml.encoder.encode_document", DeprecationWarning)
    return bioc.biocxml.encoder.encode_document(obj)


Instruction:
Can you generate the documentation for the following Python 3 function
def encode_passage(obj):
    warnings.warn("deprecated. Please use bioc.biocxml.encoder.encode_passage", DeprecationWarning)
    return bioc.biocxml.encoder.encode_passage(obj)

Response: Encode a single passage.


Instruction:
Create a Python 3 function for
encoding a single sentence.

Response:
def encode_sentence(obj):
    """Encode a single sentence."""
    warnings.warn("deprecated. Please use bioc.biocxml.encoder.encode_sentence", DeprecationWarning)
    return bioc.biocxml.encoder.encode_sentence(obj)


Instruction:
Here you have a function in Python 3, explain what it does
def encode_annotation(obj):
    warnings.warn("deprecated. Please use bioc.biocxml.encoder.encode_annotation",
                  DeprecationWarning)
    return bioc.biocxml.encoder.encode_annotation(obj)

Response: Encode a single annotation.


Instruction:
How would you explain what the following Python 3 function does
def encode_relation(obj):
    warnings.warn("deprecated. Please use bioc.biocxml.encoder.encode_relation", DeprecationWarning)
    return bioc.biocxml.encoder.encode_relation(obj)

Response: Encode a single relation.


Instruction:
Implement a function in Python 3 to
parse the EProcess object we get from some rekall output

Response:
def parse_eprocess(self, eprocess_data):
        """Parse the EProcess object we get from some rekall output"""
        Name = eprocess_data['_EPROCESS']['Cybox']['Name']
        PID = eprocess_data['_EPROCESS']['Cybox']['PID']
        PPID = eprocess_data['_EPROCESS']['Cybox']['Parent_PID']
        return {'Name': Name, 'PID': PID, 'PPID': PPID}


Instruction:
How would you explain what the following Python 3 function does
def execute(self, input_data):
        ''' Execute the Unzip worker '''
        raw_bytes = input_data['sample']['raw_bytes']
        zipfile_output = zipfile.ZipFile(StringIO(raw_bytes))
        payload_md5s = []
        for name in zipfile_output.namelist():
            filename = os.path.basename(name)
            payload_md5s.append(self.workbench.store_sample(zipfile_output.read(name), name, 'unknown'))
        return {'payload_md5s': payload_md5s}

Response: Execute the Unzip worker


Instruction:
Can you write a function in Python 3 where it
executes the method in the base class.

Response:
def execute(self, input_data):
        ''' Execute method '''

        # Spin up the rekall adapter
        adapter = RekallAdapter()
        adapter.set_plugin_name(self.plugin_name)
        rekall_output = adapter.execute(input_data)

        # Process the output data
        for line in rekall_output:

            if line['type'] == 'm':  # Meta
                self.output['meta'] = line['data']
            elif line['type'] == 's': # New Session (Table)
                if line['data']['name']:
                    self.current_table_name = str(line['data']['name'][1].v())
            elif line['type'] == 't': # New Table Headers (column names)
                self.column_map = {item['cname']: item['name'] if 'name' in item else item['cname'] for item in line['data']}
            elif line['type'] == 'r': # Row

                # Add the row to our current table
                row = RekallAdapter.process_row(line['data'], self.column_map)
                self.output['tables'][self.current_table_name].append(row)

                # Process Base entries
                if 'Base' in row:
                    base_info = self.parse_base(row)
                    row.update(base_info)
            else:
                print 'Got unknown line %s: %s' % (line['type'], line['data'])

        # All done
        return self.output


Instruction:
Write a Python 3 function for
helping on Workbench CLI

Response:
def help_cli(self):
        """ Help on Workbench CLI """
        help = '%sWelcome to Workbench CLI Help:%s' % (color.Yellow, color.Normal)
        help += '\n\t%s> help cli_basic %s for getting started help' % (color.Green, color.LightBlue)
        help += '\n\t%s> help workers %s for help on available workers' % (color.Green, color.LightBlue)
        help += '\n\t%s> help search %s for help on searching samples' % (color.Green, color.LightBlue)
        help += '\n\t%s> help dataframe %s for help on making dataframes' % (color.Green, color.LightBlue)
        help += '\n\t%s> help commands %s for help on workbench commands' % (color.Green, color.LightBlue)
        help += '\n\t%s> help topic %s where topic can be a help, command or worker' % (color.Green, color.LightBlue)
        help += '\n\n%sNote: cli commands are transformed into python calls' % (color.Yellow)
        help += '\n\t%s> help cli_basic --> help("cli_basic")%s' % (color.Green, color.Normal)
        return help


Instruction:
Write a Python 3 script to
help for Workbench CLI Basics

Response:
def help_cli_basic(self):
        """ Help for Workbench CLI Basics """
        help =  '%sWorkbench: Getting started...' % (color.Yellow)
        help += '\n%sLoad in a sample:'  % (color.Green)
        help += '\n\t%s> load_sample /path/to/file' % (color.LightBlue)
        help += '\n\n%sNotice the prompt now shows the md5 of the sample...'% (color.Yellow)
        help += '\n%sRun workers on the sample:'  % (color.Green)
        help += '\n\t%s> view' % (color.LightBlue)
        help += '\n%sType the \'help workers\' or the first part of the worker <tab>...'  % (color.Green)
        help += '\n\t%s> help workers (lists all possible workers)' % (color.LightBlue)
        help += '\n\t%s> pe_<tab> (will give you pe_classifier, pe_deep_sim, pe_features, pe_indicators, pe_peid)%s' % (color.LightBlue, color.Normal)
        return help


Instruction:
Explain what the following Python 3 code does
def help_cli_search(self):
        help =  '%sSearch: %s returns sample_sets, a sample_set is a set/list of md5s.' % (color.Yellow, color.Green)
        help += '\n\n\t%sSearch for all samples in the database that are known bad pe files,'  % (color.Green)
        help += '\n\t%sthis command returns the sample_set containing the matching items'% (color.Green)
        help += '\n\t%s> my_bad_exes = search([\'bad\', \'exe\'])' % (color.LightBlue)
        help += '\n\n\t%sRun workers on this sample_set:'  % (color.Green)
        help += '\n\t%s> pe_outputs = pe_features(my_bad_exes) %s' % (color.LightBlue, color.Normal)
        help += '\n\n\t%sLoop on the generator (or make a DataFrame see >help dataframe)'  % (color.Green)
        help += '\n\t%s> for output in pe_outputs: %s' % (color.LightBlue, color.Normal)
        help += '\n\t\t%s print output %s' % (color.LightBlue, color.Normal)
        return help

Response: Help for Workbench CLI Search


Instruction:
Here you have a function in Python 3, explain what it does
def help_dataframe(self):
        help =  '%sMaking a DataFrame: %s how to make a dataframe from raw data (pcap, memory, pe files)' % (color.Yellow, color.Green)
        help += '\n\t%sNote: for memory_image and pe_files see > help dataframe_memory or dataframe_pe'  % (color.Green)
        help += '\n\n%sPCAP Example:'  % (color.Green)
        help += '\n\t%s> load_sample /path/to/pcap/gold_xxx.pcap [\'bad\', \'threatglass\']' % (color.LightBlue)
        help += '\n\t%s> view     # view is your friend use it often' % (color.LightBlue)
        help += '\n\n%sGrab the http_log from the pcap (also play around with other logs):'  % (color.Green)
        help += '\n\t%s> http_log_md5 = view()[\'view\'][\'bro_logs\'][\'http_log\']' % (color.LightBlue)
        help += '\n\t%s> http_log_md5 (returns the md5 of the http_log)' % (color.LightBlue)
        help += '\n\n%sStream back the ^contents^ of the http_log:'  % (color.Green)
        help += '\n\t%s> http_log = stream_sample(http_log_md5)' % (color.LightBlue)
        help += '\n\n%sPut the http_log into a dataframe:'  % (color.Green)
        help += '\n\t%s> http_df = pd.DataFrame(http_log)' % (color.LightBlue)
        help += '\n\t%s> http_df.head()' % (color.LightBlue)
        help += '\n\t%s> http_df.groupby([\'host\',\'id.resp_h\',\'resp_mime_types\'])[[\'response_body_len\']].sum()' % (color.LightBlue)
        help += '\n\t%s> http_df.describe() %s' % (color.LightBlue, color.Normal)
        return help

Response: Help for making a DataFrame with Workbench CLI


Instruction:
Can you create a Python 3 function that
helps for making a dataframe with Workbench CLI

Response:
def help_dataframe_memory(self):
        """ Help for making a DataFrame with Workbench CLI """
        help =  '%sMaking a DataFrame: %s how to make a dataframe from memory_forensics sample' % (color.Yellow, color.Green)
        help += '\n\n%sMemory Images Example:'  % (color.Green)
        help += '\n\t%s> load_sample /path/to/pcap/exemplar4.vmem [\'bad\', \'aptz13\']' % (color.LightBlue)
        help += '\n\t%s> view # view is your friend use it often' % (color.LightBlue)
        help += '\n\t%s> <<< TODO :) >>> %s' % (color.LightBlue, color.Normal)
        return help


Instruction:
Write a Python 3 function that can
help for making a DataFrame from PE files

Response:
def help_dataframe_pe(self):
        """ Help for making a DataFrame with Workbench CLI """
        help =  '%sMaking a DataFrame: %s how to make a dataframe from pe files' % (color.Yellow, color.Green)
        help += '\n\n%sPE Files Example (loading a directory):'  % (color.Green)
        help += '\n\t%s> load_sample /path/to/pe/bad [\'bad\', \'case_69\']' % (color.LightBlue)
        help += '\n\n\t%sSearch for all samples in the database that are pe files,'  % (color.Green)
        help += '\n\t%sthis command returns the sample_set containing the matching items'% (color.Green)
        help += '\n\t%s> my_exes = search([\'exe\'])' % (color.LightBlue)
        help += '\n\n\t%sRun workers on this sample_set:'  % (color.Green)
        help += '\n\t%s> pe_outputs = set_work_request(\'pe_features\', my_exes, [\'md5\', \'dense_features.*\', \'tags\'])' % (color.LightBlue)
        help += '\n\n\t%sMake a DataFrame:'  % (color.Green)
        help += '\n\t%s> pe_df = pd.DataFrame(pe_outputs) %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.head() %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df = flatten_tags(pe_df) %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.hist(\'check_sum\',\'tags\') %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.bloxplot(\'check_sum\',\'tags\') %s' % (color.LightBlue, color.Normal)
        return help


Instruction:
Make a summary of the following Python 3 code
def _all_help_methods(self):
        methods = {name:method for name, method in inspect.getmembers(self, predicate=inspect.isroutine) if not name.startswith('_')}
        return methods

Response: Returns a list of all the Workbench commands


Instruction:
Can you write a function in Python 3 where it
executes the view_pcap_deep method

Response:
def execute(self, input_data):
        ''' ViewPcapDeep execute method '''

        # Copy info from input
        view = input_data['view_pcap']

        # Grab a couple of handles
        extracted_files = input_data['view_pcap']['extracted_files']

        # Dump a couple of fields
        del view['extracted_files']

        # Grab additional info about the extracted files
        view['extracted_files'] = [self.workbench.work_request('meta_deep', md5,
            ['md5', 'sha256', 'entropy', 'ssdeep', 'file_size', 'file_type']) for md5 in extracted_files]

        return view


Instruction:
Can you tell what is the following Python 3 function doing
def execute(self, input_data):
        ''' Execute Method '''

        # View on all the meta data files in the sample
        fields = ['filename', 'md5', 'length', 'customer', 'import_time', 'type_tag']
        view = {key:input_data['meta'][key] for key in fields}
        return view

Response: Execute method that returns the sample s meta data files


Instruction:
Write a Python 3 function for
executing the action for the info object

Response:
def execute(self, input_data):
        """ Info objects all have a type_tag of ('help','worker','command', or 'other') """
        input_data = input_data['info']
        type_tag = input_data['type_tag']
        if type_tag == 'help':
            return {'help': input_data['help'], 'type_tag': input_data['type_tag']}
        elif type_tag == 'worker':
            out_keys = ['name', 'dependencies', 'docstring', 'type_tag']
            return {key: value for key, value in input_data.iteritems() if key in out_keys}
        elif type_tag == 'command':
            out_keys = ['command', 'sig', 'docstring', 'type_tag']
            return {key: value for key, value in input_data.iteritems() if key in out_keys}
        elif type_tag == 'other':
            return input_data
        else:
            print 'Got a malformed info object %s' % input_data
            return input_data


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse_collection(obj: dict) -> BioCCollection:
    collection = BioCCollection()
    collection.source = obj['source']
    collection.date = obj['date']
    collection.key = obj['key']
    collection.infons = obj['infons']
    for doc in obj['documents']:
        collection.add_document(parse_doc(doc))
    return collection

Response: Deserialize a dict obj to a BioCCollection object


Instruction:
Explain what the following Python 3 code does
def parse_sentence(obj: dict) -> BioCSentence:
    sentence = BioCSentence()
    sentence.offset = obj['offset']
    sentence.infons = obj['infons']
    sentence.text = obj['text']
    for annotation in obj['annotations']:
        sentence.add_annotation(parse_annotation(annotation))
    for relation in obj['relations']:
        sentence.add_relation(parse_relation(relation))
    return sentence

Response: Deserialize a dict obj to a BioCSentence object


Instruction:
Can you generate the documentation for the following Python 3 function
def parse_doc(obj: dict) -> BioCDocument:
    doc = BioCDocument()
    doc.id = obj['id']
    doc.infons = obj['infons']
    for passage in obj['passages']:
        doc.add_passage(parse_passage(passage))
    for annotation in obj['annotations']:
        doc.add_annotation(parse_annotation(annotation))
    for relation in obj['relations']:
        doc.add_relation(parse_relation(relation))
    return doc

Response: Deserialize a dict obj to a BioCDocument object


Instruction:
Can you generate the documentation for the following Python 3 function
def loads(s: str, **kwargs) -> BioCCollection:
    obj = json.loads(s, **kwargs)
    return parse_collection(obj)

Response: Deserialize a string containing a JSON document into a collection of BioC objects.