Instruction:
Here you have a function in Python 3, explain what it does
def save_unmet(self, job):
        msg_id = job.msg_id
        self.depending[msg_id] = job
        # track the ids in follow or after, but not those already finished
        for dep_id in job.after.union(job.follow).difference(self.all_done):
            if dep_id not in self.graph:
                self.graph[dep_id] = set()
            self.graph[dep_id].add(msg_id)

Response: Save a message for later submission when its dependencies are met.


Instruction:
Here you have a function in Python 3, explain what it does
def submit_task(self, job, indices=None):
        if indices:
            loads = [self.loads[i] for i in indices]
        else:
            loads = self.loads
        idx = self.scheme(loads)
        if indices:
            idx = indices[idx]
        target = self.targets[idx]
        # print (target, map(str, msg[:3]))
        # send job to the engine
        self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
        self.engine_stream.send_multipart(job.raw_msg, copy=False)
        # update load
        self.add_job(idx)
        self.pending[target][job.msg_id] = job
        # notify Hub
        content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii'))
        self.session.send(self.mon_stream, 'task_destination', content=content,
                        ident=[b'tracktask',self.ident])

Response: Submit a task to any of a subset of our targets.


Instruction:
Make a summary of the following Python 3 code
def dispatch_result(self, raw_msg):
        try:
            idents,msg = self.session.feed_identities(raw_msg, copy=False)
            msg = self.session.unserialize(msg, content=False, copy=False)
            engine = idents[0]
            try:
                idx = self.targets.index(engine)
            except ValueError:
                pass # skip load-update for dead engines
            else:
                self.finish_job(idx)
        except Exception:
            self.log.error("task::Invaid result: %r", raw_msg, exc_info=True)
            return

        header = msg['header']
        parent = msg['parent_header']
        if header.get('dependencies_met', True):
            success = (header['status'] == 'ok')
            msg_id = parent['msg_id']
            retries = self.retries[msg_id]
            if not success and retries > 0:
                # failed
                self.retries[msg_id] = retries - 1
                self.handle_unmet_dependency(idents, parent)
            else:
                del self.retries[msg_id]
                # relay to client and update graph
                self.handle_result(idents, parent, raw_msg, success)
                # send to Hub monitor
                self.mon_stream.send_multipart([b'outtask']+raw_msg, copy=False)
        else:
            self.handle_unmet_dependency(idents, parent)

Response: dispatch method for result replies


Instruction:
Can you generate a brief explanation for the following Python 3 code
def handle_result(self, idents, parent, raw_msg, success=True):
        # first, relay result to client
        engine = idents[0]
        client = idents[1]
        # swap_ids for ROUTER-ROUTER mirror
        raw_msg[:2] = [client,engine]
        # print (map(str, raw_msg[:4]))
        self.client_stream.send_multipart(raw_msg, copy=False)
        # now, update our data structures
        msg_id = parent['msg_id']
        self.pending[engine].pop(msg_id)
        if success:
            self.completed[engine].add(msg_id)
            self.all_completed.add(msg_id)
        else:
            self.failed[engine].add(msg_id)
            self.all_failed.add(msg_id)
        self.all_done.add(msg_id)
        self.destinations[msg_id] = engine

        self.update_graph(msg_id, success)

Response: handle a real task result either success or failure


Instruction:
Write a Python 3 function for
handling an unmet dependency

Response:
def handle_unmet_dependency(self, idents, parent):
        """handle an unmet dependency"""
        engine = idents[0]
        msg_id = parent['msg_id']

        job = self.pending[engine].pop(msg_id)
        job.blacklist.add(engine)

        if job.blacklist == job.targets:
            self.depending[msg_id] = job
            self.fail_unreachable(msg_id)
        elif not self.maybe_run(job):
            # resubmit failed
            if msg_id not in self.all_failed:
                # put it back in our dependency tree
                self.save_unmet(job)

        if self.hwm:
            try:
                idx = self.targets.index(engine)
            except ValueError:
                pass # skip load-update for dead engines
            else:
                if self.loads[idx] == self.hwm-1:
                    self.update_graph(None)


Instruction:
Given the following Python 3 function, write the documentation
def update_graph(self, dep_id=None, success=True):
        # print ("\n\n***********")
        # pprint (dep_id)
        # pprint (self.graph)
        # pprint (self.depending)
        # pprint (self.all_completed)
        # pprint (self.all_failed)
        # print ("\n\n***********\n\n")
        # update any jobs that depended on the dependency
        jobs = self.graph.pop(dep_id, [])

        # recheck *all* jobs if
        # a) we have HWM and an engine just become no longer full
        # or b) dep_id was given as None

        if dep_id is None or self.hwm and any( [ load==self.hwm-1 for load in self.loads ]):
            jobs = self.depending.keys()

        for msg_id in sorted(jobs, key=lambda msg_id: self.depending[msg_id].timestamp):
            job = self.depending[msg_id]

            if job.after.unreachable(self.all_completed, self.all_failed)\
                    or job.follow.unreachable(self.all_completed, self.all_failed):
                self.fail_unreachable(msg_id)

            elif job.after.check(self.all_completed, self.all_failed): # time deps met, maybe run
                if self.maybe_run(job):

                    self.depending.pop(msg_id)
                    for mid in job.dependents:
                        if mid in self.graph:
                            self.graph[mid].remove(msg_id)

Response: Update the dependency graph and submit any jobs that just became runable.


Instruction:
Can you implement a function in Python 3 that
calls after self. targets [ idx ] just got the job with header. Override with subclasses.

Response:
def add_job(self, idx):
        """Called after self.targets[idx] just got the job with header.
        Override with subclasses.  The default ordering is simple LRU.
        The default loads are the number of outstanding jobs."""
        self.loads[idx] += 1
        for lis in (self.targets, self.loads):
            lis.append(lis.pop(idx))


Instruction:
Given the following Python 3 function, write the documentation
def logstart(self, logfname=None, loghead=None, logmode=None,
                 log_output=False, timestamp=False, log_raw_input=False):
        """Generate a new log-file with a default header.

        Raises RuntimeError if the log has already been started"""

        if self.logfile is not None:
            raise RuntimeError('Log file is already active: %s' %
                               self.logfname)

        # The parameters can override constructor defaults
        if logfname is not None: self.logfname = logfname
        if loghead is not None: self.loghead = loghead
        if logmode is not None: self.logmode = logmode

        # Parameters not part of the constructor
        self.timestamp = timestamp
        self.log_output = log_output
        self.log_raw_input = log_raw_input

        # init depending on the log mode requested
        isfile = os.path.isfile
        logmode = self.logmode

        if logmode == 'append':
            self.logfile = io.open(self.logfname, 'a', encoding='utf-8')

        elif logmode == 'backup':
            if isfile(self.logfname):
                backup_logname = self.logfname+'~'
                # Manually remove any old backup, since os.rename may fail
                # under Windows.
                if isfile(backup_logname):
                    os.remove(backup_logname)
                os.rename(self.logfname,backup_logname)
            self.logfile = io.open(self.logfname, 'w', encoding='utf-8')

        elif logmode == 'global':
            self.logfname = os.path.join(self.home_dir,self.logfname)
            self.logfile = io.open(self.logfname, 'a', encoding='utf-8')

        elif logmode == 'over':
            if isfile(self.logfname):
                os.remove(self.logfname)
            self.logfile = io.open(self.logfname,'w', encoding='utf-8')

        elif logmode == 'rotate':
            if isfile(self.logfname):
                if isfile(self.logfname+'.001~'):
                    old = glob.glob(self.logfname+'.*~')
                    old.sort()
                    old.reverse()
                    for f in old:
                        root, ext = os.path.splitext(f)
                        num = int(ext[1:-1])+1
                        os.rename(f, root+'.'+`num`.zfill(3)+'~')
                os.rename(self.logfname, self.logfname+'.001~')
            self.logfile = io.open(self.logfname, 'w', encoding='utf-8')

        if logmode != 'append':
            self.logfile.write(self.loghead)

        self.logfile.flush()
        self.log_active = True

Response: This function creates a new log file with the specified header and logmode.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def switch_log(self,val):

        if val not in [False,True,0,1]:
            raise ValueError, \
                  'Call switch_log ONLY with a boolean argument, not with:',val

        label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}

        if self.logfile is None:
            print """
Logging hasn't been started yet (use logstart for that).

%logon/%logoff are for temporarily starting and stopping logging for a logfile
which already exists. But you must first start the logging process with
%logstart (optionally giving a logfile name)."""

        else:
            if self.log_active == val:
                print 'Logging is already',label[val]
            else:
                print 'Switching logging',label[val]
                self.log_active = not self.log_active
                self.log_active_out = self.log_active

Response: Switch logging on or off. val should be ONLY a boolean argument.


Instruction:
Here you have a function in Python 3, explain what it does
def logstate(self):
        if self.logfile is None:
            print 'Logging has not been activated.'
        else:
            state = self.log_active and 'active' or 'temporarily suspended'
            print 'Filename       :',self.logfname
            print 'Mode           :',self.logmode
            print 'Output logging :',self.log_output
            print 'Raw input log  :',self.log_raw_input
            print 'Timestamping   :',self.timestamp
            print 'State          :',state

Response: Print a status message about the logger.


Instruction:
Given the following Python 3 function, write the documentation
def log(self, line_mod, line_ori):

        # Write the log line, but decide which one according to the
        # log_raw_input flag, set when the log is started.
        if self.log_raw_input:
            self.log_write(line_ori)
        else:
            self.log_write(line_mod)

Response: Write the sources to a log.


Instruction:
Here you have a function in Python 3, explain what it does
def log_write(self, data, kind='input'):

        #print 'data: %r' % data # dbg
        if self.log_active and data:
            write = self.logfile.write
            if kind=='input':
                if self.timestamp:
                    write(str_to_unicode(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
                                        time.localtime())))
                write(data)
            elif kind=='output' and self.log_output:
                odata = u'\n'.join([u'#[Out]# %s' % s
                                   for s in data.splitlines()])
                write(u'%s\n' % odata)
            self.logfile.flush()

Response: Write data to the log file if active


Instruction:
Here you have a function in Python 3, explain what it does
def logstop(self):

        if self.logfile is not None:
            self.logfile.close()
            self.logfile = None
        else:
            print "Logging hadn't been started."
        self.log_active = False

Response: Fully stop logging and close log file.


Instruction:
Here you have a function in Python 3, explain what it does
def new_worksheet(name=None, cells=None):
    ws = NotebookNode()
    if name is not None:
        ws.name = unicode(name)
    if cells is None:
        ws.cells = []
    else:
        ws.cells = list(cells)
    return ws

Response: Create a new worksheet by name with a list of cells.


Instruction:
Can you tell what is the following Python 3 function doing
def new_notebook(metadata=None, worksheets=None):
    nb = NotebookNode()
    nb.nbformat = 2
    if worksheets is None:
        nb.worksheets = []
    else:
        nb.worksheets = list(worksheets)
    if metadata is None:
        nb.metadata = new_metadata()
    else:
        nb.metadata = NotebookNode(metadata)
    return nb

Response: Create a notebook by name id and a list of worksheets.


Instruction:
Implement a function in Python 3 to
add a target string for dispatching.

Response:
def add_s(self, s, obj, priority= 0 ):
        """ Adds a target 'string' for dispatching """

        chain = self.strs.get(s, CommandChainDispatcher())
        chain.add(obj,priority)
        self.strs[s] = chain


Instruction:
Can you write a function in Python 3 where it
adds a target regexp for dispatching.

Response:
def add_re(self, regex, obj, priority= 0 ):
        """ Adds a target regexp for dispatching """

        chain = self.regexs.get(regex, CommandChainDispatcher())
        chain.add(obj,priority)
        self.regexs[regex] = chain


Instruction:
Can you implement a function in Python 3 that
gets a seq of Commandchain objects that match the given key

Response:
def dispatch(self, key):
        """ Get a seq of Commandchain objects that match key """
        if key in self.strs:
            yield self.strs[key]

        for r, obj in self.regexs.items():
            if re.match(r, key):
                yield obj
            else:
                #print "nomatch",key  # dbg
                pass


Instruction:
Can you implement a function in Python 3 that
yields all value targets with priority

Response:
def flat_matches(self, key):
        """ Yield all 'value' targets, without priority """
        for val in self.dispatch(key):
            for el in val:
                yield el[1] # only value, no priority
        return


Instruction:
Can you tell what is the following Python 3 function doing
def _notebook_dir_changed(self, name, old, new):
        if os.path.exists(new) and not os.path.isdir(new):
            raise TraitError("notebook dir %r is not a directory" % new)
        if not os.path.exists(new):
            self.log.info("Creating notebook dir %s", new)
            try:
                os.mkdir(new)
            except:
                raise TraitError("Couldn't create notebook dir %r" % new)

Response: check if the notebook dir is a directory and create it if it doesn t exist


Instruction:
Can you generate a brief explanation for the following Python 3 code
def list_notebooks(self):
        names = glob.glob(os.path.join(self.notebook_dir,
                                       '*' + self.filename_ext))
        names = [os.path.splitext(os.path.basename(name))[0]
                 for name in names]

        data = []
        for name in names:
            if name not in self.rev_mapping:
                notebook_id = self.new_notebook_id(name)
            else:
                notebook_id = self.rev_mapping[name]
            data.append(dict(notebook_id=notebook_id,name=name))
        data = sorted(data, key=lambda item: item['name'])
        return data

Response:
List all notebooks in the notebook dir. This returns a list of dicts of the form ::

            dict ( notebook_id = notebook name


Instruction:
Can you generate a brief explanation for the following Python 3 code
def new_notebook_id(self, name):
        # TODO: the following will give stable urls for notebooks, but unless
        # the notebooks are immediately redirected to their new urls when their
        # filemname changes, nasty inconsistencies result.  So for now it's
        # disabled and instead we use a random uuid4() call.  But we leave the
        # logic here so that we can later reactivate it, whhen the necessary
        # url redirection code is written.
        #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,
        #                 'file://'+self.get_path_by_name(name).encode('utf-8')))

        notebook_id = unicode(uuid.uuid4())

        self.mapping[notebook_id] = name
        self.rev_mapping[name] = notebook_id
        return notebook_id

Response: Generate a new notebook_id for a name and store its mappings.


Instruction:
Can you create a Python 3 function that
deletes a notebook s id only. This doesn t delete the actual notebook.

Response:
def delete_notebook_id(self, notebook_id):
        """Delete a notebook's id only. This doesn't delete the actual notebook."""
        name = self.mapping[notebook_id]
        del self.mapping[notebook_id]
        del self.rev_mapping[name]


Instruction:
Write a Python 3 function for
doing a notebook exist?

Response:
def notebook_exists(self, notebook_id):
        """Does a notebook exist?"""
        if notebook_id not in self.mapping:
            return False
        path = self.get_path_by_name(self.mapping[notebook_id])
        return os.path.isfile(path)


Instruction:
Write a Python 3 function for
returning a full path to a notebook given its notebook_id.

Response:
def find_path(self, notebook_id):
        """Return a full path to a notebook given its notebook_id."""
        try:
            name = self.mapping[notebook_id]
        except KeyError:
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        return self.get_path_by_name(name)


Instruction:
Can you write a function in Python 3 where it
returns a full path to a notebook given its name.

Response:
def get_path_by_name(self, name):
        """Return a full path to a notebook given its name."""
        filename = name + self.filename_ext
        path = os.path.join(self.notebook_dir, filename)
        return path


Instruction:
Make a summary of the following Python 3 code
def get_notebook(self, notebook_id, format=u'json'):
        format = unicode(format)
        if format not in self.allowed_formats:
            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
        last_modified, nb = self.get_notebook_object(notebook_id)
        kwargs = {}
        if format == 'json':
            # don't split lines for sending over the wire, because it
            # should match the Python in-memory format.
            kwargs['split_lines'] = False
        data = current.writes(nb, format, **kwargs)
        name = nb.metadata.get('name','notebook')
        return last_modified, name, data

Response: Get the representation of a notebook in format by notebook_id.


Instruction:
Write a Python 3 function for
getting the NotebookNode representation of a notebook by notebook_id.

Response:
def get_notebook_object(self, notebook_id):
        """Get the NotebookNode representation of a notebook by notebook_id."""
        path = self.find_path(notebook_id)
        if not os.path.isfile(path):
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        info = os.stat(path)
        last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime)
        with open(path,'r') as f:
            s = f.read()
            try:
                # v1 and v2 and json in the .ipynb files.
                nb = current.reads(s, u'json')
            except:
                raise web.HTTPError(500, u'Unreadable JSON notebook.')
        # Always use the filename as the notebook name.
        nb.metadata.name = os.path.splitext(os.path.basename(path))[0]
        return last_modified, nb


Instruction:
Can you implement a function in Python 3 that
saves a new notebook and return its notebook_id.

Response:
def save_new_notebook(self, data, name=None, format=u'json'):
        """Save a new notebook and return its notebook_id.

        If a name is passed in, it overrides any values in the notebook data
        and the value in the data is updated to use that value.
        """
        if format not in self.allowed_formats:
            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)

        try:
            nb = current.reads(data.decode('utf-8'), format)
        except:
            raise web.HTTPError(400, u'Invalid JSON data')

        if name is None:
            try:
                name = nb.metadata.name
            except AttributeError:
                raise web.HTTPError(400, u'Missing notebook name')
        nb.metadata.name = name

        notebook_id = self.new_notebook_id(name)
        self.save_notebook_object(notebook_id, nb)
        return notebook_id


Instruction:
Write a Python 3 script to
save an existing notebook by notebook_id.

Response:
def save_notebook(self, notebook_id, data, name=None, format=u'json'):
        """Save an existing notebook by notebook_id."""
        if format not in self.allowed_formats:
            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)

        try:
            nb = current.reads(data.decode('utf-8'), format)
        except:
            raise web.HTTPError(400, u'Invalid JSON data')

        if name is not None:
            nb.metadata.name = name
        self.save_notebook_object(notebook_id, nb)


Instruction:
Write a Python 3 script to
save an existing notebook object by notebook_id.

Response:
def save_notebook_object(self, notebook_id, nb):
        """Save an existing notebook object by notebook_id."""
        if notebook_id not in self.mapping:
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        old_name = self.mapping[notebook_id]
        try:
            new_name = nb.metadata.name
        except AttributeError:
            raise web.HTTPError(400, u'Missing notebook name')
        path = self.get_path_by_name(new_name)
        try:
            with open(path,'w') as f:
                current.write(nb, f, u'json')
        except Exception as e:
            raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)
        # save .py script as well
        if self.save_script:
            pypath = os.path.splitext(path)[0] + '.py'
            try:
                with io.open(pypath,'w', encoding='utf-8') as f:
                    current.write(nb, f, u'py')
            except Exception as e:
                raise web.HTTPError(400, u'Unexpected error while saving notebook as script: %s' % e)

        if old_name != new_name:
            old_path = self.get_path_by_name(old_name)
            if os.path.isfile(old_path):
                os.unlink(old_path)
            if self.save_script:
                old_pypath = os.path.splitext(old_path)[0] + '.py'
                if os.path.isfile(old_pypath):
                    os.unlink(old_pypath)
            self.mapping[notebook_id] = new_name
            self.rev_mapping[new_name] = notebook_id
            del self.rev_mapping[old_name]


Instruction:
Can you implement a function in Python 3 that
deletes notebook by notebook_id.

Response:
def delete_notebook(self, notebook_id):
        """Delete notebook by notebook_id."""
        path = self.find_path(notebook_id)
        if not os.path.isfile(path):
            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
        os.unlink(path)
        self.delete_notebook_id(notebook_id)


Instruction:
How would you implement a function in Python 3 that
returns a non - used filename of the form basename<int >.

Response:
def increment_filename(self, basename):
        """Return a non-used filename of the form basename<int>.

        This searches through the filenames (basename0, basename1, ...)
        until is find one that is not already being used. It is used to
        create Untitled and Copy names that are unique.
        """
        i = 0
        while True:
            name = u'%s%i' % (basename,i)
            path = self.get_path_by_name(name)
            if not os.path.isfile(path):
                break
            else:
                i = i+1
        return path, name


Instruction:
Make a summary of the following Python 3 code
def new_notebook(self):
        path, name = self.increment_filename('Untitled')
        notebook_id = self.new_notebook_id(name)
        metadata = current.new_metadata(name=name)
        nb = current.new_notebook(metadata=metadata)
        with open(path,'w') as f:
            current.write(nb, f, u'json')
        return notebook_id

Response: Create a new notebook and return its notebook_id.


Instruction:
Implement a function in Python 3 to
copy an existing notebook and return its notebook_id.

Response:
def copy_notebook(self, notebook_id):
        """Copy an existing notebook and return its notebook_id."""
        last_mod, nb = self.get_notebook_object(notebook_id)
        name = nb.metadata.name + '-Copy'
        path, name = self.increment_filename(name)
        nb.metadata.name = name
        notebook_id = self.new_notebook_id(name)
        self.save_notebook_object(notebook_id, nb)
        return notebook_id


Instruction:
Can you implement a function in Python 3 that
yields all tokens that are part of the given line.

Response:
def phys_tokens(toks):
    """Return all physical tokens, even line continuations.

    tokenize.generate_tokens() doesn't return a token for the backslash that
    continues lines.  This wrapper provides those tokens so that we can
    re-create a faithful representation of the original source.

    Returns the same values as generate_tokens()

    """
    last_line = None
    last_lineno = -1
    last_ttype = None
    for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:
        if last_lineno != elineno:
            if last_line and last_line.endswith("\\\n"):
                # We are at the beginning of a new line, and the last line
                # ended with a backslash.  We probably have to inject a
                # backslash token into the stream. Unfortunately, there's more
                # to figure out.  This code::
                #
                #   usage = """\
                #   HEY THERE
                #   """
                #
                # triggers this condition, but the token text is::
                #
                #   '"""\\\nHEY THERE\n"""'
                #
                # so we need to figure out if the backslash is already in the
                # string token or not.
                inject_backslash = True
                if last_ttype == tokenize.COMMENT:
                    # Comments like this \
                    # should never result in a new token.
                    inject_backslash = False
                elif ttype == token.STRING:
                    if "\n" in ttext and ttext.split('\n', 1)[0][-1] == '\\':
                        # It's a multiline string and the first line ends with
                        # a backslash, so we don't need to inject another.
                        inject_backslash = False
                if inject_backslash:
                    # Figure out what column the backslash is in.
                    ccol = len(last_line.split("\n")[-2]) - 1
                    # Yield the token, with a fake token type.
                    yield (
                        99999, "\\\n",
                        (slineno, ccol), (slineno, ccol+2),
                        last_line
                        )
            last_line = ltext
            last_ttype = ttype
        yield ttype, ttext, (slineno, scol), (elineno, ecol), ltext
        last_lineno = elineno


Instruction:
Make a summary of the following Python 3 code
def source_token_lines(source):
    ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
    line = []
    col = 0
    source = source.expandtabs(8).replace('\r\n', '\n')
    tokgen = generate_tokens(source)
    for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
        mark_start = True
        for part in re.split('(\n)', ttext):
            if part == '\n':
                yield line
                line = []
                col = 0
                mark_end = False
            elif part == '':
                mark_end = False
            elif ttype in ws_tokens:
                mark_end = False
            else:
                if mark_start and scol > col:
                    line.append(("ws", " " * (scol - col)))
                    mark_start = False
                tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
                if ttype == token.NAME and keyword.iskeyword(ttext):
                    tok_class = "key"
                line.append((tok_class, part))
                mark_end = True
            scol = 0
        if mark_end:
            col = ecol

    if line:
        yield line

Response: Yields a series of lines one for each line in source.


Instruction:
Make a summary of the following Python 3 code
def source_encoding(source):
    # Note: this function should never be called on Python 3, since py3 has
    # built-in tools to do this.
    assert sys.version_info < (3, 0)

    # This is mostly code adapted from Py3.2's tokenize module.

    cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)")

    # Do this so the detect_encode code we copied will work.
    readline = iter(source.splitlines(True)).next

    def _get_normal_name(orig_enc):
        """Imitates get_normal_name in tokenizer.c."""
        # Only care about the first 12 characters.
        enc = orig_enc[:12].lower().replace("_", "-")
        if re.match(r"^utf-8($|-)", enc):
            return "utf-8"
        if re.match(r"^(latin-1|iso-8859-1|iso-latin-1)($|-)", enc):
            return "iso-8859-1"
        return orig_enc

    # From detect_encode():
    # It detects the encoding from the presence of a utf-8 bom or an encoding
    # cookie as specified in pep-0263.  If both a bom and a cookie are present,
    # but disagree, a SyntaxError will be raised.  If the encoding cookie is an
    # invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
    # 'utf-8-sig' is returned.

    # If no encoding is specified, then the default will be returned.  The
    # default varied with version.

    if sys.version_info <= (2, 4):
        default = 'iso-8859-1'
    else:
        default = 'ascii'

    bom_found = False
    encoding = None

    def read_or_stop():
        """Get the next source line, or ''."""
        try:
            return readline()
        except StopIteration:
            return ''

    def find_cookie(line):
        """Find an encoding cookie in `line`."""
        try:
            line_string = line.decode('ascii')
        except UnicodeDecodeError:
            return None

        matches = cookie_re.findall(line_string)
        if not matches:
            return None
        encoding = _get_normal_name(matches[0])
        try:
            codec = codecs.lookup(encoding)
        except LookupError:
            # This behaviour mimics the Python interpreter
            raise SyntaxError("unknown encoding: " + encoding)

        if bom_found:
            # codecs in 2.3 were raw tuples of functions, assume the best.
            codec_name = getattr(codec, 'name', encoding)
            if codec_name != 'utf-8':
                # This behaviour mimics the Python interpreter
                raise SyntaxError('encoding problem: utf-8')
            encoding += '-sig'
        return encoding

    first = read_or_stop()
    if first.startswith(codecs.BOM_UTF8):
        bom_found = True
        first = first[3:]
        default = 'utf-8-sig'
    if not first:
        return default

    encoding = find_cookie(first)
    if encoding:
        return encoding

    second = read_or_stop()
    if not second:
        return default

    encoding = find_cookie(second)
    if encoding:
        return encoding

    return default

Response: Determine the encoding for source.


Instruction:
Can you create a Python 3 function that
loads the default config file from the default ipython_dir.

Response:
def load_default_config(ipython_dir=None):
    """Load the default config file from the default ipython_dir.

    This is useful for embedded shells.
    """
    if ipython_dir is None:
        ipython_dir = get_ipython_dir()
    profile_dir = os.path.join(ipython_dir, 'profile_default')
    cl = PyFileConfigLoader(default_config_file_name, profile_dir)
    try:
        config = cl.load_config()
    except ConfigFileNotFound:
        # no config found
        config = Config()
    return config


Instruction:
How would you explain what the following Python 3 function does
def make_report(self,traceback):

        sec_sep = self.section_sep
        # Start with parent report
        report = [super(IPAppCrashHandler, self).make_report(traceback)]
        # Add interactive-specific info we may have
        rpt_add = report.append
        try:
            rpt_add(sec_sep+"History of session input:")
            for line in self.app.shell.user_ns['_ih']:
                rpt_add(line)
            rpt_add('\n*** Last line of input (may not be in above history):\n')
            rpt_add(self.app.shell._last_input_line+'\n')
        except:
            pass

        return ''.join(report)

Response: Return a string containing a crash report.


Instruction:
Make a summary of the following Python 3 code
def _classes_default(self):
        return [
            InteractiveShellApp, # ShellApp comes before TerminalApp, because
            self.__class__,      # it will also affect subclasses (e.g. QtConsole)
            TerminalInteractiveShell,
            PromptManager,
            HistoryManager,
            ProfileDir,
            PlainTextFormatter,
            IPCompleter,
            ScriptMagics,
        ]

Response: Returns a list of classes that can be used for this instance of TerminalIPythonApp.


Instruction:
Make a summary of the following Python 3 code
def parse_command_line(self, argv=None):

        argv = sys.argv[1:] if argv is None else argv

        if '-pylab' in argv:
            # deprecated `-pylab` given,
            # warn and transform into current syntax
            argv = argv[:] # copy, don't clobber
            idx = argv.index('-pylab')
            warn.warn("`-pylab` flag has been deprecated.\n"
            "    Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
            sub = '--pylab'
            if len(argv) > idx+1:
                # check for gui arg, as in '-pylab qt'
                gui = argv[idx+1]
                if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
                    sub = '--pylab='+gui
                    argv.pop(idx+1)
            argv[idx] = sub

        return super(TerminalIPythonApp, self).parse_command_line(argv)

Response: override to allow old - pylab flag with deprecation warning


Instruction:
Can you generate the documentation for the following Python 3 function
def initialize(self, argv=None):
        super(TerminalIPythonApp, self).initialize(argv)
        if self.subapp is not None:
            # don't bother initializing further, starting subapp
            return
        if not self.ignore_old_config:
            check_for_old_config(self.ipython_dir)
        # print self.extra_args
        if self.extra_args and not self.something_to_run:
            self.file_to_run = self.extra_args[0]
        self.init_path()
        # create the shell
        self.init_shell()
        # and draw the banner
        self.init_banner()
        # Now a variety of things that happen after the banner is printed.
        self.init_gui_pylab()
        self.init_extensions()
        self.init_code()

Response: Do actions after construct but before starting the app.


Instruction:
Given the following Python 3 function, write the documentation
def init_shell(self):
        # Create an InteractiveShell instance.
        # shell.display_banner should always be False for the terminal
        # based app, because we call shell.show_banner() by hand below
        # so the banner shows *before* all extension loading stuff.
        self.shell = TerminalInteractiveShell.instance(config=self.config,
                        display_banner=False, profile_dir=self.profile_dir,
                        ipython_dir=self.ipython_dir)
        self.shell.configurables.append(self)

Response: initialize the InteractiveShell instance


Instruction:
Implement a function in Python 3 to
call when the pylab attribute is changed.

Response:
def _pylab_changed(self, name, old, new):
        """Replace --pylab='inline' with --pylab='auto'"""
        if new == 'inline':
            warn.warn("'inline' not available as pylab backend, "
                      "using 'auto' instead.\n")
            self.pylab = 'auto'


Instruction:
Write a Python 3 script to
return the class name of an object.

Response:
def class_of ( object ):
    """ Returns a string containing the class name of an object with the
    correct indefinite article ('a' or 'an') preceding it (e.g., 'an Image',
    'a PlotValue').
    """
    if isinstance( object, basestring ):
        return add_article( object )

    return add_article( object.__class__.__name__ )


Instruction:
Create a Python 3 function to
return a string representation of a value and its type for readable error messages.

Response:
def repr_type(obj):
    """ Return a string representation of a value and its type for readable
    error messages.
    """
    the_type = type(obj)
    if (not py3compat.PY3) and the_type is InstanceType:
        # Old-style class.
        the_type = obj.__class__
    msg = '%r %r' % (obj, the_type)
    return msg


Instruction:
How would you code a function in Python 3 to
convert the name argument to a list of names.

Response:
def parse_notifier_name(name):
    """Convert the name argument to a list of names.

    Examples
    --------

    >>> parse_notifier_name('a')
    ['a']
    >>> parse_notifier_name(['a','b'])
    ['a', 'b']
    >>> parse_notifier_name(None)
    ['anytrait']
    """
    if isinstance(name, str):
        return [name]
    elif name is None:
        return ['anytrait']
    elif isinstance(name, (list, tuple)):
        for n in name:
            assert isinstance(n, str), "names must be strings"
        return name


Instruction:
Implement a Python 3 function for
setting the default value on a per instance basis.

Response:
def set_default_value(self, obj):
        """Set the default value on a per instance basis.

        This method is called by :meth:`instance_init` to create and
        validate the default value.  The creation and validation of
        default values must be delayed until the parent :class:`HasTraits`
        class has been instantiated.
        """
        # Check for a deferred initializer defined in the same class as the
        # trait declaration or above.
        mro = type(obj).mro()
        meth_name = '_%s_default' % self.name
        for cls in mro[:mro.index(self.this_class)+1]:
            if meth_name in cls.__dict__:
                break
        else:
            # We didn't find one. Do static initialization.
            dv = self.get_default_value()
            newdv = self._validate(obj, dv)
            obj._trait_values[self.name] = newdv
            return
        # Complete the dynamic initialization.
        obj._trait_dyn_inits[self.name] = cls.__dict__[meth_name]


Instruction:
Write a Python 3 script for
getting a list of all the traits of this class.

Response:
def class_traits(cls, **metadata):
        """Get a list of all the traits of this class.

        This method is just like the :meth:`traits` method, but is unbound.

        The TraitTypes returned don't know anything about the values
        that the various HasTrait's instances are holding.

        This follows the same algorithm as traits does and does not allow
        for any simple way of specifying merely that a metadata name
        exists, but has any value.  This is because get_metadata returns
        None if a metadata key doesn't exist.
        """
        traits = dict([memb for memb in getmembers(cls) if \
                     isinstance(memb[1], TraitType)])

        if len(metadata) == 0:
            return traits

        for meta_name, meta_eval in metadata.items():
            if type(meta_eval) is not FunctionType:
                metadata[meta_name] = _SimpleTest(meta_eval)

        result = {}
        for name, trait in traits.items():
            for meta_name, meta_eval in metadata.items():
                if not meta_eval(trait.get_metadata(meta_name)):
                    break
            else:
                result[name] = trait

        return result


Instruction:
Can you write a function in Python 3 where it
gets metadata values for trait by key.

Response:
def trait_metadata(self, traitname, key):
        """Get metadata values for trait by key."""
        try:
            trait = getattr(self.__class__, traitname)
        except AttributeError:
            raise TraitError("Class %s does not have a trait named %s" %
                                (self.__class__.__name__, traitname))
        else:
            return trait.get_metadata(key)


Instruction:
Write a Python 3 function for
validating that the value is a valid object instance.

Response:
def validate(self, obj, value):
        """Validates that the value is a valid object instance."""
        try:
            if issubclass(value, self.klass):
                return value
        except:
            if (value is None) and (self._allow_none):
                return value

        self.error(obj, value)


Instruction:
Can you implement a function in Python 3 that
returns a description of the trait.

Response:
def info(self):
        """ Returns a description of the trait."""
        if isinstance(self.klass, basestring):
            klass = self.klass
        else:
            klass = self.klass.__name__
        result = 'a subclass of ' + klass
        if self._allow_none:
            return result + ' or None'
        return result


Instruction:
Can you tell what is the following Python 3 function doing
def info(self):
        result = 'any of ' + repr(self.values)
        if self._allow_none:
            return result + ' or None'
        return result

Response: Returns a description of the trait.


Instruction:
Make a summary of the following Python 3 code
def require(*mods):
    names = []
    for mod in mods:
        if isinstance(mod, ModuleType):
            mod = mod.__name__

        if isinstance(mod, basestring):
            names.append(mod)
        else:
            raise TypeError("names must be modules or module names, not %s"%type(mod))

    return depend(_require, *names)

Response: Simple decorator for requiring names to be importable.


Instruction:
Given the following Python 3 function, write the documentation
def check(self, completed, failed=None):
        if len(self) == 0:
            return True
        against = set()
        if self.success:
            against = completed
        if failed is not None and self.failure:
            against = against.union(failed)
        if self.all:
            return self.issubset(against)
        else:
            return not self.isdisjoint(against)

Response: check whether our dependencies have been met.


Instruction:
Implement a function in Python 3 to
return whether this dependency has become impossible.

Response:
def unreachable(self, completed, failed=None):
        """return whether this dependency has become impossible."""
        if len(self) == 0:
            return False
        against = set()
        if not self.success:
            against = completed
        if failed is not None and not self.failure:
            against = against.union(failed)
        if self.all:
            return not self.isdisjoint(against)
        else:
            return self.issubset(against)


Instruction:
Explain what the following Python 3 code does
def as_dict(self):
        return dict(
            dependencies=list(self),
            all=self.all,
            success=self.success,
            failure=self.failure
        )

Response: Represent this dependency as a dict. For json compatibility.


Instruction:
Write a Python 3 script for
returning a Solver instance

Response:
def Ainv(self):
        'Returns a Solver instance'

        if not hasattr(self, '_Ainv'):
            self._Ainv = self.Solver(self.A)
        return self._Ainv


Instruction:
Write a Python 3 function for
returning a Solver instance

Response:
def Ainv(self):
        'Returns a Solver instance'

        if getattr(self, '_Ainv', None) is None:
            self._Ainv = self.Solver(self.A, 13)
            self._Ainv.run_pardiso(12)
        return self._Ainv


Instruction:
Implement a function in Python 3 to
construct a dict representation of a binary tree

Response:
def bintree(ids, parent=None):
    """construct {child:parent} dict representation of a binary tree

    keys are the nodes in the tree, and values are the parent of each node.

    The root node has parent `parent`, default: None.

    >>> tree = bintree(range(7))
    >>> tree
    {0: None, 1: 0, 2: 1, 3: 1, 4: 0, 5: 4, 6: 4}
    >>> print_bintree(tree)
    0
      1
        2
        3
      4
        5
        6
    """
    parents = {}
    n = len(ids)
    if n == 0:
        return parents
    root = ids[0]
    parents[root] = parent
    if len(ids) == 1:
        return parents
    else:
        ids = ids[1:]
        n = len(ids)
        left = bintree(ids[:n/2], parent=root)
        right = bintree(ids[n/2:], parent=root)
        parents.update(left)
        parents.update(right)
    return parents


Instruction:
Explain what the following Python 3 code does
def reverse_bintree(parents):
    children = {}
    for child,parent in parents.iteritems():
        if parent is None:
            children[None] = child
            continue
        elif parent not in children:
            children[parent] = []
        children[parent].append(child)

    return children

Response: reverse bintree - > tree


Instruction:
Can you implement a function in Python 3 that
gets the depth of an element in the tree

Response:
def depth(n, tree):
    """get depth of an element in the tree"""
    d = 0
    parent = tree[n]
    while parent is not None:
        d += 1
        parent = tree[parent]
    return d


Instruction:
Create a Python 3 function to
print a binary tree

Response:
def print_bintree(tree, indent='  '):
    """print a binary tree"""
    for n in sorted(tree.keys()):
        print "%s%s" % (indent * depth(n,tree), n)


Instruction:
Write a Python 3 script for
accepting either IP address or dns name and return IP

Response:
def disambiguate_dns_url(url, location):
    """accept either IP address or dns name, and return IP"""
    if not ip_pat.match(location):
        location = socket.gethostbyname(location)
    return disambiguate_url(url, location)


Instruction:
Here you have a function in Python 3, explain what it does
def connect(self, peers, btree, pub_url, root_id=0):

        # count the number of children we have
        self.nchildren = btree.values().count(self.id)

        if self.root:
            return # root only binds

        root_location = peers[root_id][-1]
        self.sub.connect(disambiguate_dns_url(pub_url, root_location))

        parent = btree[self.id]

        tree_url, location = peers[parent]
        self.upstream.connect(disambiguate_dns_url(tree_url, location))

Response:
connect to peers.  `peers` will be a dict of 4-tuples, keyed by name.
        {peer : (ident, addr, pub_addr, location)}
        where peer is the name, ident is the XREP identity, addr,pub_addr are the


Instruction:
Implement a function in Python 3 to
parallel reduce on binary tree

Response:
def reduce(self, f, value, flat=True, all=False):
        """parallel reduce on binary tree

        if flat:
            value is an entry in the sequence
        else:
            value is a list of entries in the sequence

        if all:
            broadcast final result to all nodes
        else:
            only root gets final result
        """
        if not flat:
            value = reduce(f, value)

        for i in range(self.nchildren):
            value = f(value, self.recv_downstream())

        if not self.root:
            self.send_upstream(value)

        if all:
            if self.root:
                self.publish(value)
            else:
                value = self.consume()
        return value


Instruction:
Given the following Python 3 function, write the documentation
def allreduce(self, f, value, flat=True):
        return self.reduce(f, value, flat=flat, all=True)

Response: parallel reduce followed by broadcast of the result


Instruction:
Explain what the following Python 3 code does
def init_hub(self):
        client_iface = "%s://%s:" % (self.client_transport, self.client_ip) + "%i"
        engine_iface = "%s://%s:" % (self.engine_transport, self.engine_ip) + "%i"

        ctx = self.context
        loop = self.loop

        # Registrar socket
        q = ZMQStream(ctx.socket(zmq.ROUTER), loop)
        q.bind(client_iface % self.regport)
        self.log.info("Hub listening on %s for registration.", client_iface % self.regport)
        if self.client_ip != self.engine_ip:
            q.bind(engine_iface % self.regport)
            self.log.info("Hub listening on %s for registration.", engine_iface % self.regport)

        ### Engine connections ###

        # heartbeat
        hpub = ctx.socket(zmq.PUB)
        hpub.bind(engine_iface % self.hb[0])
        hrep = ctx.socket(zmq.ROUTER)
        hrep.bind(engine_iface % self.hb[1])
        self.heartmonitor = HeartMonitor(loop=loop, config=self.config, log=self.log,
                                pingstream=ZMQStream(hpub,loop),
                                pongstream=ZMQStream(hrep,loop)
                            )

        ### Client connections ###
        # Notifier socket
        n = ZMQStream(ctx.socket(zmq.PUB), loop)
        n.bind(client_iface%self.notifier_port)

        ### build and launch the queues ###

        # monitor socket
        sub = ctx.socket(zmq.SUB)
        sub.setsockopt(zmq.SUBSCRIBE, b"")
        sub.bind(self.monitor_url)
        sub.bind('inproc://monitor')
        sub = ZMQStream(sub, loop)

        # connect the db
        db_class = _db_shortcuts.get(self.db_class.lower(), self.db_class)
        self.log.info('Hub using DB backend: %r', (db_class.split('.')[-1]))
        self.db = import_item(str(db_class))(session=self.session.session,
                                            config=self.config, log=self.log)
        time.sleep(.25)
        try:
            scheme = self.config.TaskScheduler.scheme_name
        except AttributeError:
            from .scheduler import TaskScheduler
            scheme = TaskScheduler.scheme_name.get_default_value()
        # build connection dicts
        self.engine_info = {
            'control' : engine_iface%self.control[1],
            'mux': engine_iface%self.mux[1],
            'heartbeat': (engine_iface%self.hb[0], engine_iface%self.hb[1]),
            'task' : engine_iface%self.task[1],
            'iopub' : engine_iface%self.iopub[1],
            # 'monitor' : engine_iface%self.mon_port,
            }

        self.client_info = {
            'control' : client_iface%self.control[0],
            'mux': client_iface%self.mux[0],
            'task' : (scheme, client_iface%self.task[0]),
            'iopub' : client_iface%self.iopub[0],
            'notification': client_iface%self.notifier_port
            }
        self.log.debug("Hub engine addrs: %s", self.engine_info)
        self.log.debug("Hub client addrs: %s", self.client_info)

        # resubmit stream
        r = ZMQStream(ctx.socket(zmq.DEALER), loop)
        url = util.disambiguate_url(self.client_info['task'][-1])
        r.setsockopt(zmq.IDENTITY, self.session.bsession)
        r.connect(url)

        self.hub = Hub(loop=loop, session=self.session, monitor=sub, heartmonitor=self.heartmonitor,
                query=q, notifier=n, resubmit=r, db=self.db,
                engine_info=self.engine_info, client_info=self.client_info,
                log=self.log)

Response: Initialize the hub with the same client and engine interfaces.


Instruction:
Can you create a Python 3 function that
turns any valid targets argument into a list of integer ids

Response:
def _validate_targets(self, targets):
        """turn any valid targets argument into a list of integer ids"""
        if targets is None:
            # default to all
            return self.ids

        if isinstance(targets, (int,str,unicode)):
            # only one target specified
            targets = [targets]
        _targets = []
        for t in targets:
            # map raw identities to ids
            if isinstance(t, (str,unicode)):
                t = self.by_ident.get(cast_bytes(t), t)
            _targets.append(t)
        targets = _targets
        bad_targets = [ t for t in targets if t not in self.ids ]
        if bad_targets:
            raise IndexError("No Such Engine: %r" % bad_targets)
        if not targets:
            raise IndexError("No Engines Registered")
        return targets


Instruction:
Can you write a function in Python 3 where it
routes registration requests and queries from clients.

Response:
def dispatch_query(self, msg):
        """Route registration requests and queries from clients."""
        try:
            idents, msg = self.session.feed_identities(msg)
        except ValueError:
            idents = []
        if not idents:
            self.log.error("Bad Query Message: %r", msg)
            return
        client_id = idents[0]
        try:
            msg = self.session.unserialize(msg, content=True)
        except Exception:
            content = error.wrap_exception()
            self.log.error("Bad Query Message: %r", msg, exc_info=True)
            self.session.send(self.query, "hub_error", ident=client_id,
                    content=content)
            return
        # print client_id, header, parent, content
        #switch on message type:
        msg_type = msg['header']['msg_type']
        self.log.info("client::client %r requested %r", client_id, msg_type)
        handler = self.query_handlers.get(msg_type, None)
        try:
            assert handler is not None, "Bad Message Type: %r" % msg_type
        except:
            content = error.wrap_exception()
            self.log.error("Bad Message Type: %r", msg_type, exc_info=True)
            self.session.send(self.query, "hub_error", ident=client_id,
                    content=content)
            return

        else:
            handler(idents, msg)


Instruction:
Given the following Python 3 function, write the documentation
def handle_new_heart(self, heart):
        self.log.debug("heartbeat::handle_new_heart(%r)", heart)
        if heart not in self.incoming_registrations:
            self.log.info("heartbeat::ignoring new heart: %r", heart)
        else:
            self.finish_registration(heart)

Response:
handler to attach to heartbeater.
        Called when a new heart starts to be beat.
            Called when a new heart starts to be beat.
            Triggers completion of registration.


Instruction:
Can you tell what is the following Python 3 function doing
def handle_heart_failure(self, heart):
        self.log.debug("heartbeat::handle_heart_failure(%r)", heart)
        eid = self.hearts.get(heart, None)
        queue = self.engines[eid].queue
        if eid is None or self.keytable[eid] in self.dead_engines:
            self.log.info("heartbeat::ignoring heart failure %r (not an engine or already dead)", heart)
        else:
            self.unregister_engine(heart, dict(content=dict(id=eid, queue=queue)))

Response: handle_heart_failure - handle a heartbeat failure.


Instruction:
Can you tell what is the following Python 3 function doing
def save_task_request(self, idents, msg):
        client_id = idents[0]

        try:
            msg = self.session.unserialize(msg)
        except Exception:
            self.log.error("task::client %r sent invalid task message: %r",
                    client_id, msg, exc_info=True)
            return
        record = init_record(msg)

        record['client_uuid'] = client_id.decode('ascii')
        record['queue'] = 'task'
        header = msg['header']
        msg_id = header['msg_id']
        self.pending.add(msg_id)
        self.unassigned.add(msg_id)
        try:
            # it's posible iopub arrived first:
            existing = self.db.get_record(msg_id)
            if existing['resubmitted']:
                for key in ('submitted', 'client_uuid', 'buffers'):
                    # don't clobber these keys on resubmit
                    # submitted and client_uuid should be different
                    # and buffers might be big, and shouldn't have changed
                    record.pop(key)
                    # still check content,header which should not change
                    # but are not expensive to compare as buffers

            for key,evalue in existing.iteritems():
                if key.endswith('buffers'):
                    # don't compare buffers
                    continue
                rvalue = record.get(key, None)
                if evalue and rvalue and evalue != rvalue:
                    self.log.warn("conflicting initial state for record: %r:%r <%r> %r", msg_id, rvalue, key, evalue)
                elif evalue and not rvalue:
                    record[key] = evalue
            try:
                self.db.update_record(msg_id, record)
            except Exception:
                self.log.error("DB Error updating record %r", msg_id, exc_info=True)
        except KeyError:
            try:
                self.db.add_record(msg_id, record)
            except Exception:
                self.log.error("DB Error adding record %r", msg_id, exc_info=True)
        except Exception:
            self.log.error("DB Error saving task request %r", msg_id, exc_info=True)

Response: Save the submission of a task.


Instruction:
Write a Python 3 script for
saving the result of a completed task.

Response:
def save_task_result(self, idents, msg):
        """save the result of a completed task."""
        client_id = idents[0]
        try:
            msg = self.session.unserialize(msg)
        except Exception:
            self.log.error("task::invalid task result message send to %r: %r",
                    client_id, msg, exc_info=True)
            return

        parent = msg['parent_header']
        if not parent:
            # print msg
            self.log.warn("Task %r had no parent!", msg)
            return
        msg_id = parent['msg_id']
        if msg_id in self.unassigned:
            self.unassigned.remove(msg_id)

        header = msg['header']
        engine_uuid = header.get('engine', u'')
        eid = self.by_ident.get(cast_bytes(engine_uuid), None)

        status = header.get('status', None)

        if msg_id in self.pending:
            self.log.info("task::task %r finished on %s", msg_id, eid)
            self.pending.remove(msg_id)
            self.all_completed.add(msg_id)
            if eid is not None:
                if status != 'aborted':
                    self.completed[eid].append(msg_id)
                if msg_id in self.tasks[eid]:
                    self.tasks[eid].remove(msg_id)
            completed = header['date']
            started = header.get('started', None)
            result = {
                'result_header' : header,
                'result_content': msg['content'],
                'started' : started,
                'completed' : completed,
                'received' : datetime.now(),
                'engine_uuid': engine_uuid,
            }

            result['result_buffers'] = msg['buffers']
            try:
                self.db.update_record(msg_id, result)
            except Exception:
                self.log.error("DB Error saving task request %r", msg_id, exc_info=True)

        else:
            self.log.debug("task::unknown task %r finished", msg_id)


Instruction:
Can you implement a function in Python 3 that
saves an iopub message into the database

Response:
def save_iopub_message(self, topics, msg):
        """save an iopub message into the db"""
        # print (topics)
        try:
            msg = self.session.unserialize(msg, content=True)
        except Exception:
            self.log.error("iopub::invalid IOPub message", exc_info=True)
            return

        parent = msg['parent_header']
        if not parent:
            self.log.warn("iopub::IOPub message lacks parent: %r", msg)
            return
        msg_id = parent['msg_id']
        msg_type = msg['header']['msg_type']
        content = msg['content']

        # ensure msg_id is in db
        try:
            rec = self.db.get_record(msg_id)
        except KeyError:
            rec = empty_record()
            rec['msg_id'] = msg_id
            self.db.add_record(msg_id, rec)
        # stream
        d = {}
        if msg_type == 'stream':
            name = content['name']
            s = rec[name] or ''
            d[name] = s + content['data']

        elif msg_type == 'pyerr':
            d['pyerr'] = content
        elif msg_type == 'pyin':
            d['pyin'] = content['code']
        elif msg_type in ('display_data', 'pyout'):
            d[msg_type] = content
        elif msg_type == 'status':
            pass
        else:
            self.log.warn("unhandled iopub msg_type: %r", msg_type)

        if not d:
            return

        try:
            self.db.update_record(msg_id, d)
        except Exception:
            self.log.error("DB Error saving iopub message %r", msg_id, exc_info=True)


Instruction:
Write a Python 3 script to
reply with connection addresses for clients.

Response:
def connection_request(self, client_id, msg):
        """Reply with connection addresses for clients."""
        self.log.info("client::client %r connected", client_id)
        content = dict(status='ok')
        content.update(self.client_info)
        jsonable = {}
        for k,v in self.keytable.iteritems():
            if v not in self.dead_engines:
                jsonable[str(k)] = v.decode('ascii')
        content['engines'] = jsonable
        self.session.send(self.query, 'connection_reply', content, parent=msg, ident=client_id)


Instruction:
Can you write a function in Python 3 where it
registers a new engine and create the sockets necessary.

Response:
def register_engine(self, reg, msg):
        """Register a new engine."""
        content = msg['content']
        try:
            queue = cast_bytes(content['queue'])
        except KeyError:
            self.log.error("registration::queue not specified", exc_info=True)
            return
        heart = content.get('heartbeat', None)
        if heart:
            heart = cast_bytes(heart)
        """register a new engine, and create the socket(s) necessary"""
        eid = self._next_id
        # print (eid, queue, reg, heart)

        self.log.debug("registration::register_engine(%i, %r, %r, %r)", eid, queue, reg, heart)

        content = dict(id=eid,status='ok')
        content.update(self.engine_info)
        # check if requesting available IDs:
        if queue in self.by_ident:
            try:
                raise KeyError("queue_id %r in use" % queue)
            except:
                content = error.wrap_exception()
                self.log.error("queue_id %r in use", queue, exc_info=True)
        elif heart in self.hearts: # need to check unique hearts?
            try:
                raise KeyError("heart_id %r in use" % heart)
            except:
                self.log.error("heart_id %r in use", heart, exc_info=True)
                content = error.wrap_exception()
        else:
            for h, pack in self.incoming_registrations.iteritems():
                if heart == h:
                    try:
                        raise KeyError("heart_id %r in use" % heart)
                    except:
                        self.log.error("heart_id %r in use", heart, exc_info=True)
                        content = error.wrap_exception()
                    break
                elif queue == pack[1]:
                    try:
                        raise KeyError("queue_id %r in use" % queue)
                    except:
                        self.log.error("queue_id %r in use", queue, exc_info=True)
                        content = error.wrap_exception()
                    break

        msg = self.session.send(self.query, "registration_reply",
                content=content,
                ident=reg)

        if content['status'] == 'ok':
            if heart in self.heartmonitor.hearts:
                # already beating
                self.incoming_registrations[heart] = (eid,queue,reg[0],None)
                self.finish_registration(heart)
            else:
                purge = lambda : self._purge_stalled_registration(heart)
                dc = ioloop.DelayedCallback(purge, self.registration_timeout, self.loop)
                dc.start()
                self.incoming_registrations[heart] = (eid,queue,reg[0],dc)
        else:
            self.log.error("registration::registration %i failed: %r", eid, content['evalue'])
        return eid


Instruction:
How would you code a function in Python 3 to
handle stranded messages from the queue.

Response:
def _handle_stranded_msgs(self, eid, uuid):
        """Handle messages known to be on an engine when the engine unregisters.

        It is possible that this will fire prematurely - that is, an engine will
        go down after completing a result, and the client will be notified
        that the result failed and later receive the actual result.
        """

        outstanding = self.queues[eid]

        for msg_id in outstanding:
            self.pending.remove(msg_id)
            self.all_completed.add(msg_id)
            try:
                raise error.EngineError("Engine %r died while running task %r" % (eid, msg_id))
            except:
                content = error.wrap_exception()
            # build a fake header:
            header = {}
            header['engine'] = uuid
            header['date'] = datetime.now()
            rec = dict(result_content=content, result_header=header, result_buffers=[])
            rec['completed'] = header['date']
            rec['engine_uuid'] = uuid
            try:
                self.db.update_record(msg_id, rec)
            except Exception:
                self.log.error("DB Error handling stranded msg %r", msg_id, exc_info=True)


Instruction:
Given the following Python 3 function, write the documentation
def finish_registration(self, heart):
        try:
            (eid,queue,reg,purge) = self.incoming_registrations.pop(heart)
        except KeyError:
            self.log.error("registration::tried to finish nonexistant registration", exc_info=True)
            return
        self.log.info("registration::finished registering engine %i:%r", eid, queue)
        if purge is not None:
            purge.stop()
        control = queue
        self.ids.add(eid)
        self.keytable[eid] = queue
        self.engines[eid] = EngineConnector(id=eid, queue=queue, registration=reg,
                                    control=control, heartbeat=heart)
        self.by_ident[queue] = eid
        self.queues[eid] = list()
        self.tasks[eid] = list()
        self.completed[eid] = list()
        self.hearts[heart] = eid
        content = dict(id=eid, queue=self.engines[eid].queue.decode('ascii'))
        if self.notifier:
            self.session.send(self.notifier, "registration_notification", content=content)
        self.log.info("engine::Engine Connected: %i", eid)

Response: Finishes the engine registration.


Instruction:
Can you tell what is the following Python 3 function doing
def queue_status(self, client_id, msg):
        content = msg['content']
        targets = content['targets']
        try:
            targets = self._validate_targets(targets)
        except:
            content = error.wrap_exception()
            self.session.send(self.query, "hub_error",
                    content=content, ident=client_id)
            return
        verbose = content.get('verbose', False)
        content = dict(status='ok')
        for t in targets:
            queue = self.queues[t]
            completed = self.completed[t]
            tasks = self.tasks[t]
            if not verbose:
                queue = len(queue)
                completed = len(completed)
                tasks = len(tasks)
            content[str(t)] = {'queue': queue, 'completed': completed , 'tasks': tasks}
        content['unassigned'] = list(self.unassigned) if verbose else len(self.unassigned)
        # print (content)
        self.session.send(self.query, "queue_reply", content=content, ident=client_id)

Response: Return the status of one or more targets.


Instruction:
Can you generate the documentation for the following Python 3 function
def purge_results(self, client_id, msg):
        content = msg['content']
        self.log.info("Dropping records with %s", content)
        msg_ids = content.get('msg_ids', [])
        reply = dict(status='ok')
        if msg_ids == 'all':
            try:
                self.db.drop_matching_records(dict(completed={'$ne':None}))
            except Exception:
                reply = error.wrap_exception()
        else:
            pending = filter(lambda m: m in self.pending, msg_ids)
            if pending:
                try:
                    raise IndexError("msg pending: %r" % pending[0])
                except:
                    reply = error.wrap_exception()
            else:
                try:
                    self.db.drop_matching_records(dict(msg_id={'$in':msg_ids}))
                except Exception:
                    reply = error.wrap_exception()

            if reply['status'] == 'ok':
                eids = content.get('engine_ids', [])
                for eid in eids:
                    if eid not in self.engines:
                        try:
                            raise IndexError("No such engine: %i" % eid)
                        except:
                            reply = error.wrap_exception()
                        break
                    uid = self.engines[eid].queue
                    try:
                        self.db.drop_matching_records(dict(engine_uuid=uid, completed={'$ne':None}))
                    except Exception:
                        reply = error.wrap_exception()
                        break

        self.session.send(self.query, 'purge_reply', content=reply, ident=client_id)

Response:
Purge results from memory. This method is more valuable before we move
        to a DB based message storage mechanism. This method is more valuable before we move
        to a DB based message storage mechanism. This method is more valuable before we move
        to a DB based message storage mechanism.


Instruction:
Given the following Python 3 function, write the documentation
def resubmit_task(self, client_id, msg):
        def finish(reply):
            self.session.send(self.query, 'resubmit_reply', content=reply, ident=client_id)

        content = msg['content']
        msg_ids = content['msg_ids']
        reply = dict(status='ok')
        try:
            records = self.db.find_records({'msg_id' : {'$in' : msg_ids}}, keys=[
                'header', 'content', 'buffers'])
        except Exception:
            self.log.error('db::db error finding tasks to resubmit', exc_info=True)
            return finish(error.wrap_exception())

        # validate msg_ids
        found_ids = [ rec['msg_id'] for rec in records ]
        pending_ids = [ msg_id for msg_id in found_ids if msg_id in self.pending ]
        if len(records) > len(msg_ids):
            try:
                raise RuntimeError("DB appears to be in an inconsistent state."
                    "More matching records were found than should exist")
            except Exception:
                return finish(error.wrap_exception())
        elif len(records) < len(msg_ids):
            missing = [ m for m in msg_ids if m not in found_ids ]
            try:
                raise KeyError("No such msg(s): %r" % missing)
            except KeyError:
                return finish(error.wrap_exception())
        elif pending_ids:
            pass
            # no need to raise on resubmit of pending task, now that we
            # resubmit under new ID, but do we want to raise anyway?
            # msg_id = invalid_ids[0]
            # try:
            #     raise ValueError("Task(s) %r appears to be inflight" % )
            # except Exception:
            #     return finish(error.wrap_exception())

        # mapping of original IDs to resubmitted IDs
        resubmitted = {}

        # send the messages
        for rec in records:
            header = rec['header']
            msg = self.session.msg(header['msg_type'], parent=header)
            msg_id = msg['msg_id']
            msg['content'] = rec['content']

            # use the old header, but update msg_id and timestamp
            fresh = msg['header']
            header['msg_id'] = fresh['msg_id']
            header['date'] = fresh['date']
            msg['header'] = header

            self.session.send(self.resubmit, msg, buffers=rec['buffers'])

            resubmitted[rec['msg_id']] = msg_id
            self.pending.add(msg_id)
            msg['buffers'] = rec['buffers']
            try:
                self.db.add_record(msg_id, init_record(msg))
            except Exception:
                self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True)

        finish(dict(status='ok', resubmitted=resubmitted))

        # store the new IDs in the Task DB
        for msg_id, resubmit_id in resubmitted.iteritems():
            try:
                self.db.update_record(msg_id, {'resubmitted' : resubmit_id})
            except Exception:
                self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True)

Response: Resubmit one or more tasks.


Instruction:
Can you generate the documentation for the following Python 3 function
def _extract_record(self, rec):
        io_dict = {}
        for key in ('pyin', 'pyout', 'pyerr', 'stdout', 'stderr'):
                io_dict[key] = rec[key]
        content = { 'result_content': rec['result_content'],
                            'header': rec['header'],
                            'result_header' : rec['result_header'],
                            'received' : rec['received'],
                            'io' : io_dict,
                          }
        if rec['result_buffers']:
            buffers = map(bytes, rec['result_buffers'])
        else:
            buffers = []

        return content, buffers

Response: decompose a TaskRecord dict into subsection of reply for get_result


Instruction:
Can you generate the documentation for the following Python 3 function
def get_results(self, client_id, msg):
        content = msg['content']
        msg_ids = sorted(set(content['msg_ids']))
        statusonly = content.get('status_only', False)
        pending = []
        completed = []
        content = dict(status='ok')
        content['pending'] = pending
        content['completed'] = completed
        buffers = []
        if not statusonly:
            try:
                matches = self.db.find_records(dict(msg_id={'$in':msg_ids}))
                # turn match list into dict, for faster lookup
                records = {}
                for rec in matches:
                    records[rec['msg_id']] = rec
            except Exception:
                content = error.wrap_exception()
                self.session.send(self.query, "result_reply", content=content,
                                                    parent=msg, ident=client_id)
                return
        else:
            records = {}
        for msg_id in msg_ids:
            if msg_id in self.pending:
                pending.append(msg_id)
            elif msg_id in self.all_completed:
                completed.append(msg_id)
                if not statusonly:
                    c,bufs = self._extract_record(records[msg_id])
                    content[msg_id] = c
                    buffers.extend(bufs)
            elif msg_id in records:
                if rec['completed']:
                    completed.append(msg_id)
                    c,bufs = self._extract_record(records[msg_id])
                    content[msg_id] = c
                    buffers.extend(bufs)
                else:
                    pending.append(msg_id)
            else:
                try:
                    raise KeyError('No such message: '+msg_id)
                except:
                    content = error.wrap_exception()
                break
        self.session.send(self.query, "result_reply", content=content,
                                            parent=msg, ident=client_id,
                                            buffers=buffers)

Response: Get the result of 1 or more messages.


Instruction:
How would you code a function in Python 3 to
get a list of all msg_ids in our DB records

Response:
def get_history(self, client_id, msg):
        """Get a list of all msg_ids in our DB records"""
        try:
            msg_ids = self.db.get_history()
        except Exception as e:
            content = error.wrap_exception()
        else:
            content = dict(status='ok', history=msg_ids)

        self.session.send(self.query, "history_reply", content=content,
                                            parent=msg, ident=client_id)


Instruction:
Can you create a Python 3 function that
performs a raw query on the task record database.

Response:
def db_query(self, client_id, msg):
        """Perform a raw query on the task record database."""
        content = msg['content']
        query = content.get('query', {})
        keys = content.get('keys', None)
        buffers = []
        empty = list()
        try:
            records = self.db.find_records(query, keys)
        except Exception as e:
            content = error.wrap_exception()
        else:
            # extract buffers from reply content:
            if keys is not None:
                buffer_lens = [] if 'buffers' in keys else None
                result_buffer_lens = [] if 'result_buffers' in keys else None
            else:
                buffer_lens = None
                result_buffer_lens = None

            for rec in records:
                # buffers may be None, so double check
                b = rec.pop('buffers', empty) or empty
                if buffer_lens is not None:
                    buffer_lens.append(len(b))
                    buffers.extend(b)
                rb = rec.pop('result_buffers', empty) or empty
                if result_buffer_lens is not None:
                    result_buffer_lens.append(len(rb))
                    buffers.extend(rb)
            content = dict(status='ok', records=records, buffer_lens=buffer_lens,
                                    result_buffer_lens=result_buffer_lens)
        # self.log.debug (content)
        self.session.send(self.query, "db_reply", content=content,
                                            parent=msg, ident=client_id,
                                            buffers=buffers)


Instruction:
Can you implement a function in Python 3 that
changes the path of the current directory to newdir

Response:
def cd(self, newdir):
        """
        go to the path
        """
        prevdir = os.getcwd()
        os.chdir(newdir)
        try:
            yield
        finally:
            os.chdir(prevdir)


Instruction:
Create a Python 3 function to
decode a command output into a ParsedCompletedCommand object.

Response:
def decode_cmd_out(self, completed_cmd):
        """
        return a standard message
        """
        try:
            stdout = completed_cmd.stdout.encode('utf-8').decode()
        except AttributeError:
            try:
                stdout = str(bytes(completed_cmd.stdout), 'big5').strip()
            except AttributeError:
                stdout = str(bytes(completed_cmd.stdout).decode('utf-8')).strip()
        try:
            stderr = completed_cmd.stderr.encode('utf-8').decode()
        except AttributeError:
            try:
                stderr = str(bytes(completed_cmd.stderr), 'big5').strip()
            except AttributeError:
                stderr = str(bytes(completed_cmd.stderr).decode('utf-8')).strip()
        return ParsedCompletedCommand(
            completed_cmd.returncode,
            completed_cmd.args,
            stdout,
            stderr
        )


Instruction:
Can you implement a function in Python 3 that
runs command under the r - root directory

Response:
def run_command_under_r_root(self, cmd, catched=True):
        """
        subprocess run on here
        """
        RPATH = self.path
        with self.cd(newdir=RPATH):
            if catched:
                process = sp.run(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
            else:
                process = sp.run(cmd)
            return process


Instruction:
Here you have a function in Python 3, explain what it does
def execute(self):
        rprocess = OrderedDict()
        commands = OrderedDict([
            (self.file, ['Rscript', self.file] + self.cmd),
        ])
        for cmd_name, cmd in commands.items():
            rprocess[cmd_name] = self.run_command_under_r_root(cmd)

        return self.decode_cmd_out(completed_cmd=rprocess[self.file])

Response: Execute R script and return a list of results.


Instruction:
How would you explain what the following Python 3 function does
def _set_kernel_manager(self, kernel_manager):
        # Disconnect the old kernel manager, if necessary.
        old_manager = self._kernel_manager
        if old_manager is not None:
            old_manager.started_kernel.disconnect(self._started_kernel)
            old_manager.started_channels.disconnect(self._started_channels)
            old_manager.stopped_channels.disconnect(self._stopped_channels)

            # Disconnect the old kernel manager's channels.
            old_manager.sub_channel.message_received.disconnect(self._dispatch)
            old_manager.shell_channel.message_received.disconnect(self._dispatch)
            old_manager.stdin_channel.message_received.disconnect(self._dispatch)
            old_manager.hb_channel.kernel_died.disconnect(
                self._handle_kernel_died)

            # Handle the case where the old kernel manager is still listening.
            if old_manager.channels_running:
                self._stopped_channels()

        # Set the new kernel manager.
        self._kernel_manager = kernel_manager
        if kernel_manager is None:
            return

        # Connect the new kernel manager.
        kernel_manager.started_kernel.connect(self._started_kernel)
        kernel_manager.started_channels.connect(self._started_channels)
        kernel_manager.stopped_channels.connect(self._stopped_channels)

        # Connect the new kernel manager's channels.
        kernel_manager.sub_channel.message_received.connect(self._dispatch)
        kernel_manager.shell_channel.message_received.connect(self._dispatch)
        kernel_manager.stdin_channel.message_received.connect(self._dispatch)
        kernel_manager.hb_channel.kernel_died.connect(self._handle_kernel_died)

        # Handle the case where the kernel manager started channels before
        # we connected.
        if kernel_manager.channels_running:
            self._started_channels()

Response: Connect to the current kernel manager and set a new kernel manager.


Instruction:
Can you tell what is the following Python 3 function doing
def _dispatch(self, msg):
        msg_type = msg['header']['msg_type']
        handler = getattr(self, '_handle_' + msg_type, None)
        if handler:
            handler(msg)

Response:
Dispatches the message to the frontend handler associated with the message type of the
            given message.


Instruction:
Write a Python 3 script for
returning whether a reply from the kernel originated from a request from this frontend.

Response:
def _is_from_this_session(self, msg):
        """ Returns whether a reply from the kernel originated from a request
            from this frontend.
        """
        session = self._kernel_manager.session.session
        parent = msg['parent_header']
        if not parent:
            # if the message has no parent, assume it is meant for all frontends
            return True
        else:
            return parent.get('session') == session


Instruction:
Here you have a function in Python 3, explain what it does
def report(self, morfs, directory=None):
        self.report_files(self.annotate_file, morfs, directory)

Response:
Run the report.

        See `coverage.report()` for arguments.


Instruction:
Given the following Python 3 function, write the documentation
def annotate_file(self, cu, analysis):
        if not cu.relative:
            return

        filename = cu.filename
        source = cu.source_file()
        if self.directory:
            dest_file = os.path.join(self.directory, cu.flat_rootname())
            dest_file += ".py,cover"
        else:
            dest_file = filename + ",cover"
        dest = open(dest_file, 'w')

        statements = sorted(analysis.statements)
        missing = sorted(analysis.missing)
        excluded = sorted(analysis.excluded)

        lineno = 0
        i = 0
        j = 0
        covered = True
        while True:
            line = source.readline()
            if line == '':
                break
            lineno += 1
            while i < len(statements) and statements[i] < lineno:
                i += 1
            while j < len(missing) and missing[j] < lineno:
                j += 1
            if i < len(statements) and statements[i] == lineno:
                covered = j >= len(missing) or missing[j] > lineno
            if self.blank_re.match(line):
                dest.write('  ')
            elif self.else_re.match(line):
                # Special logic for lines containing only 'else:'.
                if i >= len(statements) and j >= len(missing):
                    dest.write('! ')
                elif i >= len(statements) or j >= len(missing):
                    dest.write('> ')
                elif statements[i] == missing[j]:
                    dest.write('! ')
                else:
                    dest.write('> ')
            elif lineno in excluded:
                dest.write('- ')
            elif covered:
                dest.write('> ')
            else:
                dest.write('! ')
            dest.write(line)
        source.close()
        dest.close()

Response: Annotate a single file with the information from the analysis.


Instruction:
Can you generate the documentation for the following Python 3 function
def find(name):
    '''
        returns a list of tuples (package_name, description) for apt-cache search results
    '''
    cmd = 'apt-cache search %s' % name
    args = shlex.split(cmd)
    try:
        output = subprocess.check_output(args)
    except CalledProcessError:
        return []
    lines = output.splitlines()
    packages = []
    for line in lines:
        package, _, desc = line.partition(' - ')
        packages.append((package, desc))
    return packages

Response: returns a list of tuples ( package_name description ) for apt - cache search results


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_installed_version(name):
    '''
        returns installed package version and None if package is not installed
    '''
    pattern = re.compile(r'''Installed:\s+(?P<version>.*)''')
    cmd = 'apt-cache policy %s' % name
    args = shlex.split(cmd)
    try:
        output = subprocess.check_output(args)
        if not output:
            return None
    except CalledProcessError:
        return None
    # check output
    match = pattern.search(output)
    if match:
        version = match.groupdict()['version']
        if version == '(none)':
            return None
        else:
            return version

Response: Returns installed version and None if package is not installed


Instruction:
Can you generate a brief explanation for the following Python 3 code
def squash_unicode(obj):
    if isinstance(obj,dict):
        for key in obj.keys():
            obj[key] = squash_unicode(obj[key])
            if isinstance(key, unicode):
                obj[squash_unicode(key)] = obj.pop(key)
    elif isinstance(obj, list):
        for i,v in enumerate(obj):
            obj[i] = squash_unicode(v)
    elif isinstance(obj, unicode):
        obj = obj.encode('utf8')
    return obj

Response: coerce unicode back to bytestrings.


Instruction:
Write a Python 3 script for
setting the default behavior for a config environment to be secure.

Response:
def default_secure(cfg):
    """Set the default behavior for a config environment to be secure.

    If Session.key/keyfile have not been set, set Session.key to
    a new random UUID.
    """

    if 'Session' in cfg:
        if 'key' in cfg.Session or 'keyfile' in cfg.Session:
            return
    # key/keyfile not specified, generate new UUID:
    cfg.Session.key = str_to_bytes(str(uuid.uuid4()))


Instruction:
Write a Python 3 script for
giving a message or header return the header.

Response:
def extract_header(msg_or_header):
    """Given a message or header, return the header."""
    if not msg_or_header:
        return {}
    try:
        # See if msg_or_header is the entire message.
        h = msg_or_header['header']
    except KeyError:
        try:
            # See if msg_or_header is just the header
            h = msg_or_header['msg_id']
        except KeyError:
            raise
        else:
            h = msg_or_header
    if not isinstance(h, dict):
        h = dict(h)
    return h


Instruction:
How would you code a function in Python 3 to
check packers for binary data and datetime support.

Response:
def _check_packers(self):
        """check packers for binary data and datetime support."""
        pack = self.pack
        unpack = self.unpack

        # check simple serialization
        msg = dict(a=[1,'hi'])
        try:
            packed = pack(msg)
        except Exception:
            raise ValueError("packer could not serialize a simple message")

        # ensure packed message is bytes
        if not isinstance(packed, bytes):
            raise ValueError("message packed to %r, but bytes are required"%type(packed))

        # check that unpack is pack's inverse
        try:
            unpacked = unpack(packed)
        except Exception:
            raise ValueError("unpacker could not handle the packer's output")

        # check datetime support
        msg = dict(t=datetime.now())
        try:
            unpacked = unpack(pack(msg))
        except Exception:
            self.pack = lambda o: pack(squash_dates(o))
            self.unpack = lambda s: extract_dates(unpack(s))


Instruction:
Can you generate the documentation for the following Python 3 function
def sign(self, msg_list):
        if self.auth is None:
            return b''
        h = self.auth.copy()
        for m in msg_list:
            h.update(m)
        return str_to_bytes(h.hexdigest())

Response: Sign a message with HMAC digest.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def serialize(self, msg, ident=None):
        content = msg.get('content', {})
        if content is None:
            content = self.none
        elif isinstance(content, dict):
            content = self.pack(content)
        elif isinstance(content, bytes):
            # content is already packed, as in a relayed message
            pass
        elif isinstance(content, unicode):
            # should be bytes, but JSON often spits out unicode
            content = content.encode('utf8')
        else:
            raise TypeError("Content incorrect type: %s"%type(content))

        real_message = [self.pack(msg['header']),
                        self.pack(msg['parent_header']),
                        content
        ]

        to_send = []

        if isinstance(ident, list):
            # accept list of idents
            to_send.extend(ident)
        elif ident is not None:
            to_send.append(ident)
        to_send.append(DELIM)

        signature = self.sign(real_message)
        to_send.append(signature)

        to_send.extend(real_message)

        return to_send

Response: Serialize the message components to bytes.


Instruction:
Explain what the following Python 3 code does
def send(self, stream, msg_or_type, content=None, parent=None, ident=None,
             buffers=None, subheader=None, track=False, header=None):
        """Build and send a message via stream or socket.

        The message format used by this function internally is as follows:

        [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
         buffer1,buffer2,...]

        The serialize/unserialize methods convert the nested message dict into this
        format.

        Parameters
        ----------

        stream : zmq.Socket or ZMQStream
            The socket-like object used to send the data.
        msg_or_type : str or Message/dict
            Normally, msg_or_type will be a msg_type unless a message is being
            sent more than once. If a header is supplied, this can be set to
            None and the msg_type will be pulled from the header.

        content : dict or None
            The content of the message (ignored if msg_or_type is a message).
        header : dict or None
            The header dict for the message (ignores if msg_to_type is a message).
        parent : Message or dict or None
            The parent or parent header describing the parent of this message
            (ignored if msg_or_type is a message).
        ident : bytes or list of bytes
            The zmq.IDENTITY routing path.
        subheader : dict or None
            Extra header keys for this message's header (ignored if msg_or_type
            is a message).
        buffers : list or None
            The already-serialized buffers to be appended to the message.
        track : bool
            Whether to track.  Only for use with Sockets, because ZMQStream
            objects cannot track messages.

        Returns
        -------
        msg : dict
            The constructed message.
        (msg,tracker) : (dict, MessageTracker)
            if track=True, then a 2-tuple will be returned,
            the first element being the constructed
            message, and the second being the MessageTracker

        """

        if not isinstance(stream, (zmq.Socket, ZMQStream)):
            raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream))
        elif track and isinstance(stream, ZMQStream):
            raise TypeError("ZMQStream cannot track messages")

        if isinstance(msg_or_type, (Message, dict)):
            # We got a Message or message dict, not a msg_type so don't
            # build a new Message.
            msg = msg_or_type
        else:
            msg = self.msg(msg_or_type, content=content, parent=parent,
                           subheader=subheader, header=header)

        buffers = [] if buffers is None else buffers
        to_send = self.serialize(msg, ident)
        flag = 0
        if buffers:
            flag = zmq.SNDMORE
            _track = False
        else:
            _track=track
        if track:
            tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)
        else:
            tracker = stream.send_multipart(to_send, flag, copy=False)
        for b in buffers[:-1]:
            stream.send(b, flag, copy=False)
        if buffers:
            if track:
                tracker = stream.send(buffers[-1], copy=False, track=track)
            else:
                tracker = stream.send(buffers[-1], copy=False)

        # omsg = Message(msg)
        if self.debug:
            pprint.pprint(msg)
            pprint.pprint(to_send)
            pprint.pprint(buffers)

        msg['tracker'] = tracker

        return msg

Response: Builds and sends a message via a socket or a message.


Instruction:
Here you have a function in Python 3, explain what it does
def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
        to_send = []
        if isinstance(ident, bytes):
            ident = [ident]
        if ident is not None:
            to_send.extend(ident)

        to_send.append(DELIM)
        to_send.append(self.sign(msg_list))
        to_send.extend(msg_list)
        stream.send_multipart(msg_list, flags, copy=copy)

Response: Send a message via the raw socket.


Instruction:
How would you explain what the following Python 3 function does
def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
        if isinstance(socket, ZMQStream):
            socket = socket.socket
        try:
            msg_list = socket.recv_multipart(mode, copy=copy)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # We can convert EAGAIN to None as we know in this case
                # recv_multipart won't return None.
                return None,None
            else:
                raise
        # split multipart message into identity list and message dict
        # invalid large messages can cause very expensive string comparisons
        idents, msg_list = self.feed_identities(msg_list, copy)
        try:
            return idents, self.unserialize(msg_list, content=content, copy=copy)
        except Exception as e:
            # TODO: handle it
            raise e

Response: Receive and unpack a message from a socket.