Instruction:
Can you implement a function in Python 3 that
creates a temporary project for each region in enabled_regions and creates a temporary project for each region in enabled_regions.

Response:
def _create_temporary_projects(enabled_regions, args):
    """
    Creates a temporary project needed to build an underlying workflow
    for a global workflow. Returns a dictionary with region names as keys
    and project IDs as values

    The regions in which projects will be created can be:
    i. regions specified in dxworkflow.json "regionalOptions"
    ii. regions specified as an argument to "dx build"
    iii. current context project, if None of the above are set
    If both args and dxworkflow.json specify regions, they must match.
    """
    # Create one temp project in each region
    projects_by_region = {}  # Project IDs by region
    for region in enabled_regions:
        try:
            project_input = {"name": "Temporary build project for dx build global workflow",
                             "region": region}
            if args.bill_to:
                project_input["billTo"] = args.bill_to
            temp_project = dxpy.api.project_new(project_input)["id"]
            projects_by_region[region] = temp_project
            logger.debug("Created temporary project {} to build in".format(temp_project))
        except:
            # Clean up any temp projects that might have been created
            if projects_by_region:
                dxpy.executable_builder.delete_temporary_projects(projects_by_region.values())
            err_exit()
    return projects_by_region


Instruction:
Given the following Python 3 function, write the documentation
def _build_underlying_workflows(enabled_regions, json_spec, args):
    projects_by_region = _create_temporary_projects(enabled_regions, args)
    workflows_by_region = {}

    try:
        for region, project in projects_by_region.items():
            json_spec['project'] = project
            workflow_id = _build_regular_workflow(json_spec)
            logger.debug("Created workflow " + workflow_id + " successfully")
            workflows_by_region[region] = workflow_id
    except:
        # Clean up
        if projects_by_region:
            dxpy.executable_builder.delete_temporary_projects(projects_by_region.values())
        raise

    return workflows_by_region, projects_by_region

Response: Creates a workflow in a temporary project for each enabled region.


Instruction:
Implement a Python 3 function for
building a global workflow for each enabled region and creates a new global workflow on the platform based on these workflows.

Response:
def _build_global_workflow(json_spec, args):
    """
    Creates a workflow in a temporary project for each enabled region
    and builds a global workflow on the platform based on these workflows.
    """
    # First determine in which regions the global workflow needs to be available
    enabled_regions = _get_validated_enabled_regions(json_spec, args.region)

    # Verify all the stages are also enabled in these regions
    # TODO: Add support for dx building multi-region global workflows with applets
    _assert_executable_regions_match(enabled_regions, json_spec)

    workflows_by_region, projects_by_region = {}, {}  # IDs by region
    try:
        # prepare "regionalOptions" field for the globalworkflow/new input
        workflows_by_region, projects_by_region = \
            _build_underlying_workflows(enabled_regions, json_spec, args)
        regional_options = {}
        for region, workflow_id in workflows_by_region.items():
            regional_options[region] = {'workflow': workflow_id}
        json_spec.update({'regionalOptions': regional_options})

        # leave only fields that are actually used to build the workflow
        gwf_provided_keys = GLOBALWF_SUPPORTED_KEYS.intersection(set(json_spec.keys()))
        gwf_final_json = dict((k, v) for k, v in json_spec.items() if k in gwf_provided_keys)

        # we don't want to print the whole documentation to the screen so we'll remove these fields
        print_spec = copy.deepcopy(gwf_final_json)
        if "description" in gwf_final_json:
            del print_spec["description"]
        if "developerNotes" in gwf_final_json:
            del print_spec["developerNotes"]
        logger.info("Will create global workflow with spec: {}".format(json.dumps(print_spec)))

        # Create a new global workflow version on the platform
        global_workflow_id = dxpy.api.global_workflow_new(gwf_final_json)["id"]

        logger.info("Uploaded global workflow {n}/{v} successfully".format(n=gwf_final_json["name"],
                                                                           v=gwf_final_json["version"]))
        logger.info("You can publish this workflow with:")
        logger.info("  dx publish {n}/{v}".format(n=gwf_final_json["name"],
                                                  v=gwf_final_json["version"]))
    finally:
        # Clean up
        if projects_by_region:
            dxpy.executable_builder.delete_temporary_projects(projects_by_region.values())

    # Set any additional fields on the created workflow
    try:
        _set_categories_on_workflow(global_workflow_id, gwf_final_json.get("categories", []))
    except:
        logger.warn(
            "The workflow {n}/{v} was created but setting categories failed".format(n=gwf_final_json['name'],
                                                                                v=gwf_final_json['version']))
        raise

    return global_workflow_id


Instruction:
Create a Python 3 function to
build or updates a workflow on the platform.

Response:
def _build_or_update_workflow(json_spec, args):
    """
    Creates or updates a workflow on the platform.
    Returns the workflow ID, or None if the workflow cannot be created.
    """
    try:
        if args.mode == 'workflow':
            json_spec = _get_validated_json(json_spec, args)
            workflow_id = _build_regular_workflow(json_spec)
        elif args.mode == 'globalworkflow':
            # Verify if the global workflow already exists and if the user has developer rights to it
            # If the global workflow name doesn't exist, the user is free to build it
            # If the name does exist two things can be done:
            # * either update the requested version, if this version already exists
            # * or create the version if it doesn't exist
            existing_workflow = dxpy.executable_builder.verify_developer_rights('globalworkflow-' + json_spec['name'])
            if existing_workflow and _version_exists(json_spec,
                                                     existing_workflow.name,
                                                     existing_workflow.version):
                workflow_id = _update_global_workflow(json_spec, args, existing_workflow.id)
            else:
                json_spec = _get_validated_json(json_spec, args)
                workflow_id = _build_global_workflow(json_spec, args)
        else:
            raise WorkflowBuilderException("Unrecognized workflow type: {}".format(args.mode))
    except dxpy.exceptions.DXAPIError as e:
        raise e
    return workflow_id


Instruction:
Given the following Python 3 function, write the documentation
def build(args, parser):

    if args is None:
        raise Exception("arguments not provided")

    try:
        json_spec = _parse_executable_spec(args.src_dir, "dxworkflow.json", parser)
        workflow_id = _build_or_update_workflow(json_spec, args)
        _print_output(workflow_id, args)
    except WorkflowBuilderException as e:
        print("Error: %s" % (e.args,), file=sys.stderr)
        sys.exit(3)

Response: Builds a new global workflow based on the given arguments.


Instruction:
How would you explain what the following Python 3 function does
def cp_to_noexistent_destination(args, dest_path, dx_dest, dest_proj):
    ''' Copy the source to a destination that does not currently
    exist. This involves creating the target file/folder.
    '''
    # Destination folder path is new => renaming
    if len(args.sources) != 1:
        # Can't copy and rename more than one object
        raise DXCLIError('The destination folder does not exist')
    last_slash_pos = get_last_pos_of_char('/', dest_path)
    if last_slash_pos == 0:
        dest_folder = '/'
    else:
        dest_folder = dest_path[:last_slash_pos]
    dest_name = dest_path[last_slash_pos + 1:].replace('\/', '/')
    try:
        dx_dest.list_folder(folder=dest_folder, only='folders')
    except dxpy.DXAPIError as details:
        if details.code == requests.codes['not_found']:
            raise DXCLIError('The destination folder does not exist')
        else:
            raise
    except:
        err_exit()

    # Clone and rename either the data object or the folder.
    # src_result is None if it could not be resolved to an object.
    src_proj, src_path, src_results = try_call(resolve_existing_path,
                                               args.sources[0],
                                               allow_mult=True, all_mult=args.all)

    if src_proj == dest_proj:
        if is_hashid(args.sources[0]):
            # This is the only case in which the source project is
            # purely assumed, so give a better error message.
            raise DXCLIError(fill('Error: You must specify a source project for ' + args.sources[0]))
        else:
            raise DXCLIError(fill('A source path and the destination path resolved to the ' +
                                'same project or container.  Please specify different source ' +
                                'and destination containers, e.g.') +
                             '\n  dx cp source-project:source-id-or-path dest-project:dest-path')

    if src_results is None:
        try:
            contents = dxpy.api.project_list_folder(src_proj,
                                                    {"folder": src_path, "includeHidden": True})
            dxpy.api.project_new_folder(dest_proj, {"folder": dest_path})
            exists = dxpy.api.project_clone(src_proj,
                                            {"folders": contents['folders'],
                                             "objects": [result['id'] for result in contents['objects']],
                                             "project": dest_proj,
                                             "destination": dest_path})['exists']
            if len(exists) > 0:
                print(fill('The following objects already existed in the destination ' +
                           'container and were not copied:') + '\n ' + '\n '.join(exists))
                return
        except:
            err_exit()
    else:
        try:
            exists = dxpy.api.project_clone(src_proj,
                                            {"objects": [result['id'] for result in src_results],
                                             "project": dest_proj,
                                             "destination": dest_folder})['exists']
            if len(exists) > 0:
                print(fill('The following objects already existed in the destination ' +
                           'container and were not copied:') + '\n ' + '\n '.join(exists))
            for result in src_results:
                if result['id'] not in exists:
                    dxpy.DXHTTPRequest('/' + result['id'] + '/rename',
                                       {"project": dest_proj,
                                        "name": dest_name})
            return
        except:
            err_exit()

Response: Copy the source to a destination that does not currently have a corresponding object.


Instruction:
How would you code a function in Python 3 to
return the file size in readable form.

Response:
def _readable_part_size(num_bytes):
    "Returns the file size in readable form."
    B = num_bytes
    KB = float(1024)
    MB = float(KB * 1024)
    GB = float(MB * 1024)
    TB = float(GB * 1024)

    if B < KB:
        return '{0} {1}'.format(B, 'bytes' if B != 1 else 'byte')
    elif KB <= B < MB:
        return '{0:.2f} KiB'.format(B/KB)
    elif MB <= B < GB:
        return '{0:.2f} MiB'.format(B/MB)
    elif GB <= B < TB:
        return '{0:.2f} GiB'.format(B/GB)
    elif TB <= B:
        return '{0:.2f} TiB'.format(B/TB)


Instruction:
Here you have a function in Python 3, explain what it does
def _new(self, dx_hash, media_type=None, **kwargs):

        if media_type is not None:
            dx_hash["media"] = media_type

        resp = dxpy.api.file_new(dx_hash, **kwargs)
        self.set_ids(resp["id"], dx_hash["project"])

Response: Create a new remote file with the given hash and media type.


Instruction:
How would you code a function in Python 3 to
set the IDs of the currently stored object.

Response:
def set_ids(self, dxid, project=None):
        '''
        :param dxid: Object ID
        :type dxid: string
        :param project: Project ID
        :type project: string

        Discards the currently stored ID and associates the handler with
        *dxid*. As a side effect, it also flushes the buffer for the
        previous file object if the buffer is nonempty.
        '''
        if self._dxid is not None:
            self.flush()

        DXDataObject.set_ids(self, dxid, project)

        # Reset state
        self._pos = 0
        self._file_length = None
        self._cur_part = 1
        self._num_uploaded_parts = 0


Instruction:
How would you implement a function in Python 3 that
seeks to a specific offset in the file.

Response:
def seek(self, offset, from_what=os.SEEK_SET):
        '''
        :param offset: Position in the file to seek to
        :type offset: integer

        Seeks to *offset* bytes from the beginning of the file.  This is a no-op if the file is open for writing.

        The position is computed from adding *offset* to a reference point; the reference point is selected by the
        *from_what* argument. A *from_what* value of 0 measures from the beginning of the file, 1 uses the current file
        position, and 2 uses the end of the file as the reference point. *from_what* can be omitted and defaults to 0,
        using the beginning of the file as the reference point.
        '''
        if from_what == os.SEEK_SET:
            reference_pos = 0
        elif from_what == os.SEEK_CUR:
            reference_pos = self._pos
        elif from_what == os.SEEK_END:
            if self._file_length == None:
                desc = self.describe()
                self._file_length = int(desc["size"])
            reference_pos = self._file_length
        else:
            raise DXFileError("Invalid value supplied for from_what")

        orig_pos = self._pos
        self._pos = reference_pos + offset

        in_buf = False
        orig_buf_pos = self._read_buf.tell()
        if offset < orig_pos:
            if orig_buf_pos > orig_pos - offset:
                # offset is less than original position but within the buffer
                in_buf = True
        else:
            buf_len = dxpy.utils.string_buffer_length(self._read_buf)
            if buf_len - orig_buf_pos > offset - orig_pos:
                # offset is greater than original position but within the buffer
                in_buf = True

        if in_buf:
            # offset is within the buffer (at least one byte following
            # the offset can be read directly out of the buffer)
            self._read_buf.seek(orig_buf_pos - orig_pos + offset)
        elif offset == orig_pos:
            # This seek is a no-op (the cursor is just past the end of
            # the read buffer and coincides with the desired seek
            # position). We don't have the data ready, but the request
            # for the data starting here is already in flight.
            #
            # Detecting this case helps to optimize for sequential read
            # access patterns.
            pass
        else:
            # offset is outside the buffer-- reset buffer and queues.
            # This is the failsafe behavior
            self._read_buf = BytesIO()
            # TODO: if the offset is within the next response(s), don't throw out the queues
            self._request_iterator, self._response_iterator = None, None


Instruction:
Here you have a function in Python 3, explain what it does
def flush(self, multithread=True, **kwargs):
        '''
        Flushes the internal write buffer.
        '''
        if self._write_buf.tell() > 0:
            data = self._write_buf.getvalue()
            self._write_buf = BytesIO()

            if multithread:
                self._async_upload_part_request(data, index=self._cur_part, **kwargs)
            else:
                self.upload_part(data, self._cur_part, **kwargs)

            self._cur_part += 1

        if len(self._http_threadpool_futures) > 0:
            dxpy.utils.wait_for_all_futures(self._http_threadpool_futures)
            try:
                for future in self._http_threadpool_futures:
                    if future.exception() != None:
                        raise future.exception()
            finally:
                self._http_threadpool_futures = set()

Response: Flushes the internal write buffer.


Instruction:
Can you tell what is the following Python 3 function doing
def _write2(self, data, multithread=True, **kwargs):
        '''
        :param data: Data to be written
        :type data: str or mmap object
        :param multithread: If True, sends multiple write requests asynchronously
        :type multithread: boolean

        Writes the data *data* to the file.

        .. note::

            Writing to remote files is append-only. Using :meth:`seek`
            does not affect where the next :meth:`write` will occur.

        '''
        if not USING_PYTHON2:
            assert(isinstance(data, bytes))

        self._ensure_write_bufsize(**kwargs)

        def write_request(data_for_write_req):
            if multithread:
                self._async_upload_part_request(data_for_write_req, index=self._cur_part, **kwargs)
            else:
                self.upload_part(data_for_write_req, self._cur_part, **kwargs)
            self._cur_part += 1

        if self._write_buf.tell() == 0 and self._write_bufsize == len(data):
            # In the special case of a write that is the same size as
            # our write buffer size, and no unflushed data in the
            # buffer, just directly dispatch the write and bypass the
            # write buffer.
            #
            # This saves a buffer copy, which is especially helpful if
            # 'data' is actually mmap'd from a file.
            #
            # TODO: an additional optimization could be made to allow
            # the last request from an mmap'd upload to take this path
            # too (in general it won't because it's not of length
            # _write_bufsize). This is probably inconsequential though.
            write_request(data)
            return

        remaining_space = self._write_bufsize - self._write_buf.tell()

        if len(data) <= remaining_space:
            self._write_buf.write(data)
        else:
            self._write_buf.write(data[:remaining_space])

            temp_data = self._write_buf.getvalue()
            self._write_buf = BytesIO()
            write_request(temp_data)

            # TODO: check if repeat string splitting is bad for
            # performance when len(data) >> _write_bufsize
            self.write(data[remaining_space:], **kwargs)

Response: Write to the file.


Instruction:
Here you have a function in Python 3, explain what it does
def write(self, data, multithread=True, **kwargs):
        '''
        :param data: Data to be written
        :type data: str or mmap object
        :param multithread: If True, sends multiple write requests asynchronously
        :type multithread: boolean

        Writes the data *data* to the file.

        .. note::

            Writing to remote files is append-only. Using :meth:`seek`
            does not affect where the next :meth:`write` will occur.

        '''
        if USING_PYTHON2:
            self._write2(data, multithread=multithread, **kwargs)
        else:
            # In python3, the underlying system methods use the 'bytes' type, not 'string'
            #
            # This is, hopefully, a temporary hack. It is not a good idea for two reasons:
            # 1) Performance, we need to make a pass on the data, and need to allocate
            #    another buffer of similar size
            # 2) The types are wrong. The "bytes" type should be visible to the caller
            #    of the write method, instead of being hidden.

            # Should we throw an exception if the file is opened in binary mode,
            # and the data is unicode/text?
            if isinstance(data, str):
                bt = data.encode("utf-8")
            elif isinstance(data, bytearray):
                bt = bytes(data)
            elif isinstance(data, bytes):
                bt = data
            elif isinstance(data, mmap.mmap):
                bt = bytes(data)
            else:
                raise DXFileError("Invalid type {} for write data argument".format(type(data)))
            assert(isinstance(bt, bytes))
            self._write2(bt, multithread=multithread, **kwargs)

Response: Writes the data to the file.


Instruction:
Can you tell what is the following Python 3 function doing
def close(self, block=False, **kwargs):
        '''
        :param block: If True, this function blocks until the remote file has closed.
        :type block: boolean

        Attempts to close the file.

        .. note:: The remote file cannot be closed until all parts have
           been fully uploaded. An exception will be thrown if this is
           not the case.
        '''
        self.flush(**kwargs)

        # Also populates emptyLastPartAllowed
        self._ensure_write_bufsize(**kwargs)

        if self._num_uploaded_parts == 0 and self._empty_last_part_allowed:
            # We haven't uploaded any parts in this session.
            # In case no parts have been uploaded at all and region
            # settings allow last empty part upload, try to upload
            # an empty part (otherwise files with 0 parts cannot be closed).
            try:
                if USING_PYTHON2:
                    self.upload_part('', 1, **kwargs)
                else:
                    self.upload_part(b'', 1, **kwargs)
            except dxpy.exceptions.InvalidState:
                pass

        if 'report_progress_fn' in kwargs:
            del kwargs['report_progress_fn']

        dxpy.api.file_close(self._dxid, **kwargs)

        if block:
            self._wait_on_close(**kwargs)

Response: Closes the remote file.


Instruction:
Explain what the following Python 3 code does
def upload_part(self, data, index=None, display_progress=False, report_progress_fn=None, **kwargs):
        if not USING_PYTHON2:
            # In python3, the underlying system methods use the 'bytes' type, not 'string'
            assert(isinstance(data, bytes))

        req_input = {}
        if index is not None:
            req_input["index"] = int(index)

        md5 = hashlib.md5()
        if hasattr(data, 'seek') and hasattr(data, 'tell'):
            # data is a buffer; record initial position (so we can rewind back)
            rewind_input_buffer_offset = data.tell()
            while True:
                bytes_read = data.read(MD5_READ_CHUNK_SIZE)
                if bytes_read:
                    md5.update(bytes_read)
                else:
                    break
            # rewind the buffer to original position
            data.seek(rewind_input_buffer_offset)
        else:
            md5.update(data)

        req_input["md5"] = md5.hexdigest()
        req_input["size"] = len(data)

        def get_upload_url_and_headers():
            # This function is called from within a retry loop, so to avoid amplifying the number of retries
            # geometrically, we decrease the allowed number of retries for the nested API call every time.
            if 'max_retries' not in kwargs:
                kwargs['max_retries'] = dxpy.DEFAULT_RETRIES
            elif kwargs['max_retries'] > 0:
                kwargs['max_retries'] -= 1

            if "timeout" not in kwargs:
                kwargs["timeout"] = FILE_REQUEST_TIMEOUT

            resp = dxpy.api.file_upload(self._dxid, req_input, **kwargs)
            url = resp["url"]
            return url, _validate_headers(resp.get("headers", {}))

        # The file upload API requires us to get a pre-authenticated upload URL (and headers for it) every time we
        # attempt an upload. Because DXHTTPRequest will retry requests under retryable conditions, we give it a callback
        # to ask us for a new upload URL every time it attempts a request (instead of giving them directly).
        dxpy.DXHTTPRequest(get_upload_url_and_headers,
                           data,
                           jsonify_data=False,
                           prepend_srv=False,
                           always_retry=True,
                           timeout=FILE_REQUEST_TIMEOUT,
                           auth=None,
                           method='PUT')

        self._num_uploaded_parts += 1

        if display_progress:
            warn(".")

        if report_progress_fn is not None:
            report_progress_fn(self, len(data))

Response: Uploads a part of data to the local file system.


Instruction:
Write a Python 3 function for
generating a download URL for the specified file and optional duration.

Response:
def get_download_url(self, duration=None, preauthenticated=False, filename=None, project=None, **kwargs):
        """
        :param duration: number of seconds for which the generated URL will be
            valid, should only be specified when preauthenticated is True
        :type duration: int
        :param preauthenticated: if True, generates a 'preauthenticated'
            download URL, which embeds authentication info in the URL and does
            not require additional headers
        :type preauthenticated: bool
        :param filename: desired filename of the downloaded file
        :type filename: str
        :param project: ID of a project containing the file (the download URL
            will be associated with this project, and this may affect which
            billing account is billed for this download).
            If no project is specified, an attempt will be made to verify if the file is
            in the project from the DXFile handler (as specified by the user or
            the current project stored in dxpy.WORKSPACE_ID). Otherwise, no hint is supplied.
            This fall back behavior does not happen inside a job environment.
            A non preauthenticated URL is only valid as long as the user has
            access to that project and the project contains that file.
        :type project: str
        :returns: download URL and dict containing HTTP headers to be supplied
            with the request
        :rtype: tuple (str, dict)
        :raises: :exc:`~dxpy.exceptions.ResourceNotFound` if a project context was
            given and the file was not found in that project context.
        :raises: :exc:`~dxpy.exceptions.ResourceNotFound` if no project context was
            given and the file was not found in any projects.

        Obtains a URL that can be used to directly download the associated
        file.

        """
        args = {"preauthenticated": preauthenticated}

        if duration is not None:
            args["duration"] = duration
        if filename is not None:
            args["filename"] = filename

        # If project=None, we fall back to the project attached to this handler
        # (if any). If this is supplied, it's treated as a hint: if it's a
        # project in which this file exists, it's passed on to the
        # apiserver. Otherwise, NO hint is supplied. In principle supplying a
        # project in the handler that doesn't contain this file ought to be an
        # error, but it's this way for backwards compatibility. We don't know
        # who might be doing downloads and creating handlers without being
        # careful that the project encoded in the handler contains the file
        # being downloaded. They may now rely on such behavior.
        if project is None and 'DX_JOB_ID' not in os.environ:
            project_from_handler = self.get_proj_id()
            if project_from_handler and object_exists_in_project(self.get_id(), project_from_handler):
                project = project_from_handler

        if project is not None and project is not DXFile.NO_PROJECT_HINT:
            args["project"] = project

        # Test hook to write 'project' argument passed to API call to a
        # local file
        if '_DX_DUMP_BILLED_PROJECT' in os.environ:
            with open(os.environ['_DX_DUMP_BILLED_PROJECT'], "w") as fd:
                if project is not None and project != DXFile.NO_PROJECT_HINT:
                    fd.write(project)

        with self._url_download_mutex:
            if self._download_url is None or self._download_url_expires < time.time():
                # The idea here is to cache a download URL for the entire file, that will
                # be good for a few minutes. This avoids each thread having to ask the
                # server for a URL, increasing server load.
                #
                # To avoid thread race conditions, this check/update procedure is protected
                # with a lock.

                # logging.debug("Download URL unset or expired, requesting a new one")
                if "timeout" not in kwargs:
                    kwargs["timeout"] = FILE_REQUEST_TIMEOUT
                resp = dxpy.api.file_download(self._dxid, args, **kwargs)
                self._download_url = resp["url"]
                self._download_url_headers = _validate_headers(resp.get("headers", {}))
                if preauthenticated:
                    self._download_url_expires = resp["expires"]/1000 - 60  # Try to account for drift
                else:
                    self._download_url_expires = 32503680000  # doesn't expire (year 3000)

            # Make a copy, ensuring each thread has its own mutable
            # version of the headers.  Note: python strings are
            # immutable, so we can safely give a reference to the
            # download url.
            retval_download_url = self._download_url
            retval_download_url_headers = copy.copy(self._download_url_headers)

        return retval_download_url, retval_download_url_headers


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _read2(self, length=None, use_compression=None, project=None, **kwargs):
        '''
        :param length: Maximum number of bytes to be read
        :type length: integer
        :param project: project to use as context for this download (may affect
            which billing account is billed for this download). If specified,
            must be a project in which this file exists. If not specified, the
            project ID specified in the handler is used for the download, IF it
            contains this file. If set to DXFile.NO_PROJECT_HINT, no project ID
            is supplied for the download, even if the handler specifies a
            project ID.
        :type project: str or None
        :rtype: string
        :raises: :exc:`~dxpy.exceptions.ResourceNotFound` if *project* is supplied
           and it does not contain this file

        Returns the next *length* bytes, or all the bytes until the end of file
        (if no *length* is given or there are fewer than *length* bytes left in
        the file).

        .. note:: After the first call to read(), the project arg and
           passthrough kwargs are not respected while using the same response
           iterator (i.e. until next seek).

        '''
        if self._file_length == None:
            desc = self.describe(**kwargs)
            if desc["state"] != "closed":
                raise DXFileError("Cannot read from file until it is in the closed state")
            self._file_length = int(desc["size"])

        # If running on a worker, wait for the first file download chunk
        # to come back before issuing any more requests. This ensures
        # that all subsequent requests can take advantage of caching,
        # rather than having all of the first DXFILE_HTTP_THREADS
        # requests simultaneously hit a cold cache. Enforce a minimum
        # size for this heuristic so we don't incur the overhead for
        # tiny files (which wouldn't contribute as much to the load
        # anyway).
        get_first_chunk_sequentially = (self._file_length > 128 * 1024 and self._pos == 0 and dxpy.JOB_ID)

        if self._pos == self._file_length:
            return b""

        if length == None or length > self._file_length - self._pos:
            length = self._file_length - self._pos

        buf = self._read_buf
        buf_remaining_bytes = dxpy.utils.string_buffer_length(buf) - buf.tell()
        if length <= buf_remaining_bytes:
            self._pos += length
            return buf.read(length)
        else:
            orig_buf_pos = buf.tell()
            orig_file_pos = self._pos
            buf.seek(0, os.SEEK_END)
            self._pos += buf_remaining_bytes
            while self._pos < orig_file_pos + length:
                remaining_len = orig_file_pos + length - self._pos

                if self._response_iterator is None:
                    self._request_iterator = self._generate_read_requests(
                        start_pos=self._pos, project=project, **kwargs)

                content = self._next_response_content(get_first_chunk_sequentially=get_first_chunk_sequentially)

                if len(content) < remaining_len:
                    buf.write(content)
                    self._pos += len(content)
                else: # response goes beyond requested length
                    buf.write(content[:remaining_len])
                    self._pos += remaining_len
                    self._read_buf = BytesIO()
                    self._read_buf.write(content[remaining_len:])
                    self._read_buf.seek(0)
            buf.seek(orig_buf_pos)
            return buf.read()

Response: Reads a file from the file and returns the next length bytes.


Instruction:
Here you have a function in Python 3, explain what it does
def setup_ssh_tunnel(job_id, local_port, remote_port):
    cmd = ['dx', 'ssh', '--suppress-running-check', job_id, '-o', 'StrictHostKeyChecking no']
    cmd += ['-f', '-L', '{0}:localhost:{1}'.format(local_port, remote_port), '-N']
    subprocess.check_call(cmd)

Response: Setup an ssh tunnel to the given job - id.


Instruction:
Create a Python 3 function for
polling for the server to start running and post the SERVER_READY_TAG.

Response:
def poll_for_server_running(job_id):
    """
    Poll for the job to start running and post the SERVER_READY_TAG.
    """
    sys.stdout.write('Waiting for server in {0} to initialize ...'.format(job_id))
    sys.stdout.flush()
    desc = dxpy.describe(job_id)
    # Keep checking until the server has begun or it has failed.
    while(SERVER_READY_TAG not in desc['tags'] and desc['state'] != 'failed'):
        time.sleep(SLEEP_PERIOD)
        sys.stdout.write('.')
        sys.stdout.flush()
        desc = dxpy.describe(job_id)

    # If the server job failed, provide friendly advice.
    if desc['state'] == 'failed':
        msg = RED('Error:') + ' Server failed to run.\n'
        msg += 'You may want to check the job logs by running:'
        msg += BOLD('dx watch {0}'.format(job_id))
        err_exit(msg)


Instruction:
Write a Python 3 function that can
open the given command and use the OS to automatically open the appropriate resource.

Response:
def multi_platform_open(cmd):
    """
    Take the given command and use the OS to automatically open the appropriate
    resource.  For instance, if a URL is provided, this will have the OS automatically
    open the URL in the default web browser.
    """
    if platform == "linux" or platform == "linux2":
        cmd = ['xdg-open', cmd]
    elif platform == "darwin":
        cmd = ['open', cmd]
    elif platform == "win32":
        cmd = ['start', cmd]
    subprocess.check_call(cmd)


Instruction:
Here you have a function in Python 3, explain what it does
def get_notebook_app_versions():
    notebook_apps = dxpy.find_apps(name=NOTEBOOK_APP, all_versions=True)
    versions = [str(dxpy.describe(app['id'])['version']) for app in notebook_apps]
    return versions

Response: Get the valid version numbers of the notebook app.


Instruction:
Explain what the following Python 3 code does
def run_notebook(args, ssh_config_check):
    # Check that ssh is setup.  Currently notebooks require ssh for tunelling.
    ssh_config_check()
    if args.only_check_config:
        return

    # If the user requested a specific version of the notebook server,
    # get the executable id.
    if args.version is not None:
        executable = get_app_from_path('app-{0}/{1}'.format(NOTEBOOK_APP, args.version))
        if executable is not None and 'id' in executable:
            executable = executable['id']
        else:
            msg = RED('Warning:') + ' Invalid notebook version: {0}\nValid versions are: '.format(args.version)
            msg += BOLD('{0}'.format(str(get_notebook_app_versions())))
            err_exit(msg)
    else:
        executable = 'app-{0}'.format(NOTEBOOK_APP)

    # Compose the command to launch the notebook
    cmd = ['dx', 'run', executable, '-inotebook_type={0}'.format(args.notebook_type)]
    cmd += ['-iinput_files={0}'.format(f) for f in args.notebook_files]
    cmd += ['-itimeout={0}'.format(args.timeout), '-y', '--brief', '--allow-ssh', '--instance-type', args.instance_type]
    if args.spark:
        cmd += ['-iinstall_spark=true']
    if args.snapshot:
        cmd += ['-isnapshot={0}'.format(args.snapshot)]
    job_id = subprocess.check_output(cmd).strip()

    poll_for_server_running(job_id)

    if args.notebook_type in {'jupyter', 'jupyter_lab', 'jupyter_notebook'}:
        remote_port = 8888

    setup_ssh_tunnel(job_id, args.port, remote_port)

    if args.open_server:
        multi_platform_open('http://localhost:{0}'.format(args.port))
        print('A web browser should have opened to connect you to your notebook.')
    print('If no browser appears, or if you need to reopen a browser at any point, you should be able to point your browser to http://localhost:{0}'.format(args.port))

Response: Launch the notebook server.


Instruction:
Write a Python 3 function that can
return the best fit of the given word in the given list of known words.

Response:
def correct(word, known_words):
    """
    :param word: Word to correct
    :type word: string
    :param known_words: List of known words
    :type known_words: iterable of strings

    Given **word**, suggests a correction from **known_words**. If no reasonably close correction is found, returns
    **word**.
    """
    NWORDS = _train(known_words)
    candidates = _known([word], NWORDS) or _known(_edits1(word), NWORDS) or _known_edits2(word, NWORDS) or [word]
    return max(candidates, key=NWORDS.get)


Instruction:
Write a Python 3 script to
validate user supplied filters.

Response:
def _validate_filters(cls, filters, model_class):
    """Validate user supplied filters.

    Validate filters are on existing properties and filter values
    have valid semantics.

    Args:
      filters: user supplied filters. Each filter should be a list or tuple of
        format (<property_name_as_str>, <query_operator_as_str>,
        <value_of_certain_type>). Value type is up to the property's type.
      model_class: the db.Model class for the entity type to apply filters on.

    Raises:
      BadReaderParamsError: if any filter is invalid in any way.
    """
    if not filters:
      return

    properties = model_class.properties()

    for f in filters:
      prop, _, val = f
      if prop not in properties:
        raise errors.BadReaderParamsError(
            "Property %s is not defined for entity type %s",
            prop, model_class.kind())

      # Validate the value of each filter. We need to know filters have
      # valid value to carry out splits.
      try:
        properties[prop].validate(val)
      except db.BadValueError, e:
        raise errors.BadReaderParamsError(e)


Instruction:
Implement a function in Python 3 to
validate ndb. Model filters.

Response:
def _validate_filters_ndb(cls, filters, model_class):
    """Validate ndb.Model filters."""
    if not filters:
      return

    properties = model_class._properties

    for f in filters:
      prop, _, val = f
      if prop not in properties:
        raise errors.BadReaderParamsError(
            "Property %s is not defined for entity type %s",
            prop, model_class._get_kind())

      # Validate the value of each filter. We need to know filters have
      # valid value to carry out splits.
      try:
        properties[prop]._do_validate(val)
      except db.BadValueError, e:
        raise errors.BadReaderParamsError(e)


Instruction:
Given the following Python 3 function, write the documentation
def _normalize_entity(value):
  if ndb is not None and isinstance(value, ndb.Model):
    return None
  if getattr(value, "_populate_internal_entity", None):
    return value._populate_internal_entity()
  return value

Response: Return an entity from an entity or model instance.


Instruction:
Implement a function in Python 3 to
return a key from an entity model instance key or key string.

Response:
def _normalize_key(value):
  """Return a key from an entity, model instance, key, or key string."""
  if ndb is not None and isinstance(value, (ndb.Model, ndb.Key)):
    return None
  if getattr(value, "key", None):
    return value.key()
  elif isinstance(value, basestring):
    return datastore.Key(value)
  else:
    return value


Instruction:
Explain what the following Python 3 code does
def append(self, item):
    if self.should_flush():
      self.flush()
    self.items.append(item)

Response: Adds a new item to the list. If needed flush existing items and clear existing items.


Instruction:
Can you generate the documentation for the following Python 3 function
def put(self, entity):
    actual_entity = _normalize_entity(entity)
    if actual_entity is None:
      return self.ndb_put(entity)
    self.puts.append(actual_entity)

Response: Registers entity to datastore.


Instruction:
Create a Python 3 function to
like put but for NDB entities.

Response:
def ndb_put(self, entity):
    """Like put(), but for NDB entities."""
    assert ndb is not None and isinstance(entity, ndb.Model)
    self.ndb_puts.append(entity)


Instruction:
How would you implement a function in Python 3 that
registers entity to be deleted from datastore.

Response:
def delete(self, entity):
    """Registers entity to delete from datastore.

    Args:
      entity: an entity, model instance, or key to delete.
    """
    key = _normalize_key(entity)
    if key is None:
      return self.ndb_delete(entity)
    self.deletes.append(key)


Instruction:
Make a summary of the following Python 3 code
def ndb_delete(self, entity_or_key):
    if ndb is not None and isinstance(entity_or_key, ndb.Model):
      key = entity_or_key.key
    else:
      key = entity_or_key
    self.ndb_deletes.append(key)

Response: Like delete but for NDB entities and keys.


Instruction:
Make a summary of the following Python 3 code
def flush(self):
    self.puts.flush()
    self.deletes.flush()
    self.ndb_puts.flush()
    self.ndb_deletes.flush()

Response: Flush all changed to datastore.


Instruction:
Make a summary of the following Python 3 code
def _flush_puts(self, items, options):
    datastore.Put(items, config=self._create_config(options))

Response: Flushes all puts to datastore.


Instruction:
Make a summary of the following Python 3 code
def _flush_deletes(self, items, options):
    datastore.Delete(items, config=self._create_config(options))

Response: Flush all deletes to datastore.


Instruction:
Can you generate the documentation for the following Python 3 function
def _flush_ndb_puts(self, items, options):
    assert ndb is not None
    ndb.put_multi(items, config=self._create_config(options))

Response: Flush all NDB puts to datastore.


Instruction:
How would you code a function in Python 3 to
flush all deletes to datastore.

Response:
def _flush_ndb_deletes(self, items, options):
    """Flush all deletes to datastore."""
    assert ndb is not None
    ndb.delete_multi(items, config=self._create_config(options))


Instruction:
Create a Python 3 function for
creating a datastore Config.

Response:
def _create_config(self, options):
    """Creates datastore Config.

    Returns:
      A datastore_rpc.Configuration instance.
    """
    return datastore.CreateConfig(deadline=options["deadline"],
                                  force_writes=self.force_writes)


Instruction:
How would you code a function in Python 3 to
increment the value of a counter.

Response:
def increment(self, counter_name, delta=1):
    """Increment counter value.

    Args:
      counter_name: name of the counter as string.
      delta: increment delta as int.
    """
    self._shard_state.counters_map.increment(counter_name, delta)


Instruction:
Write a Python 3 function for
serializing all states into json form.

Response:
def to_json(self):
    """Serializes all states into json form.

    Returns:
      all states in json-compatible map.
    """
    cursor = self._get_cursor()
    cursor_object = False
    if cursor and isinstance(cursor, datastore_query.Cursor):
      cursor = cursor.to_websafe_string()
      cursor_object = True
    return {"key_range": self._key_range.to_json(),
            "query_spec": self._query_spec.to_json(),
            "cursor": cursor,
            "cursor_object": cursor_object}


Instruction:
Can you tell what is the following Python 3 function doing
def find_mapreduce_yaml(status_file=__file__):
  checked = set()
  yaml = _find_mapreduce_yaml(os.path.dirname(status_file), checked)
  if not yaml:
    yaml = _find_mapreduce_yaml(os.getcwd(), checked)
  return yaml

Response: Traverse the directory trees to find mapreduce. yaml file.


Instruction:
Here you have a function in Python 3, explain what it does
def _find_mapreduce_yaml(start, checked):
  dir = start
  while dir not in checked:
    checked.add(dir)
    for mr_yaml_name in MR_YAML_NAMES:
      yaml_path = os.path.join(dir, mr_yaml_name)
      if os.path.exists(yaml_path):
        return yaml_path
    dir = os.path.dirname(dir)
  return None

Response:
Traverse the directory tree identified by start until a directory already
 in checked is encountered.


Instruction:
Can you tell what is the following Python 3 function doing
def parse_mapreduce_yaml(contents):
  try:
    builder = yaml_object.ObjectBuilder(MapReduceYaml)
    handler = yaml_builder.BuilderHandler(builder)
    listener = yaml_listener.EventListener(handler)
    listener.Parse(contents)

    mr_info = handler.GetResults()
  except (ValueError, yaml_errors.EventError), e:
    raise errors.BadYamlError(e)

  if len(mr_info) < 1:
    raise errors.BadYamlError("No configs found in mapreduce.yaml")
  if len(mr_info) > 1:
    raise errors.MultipleDocumentsInMrYaml("Found %d YAML documents" %
                                           len(mr_info))

  jobs = mr_info[0]
  job_names = set(j.name for j in jobs.mapreduce)
  if len(jobs.mapreduce) != len(job_names):
    raise errors.BadYamlError(
        "Overlapping mapreduce names; names must be unique")

  return jobs

Response:
Parses mapreduce. yaml file contents.
    returns a new object with all the data from original file.


Instruction:
Can you implement a function in Python 3 that
locates mapreduce. yaml loads and parses its info.

Response:
def get_mapreduce_yaml(parse=parse_mapreduce_yaml):
  """Locates mapreduce.yaml, loads and parses its info.

  Args:
    parse: Used for testing.

  Returns:
    MapReduceYaml object.

  Raises:
    errors.BadYamlError: when contents is not a valid mapreduce.yaml file or the
    file is missing.
  """
  mr_yaml_path = find_mapreduce_yaml()
  if not mr_yaml_path:
    raise errors.MissingYamlError()
  mr_yaml_file = open(mr_yaml_path)
  try:
    return parse(mr_yaml_file.read())
  finally:
    mr_yaml_file.close()


Instruction:
Can you tell what is the following Python 3 function doing
def to_dict(mapreduce_yaml):
    all_configs = []
    for config in mapreduce_yaml.mapreduce:
      out = {
          "name": config.name,
          "mapper_input_reader": config.mapper.input_reader,
          "mapper_handler": config.mapper.handler,
      }
      if config.mapper.params_validator:
        out["mapper_params_validator"] = config.mapper.params_validator
      if config.mapper.params:
        param_defaults = {}
        for param in config.mapper.params:
          param_defaults[param.name] = param.default or param.value
        out["mapper_params"] = param_defaults
      if config.params:
        param_defaults = {}
        for param in config.params:
          param_defaults[param.name] = param.default or param.value
        out["params"] = param_defaults
      if config.mapper.output_writer:
        out["mapper_output_writer"] = config.mapper.output_writer
      all_configs.append(out)

    return all_configs

Response: Converts a MapReduceYaml file into a JSON - encodable dictionary.


Instruction:
Make a summary of the following Python 3 code
def validate(cls, job_config):
    if job_config.input_reader_cls != cls:
      raise errors.BadReaderParamsError(
          "Expect input reader class %r, got %r." %
          (cls, job_config.input_reader_cls))

Response:
Validates relevant parameters.

    This method can validate fields which it deems relevant.

    Args:
      job_config: an instance of map_job.JobConfig.

    Raises:
      errors.BadReaderParamsError: required parameters are missing or invalid.


Instruction:
Here you have a function in Python 3, explain what it does
def _sort_records_map(records):
  ctx = context.get()
  l = len(records)
  key_records = [None] * l

  logging.debug("Parsing")
  for i in range(l):
    proto = kv_pb.KeyValue()
    proto.ParseFromString(records[i])
    key_records[i] = (proto.key(), records[i])

  logging.debug("Sorting")
  key_records.sort(cmp=_compare_keys)

  logging.debug("Writing")
  mapper_spec = ctx.mapreduce_spec.mapper
  params = input_readers._get_params(mapper_spec)
  bucket_name = params.get("bucket_name")
  filename = (ctx.mapreduce_spec.name + "/" + ctx.mapreduce_id + "/output-" +
              ctx.shard_id + "-" + str(int(time.time())))
  full_filename = "/%s/%s" % (bucket_name, filename)
  filehandle = cloudstorage.open(full_filename, mode="w")
  with output_writers.GCSRecordsPool(filehandle, ctx=ctx) as pool:
    for key_record in key_records:
      pool.append(key_record[1])

  logging.debug("Finalizing")
  filehandle.close()

  entity = _OutputFile(key_name=full_filename,
                       parent=_OutputFile.get_root_key(ctx.mapreduce_id))
  entity.put()

Response: Map function sorting records.


Instruction:
Write a Python 3 function for
splitting input into multiple shards.

Response:
def split_input(cls, mapper_spec):
    """Split input into multiple shards."""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]


Instruction:
Here you have a function in Python 3, explain what it does
def validate(cls, mapper_spec):
    if mapper_spec.input_reader_class() != cls:
      raise errors.BadReaderParamsError("Input reader class mismatch")
    params = mapper_spec.params
    if cls.FILES_PARAM not in params:
      raise errors.BadReaderParamsError("Missing files parameter.")

Response: Validate reader parameters in mapper_spec.


Instruction:
Can you tell what is the following Python 3 function doing
def validate(cls, mapper_spec):
    if mapper_spec.output_writer_class() != cls:
      raise errors.BadWriterParamsError("Output writer class mismatch")
    params = output_writers._get_params(mapper_spec)
    # Bucket Name is required
    if cls.BUCKET_NAME_PARAM not in params:
      raise errors.BadWriterParamsError(
          "%s is required for the _HashingGCSOutputWriter" %
          cls.BUCKET_NAME_PARAM)

Response: Validates the mapper specification.


Instruction:
Given the following Python 3 function, write the documentation
def to_json(self):
    # Use the member variable (since we don't have access to the context) to
    # flush each pool to minimize the size of each filehandle before we
    # serialize it.
    for pool in self._pools:
      if pool is not None:
        pool.flush(True)
    return {"filehandles": pickle.dumps(self._filehandles)}

Response: Returns the json -izable version of the OutputWriter state to serialize in json.


Instruction:
Implement a function in Python 3 to
get the list of filenames for the shard.

Response:
def get_filenames(cls, mapreduce_state):
    """See parent class."""
    shards = mapreduce_state.mapreduce_spec.mapper.shard_count
    filenames = []
    for _ in range(shards):
      filenames.append([None] * shards)
    shard_states = model.ShardState.find_all_by_mapreduce_state(mapreduce_state)
    for x, shard_state in enumerate(shard_states):
      shard_filenames = shard_state.writer_state["shard_filenames"]
      for y in range(shards):
        filenames[y][x] = shard_filenames[y]
    return filenames


Instruction:
Can you create a Python 3 function that
writes data to a single file.

Response:
def write(self, data):
    """Write data.

    Args:
      data: actual data yielded from handler. Type is writer-specific.
    """
    ctx = context.get()
    if len(data) != 2:
      logging.error("Got bad tuple of length %d (2-tuple expected): %s",
                    len(data), data)

    try:
      key = str(data[0])
      value = str(data[1])
    except TypeError:
      logging.error("Expecting a tuple, but got %s: %s",
                    data.__class__.__name__, data)

    file_index = key.__hash__() % len(self._filehandles)

    # Work-around: Since we don't have access to the context in the to_json()
    # function, but we need to flush each pool before we serialize the
    # filehandle, we rely on a member variable instead of using context for
    # pool management.
    pool = self._pools[file_index]
    if pool is None:
      filehandle = self._filehandles[file_index]
      pool = output_writers.GCSRecordsPool(filehandle=filehandle, ctx=ctx)
      self._pools[file_index] = pool

    proto = kv_pb.KeyValue()
    proto.set_key(key)
    proto.set_value(value)
    pool.append(proto.Encode())


Instruction:
Can you tell what is the following Python 3 function doing
def _get_mapper_params(self):
    reader_params = self.input_reader_cls.params_to_json(
        self.input_reader_params)
    # TODO(user): Do the same for writer params.
    return {"input_reader": reader_params,
            "output_writer": self.output_writer_params}

Response: Converts self to model. MapperSpec. params.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _get_mapper_spec(self):
    # pylint: disable=g-import-not-at-top
    from mapreduce import model

    return model.MapperSpec(
        handler_spec=util._obj_to_path(self.mapper),
        input_reader_spec=util._obj_to_path(self.input_reader_cls),
        params=self._get_mapper_params(),
        shard_count=self.shard_count,
        output_writer_spec=util._obj_to_path(self.output_writer_cls))

Response: Converts self to model. MapperSpec.


Instruction:
Write a Python 3 script to
convert self to model. MapreduceSpec. params.

Response:
def _get_mr_params(self):
    """Converts self to model.MapreduceSpec.params."""
    return {"force_writes": self._force_writes,
            "done_callback": self.done_callback_url,
            "user_params": self.user_params,
            "shard_max_attempts": self.shard_max_attempts,
            "task_max_attempts": self._task_max_attempts,
            "task_max_data_processing_attempts":
                self._task_max_data_processing_attempts,
            "queue_name": self.queue_name,
            "base_path": self._base_path,
            "app_id": self._app,
            "api_version": self._api_version}


Instruction:
Here you have a function in Python 3, explain what it does
def _get_default_mr_params(cls):
    cfg = cls(_lenient=True)
    mr_params = cfg._get_mr_params()
    mr_params["api_version"] = 0
    return mr_params

Response: Gets default values for old API.


Instruction:
How would you code a function in Python 3 to
convert MapreduceSpec to JobConfig.

Response:
def _to_map_job_config(cls,
                         mr_spec,
                         # TODO(user): Remove this parameter after it can be
                         # read from mr_spec.
                         queue_name):
    """Converts model.MapreduceSpec back to JobConfig.

    This method allows our internal methods to use JobConfig directly.
    This method also allows us to expose JobConfig as an API during execution,
    despite that it is not saved into datastore.

    Args:
      mr_spec: model.MapreduceSpec.
      queue_name: queue name.

    Returns:
      The JobConfig object for this job.
    """
    mapper_spec = mr_spec.mapper
    # 0 means all the old APIs before api_version is introduced.
    api_version = mr_spec.params.get("api_version", 0)
    old_api = api_version == 0

    # Deserialize params from json if input_reader/output_writer are new API.
    input_reader_cls = mapper_spec.input_reader_class()
    input_reader_params = input_readers._get_params(mapper_spec)
    if issubclass(input_reader_cls, input_reader.InputReader):
      input_reader_params = input_reader_cls.params_from_json(
          input_reader_params)

    output_writer_cls = mapper_spec.output_writer_class()
    output_writer_params = output_writers._get_params(mapper_spec)
    # TODO(user): Call json (de)serialization for writer.
    # if (output_writer_cls and
    #     issubclass(output_writer_cls, output_writer.OutputWriter)):
    #   output_writer_params = output_writer_cls.params_from_json(
    #       output_writer_params)

    # We can not always convert MapreduceSpec generated by older API
    # to JobConfig. Thus, mr framework should use/expose the returned JobConfig
    # object with caution when a job is started with an old API.
    # In this case, this method only tries not to blow up and assemble a
    # JobConfig object as accurate as possible.
    return cls(_lenient=old_api,
               job_name=mr_spec.name,
               job_id=mr_spec.mapreduce_id,
               # handler_spec from older API may not have map_job.Mapper type.
               mapper=util.for_name(mapper_spec.handler_spec),
               input_reader_cls=input_reader_cls,
               input_reader_params=input_reader_params,
               output_writer_cls=output_writer_cls,
               output_writer_params=output_writer_params,
               shard_count=mapper_spec.shard_count,
               queue_name=queue_name,
               user_params=mr_spec.params.get("user_params"),
               shard_max_attempts=mr_spec.params.get("shard_max_attempts"),
               done_callback_url=mr_spec.params.get("done_callback"),
               _force_writes=mr_spec.params.get("force_writes"),
               _base_path=mr_spec.params["base_path"],
               _task_max_attempts=mr_spec.params.get("task_max_attempts"),
               _task_max_data_processing_attempts=(
                   mr_spec.params.get("task_max_data_processing_attempts")),
               _hooks_cls=util.for_name(mr_spec.hooks_class_name),
               _app=mr_spec.params.get("app_id"),
               _api_version=api_version)


Instruction:
Here you have a function in Python 3, explain what it does
def __write_record(self, record_type, data):
    length = len(data)

    crc = crc32c.crc_update(crc32c.CRC_INIT, [record_type])
    crc = crc32c.crc_update(crc, data)
    crc = crc32c.crc_finalize(crc)

    self.__writer.write(
        struct.pack(_HEADER_FORMAT, _mask_crc(crc), length, record_type))
    self.__writer.write(data)
    self.__position += _HEADER_LENGTH + length

Response: Write a single physical record.


Instruction:
Here you have a function in Python 3, explain what it does
def write(self, data):
    block_remaining = _BLOCK_SIZE - self.__position % _BLOCK_SIZE

    if block_remaining < _HEADER_LENGTH:
      # Header won't fit into remainder
      self.__writer.write('\x00' * block_remaining)
      self.__position += block_remaining
      block_remaining = _BLOCK_SIZE

    if block_remaining < len(data) + _HEADER_LENGTH:
      first_chunk = data[:block_remaining - _HEADER_LENGTH]
      self.__write_record(_RECORD_TYPE_FIRST, first_chunk)
      data = data[len(first_chunk):]

      while True:
        block_remaining = _BLOCK_SIZE - self.__position % _BLOCK_SIZE
        if block_remaining >= len(data) + _HEADER_LENGTH:
          self.__write_record(_RECORD_TYPE_LAST, data)
          break
        else:
          chunk = data[:block_remaining - _HEADER_LENGTH]
          self.__write_record(_RECORD_TYPE_MIDDLE, chunk)
          data = data[len(chunk):]
    else:
      self.__write_record(_RECORD_TYPE_FULL, data)

Response: Writes a single record.


Instruction:
Write a Python 3 function for
padding current block with 0.

Response:
def _pad_block(self):
    """Pad block with 0.

    Pad current block with 0. Reader will simply treat these as corrupted
    record and skip the block.

    This method is idempotent.
    """
    pad_length = _BLOCK_SIZE - self.__position % _BLOCK_SIZE
    if pad_length and pad_length != _BLOCK_SIZE:
      self.__writer.write('\x00' * pad_length)
      self.__position += pad_length


Instruction:
Make a summary of the following Python 3 code
def __try_read_record(self):
    block_remaining = _BLOCK_SIZE - self.__reader.tell() % _BLOCK_SIZE
    if block_remaining < _HEADER_LENGTH:
      return ('', _RECORD_TYPE_NONE)

    header = self.__reader.read(_HEADER_LENGTH)
    if len(header) != _HEADER_LENGTH:
      raise EOFError('Read %s bytes instead of %s' %
                     (len(header), _HEADER_LENGTH))

    (masked_crc, length, record_type) = struct.unpack(_HEADER_FORMAT, header)
    crc = _unmask_crc(masked_crc)

    if length + _HEADER_LENGTH > block_remaining:
      # A record can't be bigger than one block.
      raise errors.InvalidRecordError('Length is too big')

    data = self.__reader.read(length)
    if len(data) != length:
      raise EOFError('Not enough data read. Expected: %s but got %s' %
                     (length, len(data)))

    if record_type == _RECORD_TYPE_NONE:
      return ('', record_type)

    actual_crc = crc32c.crc_update(crc32c.CRC_INIT, [record_type])
    actual_crc = crc32c.crc_update(actual_crc, data)
    actual_crc = crc32c.crc_finalize(actual_crc)

    if actual_crc != crc:
      raise errors.InvalidRecordError('Data crc does not match')
    return (data, record_type)

Response: Try reading a record.


Instruction:
Can you generate the documentation for the following Python 3 function
def __sync(self):
    pad_length = _BLOCK_SIZE - self.__reader.tell() % _BLOCK_SIZE
    if pad_length and pad_length != _BLOCK_SIZE:
      data = self.__reader.read(pad_length)
      if len(data) != pad_length:
        raise EOFError('Read %d bytes instead of %d' %
                       (len(data), pad_length))

Response: Skip reader to the block boundary.


Instruction:
How would you implement a function in Python 3 that
reads a single record from the current position in reader.

Response:
def read(self):
    """Reads record from current position in reader.

    Returns:
      original bytes stored in a single record.
    """
    data = None
    while True:
      last_offset = self.tell()
      try:
        (chunk, record_type) = self.__try_read_record()
        if record_type == _RECORD_TYPE_NONE:
          self.__sync()
        elif record_type == _RECORD_TYPE_FULL:
          if data is not None:
            logging.warning(
                "Ordering corruption: Got FULL record while already "
                "in a chunk at offset %d", last_offset)
          return chunk
        elif record_type == _RECORD_TYPE_FIRST:
          if data is not None:
            logging.warning(
                "Ordering corruption: Got FIRST record while already "
                "in a chunk at offset %d", last_offset)
          data = chunk
        elif record_type == _RECORD_TYPE_MIDDLE:
          if data is None:
            logging.warning(
                "Ordering corruption: Got MIDDLE record before FIRST "
                "record at offset %d", last_offset)
          else:
            data += chunk
        elif record_type == _RECORD_TYPE_LAST:
          if data is None:
            logging.warning(
                "Ordering corruption: Got LAST record but no chunk is in "
                "progress at offset %d", last_offset)
          else:
            result = data + chunk
            data = None
            return result
        else:
          raise errors.InvalidRecordError(
              "Unsupported record type: %s" % record_type)

      except errors.InvalidRecordError, e:
        logging.warning("Invalid record encountered at %s (%s). Syncing to "
                        "the next block", last_offset, e)
        data = None
        self.__sync()


Instruction:
Write a Python 3 function for
starting a mapreduce job. Args: job_name: mapreduce name. Only for display purpose. handler_spec: fully qualified name to your map function/class. input_reader_spec: fully qualified name to input reader class. output_writer_spec: fully qualified name to output writer class. params: a dictionary of parameters for input reader and output writer initialization. shards: number of shards. This provides a guide to mapreduce. The real number of shards is determined by how input are splited.

Response:
def run(self,
          job_name,
          handler_spec,
          input_reader_spec,
          output_writer_spec=None,
          params=None,
          shards=None,
          base_path=None):
    """Start a mapreduce job.

    Args:
      job_name: mapreduce name. Only for display purpose.
      handler_spec: fully qualified name to your map function/class.
      input_reader_spec: fully qualified name to input reader class.
      output_writer_spec: fully qualified name to output writer class.
      params: a dictionary of parameters for input reader and output writer
        initialization.
      shards: number of shards. This provides a guide to mapreduce. The real
        number of shards is determined by how input are splited.
    """
    if shards is None:
      shards = parameters.config.SHARD_COUNT
    if base_path is None:
      base_path = parameters.config.BASE_PATH
    mapreduce_id = control.start_map(
        job_name,
        handler_spec,
        input_reader_spec,
        params or {},
        mapreduce_parameters={
            "done_callback": self.get_callback_url(),
            "done_callback_method": "GET",
            "pipeline_id": self.pipeline_id,
            "base_path": base_path,
        },
        shard_count=shards,
        output_writer_spec=output_writer_spec,
        queue_name=self.queue_name,
        )
    self.fill(self.outputs.job_id, mapreduce_id)
    self.set_status(console_url="%s/detail?mapreduce_id=%s" % (
        (base_path, mapreduce_id)))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def start_map(name,
              handler_spec,
              reader_spec,
              mapper_parameters,
              shard_count=None,
              output_writer_spec=None,
              mapreduce_parameters=None,
              base_path=None,
              queue_name=None,
              eta=None,
              countdown=None,
              hooks_class_name=None,
              _app=None,
              in_xg_transaction=False):
  """Start a new, mapper-only mapreduce.

  Deprecated! Use map_job.start instead.

  If a value can be specified both from an explicit argument and from
  a dictionary, the value from the explicit argument wins.

  Args:
    name: mapreduce name. Used only for display purposes.
    handler_spec: fully qualified name of mapper handler function/class to call.
    reader_spec: fully qualified name of mapper reader to use
    mapper_parameters: dictionary of parameters to pass to mapper. These are
      mapper-specific and also used for reader/writer initialization.
      Should have format {"input_reader": {}, "output_writer":{}}. Old
      deprecated style does not have sub dictionaries.
    shard_count: number of shards to create.
    mapreduce_parameters: dictionary of mapreduce parameters relevant to the
      whole job.
    base_path: base path of mapreduce library handler specified in app.yaml.
      "/mapreduce" by default.
    queue_name: taskqueue queue name to be used for mapreduce tasks.
      see util.get_queue_name.
    eta: absolute time when the MR should execute. May not be specified
      if 'countdown' is also supplied. This may be timezone-aware or
      timezone-naive.
    countdown: time in seconds into the future that this MR should execute.
      Defaults to zero.
    hooks_class_name: fully qualified name of a hooks.Hooks subclass.
    in_xg_transaction: controls what transaction scope to use to start this MR
      job. If True, there has to be an already opened cross-group transaction
      scope. MR will use one entity group from it.
      If False, MR will create an independent transaction to start the job
      regardless of any existing transaction scopes.

  Returns:
    mapreduce id as string.
  """
  if shard_count is None:
    shard_count = parameters.config.SHARD_COUNT

  if mapper_parameters:
    mapper_parameters = dict(mapper_parameters)

  # Make sure this old API fill all parameters with default values.
  mr_params = map_job.JobConfig._get_default_mr_params()
  if mapreduce_parameters:
    mr_params.update(mapreduce_parameters)

  # Override default values if user specified them as arguments.
  if base_path:
    mr_params["base_path"] = base_path
  mr_params["queue_name"] = util.get_queue_name(queue_name)

  mapper_spec = model.MapperSpec(handler_spec,
                                 reader_spec,
                                 mapper_parameters,
                                 shard_count,
                                 output_writer_spec=output_writer_spec)

  if in_xg_transaction and not db.is_in_transaction():
    logging.warning("Expects an opened xg transaction to start mapreduce "
                    "when transactional is True.")

  return handlers.StartJobHandler._start_map(
      name,
      mapper_spec,
      mr_params,
      # TODO(user): Now that "queue_name" is part of mr_params.
      # Remove all the other ways to get queue_name after one release.
      queue_name=mr_params["queue_name"],
      eta=eta,
      countdown=countdown,
      hooks_class_name=hooks_class_name,
      _app=_app,
      in_xg_transaction=in_xg_transaction)

Response: Start a new mapreduce job.


Instruction:
Can you implement a function in Python 3 that
gets status enum. Returns: One of the status enum.

Response:
def get_status(self):
    """Get status enum.

    Returns:
      One of the status enum.
    """
    self.__update_state()
    if self._state.active:
      return self.RUNNING
    else:
      return self._state.result_status


Instruction:
Write a Python 3 script to
get the value of the named counter from this job.

Response:
def get_counter(self, counter_name, default=0):
    """Get the value of the named counter from this job.

    When a job is running, counter values won't be very accurate.

    Args:
      counter_name: name of the counter in string.
      default: default value if the counter doesn't exist.

    Returns:
      Value in int of the named counter.
    """
    self.__update_state()
    return self._state.counters_map.get(counter_name, default)


Instruction:
Explain what the following Python 3 code does
def get_outputs(self):
    assert self.SUCCESS == self.get_status()
    ss = model.ShardState.find_all_by_mapreduce_state(self._state)
    for s in ss:
      yield iter(s.writer_state.get("outs", []))

Response: Get outputs of this job.


Instruction:
Can you generate the documentation for the following Python 3 function
def submit(cls, job_config, in_xg_transaction=False):
    cls.__validate_job_config(job_config)
    mapper_spec = job_config._get_mapper_spec()

    # Create mr spec.
    mapreduce_params = job_config._get_mr_params()
    mapreduce_spec = model.MapreduceSpec(
        job_config.job_name,
        job_config.job_id,
        mapper_spec.to_json(),
        mapreduce_params,
        util._obj_to_path(job_config._hooks_cls))

    # Save states and enqueue task.
    if in_xg_transaction:
      propagation = db.MANDATORY
    else:
      propagation = db.INDEPENDENT

    state = None
    @db.transactional(propagation=propagation)
    def _txn():
      state = cls.__create_and_save_state(job_config, mapreduce_spec)
      cls.__add_kickoff_task(job_config, mapreduce_spec)
      return state

    state = _txn()
    return cls(state)

Response: Submit the job to run.


Instruction:
Can you generate the documentation for the following Python 3 function
def __update_state(self):
    # Only if the job was not in a terminal state.
    if self._state.active:
      self._state = self.__get_state_by_id(self.job_config.job_id)

Response: Updates the state of the job.


Instruction:
Can you generate the documentation for the following Python 3 function
def __get_state_by_id(cls, job_id):
    state = model.MapreduceState.get_by_job_id(job_id)
    if state is None:
      raise ValueError("Job state for job %s is missing." % job_id)
    return state

Response: Get the job state by id.


Instruction:
Can you tell what is the following Python 3 function doing
def __create_and_save_state(cls, job_config, mapreduce_spec):
    state = model.MapreduceState.create_new(job_config.job_id)
    state.mapreduce_spec = mapreduce_spec
    state.active = True
    state.active_shards = 0
    state.app_id = job_config._app
    config = datastore_rpc.Configuration(force_writes=job_config._force_writes)
    state.put(config=config)
    return state

Response: Create and save the map job state to datastore.


Instruction:
Write a Python 3 function that can
add kickoff task to taskqueue.

Response:
def __add_kickoff_task(cls, job_config, mapreduce_spec):
    """Add kickoff task to taskqueue.

    Args:
      job_config: map_job.JobConfig.
      mapreduce_spec: model.MapreduceSpec,
    """
    params = {"mapreduce_id": job_config.job_id}
    # Task is not named so that it can be added within a transaction.
    kickoff_task = taskqueue.Task(
        # TODO(user): Perhaps make this url a computed field of job_config.
        url=job_config._base_path + "/kickoffjob_callback/" + job_config.job_id,
        headers=util._get_task_headers(job_config.job_id),
        params=params)
    if job_config._hooks_cls:
      hooks = job_config._hooks_cls(mapreduce_spec)
      try:
        hooks.enqueue_kickoff_task(kickoff_task, job_config.queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(job_config.queue_name, transactional=True)


Instruction:
Explain what the following Python 3 code does
def to_json_str(self):
    _json = self.to_json()
    try:
      return json.dumps(_json, sort_keys=True, cls=JsonEncoder)
    except:
      logging.exception("Could not serialize JSON: %r", _json)
      raise

Response: Convert data to json string representation.


Instruction:
Can you implement a function in Python 3 that
converts json string representation into object.

Response:
def from_json_str(cls, json_str):
    """Convert json string representation into class instance.

    Args:
      json_str: json representation as string.

    Returns:
      New instance of the class with data loaded from json string.
    """
    return cls.from_json(json.loads(json_str, cls=JsonDecoder))


Instruction:
Can you write a function in Python 3 where it
gets value for the datastore.

Response:
def get_value_for_datastore(self, model_instance):
    """Gets value for datastore.

    Args:
      model_instance: instance of the model class.

    Returns:
      datastore-compatible value.
    """
    value = super(JsonProperty, self).get_value_for_datastore(model_instance)
    if not value:
      return None
    json_value = value
    if not isinstance(value, dict):
      json_value = value.to_json()
    if not json_value:
      return None
    return datastore_types.Text(json.dumps(
        json_value, sort_keys=True, cls=JsonEncoder))


Instruction:
Explain what the following Python 3 code does
def make_value_from_datastore(self, value):

    if value is None:
      return None
    _json = json.loads(value, cls=JsonDecoder)
    if self.data_type == dict:
      return _json
    return self.data_type.from_json(_json)

Response: Convert value from datastore representation.


Instruction:
Write a Python 3 script to
validate the specified value.

Response:
def validate(self, value):
    """Validate value.

    Args:
      value: model value.

    Returns:
      Whether the specified value is valid data type value.

    Raises:
      BadValueError: when value is not of self.data_type type.
    """
    if value is not None and not isinstance(value, self.data_type):
      raise datastore_errors.BadValueError(
          "Property %s must be convertible to a %s instance (%s)" %
          (self.name, self.data_type, value))
    return super(JsonProperty, self).validate(value)


Instruction:
Here you have a function in Python 3, explain what it does
def add(self, queue_name, transactional=False):
    task = self.to_task()
    task.add(queue_name, transactional)

Response: Add a task to the queue.


Instruction:
Write a Python 3 function that can
convert to a taskqueue task.

Response:
def to_task(self):
    """Convert to a taskqueue task."""
    # Never pass params to taskqueue.Task. Use payload instead. Otherwise,
    # it's up to a particular taskqueue implementation to generate
    # payload from params. It could blow up payload size over limit.
    return taskqueue.Task(
        url=self.url,
        payload=self._payload,
        name=self.name,
        eta=self.eta,
        countdown=self.countdown,
        headers=self._headers)


Instruction:
Here you have a function in Python 3, explain what it does
def decode_payload(cls, request):
    # TODO(user): Pass mr_id into headers. Otherwise when payload decoding
    # failed, we can't abort a mr.
    if request.headers.get(cls.PAYLOAD_VERSION_HEADER) != cls.PAYLOAD_VERSION:
      raise DeprecationWarning(
          "Task is generated by an older incompatible version of mapreduce. "
          "Please kill this job manually")
    return cls._decode_payload(request.body)

Response: Decode the payload of a Huge task.


Instruction:
Can you implement a function in Python 3 that
increments the value of a counter.

Response:
def increment(self, counter_name, delta):
    """Increment counter value.

    Args:
      counter_name: counter name as String.
      delta: increment delta as Integer.

    Returns:
      new counter value.
    """
    current_value = self.counters.get(counter_name, 0)
    new_value = current_value + delta
    self.counters[counter_name] = new_value
    return new_value


Instruction:
Given the following Python 3 function, write the documentation
def add_map(self, counters_map):
    for counter_name in counters_map.counters:
      self.increment(counter_name, counters_map.counters[counter_name])

Response: Adds all counters from the passed map to the current object.


Instruction:
Can you generate the documentation for the following Python 3 function
def sub_map(self, counters_map):
    for counter_name in counters_map.counters:
      self.increment(counter_name, -counters_map.counters[counter_name])

Response: Subtracts all counters from the passed map.


Instruction:
Write a Python 3 function for
serializing this MapperSpec into a json -izable object.

Response:
def to_json(self):
    """Serializes this MapperSpec into a json-izable object."""
    result = {
        "mapper_handler_spec": self.handler_spec,
        "mapper_input_reader": self.input_reader_spec,
        "mapper_params": self.params,
        "mapper_shard_count": self.shard_count
    }
    if self.output_writer_spec:
      result["mapper_output_writer"] = self.output_writer_spec
    return result


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_hooks(self):
    if self.__hooks is None and self.hooks_class_name is not None:
      hooks_class = util.for_name(self.hooks_class_name)
      if not isinstance(hooks_class, type):
        raise ValueError("hooks_class_name must refer to a class, got %s" %
                         type(hooks_class).__name__)
      if not issubclass(hooks_class, hooks.Hooks):
        raise ValueError(
            "hooks_class_name must refer to a hooks.Hooks subclass")
      self.__hooks = hooks_class(self)

    return self.__hooks

Response: Returns a hooks. Hooks class or None if no hooks class has been set.


Instruction:
Can you write a function in Python 3 where it
serializes all data in this mapreduce spec into json form.

Response:
def to_json(self):
    """Serializes all data in this mapreduce spec into json form.

    Returns:
      data in json format.
    """
    mapper_spec = self.mapper.to_json()
    return {
        "name": self.name,
        "mapreduce_id": self.mapreduce_id,
        "mapper_spec": mapper_spec,
        "params": self.params,
        "hooks_class_name": self.hooks_class_name,
    }


Instruction:
Write a Python 3 function for
creating a MapreduceSpec from the json representation of a MapreduceSpec.

Response:
def from_json(cls, json):
    """Create new MapreduceSpec from the json, encoded by to_json.

    Args:
      json: json representation of MapreduceSpec.

    Returns:
      an instance of MapreduceSpec with all data deserialized from json.
    """
    mapreduce_spec = cls(json["name"],
                         json["mapreduce_id"],
                         json["mapper_spec"],
                         json.get("params"),
                         json.get("hooks_class_name"))
    return mapreduce_spec


Instruction:
Here you have a function in Python 3, explain what it does
def _get_mapreduce_spec(cls, mr_id):
    key = 'GAE-MR-spec: %s' % mr_id
    spec_json = memcache.get(key)
    if spec_json:
      return cls.from_json(spec_json)
    state = MapreduceState.get_by_job_id(mr_id)
    spec = state.mapreduce_spec
    spec_json = spec.to_json()
    memcache.set(key, spec_json)
    return spec

Response: Get Mapreduce spec from mr id.


Instruction:
Write a Python 3 function for
retrieving the Key for a Job.

Response:
def get_key_by_job_id(cls, mapreduce_id):
    """Retrieves the Key for a Job.

    Args:
      mapreduce_id: The job to retrieve.

    Returns:
      Datastore Key that can be used to fetch the MapreduceState.
    """
    return db.Key.from_path(cls.kind(), str(mapreduce_id))


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_processed_counts(self, shards_processed, shards_status):
    chart = google_chart_api.BarChart()

    def filter_status(status_to_filter):
      return [count if status == status_to_filter else 0
              for count, status in zip(shards_processed, shards_status)]

    if shards_status:
      # Each index will have only one non-zero count, so stack them to color-
      # code the bars by status
      # These status values are computed in _update_state_from_shard_states,
      # in mapreduce/handlers.py.
      chart.stacked = True
      chart.AddBars(filter_status("unknown"), color="404040")
      chart.AddBars(filter_status("success"), color="00ac42")
      chart.AddBars(filter_status("running"), color="3636a9")
      chart.AddBars(filter_status("aborted"), color="e29e24")
      chart.AddBars(filter_status("failed"), color="f6350f")
    else:
      chart.AddBars(shards_processed)

    shard_count = len(shards_processed)

    if shard_count > 95:
      # Auto-spacing does not work for large numbers of shards.
      pixels_per_shard = 700.0 / shard_count
      bar_thickness = int(pixels_per_shard * .9)

      chart.style = bar_chart.BarChartStyle(bar_thickness=bar_thickness,
        bar_gap=0.1, use_fractional_gap_spacing=True)

    if shards_processed and shard_count <= 95:
      # Adding labels puts us in danger of exceeding the URL length, only
      # do it when we have a small amount of data to plot.
      # Only 16 labels on the whole chart.
      stride_length = max(1, shard_count / 16)
      chart.bottom.labels = []
      for x in xrange(shard_count):
        if (x % stride_length == 0 or
            x == shard_count - 1):
          chart.bottom.labels.append(x)
        else:
          chart.bottom.labels.append("")
      chart.left.labels = ["0", str(max(shards_processed))]
      chart.left.min = 0

    self.chart_width = min(700, max(300, shard_count * 20))
    self.chart_url = chart.display.Url(self.chart_width, 200)

Response: Updates a chart url to display processed count for each shard.


Instruction:
Write a Python 3 function for
creating a new MapreduceState object.

Response:
def create_new(mapreduce_id=None,
                 gettime=datetime.datetime.now):
    """Create a new MapreduceState.

    Args:
      mapreduce_id: Mapreduce id as string.
      gettime: Used for testing.
    """
    if not mapreduce_id:
      mapreduce_id = MapreduceState.new_mapreduce_id()
    state = MapreduceState(key_name=mapreduce_id,
                           last_poll_time=gettime())
    state.set_processed_counts([], [])
    return state


Instruction:
Explain what the following Python 3 code does
def reset_for_retry(self, output_writer):
    self.input_reader = self.initial_input_reader
    self.slice_id = 0
    self.retries += 1
    self.output_writer = output_writer
    self.handler = self.mapreduce_spec.mapper.handler

Response: Reset self for shard retry.


Instruction:
Can you write a function in Python 3 where it
advances relavent states for next slice.

Response:
def advance_for_next_slice(self, recovery_slice=False):
    """Advance relavent states for next slice.

    Args:
      recovery_slice: True if this slice is running recovery logic.
        See handlers.MapperWorkerCallbackHandler._attempt_slice_recovery
        for more info.
    """
    if recovery_slice:
      self.slice_id += 2
      # Restore input reader to the beginning of the slice.
      self.input_reader = self.input_reader.from_json(self._input_reader_json)
    else:
      self.slice_id += 1


Instruction:
Can you generate the documentation for the following Python 3 function
def to_dict(self):
    result = {"mapreduce_spec": self.mapreduce_spec.to_json_str(),
              "shard_id": self.shard_id,
              "slice_id": str(self.slice_id),
              "input_reader_state": self.input_reader.to_json_str(),
              "initial_input_reader_state":
              self.initial_input_reader.to_json_str(),
              "retries": str(self.retries)}
    if self.output_writer:
      result["output_writer_state"] = self.output_writer.to_json_str()
    serialized_handler = util.try_serialize_handler(self.handler)
    if serialized_handler:
      result["serialized_handler"] = serialized_handler
    return result

Response: Convert state to dictionary to save in task payload.


Instruction:
Explain what the following Python 3 code does
def from_request(cls, request):
    mapreduce_spec = MapreduceSpec.from_json_str(request.get("mapreduce_spec"))
    mapper_spec = mapreduce_spec.mapper
    input_reader_spec_dict = json.loads(request.get("input_reader_state"),
                                        cls=json_util.JsonDecoder)
    input_reader = mapper_spec.input_reader_class().from_json(
        input_reader_spec_dict)
    initial_input_reader_spec_dict = json.loads(
        request.get("initial_input_reader_state"), cls=json_util.JsonDecoder)
    initial_input_reader = mapper_spec.input_reader_class().from_json(
        initial_input_reader_spec_dict)

    output_writer = None
    if mapper_spec.output_writer_class():
      output_writer = mapper_spec.output_writer_class().from_json(
          json.loads(request.get("output_writer_state", "{}"),
                     cls=json_util.JsonDecoder))
      assert isinstance(output_writer, mapper_spec.output_writer_class()), (
          "%s.from_json returned an instance of wrong class: %s" % (
              mapper_spec.output_writer_class(),
              output_writer.__class__))

    handler = util.try_deserialize_handler(request.get("serialized_handler"))
    if not handler:
      handler = mapreduce_spec.mapper.handler

    return cls(mapreduce_spec.params["base_path"],
               mapreduce_spec,
               str(request.get("shard_id")),
               int(request.get("slice_id")),
               input_reader,
               initial_input_reader,
               output_writer=output_writer,
               retries=int(request.get("retries")),
               handler=handler)

Response: Create TransientShardState from webapp request.


Instruction:
Given the following Python 3 function, write the documentation
def reset_for_retry(self):
    self.retries += 1
    self.last_work_item = ""
    self.active = True
    self.result_status = None
    self.input_finished = False
    self.counters_map = CountersMap()
    self.slice_id = 0
    self.slice_start_time = None
    self.slice_request_id = None
    self.slice_retries = 0
    self.acquired_once = False

Response: Reset self for shard retry.


Instruction:
Explain what the following Python 3 code does
def advance_for_next_slice(self, recovery_slice=False):
    self.slice_start_time = None
    self.slice_request_id = None
    self.slice_retries = 0
    self.acquired_once = False
    if recovery_slice:
      self.slice_id += 2
    else:
      self.slice_id += 1

Response: Advance self for next slice.


Instruction:
Can you tell what is the following Python 3 function doing
def copy_from(self, other_state):
    for prop in self.properties().values():
      setattr(self, prop.name, getattr(other_state, prop.name))

Response: Copy data from another shard state entity to self.


Instruction:
Can you implement a function in Python 3 that
finds all shard states for given mapreduce state.

Response:
def find_all_by_mapreduce_state(cls, mapreduce_state):
    """Find all shard states for given mapreduce.

    Args:
      mapreduce_state: MapreduceState instance

    Yields:
      shard states sorted by shard id.
    """
    keys = cls.calculate_keys_by_mapreduce_state(mapreduce_state)
    i = 0
    while i < len(keys):
      @db.non_transactional
      def no_tx_get(i):
        return db.get(keys[i:i+cls._MAX_STATES_IN_MEMORY])
      # We need a separate function to so that we can mix non-transactional and
      # use be a generator
      states = no_tx_get(i)
      for s in states:
        i += 1
        if s is not None:
          yield s


Instruction:
Given the following Python 3 function, write the documentation
def calculate_keys_by_mapreduce_state(cls, mapreduce_state):
    if mapreduce_state is None:
      return []

    keys = []
    for i in range(mapreduce_state.mapreduce_spec.mapper.shard_count):
      shard_id = cls.shard_id_from_number(mapreduce_state.key().name(), i)
      keys.append(cls.get_key_by_shard_id(shard_id))
    return keys

Response: Calculate all shard states keys for given mapreduce.


Instruction:
Implement a Python 3 function for
creating a new shard state.

Response:
def create_new(cls, mapreduce_id, shard_number):
    """Create new shard state.

    Args:
      mapreduce_id: unique mapreduce id as string.
      shard_number: shard number for which to create shard state.

    Returns:
      new instance of ShardState ready to put into datastore.
    """
    shard_id = cls.shard_id_from_number(mapreduce_id, shard_number)
    state = cls(key_name=shard_id,
                mapreduce_id=mapreduce_id)
    return state


Instruction:
Can you write a function in Python 3 where it
retrieves the Key for a given mapreduce ID.

Response:
def get_key_by_job_id(cls, mapreduce_id):
    """Retrieves the Key for a mapreduce ID.

    Args:
      mapreduce_id: The job to fetch.

    Returns:
      Datastore Key for the command for the given job ID.
    """
    return db.Key.from_path(cls.kind(), "%s:%s" % (mapreduce_id, cls._KEY_NAME))


Instruction:
Write a Python 3 function that can
abort a job in a sequence.

Response:
def abort(cls, mapreduce_id, **kwargs):
    """Causes a job to abort.

    Args:
      mapreduce_id: The job to abort. Not verified as a valid job.
    """
    cls(key_name="%s:%s" % (mapreduce_id, cls._KEY_NAME),
        command=cls.ABORT).put(**kwargs)


Instruction:
How would you explain what the following Python 3 function does
def initialize(self, request, response):
    super(TaskQueueHandler, self).initialize(request, response)

    # Check request is from taskqueue.
    if "X-AppEngine-QueueName" not in self.request.headers:
      logging.error(self.request.headers)
      logging.error("Task queue handler received non-task queue request")
      self.response.set_status(
          403, message="Task queue handler received non-task queue request")
      return

    # Check task has not been retried too many times.
    if self.task_retry_count() + 1 > parameters.config.TASK_MAX_ATTEMPTS:
      logging.error(
          "Task %s has been attempted %s times. Dropping it permanently.",
          self.request.headers["X-AppEngine-TaskName"],
          self.task_retry_count() + 1)
      self._drop_gracefully()
      return

    try:
      self._preprocess()
      self._preprocess_success = True
    # pylint: disable=bare-except
    except:
      self._preprocess_success = False
      logging.error(
          "Preprocess task %s failed. Dropping it permanently.",
          self.request.headers["X-AppEngine-TaskName"])
      self._drop_gracefully()

Response: Initialize the task queue.


Instruction:
How would you explain what the following Python 3 function does
def retry_task(self):
    self.response.set_status(httplib.SERVICE_UNAVAILABLE, "Retry task")
    self.response.clear()

Response: Ask taskqueue to retry this task.


Instruction:
Can you write a function in Python 3 where it
returns the base path for all mapreduce - related urls.

Response:
def base_path(self):
    """Base path for all mapreduce-related urls.

    JSON handlers are mapped to /base_path/command/command_name thus they
    require special treatment.

    Raises:
      BadRequestPathError: if the path does not end with "/command".

    Returns:
      The base path.
    """
    path = self.request.path
    base_path = path[:path.rfind("/")]
    if not base_path.endswith("/command"):
      raise BadRequestPathError(
          "Json handlers should have /command path prefix")
    return base_path[:base_path.rfind("/")]


Instruction:
Given the following Python 3 function, write the documentation
def read(self, n):
    if self._EOF:
      return ""

    while self._seg_index <= self._last_seg_index:
      result = self._read_from_seg(n)
      if result != "":
        return result
      else:
        self._next_seg()

    self._EOF = True
    return ""

Response: Reads n bytes from the segments in the file.


Instruction:
Make a summary of the following Python 3 code
def _read_from_seg(self, n):
    result = self._seg.read(size=n)
    if result == "":
      return result
    offset = self._seg.tell()
    if offset > self._seg_valid_length:
      extra = offset - self._seg_valid_length
      result = result[:-1*extra]
    self._offset += len(result)
    return result

Response: Read n bytes from the current seg.


Instruction:
Write a Python 3 script for
returning a key name lexically ordered by time descending.

Response:
def _get_descending_key(gettime=time.time):
  """Returns a key name lexically ordered by time descending.

  This lets us have a key name for use with Datastore entities which returns
  rows in time descending order when it is scanned in lexically ascending order,
  allowing us to bypass index building for descending indexes.

  Args:
    gettime: Used for testing.

  Returns:
    A string with a time descending key.
  """
  now_descending = int((_FUTURE_TIME - gettime()) * 100)
  request_id_hash = os.environ.get("REQUEST_ID_HASH")
  if not request_id_hash:
    request_id_hash = str(random.getrandbits(32))
  return "%d%s" % (now_descending, request_id_hash)


Instruction:
Create a Python 3 function for
getting the Host header value for all mr tasks.

Response:
def _get_task_host():
  """Get the Host header value for all mr tasks.

  Task Host header determines which instance this task would be routed to.

  Current version id format is: v7.368834058928280579
  Current module id is just the module's name. It could be "default"
  Default version hostname is app_id.appspot.com

  Returns:
    A complete host name is of format version.module.app_id.appspot.com
  If module is the default module, just version.app_id.appspot.com. The reason
  is if an app doesn't have modules enabled and the url is
  "version.default.app_id", "version" is ignored and "default" is used as
  version. If "default" version doesn't exist, the url is routed to the
  default version.
  """
  version = os.environ["CURRENT_VERSION_ID"].split(".")[0]
  default_host = os.environ["DEFAULT_VERSION_HOSTNAME"]
  module = os.environ["CURRENT_MODULE_ID"]
  if os.environ["CURRENT_MODULE_ID"] == "default":
    return "%s.%s" % (version, default_host)
  return "%s.%s.%s" % (version, module, default_host)


Instruction:
Given the following Python 3 function, write the documentation
def get_queue_name(queue_name):
  if queue_name:
    return queue_name
  queue_name = os.environ.get("HTTP_X_APPENGINE_QUEUENAME",
                              parameters.config.QUEUE_NAME)
  if len(queue_name) > 1 and queue_name[0:2] == "__":
    # We are currently in some special queue. E.g. __cron.
    return parameters.config.QUEUE_NAME
  else:
    return queue_name

Response: Determine which queue MR should run on.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def total_seconds(td):
  secs = td.seconds + td.days * 24 * 3600
  if td.microseconds:
    secs += 1
  return secs

Response: convert a timedelta to seconds.