Instruction:
Can you implement a function in Python 3 that
extends a jsonschema validation class with a default - setting mechanism.

Response:
def extend_with_default(validator_class):
    """
    Add a default-setting mechanism to a ``jsonschema`` validation class.
    """
    validate_properties = validator_class.VALIDATORS["properties"]

    def set_defaults(validator, properties, instance, schema):
        """
        Set defaults in subschemas
        """
        for prop, subschema in properties.items():
            if "default" in subschema:
                instance.setdefault(prop, subschema["default"])

        for error in validate_properties(validator, properties, instance, schema):
            yield error

    return validators.extend(validator_class, {"properties": set_defaults})


Instruction:
How would you implement a function in Python 3 that
validates an object against a schema

Response:
def validate(obj, schema):
        """
        Validate an object against a schema

        Args:
            obj (dict):
            schema (dict):
        """
        if isinstance(obj, str):
            obj = json.loads(obj)
        return JsonValidator(schema)._validate(obj)


Instruction:
Can you tell what is the following Python 3 function doing
def _validate(self, obj):
        report = ValidationReport()
        if not self.validator.is_valid(obj):
            for v in self.validator.iter_errors(obj):
                report.add_error("[%s] %s" % ('.'.join(str(vv) for vv in v.path), v.message))
        return report

Response: Validate the object and return a ValidationReport object


Instruction:
Explain what the following Python 3 code does
def run_processor(
        processorClass,
        ocrd_tool=None,
        mets_url=None,
        resolver=None,
        workspace=None,
        page_id=None,
        log_level=None,
        input_file_grp=None,
        output_file_grp=None,
        parameter=None,
        working_dir=None,
): # pylint: disable=too-many-locals
    """
    Create a workspace for mets_url and run processor through it

    Args:
        parameter (string): URL to the parameter
    """
    workspace = _get_workspace(
        workspace,
        resolver,
        mets_url,
        working_dir
    )
    if parameter is not None:
        if not '://' in parameter:
            fname = os.path.abspath(parameter)
        else:
            fname = workspace.download_url(parameter)
        with open(fname, 'r') as param_json_file:
            parameter = json.load(param_json_file)
    else:
        parameter = {}
    log.debug("Running processor %s", processorClass)
    processor = processorClass(
        workspace,
        ocrd_tool=ocrd_tool,
        page_id=page_id,
        input_file_grp=input_file_grp,
        output_file_grp=output_file_grp,
        parameter=parameter
    )
    ocrd_tool = processor.ocrd_tool
    name = '%s v%s' % (ocrd_tool['executable'], processor.version)
    otherrole = ocrd_tool['steps'][0]
    log.debug("Processor instance %s (%s doing %s)", processor, name, otherrole)
    processor.process()
    workspace.mets.add_agent(
        name=name,
        _type='OTHER',
        othertype='SOFTWARE',
        role='OTHER',
        otherrole=otherrole
    )
    workspace.save_mets()
    return processor

Response: Runs a processor through the mets_url and saves the result in a workspace.


Instruction:
Write a Python 3 script to
run MP CLI through it

Response:
def run_cli(
        executable,
        mets_url=None,
        resolver=None,
        workspace=None,
        page_id=None,
        log_level=None,
        input_file_grp=None,
        output_file_grp=None,
        parameter=None,
        working_dir=None,
):
    """
    Create a workspace for mets_url and run MP CLI through it
    """
    workspace = _get_workspace(workspace, resolver, mets_url, working_dir)
    args = [executable, '--working-dir', workspace.directory]
    args += ['--mets', mets_url]
    if log_level:
        args += ['--log-level', log_level]
    if page_id:
        args += ['--page-id', page_id]
    if input_file_grp:
        args += ['--input-file-grp', input_file_grp]
    if output_file_grp:
        args += ['--output-file-grp', output_file_grp]
    if parameter:
        args += ['--parameter', parameter]
    log.debug("Running subprocess '%s'", ' '.join(args))
    return subprocess.call(args)


Instruction:
Given the following Python 3 function, write the documentation
def input_files(self):
        return self.workspace.mets.find_files(fileGrp=self.input_file_grp, pageId=self.page_id)

Response: List the input files in the workspace


Instruction:
How would you implement a function in Python 3 that
creates a PcGtsType object representing a single image.

Response:
def page_from_image(input_file):
    """
    Create `OcrdPage </../../ocrd_models/ocrd_models.ocrd_page.html>`_
    from an `OcrdFile </../../ocrd_models/ocrd_models.ocrd_file.html>`_
    representing an image (i.e. should have ``mimetype`` starting with ``image/``).

    Arguments:
        * input_file (OcrdFile):
    """
    if input_file.local_filename is None:
        raise Exception("input_file must have 'local_filename' property")
    exif = exif_from_filename(input_file.local_filename)
    now = datetime.now()
    return PcGtsType(
        Metadata=MetadataType(
            Creator="OCR-D/core %s" % VERSION,
            Created=now,
            LastChange=now
        ),
        Page=PageType(
            imageWidth=exif.width,
            imageHeight=exif.height,
            # XXX brittle
            imageFilename=input_file.url if input_file.url is not None else 'file://' + input_file.local_filename
        )
    )


Instruction:
Given the following Python 3 function, write the documentation
def page_from_file(input_file):
    #  print("PARSING PARSING '%s'" % input_file)
    if input_file.mimetype.startswith('image'):
        return page_from_image(input_file)
    if input_file.mimetype == MIMETYPE_PAGE:
        return parse(input_file.local_filename, silence=True)
    raise Exception("Unsupported mimetype '%s'" % input_file.mimetype)

Response: Create a new PAGE - XML from a METS file representing a PAGE - XML or an image.


Instruction:
How would you explain what the following Python 3 function does
def concat_padded(base, *args):
    ret = base
    for n in args:
        if is_string(n):
            ret = "%s_%s" % (ret, n)
        else:
            ret = "%s_%04i"  % (ret, n + 1)
    return ret

Response: Concatenate string and zero - padded 4 digit number of a base element.


Instruction:
Can you write a function in Python 3 where it
constructs a polygon representation from a rectangle described as a dict with keys x y w h.

Response:
def points_from_xywh(box):
    """
    Constructs a polygon representation from a rectangle described as a dict with keys x, y, w, h.
    """
    x, y, w, h = box['x'], box['y'], box['w'], box['h']
    # tesseract uses a different region representation format
    return "%i,%i %i,%i %i,%i %i,%i" % (
        x, y,
        x + w, y,
        x + w, y + h,
        x, y + h
    )


Instruction:
Write a Python 3 script to
construct a polygon representation from a list of y0 x0 y1 x1 coordinates.

Response:
def points_from_y0x0y1x1(yxyx):
    """
    Constructs a polygon representation from a rectangle described as a list [y0, x0, y1, x1]
    """
    y0 = yxyx[0]
    x0 = yxyx[1]
    y1 = yxyx[2]
    x1 = yxyx[3]
    return "%s,%s %s,%s %s,%s %s,%s" % (
        x0, y0,
        x1, y0,
        x1, y1,
        x0, y1
    )


Instruction:
Implement a function in Python 3 to
construct a polygon representation from a list of x0 y0 x1 y1 coordinates.

Response:
def points_from_x0y0x1y1(xyxy):
    """
    Constructs a polygon representation from a rectangle described as a list [x0, y0, x1, y1]
    """
    x0 = xyxy[0]
    y0 = xyxy[1]
    x1 = xyxy[2]
    y1 = xyxy[3]
    return "%s,%s %s,%s %s,%s %s,%s" % (
        x0, y0,
        x1, y0,
        x1, y1,
        x0, y1
    )


Instruction:
Can you generate the documentation for the following Python 3 function
def polygon_from_points(points):
    polygon = []
    for pair in points.split(" "):
        x_y = pair.split(",")
        polygon.append([float(x_y[0]), float(x_y[1])])
    return polygon

Response: Constructs a numpy - compatible polygon from a page representation.


Instruction:
How would you implement a function in Python 3 that
unzips a ZIP file to a directory

Response:
def unzip_file_to_dir(path_to_zip, output_directory):
    """
    Extract a ZIP archive to a directory
    """
    z = ZipFile(path_to_zip, 'r')
    z.extractall(output_directory)
    z.close()


Instruction:
Can you write a function in Python 3 where it
constructs a dictionary representing a rectangle with keys x y w h where the keys x y w h are the number of points in the image.

Response:
def xywh_from_points(points):
    """
    Constructs an dict representing a rectangle with keys x, y, w, h
    """
    xys = [[int(p) for p in pair.split(',')] for pair in points.split(' ')]
    minx = sys.maxsize
    miny = sys.maxsize
    maxx = 0
    maxy = 0
    for xy in xys:
        if xy[0] < minx:
            minx = xy[0]
        if xy[0] > maxx:
            maxx = xy[0]
        if xy[1] < miny:
            miny = xy[1]
        if xy[1] > maxy:
            maxy = xy[1]

    return {
        'x': minx,
        'y': miny,
        'w': maxx - minx,
        'h': maxy - miny,
    }


Instruction:
Given the following Python 3 function, write the documentation
def _validate_profile(self, bag):
        if not self.profile_validator.validate(bag):
            raise Exception(str(self.profile_validator.report))

Response: Validate that the bag - info fields are valid.


Instruction:
Given the following Python 3 function, write the documentation
def _validate_bag(self, bag, **kwargs):
        failed = None
        try:
            bag.validate(**kwargs)
        except BagValidationError as e:
            failed = e
            #  for d in e.details:
            #      if isinstance(d, ChecksumMismatch):
            #          log.error("Validation Error: expected %s to have %s checksum of %s but found %s", d.path, d.algorithm, d.expected, d.found)
            #      else:
            #          log.error("Validation Error: %s", d)
        if failed:
            raise BagValidationError("%s" % failed)

Response: Validate BagIt (checksums, payload.oxum etc)


Instruction:
Implement a Python 3 function for
validating an OCRD - ZIP file for profile bag and workspace.

Response:
def validate(self, skip_checksums=False, skip_bag=False, skip_unzip=False, skip_delete=False, processes=2):
        """
        Validate an OCRD-ZIP file for profile, bag and workspace conformance

        Arguments:
            skip_bag (boolean): Whether to skip all checks of manifests and files
            skip_checksums (boolean): Whether to omit checksum checks but still check basic BagIt conformance
            skip_unzip (boolean): Whether the OCRD-ZIP is unzipped, i.e. a directory
            skip_delete (boolean): Whether to skip deleting the unpacked OCRD-ZIP dir after valdiation
            processes (integer): Number of processes used for checksum validation

        """
        if skip_unzip:
            bagdir = self.path_to_zip
            skip_delete = True
        else:
            #  try:
            self.profile_validator.validate_serialization(self.path_to_zip)
            #  except IOError as err:
            #      raise err
            #  except ProfileValidationError as err:
            #      self.report.add_error(err.value)
            bagdir = mkdtemp(prefix=TMP_BAGIT_PREFIX)
            unzip_file_to_dir(self.path_to_zip, bagdir)

        try:
            bag = Bag(bagdir)
            self._validate_profile(bag)

            if not skip_bag:
                self._validate_bag(bag, fast=skip_checksums, processes=processes)

        finally:
            if not skip_delete:
                # remove tempdir
                rmtree(bagdir)
        return self.report


Instruction:
Here you have a function in Python 3, explain what it does
def quote_xml(inStr):
    "Escape markup chars, but do not modify CDATA sections."
    if not inStr:
        return ''
    s1 = (isinstance(inStr, BaseStrType_) and inStr or '%s' % inStr)
    s2 = ''
    pos = 0
    matchobjects = CDATA_pattern_.finditer(s1)
    for mo in matchobjects:
        s3 = s1[pos:mo.start()]
        s2 += quote_xml_aux(s3)
        s2 += s1[mo.start():mo.end()]
        pos = mo.end()
    s3 = s1[pos:]
    s2 += quote_xml_aux(s3)
    return s2

Response: Escape markup chars but do not modify CDATA sections.


Instruction:
How would you explain what the following Python 3 function does
def parseString(inString, silence=False):
    '''Parse a string, create the object tree, and export it.

    Arguments:
    - inString -- A string.  This XML fragment should not start
      with an XML declaration containing an encoding.
    - silence -- A boolean.  If False, export the object.
    Returns -- The root object in the tree.
    '''
    parser = None
    rootNode= parsexmlstring_(inString, parser)
    rootTag, rootClass = get_root_tag(rootNode)
    if rootClass is None:
        rootTag = 'PcGts'
        rootClass = PcGts
    rootObj = rootClass.factory()
    rootObj.build(rootNode)
    # Enable Python to collect the space used by the DOM.
    if not silence:
        sys.stdout.write('<?xml version="1.0" ?>\n')
        rootObj.export(
            sys.stdout, 0, name_=rootTag,
            namespacedef_='xmlns:pc="http://schema.primaresearch.org/PAGE/gts/pagecontent/2018-07-15"')
    return rootObj

Response: Parse a string and create the object tree and export it.


Instruction:
Write a Python 3 function that can
parse parameters with fallback to defaults and output as shell - evalable assignments to params var.

Response:
def ocrd_tool_tool_parse_params(ctx, parameters, json):
    """
    Parse parameters with fallback to defaults and output as shell-eval'able assignments to params var.
    """
    if parameters is None or parameters == "":
        parameters = {}
    else:
        with open(parameters, 'r') as f:
            parameters = loads(f.read())
    parameterValidator = ParameterValidator(ctx.json['tools'][ctx.tool_name])
    report = parameterValidator.validate(parameters)
    if not report.is_valid:
        print(report.to_xml())
        sys.exit(1)
    if json:
        print(dumps(parameters))
    else:
        for k in parameters:
            print('params["%s"]="%s"' % (k, parameters[k]))


Instruction:
Implement a function in Python 3 to
set the OTHERTYPE attribute value.

Response:
def othertype(self, othertype):
        """
        Set the ``OTHERTYPE`` attribute value.
        """
        if othertype is not None:
            self._el.set('TYPE', 'OTHER')
            self._el.set('OTHERTYPE', othertype)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def otherrole(self, otherrole):
        if otherrole is not None:
            self._el.set('ROLE', 'OTHER')
            self._el.set('OTHERROLE', otherrole)

Response: Sets the OTHERROLE attribute value.


Instruction:
Can you implement a function in Python 3 that
gets the name of the user s company.

Response:
def name(self):
        """
        Get the ``mets:name`` element value.
        """
        el_name = self._el.find('mets:name', NS)
        if el_name is not None:
            return el_name.text


Instruction:
Can you generate a brief explanation for the following Python 3 code
def name(self, name):
        if name is not None:
            el_name = self._el.find('mets:name', NS)
            if el_name is None:
                el_name = ET.SubElement(self._el, TAG_METS_NAME)
            el_name.text = name

Response: Sets the name of the user.


Instruction:
Can you write a function in Python 3 where it
validates a parameter dict against a parameter schema from an ocrd - tool. json

Response:
def validate(self, *args, **kwargs): # pylint: disable=arguments-differ
        """
        Validate a parameter dict against a parameter schema from an ocrd-tool.json

        Args:
            obj (dict):
            schema (dict):
        """
        return super(ParameterValidator, self)._validate(*args, **kwargs)


Instruction:
Can you implement a function in Python 3 that
checks whether the text results on an element is consistent with its child element text results.

Response:
def handle_inconsistencies(node, strictness, strategy, report):
    """
    Check whether the text results on an element is consistent with its child element text results.
    """
    if isinstance(node, PcGtsType):
        node = node.get_Page()
    elif isinstance(node, GlyphType):
        return report

    _, tag, getter, concatenate_with = [x for x in _HIERARCHY if isinstance(node, x[0])][0]
    children_are_consistent = True
    children = getattr(node, getter)()
    for child in children:
        errors_before = len(report.errors)
        handle_inconsistencies(child, strictness, strategy, report)
        if len(report.errors) > errors_before:
            children_are_consistent = False
    if concatenate_with is not None:
        concatenated_children = concatenate_children(node, concatenate_with, strategy)
        text_results = get_text(node, strategy)
        if concatenated_children and text_results and concatenated_children != text_results:
            if strictness == 'fix':
                set_text(node, concatenated_children, strategy)
                #  if children_are_consistent:
                #  else:
                #      # TODO fix text results recursively
                #      report.add_warning("Fixing inconsistencies recursively not implemented")
            elif strictness == 'lax':
                if not compare_without_whitespace(concatenated_children, text_results):
                    report.add_error(ConsistencyError(tag, node.id, text_results, concatenated_children))
            else:
                report.add_error(ConsistencyError(tag, node.id, text_results, concatenated_children))
    return report


Instruction:
Here you have a function in Python 3, explain what it does
def concatenate_children(node, concatenate_with, strategy):
    _, _, getter, concatenate_with = [x for x in _HIERARCHY if isinstance(node, x[0])][0]
    tokens = [get_text(x, strategy) for x in getattr(node, getter)()]
    return concatenate_with.join(tokens).strip()

Response: Concatenate children of node according to https://ocr - d. github. io. page#consistency - of - text - results - on - different - levels


Instruction:
Explain what the following Python 3 code does
def get_text(node, strategy):
    textEquivs = node.get_TextEquiv()
    if not textEquivs:
        log.debug("No text results on %s %s", node, node.id)
        return ''
    #  elif strategy == 'index1':
    else:
        if len(textEquivs) > 1:
            index1 = [x for x in textEquivs if x.index == 1]
            if index1:
                return index1[0].get_Unicode().strip()
        return textEquivs[0].get_Unicode().strip()

Response: Get the most confident text results for a node.


Instruction:
Can you tell what is the following Python 3 function doing
def set_text(node, text, strategy):
    text = text.strip()
    textEquivs = node.get_TextEquiv()
    if not textEquivs:
        node.add_TextEquiv(TextEquivType(Unicode=text))
    #  elif strategy == 'index1':
    else:
        if len(textEquivs) > 1:
            index1 = [x for x in textEquivs if x.index == 1]
            if index1:
                index1[0].set_Unicode(text)
                return
        textEquivs[0].set_Unicode(text)

Response: Set the most confident text results for a node.


Instruction:
Create a Python 3 function for
validating a PAGE file for consistency by filename OcrdPage or OcrdFile directly.

Response:
def validate(filename=None, ocrd_page=None, ocrd_file=None, strictness='strict', strategy='index1'):
        """
        Validates a PAGE file for consistency by filename, OcrdFile or passing OcrdPage directly.

        Arguments:
            filename (string): Path to PAGE
            ocrd_page (OcrdPage): OcrdPage instance
            ocrd_file (OcrdFile): OcrdFile instance wrapping OcrdPage
            strictness (string): 'strict', 'lax', 'fix' or 'off'
            strategy (string): Currently only 'index1'

        Returns:
            report (:class:`ValidationReport`) Report on the validity
        """
        if ocrd_page:
            validator = PageValidator(ocrd_page, strictness, strategy)
        elif ocrd_file:
            validator = PageValidator(page_from_file(ocrd_file), strictness, strategy)
        elif filename:
            validator = PageValidator(parse(filename, silence=True), strictness, strategy)
        else:
            raise Exception("At least one of ocrd_page, ocrd_file or filename must be set")
        return validator._validate()


Instruction:
Can you write a function in Python 3 where it
validates the current page and report.

Response:
def _validate(self):
        """
        Do the actual validation
        """
        if self.strictness == 'off':
            return self.report
        handle_inconsistencies(self.page, self.strictness, self.strategy, self.report)
        return self.report


Instruction:
Can you create a Python 3 function that
implements MP CLI. Usage:: import ocrd_click_cli from ocrd.utils @click.command() @ocrd_click_cli def cli(mets_url): print(mets_url)

Response:
def ocrd_cli_options(f):
    """
    Implement MP CLI.

    Usage::

        import ocrd_click_cli from ocrd.utils

        @click.command()
        @ocrd_click_cli
        def cli(mets_url):
            print(mets_url)
    """
    params = [
        click.option('-m', '--mets', help="METS URL to validate"),
        click.option('-w', '--working-dir', help="Working Directory"),
        click.option('-I', '--input-file-grp', help='File group(s) used as input.', default='INPUT'),
        click.option('-O', '--output-file-grp', help='File group(s) used as output.', default='OUTPUT'),
        click.option('-g', '--page-id', help="ID(s) of the pages to process"),
        click.option('-p', '--parameter', type=click.Path()),
        click.option('-J', '--dump-json', help="Dump tool description as JSON and exit", is_flag=True, default=False),
        loglevel_option,
        click.option('-V', '--version', help="Show version", is_flag=True, default=False)
    ]
    for param in params:
        param(f)
    return f


Instruction:
Can you implement a function in Python 3 that
merges another report into this one.

Response:
def merge_report(self, otherself):
        """
        Merge another report into this one.
        """
        self.notices += otherself.notices
        self.warnings += otherself.warnings
        self.errors += otherself.errors


Instruction:
How would you implement a function in Python 3 that
processes a series of tasks in the cli

Response:
def process_cli(log_level, mets, page_id, tasks):
    """
    Process a series of tasks
    """
    log = getLogger('ocrd.cli.process')

    run_tasks(mets, log_level, page_id, tasks)
    log.info("Finished")


Instruction:
Implement a function in Python 3 to
bag a new NCSC - ZIP workspace at DEST.

Response:
def bag(directory, mets_basename, dest, identifier, in_place, manifestation_depth, mets, base_version_checksum, tag_file, skip_zip, processes):
    """
    Bag workspace as OCRD-ZIP at DEST
    """
    resolver = Resolver()
    workspace = Workspace(resolver, directory=directory, mets_basename=mets_basename)
    workspace_bagger = WorkspaceBagger(resolver)
    workspace_bagger.bag(
        workspace,
        dest=dest,
        ocrd_identifier=identifier,
        ocrd_manifestation_depth=manifestation_depth,
        ocrd_mets=mets,
        ocrd_base_version_checksum=base_version_checksum,
        processes=processes,
        tag_files=tag_file,
        skip_zip=skip_zip,
        in_place=in_place
    )


Instruction:
Write a Python 3 script for
spilling a single file into a single file in a directory.

Response:
def spill(directory, src):
    """
    Spill/unpack OCRD-ZIP bag at SRC to DEST

    SRC must exist an be an OCRD-ZIP
    DEST must not exist and be a directory
    """
    resolver = Resolver()
    workspace_bagger = WorkspaceBagger(resolver)
    workspace = workspace_bagger.spill(src, directory)
    print(workspace)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def validate(src, **kwargs):
    resolver = Resolver()
    validator = OcrdZipValidator(resolver, src)
    report = validator.validate(**kwargs)
    print(report)
    if not report.is_valid:
        sys.exit(1)

Response:
Validate OCRD - ZIP
    SRC must exist an be an OCRD - ZIP file or a directory.


Instruction:
Explain what the following Python 3 code does
def restore(self, chksum, choose_first=False):
        log = getLogger('ocrd.workspace_backup.restore')
        bak = None
        candidates = glob(join(self.backup_directory, '%s*' % chksum))
        if not candidates:
            log.error("No backup found: %s" % chksum)
            return
        if len(candidates) > 1 and not choose_first:
            raise Exception("Not unique, could be\n%s" % '\n'.join(candidates))
        bak = candidates[0]
        self.add()
        log.info("Restoring from %s/mets.xml" % bak)
        src = join(bak, 'mets.xml')
        dest = self.workspace.mets_target
        log.debug('cp "%s" "%s"', src, dest)
        copy(src, dest)
        self.workspace.reload_mets()

Response: Restore the mets. xml to the previous state.


Instruction:
How would you code a function in Python 3 to
create a new backup in self. backup_directory

Response:
def add(self):
        """
        Create a backup in <self.backup_directory>
        """
        log = getLogger('ocrd.workspace_backup.add')
        mets_str = self.workspace.mets.to_xml()
        chksum = _chksum(mets_str)
        backups = self.list()
        if backups and backups[0].chksum == chksum:
            log.info('No changes since last backup: %s' % backups[0])
        else:
            timestamp = datetime.now().timestamp()
            d = join(self.backup_directory, '%s.%s' % (chksum, timestamp))
            mets_file = join(d, 'mets.xml')
            log.info("Backing up to %s" % mets_file)
            makedirs(d)
            with open(mets_file, 'wb') as f:
                f.write(mets_str)
        return chksum


Instruction:
Create a Python 3 function for
listing all backups as WorkspaceBackup objects sorted descending by lastmod.

Response:
def list(self):
        """
        List all backups as WorkspaceBackup objects, sorted descending by lastmod.
        """
        backups = []
        for d in glob(join(self.backup_directory, '*')):
            backups.append(WorkspaceBackup.from_path(d))
        backups.sort(key=lambda b: b.lastmod, reverse=True)
        return backups


Instruction:
Implement a function in Python 3 to
restore to last version of the backup

Response:
def undo(self):
        """
        Restore to last version
        """
        log = getLogger('ocrd.workspace_backup.undo')
        backups = self.list()
        if backups:
            last_backup = backups[0]
            self.restore(last_backup.chksum, choose_first=True)
        else:
            log.info("No backups, nothing to undo.")


Instruction:
Make a summary of the following Python 3 code
def to_xml(el):
    sio = StringIO()
    el.export(sio, 0, name_='PcGts', namespacedef_='xmlns:pc="%s"' % NAMESPACES['page'])
    return '<?xml version="1.0" encoding="UTF-8"?>\n' + sio.getvalue()

Response: Serialize a PCC element into an XML document.


Instruction:
Can you implement a function in Python 3 that
gets the name of a python logging level.

Response:
def getLevelName(lvl):
    """
    Get (numerical) python logging level for (string) spec-defined log level name.
    """
    lvl = _ocrdLevel2pythonLevel.get(lvl, lvl)
    return logging.getLevelName(lvl)


Instruction:
Can you implement a function in Python 3 that
overrides all loggers to include lvl and above.

Response:
def setOverrideLogLevel(lvl):
    """
    Override all logger filter levels to include lvl and above.

    - Set root logger level
    - iterates all existing loggers and sets their log level to ``NOTSET``.

    Args:
        lvl (string): Log level name.
    """
    if lvl is None:
        return
    logging.info('Overriding log level globally to %s', lvl)
    lvl = getLevelName(lvl)
    global _overrideLogLevel # pylint: disable=global-statement
    _overrideLogLevel = lvl
    logging.getLogger('').setLevel(lvl)
    for loggerName in logging.Logger.manager.loggerDict:
        logger = logging.Logger.manager.loggerDict[loggerName]
        if isinstance(logger, logging.PlaceHolder):
            continue
        logger.setLevel(logging.NOTSET)


Instruction:
Create a Python 3 function for
wrapping around logging. getLogger that respects overrideLogLevel

Response:
def getLogger(*args, **kwargs):
    """
    Wrapper around ``logging.getLogger`` that respects `overrideLogLevel <#setOverrideLogLevel>`_.
    """
    logger = logging.getLogger(*args, **kwargs)
    if _overrideLogLevel is not None:
        logger.setLevel(logging.NOTSET)
    return logger


Instruction:
Explain what the following Python 3 code does
def initLogging():
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s.%(msecs)03d %(levelname)s %(name)s - %(message)s',
        datefmt='%H:%M:%S')
    logging.getLogger('').setLevel(logging.INFO)
    #  logging.getLogger('ocrd.resolver').setLevel(logging.INFO)
    #  logging.getLogger('ocrd.resolver.download_to_directory').setLevel(logging.INFO)
    #  logging.getLogger('ocrd.resolver.add_files_to_mets').setLevel(logging.INFO)
    logging.getLogger('PIL').setLevel(logging.INFO)

    # Allow overriding

    CONFIG_PATHS = [
        os.path.curdir,
        os.path.join(os.path.expanduser('~')),
        '/etc',
    ]

    for p in CONFIG_PATHS:
        config_file = os.path.join(p, 'ocrd_logging.py')
        if os.path.exists(config_file):
            logging.info("Loading logging configuration from '%s'", config_file)
            with open(config_file) as f:
                code = compile(f.read(), config_file, 'exec')
                exec(code, globals(), locals())

Response: Initializes the logging module


Instruction:
Make a summary of the following Python 3 code
def bag(self,
            workspace,
            ocrd_identifier,
            dest=None,
            ocrd_mets='mets.xml',
            ocrd_manifestation_depth='full',
            ocrd_base_version_checksum=None,
            processes=1,
            skip_zip=False,
            in_place=False,
            tag_files=None
           ):
        """
        Bag a workspace

        See https://ocr-d.github.com/ocrd_zip#packing-a-workspace-as-ocrd-zip

        Arguments:
            workspace (ocrd.Workspace): workspace to bag
            ord_identifier (string): Ocrd-Identifier in bag-info.txt
            dest (string): Path of the generated OCRD-ZIP.
            ord_mets (string): Ocrd-Mets in bag-info.txt
            ord_manifestation_depth (string): Ocrd-Manifestation-Depth in bag-info.txt
            ord_base_version_checksum (string): Ocrd-Base-Version-Checksum in bag-info.txt
            processes (integer): Number of parallel processes checksumming
            skip_zip (boolean): Whether to leave directory unzipped
            in_place (boolean): Whether to **replace** the workspace with its BagIt variant
            tag_files (list<string>): Path names of additional tag files to be bagged at the root of the bag
        """
        if ocrd_manifestation_depth not in ('full', 'partial'):
            raise Exception("manifestation_depth must be 'full' or 'partial'")
        if in_place and (dest is not None):
            raise Exception("Setting 'dest' and 'in_place' is a contradiction")
        if in_place and not skip_zip:
            raise Exception("Setting 'skip_zip' and not 'in_place' is a contradiction")

        if tag_files is None:
            tag_files = []

        # create bagdir
        bagdir = mkdtemp(prefix=TMP_BAGIT_PREFIX)

        if dest is None:
            if in_place:
                dest = workspace.directory
            elif not skip_zip:
                dest = '%s.ocrd.zip' % workspace.directory
            else:
                dest = '%s.ocrd' % workspace.directory

        log.info("Bagging %s to %s (temp dir %s)", workspace.directory, '(in-place)' if in_place else dest, bagdir)

        # create data dir
        makedirs(join(bagdir, 'data'))

        # create bagit.txt
        with open(join(bagdir, 'bagit.txt'), 'wb') as f:
            f.write(BAGIT_TXT.encode('utf-8'))

        # create manifests
        total_bytes, total_files = self._bag_mets_files(workspace, bagdir, ocrd_manifestation_depth, ocrd_mets, processes)

        # create bag-info.txt
        bag = Bag(bagdir)
        self._set_bag_info(bag, total_bytes, total_files, ocrd_identifier, ocrd_manifestation_depth, ocrd_base_version_checksum)

        for tag_file in tag_files:
            copyfile(tag_file, join(bagdir, basename(tag_file)))

        # save bag
        bag.save()

        # ZIP it
        self._serialize_bag(workspace, bagdir, dest, in_place, skip_zip)

        log.info('Created bag at %s', dest)
        return dest

Response: Bag a workspace into a new file.


Instruction:
Can you generate the documentation for the following Python 3 function
def spill(self, src, dest):
        #  print(dest)

        if exists(dest) and not isdir(dest):
            raise Exception("Not a directory: %s" % dest)

        # If dest is an existing directory, try to derive its name from src
        if isdir(dest):
            workspace_name = re.sub(r'(\.ocrd)?\.zip$', '', basename(src))
            new_dest = join(dest, workspace_name)
            if exists(new_dest):
                raise Exception("Directory exists: %s" % new_dest)
            dest = new_dest

        log.info("Spilling %s to %s", src, dest)

        bagdir = mkdtemp(prefix=TMP_BAGIT_PREFIX)
        unzip_file_to_dir(src, bagdir)

        datadir = join(bagdir, 'data')
        for root, _, files in walk(datadir):
            for f in files:
                srcfile = join(root, f)
                destdir = join(dest, relpath(root, datadir))
                destfile = join(destdir, f)
                if not exists(destdir):
                    makedirs(destdir)
                log.debug("Copy %s -> %s", srcfile, destfile)
                copyfile(srcfile, destfile)

        # TODO copy allowed tag files if present

        # TODO validate bagit

        # Drop tempdir
        rmtree(bagdir)

        # Create workspace
        workspace = Workspace(self.resolver, directory=dest)

        # TODO validate workspace

        return workspace

Response: Spill a workspace from src into dest.


Instruction:
Create a Python 3 function for
downloading a URL to the workspace.

Response:
def download_url(self, url, **kwargs):
        """
        Download a URL to the workspace.

        Args:
            url (string): URL to download to directory
            **kwargs : See :py:mod:`ocrd.resolver.Resolver`

        Returns:
            The local filename of the downloaded file
        """
        if self.baseurl and '://' not in url:
            url = join(self.baseurl, url)
        return self.resolver.download_to_directory(self.directory, url, **kwargs)


Instruction:
Can you tell what is the following Python 3 function doing
def download_file(self, f):
        #  os.chdir(self.directory)
        #  log.info('f=%s' % f)
        oldpwd = os.getcwd()
        try:
            os.chdir(self.directory)
            if is_local_filename(f.url):
                f.local_filename = abspath(f.url)
            else:
                if f.local_filename:
                    log.debug("Already downloaded: %s", f.local_filename)
                else:
                    f.local_filename = self.download_url(f.url, basename='%s/%s' % (f.fileGrp, f.ID))
        finally:
            os.chdir(oldpwd)

        #  print(f)
        return f

Response: Download a file to the workspace.


Instruction:
Write a Python 3 function for
adding an output file to the Mets output section.

Response:
def add_file(self, file_grp, content=None, **kwargs):
        """
        Add an output file. Creates an :class:`OcrdFile` to pass around and adds that to the
        OcrdMets OUTPUT section.
        """
        log.debug(
            'outputfile file_grp=%s local_filename=%s content=%s',
            file_grp,
            kwargs.get('local_filename'),
            content is not None)
        if content is not None and 'local_filename' not in kwargs:
            raise Exception("'content' was set but no 'local_filename'")

        oldpwd = os.getcwd()
        try:
            os.chdir(self.directory)
            if 'local_filename' in kwargs:
                local_filename_dir = kwargs['local_filename'].rsplit('/', 1)[0]
                if not os.path.isdir(local_filename_dir):
                    os.makedirs(local_filename_dir)
                if 'url' not in kwargs:
                    kwargs['url'] = kwargs['local_filename']

            #  print(kwargs)
            ret = self.mets.add_file(file_grp, **kwargs)

            if content is not None:
                with open(kwargs['local_filename'], 'wb') as f:
                    if isinstance(content, str):
                        content = bytes(content, 'utf-8')
                    f.write(content)
        finally:
            os.chdir(oldpwd)

        return ret


Instruction:
Create a Python 3 function for
writing out the METS file.

Response:
def save_mets(self):
        """
        Write out the current state of the METS file.
        """
        log.info("Saving mets '%s'" % self.mets_target)
        if self.automatic_backup:
            WorkspaceBackupManager(self).add()
        with open(self.mets_target, 'wb') as f:
            f.write(self.mets.to_xml(xmllint=True))


Instruction:
Given the following Python 3 function, write the documentation
def resolve_image_exif(self, image_url):
        files = self.mets.find_files(url=image_url)
        if files:
            image_filename = self.download_file(files[0]).local_filename
        else:
            image_filename = self.download_url(image_url)

        if image_url not in self.image_cache['exif']:
            self.image_cache['exif'][image_url] = OcrdExif(Image.open(image_filename))
        return self.image_cache['exif'][image_url]

Response: Resolve an image URL to an EXIF metadata object.


Instruction:
How would you code a function in Python 3 to
resolve an image URL to a PIL image.

Response:
def resolve_image_as_pil(self, image_url, coords=None):
        """
        Resolve an image URL to a PIL image.

        Args:
            coords (list) : Coordinates of the bounding box to cut from the image

        Returns:
            Image or region in image as PIL.Image
        """
        files = self.mets.find_files(url=image_url)
        if files:
            image_filename = self.download_file(files[0]).local_filename
        else:
            image_filename = self.download_url(image_url)

        if image_url not in self.image_cache['pil']:
            self.image_cache['pil'][image_url] = Image.open(image_filename)

        pil_image = self.image_cache['pil'][image_url]

        if coords is None:
            return pil_image
        if image_url not in self.image_cache['cv2']:
            log.debug("Converting PIL to OpenCV: %s", image_url)
            color_conversion = cv2.COLOR_GRAY2BGR if pil_image.mode in ('1', 'L') else  cv2.COLOR_RGB2BGR
            pil_as_np_array = np.array(pil_image).astype('uint8') if pil_image.mode == '1' else np.array(pil_image)
            self.image_cache['cv2'][image_url] = cv2.cvtColor(pil_as_np_array, color_conversion)
        cv2_image = self.image_cache['cv2'][image_url]
        poly = np.array(coords, np.int32)
        log.debug("Cutting region %s from %s", coords, image_url)
        region_cut = cv2_image[
            np.min(poly[:, 1]):np.max(poly[:, 1]),
            np.min(poly[:, 0]):np.max(poly[:, 0])
        ]
        return Image.fromarray(region_cut)


Instruction:
Write a Python 3 script to
serialize all properties as XML

Response:
def to_xml(self):
        """
        Serialize all properties as XML
        """
        ret = '<exif>'
        for k in self.__dict__:
            ret += '<%s>%s</%s>' % (k, self.__dict__[k], k)
        ret += '</exif>'
        return ret


Instruction:
Make a summary of the following Python 3 code
def basename_without_extension(self):
        ret = self.basename.rsplit('.', 1)[0]
        if ret.endswith('.tar'):
            ret = ret[0:len(ret)-4]
        return ret

Response: Get the os. path. basename of the local file without the extension removed.


Instruction:
Can you write a function in Python 3 where it
gets the ID of the physical page this file manifests belongs to.

Response:
def pageId(self):
        """
        Get the ID of the physical page this file manifests.
        """
        if self.mets is None:
            raise Exception("OcrdFile %s has no member 'mets' pointing to parent OcrdMets" % self)
        return self.mets.get_physical_page_for_file(self)


Instruction:
How would you implement a function in Python 3 that
sets the ID of the physical page this file manifests.

Response:
def pageId(self, pageId):
        """
        Set the ID of the physical page this file manifests.
        """
        if pageId is None:
            return
        if self.mets is None:
            raise Exception("OcrdFile %s has no member 'mets' pointing to parent OcrdMets" % self)
        self.mets.set_physical_page_for_file(pageId, self)


Instruction:
Here you have a function in Python 3, explain what it does
def url(self):
        el_FLocat = self._el.find(TAG_METS_FLOCAT)
        if el_FLocat is not None:
            return el_FLocat.get("{%s}href" % NS["xlink"])
        return ''

Response: Get the xlink href of this file.


Instruction:
Implement a function in Python 3 to
set the xlink : href of this file.

Response:
def url(self, url):
        """
        Set the ``xlink:href`` of this file.
        """
        if url is None:
            return
        el_FLocat = self._el.find('mets:FLocat', NS)
        if el_FLocat is None:
            el_FLocat = ET.SubElement(self._el, TAG_METS_FLOCAT)
        el_FLocat.set("{%s}href" % NS["xlink"], url)


Instruction:
Given the following Python 3 function, write the documentation
def empty_mets():
        tpl = METS_XML_EMPTY.decode('utf-8')
        tpl = tpl.replace('{{ VERSION }}', VERSION)
        tpl = tpl.replace('{{ NOW }}', '%s' % datetime.now())
        return OcrdMets(content=tpl.encode('utf-8'))

Response: Create an empty METS file from bundled template.


Instruction:
Here you have a function in Python 3, explain what it does
def unique_identifier(self):
        for t in IDENTIFIER_PRIORITY:
            found = self._tree.getroot().find('.//mods:identifier[@type="%s"]' % t, NS)
            if found is not None:
                return found.text

Response: Get the unique identifier of the moderation.


Instruction:
Implement a Python 3 function for
setting the unique identifier by looking through mods. identifier and then looking through the identifier.

Response:
def unique_identifier(self, purl):
        """
        Set the unique identifier by looking through ``mods:identifier``

        See `specs <https://ocr-d.github.io/mets#unique-id-for-the-document-processed>`_ for details.
        """
        id_el = None
        for t in IDENTIFIER_PRIORITY:
            id_el = self._tree.getroot().find('.//mods:identifier[@type="%s"]' % t, NS)
            if id_el is not None:
                break
        if id_el is None:
            mods = self._tree.getroot().find('.//mods:mods', NS)
            id_el = ET.SubElement(mods, TAG_MODS_IDENTIFIER)
            id_el.set('type', 'purl')
        id_el.text = purl


Instruction:
Implement a function in Python 3 to
list all OcrdAgent elements

Response:
def agents(self):
        """
        List all `OcrdAgent </../../ocrd_models/ocrd_models.ocrd_agent.html>`_
        """
        return [OcrdAgent(el_agent) for el_agent in self._tree.getroot().findall('mets:metsHdr/mets:agent', NS)]


Instruction:
Write a Python 3 script to
add an OcrdAgent to the list of agents in the metsHdr.

Response:
def add_agent(self, *args, **kwargs):
        """
        Add an `OcrdAgent </../../ocrd_models/ocrd_models.ocrd_agent.html>`_ to the list of agents in the metsHdr.
        """
        el_metsHdr = self._tree.getroot().find('.//mets:metsHdr', NS)
        if el_metsHdr is None:
            el_metsHdr = ET.Element(TAG_METS_METSHDR)
            self._tree.getroot().insert(0, el_metsHdr)
        #  assert(el_metsHdr is not None)
        el_agent = ET.SubElement(el_metsHdr, TAG_METS_AGENT)
        #  print(ET.tostring(el_metsHdr))
        return OcrdAgent(el_agent, *args, **kwargs)


Instruction:
Write a Python 3 script to
list the USE attributes of all fileGrp.

Response:
def file_groups(self):
        """
        List the ``USE`` attributes of all ``mets:fileGrp``.
        """
        return [el.get('USE') for el in self._tree.getroot().findall('.//mets:fileGrp', NS)]


Instruction:
Can you write a function in Python 3 where it
searches METS document for files of a given file.

Response:
def find_files(self, ID=None, fileGrp=None, pageId=None, mimetype=None, url=None, local_only=False):
        """
        Search ``mets:file`` in this METS document.

        Args:
            ID (string) : ID of the file
            fileGrp (string) : USE of the fileGrp to list files of
            pageId (string) : ID of physical page manifested by matching files
            url (string) : @xlink:href of mets:Flocat of mets:file
            mimetype (string) : MIMETYPE of matching files
            local (boolean) : Whether to restrict results to local files, i.e. file://-URL

        Return:
            List of files.
        """
        ret = []
        fileGrp_clause = '' if fileGrp is None else '[@USE="%s"]' % fileGrp
        file_clause = ''
        if ID is not None:
            file_clause += '[@ID="%s"]' % ID
        if mimetype is not None:
            file_clause += '[@MIMETYPE="%s"]' % mimetype
        if url is not None:
            file_clause += '[mets:FLocat[@xlink:href = "%s"]]' % url
        # TODO lxml says invalid predicate. I disagree
        #  if local_only:
        #      file_clause += "[mets:FLocat[starts-with(@xlink:href, 'file://')]]"

        # Search
        file_ids = self._tree.getroot().xpath("//mets:fileGrp%s/mets:file%s/@ID" % (fileGrp_clause, file_clause), namespaces=NS)
        if pageId is not None:
            by_pageid = self._tree.getroot().xpath('//mets:div[@TYPE="page"][@ID="%s"]/mets:fptr/@FILEID' % pageId, namespaces=NS)
            file_ids = [i for i in by_pageid if i in file_ids]

        # instantiate / get from cache
        for file_id in file_ids:
            el = self._tree.getroot().find('.//mets:file[@ID="%s"]' % file_id, NS)
            if file_id not in self._file_by_id:
                self._file_by_id[file_id] = OcrdFile(el, mets=self)

            # If only local resources should be returned and file is neither a
            # file:// URL nor a file path: skip the file
            url = self._file_by_id[file_id].url
            if local_only and not (url.startswith('file://') or '://' not in url):
                continue
            ret.append(self._file_by_id[file_id])
        return ret


Instruction:
Can you generate the documentation for the following Python 3 function
def add_file_group(self, fileGrp):
        el_fileSec = self._tree.getroot().find('mets:fileSec', NS)
        if el_fileSec is None:
            el_fileSec = ET.SubElement(self._tree.getroot(), TAG_METS_FILESEC)
        el_fileGrp = el_fileSec.find('mets:fileGrp[@USE="%s"]' % fileGrp, NS)
        if el_fileGrp is None:
            el_fileGrp = ET.SubElement(el_fileSec, TAG_METS_FILEGRP)
            el_fileGrp.set('USE', fileGrp)
        return el_fileGrp

Response: Adds a new filegroup to the file group.


Instruction:
Can you tell what is the following Python 3 function doing
def add_file(self, fileGrp, mimetype=None, url=None, ID=None, pageId=None, force=False, local_filename=None, **kwargs):
        if not ID:
            raise Exception("Must set ID of the mets:file")
        el_fileGrp = self._tree.getroot().find(".//mets:fileGrp[@USE='%s']" % (fileGrp), NS)
        if el_fileGrp is None:
            el_fileGrp = self.add_file_group(fileGrp)
        if ID is not None and self.find_files(ID=ID) != []:
            if not force:
                raise Exception("File with ID='%s' already exists" % ID)
            mets_file = self.find_files(ID=ID)[0]
        else:
            mets_file = OcrdFile(ET.SubElement(el_fileGrp, TAG_METS_FILE), mets=self)
        mets_file.url = url
        mets_file.mimetype = mimetype
        mets_file.ID = ID
        mets_file.pageId = pageId
        mets_file.local_filename = local_filename

        self._file_by_id[ID] = mets_file

        return mets_file

Response: Add a file to the Mets object.


Instruction:
Create a Python 3 function to
set the physical page for a file.

Response:
def set_physical_page_for_file(self, pageId, ocrd_file, order=None, orderlabel=None):
        """
        Create a new physical page
        """
        #  print(pageId, ocrd_file)
        # delete any page mapping for this file.ID
        for el_fptr in self._tree.getroot().findall(
                'mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"]/mets:fptr[@FILEID="%s"]' %
                ocrd_file.ID, namespaces=NS):
            el_fptr.getparent().remove(el_fptr)

        # find/construct as necessary
        el_structmap = self._tree.getroot().find('mets:structMap[@TYPE="PHYSICAL"]', NS)
        if el_structmap is None:
            el_structmap = ET.SubElement(self._tree.getroot(), TAG_METS_STRUCTMAP)
            el_structmap.set('TYPE', 'PHYSICAL')
        el_seqdiv = el_structmap.find('mets:div[@TYPE="physSequence"]', NS)
        if el_seqdiv is None:
            el_seqdiv = ET.SubElement(el_structmap, TAG_METS_DIV)
            el_seqdiv.set('TYPE', 'physSequence')
        el_pagediv = el_seqdiv.find('mets:div[@ID="%s"]' % pageId, NS)
        if el_pagediv is None:
            el_pagediv = ET.SubElement(el_seqdiv, TAG_METS_DIV)
            el_pagediv.set('TYPE', 'page')
            el_pagediv.set('ID', pageId)
            if order:
                el_pagediv.set('ORDER', order)
            if orderlabel:
                el_pagediv.set('ORDERLABEL', orderlabel)
        el_fptr = ET.SubElement(el_pagediv, TAG_METS_FPTR)
        el_fptr.set('FILEID', ocrd_file.ID)


Instruction:
Implement a function in Python 3 to
get the physical page for a given ocrd_file

Response:
def get_physical_page_for_file(self, ocrd_file):
        """
        Get the pageId for a ocrd_file
        """
        ret = self._tree.getroot().xpath(
            '/mets:mets/mets:structMap[@TYPE="PHYSICAL"]/mets:div[@TYPE="physSequence"]/mets:div[@TYPE="page"][./mets:fptr[@FILEID="%s"]]/@ID' %
            ocrd_file.ID, namespaces=NS)
        if ret:
            return ret[0]


Instruction:
Explain what the following Python 3 code does
def _resolve_workspace(self):
        if self.workspace is None:
            self.workspace = self.resolver.workspace_from_url(self.mets_url, baseurl=self.src_dir, download=self.download)
            self.mets = self.workspace.mets

Response: Clone workspace from mets_url unless workspace was provided.


Instruction:
Can you generate the documentation for the following Python 3 function
def _validate_pixel_density(self):
        for f in [f for f in self.mets.find_files() if f.mimetype.startswith('image/')]:
            if not f.local_filename and not self.download:
                self.report.add_notice("Won't download remote image <%s>" % f.url)
                continue
            exif = self.workspace.resolve_image_exif(f.url)
            for k in ['xResolution', 'yResolution']:
                v = exif.__dict__.get(k)
                if v is None or v <= 72:
                    self.report.add_error("Image %s: %s (%s pixels per %s) is too low" % (f.ID, k, v, exif.resolutionUnit))

Response: Validate image pixel density of images.


Instruction:
Given the following Python 3 function, write the documentation
def _validate_mets_file_group_names(self):
        for fileGrp in self.mets.file_groups:
            if not fileGrp.startswith(FILE_GROUP_PREFIX):
                self.report.add_notice("fileGrp USE does not begin with '%s': %s" % (FILE_GROUP_PREFIX, fileGrp))
            else:
                # OCR-D-FOO-BAR -> ('FOO', 'BAR')
                # \____/\_/ \_/
                #   |    |   |
                # Prefix |  Name
                #     Category
                category = fileGrp[len(FILE_GROUP_PREFIX):]
                name = None
                if '-' in category:
                    category, name = category.split('-', 1)
                if category not in FILE_GROUP_CATEGORIES:
                    self.report.add_error("Unspecified USE category '%s' in fileGrp '%s'" % (category, fileGrp))
                if name is not None and not re.match(r'^[A-Z0-9-]{3,}$', name):
                    self.report.add_error("Invalid USE name '%s' in fileGrp '%s'" % (name, fileGrp))

Response: Validate USE attributes of file groups.


Instruction:
Write a Python 3 script for
validating that the files in the Mets database are sane.

Response:
def _validate_mets_files(self):
        """
        Validate ``mets:file`` URLs are sane.
        """
        if not self.mets.find_files():
            self.report.add_error("No files")
        for f in self.mets.find_files():
            if f._el.get('GROUPID'): # pylint: disable=protected-access
                self.report.add_notice("File '%s' has GROUPID attribute - document might need an update" % f.ID)
            if not f.pageId:
                self.report.add_error("File '%s' does not manifest any physical page." % f.ID)
            if 'url' not in self.skip and ':/' in f.url:
                if re.match(r'^file:/[^/]', f.url):
                    self.report.add_warning("File '%s' has an invalid (Java-specific) file URL '%s'" % (f.ID, f.url))
                scheme = f.url[0:f.url.index(':')]
                if scheme not in ('http', 'https', 'file'):
                    self.report.add_warning("File '%s' has non-HTTP, non-file URL '%s'" % (f.ID, f.url))


Instruction:
Write a Python 3 script for
running PageValidator on the PAGE-XML documents referenced in the METS.

Response:
def _validate_page(self):
        """
        Run PageValidator on the PAGE-XML documents referenced in the METS.
        """
        for ocrd_file in self.mets.find_files(mimetype=MIMETYPE_PAGE, local_only=True):
            self.workspace.download_file(ocrd_file)
            page_report = PageValidator.validate(ocrd_file=ocrd_file, strictness=self.page_strictness)
            self.report.merge_report(page_report)


Instruction:
How would you explain what the following Python 3 function does
def create_log_action(self, **kwargs):
        instance = kwargs.get('instance', None)
        if instance is not None:
            del kwargs['instance']

        request = kwargs.get('request', None)
        if request is not None:
            del kwargs['request']
            # Let's grab the current IP of the user.
            x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
            if x_forwarded_for:
                remote_ip = x_forwarded_for.split(',')[0]
            else:
                remote_ip = request.META.get('REMOTE_ADDR')
            kwargs.setdefault('remote_ip', remote_ip)

        if instance is not None:
            pk = self._get_pk_value(instance)

            kwargs.setdefault(
                'content_type',
                ContentType.objects.get_for_model(instance)
            )
            kwargs.setdefault('object_pk', pk)
            kwargs.setdefault('object_repr', smart_text(instance))

            if isinstance(pk, integer_types):
                kwargs.setdefault('object_id', pk)

            get_object_extra_info = getattr(
                instance,
                'get_object_extra_info',
                None
            )

            if callable(get_object_extra_info):
                kwargs.setdefault('object_extra_info', get_object_extra_info())

            # Delete log entries with the same pk as a newly created model.
            # This should only be necessary when an pk is used twice.
            if kwargs.get('action', None) is app_conf.CREATE:
                is_obj_exists = self.filter(
                    content_type=kwargs.get('content_type'),
                    object_id=kwargs.get('object_id')
                ).exists()

                if kwargs.get('object_id', None) is not None and is_obj_exists:
                    self.filter(
                        content_type=kwargs.get('content_type'),
                        object_id=kwargs.get('object_id')
                    ).delete()
                else:
                    self.filter(
                        content_type=kwargs.get('content_type'),
                        object_pk=kwargs.get('object_pk', '')
                    ).delete()

        action_log = self.create(**kwargs)
        action_logged.send(sender=LogAction, action=action_log)
        return action_log

Response: This method creates a new log entry.


Instruction:
Make a summary of the following Python 3 code
def get_for_objects(self, queryset):
        if not isinstance(queryset, QuerySet) or queryset.count() == 0:
            return self.none()

        content_type = ContentType.objects.get_for_model(queryset.model)
        primary_keys = queryset.values_list(queryset.model._meta.pk.name, flat=True)

        if isinstance(primary_keys[0], integer_types):
            return self.filter(content_type=content_type).filter(Q(object_id__in=primary_keys)).distinct()
        else:
            return self.filter(content_type=content_type).filter(Q(object_pk__in=primary_keys)).distinct()

Response: Get the log entries for the objects in the specified queryset.


Instruction:
Can you tell what is the following Python 3 function doing
def register(self, model, include_fields=[], exclude_fields=[]):
        if issubclass(model, Model):
            self._registry[model] = {
                'include_fields': include_fields,
                'exclude_fields': exclude_fields,
            }
            self._connect_signals(model)
        else:
            raise TypeError("Supplied model is not a valid model.")

Response: Register a model with actionslog.


Instruction:
How would you explain what the following Python 3 function does
def track_field(field):
    from actionslog.models import LogAction
    # Do not track many to many relations
    if field.many_to_many:
        return False

    # Do not track relations to LogAction
    if getattr(field, 'rel', None) is not None and field.rel.to == LogAction:
        return False

    return True

Response: Returns whether the given field should be tracked by Actionslog.


Instruction:
Implement a Python 3 function for
calculating the differences between two model instances.

Response:
def model_instance_diff(old, new):
    """
    Calculates the differences between two model instances. One of the instances may be ``None`` (i.e., a newly
    created model or deleted model). This will cause all fields with a value to have changed (from ``None``).

    :param old: The old state of the model instance.
    :type old: Model
    :param new: The new state of the model instance.
    :type new: Model
    :return: A dictionary with the names of the changed fields as keys and a two tuple of the old and new field values
             as value.
    :rtype: dict
    """
    from actionslog.registry import actionslog

    if not(old is None or isinstance(old, Model)):
        raise TypeError("The supplied old instance is not a valid model instance.")
    if not(new is None or isinstance(new, Model)):
        raise TypeError("The supplied new instance is not a valid model instance.")

    diff = {}

    if old is not None and new is not None:
        fields = set(old._meta.fields + new._meta.fields)
        model_fields = actionslog.get_model_fields(new._meta.model)
    elif old is not None:
        fields = set(get_fields_in_model(old))
        model_fields = actionslog.get_model_fields(old._meta.model)
    elif new is not None:
        fields = set(get_fields_in_model(new))
        model_fields = actionslog.get_model_fields(new._meta.model)
    else:
        fields = set()
        model_fields = None

    # Check if fields must be filtered
    if model_fields and (model_fields['include_fields'] or model_fields['exclude_fields']) and fields:
        filtered_fields = []
        if model_fields['include_fields']:
            filtered_fields = [field for field in fields
                               if field.name in model_fields['include_fields']]
        else:
            filtered_fields = fields
        if model_fields['exclude_fields']:
            filtered_fields = [field for field in filtered_fields
                               if field.name not in model_fields['exclude_fields']]
        fields = filtered_fields

    for field in fields:
        try:
            old_value = smart_text(getattr(old, field.name, None))
        except ObjectDoesNotExist:
            old_value = field.default if field.default is not NOT_PROVIDED else None

        try:
            new_value = smart_text(getattr(new, field.name, None))
        except ObjectDoesNotExist:
            new_value = None

        if old_value != new_value:
            diff[field.name] = (smart_text(old_value), smart_text(new_value))

    if len(diff) == 0:
        diff = None

    return diff


Instruction:
Can you write a function in Python 3 where it
sets the current user from the request and prepares and connects a signal receiver with the user already attached to it.

Response:
def process_request(self, request):
        """
        Gets the current user from the request and prepares and connects a signal receiver with the user already
        attached to it.
        """
        # Initialize thread local storage
        threadlocal.actionslog = {
            'signal_duid': (self.__class__, time.time()),
            'remote_ip': request.META.get('REMOTE_ADDR'),
        }

        # In case of proxy, set 'original' address
        if request.META.get('HTTP_X_FORWARDED_FOR'):
            threadlocal.actionslog['remote_ip'] = request.META.get('HTTP_X_FORWARDED_FOR').split(',')[0]

        # Connect signal for automatic logging
        if hasattr(request, 'user') and hasattr(request.user, 'is_authenticated') and request.user.is_authenticated():
            set_user = curry(self.set_user, request.user)
            pre_save.connect(set_user, sender=LogAction, dispatch_uid=threadlocal.actionslog['signal_duid'], weak=False)


Instruction:
How would you explain what the following Python 3 function does
def process_response(self, request, response):
        if hasattr(threadlocal, 'actionslog'):
            pre_save.disconnect(sender=LogAction, dispatch_uid=threadlocal.actionslog['signal_duid'])

        return response

Response: Disconnects the signal receiver to prevent it from staying active.


Instruction:
How would you implement a function in Python 3 that
disconnects the signal receiver to prevent it from staying active in case of an exception.

Response:
def process_exception(self, request, exception):
        """
        Disconnects the signal receiver to prevent it from staying active in case of an exception.
        """
        if hasattr(threadlocal, 'actionslog'):
            pre_save.disconnect(sender=LogAction, dispatch_uid=threadlocal.actionslog['signal_duid'])

        return None


Instruction:
Write a Python 3 function that can
set the user of the log action instance.

Response:
def set_user(user, sender, instance, **kwargs):
        """
        Signal receiver with an extra, required 'user' kwarg. This method becomes a real (valid) signal receiver when
        it is curried with the user.
        """
        try:
            app_label, model_name = settings.AUTH_USER_MODEL.split('.')
            auth_user_model = apps.get_model(app_label, model_name)
        except ValueError:
            auth_user_model = apps.get_model('auth', 'user')
        if sender == LogAction and isinstance(user, auth_user_model) and instance.user is None:
            instance.user = user
        if hasattr(threadlocal, 'actionslog'):
            instance.remote_ip = threadlocal.actionslog['remote_ip']


Instruction:
Write a Python 3 function for
signaling receiver that creates a log entry when a model instance is first saved to the database.

Response:
def action_log_create(sender, instance, created, **kwargs):
    """
    Signal receiver that creates a log entry when a model instance is first saved to the database.

    Direct use is discouraged, connect your model through :py:func:`actionslog.registry.register` instead.
    """
    if created:
        changes = model_instance_diff(None, instance)

        log_entry = LogAction.objects.create_log_action(
            instance=instance,
            action=LogAction.CREATE,
            changes=json.dumps(changes),
        )


Instruction:
Write a Python 3 function for
signaling receiver that creates a log entry when a model instance is changed and saved to the database.

Response:
def action_log_update(sender, instance, **kwargs):
    """
    Signal receiver that creates a log entry when a model instance is changed and saved to the database.

    Direct use is discouraged, connect your model through :py:func:`actionslog.registry.register` instead.
    """
    if instance.pk is not None:
        try:
            old = sender.objects.get(pk=instance.pk)
        except sender.DoesNotExist:
            pass
        else:
            new = instance

            changes = model_instance_diff(old, new)

            # Log an entry only if there are changes
            if changes:
                log_entry = LogAction.objects.create_log_action(
                    instance=instance,
                    action=LogAction.UPDATE,
                    changes=json.dumps(changes),
                )


Instruction:
Can you tell what is the following Python 3 function doing
def action_log_delete(sender, instance, **kwargs):
    if instance.pk is not None:
        changes = model_instance_diff(instance, None)

        log_entry = LogAction.objects.create_log_action(
            instance=instance,
            action=LogAction.DELETE,
            changes=json.dumps(changes),
        )

Response: Signal receiver that creates a log entry when a model instance is deleted from the database.


Instruction:
Can you generate the documentation for the following Python 3 function
def request(self, method, api_url, params={}, **kwargs):

        LOG.debug("axapi_http: full url = %s", self.url_base + api_url)
        LOG.debug("axapi_http: %s url = %s", method, api_url)
        LOG.debug("axapi_http: params = %s", json.dumps(logutils.clean(params), indent=4))

        # Set "data" variable for the request
        if params:
            extra_params = kwargs.get('axapi_args', {})
            params_copy = merge_dicts(params, extra_params)
            LOG.debug("axapi_http: params_all = %s", logutils.clean(params_copy))

            payload = json.dumps(params_copy)
        else:
            try:
                payload = kwargs.pop('payload', None)
                self.headers = dict(self.HEADERS, **kwargs.pop('headers', {}))
                LOG.debug("axapi_http: headers_all = %s", logutils.clean(self.headers))
            except KeyError:
                payload = None

        max_retries = kwargs.get('max_retries', self.max_retries)
        timeout = kwargs.get('timeout', self.timeout)

        # Create session to set HTTPAdapter or SSLAdapter
        session = Session()
        if self.port == 443:
            # Add adapter for any https session to force TLS1_0 connection for v21 of AXAPI
            session.mount('https://', SSLAdapter(max_retries=max_retries))
        else:
            session.mount('http://', HTTPAdapter(max_retries=max_retries))
        session_request = getattr(session, method.lower())

        # Make actual request and handle any errors
        try:
            device_response = session_request(
                self.url_base + api_url, verify=False, data=payload, headers=self.HEADERS, timeout=timeout
            )
        except (Exception) as e:
            LOG.error("acos_client failing with error %s after %s retries", e.__class__.__name__, max_retries)
            raise e
        finally:
            session.close()

        # Log if the reponse is one of the known broken response
        if device_response in broken_replies:
            device_response = broken_replies[device_response]
            LOG.debug("axapi_http: broken reply, new response: %s", logutils.clean(device_response))

        # Validate json response
        try:
            json_response = device_response.json()
            LOG.debug("axapi_http: data = %s", json.dumps(logutils.clean(json_response), indent=4))
        except ValueError as e:
            # The response is not JSON but it still succeeded.
            LOG.debug("axapi_http: json = %s", e)
            return device_response

        # Handle "fail" responses returned by AXAPI
        if 'response' in json_response and 'status' in json_response['response']:
            if json_response['response']['status'] == 'fail':
                    acos_responses.raise_axapi_ex(json_response, action=extract_method(api_url))

        # Return json portion of response
        return json_response

Response: Generate the API call to the device.


Instruction:
Can you tell what is the following Python 3 function doing
def create(self, host_list=[], serial=None, instance_name=None, use_mgmt_port=False,
               interval=None, bandwidth_base=None, bandwidth_unrestricted=None):
        """Creates a license manager entry

        Keyword arguments:
        instance_name -- license manager instance name
        host_list -- list(dict) a list of dictionaries of the format:
            {'ip': '127.0.0.1', 'port': 443}
        serial - (str) appliance serial number
        use_mgmt_port - (bool) use management for license interactions
        interval - (int) 1=Monthly, 2=Daily, 3=Hourly
        bandwidth_base - (int) Configure feature bandwidth base (Mb)
            Valid range - 10-102400
        bandwidth_unrestricted - (bool) Set the bandwidth to maximum
        """
        payload = self._build_payload(host_list=host_list, serial=serial,
                                      instance_name=instance_name,
                                      use_mgmt_port=use_mgmt_port,
                                      interval=interval, bandwidth_base=bandwidth_base,
                                      bandwidth_unrestricted=bandwidth_unrestricted)
        return self._post(self.url_base, payload)

Response: Creates a new entry in the license manager.


Instruction:
Make a summary of the following Python 3 code
def update(self, host_list=[], serial=None, instance_name=None, use_mgmt_port=False,
               interval=None, bandwidth_base=None, bandwidth_unrestricted=None):
        """Update a license manager entry

        Keyword arguments:
        instance_name -- license manager instance name
        host_list -- list(dict) a list of dictionaries of the format:
            {'ip': '127.0.0.1', 'port': 443}
        serial - (str) appliance serial number
        use_mgmt_port - (bool) use management for license interactions
        interval - (int) 1=Monthly, 2=Daily, 3=Hourly
        bandwidth_base - (int) Configure feature bandwidth base (Mb)
            Valid range - 10-102400
        bandwidth_unrestricted - (bool) Set the bandwidth to maximum
        """

        return self.create(host_list=host_list, serial=serial, instance_name=instance_name,
                           use_mgmt_port=use_mgmt_port,
                           interval=interval, bandwidth_base=bandwidth_base,
                           bandwidth_unrestricted=bandwidth_unrestricted)

Response: Update a license manager entry.


Instruction:
Can you create a Python 3 function that
switches the device - context of a specific object.

Response:
def switch(self, device_id, obj_slot_id):
        """Switching of device-context"""
        payload = {
            "device-context": self._build_payload(device_id, obj_slot_id)
        }

        return self._post(self.url_prefix, payload)


Instruction:
Here you have a function in Python 3, explain what it does
def create(self, destination, mask, next_hops=[]):
        payload = {
            "rib": self._build_payload(destination, mask, next_hops)
        }

        return self._post(self.url_prefix, payload)

Response: Create a new route to the specified destination.


Instruction:
Write a Python 3 function that can
check if a directory contains valid VASP input.

Response:
def contains_vasp_input(dir_name):
    """
    Checks if a directory contains valid VASP input.

    Args:
        dir_name:
            Directory name to check.

    Returns:
        True if directory contains all four VASP input files (INCAR, POSCAR,
        KPOINTS and POTCAR).
    """
    for f in ["INCAR", "POSCAR", "POTCAR", "KPOINTS"]:
        if not os.path.exists(os.path.join(dir_name, f)) and \
                not os.path.exists(os.path.join(dir_name, f + ".orig")):
            return False
    return True


Instruction:
Write a Python 3 script for
returning the URI path for a directory.

Response:
def get_uri(dir_name):
    """
    Returns the URI path for a directory. This allows files hosted on
    different file servers to have distinct locations.

    Args:
        dir_name:
            A directory name.

    Returns:
        Full URI path, e.g., fileserver.host.com:/full/path/of/dir_name.
    """
    fullpath = os.path.abspath(dir_name)
    try:
        hostname = socket.gethostbyaddr(socket.gethostname())[0]
    except:
        hostname = socket.gethostname()
    return "{}:{}".format(hostname, fullpath)


Instruction:
Can you tell what is the following Python 3 function doing
def assimilate(self, path):
        try:
            d = self.get_task_doc(path)
            if self.mapi_key is not None and d["state"] == "successful":
                self.calculate_stability(d)
            tid = self._insert_doc(d)
            return tid
        except Exception as ex:
            import traceback
            logger.error(traceback.format_exc())
            return False

Response: Assimilate a vasp run into the database.


Instruction:
How would you explain what the following Python 3 function does
def get_task_doc(self, path):
        logger.info("Getting task doc for base dir :{}".format(path))
        files = os.listdir(path)
        vasprun_files = OrderedDict()
        if "STOPCAR" in files:
            #Stopped runs. Try to parse as much as possible.
            logger.info(path + " contains stopped run")
        for r in self.runs:
            if r in files: #try subfolder schema
                for f in os.listdir(os.path.join(path, r)):
                    if fnmatch(f, "vasprun.xml*"):
                        vasprun_files[r] = os.path.join(r, f)
            else: #try extension schema
                for f in files:
                    if fnmatch(f, "vasprun.xml.{}*".format(r)):
                        vasprun_files[r] = f
        if len(vasprun_files) == 0:
            for f in files: #get any vasprun from the folder
                if fnmatch(f, "vasprun.xml*") and \
                        f not in vasprun_files.values():
                    vasprun_files['standard'] = f

        if len(vasprun_files) > 0:
            d = self.generate_doc(path, vasprun_files)
            if not d:
                d = self.process_killed_run(path)
            self.post_process(path, d)
        elif (not (path.endswith("relax1") or
              path.endswith("relax2"))) and contains_vasp_input(path):
            #If not Materials Project style, process as a killed run.
            logger.warning(path + " contains killed run")
            d = self.process_killed_run(path)
            self.post_process(path, d)
        else:
            raise ValueError("No VASP files found!")

        return d

Response: Get the entire task doc for a given path.


Instruction:
Create a Python 3 function for
processing a killed vasp run.

Response:
def process_killed_run(self, dir_name):
        """
        Process a killed vasp run.
        """
        fullpath = os.path.abspath(dir_name)
        logger.info("Processing Killed run " + fullpath)
        d = {"dir_name": fullpath, "state": "killed", "oszicar": {}}

        for f in os.listdir(dir_name):
            filename = os.path.join(dir_name, f)
            if fnmatch(f, "INCAR*"):
                try:
                    incar = Incar.from_file(filename)
                    d["incar"] = incar.as_dict()
                    d["is_hubbard"] = incar.get("LDAU", False)
                    if d["is_hubbard"]:
                        us = np.array(incar.get("LDAUU", []))
                        js = np.array(incar.get("LDAUJ", []))
                        if sum(us - js) == 0:
                            d["is_hubbard"] = False
                            d["hubbards"] = {}
                    else:
                        d["hubbards"] = {}
                    if d["is_hubbard"]:
                        d["run_type"] = "GGA+U"
                    elif incar.get("LHFCALC", False):
                        d["run_type"] = "HF"
                    else:
                        d["run_type"] = "GGA"
                except Exception as ex:
                    print(str(ex))
                    logger.error("Unable to parse INCAR for killed run {}."
                                 .format(dir_name))
            elif fnmatch(f, "KPOINTS*"):
                try:
                    kpoints = Kpoints.from_file(filename)
                    d["kpoints"] = kpoints.as_dict()
                except:
                    logger.error("Unable to parse KPOINTS for killed run {}."
                                 .format(dir_name))
            elif fnmatch(f, "POSCAR*"):
                try:
                    s = Poscar.from_file(filename).structure
                    comp = s.composition
                    el_amt = s.composition.get_el_amt_dict()
                    d.update({"unit_cell_formula": comp.as_dict(),
                              "reduced_cell_formula": comp.to_reduced_dict,
                              "elements": list(el_amt.keys()),
                              "nelements": len(el_amt),
                              "pretty_formula": comp.reduced_formula,
                              "anonymous_formula": comp.anonymized_formula,
                              "nsites": comp.num_atoms,
                              "chemsys": "-".join(sorted(el_amt.keys()))})
                    d["poscar"] = s.as_dict()
                except:
                    logger.error("Unable to parse POSCAR for killed run {}."
                                 .format(dir_name))
            elif fnmatch(f, "POTCAR*"):
                try:
                    potcar = Potcar.from_file(filename)
                    d["pseudo_potential"] = {
                        "functional": potcar.functional.lower(),
                        "pot_type": "paw",
                        "labels": potcar.symbols}
                except:
                    logger.error("Unable to parse POTCAR for killed run in {}."
                                 .format(dir_name))
            elif fnmatch(f, "OSZICAR"):
                try:
                    d["oszicar"]["root"] = \
                        Oszicar(os.path.join(dir_name, f)).as_dict()
                except:
                    logger.error("Unable to parse OSZICAR for killed run in {}."
                                 .format(dir_name))
            elif re.match("relax\d", f):
                if os.path.exists(os.path.join(dir_name, f, "OSZICAR")):
                    try:
                        d["oszicar"][f] = Oszicar(
                            os.path.join(dir_name, f, "OSZICAR")).as_dict()
                    except:
                        logger.error("Unable to parse OSZICAR for killed "
                                     "run in {}.".format(dir_name))
        return d


Instruction:
Implement a Python 3 function for
processing a vasprun. xml file.

Response:
def process_vasprun(self, dir_name, taskname, filename):
        """
        Process a vasprun.xml file.
        """
        vasprun_file = os.path.join(dir_name, filename)
        if self.parse_projected_eigen and (self.parse_projected_eigen != 'final' or \
                             taskname == self.runs[-1]):
            parse_projected_eigen = True
        else:
            parse_projected_eigen = False
        r = Vasprun(vasprun_file,parse_projected_eigen=parse_projected_eigen)
        d = r.as_dict()
        d["dir_name"] = os.path.abspath(dir_name)
        d["completed_at"] = \
            str(datetime.datetime.fromtimestamp(os.path.getmtime(
                vasprun_file)))
        d["cif"] = str(CifWriter(r.final_structure))
        d["density"] = r.final_structure.density
        if self.parse_dos and (self.parse_dos != 'final' \
                               or taskname == self.runs[-1]):
            try:
                d["dos"] = r.complete_dos.as_dict()
            except Exception:
                logger.warning("No valid dos data exist in {}.\n Skipping dos"
                               .format(dir_name))
        if taskname == "relax1" or taskname == "relax2":
            d["task"] = {"type": "aflow", "name": taskname}
        else:
            d["task"] = {"type": taskname, "name": taskname}
        d["oxide_type"] = oxide_type(r.final_structure)
        return d


Instruction:
Make a summary of the following Python 3 code
def generate_doc(self, dir_name, vasprun_files):
        try:
            fullpath = os.path.abspath(dir_name)
            # Defensively copy the additional fields first.  This is a MUST.
            # Otherwise, parallel updates will see the same object and inserts
            # will be overridden!!
            d = {k: v for k, v in self.additional_fields.items()}
            d["dir_name"] = fullpath
            d["schema_version"] = VaspToDbTaskDrone.__version__
            d["calculations"] = [
                self.process_vasprun(dir_name, taskname, filename)
                for taskname, filename in vasprun_files.items()]
            d1 = d["calculations"][0]
            d2 = d["calculations"][-1]

            # Now map some useful info to the root level.
            for root_key in ["completed_at", "nsites", "unit_cell_formula",
                             "reduced_cell_formula", "pretty_formula",
                             "elements", "nelements", "cif", "density",
                             "is_hubbard", "hubbards", "run_type"]:
                d[root_key] = d2[root_key]
            d["chemsys"] = "-".join(sorted(d2["elements"]))

            # store any overrides to the exchange correlation functional
            xc = d2["input"]["incar"].get("GGA")
            if xc:
                xc = xc.upper()
            d["input"] = {"crystal": d1["input"]["crystal"],
                          "is_lasph": d2["input"]["incar"].get("LASPH", False),
                          "potcar_spec": d1["input"].get("potcar_spec"),
                          "xc_override": xc}
            vals = sorted(d2["reduced_cell_formula"].values())
            d["anonymous_formula"] = {string.ascii_uppercase[i]: float(vals[i])
                                      for i in range(len(vals))}
            d["output"] = {
                "crystal": d2["output"]["crystal"],
                "final_energy": d2["output"]["final_energy"],
                "final_energy_per_atom": d2["output"]["final_energy_per_atom"]}
            d["name"] = "aflow"
            p = d2["input"]["potcar_type"][0].split("_")
            pot_type = p[0]
            functional = "lda" if len(pot_type) == 1 else "_".join(p[1:])
            d["pseudo_potential"] = {"functional": functional.lower(),
                                     "pot_type": pot_type.lower(),
                                     "labels": d2["input"]["potcar"]}
            if len(d["calculations"]) == len(self.runs) or \
                    list(vasprun_files.keys())[0] != "relax1":
                d["state"] = "successful" if d2["has_vasp_completed"] \
                    else "unsuccessful"
            else:
                d["state"] = "stopped"
            d["analysis"] = get_basic_analysis_and_error_checks(d)

            sg = SpacegroupAnalyzer(Structure.from_dict(d["output"]["crystal"]),
                                    0.1)
            d["spacegroup"] = {"symbol": sg.get_space_group_symbol(),
                               "number": sg.get_space_group_number(),
                               "point_group": sg.get_point_group_symbol(),
                               "source": "spglib",
                               "crystal_system": sg.get_crystal_system(),
                               "hall": sg.get_hall()}
            d["oxide_type"] = d2["oxide_type"]
            d["last_updated"] = datetime.datetime.today()
            return d
        except Exception as ex:
            import traceback
            print(traceback.format_exc())
            logger.error("Error in " + os.path.abspath(dir_name) +
                         ".\n" + traceback.format_exc())

            return None

Response: Generate a doc for a single vasp run.


Instruction:
How would you code a function in Python 3 to
return a list of valid paths for the given path.

Response:
def get_valid_paths(self, path):
        """
        There are some restrictions on the valid directory structures:

        1. There can be only one vasp run in each directory. Nested directories
           are fine.
        2. Directories designated "relax1", "relax2" are considered to be 2
           parts of an aflow style run.
        3. Directories containing vasp output with ".relax1" and ".relax2" are
           also considered as 2 parts of an aflow style run.
        """
        (parent, subdirs, files) = path
        if set(self.runs).intersection(subdirs):
            return [parent]
        if not any([parent.endswith(os.sep + r) for r in self.runs]) and \
                len(glob.glob(os.path.join(parent, "vasprun.xml*"))) > 0:
            return [parent]
        return []


Instruction:
Given the following Python 3 function, write the documentation
def total_size(o, handlers={}, verbose=False, count=False):
    # How to make different types of objects iterable
    dict_handler = lambda d: chain.from_iterable(d.items())
    all_handlers = {tuple: iter,
                    list: iter,
                    deque: iter,
                    dict: dict_handler,
                    set: iter,
                    frozenset: iter}
    all_handlers.update(handlers)     # user handlers take precedence
    seen = set()                      # track which object id's have already been seen
    default_size = getsizeof(0)       # estimate sizeof object without __sizeof__

    def sizeof(o):
        "Calculate size of `o` and all its children"
        if id(o) in seen:             # do not double count the same object
            return 0
        seen.add(id(o))
        if count:
            s = 1
        else:
            s = getsizeof(o, default_size)
        # If `o` is iterable, add size of its members
        for typ, handler in all_handlers.items():
            if isinstance(o, typ):
                s += sum(map(sizeof, handler(o)))
                break
        return s

    return sizeof(o)

Response: Returns the approximate memory footprint an object and all of its children.


Instruction:
Implement a function in Python 3 to
parse argument string as key = value pairs separated by commas.

Response:
def args_kvp_nodup(s):
    """Parse argument string as key=value pairs separated by commas.

    :param s: Argument string
    :return: Parsed value
    :rtype: dict
    :raises: ValueError for format violations or a duplicated key.
    """
    if s is None:
        return {}
    d = {}
    for item in [e.strip() for e in s.split(",")]:
        try:
            key, value = item.split("=", 1)
        except ValueError:
            msg = "argument item '{}' not in form key=value".format(item)
            if _argparse_is_dumb:
                _alog.warn(msg)
            raise ValueError(msg)
        if key in d:
            msg = "Duplicate key for '{}' not allowed".format(key)
            if _argparse_is_dumb:
                _alog.warn(msg)
            raise ValueError(msg)
        d[key] = value
    return d


Instruction:
Can you generate a brief explanation for the following Python 3 code
def walk(self, o):
        if isinstance(o, dict):
            d = o if self._dx is None else self._dx(o)
            return {k: self.walk(v) for k, v in d.items()}
        elif isinstance(o, list):
            return [self.walk(v) for v in o]
        else:
            return o if self._vx is None else self._vx(o)

Response: Walk a dict & transform.


Instruction:
Make a summary of the following Python 3 code
def dict_expand(o):
        r = {}
        for k, v in o.items():
            if isinstance(k, str):
                k = k.replace('$', '_')
            if "." in k:
                sub_r, keys = r, k.split('.')
                # create sub-dicts until last part of key
                for k2 in keys[:-1]:
                    sub_r[k2] = {}
                    sub_r = sub_r[k2]  # descend
                    # assign last part of key to value
                sub_r[keys[-1]] = v
            else:
                r[k] = v
        return r

Response: Expand keys in a dict with. in them into a dict with. in them into a dict with the keys a. b. c and foo.


Instruction:
Implement a Python 3 function for
switching to another collection.

Response:
def collection_name(self, value):
        """Switch to another collection.
        Note that you may have to set the aliases and default properties if the
        schema of the new collection differs from the current collection.
        """
        self._collection_name = value
        self._mongo_coll = self.db[value]
        self.collection = TrackedCollection(self._mongo_coll, operation=self._t_op,
                                            field=self._t_field)


Instruction:
Make a summary of the following Python 3 code
def tracked_find(self, *args, **kwargs):
        _log.info("tracked_find.begin")
        # if tracking is off, just call find (ie do nothing)
        if self._tracking_off:
            _log.info("tracked_find.end, tracking=off")
            return self._coll_find(*args, **kwargs)
        # otherwise do somethin' real
        # fish 'filter' out of args or kwargs
        if len(args) > 0:
            filt = args[0]
        else:
            if 'filter' not in kwargs:
                kwargs['filter'] = {}
            filt = kwargs['filter']
        # update filter with tracker query
        filt.update(self._mark.query)
        # delegate to "real" find()
        _log.info("tracked_find.end, call: {}.find(args={} kwargs={})".format(self._coll.name, args, kwargs))
        return self._coll_find(*args, **kwargs)

Response: Replacement for regular find.


Instruction:
How would you implement a function in Python 3 that
updates the position of the record in the collection.

Response:
def update(self):
        """Update the position of the mark in the collection.

        :return: this object, for chaining
        :rtype: Mark
        """
        rec = self._c.find_one({}, {self._fld: 1}, sort=[(self._fld, -1)], limit=1)
        if rec is None:
            self._pos = self._empty_pos()
        elif not self._fld in rec:
            _log.error("Tracking field not found. field={} collection={}"
                       .format(self._fld, self._c.name))
            _log.warn("Continuing without tracking")
            self._pos = self._empty_pos()
        else:
            self._pos = {self._fld: rec[self._fld]}
        return self


Instruction:
Can you generate a brief explanation for the following Python 3 code
def as_dict(self):
        return {self.FLD_OP: self._op.name,
                self.FLD_MARK: self._pos,
                self.FLD_FLD: self._fld}

Response: Representation as a dict for JSON serialization.


Instruction:
Write a Python 3 function that can
construct a Mark object from a dictionary.

Response:
def from_dict(cls, coll, d):
        """Construct from dict

        :param coll: Collection for the mark
        :param d: Input
        :type d: dict
        :return: new instance
        :rtype: Mark
        """
        return Mark(collection=coll, operation=Operation[d[cls.FLD_OP]],
                    pos=d[cls.FLD_MARK], field=d[cls.FLD_FLD])


Instruction:
Can you generate a brief explanation for the following Python 3 code
def query(self):
        q = {}
        for field, value in self._pos.items():
            if value is None:
                q.update({field: {'$exists': True}})
            else:
                q.update({field: {'$gt': value}})
        return q

Response:
A mongdb query expression to find all records with higher values
        for this mark s fields in the collection.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def create(self):
        if self._track is None:
            self._track = self.db[self.tracking_collection_name]

Response: Create tracking collection. Does nothing if tracking collection already exists.


Instruction:
Given the following Python 3 function, write the documentation
def save(self, mark):
        self._check_exists()
        obj = mark.as_dict()
        try:
            # Make a 'filter' to find/update existing record, which uses
            # the field name and operation (but not the position).
            filt = {k: obj[k] for k in (mark.FLD_FLD, mark.FLD_OP)}
            _log.debug("save: upsert-spec={} upsert-obj={}".format(filt, obj))
            self._track.update(filt, obj, upsert=True)
        except pymongo.errors.PyMongoError as err:
            raise DBError("{}".format(err))

Response: Save a position in this collection.


Instruction:
Make a summary of the following Python 3 code
def retrieve(self, operation, field=None):
        obj = self._get(operation, field)
        if obj is None:
            # empty Mark instance
            return Mark(collection=self.collection, operation=operation, field=field)
        return Mark.from_dict(self.collection, obj)

Response: Retrieve a position in this collection.


Instruction:
Can you implement a function in Python 3 that
gets the tracked position for a given operation and field.

Response:
def _get(self, operation, field):
        """Get tracked position for a given operation and field."""
        self._check_exists()
        query = {Mark.FLD_OP: operation.name,
                 Mark.FLD_MARK + "." + field: {"$exists": True}}
        return self._track.find_one(query)