Instruction:
Implement a function in Python 3 to
parse the response from the last. fm API call.

Response:
async def parseResults(self, api_data):
    """ See CoverSource.parseResults. """
    results = []

    # get xml results list
    xml_text = api_data.decode("utf-8")
    xml_root = xml.etree.ElementTree.fromstring(xml_text)
    status = xml_root.get("status")
    if status != "ok":
      raise Exception("Unexpected Last.fm response status: %s" % (status))
    img_elements = xml_root.findall("album/image")

    # build results from xml
    thumbnail_url = None
    thumbnail_size = None
    for img_element in img_elements:
      img_url = img_element.text
      if not img_url:
        # last.fm returns empty image tag for size it does not have
        continue
      lfm_size = img_element.get("size")
      if lfm_size == "mega":
        check_metadata = CoverImageMetadata.SIZE
      else:
        check_metadata = CoverImageMetadata.NONE
      try:
        size = __class__.SIZES[lfm_size]
      except KeyError:
        continue
      if (size[0] <= MAX_THUMBNAIL_SIZE) and ((thumbnail_size is None) or (size[0] < thumbnail_size)):
        thumbnail_url = img_url
        thumbnail_size = size[0]
      format = os.path.splitext(img_url)[1][1:].lower()
      format = SUPPORTED_IMG_FORMATS[format]
      results.append(LastFmCoverSourceResult(img_url,
                                             size,
                                             format,
                                             thumbnail_url=thumbnail_url,
                                             source=self,
                                             check_metadata=check_metadata))

    return results


Instruction:
How would you implement a function in Python 3 that
searches and download a cover.

Response:
async def search_and_download(album, artist, format, size, out_filepath, *, size_tolerance_prct, amazon_tlds, no_lq_sources,
                              async_loop):
  """ Search and download a cover, return True if success, False instead. """
  # register sources
  source_args = (size, size_tolerance_prct)
  cover_sources = [sources.LastFmCoverSource(*source_args),
                   sources.AmazonCdCoverSource(*source_args),
                   sources.AmazonDigitalCoverSource(*source_args)]
  for tld in amazon_tlds:
    cover_sources.append(sources.AmazonCdCoverSource(*source_args, tld=tld))
  if not no_lq_sources:
    cover_sources.append(sources.GoogleImagesWebScrapeCoverSource(*source_args))

  # schedule search work
  search_futures = []
  for cover_source in cover_sources:
    coroutine = cover_source.search(album, artist)
    future = asyncio.ensure_future(coroutine, loop=async_loop)
    search_futures.append(future)

  # wait for it
  await asyncio.wait(search_futures, loop=async_loop)

  # get results
  results = []
  for future in search_futures:
    source_results = future.result()
    results.extend(source_results)

  # sort results
  results = await CoverSourceResult.preProcessForComparison(results, size, size_tolerance_prct)
  results.sort(reverse=True,
               key=functools.cmp_to_key(functools.partial(CoverSourceResult.compare,
                                                          target_size=size,
                                                          size_tolerance_prct=size_tolerance_prct)))
  if not results:
    logging.getLogger("Main").info("No results")

  # download
  for result in results:
    try:
      await result.get(format, size, size_tolerance_prct, out_filepath)
    except Exception as e:
      logging.getLogger("Main").warning("Download of %s failed: %s %s" % (result,
                                                                          e.__class__.__qualname__,
                                                                          e))
      continue
    else:
      return True

  return False


Instruction:
Create a Python 3 function for
parsing the results from the CoverSource API.

Response:
async def parseResults(self, api_data):
    """ See CoverSource.parseResults. """
    results = []

    # parse page
    parser = lxml.etree.HTMLParser()
    html = lxml.etree.XML(api_data.decode("utf-8"), parser)

    for page_struct_version, result_selector in enumerate(__class__.RESULTS_SELECTORS):
      result_nodes = result_selector(html)
      if result_nodes:
        break

    for rank, result_node in enumerate(result_nodes, 1):
      # get thumbnail & full image url
      img_node = __class__.IMG_SELECTORS[page_struct_version](result_node)[0]
      thumbnail_url = img_node.get("src")
      thumbnail_url = thumbnail_url.replace("Stripe-Prime-Only", "")
      url_parts = thumbnail_url.rsplit(".", 2)
      img_url = ".".join((url_parts[0], url_parts[2]))

      # assume size is fixed
      size = (500, 500)

      # try to get higher res image...
      if self.target_size > size[0]:  # ...but only if needed
        self.logger.debug("Looking for optimal subimages configuration...")
        product_url = __class__.LINK_SELECTOR(result_node)[0].get("href")
        product_url = urllib.parse.urlsplit(product_url)
        product_id = product_url.path.split("/")[3]

        # TODO don't pick up highest res image if user asked less?
        for amazon_img_format in AMAZON_DIGITAL_IMAGE_FORMATS:
          # TODO review this, it seem to always fail now
          self.logger.debug("Trying %u subimages..." % (amazon_img_format.slice_count ** 2))
          urls = tuple(self.generateImgUrls(product_id,
                                            __class__.DYNAPI_KEY,
                                            amazon_img_format.id,
                                            amazon_img_format.slice_count))
          url_ok = await self.probeUrl(urls[-1])
          if not url_ok:
            # images at this size are not available
            continue

          # images at this size are available
          img_url = urls
          size = (amazon_img_format.total_res,) * 2
          break

      # assume format is always jpg
      format = CoverImageFormat.JPEG

      # add result
      results.append(AmazonDigitalCoverSourceResult(img_url,
                                                    size,
                                                    format,
                                                    thumbnail_url=thumbnail_url,
                                                    source=self,
                                                    rank=rank,
                                                    check_metadata=CoverImageMetadata.SIZE))

    return results


Instruction:
How would you explain what the following Python 3 function does
def generateImgUrls(self, product_id, dynapi_key, format_id, slice_count):
    for x in range(slice_count):
      for y in range(slice_count):
        yield ("http://z2-ec2.images-amazon.com/R/1/a=" + product_id +
               "+c=" + dynapi_key +
               "+d=_SCR%28" + str(format_id) + "," + str(x) + "," + str(y) + "%29_=.jpg")

Response: Generate URLs for subimages of a product.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def retrier(*, max_attempts, sleeptime, max_sleeptime, sleepscale=1.5, jitter=0.2):
  assert(max_attempts > 1)
  assert(sleeptime >= 0)
  assert(0 <= jitter <= sleeptime)
  assert(sleepscale >= 1)

  cur_sleeptime = min(max_sleeptime, sleeptime)

  for attempt in range(max_attempts):
    cur_jitter = random.randint(int(-jitter * 1000), int(jitter * 1000)) / 1000
    yield max(0, cur_sleeptime + cur_jitter)
    cur_sleeptime = min(max_sleeptime, cur_sleeptime * sleepscale)

Response: Generator yielding time to wait for a given amount of time.


Instruction:
Can you implement a function in Python 3 that
downloads cover and process it.

Response:
async def get(self, target_format, target_size, size_tolerance_prct, out_filepath):
    """ Download cover and process it. """
    if self.source_quality.value <= CoverSourceQuality.LOW.value:
      logging.getLogger("Cover").warning("Cover is from a potentially unreliable source and may be unrelated to the search")

    images_data = []
    for i, url in enumerate(self.urls):
      # download
      logging.getLogger("Cover").info("Downloading cover '%s' (part %u/%u)..." % (url, i + 1, len(self.urls)))
      headers = {}
      self.source.updateHttpHeaders(headers)

      async def pre_cache_callback(img_data):
        return await __class__.crunch(img_data, self.format)

      store_in_cache_callback, image_data = await self.source.http.query(url,
                                                                         headers=headers,
                                                                         verify=False,
                                                                         cache=__class__.image_cache,
                                                                         pre_cache_callback=pre_cache_callback)

      # store immediately in cache
      await store_in_cache_callback()

      # append for multi images
      images_data.append(image_data)

    need_format_change = (self.format != target_format)
    need_size_change = ((max(self.size) > target_size) and
                        (abs(max(self.size) - target_size) >
                         target_size * size_tolerance_prct / 100))
    need_join = len(images_data) > 1
    if need_join or need_format_change or need_size_change:
      # post process
      image_data = self.postProcess(images_data,
                                    target_format if need_format_change else None,
                                    target_size if need_size_change else None)

      # crunch image again
      image_data = await __class__.crunch(image_data, target_format)

    # write it
    with open(out_filepath, "wb") as file:
      file.write(image_data)


Instruction:
How would you implement a function in Python 3 that
converts binary data to a target format and size and return the processed data.

Response:
def postProcess(self, images_data, new_format, new_size):
    """ Convert image binary data to a target format and/or size (None if no conversion needed), and return the processed data. """
    if len(images_data) == 1:
      in_bytes = io.BytesIO(images_data[0])
      img = PIL.Image.open(in_bytes)
      if img.mode != "RGB":
        img = img.convert("RGB")

    else:
      # images need to be joined before further processing
      logging.getLogger("Cover").info("Joining %u images..." % (len(images_data)))
      # TODO find a way to do this losslessly for JPEG
      new_img = PIL.Image.new("RGB", self.size)
      assert(is_square(len(images_data)))
      sq = int(math.sqrt(len(images_data)))

      images_data_it = iter(images_data)
      img_sizes = {}
      for x in range(sq):
        for y in range(sq):
          current_image_data = next(images_data_it)
          img_stream = io.BytesIO(current_image_data)
          img = PIL.Image.open(img_stream)
          img_sizes[(x, y)] = img.size
          box = [0, 0]
          if x > 0:
            for px in range(x):
              box[0] += img_sizes[(px, y)][0]
          if y > 0:
            for py in range(y):
              box[1] += img_sizes[(x, py)][1]
          box.extend((box[0] + img.size[0], box[1] + img.size[1]))
          new_img.paste(img, box=tuple(box))
      img = new_img

    out_bytes = io.BytesIO()
    if new_size is not None:
      logging.getLogger("Cover").info("Resizing from %ux%u to %ux%u..." % (self.size[0], self.size[1], new_size, new_size))
      img = img.resize((new_size, new_size), PIL.Image.LANCZOS)
      # apply unsharp filter to remove resize blur (equivalent to (images/graphics)magick -unsharp 1.5x1+0.7+0.02)
      # we don't use PIL.ImageFilter.SHARPEN or PIL.ImageEnhance.Sharpness because we want precise control over
      # parameters
      unsharper = PIL.ImageFilter.UnsharpMask(radius=1.5, percent=70, threshold=5)
      img = img.filter(unsharper)
    if new_format is not None:
      logging.getLogger("Cover").info("Converting to %s..." % (new_format.name.upper()))
      target_format = new_format
    else:
      target_format = self.format
    img.save(out_bytes,
             format=target_format.name,
             quality=90,
             optimize=True)
    return out_bytes.getvalue()


Instruction:
Make a summary of the following Python 3 code
async def updateImageMetadata(self):
    assert(self.needMetadataUpdate())

    width_sum, height_sum = 0, 0

    # only download metadata for the needed images to get full size
    idxs = []
    assert(is_square(len(self.urls)))
    sq = int(math.sqrt(len(self.urls)))
    for x in range(sq):
      for y in range(sq):
        if x == y:
          idxs.append((x * sq + y, x, y))

    for idx, x, y in idxs:
      url = self.urls[idx]
      format, width, height = None, None, None

      try:
        format, width, height = pickle.loads(__class__.metadata_cache[url])
      except KeyError:
        # cache miss
        pass
      except Exception as e:
        logging.getLogger("Cover").warning("Unable to load metadata for URL '%s' from cache: %s %s" % (url,
                                                                                e.__class__.__qualname__,
                                                                                e))
      else:
        # cache hit
        logging.getLogger("Cover").debug("Got metadata for URL '%s' from cache" % (url))
        if format is not None:
          self.setFormatMetadata(format)

      if (self.needMetadataUpdate(CoverImageMetadata.FORMAT) or
              (self.needMetadataUpdate(CoverImageMetadata.SIZE) and ((width is None) or (height is None)))):
        # download
        logging.getLogger("Cover").debug("Downloading file header for URL '%s'..." % (url))
        try:
          headers = {}
          self.source.updateHttpHeaders(headers)
          response = await self.source.http.fastStreamedQuery(url,
                                                              headers=headers,
                                                              verify=False)
          try:
            if self.needMetadataUpdate(CoverImageMetadata.FORMAT):
              # try to get format from response
              format = __class__.guessImageFormatFromHttpResponse(response)
              if format is not None:
                self.setFormatMetadata(format)

            if self.needMetadataUpdate():
              # try to get metadata from HTTP data
              metadata = await __class__.guessImageMetadataFromHttpData(response)
              if metadata is not None:
                format, width, height = metadata
                if format is not None:
                  self.setFormatMetadata(format)

          finally:
            response.release()

        except Exception as e:
          logging.getLogger("Cover").warning("Failed to get file metadata for URL '%s' "
                                             "(%s %s)" % (url,
                                                          e.__class__.__qualname__,
                                                          e))

        if self.needMetadataUpdate():  # did we fail to get needed metadata at this point?
          if ((self.format is None) or
                  ((self.size is None) and ((width is None) or (height is None)))):
            # if we get here, file is probably not reachable, or not even an image
            logging.getLogger("Cover").debug("Unable to get file metadata from file or HTTP headers for URL '%s', "
                                             "skipping this result" % (url))
            return

          if ((self.format is not None) and
                  ((self.size is not None) and (width is None) and (height is None))):
            logging.getLogger("Cover").debug("Unable to get file metadata from file or HTTP headers for URL '%s', "
                                             "falling back to API data" % (url))
            self.check_metadata = CoverImageMetadata.NONE
            self.reliable_metadata = False
            return

        # save it to cache
        __class__.metadata_cache[url] = pickle.dumps((format, width, height))

      # sum sizes
      if (width is not None) and (height is not None):
        width_sum += width
        height_sum += height

    if self.needMetadataUpdate(CoverImageMetadata.SIZE) and (width_sum > 0) and (height_sum > 0):
      self.setSizeMetadata((width_sum, height_sum))

Response: Partially download image file to get its real metadata or get it from cache.


Instruction:
Implement a Python 3 function for
setting format image metadata to what has been reliably identified.

Response:
def setFormatMetadata(self, format):
    """ Set format image metadata to what has been reliably identified. """
    assert((self.needMetadataUpdate(CoverImageMetadata.FORMAT)) or
           (self.format is format))
    self.format = format
    self.check_metadata &= ~CoverImageMetadata.FORMAT


Instruction:
Create a Python 3 function to
set size image metadata to what has been reliably identified.

Response:
def setSizeMetadata(self, size):
    """ Set size image metadata to what has been reliably identified. """
    assert((self.needMetadataUpdate(CoverImageMetadata.SIZE)) or
           (self.size == size))
    self.size = size
    self.check_metadata &= ~CoverImageMetadata.SIZE


Instruction:
Here you have a function in Python 3, explain what it does
async def updateSignature(self):
    assert(self.thumbnail_sig is None)

    if self.thumbnail_url is None:
      logging.getLogger("Cover").warning("No thumbnail available for %s" % (self))
      return

    # download
    logging.getLogger("Cover").debug("Downloading cover thumbnail '%s'..." % (self.thumbnail_url))
    headers = {}
    self.source.updateHttpHeaders(headers)

    async def pre_cache_callback(img_data):
      return await __class__.crunch(img_data, CoverImageFormat.JPEG, silent=True)

    try:
      store_in_cache_callback, image_data = await self.source.http.query(self.thumbnail_url,
                                                                         cache=__class__.image_cache,
                                                                         headers=headers,
                                                                         pre_cache_callback=pre_cache_callback)
    except Exception as e:
      logging.getLogger("Cover").warning("Download of '%s' failed: %s %s" % (self.thumbnail_url,
                                                                             e.__class__.__qualname__,
                                                                             e))
      return

    # compute sig
    logging.getLogger("Cover").debug("Computing signature of %s..." % (self))
    try:
      self.thumbnail_sig = __class__.computeImgSignature(image_data)
    except Exception as e:
      logging.getLogger("Cover").warning("Failed to compute signature of '%s': %s %s" % (self,
                                                                                e.__class__.__qualname__,
                                                                                e))
    else:
      await store_in_cache_callback()

Response: Calculate a cover s signature using its thumbnail url.


Instruction:
Given the following Python 3 function, write the documentation
def compare(first, second, *, target_size, size_tolerance_prct):
    for c in (first, second):
      assert(c.format is not None)
      assert(isinstance(c.size[0], int) and isinstance(c.size[1], int))

    # prefer square covers #1
    delta_ratio1 = abs(first.size[0] / first.size[1] - 1)
    delta_ratio2 = abs(second.size[0] / second.size[1] - 1)
    if abs(delta_ratio1 - delta_ratio2) > 0.15:
      return -1 if (delta_ratio1 > delta_ratio2) else 1

    # prefer similar to reference
    sr1 = first.is_similar_to_reference
    sr2 = second.is_similar_to_reference
    if sr1 and (not sr2):
      return 1
    if (not sr1) and sr2:
      return -1

    # prefer size above preferred
    delta_size1 = ((first.size[0] + first.size[1]) / 2) - target_size
    delta_size2 = ((second.size[0] + second.size[1]) / 2) - target_size
    if (((delta_size1 < 0) and (delta_size2 >= 0)) or
            (delta_size1 >= 0) and (delta_size2 < 0)):
      return -1 if (delta_size1 < delta_size2) else 1

    # if both below target size, prefer closest
    if (delta_size1 < 0) and (delta_size2 < 0) and (delta_size1 != delta_size2):
      return -1 if (delta_size1 < delta_size2) else 1

    # prefer covers of most reliable source
    qs1 = first.source_quality.value
    qs2 = second.source_quality.value
    if qs1 != qs2:
      return -1 if (qs1 < qs2) else 1

    # prefer best ranked
    if ((first.rank is not None) and
            (second.rank is not None) and
            (first.__class__ is second.__class__) and
            (first.rank != second.rank)):
      return -1 if (first.rank > second.rank) else 1

    # prefer reliable metadata
    if first.reliable_metadata != second.reliable_metadata:
      return 1 if first.reliable_metadata else -1

    # prefer covers with less images to join
    ic1 = len(first.urls)
    ic2 = len(second.urls)
    if ic1 != ic2:
      return -1 if (ic1 > ic2) else 1

    # prefer the preferred size
    if abs(delta_size1) != abs(delta_size2):
      return -1 if (abs(delta_size1) > abs(delta_size2)) else 1

    # prefer png
    if first.format != second.format:
      return -1 if (second.format is CoverImageFormat.PNG) else 1

    # prefer square covers #2
    if (delta_ratio1 != delta_ratio2):
      return -1 if (delta_ratio1 > delta_ratio2) else 1

    # fuck, they are the same!
    return 0

Response: Compare two cover relevance and quality.


Instruction:
How would you code a function in Python 3 to
crunch image data and return the processed data.

Response:
async def crunch(image_data, format, silent=False):
    """ Crunch image data, and return the processed data, or orignal data if operation failed. """
    if (((format is CoverImageFormat.PNG) and (not HAS_OPTIPNG)) or
            ((format is CoverImageFormat.JPEG) and (not HAS_JPEGOPTIM))):
      return image_data
    with mkstemp_ctx.mkstemp(suffix=".%s" % (format.name.lower())) as tmp_out_filepath:
      if not silent:
        logging.getLogger("Cover").info("Crunching %s image..." % (format.name.upper()))
      with open(tmp_out_filepath, "wb") as tmp_out_file:
        tmp_out_file.write(image_data)
      size_before = len(image_data)
      if format is CoverImageFormat.PNG:
        cmd = ["optipng", "-quiet", "-o1"]
      elif format is CoverImageFormat.JPEG:
        cmd = ["jpegoptim", "-q", "--strip-all"]
      cmd.append(tmp_out_filepath)
      p = await asyncio.create_subprocess_exec(*cmd,
                                               stdin=asyncio.subprocess.DEVNULL,
                                               stdout=asyncio.subprocess.DEVNULL,
                                               stderr=asyncio.subprocess.DEVNULL)
      await p.wait()
      if p.returncode != 0:
        if not silent:
          logging.getLogger("Cover").warning("Crunching image failed")
        return image_data
      with open(tmp_out_filepath, "rb") as tmp_out_file:
        crunched_image_data = tmp_out_file.read()
      size_after = len(crunched_image_data)
      pct_saved = 100 * (size_before - size_after) / size_before
      if not silent:
        logging.getLogger("Cover").debug("Crunching image saved %.2f%% filesize" % (pct_saved))
    return crunched_image_data


Instruction:
Can you generate a brief explanation for the following Python 3 code
def guessImageMetadataFromData(img_data):
    format, width, height = None, None, None
    img_stream = io.BytesIO(img_data)
    try:
      img = PIL.Image.open(img_stream)
    except IOError:
      format = imghdr.what(None, h=img_data)
      format = SUPPORTED_IMG_FORMATS.get(format, None)
    else:
      format = img.format.lower()
      format = SUPPORTED_IMG_FORMATS.get(format, None)
      width, height = img.size
    return format, width, height

Response: Identify an image format and size from its first bytes.


Instruction:
Given the following Python 3 function, write the documentation
async def guessImageMetadataFromHttpData(response):
    metadata = None
    img_data = bytearray()

    while len(img_data) < CoverSourceResult.MAX_FILE_METADATA_PEEK_SIZE:
      new_img_data = await response.content.read(__class__.METADATA_PEEK_SIZE_INCREMENT)
      if not new_img_data:
        break
      img_data.extend(new_img_data)

      metadata = __class__.guessImageMetadataFromData(img_data)
      if (metadata is not None) and all(metadata):
        return metadata

    return metadata

Response: Identify an image format and size from the beginning of its HTTP data.


Instruction:
Implement a function in Python 3 to
guess file format from HTTP response.

Response:
def guessImageFormatFromHttpResponse(response):
    """ Guess file format from HTTP response, return format or None. """
    extensions = []

    # try to guess extension from response content-type header
    try:
      content_type = response.headers["Content-Type"]
    except KeyError:
      pass
    else:
      ext = mimetypes.guess_extension(content_type, strict=False)
      if ext is not None:
        extensions.append(ext)

    # try to extract extension from URL
    urls = list(response.history) + [response.url]
    for url in map(str, urls):
      ext = os.path.splitext(urllib.parse.urlsplit(url).path)[-1]
      if (ext is not None) and (ext not in extensions):
        extensions.append(ext)

    # now guess from the extensions
    for ext in extensions:
      try:
        return SUPPORTED_IMG_FORMATS[ext[1:]]
      except KeyError:
        pass


Instruction:
Can you implement a function in Python 3 that
processes the results to prepare them for comparison and sorting.

Response:
async def preProcessForComparison(results, target_size, size_tolerance_prct):
    """ Process results to prepare them for future comparison and sorting. """
    # find reference (=image most likely to match target cover ignoring factors like size and format)
    reference = None
    for result in results:
      if result.source_quality is CoverSourceQuality.REFERENCE:
        if ((reference is None) or
            (CoverSourceResult.compare(result,
                                       reference,
                                       target_size=target_size,
                                       size_tolerance_prct=size_tolerance_prct) > 0)):
          reference = result

    # remove results that are only refs
    results = list(itertools.filterfalse(operator.attrgetter("is_only_reference"), results))

    # remove duplicates
    no_dup_results = []
    for result in results:
      is_dup = False
      for result_comp in results:
        if ((result_comp is not result) and
            (result_comp.urls == result.urls) and
            (__class__.compare(result,
                               result_comp,
                               target_size=target_size,
                               size_tolerance_prct=size_tolerance_prct) < 0)):
          is_dup = True
          break
      if not is_dup:
        no_dup_results.append(result)
    dup_count = len(results) - len(no_dup_results)
    if dup_count > 0:
      logging.getLogger("Cover").info("Removed %u duplicate results" % (dup_count))
      results = no_dup_results

    if reference is not None:
      logging.getLogger("Cover").info("Reference is: %s" % (reference))
      reference.is_similar_to_reference = True

      # calculate sigs
      futures = []
      for result in results:
        coroutine = result.updateSignature()
        future = asyncio.ensure_future(coroutine)
        futures.append(future)
      if reference.is_only_reference:
        assert(reference not in results)
        coroutine = reference.updateSignature()
        future = asyncio.ensure_future(coroutine)
        futures.append(future)
      if futures:
        await asyncio.wait(futures)
      for future in futures:
        future.result()  # raise pending exception if any

      # compare other results to reference
      for result in results:
        if ((result is not reference) and
                (result.thumbnail_sig is not None) and
                (reference.thumbnail_sig is not None)):
          result.is_similar_to_reference = __class__.areImageSigsSimilar(result.thumbnail_sig,
                                                                         reference.thumbnail_sig)
          if result.is_similar_to_reference:
            logging.getLogger("Cover").debug("%s is similar to reference" % (result))
          else:
            logging.getLogger("Cover").debug("%s is NOT similar to reference" % (result))
    else:
      logging.getLogger("Cover").warning("No reference result found")

    return results


Instruction:
Can you tell what is the following Python 3 function doing
def computeImgSignature(image_data):
    parser = PIL.ImageFile.Parser()
    parser.feed(image_data)
    img = parser.close()
    target_size = (__class__.IMG_SIG_SIZE, __class__.IMG_SIG_SIZE)
    img.thumbnail(target_size, PIL.Image.BICUBIC)
    if img.size != target_size:
      logging.getLogger("Cover").debug("Non square thumbnail after resize to %ux%u, unable to compute signature" % target_size)
      return None
    img = img.convert(mode="RGB")
    pixels = img.getdata()
    pixel_count = target_size[0] * target_size[1]
    color_count = 3
    r = bitarray.bitarray(pixel_count * color_count)
    r.setall(False)
    for ic in range(color_count):
      mean = sum(p[ic] for p in pixels) // pixel_count
      for ip, p in enumerate(pixels):
        if p[ic] > mean:
          r[pixel_count * ic + ip] = True
    return r

Response: Calculate an image signature.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def analyze_lib(lib_dir, cover_filename, *, ignore_existing=False):
  work = {}
  stats = collections.OrderedDict(((k, 0) for k in("files", "albums", "missing covers", "errors")))
  with tqdm.tqdm(desc="Analyzing library",
                 unit="dir",
                 postfix=stats) as progress, \
          tqdm_logging.redirect_logging(progress):
    for rootpath, rel_dirpaths, rel_filepaths in os.walk(lib_dir):
      metadata = analyze_dir(stats,
                             rootpath,
                             rel_filepaths,
                             cover_filename,
                             ignore_existing=ignore_existing)
      progress.set_postfix(stats, refresh=False)
      progress.update(1)
      if all(metadata[:-1]):
        work[rootpath] = metadata[:-1]
  return work

Response: Recursively analyze a library and return a dict of path - > album.


Instruction:
Can you create a Python 3 function that
returns a tuple of album artist has_embedded_album_art from a list of audio files.

Response:
def get_metadata(audio_filepaths):
  """ Return a tuple of album, artist, has_embedded_album_art from a list of audio files. """
  artist, album, has_embedded_album_art = None, None, None
  for audio_filepath in audio_filepaths:
    try:
      mf = mutagen.File(audio_filepath)
    except Exception:
      continue
    if mf is None:
      continue

    # artist
    for key in ("albumartist", "artist",  # ogg
                "TPE1", "TPE2",  # mp3
                "aART", "\xa9ART"):  # mp4
      try:
        val = mf.get(key, None)
      except ValueError:
        val = None
      if val is not None:
        artist = val[-1]
        break

    # album
    for key in ("_album", "album",  # ogg
                "TALB",  # mp3
                "\xa9alb"):  # mp4
      try:
        val = mf.get(key, None)
      except ValueError:
        val = None
      if val is not None:
        album = val[-1]
        break

    if artist and album:
      # album art
      if isinstance(mf, mutagen.ogg.OggFileType):
        has_embedded_album_art = "metadata_block_picture" in mf
      elif isinstance(mf, mutagen.mp3.MP3):
        has_embedded_album_art = any(map(operator.methodcaller("startswith", "APIC:"), mf.keys()))
      elif isinstance(mf, mutagen.mp4.MP4):
        has_embedded_album_art = "covr" in mf

      # stop at the first file that succeeds (for performance)
      break

  return artist, album, has_embedded_album_art


Instruction:
Explain what the following Python 3 code does
def analyze_dir(stats, parent_dir, rel_filepaths, cover_filename, *, ignore_existing=False):
  no_metadata = None, None, None
  metadata = no_metadata
  audio_filepaths = []
  for rel_filepath in rel_filepaths:
    stats["files"] += 1
    try:
      ext = os.path.splitext(rel_filepath)[1][1:].lower()
    except IndexError:
      continue
    if ext in AUDIO_EXTENSIONS:
      audio_filepaths.append(os.path.join(parent_dir, rel_filepath))
  if audio_filepaths:
    stats["albums"] += 1
    if (cover_filename != EMBEDDED_ALBUM_ART_SYMBOL):
      missing = (not os.path.isfile(os.path.join(parent_dir, cover_filename))) or ignore_existing
      if missing:
        metadata = get_metadata(audio_filepaths)
    else:
      metadata = get_metadata(audio_filepaths)
      missing = (not metadata[2]) or ignore_existing
    if missing:
      stats["missing covers"] += 1
      if not all(metadata[:-1]):
        # failed to get metadata for this album
        stats["errors"] += 1
        logging.getLogger("sacad_r").error("Unable to read metadata for album directory '%s'" % (parent_dir))
    else:
      metadata = no_metadata
  return metadata

Response: Analyze a directory to get album metadata.


Instruction:
Create a Python 3 function for
embedding album art into audio files.

Response:
def embed_album_art(cover_filepath, path):
  """ Embed album art into audio files. """
  with open(cover_filepath, "rb") as f:
    cover_data = f.read()

  for filename in os.listdir(path):
    try:
      ext = os.path.splitext(filename)[1][1:].lower()
    except IndexError:
      continue

    if ext in AUDIO_EXTENSIONS:
      filepath = os.path.join(path, filename)
      mf = mutagen.File(filepath)
      if (isinstance(mf.tags, mutagen._vorbis.VComment) or
              isinstance(mf, mutagen.ogg.OggFileType)):
        picture = mutagen.flac.Picture()
        picture.data = cover_data
        picture.type = mutagen.id3.PictureType.COVER_FRONT
        picture.mime = "image/jpeg"
        encoded_data = base64.b64encode(picture.write())
        mf["metadata_block_picture"] = encoded_data.decode("ascii")
      elif (isinstance(mf.tags, mutagen.id3.ID3) or
            isinstance(mf, mutagen.id3.ID3FileType)):
        mf.tags.add(mutagen.id3.APIC(mime="image/jpeg",
                                     type=mutagen.id3.PictureType.COVER_FRONT,
                                     data=cover_data))
      elif (isinstance(mf.tags, mutagen.mp4.MP4Tags) or
            isinstance(mf, mutagen.mp4.MP4)):
        mf["covr"] = [mutagen.mp4.MP4Cover(cover_data,
                                           imageformat=mutagen.mp4.AtomDataType.JPEG)]
      mf.save()


Instruction:
Can you generate a brief explanation for the following Python 3 code
def ichunk(iterable, n):
  it = iter(iterable)
  while True:
    chunk = tuple(itertools.islice(it, n))
    if not chunk:
      return
    yield chunk

Response: Split an iterable into n - sized chunks.


Instruction:
Can you create a Python 3 function that
searches for a given album and artist and return an iterable of CoverSourceResult objects.

Response:
async def search(self, album, artist):
    """ Search for a given album/artist and return an iterable of CoverSourceResult. """
    self.logger.debug("Searching with source '%s'..." % (self.__class__.__name__))
    album = self.processAlbumString(album)
    artist = self.processArtistString(artist)
    url_data = self.getSearchUrl(album, artist)
    if isinstance(url_data, tuple):
      url, post_data = url_data
    else:
      url = url_data
      post_data = None
    try:
      store_in_cache_callback, api_data = await self.fetchResults(url, post_data)
      results = await self.parseResults(api_data)
    except Exception as e:
      # raise
      self.logger.warning("Search with source '%s' failed: %s %s" % (self.__class__.__name__,
                                                                     e.__class__.__qualname__,
                                                                     e))
      return ()
    else:
      if results:
        # only store in cache if parsing succeeds and we have results
        await store_in_cache_callback()

    # get metadata
    futures = []
    for result in filter(operator.methodcaller("needMetadataUpdate"), results):
      coroutine = result.updateImageMetadata()
      future = asyncio.ensure_future(coroutine)
      futures.append(future)
    if futures:
      await asyncio.wait(futures)
    for future in futures:
      future.result()  # raise pending exception if any

    # filter
    results_excluded_count = 0
    reference_only_count = 0
    results_kept = []
    for result in results:
      if ((result.size[0] + (self.size_tolerance_prct * self.target_size / 100) < self.target_size) or  # skip too small images
              (result.size[1] + (self.size_tolerance_prct * self.target_size / 100) < self.target_size) or
              (result.format is None) or  # unknown format
              result.needMetadataUpdate()):  # if still true, it means we failed to grab metadata, so exclude it
        if result.source_quality is CoverSourceQuality.REFERENCE:
          # we keep this result just for the reference, it will be excluded from the results
          result.is_only_reference = True
          results_kept.append(result)
          reference_only_count += 1
        else:
          results_excluded_count += 1
      else:
        results_kept.append(result)
    result_kept_count = len(results_kept) - reference_only_count

    # log
    self.logger.info("Got %u relevant (%u excluded) results from source '%s'" % (result_kept_count,
                                                                                results_excluded_count + reference_only_count,
                                                                                self.__class__.__name__))
    for result in itertools.filterfalse(operator.attrgetter("is_only_reference"), results_kept):
      self.logger.debug("%s %s%s %4dx%4d %s%s" % (result.__class__.__name__,
                                                  ("(%02d) " % (result.rank)) if result.rank is not None else "",
                                                  result.format.name,
                                                  result.size[0],
                                                  result.size[1],
                                                  result.urls[0],
                                                  " [x%u]" % (len(result.urls)) if len(result.urls) > 1 else ""))
    return results_kept


Instruction:
Implement a function in Python 3 to
fetch a tuple from an URL.

Response:
async def fetchResults(self, url, post_data=None):
    """ Get a (store in cache callback, search results) tuple from an URL. """
    if post_data is not None:
      self.logger.debug("Querying URL '%s' %s..." % (url, dict(post_data)))
    else:
      self.logger.debug("Querying URL '%s'..." % (url))
    headers = {}
    self.updateHttpHeaders(headers)
    return await self.http.query(url,
                                 post_data=post_data,
                                 headers=headers,
                                 cache=__class__.api_cache)


Instruction:
Make a summary of the following Python 3 code
async def probeUrl(self, url, response_headers=None):
    self.logger.debug("Probing URL '%s'..." % (url))
    headers = {}
    self.updateHttpHeaders(headers)
    resp_headers = {}
    resp_ok = await self.http.isReachable(url,
                                          headers=headers,
                                          response_headers=resp_headers,
                                          cache=__class__.probe_cache)

    if response_headers is not None:
      response_headers.update(resp_headers)

    return resp_ok

Response: Probe a URL and return True if it is reachable.


Instruction:
Create a Python 3 function for
replacing accentuated chars in string by their non accentuated equivalent.

Response:
def unaccentuate(s):
    """ Replace accentuated chars in string by their non accentuated equivalent. """
    return "".join(c for c in unicodedata.normalize("NFKD", s) if not unicodedata.combining(c))


Instruction:
Implement a function in Python 3 to
remove punctuation from string s.

Response:
def unpunctuate(s, *, char_blacklist=string.punctuation):
    """ Remove punctuation from string s. """
    # remove punctuation
    s = "".join(c for c in s if c not in char_blacklist)
    # remove consecutive spaces
    return " ".join(filter(None, s.split(" ")))


Instruction:
Can you generate the documentation for the following Python 3 function
async def parseResults(self, api_data):
    results = []

    # parse page
    parser = lxml.etree.HTMLParser()
    html = lxml.etree.XML(api_data.decode("utf-8", "ignore"), parser)

    for page_struct_version, result_selector in enumerate(__class__.RESULTS_SELECTORS):
      result_nodes = result_selector(html)
      if result_nodes:
        break

    for rank, result_node in enumerate(result_nodes, 1):
      try:
        img_node = __class__.IMG_SELECTORS[page_struct_version](result_node)[0]
      except IndexError:
        # no image for that product
        continue
      # get thumbnail & full image url
      thumbnail_url = img_node.get("src")
      url_parts = thumbnail_url.rsplit(".", 2)
      img_url = ".".join((url_parts[0], url_parts[2]))
      # assume size is fixed
      size = (500, 500)
      check_metadata = CoverImageMetadata.SIZE
      # try to get higher res image...
      if ((self.target_size > size[0]) and  # ...only if needed
              (rank <= 3)):  # and only for first 3 results because this is time
                             # consuming (1 more GET request per result)
        product_url = __class__.PRODUCT_LINK_SELECTORS[page_struct_version](result_node)[0].get("href")
        product_url_split = urllib.parse.urlsplit(product_url)
        if not product_url_split.scheme:
          # relative redirect url
          product_url_query = urllib.parse.parse_qsl(product_url_split.query)
          product_url_query = collections.OrderedDict(product_url_query)
          try:
            # needed if page_struct_version == 1
            product_url = product_url_query["url"]
          except KeyError:
            # page_struct_version == 0, make url absolute
            product_url = urllib.parse.urljoin(self.base_url, product_url)
          product_url_split = urllib.parse.urlsplit(product_url)
        product_url_query = urllib.parse.parse_qsl(product_url_split.query)
        product_url_query = collections.OrderedDict(product_url_query)
        try:
          # remove timestamp from url to improve future cache hit rate
          del product_url_query["qid"]
        except KeyError:
          pass
        product_url_query = urllib.parse.urlencode(product_url_query)
        product_url_no_ts = urllib.parse.urlunsplit(product_url_split[:3] + (product_url_query,) + product_url_split[4:])
        store_in_cache_callback, product_page_data = await self.fetchResults(product_url_no_ts)
        product_page_html = lxml.etree.XML(product_page_data.decode("latin-1"), parser)
        try:
          img_node = __class__.PRODUCT_PAGE_IMG_SELECTOR(product_page_html)[0]
        except IndexError:
          # unable to get better image
          pass
        else:
          better_img_url = img_node.get("data-old-hires")
          # img_node.get("data-a-dynamic-image") contains json with image urls too, but they are not larger than
          # previous 500px image and are often covered by autorip badges (can be removed by cleaning url though)
          if better_img_url:
            img_url = better_img_url
            size_url_hint = img_url.rsplit(".", 2)[1].strip("_")
            assert(size_url_hint.startswith("SL"))
            size_url_hint = int(size_url_hint[2:])
            size = (size_url_hint, size_url_hint)
            check_metadata = CoverImageMetadata.NONE
          await store_in_cache_callback()

      # assume format is always jpg
      format = CoverImageFormat.JPEG
      # add result
      results.append(AmazonCdCoverSourceResult(img_url,
                                               size,
                                               format,
                                               thumbnail_url=thumbnail_url,
                                               source=self,
                                               rank=rank,
                                               check_metadata=check_metadata))

    return results

Response: Parse the results from the Cover API.


Instruction:
Write a Python 3 function that can
call delete on all members of obj that are recognized as instances of pg objects.

Response:
def delete_all(obj):
    '''Calls `delete()` on all members of `obj` that are recognized as
    instances of `pg` objects.'''
    types = tuple([
        Shader,
        Mesh,
        VertexBuffer,
        IndexBuffer,
        Texture,
        Program,
        Context,
    ])
    for name in dir(obj):
        child = getattr(obj, name)
        if isinstance(child, types):
            child.delete()


Instruction:
Can you generate the documentation for the following Python 3 function
def _glfw_get_version(filename):
    '''
    Queries and returns the library version tuple or None by using a
    subprocess.
    '''
    version_checker_source = """
        import sys
        import ctypes

        def get_version(library_handle):
            '''
            Queries and returns the library version tuple or None.
            '''
            major_value = ctypes.c_int(0)
            major = ctypes.pointer(major_value)
            minor_value = ctypes.c_int(0)
            minor = ctypes.pointer(minor_value)
            rev_value = ctypes.c_int(0)
            rev = ctypes.pointer(rev_value)
            if hasattr(library_handle, 'glfwGetVersion'):
                library_handle.glfwGetVersion(major, minor, rev)
                version = (major_value.value,
                           minor_value.value,
                           rev_value.value)
                return version
            else:
                return None

        try:
            input_func = raw_input
        except NameError:
            input_func = input
        filename = input_func().strip()

        try:
            library_handle = ctypes.CDLL(filename)
        except OSError:
            pass
        else:
            version = get_version(library_handle)
            print(version)
    """

    args = [sys.executable, '-c', textwrap.dedent(version_checker_source)]
    process = subprocess.Popen(args, universal_newlines=True,
                               stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    out = process.communicate(_to_char_p(filename))[0]
    out = out.strip()
    if out:
        return eval(out)
    else:
        return None

Response: Function to get the version of the library.


Instruction:
Can you create a Python 3 function that
sets the error callback.

Response:
def set_error_callback(cbfun):
    '''
    Sets the error callback.

    Wrapper for:
        GLFWerrorfun glfwSetErrorCallback(GLFWerrorfun cbfun);
    '''
    global _error_callback
    previous_callback = _error_callback
    if cbfun is None:
        cbfun = 0
    c_cbfun = _GLFWerrorfun(cbfun)
    _error_callback = (cbfun, c_cbfun)
    cbfun = c_cbfun
    _glfw.glfwSetErrorCallback(cbfun)
    if previous_callback is not None and previous_callback[0] != 0:
        return previous_callback[0]


Instruction:
Make a summary of the following Python 3 code
def destroy_window(window):
    '''
    Destroys the specified window and its context.

    Wrapper for:
        void glfwDestroyWindow(GLFWwindow* window);
    '''
    _glfw.glfwDestroyWindow(window)
    window_addr = ctypes.cast(ctypes.pointer(window),
                              ctypes.POINTER(ctypes.c_ulong)).contents.value
    for callback_repository in _callback_repositories:
        if window_addr in callback_repository:
            del callback_repository[window_addr]

Response: Destroys the specified window and its context.


Instruction:
Can you tell what is the following Python 3 function doing
def unwrap(self):
        '''
        Returns a nested python sequence.
        '''
        size = self.width, self.height
        bits = self.red_bits, self.green_bits, self.blue_bits
        return size, bits, self.refresh_rate

Response: Returns a nested python sequence.


Instruction:
How would you code a function in Python 3 to
wrap a nested python sequence.

Response:
def wrap(self, gammaramp):
        '''
        Wraps a nested python sequence.
        '''
        red, green, blue = gammaramp
        size = min(len(red), len(green), len(blue))
        array_type = ctypes.c_ushort*size
        self.size = ctypes.c_uint(size)
        self.red_array = array_type()
        self.green_array = array_type()
        self.blue_array = array_type()
        for i in range(self.size):
            self.red_array[i] = int(red[i]*65535)
            self.green_array[i] = int(green[i]*65535)
            self.blue_array[i] = int(blue[i]*65535)
        pointer_type = ctypes.POINTER(ctypes.c_ushort)
        self.red = ctypes.cast(self.red_array, pointer_type)
        self.green = ctypes.cast(self.green_array, pointer_type)
        self.blue = ctypes.cast(self.blue_array, pointer_type)


Instruction:
Here you have a function in Python 3, explain what it does
def unwrap(self):
        '''
        Returns a nested python sequence.
        '''
        red = [self.red[i]/65535.0 for i in range(self.size)]
        green = [self.green[i]/65535.0 for i in range(self.size)]
        blue = [self.blue[i]/65535.0 for i in range(self.size)]
        return red, green, blue

Response: Returns a nested python sequence.


Instruction:
Can you implement a function in Python 3 that
takes a hexadecimal color value in the format 0xrrggbb and returns an r g b tuple where 0 <= r g b <= 1. 0 and 0 <= g b <= 1. 0

Response:
def hex_color(value):
    '''Accepts a hexadecimal color `value` in the format ``0xrrggbb`` and
    returns an (r, g, b) tuple where 0.0 <= r, g, b <= 1.0.
    '''
    r = ((value >> (8 * 2)) & 255) / 255.0
    g = ((value >> (8 * 1)) & 255) / 255.0
    b = ((value >> (8 * 0)) & 255) / 255.0
    return (r, g, b)


Instruction:
Can you implement a function in Python 3 that
normalizes the vector so that its length is 1.

Response:
def normalize(vector):
    '''Normalizes the `vector` so that its length is 1. `vector` can have
    any number of components.
    '''
    d = sum(x * x for x in vector) ** 0.5
    return tuple(x / d for x in vector)


Instruction:
Can you create a Python 3 function that
computes and returns the distance between two points p1 and p2.

Response:
def distance(p1, p2):
    '''Computes and returns the distance between two points, `p1` and `p2`.
    The points can have any number of components.
    '''
    return sum((a - b) ** 2 for a, b in zip(p1, p2)) ** 0.5


Instruction:
Implement a function in Python 3 to
compute the cross product of two vectors.

Response:
def cross(v1, v2):
    '''Computes the cross product of two vectors.
    '''
    return (
        v1[1] * v2[2] - v1[2] * v2[1],
        v1[2] * v2[0] - v1[0] * v2[2],
        v1[0] * v2[1] - v1[1] * v2[0],
    )


Instruction:
Can you create a Python 3 function that
computes the dot product of two vectors.

Response:
def dot(v1, v2):
    '''Computes the dot product of two vectors.
    '''
    x1, y1, z1 = v1
    x2, y2, z2 = v2
    return x1 * x2 + y1 * y2 + z1 * z2


Instruction:
Can you generate the documentation for the following Python 3 function
def interpolate(v1, v2, t):
    '''Interpolate from one vector to another.
    '''
    return add(v1, mul(sub(v2, v1), t))

Response: Interpolate from one vector to another.


Instruction:
Write a Python 3 function that can
compute a normal vector given three points.

Response:
def normal_from_points(a, b, c):
    '''Computes a normal vector given three points.
    '''
    x1, y1, z1 = a
    x2, y2, z2 = b
    x3, y3, z3 = c
    ab = (x2 - x1, y2 - y1, z2 - z1)
    ac = (x3 - x1, y3 - y1, z3 - z1)
    x, y, z = cross(ab, ac)
    d = (x * x + y * y + z * z) ** 0.5
    return (x / d, y / d, z / d)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def smooth_normals(positions, normals):
    '''Assigns an averaged normal to each position based on all of the normals
    originally used for the position.
    '''
    lookup = defaultdict(list)
    for position, normal in zip(positions, normals):
        lookup[position].append(normal)
    result = []
    for position in positions:
        tx = ty = tz = 0
        for x, y, z in lookup[position]:
            tx += x
            ty += y
            tz += z
        d = (tx * tx + ty * ty + tz * tz) ** 0.5
        result.append((tx / d, ty / d, tz / d))
    return result

Response:
Assigns an averaged normal to each position based on all of the normals
    originally used for the position.


Instruction:
Can you implement a function in Python 3 that
computes the bounding box for a list of 3 - dimensional points.

Response:
def bounding_box(positions):
    '''Computes the bounding box for a list of 3-dimensional points.
    '''
    (x0, y0, z0) = (x1, y1, z1) = positions[0]
    for x, y, z in positions:
        x0 = min(x0, x)
        y0 = min(y0, y)
        z0 = min(z0, z)
        x1 = max(x1, x)
        y1 = max(y1, y)
        z1 = max(z1, z)
    return (x0, y0, z0), (x1, y1, z1)


Instruction:
Here you have a function in Python 3, explain what it does
def recenter(positions):
    '''Returns a list of new positions centered around the origin.
    '''
    (x0, y0, z0), (x1, y1, z1) = bounding_box(positions)
    dx = x1 - (x1 - x0) / 2.0
    dy = y1 - (y1 - y0) / 2.0
    dz = z1 - (z1 - z0) / 2.0
    result = []
    for x, y, z in positions:
        result.append((x - dx, y - dy, z - dz))
    return result

Response: Returns a list of new positions centered around the origin.


Instruction:
Can you create a Python 3 function that
interleaves the elements of the provided arrays.

Response:
def interleave(*args):
    '''Interleaves the elements of the provided arrays.

        >>> a = [(0, 0), (1, 0), (2, 0), (3, 0)]
        >>> b = [(0, 0), (0, 1), (0, 2), (0, 3)]
        >>> interleave(a, b)
        [(0, 0, 0, 0), (1, 0, 0, 1), (2, 0, 0, 2), (3, 0, 0, 3)]

    This is useful for combining multiple vertex attributes into a single
    vertex buffer. The shader attributes can be assigned a slice of the
    vertex buffer.
    '''
    result = []
    for array in zip(*args):
        result.append(tuple(flatten(array)))
    return result


Instruction:
Make a summary of the following Python 3 code
def distinct(iterable, keyfunc=None):
    '''Yields distinct items from `iterable` in the order that they appear.
    '''
    seen = set()
    for item in iterable:
        key = item if keyfunc is None else keyfunc(item)
        if key not in seen:
            seen.add(key)
            yield item

Response: Yields distinct items from iterable in the order that they appear.


Instruction:
Can you generate the documentation for the following Python 3 function
def ray_triangle_intersection(v1, v2, v3, o, d):
    '''Computes the distance from a point to a triangle given a ray.
    '''
    eps = 1e-6
    e1 = sub(v2, v1)
    e2 = sub(v3, v1)
    p = cross(d, e2)
    det = dot(e1, p)
    if abs(det) < eps:
        return None
    inv = 1.0 / det
    t = sub(o, v1)
    u = dot(t, p) * inv
    if u < 0 or u > 1:
        return None
    q = cross(t, e1)
    v = dot(d, q) * inv
    if v < 0 or v > 1:
        return None
    t = dot(e2, q) * inv
    if t > eps:
        return t
    return None

Response: Computes the distance from a point to a triangle given a ray.


Instruction:
Make a summary of the following Python 3 code
def pack_list(fmt, data):
    '''Convert a Python list into a ctypes buffer.

    This appears to be faster than the typical method of creating a ctypes
    array, e.g. (c_float * len(data))(*data)
    '''
    func = struct.Struct(fmt).pack
    return create_string_buffer(''.join([func(x) for x in data]))

Response: Convert a Python list into a ctypes buffer.


Instruction:
Given the following Python 3 function, write the documentation
def click(self, jquery=False):
        if jquery:
            e = JQuery(self)
            e.click()
        else:
            super(Clickable, self).click()

Response: Click by WebElement or JQuery


Instruction:
Write a Python 3 script to
convert an instance of Cookie class from cookielib to a dict.

Response:
def convert_cookie_to_dict(cookie, keys_map=WEB_DRIVER_COOKIE_KEYS_MAP):
    """
    Converts an instance of Cookie class from cookielib to a dict.
    The names of attributes can be changed according to keys_map:.
    For example, this method can be used to create a cookie which compatible with WebDriver format.

    :param cookie: Cookie instance received from requests/sessions using url2lib or requests libraries.
    :param keys_map: The dict to map cookie attributes for different schemas. By default WebDriver format is used.
    :return:
    """

    cookie_dict = dict()

    for k in keys_map.keys():
        key = _to_unicode_if_str(keys_map[k])
        value = _to_unicode_if_str(getattr(cookie, k))
        cookie_dict[key] = value

    return cookie_dict


Instruction:
Here you have a function in Python 3, explain what it does
def add_cookies_to_web_driver(driver, cookies):
    for cookie in cookies:
        driver.add_cookie(convert_cookie_to_dict(cookie))
    return driver

Response: Adds cookies to a WebDriver session.


Instruction:
Here you have a function in Python 3, explain what it does
def configure(self, options, conf):
        self.conf = conf
        self.when = options.browser_closer_when

Response: Configure plugin. Plugin is enabled by default.


Instruction:
Create a Python 3 function to
index a path. :param root: Either a package directory, a .so or a .py module.

Response:
def index_path(self, root):
        """Index a path.

        :param root: Either a package directory, a .so or a .py module.
        """
        basename = os.path.basename(root)
        if os.path.splitext(basename)[0] != '__init__' and basename.startswith('_'):
            return
        location = self._determine_location_for(root)
        if os.path.isfile(root):
            self._index_module(root, location)
        elif os.path.isdir(root) and os.path.exists(os.path.join(root, '__init__.py')):
            self._index_package(root, location)


Instruction:
Write a Python 3 script to
get or create an index for the current object.

Response:
def get_or_create_index(self, paths=None, name=None, refresh=False):
        """
        Get index with given name from cache. Create if it doesn't exists.
        """
        if not paths:
            paths = sys.path
        if not name:
            name = 'default'
        self._name = name

        idx_dir = get_cache_dir()
        idx_file = os.path.join(idx_dir, name + '.json')

        if os.path.exists(idx_file) and not refresh:
            with open(idx_file) as fd:
                self.deserialize(fd)
        else:
            self.build_index(paths)
            with open(idx_file, 'w') as fd:
                self.serialize(fd)

        return self


Instruction:
Can you create a Python 3 function that
finds matches for a given symbol.

Response:
def symbol_scores(self, symbol):
        """Find matches for symbol.

        :param symbol: A . separated symbol. eg. 'os.path.basename'
        :returns: A list of tuples of (score, package, reference|None),
            ordered by score from highest to lowest.
        """
        scores = []
        path = []

        # sys.path              sys path          ->    import sys
        # os.path.basename      os.path basename  ->    import os.path
        # basename              os.path basename   ->   from os.path import basename
        # path.basename         os.path basename   ->   from os import path
        def fixup(module, variable):
            prefix = module.split('.')
            if variable is not None:
                prefix.append(variable)
            seeking = symbol.split('.')
            new_module = []
            while prefix and seeking[0] != prefix[0]:
                new_module.append(prefix.pop(0))
            if new_module:
                module, variable = '.'.join(new_module), prefix[0]
            else:
                variable = None
            return module, variable

        def score_walk(scope, scale):
            sub_path, score = self._score_key(scope, full_key)
            if score > 0.1:
                try:
                    i = sub_path.index(None)
                    sub_path, from_symbol = sub_path[:i], '.'.join(sub_path[i + 1:])
                except ValueError:
                    from_symbol = None
                package_path = '.'.join(path + sub_path)
                package_path, from_symbol = fixup(package_path, from_symbol)
                scores.append((score * scale, package_path, from_symbol))

            for key, subscope in scope._tree.items():
                if type(subscope) is not float:
                    path.append(key)
                    score_walk(subscope, subscope.score * scale - 0.1)
                    path.pop()

        full_key = symbol.split('.')
        score_walk(self, 1.0)
        scores.sort(reverse=True)
        return scores


Instruction:
How would you code a function in Python 3 to
return the node for a path or None.

Response:
def find(self, path):
        """Return the node for a path, or None."""
        path = path.split('.')
        node = self
        while node._parent:
            node = node._parent
        for name in path:
            node = node._tree.get(name, None)
            if node is None or type(node) is float:
                return None
        return node


Instruction:
Can you implement a function in Python 3 that
returns the location code for a path.

Response:
def location_for(self, path):
        """Return the location code for a path."""
        path = path.split('.')
        node = self
        while node._parent:
            node = node._parent
        location = node.location
        for name in path:
            tree = node._tree.get(name, None)
            if tree is None or type(tree) is float:
                return location
            location = tree.location
        return location


Instruction:
Can you generate a brief explanation for the following Python 3 code
def select_option(self, option):
        items_list = self.get_options()
        for item in items_list:
            if item.get_attribute("value") == option:
                item.click()
                break

Response: Selects the specified item from Web List


Instruction:
Can you generate a brief explanation for the following Python 3 code
def get_attribute_selected(self, attribute):
        items_list = self.get_options()
        return next(iter([item.get_attribute(attribute) for item in items_list if item.is_selected()]), None)

Response: Returns the attribute of selected item from Web List


Instruction:
Write a Python 3 function that can
search for an item in the Web List by the visible text.

Response:
def select_by_visible_text(self, text):
        """
        Performs search of selected item from Web List

        @params text - string visible text
        """
        xpath = './/option[normalize-space(.) = {0}]'.format(self._escape_string(text))
        opts = self.find_elements_by_xpath(xpath)
        matched = False
        for opt in opts:
            self._set_selected(opt)
            if not self.is_multiple:
                return
            matched = True
        # in case the target option isn't found by xpath
        # attempt to find it by direct comparison among options which contain at least the longest token from the text
        if len(opts) == 0 and ' ' in text:
            sub_string_without_space = self._get_longest_token(text)
            if sub_string_without_space == "":
                candidates = self.get_options()
            else:
                xpath = ".//option[contains(.,{0})]".format(self._escape_string(sub_string_without_space))
                candidates = self.find_elements_by_xpath(xpath)
            for candidate in candidates:
                if text == candidate.text:
                    self._set_selected(candidate)
                    if not self.is_multiple:
                        return
                    matched = True
        if not matched:
            raise NoSuchElementException("Could not locate element with visible text: " + str(text))


Instruction:
Write a Python 3 function for
wrapping the wait method of waiting library with default parameter values.

Response:
def wait(*args, **kwargs):
    """
    Wrapping 'wait()' method of 'waiting' library with default parameter values.
    WebDriverException is ignored in the expected exceptions by default.
    """
    kwargs.setdefault('sleep_seconds', (1, None))
    kwargs.setdefault('expected_exceptions', WebDriverException)
    kwargs.setdefault('timeout_seconds', webium.settings.wait_timeout)

    return wait_lib(*args, **kwargs)


Instruction:
How would you explain what the following Python 3 function does
def parse_ast(source, filename=None):
    if isinstance(source, text_type) and sys.version_info[0] == 2:
        # ast.parse() on Python 2 doesn't like encoding declarations
        # in Unicode strings
        source = CODING_COOKIE_RE.sub(r'\1', source, 1)
    return ast.parse(source, filename or '<unknown>')

Response: Parse source into a Python AST taking care of encoding.


Instruction:
How would you implement a function in Python 3 that
finds any unresolved symbols and unreferenced symbols from this scope.

Response:
def find_unresolved_and_unreferenced_symbols(self):
        """Find any unresolved symbols, and unreferenced symbols from this scope.

        :returns: ({unresolved}, {unreferenced})
        """
        unresolved = set()
        unreferenced = self._definitions.copy()
        self._collect_unresolved_and_unreferenced(set(), set(), unresolved, unreferenced,
                                                  frozenset(self._definitions), start=True)
        return unresolved, unreferenced - Scope.ALL_BUILTINS


Instruction:
How would you code a function in Python 3 to
return content in JSON format

Response:
def get_item(key):
    """Return content in cached file in JSON format"""
    CACHED_KEY_FILE = os.path.join(CURRENT_DIR, key)

    try:
        return json.loads(open(CACHED_KEY_FILE, "rb").read().decode('UTF-8'))["_"]
    except (IOError, ValueError):
        return None


Instruction:
Can you generate a brief explanation for the following Python 3 code
def set_item(key,value):
    CACHED_KEY_FILE = os.path.join(CURRENT_DIR, key)

    open(CACHED_KEY_FILE, "wb").write(json.dumps({"_": value}).encode('UTF-8'))

    return value

Response: Write JSON content from value argument to cached file and return


Instruction:
Implement a Python 3 function for
deleting the item from the cache if present

Response:
def delete_item(key):
    """Delete cached file if present"""
    CACHED_KEY_FILE = os.path.join(CURRENT_DIR, key)

    if os.path.isfile(CACHED_KEY_FILE):
        os.remove(CACHED_KEY_FILE)


Instruction:
Can you generate the documentation for the following Python 3 function
def __parse_json_data(self, data):
        if isinstance(data, dict) or isinstance(data, list):
            self._raw_data = data
            self._json_data = copy.deepcopy(self._raw_data)
        else:
            raise TypeError("Provided Data is not json")

Response:
Process Json data

        :@param data
        :@type data: json/dict

        :throws TypeError


Instruction:
How would you code a function in Python 3 to
parse the Json file and store the result in self. _json_data.

Response:
def __parse_json_file(self, file_path):
        """Process Json file data

        :@param file_path
        :@type file_path: string

        :@throws IOError
        """
        if file_path == '' or os.path.splitext(file_path)[1] != '.json':
            raise IOError('Invalid Json file')

        with open(file_path) as json_file:
            self._raw_data = json.load(json_file)

        self._json_data = copy.deepcopy(self._raw_data)


Instruction:
Can you write a function in Python 3 where it
finds value from json data

Response:
def __get_value_from_data(self, key, data):
        """Find value from json data

        :@pram key
        :@type: string

        :@pram data
        :@type data: dict

        :@return object
        :@throws KeyError
        """
        if key.isdigit():
            return data[int(key)]

        if key not in data:
            raise KeyError("Key not exists")

        return data.get(key)


Instruction:
Can you generate the documentation for the following Python 3 function
def at(self, root):
        leafs = root.strip(" ").split('.')
        for leaf in leafs:
            if leaf:
                self._json_data = self.__get_value_from_data(leaf, self._json_data)
        return self

Response: Set the object to be the value of the root where PyJsonq start


Instruction:
Can you implement a function in Python 3 that
resets the object s data according to given data or previously given raw Json data.

Response:
def reset(self, data={}):
        """JsonQuery object cen be reset to new data

        according to given data or previously given raw Json data

        :@param data: {}
        :@type data: json/dict

        :@return self
        """
        if data and (isinstance(data, dict) or isinstance(data, list)):
            self._json_data = data
        else:
            self._json_data = copy.deepcopy(self._raw_data)

        self.__reset_queries()
        return self


Instruction:
Can you tell what is the following Python 3 function doing
def __store_query(self, query_items):
        temp_index = self._current_query_index
        if len(self._queries) - 1 < temp_index:
            self._queries.append([])

        self._queries[temp_index].append(query_items)

Response: Store the query items in the where clause.


Instruction:
Given the following Python 3 function, write the documentation
def __execute_queries(self):

        def func(item):
            or_check = False
            for queries in self._queries:
                and_check = True
                for query in queries:
                    and_check &= self._matcher._match(
                        item.get(query.get('key'), None),
                        query.get('operator'),
                        query.get('value')
                    )
                or_check |= and_check
            return or_check

        self._json_data = list(filter(lambda item: func(item), self._json_data))

Response: Execute all condition and filter result data


Instruction:
Here you have a function in Python 3, explain what it does
def where(self, key, operator, value):
        self.__store_query({"key": key, "operator": operator, "value": value})
        return self

Response:
Make where clause

        :@param key
        :@param operator
        :@param value
        :@type key,operator,value: string

        :@return self


Instruction:
Can you generate the documentation for the following Python 3 function
def or_where(self, key, operator, value):
        if len(self._queries) > 0:
            self._current_query_index += 1
        self.__store_query({"key": key, "operator": operator, "value": value})
        return self

Response: Make an or where clause


Instruction:
Write a Python 3 function for
returning the nth element of the collection

Response:
def nth(self, index):
        """Getting the nth element of the collection

        :@param index
        :@type index: int

        :@return object
        """
        self.__prepare()
        return None if self.count() < math.fabs(index) else self._json_data[index]


Instruction:
Write a Python 3 script for
getting the sum of the values of the given property

Response:
def sum(self, property):
        """Getting the sum according to the given property

        :@param property
        :@type property: string

        :@return int/float
        """
        self.__prepare()
        total = 0
        for i in self._json_data:
            total += i.get(property)

        return total


Instruction:
Explain what the following Python 3 code does
def max(self, property):
        self.__prepare()
        try:
            return max(self._json_data, key=lambda x: x[property]).get(property)
        except KeyError:
            raise KeyError("Key is not exists")

Response: Gets the maximum value from the prepared data


Instruction:
How would you code a function in Python 3 to
get the average value according to given property

Response:
def avg(self, property):
        """Getting average according to given property

        :@param property
        :@type property: string

        :@return average: int/float
        """
        self.__prepare()
        return self.sum(property) / self.count()


Instruction:
Implement a function in Python 3 to
group the resulted collection to multiple chunk size

Response:
def chunk(self, size=0):
        """Group the resulted collection to multiple chunk

        :@param size: 0
        :@type size: integer

        :@return Chunked List
        """

        if size == 0:
            raise ValueError('Invalid chunk size')

        self.__prepare()
        _new_content = []

        while(len(self._json_data) > 0):
            _new_content.append(self._json_data[0:size])
            self._json_data = self._json_data[size:]

        self._json_data = _new_content

        return self._json_data


Instruction:
Can you tell what is the following Python 3 function doing
def group_by(self, property):
        self.__prepare()
        group_data = {}
        for data in self._json_data:
            if data[property] not in group_data:
                group_data[data[property]] = []
            group_data[data[property]].append(data)
        self._json_data = group_data

        return self

Response: Get the grouped result by the given property


Instruction:
Implement a function in Python 3 to
return the sorted result of the given list

Response:
def sort(self, order="asc"):
        """Getting the sorted result of the given list

        :@param order: "asc"
        :@type order: string

        :@return self
        """
        self.__prepare()
        if isinstance(self._json_data, list):
            if order == "asc":
                self._json_data = sorted(self._json_data)
            else:
                self._json_data = sorted(self._json_data, reverse=True)

        return self


Instruction:
How would you code a function in Python 3 to
get the sorted result by the given property

Response:
def sort_by(self, property, order="asc"):
        """Getting the sorted result by the given property

        :@param property, order: "asc"
        :@type property, order: string

        :@return self
        """
        self.__prepare()
        if isinstance(self._json_data, list):
            if order == "asc":
                self._json_data = sorted(
                    self._json_data,
                    key=lambda x: x.get(property)
                )
            else:
                self._json_data = sorted(
                    self._json_data,
                    key=lambda x: x.get(property),
                    reverse=True
                )

        return self


Instruction:
How would you code a function in Python 3 to
compare the given x and y based on op.

Response:
def _match(self, x, op, y):
        """Compare the given `x` and `y` based on `op`

        :@param x, y, op
        :@type x, y: mixed
        :@type op: string

        :@return bool
        :@throws ValueError
        """
        if (op not in self.condition_mapper):
            raise ValueError('Invalid where condition given')

        func = getattr(self, self.condition_mapper.get(op))
        return func(x, y)


Instruction:
How would you explain what the following Python 3 function does
def _get_base_class_names(frame):
    co, lasti = frame.f_code, frame.f_lasti
    code = co.co_code

    extends = []
    for (op, oparg) in op_stream(code, lasti):
        if op in dis.hasconst:
            if type(co.co_consts[oparg]) == str:
                extends = []
        elif op in dis.hasname:
            if dis.opname[op] == 'LOAD_NAME':
                extends.append(('name', co.co_names[oparg]))
            if dis.opname[op] == 'LOAD_ATTR':
                extends.append(('attr', co.co_names[oparg]))
            if dis.opname[op] == 'LOAD_GLOBAL':
                extends.append(('name', co.co_names[oparg]))

    items = []
    previous_item = []
    for t, s in extends:
        if t == 'name':
            if previous_item:
                items.append(previous_item)
            previous_item = [s]
        else:
            previous_item += [s]
    if previous_item:
        items.append(previous_item)
    return items

Response: Get baseclass names from the code object.


Instruction:
Write a Python 3 script to
get the charset of the message

Response:
def get_charset(message, default="utf-8"):
    """Get the message charset"""
    if message.get_content_charset():
        return message.get_content_charset()
    if message.get_charset():
        return message.get_charset()
    return default


Instruction:
Can you tell what is the following Python 3 function doing
def load_tlds():
    file = os.path.join(os.path.dirname(__file__),
                        'assets',
                        'tlds-alpha-by-domain.txt')
    with open(file) as fobj:
        return [elem for elem in fobj.read().lower().splitlines()[1:]
                if "--" not in elem]

Response: Load all legal TLD extensions from assets


Instruction:
How would you explain what the following Python 3 function does
def parse_text_urls(mesg):

    rval = []

    loc = 0

    for match in URLRE.finditer(mesg):
        if loc < match.start():
            rval.append(Chunk(mesg[loc:match.start()], None))
        # Turn email addresses into mailto: links
        email = match.group("email")
        if email and "mailto" not in email:
            mailto = "mailto:{}".format(email)
        else:
            mailto = match.group(1)
        rval.append(Chunk(None, mailto))
        loc = match.end()

    if loc < len(mesg):
        rval.append(Chunk(mesg[loc:], None))

    return rval

Response: Parse a block of text into its url and non - url components.


Instruction:
Write a Python 3 function that can
extract URL and context from a given list of strings.

Response:
def extract_with_context(lst, pred, before_context, after_context):
    """Extract URL and context from a given chunk.

    """
    rval = []

    start = 0
    length = 0
    while start < len(lst):
        usedfirst = False
        usedlast = False
        # Extend to the next match.
        while start + length < len(lst) and length < before_context + 1 \
                and not pred(lst[start + length]):
            length += 1

        # Slide to the next match.
        while start + length < len(lst) and not pred(lst[start + length]):
            start += 1

        # If there was no next match, abort here (it's easier
        # to do this than to try to detect this case later).
        if start + length == len(lst):
            break

        # Now extend repeatedly until we can't find anything.
        while start + length < len(lst) and pred(lst[start + length]):
            extendlength = 1
            # Read in the 'after' context and see if it holds a URL.
            while extendlength < after_context + 1 and start + length + \
                    extendlength < len(lst) and \
                    not pred(lst[start + length + extendlength]):
                extendlength += 1
            length += extendlength
            if start + length < len(lst) and not pred(lst[start + length]):
                # Didn't find a matching line, so we either
                # hit the end or extended to after_context + 1..
                #
                # Now read in possible 'before' context
                # from the next URL; if we don't find one,
                # we discard the readahead.
                extendlength = 1
                while extendlength < before_context and start + length + \
                        extendlength < len(lst) and \
                        not pred(lst[start + length + extendlength]):
                    extendlength += 1
                if start + length + extendlength < len(lst) and \
                        pred(lst[start + length + extendlength]):
                    length += extendlength

        if length > 0 and start + length <= len(lst):
            if start == 0:
                usedfirst = True
            if start + length == len(lst):
                usedlast = True
            rval.append((lst[start:start + length], usedfirst, usedlast))

        start += length
        length = 0
    return rval


Instruction:
How would you implement a function in Python 3 that
gives a text message extract all the URLs found in the message along with their surrounding context.

Response:
def extracturls(mesg):
    """Given a text message, extract all the URLs found in the message, along
    with their surrounding context.  The output is a list of sequences of Chunk
    objects, corresponding to the contextual regions extracted from the string.

    """
    lines = NLRE.split(mesg)

    # The number of lines of context above to provide.
    # above_context = 1
    # The number of lines of context below to provide.
    # below_context = 1

    # Plan here is to first transform lines into the form
    # [line_fragments] where each fragment is a chunk as
    # seen by parse_text_urls.  Note that this means that
    # lines with more than one entry or one entry that's
    # a URL are the only lines containing URLs.

    linechunks = [parse_text_urls(l) for l in lines]

    return extract_with_context(linechunks,
                                lambda chunk: len(chunk) > 1 or
                                (len(chunk) == 1 and chunk[0].url is not None),
                                1, 1)


Instruction:
Can you generate a brief explanation for the following Python 3 code
def extracthtmlurls(mesg):
    chunk = HTMLChunker()
    chunk.feed(mesg)
    chunk.close()
    # above_context = 1
    # below_context = 1

    def somechunkisurl(chunks):
        for chnk in chunks:
            if chnk.url is not None:
                return True
        return False

    return extract_with_context(chunk.rval, somechunkisurl, 1, 1)

Response: Extract URLs with context from html type message. Similar to extracturls.


Instruction:
Can you write a function in Python 3 where it
gives a byte string or bytes input return a string.

Response:
def decode_bytes(byt, enc='utf-8'):
    """Given a string or bytes input, return a string.

        Args: bytes - bytes or string
              enc - encoding to use for decoding the byte string.

    """
    try:
        strg = byt.decode(enc)
    except UnicodeDecodeError as err:
        strg = "Unable to decode message:\n{}\n{}".format(str(byt), err)
    except (AttributeError, UnicodeEncodeError):
        # If byt is already a string, just return it
        return byt
    return strg


Instruction:
Can you tell what is the following Python 3 function doing
def decode_msg(msg, enc='utf-8'):
    # We avoid the get_payload decoding machinery for raw
    # content-transfer-encodings potentially containing non-ascii characters,
    # such as 8bit or binary, as these are encoded using raw-unicode-escape which
    # seems to prevent subsequent utf-8 decoding.
    cte = str(msg.get('content-transfer-encoding', '')).lower()
    decode = cte not in ("8bit", "7bit", "binary")
    res = msg.get_payload(decode=decode)
    return decode_bytes(res, enc)

Response: Decodes a message fragment.


Instruction:
Make a summary of the following Python 3 code
def msgurls(msg, urlidx=1):
    # Written as a generator so I can easily choose only
    # one subpart in the future (e.g., for
    # multipart/alternative).  Actually, I might even add
    # a browser for the message structure?
    enc = get_charset(msg)
    if msg.is_multipart():
        for part in msg.get_payload():
            for chunk in msgurls(part, urlidx):
                urlidx += 1
                yield chunk
    elif msg.get_content_type() == "text/plain":
        decoded = decode_msg(msg, enc)
        for chunk in extracturls(decoded):
            urlidx += 1
            yield chunk
    elif msg.get_content_type() == "text/html":
        decoded = decode_msg(msg, enc)
        for chunk in extracthtmlurls(decoded):
            urlidx += 1
            yield chunk

Response: Yields all the urls in a message.


Instruction:
Can you implement a function in Python 3 that
shortens long URLs to fit on one line.

Response:
def shorten_url(url, cols, shorten):
    """Shorten long URLs to fit on one line.

    """
    cols = ((cols - 6) * .85)  # 6 cols for urlref and don't use while line
    if shorten is False or len(url) < cols:
        return url
    split = int(cols * .5)
    return url[:split] + "..." + url[-split:]


Instruction:
Here you have a function in Python 3, explain what it does
def grp_list(items):
    grp = []
    res = []
    for item in items:
        if isinstance(item, urwid.Divider):
            res.append(grp)
            grp = [items[0]]
        else:
            grp.append(item)
    res.append(grp)
    return res[1:]

Response: Organize list of items like a 2 3 4 2 1 etc...


Instruction:
Given the following Python 3 function, write the documentation
def splittext(text, search, attr):
    if search:
        pat = re.compile("({})".format(re.escape(search)), re.IGNORECASE)
    else:
        return text
    final = pat.split(text)
    final = [(attr, i) if i.lower() == search.lower() else i for i in final]
    return final

Response: Split a text string by search string and add Urwid display attribute to


Instruction:
Can you tell what is the following Python 3 function doing
def handle_keys(self, keys, raw):
        for j, k in enumerate(keys):
            if self.search is True:
                text = "Search: {}".format(self.search_string)
                if k == 'enter':
                    # Catch 'enter' key to prevent opening URL in mkbrowseto
                    self.enter = True
                    if not self.items:
                        self.search = False
                        self.enter = False
                    if self.search_string:
                        footer = 'search'
                    else:
                        footer = 'default'
                        text = ""
                    footerwid = urwid.AttrMap(urwid.Text(text), footer)
                    self.top.footer = footerwid
                elif k in self.activate_keys:
                    self.search_string += k
                    self._search()
                elif k == 'backspace':
                    self.search_string = self.search_string[:-1]
                    self._search()
            elif k in self.activate_keys and \
                    self.urls and \
                    self.search is False and \
                    self.help_menu is False:
                self._open_url()
            elif self.help_menu is True:
                self._help_menu()
                return []
            if k == 'up':
                # Works around bug where the up arrow goes higher than the top list
                # item and unintentionally triggers context and palette switches.
                # Remaps 'up' to 'k'
                keys[j] = 'k'
            if k == 'home':
                # Remap 'home' to 'g'. Works around small bug where 'home' takes the cursor
                # above the top list item.
                keys[j] = 'g'
        # filter backspace out before the widget, it has a weird interaction
        return [i for i in keys if i != 'backspace']

Response: Handles the keys in the list and returns a list of the keys that are handled


Instruction:
Can you write a function in Python 3 where it
handles other keyboard actions not handled by the ListBox widget.

Response:
def unhandled(self, key):
        """Handle other keyboard actions not handled by the ListBox widget.

        """
        self.key = key
        self.size = self.tui.get_cols_rows()
        if self.search is True:
            if self.enter is False and self.no_matches is False:
                if len(key) == 1 and key.isprintable():
                    self.search_string += key
                self._search()
            elif self.enter is True and not self.search_string:
                self.search = False
                self.enter = False
            return
        if not self.urls and key not in "Qq":
            return  # No other actions are useful with no URLs
        if self.help_menu is False:
            try:
                self.keys[key]()
            except KeyError:
                pass


Instruction:
Given the following Python 3 function, write the documentation
def _search_key(self):
        if self.urls:
            self.search = True
            if self.compact is True:
                self.compact = False
                self.items, self.items_com = self.items_com, self.items
        else:
            return
        self.no_matches = False
        self.search_string = ""
        # Reset the search highlighting
        self._search()
        footerwid = urwid.AttrMap(urwid.Text("Search: "), 'footer')
        self.top.footer = footerwid
        self.items = self.items_orig
        self.top.body = urwid.ListBox(self.items)

Response: Set the search key to True


Instruction:
Write a Python 3 function that can
update the number of the items in the page.

Response:
def _digits(self):
        """ 0-9 """
        self.number += self.key
        try:
            if self.compact is False:
                self.top.body.focus_position = \
                    self.items.index(self.items_com[max(int(self.number) - 1, 0)])
            else:
                self.top.body.focus_position = \
                    self.items.index(self.items[max(int(self.number) - 1, 0)])
        except IndexError:
            self.number = self.number[:-1]
        self.top.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
        if self.number:
            self._footer_start_thread("Selection: {}".format(self.number), 1)


Instruction:
Can you generate the documentation for the following Python 3 function
def _top(self):
        # Goto top of the list
        self.top.body.focus_position = 2 if self.compact is False else 0
        self.top.keypress(self.size, "")

Response: g Move to the top of the list


Instruction:
How would you explain what the following Python 3 function does
def _shorten(self):
        # Toggle shortened URL for selected item
        fpo = self.top.body.focus_position
        url_idx = len([i for i in self.items[:fpo + 1]
                       if isinstance(i, urwid.Columns)]) - 1
        if self.compact is False and fpo <= 1:
            return
        url = self.urls[url_idx]
        short = not "..." in self.items[fpo][1].label
        self.items[fpo][1].set_label(shorten_url(url, self.size[0], short))

Response: s Shortens the URL for the currently selected item.


Instruction:
How would you explain what the following Python 3 function does
def _all_escape(self):
        # Toggle all escaped URLs
        self.unesc = not self.unesc
        self.urls, self.urls_unesc = self.urls_unesc, self.urls
        urls = iter(self.urls)
        for item in self.items:
            # Each Column has (Text, Button). Update the Button label
            if isinstance(item, urwid.Columns):
                item[1].set_label(shorten_url(next(urls),
                                              self.size[0],
                                              self.shorten))

Response: u Update all escaped URLs and URLs in the tables.


Instruction:
Explain what the following Python 3 code does
def _context(self):
        # Show/hide context
        if self.search_string:
            # Reset search when toggling compact mode
            footerwid = urwid.AttrMap(urwid.Text(""), 'default')
            self.top.footer = footerwid
            self.search_string = ""
            self.items = self.items_orig
        fpo = self.top.body.focus_position
        self.items, self.items_com = self.items_com, self.items
        self.top.body = urwid.ListBox(self.items)
        self.top.body.focus_position = self._cur_focus(fpo)
        self.compact = not self.compact

Response: c Shows context in the top widget


Instruction:
Can you tell what is the following Python 3 function doing
def _palette(self):
        # Loop through available palettes
        self.palette_idx += 1
        try:
            self.loop.screen.register_palette(self.palettes[self.palette_names[self.palette_idx]])
        except IndexError:
            self.loop.screen.register_palette(self.palettes[self.palette_names[0]])
            self.palette_idx = 0
        self.loop.screen.clear()

Response: p Registers the current palette in the loop.


Instruction:
Can you create a Python 3 function that
creates config. json file if it doesn t exist.

Response:
def _config_create(self):
        """ --genconf """
        # Create ~/.config/urlscan/config.json if if doesn't exist
        if not exists(self.conf):
            try:
                # Python 2/3 compatible recursive directory creation
                os.makedirs(dirname(expanduser(self.conf)))
            except OSError as err:
                if errno.EEXIST != err.errno:
                    raise
            keys = dict(zip(self.keys.keys(),
                            [i.__name__.strip('_') for i in self.keys.values()]))
            with open(expanduser(self.conf), 'w') as pals:
                pals.writelines(json.dumps({"palettes": self.palettes, "keys": keys},
                                           indent=4))
            print("Created ~/.config/urlscan/config.json")
        else:
            print("~/.config/urlscan/config.json already exists")


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _footer_start_thread(self, text, time):
        footerwid = urwid.AttrMap(urwid.Text(text), 'footer')
        self.top.footer = footerwid
        load_thread = Thread(target=self._loading_thread, args=(time,))
        load_thread.daemon = True
        load_thread.start()

Response: Display given text in the footer. Clears after time seconds


Instruction:
Can you create a Python 3 function that
searches - Search URLs and text.

Response:
def _search(self):
        """ Search - search URLs and text.

        """
        text = "Search: {}".format(self.search_string)
        footerwid = urwid.AttrMap(urwid.Text(text), 'footer')
        self.top.footer = footerwid
        search_items = []
        for grp in self.items_org:
            done = False
            for idx, item in enumerate(grp):
                if isinstance(item, urwid.Columns):
                    for col_idx, col in enumerate(item.contents):
                        if isinstance(col[0], urwid.decoration.AttrMap):
                            grp[idx][col_idx].set_label(splittext(col[0].base_widget.label,
                                                                  self.search_string,
                                                                  ''))
                            if self.search_string.lower() in col[0].base_widget.label.lower():
                                grp[idx][col_idx].set_label(splittext(col[0].base_widget.label,
                                                                      self.search_string,
                                                                      'search'))
                                done = True
                elif isinstance(item, urwid.Text):
                    grp[idx].set_text(splittext(item.text, self.search_string, ''))
                    if self.search_string.lower() in item.text.lower():
                        grp[idx].set_text(splittext(item.text, self.search_string, 'search'))
                        done = True
            if done is True:
                search_items.extend(grp)
        self.items = search_items
        self.top.body = urwid.ListBox(self.items)
        if self.items:
            self.top.body.focus_position = 2 if self.compact is False else 0
            # Trick urwid into redisplaying the cursor
            self.top.keypress(self.tui.get_cols_rows(), "")
            self.no_matches = False
        else:
            self.no_matches = True
            footerwid = urwid.AttrMap(urwid.Text(text + "  No Matches"), 'footer')
            self.top.footer = footerwid


Instruction:
Write a Python 3 function that can
create the urwid callback function to open the web browser or call another function with the URL.

Response:
def mkbrowseto(self, url):
        """Create the urwid callback function to open the web browser or call
        another function with the URL.

        """
        # Try-except block to work around webbrowser module bug
        # https://bugs.python.org/issue31014
        try:
            browser = os.environ['BROWSER']
        except KeyError:
            pass
        else:
            del os.environ['BROWSER']
            webbrowser.register(browser, None, webbrowser.GenericBrowser(browser))
            try_idx = webbrowser._tryorder.index(browser)
            webbrowser._tryorder.insert(0, webbrowser._tryorder.pop(try_idx))

        def browse(*args):
            # double ()() to ensure self.search evaluated at runtime, not when
            # browse() is _created_. [0] is self.search, [1] is self.enter
            # self.enter prevents opening URL when in search mode
            if self._get_search()[0]() is True:
                if self._get_search()[1]() is True:
                    self.search = False
                    self.enter = False
            elif not self.run:
                webbrowser.open(url)
            elif self.run and self.pipe:
                process = Popen(shlex.split(self.run), stdout=PIPE, stdin=PIPE)
                process.communicate(input=url.encode(sys.getdefaultencoding()))
            else:
                Popen(self.run.format(url), shell=True).communicate()

            size = self.tui.get_cols_rows()
            self.draw_screen(size)
        return browse