Instruction:
Here you have a function in Python 3, explain what it does
def _work(self):
        if self._refresh_cache:
            # Inconsistent cache might cause exceptions. For example,
            # if a port has been removed, it will be known in the next
            # loop. Using the old switch port can cause exceptions.
            LOG.debug("Refreshing os_win caches...")
            self._utils.update_cache()
            self._refresh_cache = False

        if self._bound_ports or self._unbound_ports:
            eventlet.spawn_n(self._notify_plugin_on_port_updates)

        # notify plugin about port deltas
        if self._added_ports:
            LOG.debug("Agent loop has new devices!")
            self._treat_devices_added()

        if self._removed_ports:
            LOG.debug("Agent loop has lost devices...")
            self._treat_devices_removed()

Response: Process the information regarding the available ports.


Instruction:
Here you have a function in Python 3, explain what it does
def build(subparsers):

    parser = subparsers.add_parser(
        'build',
        help='Build derived packages',
        description=build.__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog='')

    parser.set_defaults(run_command=run_metapack)

    parser.add_argument('metatabfile', nargs='?',
                        help="Path or URL to a metatab file. If not provided, defaults to 'metadata.csv'. "
                          )

    parser.add_argument('-p', '--profile', help="Name of a BOTO or AWS credentails profile", required=False)

    parser.add_argument('-D', '--package-directory', help="Write Zip, Excel and CSV packages to an alternate directory",
                            required=False)

    parser.add_argument('-F', '--force', action='store_true', default=False,
                             help='Force some operations, like updating the name and building packages')

    parser.add_argument('-R', '--reuse-resources', action='store_true', default=False,
                        help='When building Filesystem package, try to reuse resources built in prior build')

    group = parser.add_mutually_exclusive_group()

    group.add_argument('-n', '--nonversion-name', action='store_true', default=False,
                        help='Write file packages with non-versioned names')

    group.add_argument('-N', '--nonversion-link', action='store_true', default=False,
                        help='Create links with nonversioned names to file packages')

    parser.set_defaults(handler=None)

    ##
    ## Derived Package Group

    derived_group = parser.add_argument_group('Derived Packages', 'Generate other types of packages')

    derived_group.add_argument('-e', '--excel', action='store_true', default=False,
                               help='Create an excel archive from a metatab file')

    derived_group.add_argument('-z', '--zip', action='store_true', default=False,
                               help='Create a zip archive from a metatab file')

    derived_group.add_argument('-f', '--filesystem', action='store_true', default=False,
                               help='Create a filesystem archive from a metatab file')

    derived_group.add_argument('-c', '--csv', action='store_true', default=False,
                               help='Create a CSV archive from a metatab file')

    ##
    ## Administration Group

    admin_group = parser.add_argument_group('Administration', 'Information and administration')

    admin_group.add_argument('--clean-cache', default=False, action='store_true',
                             help="Clean the download cache")

    admin_group.add_argument('-C', '--clean', default=False, action='store_true',
                             help="For some operations, like updating schemas, clear the section of existing terms first")

Response:
Build source packages.

    The mp build program runs all of the resources listed in a Metatab file and
    produces one or more Metapack packages with those resources localized. It
    will always try to produce a Filesystem package, and may optionally produce
    Excel, Zip and CSV packages.

    Typical usage is to be run inside a source package directory with

    .. code-block:: bash

        $ mp build

    To build all of the package types:

    .. code-block:: bash

        $ mp build -fezc

    By default, packages are built with versioned names. The
    :option:`--nonversion-name` option will create file packages with
    non-versioned name, and the :option:`--nonversioned-link` option will
    produce a non-versioned soft link pointing to the versioned file.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def metatab_derived_handler(m):
    from metapack.exc import PackageError
    from metapack.util import get_materialized_data_cache
    from shutil import rmtree

    create_list = []
    url = None

    doc = MetapackDoc(m.mt_file)

    env = get_lib_module_dict(doc)

    package_dir = m.package_root

    if m.args.package_directory:
        # If this is set, the FS package will be built to m.package_root, but the
        # file packages will be built to package_dir
        package_dir = parse_app_url(m.args.package_directory)

    update_name(m.mt_file, fail_on_missing=False, report_unchanged=False)

    process_schemas(m.mt_file, cache=m.cache, clean=m.args.clean, report_found=False)

    nv_name = m.args.nonversion_name
    nv_link = m.args.nonversion_link

    # Remove any data that may have been cached , for instance, from Jupyter notebooks
    rmtree(get_materialized_data_cache(doc), ignore_errors=True)

    reuse_resources=m.args.reuse_resources

    try:

        # Always create a filesystem package before ZIP or Excel, so we can use it as a source for
        # data for the other packages. This means that Transform processes and programs only need
        # to be run once.

        _, url, created = make_filesystem_package(m.mt_file, m.package_root, m.cache, env, m.args.force, False,
                                                  nv_link, reuse_resources=reuse_resources)
        create_list.append(('fs', url, created))

        lb_path = Path( m.package_root.fspath,'last_build')

        if created or not lb_path.exists():
            Path( m.package_root.fspath,'last_build').touch()

        m.mt_file = url

        env = {}  # Don't need it anymore, since no more programs will be run.

        if m.args.excel is not False:
            _, url, created = make_excel_package(m.mt_file, package_dir, m.cache, env, m.args.force, nv_name, nv_link)
            create_list.append(('xlsx', url, created))

        if m.args.zip is not False:
            _, url, created = make_zip_package(m.mt_file, package_dir, m.cache, env, m.args.force, nv_name, nv_link)
            create_list.append(('zip', url, created))

        if m.args.csv is not False:
            _, url, created = make_csv_package(m.mt_file, package_dir, m.cache, env, m.args.force, nv_name, nv_link)
            create_list.append(('csv', url, created))

    except PackageError as e:
        err("Failed to generate package: {}".format(e))

    index_packages(m)

    return create_list

Response: Create local Zip Excel and Filesystem packages and process the data.


Instruction:
Implement a function in Python 3 to
initialize features that are normally initialized in the CLI

Response:
def init():
    """Initialize features that are normally initialized in the CLI"""

    from metapack.appurl import SearchUrl
    import metapack as mp
    from os import environ

    SearchUrl.initialize()  # This makes the 'index:" urls work
    mp.Downloader.context.update(environ)


Instruction:
Given the following Python 3 function, write the documentation
def read_struct_file(struct_data):
    # Holders for all structs
    structs = dict()

    # Holders for the current struct
    name = None

    # Iterate over every line in the file
    for i, l in enumerate(struct_data.splitlines()):
        # Empty the line of comments, if the line is empty then skip to the
        # next line.  Split on whitespace to get the tokens.
        tokens = re_comment.sub(b"", l).strip().split()
        if len(tokens) == 0:
            continue
        elif len(tokens) == 3:
            # 3 tokens implies header data
            (key, _, value) = tokens

            if key == b"name":
                if name is not None:
                    if structs[name].size is None:
                        raise ValueError(
                            "size value missing for struct '{}'".format(name))
                    if structs[name].base is None:
                        raise ValueError(
                            "base value missing for struct '{}'".format(name))
                name = value
                structs[name] = Struct(name)
            elif key == b"size":
                structs[name].size = num(value)
            elif key == b"base":
                structs[name].base = num(value)
            else:
                raise ValueError(key)
        elif len(tokens) == 5:
            # 5 tokens implies entry in struct.
            (field, pack, offset, printf, default) = tokens

            # Convert the packing character from Perl to Python standard
            num_pack = re_numbered_pack.match(pack)
            if num_pack is not None:
                pack = (num_pack.group("num") +
                        perl_to_python_packs[num_pack.group("char")])
            else:
                pack = perl_to_python_packs[pack]

            # If the field is an array then extract the length
            length = 1
            field_exp = re_array_field.match(field)
            if field_exp is not None:
                field = field_exp.group("field")
                length = num(field_exp.group("length"))

            structs[name][field] = StructField(pack, num(offset), printf,
                                               num(default), length)
        else:
            raise ValueError(
                "line {}: Invalid syntax in struct file".format(i))

    # Final check for setting size and base
    if structs[name].size is None:
        raise ValueError(
            "size value missing for struct '{}'".format(name))
    if structs[name].base is None:
        raise ValueError(
            "base value missing for struct '{}'".format(name))

    return structs

Response: Interpret a struct file defining the location of variables in memory.


Instruction:
Can you create a Python 3 function that
converts a value from one of several bases to an int.

Response:
def num(value):
    """Convert a value from one of several bases to an int."""
    if re_hex_num.match(value):
        return int(value, base=16)
    else:
        return int(value)


Instruction:
Can you write a function in Python 3 where it
replaces the default values of specified fields in the internal structure.

Response:
def update_default_values(self, **updates):
        """Replace the default values of specified fields.

        Parameters
        ----------
        Parameters are taken as keyword-arguments of `field=new_value`.

        Raises
        ------
        KeyError
            If a field doesn't exist in the struct.
        """
        for (field, value) in six.iteritems(updates):
            fname = six.b(field)
            self[fname] = self[fname]._replace(default=value)


Instruction:
Can you create a Python 3 function that
packs the struct into a string of bytes.

Response:
def pack(self):
        """Pack the struct (and its default values) into a string of bytes.

        Returns
        -------
        :py:class:`bytes`
            Byte-string representation of struct containing default values.
        """
        # Generate a buffer big enough to hold the packed values
        data = bytearray(b"\x00" * self.size)

        # Iterate over the fields, pack each value in little-endian format and
        # insert into the buffered data.
        for field in six.itervalues(self.fields):
            packed_data = struct.pack(b"<" + field.pack_chars, field.default)
            data[field.offset:len(packed_data)+field.offset] = packed_data

        return bytes(data)


Instruction:
Here you have a function in Python 3, explain what it does
def next_future_job_delta(self) -> Optional[float]:
        job = self._get_next_future_job()
        if not job:
            return None
        return (job.at - datetime.now(timezone.utc)).total_seconds()

Response: Give the amount of seconds before the next future job is due.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def sdram_alloc_for_vertices(controller, placements, allocations,
                             core_as_tag=True, sdram_resource=SDRAM,
                             cores_resource=Cores, clear=False):
    """Allocate and return a file-like view of a region of SDRAM for each
    vertex which uses SDRAM as a resource.

    The tag assigned to each region of assigned SDRAM is the index of the
    first core that each vertex is assigned.  For example::

        placements = {vertex: (0, 5)}
        allocations = {vertex: {Cores: slice(3, 6),
                                SDRAM: slice(204, 304)}}
        sdram_allocations = sdram_alloc_for_vertices(
            controller, placements, allocations
        )

    Will allocate a 100-byte block of SDRAM for the vertex which is
    allocated cores 3-5 on chip (0, 5).  The region of SDRAM will be tagged
    `3` (because this is the index of the first core).

    Parameters
    ----------
    controller : :py:class:`rig.machine_control.MachineController`
        Controller to use to allocate the SDRAM.
    placements : {vertex: (x, y), ...}
        Mapping of vertices to the chips they have been placed on.  Same as
        produced by placers.
    allocations : {vertex: {resource: allocation, ...}, ...}
        Mapping of vertices to the resources they have been allocated.

        A block of memory of the size specified by the `sdram_resource`
        (default: :py:class:`~rig.place_and_route.SDRAM`) resource will be
        allocated for each vertex. Note that location of the supplied
        allocation is *not* used.

        When `core_as_tag=True`, the tag allocated will be the ID of the first
        core used by the vertex (indicated by the `cores_resource`, default
        :py:class:`~rig.place_and_route.Cores`), otherwise the tag will be set
        to 0.
    clear : bool
        If True the requested memory will be filled with zeros before the
        pointer is returned.  If False (the default) the memory will be left
        as-is.

    Other Parameters
    ----------------
    core_as_tag : bool
        Use the index of the first allocated core as the tag for the region of
        memory, otherwise 0 will be used.
    sdram_resource : resource (default :py:class:`~rig.place_and_route.SDRAM`)
        Key used to indicate SDRAM usage in the resources dictionary.
    cores_resource : resource (default :py:class:`~rig.place_and_route.Cores`)
        Key used to indicate cores which have been allocated in the
        allocations dictionary.

    Returns
    -------
    {vertex: :py:class:`.MemoryIO`, ...}
        A file-like object for each vertex which can be used to read and write
        to the region of SDRAM allocated to the vertex.

    Raises
    ------
    rig.machine_control.machine_controller.SpiNNakerMemoryError
        If the memory cannot be allocated, or a tag is already taken or
        invalid.
    """
    # For each vertex we perform an SDRAM alloc to get a file-like for
    # the vertex.
    vertex_memory = dict()
    for vertex, allocs in six.iteritems(allocations):
        if sdram_resource in allocs:
            sdram_slice = allocs[sdram_resource]
            assert sdram_slice.step is None

            size = sdram_slice.stop - sdram_slice.start
            x, y = placements[vertex]

            if core_as_tag:
                tag = allocs[cores_resource].start
            else:
                tag = 0

            # Get the memory
            vertex_memory[vertex] = controller.sdram_alloc_as_filelike(
                size, tag, x=x, y=y, clear=clear
            )

    return vertex_memory

Response: Allocate and return a file - like view of a region of SDRAM for each vertex.


Instruction:
Can you generate the documentation for the following Python 3 function
def advance_job_status(namespace: str, job: Job, duration: float,
                       err: Optional[Exception]):
    """Advance the status of a job depending on its execution.

    This function is called after a job has been executed. It calculates its
    next status and calls the appropriate signals.
    """
    duration = human_duration(duration)
    if not err:
        job.status = JobStatus.SUCCEEDED
        logger.info('Finished execution of %s in %s', job, duration)
        return

    if job.should_retry:
        job.status = JobStatus.NOT_SET
        job.retries += 1
        if isinstance(err, RetryException) and err.at is not None:
            job.at = err.at
        else:
            job.at = (datetime.now(timezone.utc) +
                      exponential_backoff(job.retries))

        signals.job_schedule_retry.send(namespace, job=job, err=err)

        log_args = (
            job.retries, job.max_retries + 1, job, duration,
            human_duration(
                (job.at - datetime.now(tz=timezone.utc)).total_seconds()
            )
        )
        if isinstance(err, RetryException):
            logger.info('Retry requested during execution %d/%d of %s '
                        'after %s, retry in %s', *log_args)
        else:
            logger.warning('Error during execution %d/%d of %s after %s, '
                           'retry in %s', *log_args)

        return

    job.status = JobStatus.FAILED
    signals.job_failed.send(namespace, job=job, err=err)
    logger.error(
        'Error during execution %d/%d of %s after %s',
        job.max_retries + 1, job.max_retries + 1, job, duration,
        exc_info=err
    )

Response: Advance the status of a job depending on its execution.


Instruction:
Explain what the following Python 3 code does
def tile(self, zoom, row, col):
        return Tile(self, zoom, row, col)

Response: Return a Tile object of this TilePyramid.


Instruction:
Can you tell what is the following Python 3 function doing
def matrix_width(self, zoom):
        validate_zoom(zoom)
        width = int(math.ceil(self.grid.shape.width * 2**(zoom) / self.metatiling))
        return 1 if width < 1 else width

Response: Returns the number of columns at zoom level.


Instruction:
Make a summary of the following Python 3 code
def matrix_height(self, zoom):
        validate_zoom(zoom)
        height = int(math.ceil(self.grid.shape.height * 2**(zoom) / self.metatiling))
        return 1 if height < 1 else height

Response: Returns the number of rows at zoom level.


Instruction:
Given the following Python 3 function, write the documentation
def tile_x_size(self, zoom):
        warnings.warn(DeprecationWarning("tile_x_size is deprecated"))
        validate_zoom(zoom)
        return round(self.x_size / self.matrix_width(zoom), ROUND)

Response: Return the width of a tile in SRID units at zoom level.


Instruction:
Write a Python 3 script to
return the height of a tile in SRID units at zoom level zoom.

Response:
def tile_y_size(self, zoom):
        """
        Height of a tile in SRID units at zoom level.

        - zoom: zoom level
        """
        warnings.warn(DeprecationWarning("tile_y_size is deprecated"))
        validate_zoom(zoom)
        return round(self.y_size / self.matrix_height(zoom), ROUND)


Instruction:
Make a summary of the following Python 3 code
def tile_width(self, zoom):
        warnings.warn(DeprecationWarning("tile_width is deprecated"))
        validate_zoom(zoom)
        matrix_pixel = 2**(zoom) * self.tile_size * self.grid.shape.width
        tile_pixel = self.tile_size * self.metatiling
        return matrix_pixel if tile_pixel > matrix_pixel else tile_pixel

Response: Tile width in pixel.


Instruction:
Given the following Python 3 function, write the documentation
def tile_height(self, zoom):
        warnings.warn(DeprecationWarning("tile_height is deprecated"))
        validate_zoom(zoom)
        matrix_pixel = 2**(zoom) * self.tile_size * self.grid.shape.height
        tile_pixel = self.tile_size * self.metatiling
        return matrix_pixel if tile_pixel > matrix_pixel else tile_pixel

Response: Return the height of the tile in pixel.


Instruction:
Can you write a function in Python 3 where it
returns the width of a pixel in SRID units at zoom level zoom.

Response:
def pixel_x_size(self, zoom):
        """
        Width of a pixel in SRID units at zoom level.

        - zoom: zoom level
        """
        validate_zoom(zoom)
        return round(
            (self.grid.right - self.grid.left) /
            (self.grid.shape.width * 2**zoom * self.tile_size),
            ROUND
        )


Instruction:
Can you write a function in Python 3 where it
returns the height of a pixel in SRID units at zoom level zoom.

Response:
def pixel_y_size(self, zoom):
        """
        Height of a pixel in SRID units at zoom level.

        - zoom: zoom level
        """
        validate_zoom(zoom)
        return round(
            (self.grid.top - self.grid.bottom) /
            (self.grid.shape.height * 2**zoom * self.tile_size),
            ROUND
        )


Instruction:
Here you have a function in Python 3, explain what it does
def tiles_from_bounds(self, bounds, zoom):
        validate_zoom(zoom)
        if not isinstance(bounds, tuple) or len(bounds) != 4:
            raise ValueError("bounds must be a tuple of left, bottom, right, top values")
        if not isinstance(bounds, Bounds):
            bounds = Bounds(*bounds)
        if self.is_global:
            for tile in _global_tiles_from_bounds(self, bounds, zoom):
                yield tile
        else:
            for tile in _tiles_from_cleaned_bounds(self, bounds, zoom):
                yield tile

Response: Yield all tiles intersecting with bounds.


Instruction:
Given the following Python 3 function, write the documentation
def tiles_from_bbox(self, geometry, zoom):
        validate_zoom(zoom)
        return self.tiles_from_bounds(geometry.bounds, zoom)

Response: Returns a list of metatiles intersecting with given bounding box.


Instruction:
Explain what the following Python 3 code does
def tiles_from_geom(self, geometry, zoom):
        validate_zoom(zoom)
        if geometry.is_empty:
            return
        if not geometry.is_valid:
            raise ValueError("no valid geometry: %s" % geometry.type)
        if geometry.geom_type == "Point":
            yield self.tile_from_xy(geometry.x, geometry.y, zoom)
        elif geometry.geom_type == "MultiPoint":
            for point in geometry:
                yield self.tile_from_xy(point.x, point.y, zoom)
        elif geometry.geom_type in (
            "LineString", "MultiLineString", "Polygon", "MultiPolygon",
            "GeometryCollection"
        ):
            prepared_geometry = prep(clip_geometry_to_srs_bounds(geometry, self))
            for tile in self.tiles_from_bbox(geometry, zoom):
                if prepared_geometry.intersects(tile.bbox()):
                    yield tile

Response: Yields all tiles intersecting with input geometry.


Instruction:
Write a Python 3 function that can
return a tile covering a point defined by x y and zoom level.

Response:
def tile_from_xy(self, x, y, zoom, on_edge_use="rb"):
        """
        Return tile covering a point defined by x and y values.

        - x: x coordinate
        - y: y coordinate
        - zoom: zoom level
        - on_edge_use: determine which Tile to pick if Point hits a grid edge
            - rb: right bottom (default)
            - rt: right top
            - lt: left top
            - lb: left bottom
        """
        validate_zoom(zoom)
        if x < self.left or x > self.right or y < self.bottom or y > self.top:
            raise ValueError("x or y are outside of grid bounds")
        if on_edge_use not in ["lb", "rb", "rt", "lt"]:
            raise ValueError("on_edge_use must be one of lb, rb, rt or lt")
        return _tile_from_xy(self, x, y, zoom, on_edge_use=on_edge_use)


Instruction:
Given the following Python 3 function, write the documentation
def get_ladders_metadata(session, parsed):
    ladders = {}
    for ladder in parsed.find_all('a', href=re.compile(LADDER_URL_REGEX)):
        ladders[ladder.text] = get_ladder_metadata(session, ladder['href'])
    return ladders

Response: Get metadata for all ladders.


Instruction:
Explain what the following Python 3 code does
def get_metadata(session, games):
    for data in games.values():
        parsed = make_scrape_request(session, data['url'])
    return {
        'ladders': get_ladders_metadata(session, parsed)
    }

Response: Get metadata for all games.


Instruction:
Explain what the following Python 3 code does
def update_metadata(session, path=DATA_PATH):
    with open(os.path.join(path, 'games.json')) as handle:
        games = json.loads(handle.read())

    for key, data in get_metadata(session, games).items():
        with open(os.path.join(path, '{}.json'.format(key)), 'w') as handle:
           handle.write(json.dumps(data, indent=2))

Response: Update metadata files in the given path.


Instruction:
Can you create a Python 3 function that
samples every router counter in the machine.

Response:
def sample_counters(mc, system_info):
    """Sample every router counter in the machine."""
    return {
        (x, y): mc.get_router_diagnostics(x, y) for (x, y) in system_info
    }


Instruction:
Create a Python 3 function to
return the change in counter values.

Response:
def deltas(last, now):
    """Return the change in counter values (accounting for wrap-around)."""
    return {
        xy: RouterDiagnostics(*((n - l) & 0xFFFFFFFF
                                for l, n in zip(last[xy], now[xy])))
        for xy in last
    }


Instruction:
Can you tell what is the following Python 3 function doing
def monitor_counters(mc, output, counters, detailed, f):
    # Print CSV header
    output.write("time,{}{}\n".format("x,y," if detailed else "",
                                      ",".join(counters)))

    system_info = mc.get_system_info()

    # Make an initial sample of the counters
    last_counter_values = sample_counters(mc, system_info)

    start_time = time.time()

    for _ in f():
        # Snapshot the change in counter values
        counter_values = sample_counters(mc, system_info)
        delta = deltas(last_counter_values, counter_values)
        last_counter_values = counter_values

        now = time.time() - start_time

        # Output the changes
        if detailed:
            for x, y in sorted(system_info):
                output.write("{:0.1f},{},{},{}\n".format(
                    now, x, y,
                    ",".join(str(getattr(delta[(x, y)], c))
                             for c in counters)))
        else:
            totals = [0 for _ in counters]
            for xy in sorted(system_info):
                for i, counter in enumerate(counters):
                    totals[i] += getattr(delta[xy], counter)
            output.write("{:0.1f},{}\n".format(
                now, ",".join(map(str, totals))))

Response:
Monitor the counters on a specified machine taking a snap - shot every
    time the generator f yields.


Instruction:
How would you explain what the following Python 3 function does
def press_enter(multiple=False, silent=False):

    def f():
        try:
            while True:
                if silent:
                    yield input()
                else:
                    sys.stderr.write("<press enter> ")
                    sys.stderr.flush()
                    yield input()
                if not multiple:
                    break
        except (EOFError, KeyboardInterrupt):
            # User Ctrl+D or Ctrl+C'd
            if not silent:
                # Prevents the user's terminal getting clobbered
                sys.stderr.write("\n")
                sys.stderr.flush()
            return

    return f

Response:
Return a generator function which yields every time the user presses
    return.


Instruction:
Explain what the following Python 3 code does
def listen(timeout=6.0, port=BOOT_PORT):

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    # Don't take control of this socket in the system (i.e. allow other
    # processes to bind to it) since we're listening for broadcasts.
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    # Listen for the broadcasts
    s.bind(('0.0.0.0', port))
    s.settimeout(timeout)
    try:
        message, (ipaddr, port) = s.recvfrom(512)
        return ipaddr
    except socket.timeout:
        return None

Response: Listen for a ping broadcast message from an unbooted SpiNNaker board on port 54321.


Instruction:
Can you tell what is the following Python 3 function doing
def clear_cache(m, files_processed):

    for what, reason, url, path in files_processed:
        cp = m.doc.downloader.cache_path(url)

        if m.cache.exists(cp):
            m.cache.remove(cp)

Response: Remove any files we ve uploaded from the cache.


Instruction:
Explain what the following Python 3 code does
def fill_categorical_na(df, nan_cat='NA'):
    for col in df.columns[df.isna().any()].tolist():

        if df[col].dtype.name != 'category':
            # If not categorical, fill with a blank, which creates and
            # empty cell in CSV.
            df[col] = df[col].fillna('')
        else:

            try:
                df[col].cat.add_categories([nan_cat], inplace=True)
            except ValueError:
                pass

            df[col] = df[col].fillna(nan_cat)

    return df

Response: Fill NaNs with NaNs in the categoricals of the archive with NaNs.


Instruction:
Given the following Python 3 function, write the documentation
def executor_ips(sc):
    try:
        app_id = sc.applicationId
    except AttributeError:
        app_id = sc.getConf().get('spark.app.id')
    # for getting the url (see: https://github.com/apache/spark/pull/15000)
    try:
        base_url = sc.uiWebUrl
    except AttributeError:
        base_url = sc._jsc.sc().uiWebUrl().get()
    url = base_url + '/api/v1/applications/' + app_id + '/executors'
    try:
        data = json.load(urlopen(url))
    except TypeError:
        response = urlopen(url)
        str_response = response.read().decode('utf-8')
        data = json.loads(str_response)
    ips = set(map(lambda x: x[u'hostPort'].split(':')[0], data))
    return ips

Response: gets the unique ip addresses of the executors of the current application


Instruction:
Can you tell what is the following Python 3 function doing
def change(sc=None, app_name='customSpark', master=None, wait='ips', min_cores=None, min_ips=None, timeout=30,
           refresh_rate=0.5, fail_on_timeout=False, **kwargs):
    """ Returns a new Spark Context (sc) object with added properties set

    :param sc: current SparkContext if None will create a new one
    :param app_name: name of new spark app
    :param master: url to master, if None will get from current sc
    :param wait: when to return after asking for a new sc (or max of timeout seconds):
     'ips': wait for all the previous ips that were connected to return (needs sc to not be None)
     'cores': wait for min_cores
     None: return immediately
    :param min_cores: when wait is 'cores' will wait until defaultParallelism is back to at least this value.
     if None will be set to defaultParallelism.
    :param min_ips: when wait is 'ips' will wait until number of unique executor ips is back to at least this value.
     if None will be set to the what the original sc had.
    :param timeout: max time in seconds to wait for new sc if wait is 'ips' or 'cores'
    :param fail_on_timeout: whether to fail if timeout has reached
    :param refresh_rate: how long to wait in seconds between each check of defaultParallelism
    :param kwargs:  added properties to set. In the form of key value pairs (replaces '.' with '_' in key)
                    examples: spark_task_cores='1', spark_python_worker_memory='8g'
                    see: http://spark.apache.org/docs/latest/configuration.html
    :return: a new SparkContext
    """
    # checking input
    if master is None and sc is None:
        raise ValueError('Both master and sc are None')
    if master is None:
        master = sc.getConf().get(u'spark.master')
    if wait == 'ips':
        if sc is None:
            if min_ips is None:
                min_ips = 1
        elif min_ips is None:
            min_ips = len(executor_ips(sc))
    elif wait == 'cores':
        if min_cores is None:
            if sc is None:
                logging.getLogger('pySparkUtils').info('Both sc and min_cores are None: setting target_cores to 2')
                min_cores = 2
            else:
                min_cores = sc.defaultParallelism
                logging.getLogger('pySparkUtils').info('min_cores is None: setting target_cores to: %d' % min_cores)
    elif wait is not None:
        raise ValueError("wait should be: ['ips','cores',None] got: %s" % wait)

    if sc is not None:
        logging.getLogger('pySparkUtils').info('Stopping original sc with %d cores and %d executors' %
                                               (sc.defaultParallelism, len(executor_ips(sc))))
        sc.stop()

    # building a new configuration with added arguments
    conf = SparkConf().setMaster(master).setAppName(app_name)
    for key in kwargs.keys():
        name = key.replace('_', '.', 100)
        value = kwargs[key]
        conf = conf.set(name, value)
        logging.getLogger('pySparkUtils').info('Setting %s to: %s' % (name, value))

    # starting the new context and waiting for defaultParallelism to get back to original value
    sc = SparkContext(conf=conf)
    if wait == 'cores':
        total_time = 0
        while sc.defaultParallelism < min_cores and total_time < timeout:
            time.sleep(refresh_rate)
            total_time += refresh_rate
        if fail_on_timeout and total_time >= timeout:
            sc.stop()
            raise RuntimeError('Time out reached when changing sc')
    elif wait == 'ips':
        total_time = 0
        while len(executor_ips(sc)) < min_ips and total_time < timeout:
            time.sleep(refresh_rate)
            total_time += refresh_rate
        if fail_on_timeout and total_time >= timeout:
            sc.stop()
            raise RuntimeError('Time out reached when changing sc')
    logging.getLogger('pySparkUtils').info('Returning new sc with %d cores and %d executors' %
                                           (sc.defaultParallelism, len(executor_ips(sc))))
    return sc

Response: Returns a new SparkContext object with added properties set to the properties of the SparkContext object.


Instruction:
How would you explain what the following Python 3 function does
def regroup(rdd, groups=10, check_first=False):
    if check_first:
        first = rdd.first()
        if isinstance(first, (list, tuple, collections.Iterable)):
            if len(first) != 2:
                raise ValueError('first item was not not length 2: %d' % len(first))
        else:
            raise ValueError('first item was wrong type: %s' % type(first))
    rdd = rdd.map(lambda kv: (kv[0] % groups, (kv[0], kv[1])), preservesPartitioning=True)
    return rdd.groupByKey().mapValues(list)

Response: Regroups an rdd into groups of size groups


Instruction:
Can you tell what is the following Python 3 function doing
def save_rdd_as_pickle(rdd, path, batch_size=10, overwrite=False):
    if os.path.isdir(path):
        if overwrite:
            logging.getLogger('pySparkUtils').info('Deleting files from: %s' % path)
            shutil.rmtree(path)
            logging.getLogger('pySparkUtils').info('Done deleting files from: %s' % path)
        else:
            logging.getLogger('pySparkUtils').error('Directory %s already exists '
                                                    'and overwrite is false' % path)
            raise IOError('Directory %s already exists and overwrite is false'
                          % path)
    rdd.glom().saveAsPickleFile(path, batchSize=batch_size)
    logging.getLogger('pySparkUtils').info('Saved rdd as pickle to: %s' % path)

Response: Saves an rdd to a pickle file.


Instruction:
Make a summary of the following Python 3 code
def load_rdd_from_pickle(sc, path, min_partitions=None, return_type='images'):
    if min_partitions is None:
        min_partitions = sc.defaultParallelism
    rdd = sc.pickleFile(path, minPartitions=min_partitions)
    rdd = rdd.flatMap(lambda x: x)
    if return_type == 'images':
        result = td.images.fromrdd(rdd).repartition(min_partitions)
    elif return_type == 'series':
        result = td.series.fromrdd(rdd).repartition(min_partitions)
    elif return_type == 'rdd':
        result = rdd.repartition(min_partitions)
    else:
        raise ValueError('return_type not supported: %s' % return_type)
    logging.getLogger('pySparkUtils').info('Loaded rdd from: %s as type: %s'
                                           % (path, return_type))
    return result

Response:
Loads an RDD that was saved as one pickle file per partition and returns it as a new instance of
       .


Instruction:
Make a summary of the following Python 3 code
def geo(self):
        import geopandas as gpd
        from shapely.geometry.polygon import BaseGeometry
        from shapely.wkt import loads

        gdf = gpd.GeoDataFrame(self)

        first = next(gdf.iterrows())[1].geometry

        if isinstance(first, str):
            # We have a GeoDataframe, but the geometry column is still strings, so
            # it must be converted
            shapes = [ loads(row['geometry']) for i, row in gdf.iterrows()]

        elif not isinstance(first, BaseGeometry):
            # If we are reading a metatab package, the geometry column's type should be
            # 'geometry' which will give the geometry values class type of
            # rowpipe.valuetype.geo.ShapeValue. However, there are other
            # types of objects that have a 'shape' property.

            shapes = [row['geometry'].shape for i, row in gdf.iterrows()]

        else:
            shapes = gdf['geometry']

        gdf['geometry'] = gpd.GeoSeries(shapes)
        gdf.set_geometry('geometry')
        return gdf

Response: Return a geopandas dataframe


Instruction:
Can you create a Python 3 function that
yields the rows of the table.

Response:
def rows(self):
        """Yield rows like a partition does, with a header first, then rows. """

        yield [self.index.name] + list(self.columns)

        for t in self.itertuples():
            yield list(t)


Instruction:
Write a Python 3 script for
checking whether the node_set properties match the given model type definition

Response:
def _matches_node_set_props(type_info, node_set_props):
    """
        Check whether the node_set properties match the given model type definition
    """
    matches = None
    for key in node_set_props:
        ns_val = node_set_props[key]
        if key in type_info:
            if ns_val==type_info[key]:
                if matches:
                    matches = matches and True
                else:
                    matches = True
            else:
                matches = False
    return matches


Instruction:
How would you implement a function in Python 3 that
gets a NeuroML document from a Sonata model

Response:
def get_neuroml_from_sonata(sonata_filename, id, generate_lems = True, format='xml'):
    """
    Return a NeuroMLDocument with (most of) the contents of the Sonata model
    """

    from neuroml.hdf5.NetworkBuilder import NetworkBuilder
    neuroml_handler = NetworkBuilder()

    sr = SonataReader(filename=sonata_filename, id=id)

    sr.parse(neuroml_handler)

    nml_doc = neuroml_handler.get_nml_doc()

    sr.add_neuroml_components(nml_doc)

    if format == 'xml':
        nml_file_name = '%s.net.nml'%id
        from neuroml.writers import NeuroMLWriter
        NeuroMLWriter.write(nml_doc, nml_file_name)

    elif format == 'hdf5':
        nml_file_name = '%s.net.nml.h5'%id
        from neuroml.writers import NeuroMLHdf5Writer
        NeuroMLHdf5Writer.write(nml_doc, nml_file_name)

    print_v('Written to: %s'%nml_file_name)

    if generate_lems:
        lems_file_name = sr.generate_lems_file(nml_file_name, nml_doc)

        return sr, lems_file_name, nml_file_name, nml_doc

    return nml_doc


Instruction:
Explain what the following Python 3 code does
def subs(self, path):
        #print_v('Checking for: \n  %s, \n  %s \n  in %s'%(self.substitutes,self.init_substitutes,path))
        if type(path) == int or type(path) == float:
            return path
        for s in self.init_substitutes:
            if path.startswith(s):
                path = path.replace(s,self.init_substitutes[s], 1)
        #print_v('  So far: %s'%path)
        for s in self.substitutes:
            if s in path:
                path = path.replace(s,self.substitutes[s])
        #print_v('  Returning: %s'%path)
        return path

Response: Search the strings in a config file for a substitutable value


Instruction:
Can you generate a brief explanation for the following Python 3 code
def parse(self, handler):

        ########################################################################
        #  load the main configuration scripts

        main_config_filename = os.path.abspath(self.parameters['filename'])

        config = load_json(main_config_filename)

        self.init_substitutes = {'.':'%s/'%os.path.dirname(main_config_filename),
                                 '../':'%s/'%os.path.dirname(os.path.dirname(main_config_filename))}

        self.substitutes = {'${configdir}':'%s'%os.path.dirname(main_config_filename)}

        if 'network' in config:
            self.network_config = load_json(self.subs(config['network']))
        else:
            self.network_config = config

        if 'simulation' in config:
            self.simulation_config = load_json(self.subs(config['simulation']))
        else:
            self.simulation_config = None

        for m in self.network_config['manifest']:
            path = self.subs(self.network_config['manifest'][m])
            self.substitutes[m] = path

        if 'id' in self.parameters:
            id = self.parameters['id']
        else:
            id = 'SonataNetwork'

        if id[0].isdigit():  # id like 9_cells is not a valid id for NeuroML
            id='NML2_%s'%id

        ########################################################################
        #  Feed the handler the info on the network

        self.handler = handler

        notes = "Network read in from Sonata: %s"%main_config_filename
        handler.handle_document_start(id, notes)

        handler.handle_network(id, notes)

        self.node_types = {}

        ########################################################################
        #  Get info from nodes files

        for n in self.network_config['networks']['nodes']:
            nodes_file = self.subs(n['nodes_file'])
            node_types_file = self.subs(n['node_types_file'])

            print_v("\nLoading nodes from %s and %s"%(nodes_file,node_types_file))

            h5file=tables.open_file(nodes_file,mode='r')

            print_v("Opened HDF5 file: %s"%(h5file.filename))
            self.parse_group(h5file.root.nodes)
            h5file.close()
            self.node_types[self.current_sonata_pop] = load_csv_props(node_types_file)
            self.current_sonata_pop = None

        ########################################################################
        #  Get info from edges files

        self.edges_info = {}
        self.conn_info = {}

        if 'edges' in self.network_config['networks']:
            for e in self.network_config['networks']['edges']:
                edges_file = self.subs(e['edges_file'])
                edge_types_file = self.subs(e['edge_types_file'])

                print_v("\nLoading edges from %s and %s"%(edges_file,edge_types_file))

                h5file=tables.open_file(edges_file,mode='r')

                print_v("Opened HDF5 file: %s"%(h5file.filename))
                self.parse_group(h5file.root.edges)
                h5file.close()
                self.edges_info[self.current_edge] = load_csv_props(edge_types_file)
                self.current_edge = None

        ########################################################################
        #  Use extracted node/cell info to create populations

        for sonata_pop in self.cell_info:
            types_vs_pops = {}

            for type in self.cell_info[sonata_pop]['type_count']:
                node_type_info = self.node_types[sonata_pop][type]

                model_name_type = node_type_info['model_name'] if 'model_name' in node_type_info \
                                  else (node_type_info['pop_name'] if 'pop_name' in node_type_info else node_type_info['model_type'])

                model_type = node_type_info['model_type']
                model_template = node_type_info['model_template'] if 'model_template' in node_type_info else '- None -'

                nml_pop_id = '%s_%s_%s'%(sonata_pop,model_name_type,type)

                print_v(" - Adding population: %s which has model info: %s"%(nml_pop_id, node_type_info))

                size = self.cell_info[sonata_pop]['type_count'][type]

                if model_type=='point_process' and model_template=='nrn:IntFire1':
                    raise Exception('Point process model not currently supported: %s\nTry expressing the I&F cell in NEST format with nest:iaf_psc_alpha'%model_template)
                    pop_comp = 'cell_%s'%nml_pop_id #model_template.replace(':','_')
                    self.pop_comp_info[pop_comp] = {}
                    self.pop_comp_info[pop_comp]['model_type'] = model_type

                    dynamics_params_file = self.subs(self.network_config['components']['point_neuron_models_dir']) +'/'+node_type_info['dynamics_params']
                    self.pop_comp_info[pop_comp]['dynamics_params'] = load_json(dynamics_params_file)

                elif model_type=='point_process' and model_template=='nest:iaf_psc_alpha':
                    pop_comp = 'cell_%s'%nml_pop_id # = model_template.replace(':','_')
                    self.pop_comp_info[pop_comp] = {}
                    self.pop_comp_info[pop_comp]['model_type'] = model_type
                    self.pop_comp_info[pop_comp]['model_template'] = model_template

                    dynamics_params_file = self.subs(self.network_config['components']['point_neuron_models_dir']) +'/'+node_type_info['dynamics_params']
                    self.pop_comp_info[pop_comp]['dynamics_params'] = load_json(dynamics_params_file)
                else:
                    pop_comp = DUMMY_CELL
                    self.pop_comp_info[pop_comp] = {}
                    self.pop_comp_info[pop_comp]['model_type'] = pop_comp

                self.nml_pop_vs_comps[nml_pop_id] = pop_comp

                properties = {}

                properties['type_id']=type
                properties['sonata_population']=sonata_pop
                properties['region']=sonata_pop

                for i in node_type_info:
                    properties[i]=node_type_info[i]
                    if i=='ei':
                        properties['type']=node_type_info[i].upper()

                color = '%s %s %s'%(self.myrandom.random(),self.myrandom.random(),self.myrandom.random())

                try:
                    import opencortex.utils.color as occ
                    interneuron = 'SOM' in nml_pop_id or 'PV' in nml_pop_id
                    if 'L23' in nml_pop_id:
                        color = occ.L23_INTERNEURON if interneuron else occ.L23_PRINCIPAL_CELL
                        pop.properties.append(neuroml.Property('region','L23'))
                    if 'L4' in nml_pop_id:
                        color = occ.L4_INTERNEURON if interneuron else occ.L4_PRINCIPAL_CELL
                        pop.properties.append(neuroml.Property('region','L4'))
                    if 'L5' in nml_pop_id:
                        color = occ.L5_INTERNEURON if interneuron else occ.L5_PRINCIPAL_CELL
                        pop.properties.append(neuroml.Property('region','L5'))
                    if 'L6' in nml_pop_id:
                        color = occ.L6_INTERNEURON if interneuron else occ.L6_PRINCIPAL_CELL
                        pop.properties.append(neuroml.Property('region','L6'))
                except:
                    pass # Don't specify a particular color, use random, not a problem...

                properties['color']=color
                if True or not 'locations' in self.cell_info[sonata_pop]['0']:

                    properties={}  #############  temp for LEMS...

                if model_type != 'virtual':

                    self.handler.handle_population(nml_pop_id,
                                             pop_comp,
                                             size,
                                             component_obj=None,
                                             properties=properties)

                types_vs_pops[type] = nml_pop_id
            self.cell_info[sonata_pop]['pop_count'] = {}
            self.cell_info[sonata_pop]['pop_map'] = {}

            for i in self.cell_info[sonata_pop]['types']:

                pop = types_vs_pops[self.cell_info[sonata_pop]['types'][i]]

                if not pop in self.cell_info[sonata_pop]['pop_count']:
                    self.cell_info[sonata_pop]['pop_count'][pop] = 0

                index = self.cell_info[sonata_pop]['pop_count'][pop]
                self.cell_info[sonata_pop]['pop_map'][i] = (pop, index)

                if not pop in self.nml_ids_vs_gids:
                    self.nml_ids_vs_gids[pop] = {}
                self.nml_ids_vs_gids[pop][index] = (sonata_pop, i)

                if i in self.cell_info[sonata_pop]['0']['locations']:
                    if not pop in self.nml_pops_having_locations:
                        self.nml_pops_having_locations.append(pop)
                    pos = self.cell_info[sonata_pop]['0']['locations'][i]
                    #print('Adding pos %i: %s'%(i,pos))
                    self.handler.handle_location(index,
                                                 pop,
                                                 pop_comp,
                                                 pos['x'] if 'x' in pos and pos['x'] is not None else 0,
                                                 pos['y'] if 'y' in pos and pos['y'] is not None else 0,
                                                 pos['z'] if 'z' in pos and pos['z'] is not None else 0)

                self.cell_info[sonata_pop]['pop_count'][pop]+=1

        ########################################################################
        #  Load simulation info into self.simulation_config

        if self.simulation_config:

            if self.simulation_config:
                for m in self.simulation_config['manifest']:
                    path = self.subs(self.simulation_config['manifest'][m])
                    self.substitutes[m] = path

            for s1 in ['output']:
                for k in self.simulation_config[s1]:
                    self.simulation_config[s1][k] = self.subs(self.simulation_config[s1][k])

            for s1 in ['inputs']:
                for s2 in self.simulation_config[s1]:
                    for k in self.simulation_config[s1][s2]:
                        self.simulation_config[s1][s2][k] = self.subs(self.simulation_config[s1][s2][k])

            if 'node_sets_file' in self.simulation_config:
                node_sets = load_json(self.subs(self.simulation_config['node_sets_file']))
                self.simulation_config['node_sets'] = node_sets

            if not 'node_sets' in self.simulation_config:
                self.simulation_config['node_sets'] = {}

            for sonata_pop in self.cell_info:
                self.node_set_mappings[sonata_pop] = {}
                for sindex in self.cell_info[sonata_pop]['pop_map']:
                    nml_pop = self.cell_info[sonata_pop]['pop_map'][sindex][0]
                    nml_index = self.cell_info[sonata_pop]['pop_map'][sindex][1]

                    # Add all in this sonata_pop to a 'node_set' named after the sonata_pop
                    if not nml_pop in self.node_set_mappings[sonata_pop]:
                        self.node_set_mappings[sonata_pop][nml_pop] = []
                    self.node_set_mappings[sonata_pop][nml_pop].append(nml_index)

            #pp.pprint(self.simulation_config)
            #pp.pprint(self.pop_comp_info)

            for node_set in self.simulation_config['node_sets']:
                self.node_set_mappings[node_set] = {}
                node_set_props = self.simulation_config['node_sets'][node_set]
                #print_v('===========Checking which cells in pops match node_set: %s = %s'%(node_set,node_set_props))

                for sonata_pop in self.cell_info:
                    for sindex in self.cell_info[sonata_pop]['pop_map']:
                        #print('Does %s %s match %s?'%(sonata_pop, sindex, node_set_props))

                        type = self.cell_info[sonata_pop]['types'][sindex]
                        type_info = self.node_types[sonata_pop][type]
                        nml_pop = self.cell_info[sonata_pop]['pop_map'][sindex][0]
                        nml_index = self.cell_info[sonata_pop]['pop_map'][sindex][1]

                        if 'population' in node_set_props and node_set_props['population'] == sonata_pop:
                            if 'node_id' in node_set_props and sindex in node_set_props['node_id']:
                                if not nml_pop in self.node_set_mappings[node_set]:
                                    self.node_set_mappings[node_set][nml_pop] = []
                                self.node_set_mappings[node_set][nml_pop].append(nml_index)

                        matches = _matches_node_set_props(type_info, node_set_props)
                        #print_v('Node %i in %s (NML: %s[%i]) has type %s (%s); matches: %s'%(sindex, sonata_pop, nml_pop, nml_index, type, type_info, matches))
                        if matches:
                            if not nml_pop in self.node_set_mappings[node_set]:
                                self.node_set_mappings[node_set][nml_pop] = []
                            self.node_set_mappings[node_set][nml_pop].append(nml_index)

            ##pp.pprint(self.node_set_mappings)

        ########################################################################
        #  Extract info from inputs in simulation_config
        #pp.pprint(self.simulation_config)

        for input in self.simulation_config['inputs']:
            info = self.simulation_config['inputs'][input]
            #print_v(" - Adding input: %s which has info: %s"%(input, info))

            self.input_comp_info[input] = {}
            self.input_comp_info[input][info['input_type']] = {}

            node_set = info['node_set']

            if info['input_type'] == 'current_clamp':

                comp = 'PG_%s'%input
                self.input_comp_info[input][info['input_type']][comp] = {'amp':info['amp'],'delay':info['delay'],'duration':info['duration']}

                for nml_pop_id in self.node_set_mappings[node_set]:

                    input_list_id = 'il_%s_%s'%(input,nml_pop_id)
                    indices = self.node_set_mappings[node_set][nml_pop_id]

                    self.handler.handle_input_list(input_list_id,
                                                   nml_pop_id,
                                                   comp,
                                                   len(indices))
                    count = 0
                    for index in indices:
                        self.handler.handle_single_input(input_list_id,
                                                         count,
                                                         cellId = index,
                                                         segId = 0,
                                                         fract = 0.5)
                        count+=1

            elif info['input_type'] == 'spikes':
                node_info = self.cell_info[node_set]

                from pyneuroml.plot.PlotSpikes import read_sonata_spikes_hdf5_file

                ids_times = read_sonata_spikes_hdf5_file(self.subs(info['input_file']))

                for id in ids_times:
                    times = ids_times[id]
                    if id in node_info['pop_map']:
                        nml_pop_id, cell_id = node_info['pop_map'][id]
                        print_v("Cell %i in Sonata node set %s (cell %s in nml pop %s) has %i spikes"%(id, node_set, nml_pop_id, cell_id, len(times)))

                        component = '%s__%i'%(nml_pop_id,cell_id)

                        self.input_comp_info[input][info['input_type']][component] ={'id': cell_id, 'times': times}

                        '''
                        input_list_id = 'il_%s_%i'%(input,cell_id)
                        self.handler.handle_input_list(input_list_id,
                                                       nml_pop_id,
                                                       component,
                                                       1)

                        self.handler.handle_single_input(input_list_id,
                                                          0,
                                                          cellId = cell_id,
                                                          segId = 0,
                                                          fract = 0.5)
                                                         '''
                    else:
                        print_v("Cell %i in Sonata node set %s NOT FOUND!"%(id, node_set))

            else:
                raise Exception("Sonata input type not yet supported: %s"%(info['input_type']))

        ########################################################################
        #  Use extracted edge info to create connections

        projections_created = []
        for conn in self.conn_info:

            pre_node = self.conn_info[conn]['pre_node']
            post_node = self.conn_info[conn]['post_node']

            for i in range(len(self.conn_info[conn]['pre_id'])):
                pre_id = self.conn_info[conn]['pre_id'][i]
                post_id = self.conn_info[conn]['post_id'][i]
                nsyns = self.conn_info[conn]['nsyns'][i] if 'nsyns' in self.conn_info[conn] else 1
                type = self.conn_info[conn]['edge_type_id'][i]
                #print_v('   Conn with %i syns, type %s: %s(%s) -> %s(%s)'%(nsyns,type,pre_node,pre_id,post_node,post_id))
                pre_pop,pre_i = self.cell_info[pre_node]['pop_map'][pre_id]
                post_pop,post_i = self.cell_info[post_node]['pop_map'][post_id]
                #print_v('   Mapped: Conn %s(%s) -> %s(%s)'%(pre_pop,pre_i,post_pop,post_i))
                # print self.edges_info[conn][type]
                #print self.cell_info[pre_node]
                #print 11
                #print self.node_types[pre_node]
                #print 22
                cell_type_pre = self.cell_info[pre_node]['types'][pre_id]
                #print cell_type_pre
                #print 444
                pop_type_pre = self.node_types[pre_node][cell_type_pre]['model_type']
                #print pop_type_pre
                #print 333

                synapse = self.edges_info[conn][type]['dynamics_params'].split('.')[0]
                self.syn_comp_info[synapse] = {}
                #print self.edges_info[conn][type]
                #pp.pprint(self.init_substitutes)
                #pp.pprint(self.substitutes)
                dynamics_params_file = self.subs(self.network_config['components']['synaptic_models_dir']) +'/'+self.edges_info[conn][type]['dynamics_params']
                #print_v('Adding syn %s (at %s)'%(self.edges_info[conn][type]['dynamics_params'], dynamics_params_file))

                #TODO: don't load this file every connection!!!
                self.syn_comp_info[synapse]['dynamics_params'] = load_json(dynamics_params_file)
                proj_id = '%s_%s_%s'%(pre_pop,post_pop,synapse)

                sign = self.syn_comp_info[synapse]['dynamics_params']['sign'] if 'sign' in self.syn_comp_info[synapse]['dynamics_params'] else 1
                weight = self.edges_info[conn][type]['syn_weight'] if 'syn_weight' in self.edges_info[conn][type] else 1.0

                syn_weight_edge_group_0 = self.conn_info[conn]['syn_weight_edge_group_0'][i] if 'syn_weight_edge_group_0' in self.conn_info[conn] else None

                # Assume this overrides value from csv file...
                if syn_weight_edge_group_0:
                    weight = syn_weight_edge_group_0

                #print_v('Adding syn %s (at %s), weight: %s, sign: %s, nsyns: %s'%(self.edges_info[conn][type]['dynamics_params'], dynamics_params_file, weight, sign, nsyns))

                weight_scale = 0.001
                if 'level_of_detail' in self.syn_comp_info[synapse]['dynamics_params']:
                    weight_scale = 1

                weight=weight_scale * sign * weight * nsyns

                delay = self.edges_info[conn][type]['delay'] if 'delay' in self.edges_info[conn][type] else 0

                if not pop_type_pre == 'virtual':
                    if not proj_id in projections_created:

                        self.handler.handle_projection(proj_id,
                                             pre_pop,
                                             post_pop,
                                             synapse)

                        projections_created.append(proj_id)

                    self.handler.handle_connection(proj_id,
                                                 i,
                                                 pre_pop,
                                                 post_pop,
                                                 synapse, \
                                                 pre_i, \
                                                 post_i, \
                                                 weight=weight, \
                                                 delay=delay)

                else:
                    component = '%s__%i'%(pre_pop,pre_i)
                    #print_v('   ---  Connecting %s to %s[%s]'%(component, post_pop, post_i))

                    #self.input_comp_info[input][info['input_type']][component] ={'id': cell_id, 'times': times}

                    input_list_id = 'il_%s_%s_%i_%i'%(component,post_pop,post_i,i)

                    self.handler.handle_input_list(input_list_id,
                                                   post_pop,
                                                   component,
                                                   1)

                    self.handler.handle_single_input(input_list_id,
                                                      0,
                                                      cellId = post_i,
                                                      segId = 0,
                                                      fract = 0.5,
                                                      weight=weight)

        """
        print('~~~~~~~~~~~~~~~')
        print('node_types:')
        pp.pprint(self.node_types)
        print('~~~~~~~~~~~~~~~')
        print('cell_info:')
        pp.pprint(self.cell_info)
        print('================')"""

Response: Parse the Sonata file and call the appropriate methods in the handler.


Instruction:
Create a Python 3 function to
add the components of the NeuroML cell to the doc.

Response:
def add_neuroml_components(self, nml_doc):
        """
            Based on cell & synapse properties found, create the corresponding NeuroML components
        """

        is_nest = False
        print_v("Adding NeuroML cells to: %s"%nml_doc.id)
        #pp.pprint(self.pop_comp_info)

        for c in self.pop_comp_info:
            info = self.pop_comp_info[c]

            model_template = info['model_template'] if 'model_template' in info else \
                             (info['dynamics_params']['type'] if 'dynamics_params' in info else
                             info['model_type'])

            print_v(" - Adding %s: %s"%(model_template, info))

            if info['model_type'] == 'point_process' and model_template == 'nest:iaf_psc_alpha':

                is_nest = True
                from neuroml import IF_curr_alpha
                pynn0 = IF_curr_alpha(id=c,
                                      cm=info['dynamics_params']['C_m']/1000.0,
                                      i_offset="0",
                                      tau_m=info['dynamics_params']['tau_m'],
                                      tau_refrac=info['dynamics_params']['t_ref'],
                                      tau_syn_E="1",
                                      tau_syn_I="1",
                                      v_init='-70',
                                      v_reset=info['dynamics_params']['V_reset'],
                                      v_rest=info['dynamics_params']['E_L'],
                                      v_thresh=info['dynamics_params']['V_th'])

                nml_doc.IF_curr_alpha.append(pynn0)

            elif info['model_type'] == 'point_process' and model_template == 'NEURON_IntFire1':

                contents = '''<Lems>
    <intFire1Cell id="%s" thresh="1mV" reset="0mV" tau="%sms" refract="%sms"/>
</Lems>'''%(c, info['dynamics_params']['tau']*1000, info['dynamics_params']['refrac']*1000)

                cell_file_name = '%s.xml'%c
                cell_file = open(cell_file_name,'w')
                cell_file.write(contents)
                cell_file.close()

                self.nml_includes.append(cell_file_name)
                self.nml_includes.append('../../../examples/sonatatest/IntFireCells.xml')

            else:

                from neuroml import IafRefCell
                IafRefCell0 = IafRefCell(id=DUMMY_CELL,
                           C=".2 nF",
                           thresh = "1mV",
                           reset="0mV",
                           refract="3ms",
                           leak_conductance="1.2 nS",
                           leak_reversal="0mV")

                print_v("   - Adding: %s"%IafRefCell0)
                nml_doc.iaf_ref_cells.append(IafRefCell0)

        print_v("Adding NeuroML synapses to: %s"%nml_doc.id)

        #pp.pprint(self.syn_comp_info)

        for s in self.syn_comp_info:
            dyn_params = self.syn_comp_info[s]['dynamics_params']

            print_v("     -  Syn: %s: %s"%(s, dyn_params))

            if 'level_of_detail' in dyn_params and dyn_params['level_of_detail'] == 'exp2syn':
                from neuroml import ExpTwoSynapse

                syn = ExpTwoSynapse(id=s,
                                    gbase="1nS",
                                    erev="%smV"%dyn_params['erev'],
                                    tau_rise="%sms"%dyn_params['tau1'],
                                    tau_decay="%sms"%dyn_params['tau2'])

                #print("Adding syn: %s"%syn)
                nml_doc.exp_two_synapses.append(syn)

            elif 'level_of_detail' in dyn_params and dyn_params['level_of_detail'] == 'instanteneous':

                contents = '''<Lems>
    <impulseSynapse id="%s"/>
</Lems>'''%(s)

                syn_file_name = '%s.xml'%s
                syn_file = open(syn_file_name,'w')
                syn_file.write(contents)
                syn_file.close()

                self.nml_includes.append(syn_file_name)
                #self.nml_includes.append('../examples/sonatatest/IntFireCells.xml')

            else:
                from neuroml import AlphaCurrSynapse
                pynnSynn0 = AlphaCurrSynapse(id=s, tau_syn="2")
                #print("Adding syn: %s"%pynnSynn0)
                nml_doc.alpha_curr_synapses.append(pynnSynn0)

        print_v("Adding NeuroML inputs to: %s"%nml_doc.id)

        #pp.pprint(self.input_comp_info)

        for input in self.input_comp_info:
            for input_type in self.input_comp_info[input]:
                if input_type == 'spikes':
                    for comp_id in self.input_comp_info[input][input_type]:
                        info = self.input_comp_info[input][input_type][comp_id]
                        print_v("Adding input %s: %s"%(comp_id, info.keys()))

                        nest_syn = _get_default_nest_syn(nml_doc)
                        from neuroml import TimedSynapticInput, Spike

                        tsi = TimedSynapticInput(id=comp_id, synapse=nest_syn.id, spike_target="./%s"%nest_syn.id)
                        nml_doc.timed_synaptic_inputs.append(tsi)
                        for ti in range(len(info['times'])):
                            tsi.spikes.append(Spike(id=ti, time='%sms'%info['times'][ti]))
                elif input_type == 'current_clamp':
                    from neuroml import PulseGenerator
                    for comp_id in self.input_comp_info[input][input_type]:
                        info = self.input_comp_info[input][input_type][comp_id]
                        #TODO remove when https://github.com/AllenInstitute/sonata/issues/42 is fixed!
                        amp_template = '%spA' if is_nest else '%snA' #
                        pg = PulseGenerator(id=comp_id,delay='%sms'%info['delay'],duration='%sms'%info['duration'],amplitude=amp_template%info['amp'])
                        nml_doc.pulse_generators.append(pg)


Instruction:
Write a Python 3 function that can
generate a LEMS file to use in simulations of the NeuroML file.

Response:
def generate_lems_file(self, nml_file_name, nml_doc):
        """
            Generate a LEMS file to use in simulations of the NeuroML file
        """

        #pp.pprint(self.simulation_config)
        #pp.pprint(self.pop_comp_info)
        #pp.pprint(self.node_set_mappings)

        if 'output' in self.simulation_config:
            gen_spike_saves_for_all_somas = True

        target = nml_doc.networks[0].id
        sim_id = 'Sim_%s'%target

        duration = self.simulation_config['run']['tstop']
        dt = self.simulation_config['run']['dt']
        lems_file_name = 'LEMS_%s.xml'%sim_id
        target_dir = "./"
        gen_saves_for_quantities = {}
        gen_plots_for_quantities = {}

        if 'reports' in self.simulation_config:
            if 'membrane_potential' in self.simulation_config['reports']:
                mp = self.simulation_config['reports']['membrane_potential']
                node_set = self.node_set_mappings[mp['cells']]
                for nml_pop in node_set:
                    comp = self.nml_pop_vs_comps[nml_pop]
                    ids = node_set[nml_pop]
                    display = 'Voltages_%s'%nml_pop
                    file_name = '%s.v.dat'%nml_pop

                    for id in ids:
                        quantity = '%s/%i/%s/%s'%(nml_pop,id,comp,'v')
                        if not nml_pop in self.nml_pops_having_locations:
                            quantity = '%s[%i]/%s'%(nml_pop,id,'v')

                        if not display in gen_plots_for_quantities:
                            gen_plots_for_quantities[display] = []
                        gen_plots_for_quantities[display].append(quantity)
                        if not file_name in gen_saves_for_quantities:
                            gen_saves_for_quantities[file_name] = []
                        gen_saves_for_quantities[file_name].append(quantity)

        generate_lems_file_for_neuroml(sim_id,
                                       nml_file_name,
                                       target,
                                       duration,
                                       dt,
                                       lems_file_name,
                                       target_dir,
                                       include_extra_files = self.nml_includes,
                                       gen_plots_for_all_v = False,
                                       plot_all_segments = False,
                                       gen_plots_for_quantities = gen_plots_for_quantities,   #  Dict with displays vs lists of quantity paths
                                       gen_saves_for_all_v = False,
                                       save_all_segments = False,
                                       gen_saves_for_quantities = gen_saves_for_quantities,  #  List of populations, all pops if = []
                                       gen_spike_saves_for_all_somas = gen_spike_saves_for_all_somas,
                                       report_file_name = REPORT_FILE,
                                       copy_neuroml = True,
                                       verbose=True)

        return lems_file_name


Instruction:
Can you tell what is the following Python 3 function doing
def clip_geometry_to_srs_bounds(geometry, pyramid, multipart=False):
    if not geometry.is_valid:
        raise ValueError("invalid geometry given")
    pyramid_bbox = box(*pyramid.bounds)

    # Special case for global tile pyramids if geometry extends over tile
    # pyramid boundaries (such as the antimeridian).
    if pyramid.is_global and not geometry.within(pyramid_bbox):
        inside_geom = geometry.intersection(pyramid_bbox)
        outside_geom = geometry.difference(pyramid_bbox)
        # shift outside geometry so it lies within SRS bounds
        if isinstance(outside_geom, Polygon):
            outside_geom = [outside_geom]
        all_geoms = [inside_geom]
        for geom in outside_geom:
            geom_bounds = Bounds(*geom.bounds)
            if geom_bounds.left < pyramid.left:
                geom = translate(geom, xoff=2*pyramid.right)
            elif geom_bounds.right > pyramid.right:
                geom = translate(geom, xoff=-2*pyramid.right)
            all_geoms.append(geom)
        if multipart:
            return all_geoms
        else:
            return GeometryCollection(all_geoms)

    else:
        if multipart:
            return [geometry]
        else:
            return geometry

Response: Clip input geometry to SRS bounds of given TilePyramid.


Instruction:
Can you generate the documentation for the following Python 3 function
def snap_bounds(bounds=None, tile_pyramid=None, zoom=None, pixelbuffer=0):
    bounds = Bounds(*bounds)
    validate_zoom(zoom)
    lb = _tile_from_xy(tile_pyramid, bounds.left, bounds.bottom, zoom, on_edge_use="rt")
    rt = _tile_from_xy(tile_pyramid, bounds.right, bounds.top, zoom, on_edge_use="lb")
    left, bottom, _, _ = lb.bounds(pixelbuffer)
    _, _, right, top = rt.bounds(pixelbuffer)
    return Bounds(left, bottom, right, top)

Response: Extend bounds to be aligned with union of tile bboxes.


Instruction:
Create a Python 3 function for
verifying that shape corresponds to bounds apect ratio.

Response:
def _verify_shape_bounds(shape, bounds):
    """Verify that shape corresponds to bounds apect ratio."""
    if not isinstance(shape, (tuple, list)) or len(shape) != 2:
        raise TypeError(
            "shape must be a tuple or list with two elements: %s" % str(shape)
        )
    if not isinstance(bounds, (tuple, list)) or len(bounds) != 4:
        raise TypeError(
            "bounds must be a tuple or list with four elements: %s" % str(bounds)
        )
    shape = Shape(*shape)
    bounds = Bounds(*bounds)
    shape_ratio = shape.width / shape.height
    bounds_ratio = (bounds.right - bounds.left) / (bounds.top - bounds.bottom)
    if abs(shape_ratio - bounds_ratio) > DELTA:
        min_length = min([
            (bounds.right - bounds.left) / shape.width,
            (bounds.top - bounds.bottom) / shape.height
        ])
        proposed_bounds = Bounds(
            bounds.left,
            bounds.bottom,
            bounds.left + shape.width * min_length,
            bounds.bottom + shape.height * min_length
        )
        raise ValueError(
            "shape ratio (%s) must equal bounds ratio (%s); try %s" % (
                shape_ratio, bounds_ratio, proposed_bounds
            )
        )


Instruction:
Write a Python 3 script to
return all tiles from tilepyramid intersecting with tile.

Response:
def _tile_intersecting_tilepyramid(tile, tp):
    """Return all tiles from tilepyramid intersecting with tile."""
    if tile.tp.grid != tp.grid:
        raise ValueError("Tile and TilePyramid source grids must be the same.")
    tile_metatiling = tile.tile_pyramid.metatiling
    pyramid_metatiling = tp.metatiling
    multiplier = tile_metatiling / pyramid_metatiling
    if tile_metatiling > pyramid_metatiling:
        return [
            tp.tile(
                tile.zoom,
                int(multiplier) * tile.row + row_offset,
                int(multiplier) * tile.col + col_offset
            )
            for row_offset, col_offset in product(
                range(int(multiplier)), range(int(multiplier))
            )
        ]
    elif tile_metatiling < pyramid_metatiling:
        return [tp.tile(
            tile.zoom, int(multiplier * tile.row), int(multiplier * tile.col)
        )]
    else:
        return [tp.tile(*tile.id)]


Instruction:
Can you create a Python 3 function that
yields all tiles that overlap the given bounds.

Response:
def _global_tiles_from_bounds(tp, bounds, zoom):
    """Return also Tiles if bounds cross the antimeridian."""
    seen = set()

    # clip to tilepyramid top and bottom bounds
    left, right = bounds.left, bounds.right
    top = tp.top if bounds.top > tp.top else bounds.top
    bottom = tp.bottom if bounds.bottom < tp.bottom else bounds.bottom

    if left >= tp.left and right <= tp.right:
        for tile in _tiles_from_cleaned_bounds(tp, bounds, zoom):
            yield tile

    # bounds overlap on the Western side with antimeridian
    if left < tp.left:
        for tile in chain(
            # tiles west of antimeridian
            _tiles_from_cleaned_bounds(
                tp,
                Bounds(left + (tp.right - tp.left), bottom, tp.right, top),
                zoom
            ),
            # tiles east of antimeridian
            _tiles_from_cleaned_bounds(
                tp, Bounds(tp.left, bottom, right, top), zoom
            )
        ):
            # make output tiles unique
            if tile.id not in seen:
                seen.add(tile.id)
                yield tile

    # bounds overlap on the Eastern side with antimeridian
    if right > tp.right:
        for tile in chain(
            # tiles west of antimeridian
            _tiles_from_cleaned_bounds(
                tp, Bounds(left, bottom, tp.right, top), zoom
            ),
            # tiles east of antimeridian
            _tiles_from_cleaned_bounds(
                tp,
                Bounds(tp.left, bottom, right - (tp.right - tp.left), top),
                zoom
            )
        ):
            # make output tiles unique
            if tile.id not in seen:
                seen.add(tile.id)
                yield tile


Instruction:
Can you generate a brief explanation for the following Python 3 code
def _tiles_from_cleaned_bounds(tp, bounds, zoom):
    lb = _tile_from_xy(tp, bounds.left, bounds.bottom, zoom, on_edge_use="rt")
    rt = _tile_from_xy(tp, bounds.right, bounds.top, zoom, on_edge_use="lb")
    for tile_id in product([zoom], range(rt.row, lb.row + 1), range(lb.col, rt.col + 1)):
        yield tp.tile(*tile_id)

Response: Yield all tiles intersecting with bounds.


Instruction:
Can you generate a brief explanation for the following Python 3 code
def open_args(subparsers):
    parser = subparsers.add_parser(
        'open',
        help='open a CSV resoruce with a system application',
        description=open_args.__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )

    parser.set_defaults(run_command=open_cmd)

    parser.add_argument('metatabfile', nargs='?',
                        help="Path or URL to a metatab file. If not provided, defaults to 'metadata.csv' ")

    return parser

Response: The mp open command will open a CSV resoruce with a system application such as Excel or OpenOffice.


Instruction:
Implement a function in Python 3 to
produce a shortest path tree for a given net using NER. This is the kernel of the NER algorithm. Parameters ---------- source : (x, y) The coordinate of the source vertex. destinations : iterable([(x, y), ...]) The coordinates of destination vertices. width : int Width of the system (nodes) height : int Height of the system (nodes) wrap_around : bool True if wrap-around links should be used, false if they should be avoided. radius : int Radius of area to search from each node. 20 is arbitrarily selected in the paper and shown to be acceptable in practice. Returns ------- (:py:class:`~.rig.place_and_route.routing_tree.RoutingTree`, {(x,y): :py:class:`~.rig.place_and_route.routing_tree.RoutingTree`, ...}) A RoutingTree is produced rooted at the source and visiting all destinations but which does not contain any vertices etc. For convenience, a dictionarry mapping from destination (x, y) coordinates to the associated RoutingTree is provided to allow the caller to insert these items.

Response:
def ner_net(source, destinations, width, height, wrap_around=False, radius=10):
    """Produce a shortest path tree for a given net using NER.

    This is the kernel of the NER algorithm.

    Parameters
    ----------
    source : (x, y)
        The coordinate of the source vertex.
    destinations : iterable([(x, y), ...])
        The coordinates of destination vertices.
    width : int
        Width of the system (nodes)
    height : int
        Height of the system (nodes)
    wrap_around : bool
        True if wrap-around links should be used, false if they should be
        avoided.
    radius : int
        Radius of area to search from each node. 20 is arbitrarily selected in
        the paper and shown to be acceptable in practice.

    Returns
    -------
    (:py:class:`~.rig.place_and_route.routing_tree.RoutingTree`,
     {(x,y): :py:class:`~.rig.place_and_route.routing_tree.RoutingTree`, ...})
        A RoutingTree is produced rooted at the source and visiting all
        destinations but which does not contain any vertices etc. For
        convenience, a dictionarry mapping from destination (x, y) coordinates
        to the associated RoutingTree is provided to allow the caller to insert
        these items.
    """
    # Map from (x, y) to RoutingTree objects
    route = {source: RoutingTree(source)}

    # Handle each destination, sorted by distance from the source, closest
    # first.
    for destination in sorted(destinations,
                              key=(lambda destination:
                                   shortest_mesh_path_length(
                                       to_xyz(source), to_xyz(destination))
                                   if not wrap_around else
                                   shortest_torus_path_length(
                                       to_xyz(source), to_xyz(destination),
                                       width, height))):
        # We shall attempt to find our nearest neighbouring placed node.
        neighbour = None

        # Try to find a nearby (within radius hops) node in the routing tree
        # that we can route to (falling back on just routing to the source).
        #
        # In an implementation according to the algorithm's original
        # specification looks for nodes at each point in a growing set of rings
        # of concentric hexagons. If it doesn't find any destinations this
        # means an awful lot of checks: 1261 for the default radius of 20.
        #
        # An alternative (but behaviourally identical) implementation scans the
        # list of all route nodes created so far and finds the closest node
        # which is < radius hops (falling back on the origin if no node is
        # closer than radius hops).  This implementation requires one check per
        # existing route node. In most routes this is probably a lot less than
        # 1261 since most routes will probably have at most a few hundred route
        # nodes by the time the last destination is being routed.
        #
        # Which implementation is best is a difficult question to answer:
        # * In principle nets with quite localised connections (e.g.
        #   nearest-neighbour or centroids traffic) may route slightly more
        #   quickly with the original algorithm since it may very quickly find
        #   a neighbour.
        # * In nets which connect very spaced-out destinations the second
        #   implementation may be quicker since in such a scenario it is
        #   unlikely that a neighbour will be found.
        # * In extremely high-fan-out nets (e.g. broadcasts), the original
        #   method is very likely to perform *far* better than the alternative
        #   method since most iterations will complete immediately while the
        #   alternative method must scan *all* the route vertices.
        # As such, it should be clear that neither method alone is 'best' and
        # both have degenerate performance in certain completely reasonable
        # styles of net. As a result, a simple heuristic is used to decide
        # which technique to use.
        #
        # The following micro-benchmarks are crude estimate of the
        # runtime-per-iteration of each approach (at least in the case of a
        # torus topology)::
        #
        #     $ # Original approach
        #     $ python -m timeit --setup 'x, y, w, h, r = 1, 2, 5, 10, \
        #                                     {x:None for x in range(10)}' \
        #                        'x += 1; y += 1; x %= w; y %= h; (x, y) in r'
        #     1000000 loops, best of 3: 0.207 usec per loop
        #     $ # Alternative approach
        #     $ python -m timeit --setup 'from rig.geometry import \
        #                                 shortest_torus_path_length' \
        #                        'shortest_torus_path_length( \
        #                             (0, 1, 2), (3, 2, 1), 10, 10)'
        #     1000000 loops, best of 3: 0.666 usec per loop
        #
        # From this we can approximately suggest that the alternative approach
        # is 3x more expensive per iteration. A very crude heuristic is to use
        # the original approach when the number of route nodes is more than
        # 1/3rd of the number of routes checked by the original method.
        concentric_hexagons = memoized_concentric_hexagons(radius)
        if len(concentric_hexagons) < len(route) / 3:
            # Original approach: Start looking for route nodes in a concentric
            # spiral pattern out from the destination node.
            for x, y in concentric_hexagons:
                x += destination[0]
                y += destination[1]
                if wrap_around:
                    x %= width
                    y %= height
                if (x, y) in route:
                    neighbour = (x, y)
                    break
        else:
            # Alternative approach: Scan over every route node and check to see
            # if any are < radius, picking the closest one if so.
            neighbour = None
            neighbour_distance = None
            for candidate_neighbour in route:
                if wrap_around:
                    distance = shortest_torus_path_length(
                        to_xyz(candidate_neighbour), to_xyz(destination),
                        width, height)
                else:
                    distance = shortest_mesh_path_length(
                        to_xyz(candidate_neighbour), to_xyz(destination))

                if distance <= radius and (neighbour is None or
                                           distance < neighbour_distance):
                    neighbour = candidate_neighbour
                    neighbour_distance = distance

        # Fall back on routing directly to the source if no nodes within radius
        # hops of the destination was found.
        if neighbour is None:
            neighbour = source

        # Find the shortest vector from the neighbour to this destination
        if wrap_around:
            vector = shortest_torus_path(to_xyz(neighbour),
                                         to_xyz(destination),
                                         width, height)
        else:
            vector = shortest_mesh_path(to_xyz(neighbour), to_xyz(destination))

        # The longest-dimension-first route may inadvertently pass through an
        # already connected node. If the route is allowed to pass through that
        # node it would create a cycle in the route which would be VeryBad(TM).
        # As a result, we work backward through the route and truncate it at
        # the first point where the route intersects with a connected node.
        ldf = longest_dimension_first(vector, neighbour, width, height)
        i = len(ldf)
        for direction, (x, y) in reversed(ldf):
            i -= 1
            if (x, y) in route:
                # We've just bumped into a node which is already part of the
                # route, this becomes our new neighbour and we truncate the LDF
                # route. (Note ldf list is truncated just after the current
                # position since it gives (direction, destination) pairs).
                neighbour = (x, y)
                ldf = ldf[i + 1:]
                break

        # Take the longest dimension first route.
        last_node = route[neighbour]
        for direction, (x, y) in ldf:
            this_node = RoutingTree((x, y))
            route[(x, y)] = this_node

            last_node.children.append((Routes(direction), this_node))
            last_node = this_node

    return (route[source], route)


Instruction:
Write a Python 3 script to
copy a RoutingTree and disconnects nodes which are not connected in the given machine.

Response:
def copy_and_disconnect_tree(root, machine):
    """Copy a RoutingTree (containing nothing but RoutingTrees), disconnecting
    nodes which are not connected in the machine.

    Note that if a dead chip is part of the input RoutingTree, no corresponding
    node will be included in the copy. The assumption behind this is that the
    only reason a tree would visit a dead chip is because a route passed
    through the chip and wasn't actually destined to arrive at that chip. This
    situation is impossible to confirm since the input routing trees have not
    yet been populated with vertices. The caller is responsible for being
    sensible.

    Parameters
    ----------
    root : :py:class:`~rig.place_and_route.routing_tree.RoutingTree`
        The root of the RoutingTree that contains nothing but RoutingTrees
        (i.e. no children which are vertices or links).
    machine : :py:class:`~rig.place_and_route.Machine`
        The machine in which the routes exist.

    Returns
    -------
    (root, lookup, broken_links)
        Where:
        * `root` is the new root of the tree
          :py:class:`~rig.place_and_route.routing_tree.RoutingTree`
        * `lookup` is a dict {(x, y):
          :py:class:`~rig.place_and_route.routing_tree.RoutingTree`, ...}
        * `broken_links` is a set ([(parent, child), ...]) containing all
          disconnected parent and child (x, y) pairs due to broken links.
    """
    new_root = None

    # Lookup for copied routing tree {(x, y): RoutingTree, ...}
    new_lookup = {}

    # List of missing connections in the copied routing tree [(new_parent,
    # new_child), ...]
    broken_links = set()

    # A queue [(new_parent, direction, old_node), ...]
    to_visit = deque([(None, None, root)])
    while to_visit:
        new_parent, direction, old_node = to_visit.popleft()

        if old_node.chip in machine:
            # Create a copy of the node
            new_node = RoutingTree(old_node.chip)
            new_lookup[new_node.chip] = new_node
        else:
            # This chip is dead, move all its children into the parent node
            assert new_parent is not None, \
                "Net cannot be sourced from a dead chip."
            new_node = new_parent

        if new_parent is None:
            # This is the root node
            new_root = new_node
        elif new_node is not new_parent:
            # If this node is not dead, check connectivity to parent node (no
            # reason to check connectivity between a dead node and its parent).
            if direction in links_between(new_parent.chip,
                                          new_node.chip,
                                          machine):
                # Is connected via working link
                new_parent.children.append((direction, new_node))
            else:
                # Link to parent is dead (or original parent was dead and the
                # new parent is not adjacent)
                broken_links.add((new_parent.chip, new_node.chip))

        # Copy children
        for child_direction, child in old_node.children:
            to_visit.append((new_node, child_direction, child))

    return (new_root, new_lookup, broken_links)


Instruction:
Can you write a function in Python 3 where it
uses A* to find a path from any of the sources to the sink.

Response:
def a_star(sink, heuristic_source, sources, machine, wrap_around):
    """Use A* to find a path from any of the sources to the sink.

    Note that the heuristic means that the search will proceed towards
    heuristic_source without any concern for any other sources. This means that
    the algorithm may miss a very close neighbour in order to pursue its goal
    of reaching heuristic_source. This is not considered a problem since 1) the
    heuristic source will typically be in the direction of the rest of the tree
    and near by and often the closest entity 2) it prevents us accidentally
    forming loops in the rest of the tree since we'll stop as soon as we touch
    any part of it.

    Parameters
    ----------
    sink : (x, y)
    heuristic_source : (x, y)
        An element from `sources` which is used as a guiding heuristic for the
        A* algorithm.
    sources : set([(x, y), ...])
    machine : :py:class:`~rig.place_and_route.Machine`
    wrap_around : bool
        Consider wrap-around links in heuristic distance calculations.

    Returns
    -------
    [(:py:class:`~rig.routing_table.Routes`, (x, y)), ...]
        A path starting with a coordinate in `sources` and terminating at
        connected neighbour of `sink` (i.e. the path does not include `sink`).
        The direction given is the link down which to proceed from the given
        (x, y) to arrive at the next point in the path.

    Raises
    ------
    :py:class:~rig.place_and_route.exceptions.MachineHasDisconnectedSubregion`
        If a path cannot be found.
    """
    # Select the heuristic function to use for distances
    if wrap_around:
        heuristic = (lambda node:
                     shortest_torus_path_length(to_xyz(node),
                                                to_xyz(heuristic_source),
                                                machine.width, machine.height))
    else:
        heuristic = (lambda node:
                     shortest_mesh_path_length(to_xyz(node),
                                               to_xyz(heuristic_source)))

    # A dictionary {node: (direction, previous_node}. An entry indicates that
    # 1) the node has been visited and 2) which node we hopped from (and the
    # direction used) to reach previous_node.  This may be None if the node is
    # the sink.
    visited = {sink: None}

    # The node which the tree will be reconnected to
    selected_source = None

    # A heap (accessed via heapq) of (distance, (x, y)) where distance is the
    # distance between (x, y) and heuristic_source and (x, y) is a node to
    # explore.
    to_visit = [(heuristic(sink), sink)]
    while to_visit:
        _, node = heapq.heappop(to_visit)

        # Terminate if we've found the destination
        if node in sources:
            selected_source = node
            break

        # Try all neighbouring locations. Note: link identifiers are from the
        # perspective of the neighbour, not the current node!
        for neighbour_link in Links:
            vector = neighbour_link.opposite.to_vector()
            neighbour = ((node[0] + vector[0]) % machine.width,
                         (node[1] + vector[1]) % machine.height)

            # Skip links which are broken
            if (neighbour[0], neighbour[1], neighbour_link) not in machine:
                continue

            # Skip neighbours who have already been visited
            if neighbour in visited:
                continue

            # Explore all other neighbours
            visited[neighbour] = (neighbour_link, node)
            heapq.heappush(to_visit, (heuristic(neighbour), neighbour))

    # Fail of no paths exist
    if selected_source is None:
        raise MachineHasDisconnectedSubregion(
            "Could not find path from {} to {}".format(
                sink, heuristic_source))

    # Reconstruct the discovered path, starting from the source we found and
    # working back until the sink.
    path = [(Routes(visited[selected_source][0]), selected_source)]
    while visited[path[-1][1]][1] != sink:
        node = visited[path[-1][1]][1]
        direction = Routes(visited[node][0])
        path.append((direction, node))

    return path