def StartTaskStorage(self):
    if (self._storage_type != definitions.STORAGE_TYPE_SESSION):
        raise IOError('Unsupported storage type.')
    if self._task_storage_path:
        raise IOError('Task storage path already exists.')
    output_directory = os.path.dirname(self._output_file)
    self._task_storage_path = tempfile.mkdtemp(dir=output_directory)
    self._merge_task_storage_path = os.path.join(self._task_storage_path, 'merge')
    os.mkdir(self._merge_task_storage_path)
    self._processed_task_storage_path = os.path.join(self._task_storage_path, 'processed')
    os.mkdir(self._processed_task_storage_path)

Creates a temporary path for the task storage.

Raises:
IOError: if the storage type is not supported or
if the temporary path for the task storage already exists.
OSError: if the storage type is not supported or
if the temporary path for the task storage already exists.


def mrca(self, labels):

        if not isinstance(labels,set):
            try:
                labels = set(labels)
            except:
                raise TypeError("labels must be iterable")
        l2n = self.label_to_node(labels)
        count = dict()
        for node in l2n.values():
            for a in node.traverse_ancestors():
                if a not in count:
                    count[a] = 0
                count[a] += 1
                if count[a] == len(l2n):
                    return a
        raise RuntimeError("There somehow does not exist an MRCA for the given labels")

Return the Node that is the MRCA of the nodes labeled by a label in ``labels``. If multiple nodes are labeled by a given label, only the last (preorder traversal) will be obtained

Args:
``labels`` (``set``): Set of leaf labels

Returns:
``Node``: The MRCA of the ``Node`` objects labeled by a label in ``labels``


def _event_to_pb(event):
    if isinstance(event, (TaskData, Task)):
        (key, klass) = ('task', clearly_pb2.TaskMessage)
    elif isinstance(event, (WorkerData, Worker)):
        (key, klass) = ('worker', clearly_pb2.WorkerMessage)
    else:
        raise ValueError('unknown event')
    keys = klass.DESCRIPTOR.fields_by_name.keys()
    data = {k: v for (k, v) in getattr(event, '_asdict', (lambda : {f: getattr(event, f) for f in event._fields}))().items() if (k in keys)}
    return (key, klass(**data))

Supports converting internal TaskData and WorkerData, as well as
celery Task and Worker to proto buffers messages.

Args:
event (Union[TaskData|Task|WorkerData|Worker]):

Returns:
ProtoBuf object


def get_aws_regions(*, force=False):
    from cloud_inquisitor.config import dbconfig
    global __regions
    if (force or (not __regions)):
        logger.debug('Loading list of AWS regions from static data')
        data = requests.get('https:
        rgx = re.compile(dbconfig.get('ignored_aws_regions_regexp', default='(^cn-|GLOBAL|-gov)'), re.I)
        __regions = sorted(list({x['region'] for x in data['prefixes'] if (not rgx.search(x['region']))}))
    return __regions

Load a list of AWS regions from the AWS static data.

Args:
force (`bool`): Force fetch list of regions even if we already have a cached version

Returns:
:obj:`list` of `str`


def get_vasp_input(self, vasp_input_set=MPRelaxSet, **kwargs):
    d = vasp_input_set(self.final_structure, **kwargs).get_vasp_input()
    d['transformations.json'] = json.dumps(self.as_dict())
    return d

Returns VASP input as a dict of vasp objects.

Args:
vasp_input_set (pymatgen.io.vaspio_set.VaspInputSet): input set
to create vasp input files from structures


def invite_user(self, room_id, user_id):

        body = {
            "user_id": user_id
        }
        return self._send("POST", "/rooms/" + room_id + "/invite", body)

Perform POST /rooms/$room_id/invite

Args:
room_id (str): The room ID
user_id (str): The user ID of the invitee


def compare_mim_panels(self, existing_panel, new_panel):

        existing_genes = set([gene['hgnc_id'] for gene in existing_panel['genes']])
        new_genes = set([gene['hgnc_id'] for gene in new_panel['genes']])

        return new_genes.difference(existing_genes)

Check if the latest version of OMIM differs from the most recent in database
Return all genes that where not in the previous version.

Args:
existing_panel(dict)
new_panel(dict)

Returns:
new_genes(set(str))


def strace_set_buffer_size(self, size):

        size = ctypes.c_uint32(size)
        res = self._dll.JLINK_STRACE_Control(enums.JLinkStraceCommand.SET_BUFFER_SIZE, size)
        if res < 0:
            raise errors.JLinkException('Failed to set the STRACE buffer size.')

        return None

Sets the STRACE buffer size.

Args:
self (JLink): the ``JLink`` instance.

Returns:
``None``

Raises:
JLinkException: on error.


def paths_to_str(paths, format_kwargs={}, delimiter=os.pathsep, asset_paths=False, check_paths=False):
    if (not paths):
        return ''
    if isinstance(paths, str):
        paths = paths.split(delimiter)
    processed_paths = []
    for path in paths:
        original = path
        path = path.format_map(format_kwargs)
        if (not os.path.isabs(path)):
            if (asset_paths and (':' in path)):
                try:
                    path = asset_path(path)
                except ValueError:
                    path = None
        if ((path is not None) and os.path.isdir(path)):
            processed_paths.append(path)
        elif check_paths:
            f = locals()
            printer.warning('Path does not exist: {path} (from {original})'.format_map(f))
    return delimiter.join(processed_paths)

Convert ``paths`` to a single string.

Args:
paths (str|list): A string like "/a/path:/another/path" or
a list of paths; may include absolute paths and/or asset
paths; paths that are relative will be left relative
format_kwargs (dict): Will be injected into each path
delimiter (str): The string used to separate paths
asset_paths (bool): Whether paths that look like asset paths
will be converted to absolute paths
check_paths (bool): Whether paths should be checked to ensure
they exist


def pad_shape_right_with_ones(x, ndims):
    if (not (isinstance(ndims, int) and (ndims >= 0))):
        raise ValueError('`ndims` must be a Python `integer` greater than zero. Got: {}'.format(ndims))
    if (ndims == 0):
        return x
    x = tf.convert_to_tensor(value=x)
    original_shape = x.shape
    new_shape = distribution_util.pad(tf.shape(input=x), axis=0, back=True, value=1, count=ndims)
    x = tf.reshape(x, new_shape)
    x.set_shape(original_shape.concatenate(([1] * ndims)))
    return x

Maybe add `ndims` ones to `x.shape` on the right.

If `ndims` is zero, this is a no-op; otherwise, we will create and return a
new `Tensor` whose shape is that of `x` with `ndims` ones concatenated on the
right side. If the shape of `x` is known statically, the shape of the return
value will be as well.

Args:
x: The `Tensor` we'll return a reshaping of.
ndims: Python `integer` number of ones to pad onto `x.shape`.
Returns:
If `ndims` is zero, `x`; otherwise, a `Tensor` whose shape is that of `x`
with `ndims` ones concatenated on the right side. If possible, returns a
`Tensor` whose shape is known statically.
Raises:
ValueError: if `ndims` is not a Python `integer` greater than or equal to
zero.


def _raised_cosine_window(name, default_name, window_length, periodic, dtype, a, b):
    if not dtype.is_floating:
        raise ValueError('dtype must be a floating point type. Found %s' % dtype)
    with ops.name_scope(name, default_name, [window_length, periodic]):
        window_length = ops.convert_to_tensor(window_length, dtype=dtypes.int32, name='window_length')
        window_length.shape.assert_has_rank(0)
        window_length_const = tensor_util.constant_value(window_length)
        if window_length_const == 1:
            return array_ops.ones([1], dtype=dtype)
        periodic = math_ops.cast(ops.convert_to_tensor(periodic, dtype=dtypes.bool, name='periodic'), dtypes.int32)
        periodic.shape.assert_has_rank(0)
        even = 1 - math_ops.mod(window_length, 2)
        n = math_ops.cast(window_length + periodic * even - 1, dtype=dtype)
        count = math_ops.cast(math_ops.range(window_length), dtype)
        cos_arg = constant_op.constant(2 * np.pi, dtype=dtype) * count / n
        if window_length_const is not None:
            return math_ops.cast(a - b * math_ops.cos(cos_arg), dtype=dtype)
        return cond.cond(math_ops.equal(window_length, 1), lambda: array_ops.ones([window_length], dtype=dtype), lambda: math_ops.cast(a - b * math_ops.cos(cos_arg), dtype=dtype))

Helper function for computing a raised cosine window.

Args:
name: Name to use for the scope.
default_name: Default name to use for the scope.
window_length: A scalar `Tensor` or integer indicating the window length.
periodic: A bool `Tensor` indicating whether to generate a periodic or
symmetric window.
dtype: A floating point `DType`.
a: The alpha parameter to the raised cosine window.
b: The beta parameter to the raised cosine window.

Returns:
A `Tensor` of shape `[window_length]` of type `dtype`.

Raises:
ValueError: If `dtype` is not a floating point type or `window_length` is
not scalar or `periodic` is not scalar.


def calc_stats(prices):
    if isinstance(prices, pd.Series):
        return PerformanceStats(prices)
    elif isinstance(prices, pd.DataFrame):
        return GroupStats(*[prices[x] for x in prices.columns])
    else:
        raise NotImplementedError('Unsupported type')

Calculates performance stats of a given object.

If object is Series, a PerformanceStats object is
returned. If object is DataFrame, a GroupStats object
is returned.

Args:
* prices (Series, DataFrame): Set of prices


def _change_state_prompt(mgr):

    cmd = ''
    book_range = range(1, len(mgr.books) + 1)
    ind_to_book = dict(zip(book_range, mgr.books))
    get_book = lambda cmd_str: ind_to_book[int(cmd_str.split()[1])]
    while cmd != 'q':
        print 'Books:'
        for i in book_range:
            print '\t%d: %s' % (i, ind_to_book[i])
        print 'Commands:'
        print '| start {
        print '| finish {
        print '| q           | Quit'
        cmd = safe_raw_input('> ')
        if cmd is None or cmd == 'q':
            break
        elif cmd.startswith('start '):
            book = get_book(cmd)
            initial_progress = mgr.progress[book.asin].locs[1]
            event = SetReadingEvent(book.asin, initial_progress)
        elif cmd.startswith('finish '):
            event = SetFinishedEvent(get_book(cmd).asin)
        else:
            print 'Invalid command'
            event = None
        if event is not None:
            print
            print 'REGISTERED EVENT:'
            print '  ' + str(event)
            mgr.register_events((event))
        print

Runs a prompt to change the state of books.

Registers `Event`s with `mgr` as they are requested.

Args:
mgr: A `KindleProgressMgr` object with the `books` and `progress`
fields populated.


def convert_padding(params, w_name, scope_name, inputs, layers, weights, names):
    print('Converting padding...')
    if (params['mode'] == 'constant'):
        if (params['value'] != 0.0):
            raise AssertionError('Cannot convert non-zero padding')
        if names:
            tf_name = ('PADD' + random_string(4))
        else:
            tf_name = (w_name + str(random.random()))
        padding_name = tf_name
        padding_layer = keras.layers.ZeroPadding2D(padding=((params['pads'][2], params['pads'][6]), (params['pads'][3], params['pads'][7])), name=padding_name)
        layers[scope_name] = padding_layer(layers[inputs[0]])
    elif (params['mode'] == 'reflect'):

        def target_layer(x, pads=params['pads']):
            layer = tf.pad(x, [[0, 0], [0, 0], [pads[2], pads[6]], [pads[3], pads[7]]], 'REFLECT')
            return layer
        lambda_layer = keras.layers.Lambda(target_layer)
        layers[scope_name] = lambda_layer(layers[inputs[0]])

Convert padding layer.

Args:
params: dictionary with layer parameters
w_name: name prefix in state_dict
scope_name: pytorch scope name
inputs: pytorch node inputs
layers: dictionary with keras tensors
weights: pytorch state_dict
names: use short names for keras layers


def configure_engine(self):
    try:
        self.connection.execution_options(isolation_level='SERIALIZABLE')
    except sa.exc.ArgumentError:
        LOG.debug('Unable to set isolation level to SERIALIZABLE')
    return True

Configure the databse connection.

Sets appropriate transaction isolation levels and handle errors.

Returns:
True, if we did not encounter any unrecoverable errors, else False.


def __init__(self,
               comments=r"
               cont=r"\\\s*\n",
               ml_quote=False,
               quot=(r"\"", r"'"),
               sep=r"[ \t\f\v]+",
               term=r"[\r\n]",
               verbose=0):
    r
    super(FieldParser, self).__init__()
    self.entries = []
    self.fields = []
    self.field = ""
    self.comments = AsIter(comments)
    self.cont = AsIter(cont)
    self.ml_quote = AsIter(ml_quote)
    self.quot = AsIter(quot)
    self.sep = AsIter(sep)
    self.term = AsIter(term)
    self.verbose = verbose
    self._GenStates()

r"""A generalized field-based parser.

Handles whitespace, csv etc.

Args:
comments: Line comment patterns (e.g. "#").
cont: Continuation patterns (e.g. "\\").
ml_quote: Boolean flag to allow quoted strings to span lines.
quot: Quotation patterns (e.g. "\\"" or "'").
sep: Field separator patterns (e.g. "[\\s,]").
term: Entry termination patterns (e.g. "\\n").
verbose: Enable verbose mode for the lexer. Useful for debugging.


def __format__(self, format_spec='dd'):

        text = super(Location.__base__, self).__format__(format_spec)

        if self.alt_names:
            return '%s (%s - %s)' % (self.name, ', '.join(self.alt_names),
                                     text)
        else:
            return '%s (%s)' % (self.name, text)

Extended pretty printing for location strings.

Args:
format_spec (str): Coordinate formatting system to use

Returns:
str: Human readable string representation of ``Point`` object

Raises:
ValueError: Unknown value for ``format_spec``


def _allocate_ids(cls, size=None, max=None, parent=None, **ctx_options):

    return cls._allocate_ids_async(size=size, max=max, parent=parent,
                                   **ctx_options).get_result()

Allocates a range of key IDs for this model class.

Args:
size: Number of IDs to allocate. Either size or max can be specified,
not both.
max: Maximum ID to allocate. Either size or max can be specified,
not both.
parent: Parent key for which the IDs will be allocated.
**ctx_options: Context options.

Returns:
A tuple with (start, end) for the allocated range, inclusive.


def restore_utc_timezone(datetime_str: str, original_timezone: str) -> str:
    if datetime_str.endswith('+00:00'):
        if original_timezone == SIMPLE_ZULU or original_timezone == '-00:00':
            return datetime_str.replace('+00:00', original_timezone, 1)
        if original_timezone == 'UTC':
            return datetime_str.replace('+00:00', SIMPLE_ZULU, 1)
    return datetime_str

Restores UTC '+00:00' offset with 'Z' or '-00:00' if necessary.

This is necessary because dateutil will replace the 'Z' and '-00:00' timezones
with '+00:00', which is irreversible.

Args:
datetime_str: The datetime string to examine.
original_timezone: The original timezone string on the primitive.

Returns:
The restored datetime_str. Note that this will remain unchanged if there was
no UTC timezone replacement that occurred.


def GetVolumeByIndex(self, volume_index):

    if not self._is_parsed:
      self._Parse()
      self._is_parsed = True

    if volume_index < 0 or volume_index >= len(self._volume_identifiers):
      return None

    volume_identifier = self._volume_identifiers[volume_index]
    return self._volumes[volume_identifier]

Retrieves a specific volume based on the index.

Args:
volume_index (int): index of the volume.

Returns:
Volume: a volume or None if not available.


def get_pkg_id(pkg):

    pkg = _quote(pkg)
    package_ids = []

    temp_dir = __salt__['temp.dir'](prefix='pkg-')

    try:

        cmd = 'xar -t -f {0} | grep PackageInfo'.format(pkg)
        out = __salt__['cmd.run'](cmd, python_shell=True, output_loglevel='quiet')
        files = out.split('\n')

        if 'Error opening' not in out:

            cmd = 'xar -x -f {0} {1}'.format(pkg, ' '.join(files))
            __salt__['cmd.run'](cmd, cwd=temp_dir, output_loglevel='quiet')

            for f in files:
                i = _get_pkg_id_from_pkginfo(os.path.join(temp_dir, f))
                if i:
                    package_ids.extend(i)
        else:
            package_ids = _get_pkg_id_dir(pkg)

    finally:

        __salt__['file.remove'](temp_dir)

    return package_ids

Attempt to get the package ID from a .pkg file

Args:
pkg (str): The location of the pkg file

Returns:
list: List of all of the package IDs

CLI Example:

.. code-block:: bash

salt '*' macpackage.get_pkg_id /tmp/test.pkg


def from_event(cls, event):

        return cls(
            uuid=event['uuid'],
            job_type=event['job_type'],
            event_type=event['type'],
            queue=event['queue'],
            hostname=event['hostname'],
            pid=event['pid'],
            name=event['name'],
            workflow_id=event['workflow_id'],
            event_time=event['time'],
            duration=event['duration']
        )

Create a JobEvent object from the event dictionary returned by celery.

Args:
event (dict): The dictionary as returned by celery.

Returns:
JobEvent: A fully initialized JobEvent object.


def _all_reduce_sum_gradients(self, grads_and_vars):
    replica_context = tf.distribute.get_replica_context()
    if not replica_context:
        return grads_and_vars
    grads_and_vars = list(grads_and_vars)
    filtered_grads_and_vars = filter_empty_gradients(grads_and_vars)
    if filtered_grads_and_vars:
        grads = [pair[0] for pair in filtered_grads_and_vars]
        reduced = tf.distribute.get_replica_context().all_reduce(tf.distribute.ReduceOp.SUM, grads)
    else:
        reduced = []
    reduced_with_nones = []
    reduced_pos = 0
    for g, v in grads_and_vars:
        if g is None:
            reduced_with_nones.append((None, v))
        else:
            reduced_with_nones.append((reduced[reduced_pos], v))
            reduced_pos += 1
    assert reduced_pos == len(reduced), 'Failed to add all gradients'
    return reduced_with_nones

Returns all-reduced gradients aggregated via summation.

Args:
grads_and_vars: List of (gradient, variable) pairs.

Returns:
List of (gradient, variable) pairs
where gradients have been all-reduced.


def print_summary(num_actions, failed, duration):

    num_failed = len(failed)
    print(.format(
        num_total=num_actions, num_failed=num_failed, elapsed_time=duration))

Print a small summary of the executed plan.

Args:
num_actions (int): Total size of the executed plan.
failed (:obj:`list` of :obj:`actions.Step`): List of failed actions.
duration: Time we spent executing the plan.


def comments(self, case_id=None, variant_id=None, username=None):
    logger.debug('Looking for comments')
    comment_objs = self.query(Comment)
    if case_id:
        comment_objs = comment_objs.filter_by(case_id=case_id)
    if variant_id:
        comment_objs = comment_objs.filter_by(variant_id=variant_id)
    elif case_id:
        comment_objs = comment_objs.filter_by(variant_id=None)
    return comment_objs

Return comments for a case or variant.

Args:
case_id (str): id for a related case
variant_id (Optional[str]): id for a related variant


def DataRefreshRequired(self, path=None, last=None):

    if last is None:
      if path is None:

        raise type_info.TypeValueError("Either 'path' or 'last' must"
                                       " be supplied as an argument.")

      fd = aff4.FACTORY.Open(self.root.Add(path), token=self.token)

      stat_obj = fd.Get(fd.Schema.STAT)
      if stat_obj:
        last = stat_obj.age
      else:
        last = rdfvalue.RDFDatetime(0)

    if last is None:
      return True
    last = last.AsDatetime()

    return datetime.datetime.utcnow() - last > self.max_age_before_refresh

True if we need to update this path from the client.

Args:
path: The path relative to the root to check freshness of.
last: An aff4:last attribute to check freshness of.

At least one of path or last must be supplied.

Returns:
True if the path hasn't been updated in the last
self.max_age_before_refresh seconds, else False.

Raises:
type_info.TypeValueError: If no arguments are supplied.


def search(self, scope, search, **kwargs):
    data = {'scope': scope, 'search': search}
    path = ('/projects/%s/search' % self.get_id())
    return self.manager.gitlab.http_list(path, query_data=data, **kwargs)

Search the project resources matching the provided string.'

Args:
scope (str): Scope of the search
search (str): Search string
**kwargs: Extra options to send to the server (e.g. sudo)

Raises:
GitlabAuthenticationError: If authentication is not correct
GitlabSearchError: If the server failed to perform the request

Returns:
GitlabList: A list of dicts describing the resources found.


def getConstraint(self, name):
    return lock_and_call((lambda : Constraint(self._impl.getConstraint(name))), self._lock)

Get the constraint with the corresponding name.

Args:
name: Name of the constraint to be found.

Raises:
TypeError: if the specified constraint does not exist.


def sharded_save(mesh: layout_lib.Mesh, file_prefix: Union[str, tensor_lib.Tensor], tensor_names: Union[List[str], tensor_lib.Tensor], shape_and_slices: Union[List[str], tensor_lib.Tensor], tensors: List[Union[tensor_lib.Tensor, tf_variables.Variable]]):
    with ops.device(api.device_name()):
        io_ops.save_v2(file_prefix, tensor_names, shape_and_slices, tensors)
    mesh_util.barrier(mesh.host_mesh(), 'SaveV2')
    with api.default_mesh(mesh.host_mesh()):
        merge_op = io_ops.MergeV2Checkpoints(checkpoint_prefixes=[file_prefix], destination_prefix=file_prefix, delete_old_dirs=True)
    mesh_util.barrier(mesh.host_mesh(), 'MergeV2Checkpoints')
    return merge_op

Saves given named tensor slices in a sharded, multi-client safe fashion.

The method makes sure the checkpoint directory state is correct in a sharded
mutli-client saving. Namely, we place a barrier after SaveV2 to make sure
every client has done writing the files. And another one after
MergeV2Checkpoints to make sure all Metadata is properly merged.

Upon existing, the checkpoint is completed and the all directory operations
are done.

Args:
mesh: The Mesh that contains the Tensors to save.
file_prefix: The prefix of checkpoint.
tensor_names: a list of tensor names used in save op.
shape_and_slices: a list of shape and slice specification used in save op.
The only supported value is "" as we don't support distributed saving with
slices yet.
tensors: a list of tensors used in save op. The order should match
tensor_names.

Returns:
A MergeV2Checkpoints op that merged all Metadata.


def write_int64(self, value, little_endian=True):
    if little_endian:
        endian = '<'
    else:
        endian = '>'
    return self.pack(('%sq' % endian), value)

Pack the value as a signed integer and write 8 bytes to the stream.

Args:
value:
little_endian (bool): specify the endianness. (Default) Little endian.

Returns:
int: the number of bytes written.


def get_target(self, target):

        if target not in self._target_cache:
            self._target_cache[target] = self._get_target(target)
        return self._target_cache[target]

Get the result of _get_target, cache it and return it.

Args:
target (str): target to find.

Returns:
Package/Module: package containing target or corresponding module.


def _create_all_weights(self, params):
    raise NotImplementedError

Creates and sets all optimizer weights.

Args:
params: list or tuple of `Variable` objects that will be minimized
using this optimizer.

Returns:
Specific weight values that are used in `get_updates`


def get_external_command_output(command: str) -> bytes:

    args = shlex.split(command)
    ret = subprocess.check_output(args)
    return ret

Takes a command-line command, executes it, and returns its ``stdout``
output.

Args:
command: command string

Returns:
output from the command as ``bytes``


def _tf_data_pack_sequence_as(structure, flat_sequence):
    if not (_tf_data_is_nested(flat_sequence) or isinstance(flat_sequence, list)):
        raise TypeError(f"Argument `flat_sequence` must be a sequence. Got '{type(flat_sequence).__name__}'.")
    if not _tf_data_is_nested(structure):
        if len(flat_sequence) != 1:
            raise ValueError(f'Argument `structure` is a scalar but `len(flat_sequence)`={len(flat_sequence)} > 1')
        return flat_sequence[0]
    flat_structure = _tf_data_flatten(structure)
    if len(flat_structure) != len(flat_sequence):
        raise ValueError(f'Could not pack sequence. Argument `structure` had {len(flat_structure)} elements, but argument `flat_sequence` had {len(flat_sequence)} elements. Received structure: {structure}, flat_sequence: {flat_sequence}.')
    _, packed = _tf_data_packed_nest_with_indices(structure, flat_sequence, 0)
    return sequence_like(structure, packed)

Returns a given flattened sequence packed into a nest.

If `structure` is a scalar, `flat_sequence` must be a single-element list;
in this case the return value is `flat_sequence[0]`.

Args:
structure: tuple or list constructed of scalars and/or other tuples/lists,
or a scalar.  Note: numpy arrays are considered scalars.
flat_sequence: flat sequence to pack.

Returns:
packed: `flat_sequence` converted to have the same recursive structure as
`structure`.

Raises:
ValueError: If nest and structure have different element counts.


def read_file(path):
    gen = textfile.read_separated_lines_generator(path, max_columns=6, ignore_lines_starting_with=[';;'])
    utterances = collections.defaultdict(list)
    for record in gen:
        values = record[1:len(record)]
        for i in range(len(values)):
            if ((i == 1) or (i == 2) or (i == 4)):
                values[i] = float(values[i])
        utterances[record[0]].append(values)
    return utterances


def get_metric_values_response(self):
    mv_list = self.get_metric_values()
    resp_lines = []
    for mv in mv_list:
        group_name = mv[0]
        resp_lines.append('"{}"'.format(group_name))
        mo_vals = mv[1]
        for mo_val in mo_vals:
            resp_lines.append('"{}"'.format(mo_val.resource_uri))
            resp_lines.append(str(timestamp_from_datetime(mo_val.timestamp)))
            v_list = []
            for (n, v) in mo_val.values:
                if isinstance(v, six.string_types):
                    v_str = '"{}"'.format(v)
                else:
                    v_str = str(v)
                v_list.append(v_str)
            v_line = ','.join(v_list)
            resp_lines.append(v_line)
            resp_lines.append('')
        resp_lines.append('')
    resp_lines.append('')
    return ('\n'.join(resp_lines) + '\n')

Get the faked metrics, for all metric groups and all resources that
have been prepared on the manager object of this context object, as a
string in the format needed for the "Get Metrics" operation response.

Returns:

"MetricsResponse" string as described for the "Get Metrics"
operation response.


def datetimeobj_a__d_b_Y_H_M_S_z(value):

    a, d, b, Y, t, z = value.split()
    H, M, S = t.split(":")
    return datetime.datetime(
        int(Y), _months[b.lower()], int(d), int(H), int(M), int(S),
        tzinfo=dateutil.tz.tzoffset(None, _offset(z))
    )

Convert timestamp string to a datetime object.

Timestamps strings like 'Tue, 18 Jun 2013 22:00:00 +1000' are able to be
converted by this function.

Args:
value: A timestamp string in the format '%a, %d %b %Y %H:%M:%S %z'.

Returns:
A datetime object.

Raises:
ValueError: If timestamp is invalid.
KeyError: If the abbrieviated month is invalid.


def approve(self, access_level=gitlab.DEVELOPER_ACCESS, **kwargs):

        path = '%s/%s/approve' % (self.manager.path, self.id)
        data = {'access_level': access_level}
        server_data = self.manager.gitlab.http_put(path, post_data=data,
                                                   **kwargs)
        self._update_attrs(server_data)

Approve an access request.

Args:
access_level (int): The access level for the user
**kwargs: Extra options to send to the server (e.g. sudo)

Raises:
GitlabAuthenticationError: If authentication is not correct
GitlabUpdateError: If the server fails to perform the request


def parse_gptl(file_path, var_list):
    timing_result = dict()
    if os.path.isfile(file_path):
        with open(file_path, 'r') as f:
            for var in var_list:
                for line in f:
                    if (var in line):
                        timing_result[var] = (float(line.split()[4]) / int(line.split()[2]))
                        break
    return timing_result

Read a GPTL timing file and extract some data.

Args:
file_path: the path to the GPTL timing file
var_list: a list of strings to look for in the file

Returns:
A dict containing key-value pairs of the livvkit
and the times associated with them


def default_memcache_timeout_policy(key):
    timeout = None
    if ((key is not None) and isinstance(key, model.Key)):
        modelclass = model.Model._kind_map.get(key.kind())
        if (modelclass is not None):
            policy = getattr(modelclass, '_memcache_timeout', None)
            if (policy is not None):
                if isinstance(policy, (int, long)):
                    timeout = policy
                else:
                    timeout = policy(key)
    return timeout

Default memcache timeout policy.

This defers to _memcache_timeout on the Model class.

Args:
key: Key instance.

Returns:
Memcache timeout to use (integer), or None.


def quote_identifier(identifier: str, mixed: Union[(SQLCompiler, Engine, Dialect)]) -> str:
    return get_preparer(mixed).quote(identifier)

Converts an SQL identifier to a quoted version, via the SQL dialect in
use.

Args:
identifier: the identifier to be quoted
mixed: an SQLAlchemy :class:`SQLCompiler`, :class:`Engine`, or
:class:`Dialect` object

Returns:
the quoted identifier


def learn_one(self, x: beam.Row) -> None:
    if len(x.__dict__) != 1:
        raise ValueError('IQR.learn_one expected univariate input, but got %s', str(x))
    v = next(iter(x))
    self._q1_tracker.push(v)
    self._q3_tracker.push(v)

Updates the quantile trackers with a new data point.

Args:
x: A `beam.Row` containing a single numerical value.


def recent(self):
    url = '{0}/users/ME/conversations'.format(self.skype.conn.msgsHost)
    params = {'startTime': 0, 'view': 'msnp24Equivalent', 'targetType': 'Passport|Skype|Lync|Thread'}
    resp = self.skype.conn.syncStateCall('GET', url, params, auth=SkypeConnection.Auth.RegToken).json()
    chats = {}
    for json in resp.get('conversations', []):
        cls = SkypeSingleChat
        if ('threadProperties' in json):
            info = self.skype.conn('GET', '{0}/threads/{1}'.format(self.skype.conn.msgsHost, json.get('id')), auth=SkypeConnection.Auth.RegToken, params={'view': 'msnp24Equivalent'}).json()
            json.update(info)
            cls = SkypeGroupChat
        chats[json.get('id')] = self.merge(cls.fromRaw(self.skype, json))
    return chats

Retrieve a selection of conversations with the most recent activity, and store them in the cache.

Each conversation is only retrieved once, so subsequent calls will retrieve older conversations.

Returns:
:class:`SkypeChat` list: collection of recent conversations


def _ScanVolumeSystemRootNode(
      self, scan_context, scan_node, auto_recurse=True):

    if scan_node.type_indicator == definitions.TYPE_INDICATOR_VSHADOW:

      path_spec = self.ScanForFileSystem(scan_node.path_spec.parent)
      if path_spec:
        scan_context.AddScanNode(path_spec, scan_node.parent_node)

    file_entry = resolver.Resolver.OpenFileEntry(
        scan_node.path_spec, resolver_context=self._resolver_context)

    for sub_file_entry in file_entry.sub_file_entries:
      sub_scan_node = scan_context.AddScanNode(
          sub_file_entry.path_spec, scan_node)

      if scan_node.type_indicator == definitions.TYPE_INDICATOR_VSHADOW:

        continue

      if auto_recurse or not scan_context.updated:
        self._ScanNode(scan_context, sub_scan_node, auto_recurse=auto_recurse)

Scans a volume system root node for supported formats.

Args:
scan_context (SourceScannerContext): source scanner context.
scan_node (SourceScanNode): source scan node.
auto_recurse (Optional[bool]): True if the scan should automatically
recurse as far as possible.

Raises:
ValueError: if the scan context or scan node is invalid.


def read_main_summary(spark, submission_date_s3=None, sample_id=None, mergeSchema=True, path='s3:
    base_path = path
    reader = spark.read.option('basePath', base_path)
    if mergeSchema:
        reader = reader.option('mergeSchema', 'true')
    if ((submission_date_s3 is not None) and (sample_id is None)):
        paths = ['{}/submission_date_s3={}/'.format(base_path, s) for s in submission_date_s3]
        return reader.parquet(*paths)
    if ((submission_date_s3 is not None) and (sample_id is not None)):
        paths = []
        for sd in submission_date_s3:
            for si in sample_id:
                paths.append('{}/submission_date_s3={}/sample_id={}/'.format(base_path, sd, si))
        return reader.parquet(*paths)
    if ((submission_date_s3 is None) and (sample_id is not None)):
        data = reader.parquet(base_path)
        sids = ['{}'.format(s) for s in sample_id]
        criteria = 'sample_id IN ({})'.format(','.join(sids))
        return data.where(criteria)
    return reader.parquet(base_path)

Efficiently read main_summary parquet data.

Read data from the given path, optionally filtering to a
specified set of partition values first. This can save a
time, particularly if `mergeSchema` is True.

Args:
spark: Spark session
submission_date_s3: Optional list of values to filter the
`submission_date_s3` partition. Default is to read all
partitions. Each value should be in the form `YYYYMMDD`.
sample_id: Optional list of values to filter the `sample_id`
partition. Default is to read all partitions.
mergeSchema (bool): Determines whether or not to merge the
schemas of the resulting parquet files (ie. whether to
support schema evolution or not). Default is to merge
schemas.
path (str): Location (disk or S3) from which to read data.
Default is to read from the "production" location on S3.

Returns:
A DataFrame loaded from the specified partitions.


def _validate_signing_cert(ca_certificate, certificate, crl=None):
    pyopenssl_cert = load_certificate(FILETYPE_PEM, certificate)
    pyopenssl_ca_cert = load_certificate(FILETYPE_PEM, ca_certificate)
    cert_store = X509Store()
    cert_store.add_cert(pyopenssl_ca_cert)
    if crl:
        pyopenssl_crl = load_crl(FILETYPE_PEM, crl)
        cert_store.add_crl(pyopenssl_crl)
        cert_store.set_flags((X509StoreFlags.CRL_CHECK | X509StoreFlags.CRL_CHECK_ALL))
    cert_store_context = X509StoreContext(cert_store, pyopenssl_cert)
    cert_store_context.verify_certificate()

Validate an X509 certificate using pyOpenSSL.

.. note::
pyOpenSSL is a short-term solution to certificate validation. pyOpenSSL
is basically in maintenance mode and there's a desire in upstream to move
all the functionality into cryptography.

Args:
ca_certificate (str): A PEM-encoded Certificate Authority certificate to
validate the ``certificate`` with.
certificate (str): A PEM-encoded certificate that is in need of validation.
crl (str): A PEM-encoded Certificate Revocation List which, if provided, will
be taken into account when validating the certificate.

Raises:
X509StoreContextError: If the certificate failed validation. The
exception contains the details of the error.


def play(self, wav=None, data=None, rate=16000, channels=1, width=2, block=True, spectrum=None):

        if wav:
            f = wave.open(wav, 'rb')
            rate = f.getframerate()
            channels = f.getnchannels()
            width = f.getsampwidth()

            def gen(w):
                d = w.readframes(CHUNK_SIZE)
                while d:
                    yield d
                    d = w.readframes(CHUNK_SIZE)
                w.close()

            data = gen(f)

        self.stop_event.clear()
        if block:
            self._play(data, rate, channels, width, spectrum)
        else:
            thread = threading.Thread(target=self._play, args=(data, rate, channels, width, spectrum))
            thread.start()

play wav file or raw audio (string or generator)
Args:
wav: wav file path
data: raw audio data, str or iterator
rate: sample rate, only for raw audio
channels: channel number, only for raw data
width: raw audio data width, 16 bit is 2, only for raw data
block: if true, block until audio is played.
spectrum: if true, use a spectrum analyzer thread to analyze data


def apply_theme(self, property_values):
    old_dict = self.themed_values()
    if (old_dict is property_values):
        return
    removed = set()
    if (old_dict is not None):
        removed.update(set(old_dict.keys()))
    added = set(property_values.keys())
    old_values = dict()
    for k in added.union(removed):
        old_values[k] = getattr(self, k)
    if (len(property_values) > 0):
        setattr(self, '__themed_values__', property_values)
    elif hasattr(self, '__themed_values__'):
        delattr(self, '__themed_values__')
    for (k, v) in old_values.items():
        if (k in self._unstable_themed_values):
            del self._unstable_themed_values[k]
    for (k, v) in old_values.items():
        descriptor = self.lookup(k)
        descriptor.trigger_if_changed(self, v)

Apply a set of theme values which will be used rather than
defaults, but will not override application-set values.

The passed-in dictionary may be kept around as-is and shared with
other instances to save memory (so neither the caller nor the
|HasProps| instance should modify it).

Args:
property_values (dict) : theme values to use in place of defaults

Returns:
None


def read_links(self, file, encoding=None):

        return [item[0] for item in self.iter_text(file, encoding) if item[1]]

Return an iterator of links found in the document.

Args:
file: A file object containing the document.
encoding (str): The encoding of the document.

Returns:
iterable: str


class _NonAtrousConvolution:

    def __init__(self, input_shape, filter_shape, padding, data_format=None, strides=None, name=None, num_batch_dims=1):
        if input_shape.ndims is not None:
            filter_shape = filter_shape.with_rank(input_shape.ndims - num_batch_dims + 1)
        self.padding = padding
        self.name = name
        if filter_shape.ndims is not None:
            input_shape = input_shape.with_rank(filter_shape.ndims + num_batch_dims - 1)
        if input_shape.ndims is None:
            raise ValueError(f'Rank of convolution must be known. Received: input_shape={input_shape} of rank {input_shape.rank}')
        if input_shape.ndims < 3 or input_shape.ndims - num_batch_dims + 1 > 5:
            raise ValueError(f'`input_shape.rank - num_batch_dims + 1` must be at least 3 and at most 5. Received: input_shape.rank={input_shape.rank} and num_batch_dims={num_batch_dims}')
        conv_dims = input_shape.ndims - num_batch_dims - 1
        if strides is None:
            strides = [1] * conv_dims
        elif len(strides) != conv_dims:
            raise ValueError(f'`len(strides)` should be {conv_dims}. Received: strides={strides} of length {len(strides)}')
        if conv_dims == 1:
            if data_format is None:
                data_format = 'NWC'
            elif data_format not in {'NCW', 'NWC', 'NCHW', 'NHWC'}:
                raise ValueError(f"`data_format` must be 'NWC' or 'NCW'. Received: data_format={data_format}")
            self.strides = strides[0]
            self.data_format = data_format
            self.conv_op = self._conv1d
        elif conv_dims == 2:
            if data_format is None or data_format == 'NHWC':
                data_format = 'NHWC'
                strides = [1] + list(strides) + [1]
            elif data_format == 'NCHW':
                strides = [1, 1] + list(strides)
            else:
                raise ValueError(f"`data_format` must be 'NHWC' or 'NCHW'. Received: data_format={data_format}")
            self.strides = strides
            self.data_format = data_format
            self.conv_op = conv2d
        elif conv_dims == 3:
            if data_format is None or data_format == 'NDHWC':
                strides = [1] + list(strides) + [1]
            elif data_format == 'NCDHW':
                strides = [1, 1] + list(strides)
            else:
                raise ValueError(f"`data_format` must be 'NDHWC' or 'NCDHW'. Received: data_format={data_format}")
            self.strides = strides
            self.data_format = data_format
            self.conv_op = _conv3d_expanded_batch

    def _conv1d(self, input, filter, strides, padding, data_format, name):
        return conv1d(value=input, filters=filter, stride=strides, padding=padding, data_format=data_format, name=name)

    def __call__(self, inp, filter):
        return self.conv_op(input=inp, filter=filter, strides=self.strides, padding=self.padding, data_format=self.data_format, name=self.name)

Helper class for _non_atrous_convolution.

Note that this class assumes that shapes of input and filter passed to
`__call__` are compatible with `input_shape` and filter_shape passed to the
constructor.

Args:
input_shape: static input shape, i.e. input.shape.
filter_shape: static filter shape, i.e. filter.shape.
padding: see _non_atrous_convolution.
data_format: see _non_atrous_convolution.
strides: see _non_atrous_convolution.
name: see _non_atrous_convolution.
num_batch_dims: (Optional.)  The number of batch dimensions in the input;
if not provided, the default of `1` is used.


def get_property(self, prop):
    prop = prop.split('.')
    root = self
    for p in prop:
        if (p in root):
            root = root[p]
        else:
            return None
    return root

Access nested value using dot separated keys

Args:
prop (:obj:`str`): Property in the form of dot separated keys

Returns:
Property value if exists, else `None`


def post_process(self, outputs, target_sizes):
    logger.warning_once('`post_process` is deprecated and will be removed in v5 of Transformers, please use `post_process_object_detection` instead, with `threshold=0.` for equivalent results.')
    out_logits, out_bbox = (outputs.logits, outputs.pred_boxes)
    if len(out_logits) != len(target_sizes):
        raise ValueError('Make sure that you pass in as many target sizes as the batch dimension of the logits')
    if target_sizes.shape[1] != 2:
        raise ValueError('Each element of target_sizes must contain the size (h, w) of each image of the batch')
    prob = nn.functional.softmax(out_logits, -1)
    scores, labels = prob[..., :-1].max(-1)
    boxes = center_to_corners_format(out_bbox)
    img_h, img_w = target_sizes.unbind(1)
    scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(boxes.device)
    boxes = boxes * scale_fct[:, None, :]
    results = [{'scores': s, 'labels': l, 'boxes': b} for s, l, b in zip(scores, labels, boxes)]
    return results

Converts the raw output of [`YolosForObjectDetection`] into final bounding boxes in (top_left_x, top_left_y,
bottom_right_x, bottom_right_y) format. Only supports PyTorch.

Args:
outputs ([`YolosObjectDetectionOutput`]):
Raw outputs of the model.
target_sizes (`torch.Tensor` of shape `(batch_size, 2)`):
Tensor containing the size (height, width) of each image of the batch. For evaluation, this must be the
original image size (before any data augmentation). For visualization, this should be the image size
after data augment, but before padding.
Returns:
`List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image
in the batch as predicted by the model.


def __init__(self, port):

    super(StatsServer, self).__init__(port)
    self._http_server = None
    self._server_thread = None

Instantiates a new StatsServer.

Args:
port: The TCP port that the server should listen to.


def _minimize(the_string, delim, max_size):
    if the_string.startswith(delim):
        the_string = the_string[_len(delim):]
    if (_len(the_string) > max_size):
        try:
            idx = the_string.rindex(delim, 0, max_size)
        except ValueError:
            idx = max_size
        return ([the_string[:idx]] + _minimize(the_string[idx:], delim, max_size))
    else:
        return [the_string]

Recursively split a string in the largest chunks
possible from the highest position of a delimiter all the way
to a maximum size

Args:
the_string (string): The string to split.
delim (string): The delimiter to split on.
max_size (int): The maximum size of a chunk.

Returns:
list: the minimized string in tokens

Every chunk size will be at minimum `the_string[0:idx]` where `idx`
is the highest index of `delim` found in `the_string`; and at maximum
`the_string[0:max_size]` if no `delim` was found in `the_string`.
In the latter case, the split will occur at `the_string[max_size]`
which can be any character. The function runs itself again on the rest of
`the_string` (`the_string[idx:]`) until no chunk is larger than `max_size`.


def run_inference(self, batch: Sequence[datatable.Frame], model: Union[xgboost.Booster, xgboost.XGBModel], inference_args: Optional[dict[str, Any]]=None) -> Iterable[PredictionResult]:
    return self._inference_fn(batch, model, inference_args)

Runs inferences on a batch of datatable dataframe.

Args:
batch: A sequence of examples as datatable dataframes. Each
row in a dataframe is a single example. The dimensions
must match the dimensions of the data used to train
the model.
model: XGBoost booster or XBGModel (sklearn interface). Must implement
predict(X). Where the parameter X is a datatable dataframe.
inference_args: Any additional arguments for an inference.

Returns:
An Iterable of type PredictionResult.


def load_config_at_path(path):
    if (path.exists() and path.is_file()):
        with path.open('r') as f:
            return yaml.load(f, Loader=yaml.SafeLoader)
    else:
        raise ConfigurationError("Couldn't find ballet.yml config file.")

Load config at exact path

Args:
path (path-like): path to config file

Returns:
dict: config dict


def parsed_to_ast(parsed: Parsed, errors: Errors, component_type: str = ""):

    ast = {}
    sorted_keys = sorted(parsed.keys())

    for key in sorted_keys:
        if parsed[key]["type"] == "Nested":
            nested_component_stack = ["subject", "object"]

    if component_type:
        component_stack = [component_type]
    else:
        component_stack = ["subject", "object"]

    for key in sorted_keys:
        if parsed[key]["type"] == "Function" and parsed[key]["function_level"] == "top":
            ast[component_stack.pop(0)] = parsed_function_to_ast(parsed, key)
        elif parsed[key]["type"] == "Relation" and "relation" not in ast:
            ast["relation"] = {
                "name": parsed[key]["name"],
                "type": "Relation",
                "span": key,
            }
        elif parsed[key]["type"] == "Nested":
            ast["nested"] = {}
            for nested_key in sorted_keys:
                if nested_key <= key:
                    continue

                if (
                    parsed[nested_key]["type"] == "Function"
                    and parsed[nested_key]["function_level"] == "top"
                ):
                    ast["nested"][
                        nested_component_stack.pop(0)
                    ] = parsed_function_to_ast(parsed, nested_key)
                elif (
                    parsed[nested_key]["type"] == "Relation"
                    and "relation" not in ast["nested"]
                ):
                    ast["nested"]["relation"] = {
                        "name": parsed[nested_key]["name"],
                        "type": "Relation",
                        "span": parsed[nested_key]["span"],
                    }

            return ast, errors

    return ast, errors

Convert parsed data struct to AST dictionary

Args:
parsed:
errors:
component_type: Empty string or 'subject' or 'object' to indicate that we
are parsing the subject or object field input


def setup_logging(args=None):

    logging_level = logging.WARNING
    if args is not None and args.verbose:
        logging_level = logging.INFO
    config = {"level": logging_level, "format": "jtlocalize:%(message)s"}

    if args is not None and args.log_path != "":
        config["filename"] = args.log_path

    logging.basicConfig(**config)

Setup logging module.

Args:
args (optional): The arguments returned by the argparse module.


def _prepare_input_trimaps(self, trimaps: ImageInput, device: Optional['torch.device']=None) -> list['torch.Tensor']:
    trimaps = make_list_of_images(trimaps, expected_ndims=2)
    process_image_fn = partial(self._process_image, input_data_format=ChannelDimension.FIRST, device=device)
    processed_trimaps = []
    for trimap in trimaps:
        processed_trimaps.append(torch.unsqueeze(process_image_fn(trimap), dim=0))
    return processed_trimaps

Prepare input trimaps for processing,m this can not yet deal with nested list

Args:
trimaps ('ImageInout):
The input trimaps to be process, should not be nested
device('Optional['torch.device'] defaults to 'self.device'):
The device to process the trimaps on

Returns:
list['torch.Tensor']:
Input trimaps converted to a list of tensors


def to_dict(self, remove_nones=False):

        content = {
            'uri': self.uri,
            'protocol_info': self.protocol_info,
            'import_uri': self.import_uri,
            'size': self.size,
            'duration': self.duration,
            'bitrate': self.bitrate,
            'sample_frequency': self.sample_frequency,
            'bits_per_sample': self.bits_per_sample,
            'nr_audio_channels': self.nr_audio_channels,
            'resolution': self.resolution,
            'color_depth': self.color_depth,
            'protection': self.protection,
        }
        if remove_nones:

            nones = [k for k in content if content[k] is None]
            for k in nones:
                del content[k]
        return content

Return a dict representation of the `DidlResource`.

Args:
remove_nones (bool, optional): Optionally remove dictionary
elements when their value is `None`.

Returns:
dict: a dict representing the `DidlResource`


def custom_gradient(fun):

    def __init__(self, fun):
        warnings.warn('`custom_gradient` for the openvino backend acts as a pass-through to support the forward pass. No gradient computation or modification takes place.')
        self.fun = fun

    def __call__(self, *args, **kwargs):
        outputs, _ = self.fun(*args, **kwargs)
        return outputs

Decorator for custom gradients.

Args:
fun: Forward pass function.


def scp(host, src, dst, user=None, path=None):
    if (user is None):
        arg = '{host}:{path}'.format(host=host, path=src)
    else:
        arg = '{user}@{host}:{path}'.format(user=user, host=host, path=src)
    scp_bin = which('scp', path=(path,))
    if (scp_bin is None):
        raise CommandNotFoundError("Could not find scp in '{0}'".format(path))
    (ret, out, err) = run([scp_bin, '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', arg, dst])
    if ret:
        raise ScpError(out, err)

Copy a file or directory from a remote location.

A thin wrapper around the scp (1) system command.

If the destination already exists, this will attempt to overwrite
it.

Arguments:

host (str): name of the host
src (str): path to the source file or directory.
dst (str): path to the destination file or directory.
user (str, optional): Alternative username for remote access.
If not provided, the default scp behaviour is used.
path (str, optional): Directory containing scp command. If not
provided, attempt to locate scp using which().

Raises:

CommandNotFoundError: If scp binary not found.
IOError: if transfer fails.


def fetch_local_files(destination_directory: str,
                      verbose: bool) -> prepared_env.PreparedEnv:

    staging_dir = destination_directory + '-staging'
    try:
        shutil.copytree(get_repo_root(), staging_dir)
        os.chdir(staging_dir)
        if verbose:
            print('chdir', staging_dir, file=sys.stderr)

        shell_tools.run_cmd(
            'git',
            'add',
            '--all',
            out=sys.stderr,
            log_run_to_stderr=verbose)

        shell_tools.run_cmd(
            'git',
            'commit',
            '-m', 'working changes',
            '--allow-empty',
            '--no-gpg-sign',
            None if verbose else '--quiet',
            out=sys.stderr,
            log_run_to_stderr=verbose)

        cur_commit = shell_tools.output_of('git', 'rev-parse', 'HEAD')

        os.chdir(destination_directory)
        if verbose:
            print('chdir', destination_directory, file=sys.stderr)
        shell_tools.run_cmd('git',
                            'init',
                            None if verbose else '--quiet',
                            out=sys.stderr,
                            log_run_to_stderr=verbose)
        result = _git_fetch_for_comparison(staging_dir,
                                           cur_commit,
                                           'master',
                                           verbose=verbose)
    finally:
        shutil.rmtree(staging_dir, ignore_errors=True)

    shell_tools.run_cmd(
        'git',
        'branch',
        None if verbose else '--quiet',
        'compare_commit',
        result.compare_commit_id,
        log_run_to_stderr=verbose)
    shell_tools.run_cmd(
        'git',
        'checkout',
        None if verbose else '--quiet',
        '-b',
        'actual_commit',
        result.actual_commit_id,
        log_run_to_stderr=verbose)
    return prepared_env.PreparedEnv(
        github_repo=None,
        actual_commit_id=result.actual_commit_id,
        compare_commit_id=result.compare_commit_id,
        destination_directory=destination_directory,
        virtual_env_path=None)

Uses local files to create a directory for testing and comparisons.

Args:
destination_directory: The directory where the copied files should go.
verbose: When set, more progress output is produced.

Returns:
Commit ids corresponding to content to test/compare.


def _write_entries(self, stream, entries, converter, properties=None):

    def iter_entries():
        for c in entries:
            entry = converter(c)
            if (entry is None):
                continue
            if (properties is not None):
                entry = OrderedDict(((key, value) for (key, value) in iteritems(entry) if ((key == 'id') or (key in properties))))
            (yield entry)
    self._dump(stream, list(iter_entries()))

Write iterable of entries as YAML object to stream.

Args:
stream: File-like object.
entries: Iterable of entries.
converter: Conversion function from entry to YAML object.
properties: Set of compartment properties to output (or None to
output all).


def delete_lines(self, lines):

        for k, i in enumerate(lines):
            del self[i-k]

Delete all lines with given line numbers.

Args:
lines (list): List of integers corresponding to line numbers to delete


def __ginibre_matrix(nrow, ncol=None, seed=None):
    if (ncol is None):
        ncol = nrow
    if (seed is not None):
        np.random.seed(seed)
    G = (np.random.normal(size=(nrow, ncol)) + (np.random.normal(size=(nrow, ncol)) * 1j))
    return G

Return a normally distributed complex random matrix.

Args:
nrow (int): number of rows in output matrix.
ncol (int): number of columns in output matrix.
seed (int): Optional. To set a random seed.
Returns:
ndarray: A complex rectangular matrix where each real and imaginary
entry is sampled from the normal distribution.


def get_gpubsub_publisher(config, metrics, changes_channel, **kw):
    builder = gpubsub_publisher.GPubsubPublisherBuilder(config, metrics, changes_channel, **kw)
    return builder.build_publisher()

Get a GPubsubPublisher client.

A factory function that validates configuration, creates an auth
and pubsub API client, and returns a Google Pub/Sub Publisher
provider.

Args:
config (dict): Google Cloud Pub/Sub-related configuration.
metrics (obj): :interface:`IMetricRelay` implementation.
changes_channel (asyncio.Queue): Queue to publish message to
make corrections to Cloud DNS.
kw (dict): Additional keyword arguments to pass to the
Publisher.
Returns:
A :class:`GPubsubPublisher` instance.


def course_discovery(request):
    results = {'error': _('Nothing to search')}
    status_code = 500
    search_term = request.POST.get('search_string', None)
    try:
        (size, from_, page) = _process_pagination_values(request)
        field_dictionary = _process_field_values(request)
        track.emit('edx.course_discovery.search.initiated', {'search_term': search_term, 'page_size': size, 'page_number': page})
        results = course_discovery_search(search_term=search_term, size=size, from_=from_, field_dictionary=field_dictionary)
        track.emit('edx.course_discovery.search.results_displayed', {'search_term': search_term, 'page_size': size, 'page_number': page, 'results_count': results['total']})
        status_code = 200
    except ValueError as invalid_err:
        results = {'error': six.text_type(invalid_err)}
        log.debug(six.text_type(invalid_err))
    except QueryParseError:
        results = {'error': _('Your query seems malformed. Check for unmatched quotes.')}
    except Exception as err:
        results = {'error': _('An error occurred when searching for "{search_string}"').format(search_string=search_term)}
        log.exception('Search view exception when searching for %s for user %s: %r', search_term, request.user.id, err)
    return JsonResponse(results, status=status_code)

Search for courses

Args:
request (required) - django request object

Returns:
http json response with the following fields
"took" - how many seconds the operation took
"total" - how many results were found
"max_score" - maximum score from these resutls
"results" - json array of result documents

or

"error" - displayable information about an error that occured on the server

POST Params:
"search_string" (optional) - text with which to search for courses
"page_size" (optional)- how many results to return per page (defaults to 20, with maximum cutoff at 100)
"page_index" (optional) - for which page (zero-indexed) to include results (defaults to 0)


def read_single_knmi_file(filename):

    hourly_data_obs_raw = pd.read_csv(
        filename,
        parse_dates=[['YYYYMMDD', 'HH']],
        date_parser=lambda yyyymmdd, hh: pd.datetime(int(str(yyyymmdd)[0:4]),
                                                     int(str(yyyymmdd)[4:6]),
                                                     int(str(yyyymmdd)[6:8]),
                                                     int(hh) - 1),
        skiprows=31,
        skipinitialspace=True,
        na_values='',
        keep_date_col=True,
    )

    hourly_data_obs_raw.index = hourly_data_obs_raw['YYYYMMDD_HH']
    hourly_data_obs_raw.index = hourly_data_obs_raw.index + pd.Timedelta(hours=1)

    columns_hourly = ['temp', 'precip', 'glob', 'hum', 'wind', 'ssd']

    hourly_data_obs = pd.DataFrame(
        index=hourly_data_obs_raw.index,
        columns=columns_hourly,
        data=dict(
            temp=hourly_data_obs_raw['T'] / 10 + 273.15,
            precip=hourly_data_obs_raw['RH'] / 10,
            glob=hourly_data_obs_raw['Q'] * 10000 / 3600.,
            hum=hourly_data_obs_raw['U'],
            wind=hourly_data_obs_raw['FH'] / 10,
            ssd=hourly_data_obs_raw['SQ'] * 6,
        ),
    )

    negative_values = hourly_data_obs['precip'] < 0.0
    hourly_data_obs.loc[negative_values, 'precip'] = 0.0
    return hourly_data_obs

reads a single file of KNMI's meteorological time series

data availability: www.knmi.nl/nederland-nu/klimatologie/uurgegevens

Args:
filename: the file to be opened

Returns:
pandas data frame including time series


def make_prediction_pipeline(pipeline, args):

  predicted_values, errors = (
      pipeline |
      'Read CSV Files' >>
      beam.io.ReadFromText(str(args.predict_data),
                           strip_trailing_newlines=True) |
      'Batch Input' >>
      beam.ParDo(EmitAsBatchDoFn(args.batch_size)) |
      'Run TF Graph on Batches' >>
      beam.ParDo(RunGraphDoFn(args.trained_model_dir)).with_outputs('errors', main='main'))

  ((predicted_values, errors) |
   'Format and Save' >>
   FormatAndSave(args))

Builds the prediction pipeline.

Reads the csv files, prepends a ',' if the target column is missing, run
prediction, and then prints the formated results to a file.

Args:
pipeline: the pipeline
args: command line args


def to_url(self, site='amazon', country='us'):

        try:
            try:
                url, tlds = URL_MAP[site]
            except ValueError:
                tlds = None
                url = URL_MAP[site]
        except KeyError:
            raise SiteError(site)
        inject = {'isbn': self._isbn}
        if tlds:
            if country not in tlds:
                raise CountryError(country)
            tld = tlds[country]
            if not tld:
                tld = country
            inject['tld'] = tld
        return url % inject

Generate a link to an online book site.

Args:
site (str): Site to create link to
country (str): Country specific version of ``site``

Returns:
``str``: URL on ``site`` for book

Raises:
SiteError: Unknown site value
CountryError: Unknown country value


def CheckPrintf(filename, clean_lines, linenum, error):

  line = clean_lines.elided[linenum]

  match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line)
  if match and match.group(2) != '0':

    error(filename, linenum, 'runtime/printf', 3,
          'If you can, use sizeof(%s) instead of %s as the 2nd arg '
          'to snprintf.' % (match.group(1), match.group(2)))

  if Search(r'\bsprintf\s*\(', line):
    error(filename, linenum, 'runtime/printf', 5,
          'Never use sprintf. Use snprintf instead.')
  match = Search(r'\b(strcpy|strcat)\s*\(', line)
  if match:
    error(filename, linenum, 'runtime/printf', 4,
          'Almost always, snprintf is better than %s' % match.group(1))

Check for printf related issues.

Args:
filename: The name of the current file.
clean_lines: A CleansedLines instance containing the file.
linenum: The number of the line to check.
error: The function to call with any errors found.


def Print(self, x, data, message, **kwargs):
    del data, message, kwargs
    tf.logging.warning('Warning - mtf.Print not implemented for this mesh type')
    return x

Calls tf.Print.

Args:
x: LaidOutTensor.
data: list of LaidOutTensor.
message: str.
**kwargs: keyword arguments to tf.print.

Returns:
LaidOutTensor.


class BlenderbotSmallTokenizerFast(PreTrainedTokenizerFast):
    vocab_files_names = VOCAB_FILES_NAMES
    slow_tokenizer_class = BlenderbotSmallTokenizer

    def __init__(self, vocab_file=None, merges_file=None, unk_token='<|endoftext|>', bos_token='<|endoftext|>', eos_token='<|endoftext|>', add_prefix_space=False, trim_offsets=True, **kwargs):
        super().__init__(ByteLevelBPETokenizer(vocab=vocab_file, merges=merges_file, add_prefix_space=add_prefix_space, trim_offsets=trim_offsets), bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
        self.add_prefix_space = add_prefix_space

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
        if token_ids_1 is None:
            return output
        return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]

    def create_token_type_ids_from_sequences(self, token_ids_0: List[int], token_ids_1: Optional[List[int]]=None) -> List[int]:

        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

Construct a "fast" BlenderbotSmall tokenizer (backed by HuggingFace's *tokenizers* library).

Args:
vocab_file (`str`):
Path to the vocabulary file.


def get_flag_value(self, name, default):

    value = self.__getattr__(name)
    if value is not None:
      return value
    else:
      return default

Returns the value of a flag (if not None) or a default value.

Args:
name: str, the name of a flag.
default: Default value to use if the flag value is None.

Returns:
Requested flag value or default.


def load(self, context):

    try:

      import tensorflow
    except ImportError:
      return

    from tensorboard.plugins.hparams.hparams_plugin import HParamsPlugin
    return HParamsPlugin(context)

Returns the plugin, if possible.

Args:
context: The TBContext flags.

Returns:
A HParamsPlugin instance or None if it couldn't be loaded.


def mkdir(self, path):
    self.__validate_storage_path(path, projects_allowed=False)
    parent_metadata = self.get_parent(path)
    self.api_client.create_folder(path.split('/')[(- 1)], parent_metadata['uuid'])

Create a folder in the storage service pointed by the given path.

Args:
path (str): The path of the folder to be created

Returns:
None

Raises:
StorageArgumentException: Invalid arguments
StorageForbiddenException: Server response code 403
StorageNotFoundException: Server response code 404
StorageException: other 400-600 error codes


def pad(self, video: 'torch.Tensor', padded_size: tuple[int, int], max_num_frames: int, fill: int=0, return_pixel_mask: bool=True):
    original_size = video.size()[-2:]
    padding_height = padded_size[0] - original_size[0]
    padding_width = padded_size[1] - original_size[1]
    padding_frame = max_num_frames - video.shape[0]
    if padding_width < 0 or padding_height < 0:
        raise ValueError(f'Padding dimensions are negative. Please make sure that the padded size is larger than the original size. Got padded size: {padded_size}, original size: {original_size}.')
    if original_size != padded_size:
        padding = [0, padding_width, 0, padding_height, 0, 0, 0, padding_frame]
        video = F.pad(video, padding, fill=fill)
    pixel_mask = None
    if return_pixel_mask:
        pixel_mask = torch.zeros_like(video[..., 0, :, :], dtype=torch.int64)
        pixel_mask[..., :original_size[0], :original_size[1]] = 1
    return (video, pixel_mask)

Pads the sample with empty video to the padded_size
Args:
video (`torch.Tensor`):
Video to pad.
padded_size (`Tuple[int, int]`):
Height and width to pad.
max_num_frames (`int`):
The maximum number of frames to which video will be padded.
fill (`int`, *optional*):
The value to use for the padding.
return_pixel_mask (`bool`, *optional*, defaults to `True`):
Whether to return a pixel mask.


def by_phone(self, phone, cc=None):
    (header, content) = self._http_request(self.BASE_URL, phone=phone, cc=cc)
    return json.loads(content)

Perform a Yelp Phone API Search based on phone number given.

Args:
phone    - Phone number to search by
cc       - ISO 3166-1 alpha-2 country code. (Optional)


def _pad_or_truncate_with_sequence_length(self, embeddings: tensor.Tensor, sequence_length: int) -> tensor.Tensor:
    original_sequence_length = embeddings.shape[1]
    if original_sequence_length > sequence_length:
        embeddings = array_ops.slice(embeddings, begin=[0, 0, 0], size=[-1, sequence_length, -1])
    else:
        embeddings = array_ops.pad(embeddings, paddings=[[0, 0], [0, sequence_length - original_sequence_length], [0, 0]])
    return embeddings

Pad or truncate the embedding lookup result based on the sequence length.

Args:
embeddings: A rank 3 Tensor of the embedding lookup result.
sequence_length: number of the max sequence length set in the feature
config.

Returns:
A Tensor with second last axis padded or truncated.


def color_palette_dict(self, alpha=0.35):
    color_dict = {}
    for hkl in self.all_slab_entries.keys():
        rgb_indices = [0, 1, 2]
        color = [0, 0, 0, 1]
        random.shuffle(rgb_indices)
        for (i, ind) in enumerate(rgb_indices):
            if (i == 2):
                break
            color[ind] = np.random.uniform(0, 1)
        clean_list = np.linspace(0, 1, len(self.all_slab_entries[hkl]))
        for (i, clean) in enumerate(self.all_slab_entries[hkl].keys()):
            c = copy.copy(color)
            c[rgb_indices[2]] = clean_list[i]
            color_dict[clean] = c
            for ads_entry in self.all_slab_entries[hkl][clean]:
                c_ads = copy.copy(c)
                c_ads[3] = alpha
                color_dict[ads_entry] = c_ads
    return color_dict

Helper function to assign each facet a unique color using a dictionary.

Args:
alpha (float): Degree of transparency

return (dict): Dictionary of colors (r,g,b,a) when plotting surface
energy stability. The keys are individual surface entries where
clean surfaces have a solid color while the corresponding adsorbed
surface will be transparent.


def revnet(name, x, hparams, reverse=True):
    with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
        steps = np.arange(hparams.depth)
        if reverse:
            steps = steps[::(- 1)]
        objective = 0.0
        for step in steps:
            (x, curr_obj) = revnet_step(('revnet_step_%d' % step), x, hparams, reverse=reverse)
            objective += curr_obj
        return (x, objective)

hparams.depth' steps of generative flow.

Args:
name: variable scope for the revnet block.
x: 4-D Tensor, shape=(NHWC).
hparams: HParams.
reverse: bool, forward or backward pass.
Returns:
x: 4-D Tensor, shape=(NHWC).
objective: float.


def append(self, item):
    if self.should_flush():
        self.flush()
    self.items.append(item)

Add new item to the list.

If needed, append will first flush existing items and clear existing items.

Args:
item: an item to add to the list.


def _decorate(self, atype, n, o):

        typemap = {"function": "functions",
                   "classobj": "classes",
                   "staticmethod": "methods",
                   "type": "classes"}
        from acorn.logging.decoration import decorate_obj
        try:
            otype = typemap[atype]
            decorate_obj(self.shell.user_ns, n, o, otype)

            self._logdef(n, o, otype)
            msg.okay("Auto-decorated {}: {}.".format(n, o))
        except:
            msg.err("Error auto-decorating {}: {}.".format(n, o))
            raise

Decorates the specified object for automatic logging with acorn.

Args:
atype (str): one of the types specified in :attr:`atypes`.
varobj: object instance to decorate; no additional type checking is
performed.


def disabled_function(*unused_args, **unused_kwargs):
    raise AssertionError('Cannot call disabled function.')

Disable a function.

Disable a previously defined function foo as follows:

foo = disabled_function

Any later calls to foo will raise an AssertionError.  This is used, e.g.,
in cfg.Program to prevent the addition of more nodes after we have begun
solving the graph.

Raises:
AssertionError: If something tried to call the disabled function.


def _BuildKeyHierarchy(self, subkeys, values):
    if subkeys:
        for registry_key in subkeys:
            name = registry_key.name.upper()
            if (name in self._subkeys):
                continue
            self._subkeys[name] = registry_key
            registry_key._key_path = key_paths.JoinKeyPath([self._key_path, registry_key.name])
    if values:
        for registry_value in values:
            name = registry_value.name.upper()
            if (name in self._values):
                continue
            self._values[name] = registry_value

Builds the Windows Registry key hierarchy.

Args:
subkeys (list[FakeWinRegistryKey]): list of subkeys.
values (list[FakeWinRegistryValue]): list of values.


def __init__(self, file_object, encoding='utf-8'):

    super(FileObjectInputReader, self).__init__(encoding=encoding)
    self._errors = 'strict'
    self._file_object = file_object

Initializes a file object command line interface input reader.

Args:
file_object (file): file-like object to read from.
encoding (Optional[str]): input encoding.


def is_deterministic(x: Any) -> bool:
    return not contains(x, type=NonDeterministic)

Returns if the input value is deterministic.

Example::

@pg.symbolize
def foo(x, y):
pass

assert pg.is_deterministic(1)
assert pg.is_deterministic(foo(1, 2))
assert not pg.is_deterministic(pg.oneof([1, 2]))
assert not pg.is_deterministic(foo(pg.oneof([1, 2]), 3))

Args:
x: Value to query against.

Returns:
True if value itself is not NonDeterministic and its child and nested
child fields do not contain NonDeterministic values.


def _config_parser_to_defaultdict(config_parser):

    config = defaultdict(defaultdict)
    for section, section_content in config_parser.items():
        if section != 'DEFAULT':
            for option, option_value in section_content.items():
                config[section][option] = option_value
    return config

Convert a ConfigParser to a defaultdict.

Args:
config_parser (ConfigParser): A ConfigParser.


def __init__(self, logger, lock, parser, **kwargs):

    self._logger = logger
    self._lock = lock
    self._modules = kwargs
    self._declarations = {}
    self.ARG_PARSER = parser

    self._flags, _ = parser.parse_known_args()
    self._flag_values = {}

    self.load_flag_values()

    self.reset()

Initializes the configuration state.

We have to pull everything we need from global scope into here because we
will be swapping out the module with this instance and will lose any global
references.

Args:
logger: Logger to use for logging messages within this class.
lock: Threading.lock to use for locking access to config values.
**kwargs: Modules we need to access within this class.


def conditionally_attach_managed_policies(role_name, sr_entry):

  service_type = sr_entry['type']
  if not (service_type in SERVICE_TYPE_ROLE and "aws_managed_policies" in sr_entry):
    print_if_verbose("not eligible for policies; service_type: {} is not valid for policies "
                     "or no 'aws_managed_policies' key in service registry for this role".format(service_type))
    return

  for policy_name in sr_entry['aws_managed_policies']:
    print_if_verbose("loading policy: {} for role: {}".format(policy_name, role_name))

    if CONTEXT.commit:
      try:
        CLIENTS["iam"].attach_role_policy(RoleName=role_name, PolicyArn='arn:aws:iam::aws:policy/' + policy_name)
      except:
        fail("Exception putting policy: {} onto role: {}".format(policy_name, role_name), sys.exc_info())

If 'aws_managed_policies' key lists the names of AWS managed policies to bind to the role,
attach them to the role
Args:
role_name: name of the role to attach the policies to
sr_entry: service registry entry


def update_function_configuration(self, vpc_config):
    LOG.info('Updating configuration for lambda function: %s', self.app_name)
    try:
        self.lambda_client.update_function_configuration(Environment=self.lambda_environment, FunctionName=self.app_name, Runtime=self.runtime, Role=self.role_arn, Handler=self.handler, Description=self.description, Timeout=int(self.timeout), MemorySize=int(self.memory), VpcConfig=vpc_config)
        if self.concurrency_limit:
            self.lambda_client.put_function_concurrency(FunctionName=self.app_name, ReservedConcurrentExecutions=self.concurrency_limit)
        else:
            self.lambda_client.delete_function_concurrency(FunctionName=self.app_name)
    except boto3.exceptions.botocore.exceptions.ClientError as error:
        if ('CreateNetworkInterface' in error.response['Error']['Message']):
            message = '{0} is missing "ec2:CreateNetworkInterface"'.format(self.role_arn)
            LOG.debug(message)
            raise SystemExit(message)
        raise
    LOG.info('Updating Lambda function tags')
    lambda_arn = get_lambda_arn(self.app_name, self.env, self.region)
    self.lambda_client.tag_resource(Resource=lambda_arn, Tags={'app_group': self.group, 'app_name': self.app_name})
    LOG.info('Successfully updated Lambda configuration.')

Update existing Lambda function configuration.

Args:
vpc_config (dict): Dictionary of SubnetIds and SecurityGroupsIds for using
a VPC in lambda


def train_on_batch(self, *args) -> None:

        *data, labels = args
        self._net.train_on_batch(data, labels)

Trains the model on a single batch.

Args:
*args: the list of network inputs.
Last element of `args` is the batch of targets,
all previous elements are training data batches


def _get_value_info(self, inner: tuple[_base.BaseValue, ...], ellipses: set[int], allowed_ellipses: Set[int]=frozenset()) -> tuple[tuple[int | str, ...], tuple[_base.BaseValue, ...], type[_classes.ParameterizedClass]]:
    if self.base_cls.full_name == 'typing.Protocol':
        return abstract_utils.build_generic_template(inner, self) + (_classes.ParameterizedClass,)
    if isinstance(self.base_cls, _classes.TupleClass):
        template = tuple(range(self.base_cls.tuple_length))
    elif isinstance(self.base_cls, _classes.CallableClass):
        template = tuple(range(self.base_cls.num_args)) + (abstract_utils.RET,)
    else:
        template = tuple((t.name for t in self.base_cls.template))
    self.ctx.errorlog.invalid_ellipses(self.ctx.vm.frames, ellipses - allowed_ellipses, self.name)
    last_index = len(inner) - 1
    if last_index and last_index in ellipses and (len(inner) > len(template)):
        inner = inner[:-1]
    if isinstance(self.base_cls, _classes.ParameterizedClass):
        new_inner = []
        inner_idx = 0
        subst = {}
        for k in template:
            v = self.base_cls.formal_type_parameters[k]
            if v.formal:
                params = self.ctx.annotation_utils.get_type_parameters(v)
                for param in params:
                    if param.full_name not in subst and inner_idx < len(inner):
                        subst[param.full_name] = inner[inner_idx]
                        inner_idx += 1
                new_inner.append(self._sub_annotation(v, subst))
            else:
                new_inner.append(v)
        inner = tuple(new_inner)
        if isinstance(self.base_cls, _classes.TupleClass):
            template += (abstract_utils.T,)
            inner += (self.ctx.convert.merge_values(inner),)
        elif isinstance(self.base_cls, _classes.CallableClass):
            template = template[:-1] + (abstract_utils.ARGS,) + template[-1:]
            args = inner[:-1]
            inner = args + (self.ctx.convert.merge_values(args),) + inner[-1:]
        abstract_class = type(self.base_cls)
    else:
        abstract_class = _classes.ParameterizedClass
    return (template, inner, abstract_class)

Get information about the container's inner values.

Args:
inner: The list of parameters from _build_inner().
ellipses: The set of ellipsis indices from _build_inner().
allowed_ellipses: Optionally, a set of indices at which ellipses are
allowed. If omitted, ellipses are assumed to be never allowed.

Returns:
A tuple of the template, the parameters, and the container class.


def _prefix_from_ip_string(self, ip_str):

        try:
            ip_int = self._ip_int_from_string(ip_str)
        except AddressValueError:
            raise NetmaskValueError('%s is not a valid netmask' % ip_str)

        try:
            return self._prefix_from_ip_int(ip_int)
        except NetmaskValueError:
            pass

        ip_int ^= self._ALL_ONES
        try:
            return self._prefix_from_ip_int(ip_int)
        except NetmaskValueError:
            raise NetmaskValueError('%s is not a valid netmask' % ip_str)

Turn a netmask/hostmask string into a prefix length.

Args:
ip_str: A netmask or hostmask, formatted as an IP address.

Returns:
The prefix length as an integer.

Raises:
NetmaskValueError: If the input is not a netmask or hostmask.


def indent(self, space=4):

        if not isinstance(space,int):
            raise TypeError("space must be an int")
        if space < 0:
            raise ValueError("space must be a non-negative integer")
        space = ' '*space; o = []; l = 0
        for c in self.newick():
            if c == '(':
                o.append('(\n'); l += 1; o.append(space*l)
            elif c == ')':
                o.append('\n'); l -= 1; o.append(space*l); o.append(')')
            elif c == ',':
                o.append(',\n'); o.append(space*l)
            else:
                o.append(c)
        return ''.join(o)

Return an indented Newick string, just like ``nw_indent`` in Newick Utilities

Args:
``space`` (``int``): The number of spaces a tab should equal

Returns:
``str``: An indented Newick string


def _get_dir(toml_config_setting, sawtooth_home_dir, windows_dir, default_dir):
    conf_file = os.path.join(_get_config_dir(), 'path.toml')
    if os.path.exists(conf_file):
        with open(conf_file) as fd:
            raw_config = fd.read()
        toml_config = toml.loads(raw_config)
        if (toml_config_setting in toml_config):
            return toml_config[toml_config_setting]
    if ('SAWTOOTH_HOME' in os.environ):
        return os.path.join(os.environ['SAWTOOTH_HOME'], sawtooth_home_dir)
    if (os.name == 'nt'):
        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
        return os.path.join(base_dir, windows_dir)
    return default_dir

Determines the directory path based on configuration.

Arguments:
toml_config_setting (str): The name of the config setting related
to the directory which will appear in path.toml.
sawtooth_home_dir (str): The directory under the SAWTOOTH_HOME
environment variable.  For example, for 'data' if the data
directory is $SAWTOOTH_HOME/data.
windows_dir (str): The windows path relative to the computed base
directory.
default_dir (str): The default path on Linux.

Returns:
directory (str): The path.


def load_cobra_model(self, model):

        self.model = ModelPro(model)
        for g in self.model.genes:
            if self.genes_dir:
                g.root_dir = self.genes_dir
            g.protein.pdb_file_type = self.pdb_file_type
        self.genes = self.model.genes

        log.info('{}: loaded model'.format(model.id))
        log.info('{}: number of reactions'.format(len(self.model.reactions)))
        log.info('{}: number of reactions linked to a gene'.format(ssbio.core.modelpro.true_num_reactions(self.model)))
        log.info('{}: number of genes (excluding spontaneous)'.format(ssbio.core.modelpro.true_num_genes(self.model,
                                                                                custom_spont_id=self.custom_spont_id)))
        log.info('{}: number of metabolites'.format(len(self.model.metabolites)))
        log.warning('IMPORTANT: All Gene objects have been transformed into GenePro '
                    'objects, and will be for any new ones')

Load a COBRApy Model object into the GEM-PRO project.

Args:
model (Model): COBRApy ``Model`` object


def get_facets(qhull_data, joggle=False):

    if joggle:
        return ConvexHull(qhull_data, qhull_options="QJ i").simplices
    else:
        return ConvexHull(qhull_data, qhull_options="Qt i").simplices

Get the simplex facets for the Convex hull.

Args:
qhull_data (np.ndarray): The data from which to construct the convex
hull as a Nxd array (N being number of data points and d being the
dimension)
joggle (boolean): Whether to joggle the input to avoid precision
errors.

Returns:
List of simplices of the Convex Hull.


def temp_shell_task(cls, inp, ddb_node, mpi_procs=1,
                        gkk_node=None, md_node=None, ddk_node=None, workdir=None, manager=None):

        import tempfile
        workdir = tempfile.mkdtemp() if workdir is None else workdir
        if manager is None: manager = TaskManager.from_user_config()

        return cls(inp, ddb_node,
                   gkk_node=gkk_node, md_node=md_node, ddk_node=ddk_node,
                   workdir=workdir, manager=manager.to_shell_manager(mpi_procs=mpi_procs))

Build a :class:`AnaddbTask` with a temporary workdir. The task is executed via
the shell with 1 MPI proc. Mainly used for post-processing the DDB files.

Args:
mpi_procs: Number of MPI processes to use.
anaddb_input: string with the anaddb variables.
ddb_node: The node that will produce the DDB file. Accept :class:`Task`, :class:`Work` or filepath.

See `AnaddbInit` for the meaning of the other arguments.


def _read_callback(self, data=None):

        try:
            if data is not None:
                self.__reader.feed(data)
                while True:
                    reply = self.__reader.gets()
                    if reply is not False:
                        try:
                            callback = self.__callback_queue.popleft()

                            callback(reply)
                        except IndexError:

                            self._reply_list.append(reply)
                            self._condition.notify_all()
                    else:
                        break
        except hiredis.ProtocolError:

            LOG.warning("corrupted stream => disconnect")
            self.disconnect()

Callback called when some data are read on the socket.

The buffer is given to the hiredis parser. If a reply is complete,
we put the decoded reply to on the reply queue.

Args:
data (str): string (buffer) read on the socket.


def profile_settings_args_layout_json(self, required):
    profile_args = {}
    self.db_create_table(self.input_table, self.install_json_params().keys())
    self.db_insert_record(self.input_table, self.install_json_params().keys())
    self.gen_permutations()
    try:
        for pn in self._input_permutations[self.args.permutation_id]:
            p = self.install_json_params().get(pn.get('name'))
            if (p.get('required', False) != required):
                continue
            if (p.get('type').lower() == 'boolean'):
                profile_args[p.get('name')] = pn.get('value')
            elif (p.get('type').lower() == 'choice'):
                profile_args[p.get('name')] = pn.get('value')
            elif (p.get('name') in ['api_access_id', 'api_secret_key']):
                pass
            else:
                types = '|'.join(p.get('playbookDataType', []))
                if types:
                    profile_args[p.get('name')] = p.get('default', '<{}>'.format(types))
                else:
                    profile_args[p.get('name')] = p.get('default', '')
    except IndexError:
        self.handle_error('Invalid permutation index provided.')
    return profile_args

Return args based on layout.json and conditional rendering.

Args:
required (bool): If True only required args will be returned.

Returns:
dict: Dictionary of required or optional App args.


def cat(rs: Sequence[Rotation], dim: int) -> Rotation:
    rot_mats = torch.cat([r.get_rot_mats() for r in rs], dim=dim if dim >= 0 else dim - 2)
    return Rotation(rot_mats=rot_mats, quats=None)

Concatenates rotations along one of the batch dimensions. Analogous to torch.cat().

Note that the output of this operation is always a rotation matrix, regardless of the format of input
rotations.

Args:
rs:
A list of rotation objects
dim:
The dimension along which the rotations should be concatenated
Returns:
A concatenated Rotation object in rotation matrix format


def _handle_request(self, request):
    if (request is None):
        return Response(success=False, uid=request.uid)
    action_map = {'start_dag': self._handle_start_dag, 'stop_workflow': self._handle_stop_workflow, 'join_dags': self._handle_join_dags, 'stop_dag': self._handle_stop_dag, 'is_dag_stopped': self._handle_is_dag_stopped}
    if (request.action in action_map):
        return action_map[request.action](request)
    else:
        raise RequestActionUnknown()

Handle an incoming request by forwarding it to the appropriate method.

Args:
request (Request): Reference to a request object containing the
incoming request.

Raises:
RequestActionUnknown: If the action specified in the request is not known.

Returns:
Response: A response object containing the response from the method handling
the request.


def get(self, url, headers=None, parameters=None, get_json=True):

        if self.debug:
            print("GET: %s, headers=%s" % (url, headers))

        self.headers = self._get_default_headers()
        get_parameters = self.parameters
        if get_parameters is None:

            get_parameters = {}
        if headers is not None:
            self.headers.update(headers)
        if parameters is not None:
            get_parameters.update(parameters)

        response = requests.get(url, headers=self.headers, params=get_parameters, auth=self.auth, verify=self.verify_ssl)
        json_response = self._process_json_response(response)

        return json_response if get_json is True else response

Send a GET request with custome headers and parameters

Args:
url (str): URL to send the request to
headers (str, optional): custom headers
parameters (str, optional): optional parameters

Returns:
A JSON object of the returned response if `get_json` is True,
Requests' response object otherwise


def get_target_variable(self, targets, variable):
    if isinstance(targets, str):
        targets = [targets]
    assert is_iterable(targets)
    assert isinstance(variable, basestring)
    return bjam_interface.call('get-target-variable', targets, variable)

Gets the value of `variable` on set on the first target in `targets`.

Args:
targets (str or list): one or more targets to get the variable from.
variable (str): the name of the variable

Returns:
the value of `variable` set on `targets` (list)

Example:

>>> ENGINE = get_manager().engine()
>>> ENGINE.set_target_variable(targets, 'MY-VAR', 'Hello World')
>>> ENGINE.get_target_variable(targets, 'MY-VAR')
['Hello World']

Equivalent Jam code:

MY-VAR on $(targets) = "Hello World" ;
echo [ on $(targets) return $(MY-VAR) ] ;
"Hello World"


def __init__(self, latent_size, hidden_size):

    super(EncoderDynamicFull, self).__init__()
    self.latent_size = latent_size
    self.hidden_size = hidden_size
    self.bilstm = tf.keras.layers.Bidirectional(
        tf.keras.layers.LSTM(hidden_size, return_sequences=True),
        merge_mode="sum")
    self.rnn = tf.keras.layers.SimpleRNN(hidden_size, return_sequences=True)
    self.output_layer = tf.keras.layers.Dense(2*latent_size)

Constructs a "full" encoder for `z_t`.

Args:
latent_size: An integer corresponding to the
dimensionality of the distribution.
hidden_size: Dimensionality of the LSTM, RNN, and affine function
parameters.


def sg_concat(tensor, opt):
    r
    assert opt.target is not None, 'target is mandatory.'
    opt += tf.sg_opt(axis=tensor.get_shape().ndims-1)
    target = opt.target if isinstance(opt.target, (tuple, list)) else [opt.target]
    return tf.concat([tensor] + target, opt.axis, name=opt.name)

r"""Concatenates tensors along a axis.

See `tf.concat()` in tensorflow.

Args:
tensor: A `Tensor` (automatically given by chain).
opt:
target: A `Tensor`. Must have the same rank as `tensor`, and
all dimensions except `opt.dim` must be equal.
axis : Target axis. Default is the last one.
name: If provided, replace current tensor's name.

Returns:
A `Tensor`.


def complement(self, alphabet):

        self._addsink(alphabet)
        for state in self.automaton.states():
            if self.automaton.final(state) == fst.Weight.One(self.automaton.weight_type()):
                self.automaton.set_final(state, fst.Weight.Zero(self.automaton.weight_type()))
            else:
                self.automaton.set_final(state, fst.Weight.One(self.automaton.weight_type()))

Returns the complement of DFA
Args:
alphabet (list): The input alphabet
Returns:
None


def _generate_composite(self, comp_node, keepables):
    if (comp_node.name in self.datasets):
        return
    (compositor, prereqs, optional_prereqs) = comp_node.data
    try:
        prereq_datasets = self._get_prereq_datasets(comp_node.name, prereqs, keepables)
    except KeyError:
        return
    optional_datasets = self._get_prereq_datasets(comp_node.name, optional_prereqs, keepables, skip=True)
    try:
        composite = compositor(prereq_datasets, optional_datasets=optional_datasets, **self.attrs)
        cid = DatasetID.from_dict(composite.attrs)
        self.datasets[cid] = composite
        if (comp_node.name in self.wishlist):
            self.wishlist.remove(comp_node.name)
            self.wishlist.add(cid)
        comp_node.name = cid
    except IncompatibleAreas:
        LOG.debug('Delaying generation of %s because of incompatible areas', str(compositor.id))
        preservable_datasets = set(self.datasets.keys())
        prereq_ids = set((p.name for p in prereqs))
        opt_prereq_ids = set((p.name for p in optional_prereqs))
        keepables |= (preservable_datasets & (prereq_ids | opt_prereq_ids))
        keepables.add(comp_node.name)
        return

Collect all composite prereqs and create the specified composite.

Args:
comp_node (Node): Composite Node to generate a Dataset for
keepables (set): `set` to update if any datasets are needed
when generation is continued later. This can
happen if generation is delayed to incompatible
areas which would require resampling first.


def send_datagram(self, message, address, message_type='unicast'):
    if ((self.bufsize is not 0) and (len(message) > self.bufsize)):
        raise Exception(((('Datagram is too large. Messages should be ' + 'under ') + str(self.bufsize)) + ' bytes in size.'))
    if (message_type == 'broadcast'):
        self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    elif (message_type == 'multicast'):
        self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
    try:
        logger.debug('Sending packet')
        self.sock.sendto(message, address)
        if self.stats_enabled:
            self.stats['bytes_sent'] += len(message)
    except socket.error:
        logger.error('Failed to send, [Errno 101]: Network is unreachable.')

Sends a UDP datagram packet to the requested address.

Datagrams can be sent as a "unicast", "multicast", or "broadcast"
message. Unicast messages are messages that will be sent to a single
host, multicast messages will be delivered to all hosts listening for
multicast messages, and broadcast messages will be delivered to ALL
hosts on the network.

Args:
message (str): The raw serialized packet data to send.
address (tuple): The address and port of the destination to send the
packet. E.g. (address, port)
message_type -- The type of packet to send. Can be "unicast",
"multicast", or "broadcast". Defaults to "unicast".

Returns:
None


def _prepare_for_training(self, job_name=None):

        if job_name is not None:
            self._current_job_name = job_name
        else:

            if self.base_job_name:
                base_name = self.base_job_name
            elif isinstance(self, sagemaker.algorithm.AlgorithmEstimator):
                base_name = self.algorithm_arn.split('/')[-1]
            else:
                base_name = base_name_from_image(self.train_image())

            self._current_job_name = name_from_base(base_name)

        if self.output_path is None:
            local_code = get_config_value('local.local_code', self.sagemaker_session.config)
            if self.sagemaker_session.local_mode and local_code:
                self.output_path = ''
            else:
                self.output_path = 's3:

Set any values in the estimator that need to be set before training.

Args:
* job_name (str): Name of the training job to be created. If not specified, one is generated,
using the base name given to the constructor if applicable.


def is_course_in_catalog(self, catalog_id, course_id):
    try:
        course_run_id = str(CourseKey.from_string(course_id))
    except InvalidKeyError:
        course_run_id = None
    endpoint = self.client.catalogs(catalog_id).contains
    if course_run_id:
        resp = endpoint.get(course_run_id=course_run_id)
    else:
        resp = endpoint.get(course_id=course_id)
    return resp.get('courses', {}).get(course_id, False)

Determine if the given course or course run ID is contained in the catalog with the given ID.

Args:
catalog_id (int): The ID of the catalog
course_id (str): The ID of the course or course run

Returns:
bool: Whether the course or course run is contained in the given catalog


def get_tensor(self, tensor_index, subgraph_index=0):
    return self._interpreter.GetTensor(tensor_index, subgraph_index)

Gets the value of the output tensor (get a copy).

If you wish to avoid the copy, use `tensor()`. This function cannot be used
to read intermediate results.

Args:
tensor_index: Tensor index of tensor to get. This value can be gotten from
the 'index' field in get_output_details.
subgraph_index: Index of the subgraph to fetch the tensor. Default value
is 0, which means to fetch from the primary subgraph.

Returns:
a numpy array.


def delete_endpoint(self, endpoint_name=None):

        endpoint_name = endpoint_name or self.best_training_job()
        self.sagemaker_session.delete_endpoint(endpoint_name)

Delete an Amazon SageMaker endpoint.

If an endpoint name is not specified, this defaults to looking for an endpoint that
shares a name with the best training job for deletion.

Args:
endpoint_name (str): Name of the endpoint to delete


def nav_to_vcf_dir(ftp, build):

    if build == 'b37':
        ftp.cwd(DIR_CLINVAR_VCF_B37)
    elif build == 'b38':
        ftp.cwd(DIR_CLINVAR_VCF_B38)
    else:
        raise IOError("Genome build not recognized.")

Navigate an open ftplib.FTP to appropriate directory for ClinVar VCF files.

Args:
ftp:   (type: ftplib.FTP) an open connection to ftp.ncbi.nlm.nih.gov
build: (type: string) genome build, either 'b37' or 'b38'


def update_ports(self, ports, id_or_uri, timeout=(- 1)):
    resources = merge_default_values(ports, {'type': 'port'})
    uri = (self._client.build_uri(id_or_uri) + '/update-ports')
    return self._client.update(resources, uri, timeout)

Updates the interconnect ports.

Args:
id_or_uri: Can be either the interconnect id or the interconnect uri.
ports (list): Ports to update.
timeout: Timeout in seconds. Wait for task completion by default. The timeout does not abort the operation
in OneView; it just stops waiting for its completion.

Returns:
dict: The interconnect.


def __init__(self, input_size: int, num_experts: int, top_k: int):
    super().__init__()
    self.num_experts = num_experts
    self.input_size = input_size
    self.top_k = top_k
    self.layer = nn.Linear(input_size, num_experts, bias=False)

Initialize the top-k gating mechanism.

Args:
input_size (`int`):
Size of the input.
num_experts (`int`):
Number of experts.
top_k (`int`):
Number of top experts to select.