def load_data(self, mode="train", format="csv"):

        if mode in self.data.keys():
            run_dates = pd.DatetimeIndex(start=self.start_dates[mode],
                                        end=self.end_dates[mode],freq="1D")
            run_date_str = [d.strftime("%Y%m%d-%H%M") for d in run_dates.date]
            print(run_date_str)
            all_total_track_files = sorted(glob(getattr(self, mode + "_data_path") +
                                                "*total_" + self.ensemble_name + "*." + format))
            all_step_track_files = sorted(glob(getattr(self, mode + "_data_path") +
                                               "*step_" + self.ensemble_name + "*." + format))
            total_track_files = []
            for track_file in all_total_track_files:
                file_date = track_file.split("_")[-1][:-4]
                if file_date in run_date_str:
                    total_track_files.append(track_file)
            step_track_files = []
            for step_file in all_step_track_files:
                file_date = step_file.split("_")[-1][:-4]
                if file_date in run_date_str:
                    step_track_files.append(step_file)

            self.data[mode]["total"] = pd.concat(map(pd.read_csv, total_track_files),
                                                 ignore_index=True)
            self.data[mode]["total"] = self.data[mode]["total"].fillna(value=0)
            self.data[mode]["total"] = self.data[mode]["total"].replace([np.inf, -np.inf], 0)
            self.data[mode]["step"] = pd.concat(map(pd.read_csv, step_track_files),
                                                ignore_index=True)
            self.data[mode]["step"] = self.data[mode]["step"].fillna(value=0)
            self.data[mode]["step"] = self.data[mode]["step"].replace([np.inf, -np.inf], 0)
            if mode == "forecast":
                self.data[mode]["step"] = self.data[mode]["step"].drop_duplicates("Step_ID")
            self.data[mode]["member"] = pd.read_csv(self.member_files[mode])
            self.data[mode]["combo"] = pd.merge(self.data[mode]["step"],
                                                self.data[mode]["total"],
                                                on=["Track_ID", "Ensemble_Name", "Ensemble_Member", "Run_Date"])
            self.data[mode]["combo"] = pd.merge(self.data[mode]["combo"],
                                                self.data[mode]["member"],
                                                on="Ensemble_Member")
            self.data[mode]["total_group"] = pd.merge(self.data[mode]["total"],
                                                      self.data[mode]["member"],
                                                      on="Ensemble_Member")

Load data from flat data files containing total track information and information about each timestep.
The two sets are combined using merge operations on the Track IDs. Additional member information is gathered
from the appropriate member file.
Args:
mode: "train" or "forecast"
format:  file format being used. Default is "csv"


def _compareProtos(self, batch_shape, sizes, fields, field_dict):
    for field in fields:
        values = field_dict[field.name]
        self.assertEqual(dtypes.as_dtype(values.dtype), field.dtype)
        if 'ext_value' in field.name:
            fd = test_example_pb2.PrimitiveValue()
        else:
            fd = field.value.DESCRIPTOR.fields_by_name[field.name]
        self.assertEqual(list(values.shape)[:-1], batch_shape)
        if field.name == 'message_value' or 'ext_value' in field.name:
            vs = []
            for buf in values.flat:
                msg = test_example_pb2.PrimitiveValue()
                msg.ParseFromString(buf)
                vs.append(msg)
            if 'ext_value' in field.name:
                evs = field.value.Extensions[test_example_pb2.ext_value]
            else:
                evs = getattr(field.value, field.name)
            if len(vs) != len(evs):
                self.fail('Field %s decoded %d outputs, expected %d' % (fd.name, len(vs), len(evs)))
            for v, ev in zip(vs, evs):
                self.assertEqual(v, ev)
            continue
        tf_type_to_primitive_value_field = {dtypes.bool: 'bool_value', dtypes.float32: 'float_value', dtypes.float64: 'double_value', dtypes.int8: 'int8_value', dtypes.int32: 'int32_value', dtypes.int64: 'int64_value', dtypes.string: 'string_value', dtypes.uint8: 'uint8_value', dtypes.uint32: 'uint32_value', dtypes.uint64: 'uint64_value'}
        if field.name in ['enum_value', 'enum_value_with_default']:
            tf_field_name = 'enum_value'
        else:
            tf_field_name = tf_type_to_primitive_value_field.get(field.dtype)
        if tf_field_name is None:
            self.fail('Unhandled tensorflow type %d' % field.dtype)
        self._compareValues(fd, values.flat, getattr(field.value, tf_field_name))

Compare protos of type TestValue.

Args:
batch_shape: the shape of the input tensor of serialized messages.
sizes: int matrix of repeat counts returned by decode_proto
fields: list of test_example_pb2.FieldSpec (types and expected values)
field_dict: map from field names to decoded numpy tensors of values


def meta_graph_filename(checkpoint_filename, meta_graph_suffix='meta'):
    basename = re.sub('-[\\d\\?]+-of-\\d+$', '', checkpoint_filename)
    suffixed_filename = '.'.join([basename, meta_graph_suffix])
    return suffixed_filename

Returns the meta graph filename.

Args:
checkpoint_filename: Name of the checkpoint file.
meta_graph_suffix: Suffix for `MetaGraphDef` file. Defaults to 'meta'.

Returns:
MetaGraph file name.


class MLCDEncoder(CLIPEncoder):

    def __init__(self, config: MLCDVisionConfig):

        super().__init__(config)

    def forward(self, inputs_embeds: torch.FloatTensor, position_embeddings: Tuple[torch.Tensor, torch.Tensor], attention_mask: Optional[torch.Tensor]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, return_dict: Optional[bool]=None) -> Union[Tuple, BaseModelOutput]:

        output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None
        hidden_states = inputs_embeds
        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(encoder_layer.__call__, hidden_states, position_embeddings, attention_mask, output_attentions)
            else:
                layer_outputs = encoder_layer(hidden_states=hidden_states, position_embeddings=position_embeddings, attention_mask=attention_mask, output_attentions=output_attentions)
            hidden_states = layer_outputs[0]
            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)
        if not return_dict:
            return tuple((v for v in [hidden_states, encoder_states, all_attentions] if v is not None))
        return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions)

Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`MLCDEncoderLayer`].

Args:
config: MLCDVisionConfig


def _initialize_mesh_dimension_name_to_size(self, mesh_shape):

    mesh_dimension_name_to_size = {}
    for mesh_dimension in mesh_shape.dims:
      mesh_dimension_name_to_size[mesh_dimension.name] = mesh_dimension.size
    return mesh_dimension_name_to_size

Initializer for self._mesh_dimension_name_to_size.

Args:
mesh_shape: an mtf.Shape.

Returns:
A {string: int} mapping mesh dimension names to their sizes.


def from_celery(cls, broker_dict):

        return BrokerStats(
            hostname=broker_dict['hostname'],
            port=broker_dict['port'],
            transport=broker_dict['transport'],
            virtual_host=broker_dict['virtual_host']
        )

Create a BrokerStats object from the dictionary returned by celery.

Args:
broker_dict (dict): The dictionary as returned by celery.

Returns:
BrokerStats: A fully initialized BrokerStats object.


def operate(self, point):

        affine_point = np.array([point[0], point[1], point[2], 1])
        return np.dot(self.affine_matrix, affine_point)[0:3]

Apply the operation on a point.

Args:
point: Cartesian coordinate.

Returns:
Coordinates of point after operation.


def fn(temperature_format: str):
    return -40.0

Test function

Args:
temperature_format: The temperature format to use (Choices: ["celsius", "fahrenheit"])

Returns:
The temperature


def ismount(self, path):
    path = make_string_path(path)
    if (not path):
        return False
    normed_path = self.filesystem.absnormpath(path)
    sep = self.filesystem._path_separator(path)
    if self.filesystem.is_windows_fs:
        if (self.filesystem.alternative_path_separator is not None):
            path_seps = (sep, self.filesystem._alternative_path_separator(path))
        else:
            path_seps = (sep,)
        (drive, rest) = self.filesystem.splitdrive(normed_path)
        if (drive and (drive[:1] in path_seps)):
            return ((not rest) or (rest in path_seps))
        if (rest in path_seps):
            return True
    for mount_point in self.filesystem.mount_points:
        if (normed_path.rstrip(sep) == mount_point.rstrip(sep)):
            return True
    return False

Return true if the given path is a mount point.

Args:
path: Path to filesystem object to be checked

Returns:
`True` if path is a mount point added to the fake file system.
Under Windows also returns True for drive and UNC roots
(independent of their existence).


def dp004(self, value=None):
    if (value is not None):
        try:
            value = float(value)
        except ValueError:
            raise ValueError('value {} need to be of type float for field `dp004`'.format(value))
    self._dp004 = value

Corresponds to IDD Field `dp004`
Dew-point temperature corresponding to 0.4% annual cumulative frequency of occurrence

Args:
value (float): value for IDD Field `dp004`
Unit: C
if `value` is None it will not be checked against the
specification and is assumed to be a missing value

Raises:
ValueError: if `value` is not a valid value


def tasks(self, filters=None):

        if filters is None:
            filters = {}
        filters['service'] = self.id
        return self.client.api.tasks(filters=filters)

List the tasks in this service.

Args:
filters (dict): A map of filters to process on the tasks list.
Valid filters: ``id``, ``name``, ``node``,
``label``, and ``desired-state``.

Returns:
:py:class:`list`: List of task dictionaries.

Raises:
:py:class:`docker.errors.APIError`
If the server returns an error.


def get_subject_without_validation(jwt_bu64):

    try:
        jwt_dict = get_jwt_dict(jwt_bu64)
    except JwtException as e:
        return log_jwt_bu64_info(logging.error, str(e), jwt_bu64)
    try:
        return jwt_dict['sub']
    except LookupError:
        log_jwt_dict_info(logging.error, 'Missing "sub" key', jwt_dict)

Extract subject from the JWT without validating the JWT.

- The extracted subject cannot be trusted for authn or authz.

Args:
jwt_bu64: bytes
JWT, encoded using a a URL safe flavor of Base64.

Returns:
str: The subject contained in the JWT.


def to_csv(self, filename=None, encoding=export.ENCODING, dialect=export.DIALECT, make_filename=export.MAKE_FILENAME):
    if (filename is None):
        if (make_filename is None):
            make_filename = export.MAKE_FILENAME
        infos = {'id': self._spreadsheet._id, 'title': self._spreadsheet._title, 'sheet': self._title, 'gid': self._id, 'index': self._index, 'dialect': dialect}
        if isinstance(make_filename, string_types):
            filename = (make_filename % infos)
        else:
            filename = make_filename(infos)
    with export.open_csv(filename, 'w', encoding=encoding) as fd:
        export.write_csv(fd, self._values, encoding, dialect)

Dump the worksheet to a CSV file.

Args:
filename (str): result filename (if ``None`` use ``make_filename``)
encoding (str): result string encoding
dialect (str): :mod:`csv` dialect name or object to use
make_filename: template or one-argument callable returning the filename

If ``make_filename`` is a string, it is string-interpolated with an
infos-dictionary with the fields ``id`` (spreadhseet id), ``title``
(spreadsheet title), ``sheet`` (worksheet title), ``gid`` (worksheet
id), ``index`` (worksheet index), and ``dialect`` CSV dialect to
generate the filename: ``filename = make_filename % infos``.

If ``make_filename`` is a callable, it will be called with the
infos-dictionary to generate the filename:
``filename = make_filename(infos)``.


def app_trim_memory(self, pid: (int or str), level: str='RUNNING_LOW') -> None:
    (_, error) = self._execute('-s', self.device_sn, 'shell', 'am', 'send-trim-memory', str(pid), level)
    if (error and error.startswith('Error')):
        raise ApplicationsException(error.split(':', 1)[(- 1)].strip())

Trim memory.

Args:
level: HIDDEN | RUNNING_MODERATE | BACKGROUNDRUNNING_LOW | \
MODERATE | RUNNING_CRITICAL | COMPLETE


def from_entity(entity, self_user_id):
    user_id = UserID(chat_id=entity.id.chat_id, gaia_id=entity.id.gaia_id)
    return User(user_id, entity.properties.display_name, entity.properties.first_name, entity.properties.photo_url, entity.properties.email, ((self_user_id == user_id) or (self_user_id is None)))

Construct user from ``Entity`` message.

Args:
entity: ``Entity`` message.
self_user_id (~hangups.user.UserID or None): The ID of the current
user. If ``None``, assume ``entity`` is the current user.

Returns:
:class:`~hangups.user.User` object.


def to_csv(self,
               filename=None,
               as_text=True,
               use_descriptions=False,
               dlm=",",
               header=True):

        if (filename is None):
            if (not as_text):
                raise StriplogError("You must provide a filename or set as_text to True.")
        else:
            as_text = False

        if as_text:
            output = StringIO()
        else:
            output = open(filename, 'w')

        fieldnames = ['Top', 'Base', 'Component']
        writer = csv.DictWriter(output,
                                delimiter=dlm,
                                fieldnames=fieldnames,
                                quoting=csv.QUOTE_MINIMAL)

        if header:
            writer.writeheader()

        for i in self.__list:
            if use_descriptions and i.description:
                text = i.description
            elif i.primary:
                text = i.primary.summary()
            else:
                text = ''
            data = {j: k for j, k in zip(fieldnames, [i.top.z, i.base.z, text])}
            writer.writerow(data)

        if as_text:
            return output.getvalue()

        else:
            output.close
            return None

Returns a CSV string built from the summaries of the Intervals.

Args:
use_descriptions (bool): Whether to use descriptions instead
of summaries, if available.
dlm (str): The delimiter.
header (bool): Whether to form a header row.

Returns:
str: A string of comma-separated values.


def deprocess_input(input_array, input_range=(0, 255)):
    input_array = input_array.copy()
    input_array -= input_array.mean()
    input_array /= (input_array.std() + K.epsilon())
    input_array *= 0.1
    input_array += 0.5
    input_array = np.clip(input_array, 0, 1)
    return (((input_range[1] - input_range[0]) * input_array) + input_range[0])

Utility function to scale the `input_array` to `input_range` throwing away high frequency artifacts.

Args:
input_array: An N-dim numpy array.
input_range: Specifies the input range as a `(min, max)` tuple to rescale the `input_array`.

Returns:
The rescaled `input_array`.


def _PrintVSSStoreIdentifiersOverview(
      self, volume_system, volume_identifiers):

    header = 'The following Volume Shadow Snapshots (VSS) were found:\n'
    self._output_writer.Write(header)

    column_names = ['Identifier', 'Creation Time']
    table_view = views.CLITabularTableView(column_names=column_names)

    for volume_identifier in volume_identifiers:
      volume = volume_system.GetVolumeByIdentifier(volume_identifier)
      if not volume:
        raise errors.SourceScannerError(
            'Volume missing for identifier: {0:s}.'.format(
                volume_identifier))

      volume_attribute = volume.GetAttribute('creation_time')
      filetime = dfdatetime_filetime.Filetime(timestamp=volume_attribute.value)
      creation_time = filetime.CopyToDateTimeString()

      if volume.HasExternalData():
        creation_time = '{0:s}\tWARNING: data stored outside volume'.format(
            creation_time)

      table_view.AddRow([volume.identifier, creation_time])

    self._output_writer.Write('\n')
    table_view.Write(self._output_writer)
    self._output_writer.Write('\n')

Prints an overview of VSS store identifiers.

Args:
volume_system (dfvfs.VShadowVolumeSystem): volume system.
volume_identifiers (list[str]): allowed volume identifiers.

Raises:
SourceScannerError: if a volume cannot be resolved from the volume
identifier.


def connect(address, authkey):
    TFManager.register('get_queue')
    TFManager.register('get')
    TFManager.register('set')
    m = TFManager(address, authkey=authkey)
    m.connect()
    return m

Connect to a multiprocess.Manager.

Args:
:address: unique address to the TFManager, either a unique connection string for 'local', or a (host, port) tuple for remote.
:authkey: string authorization key

Returns:
A TFManager instance referencing the remote TFManager at the supplied address.


def _should_save_model(self, epoch, batch, logs, filepath):
    logs = logs or {}
    if self.save_best_only:
        current = logs.get(self.monitor)
        if current is None:
            warnings.warn(f'Can save best model only with {self.monitor} available.', stacklevel=2)
            return True
        elif (isinstance(current, np.ndarray) or backend.is_tensor(current)) and len(current.shape) > 0:
            warnings.warn(f'Can save best model only when `monitor` is a scalar value. Received: {current}. Falling back to `save_best_only=False`.')
            return True
        else:
            best_str = 'None' if self.best is None else f'{self.best:.5f}'
            if self._is_improvement(current, self.best):
                if self.verbose > 0:
                    io_utils.print_msg(f'\nEpoch {epoch + 1}: {self.monitor} improved from {best_str} to {current:.5f}, saving model to {filepath}')
                self.best = current
                return True
            else:
                if self.verbose > 0:
                    io_utils.print_msg(f'\nEpoch {epoch + 1}: {self.monitor} did not improve from {best_str}')
                return False
    else:
        if self.verbose > 0:
            io_utils.print_msg(f'\nEpoch {epoch + 1}: saving model to {filepath}')
        return True

Determines whether the model should be saved.

The model should be saved in the following cases:

- self.save_best_only is False
- self.save_best_only is True and `monitor` is a numpy array or
backend tensor (falls back to `save_best_only=False`)
- self.save_best_only is True and `self.monitor_op(current, self.best)`
evaluates to True.

Args:
epoch: the epoch this iteration is in.
batch: the batch this iteration is in. `None` if the `save_freq`
is set to `"epoch"`.
logs: the `logs` dict passed in to `on_batch_end` or
`on_epoch_end`.
filepath: the path where the model would be saved


def _post_process_apply(self, result_data, axis, try_scale=True):

        if try_scale:
            try:
                internal_index = self.compute_index(0, result_data, True)
            except IndexError:
                internal_index = self.compute_index(0, result_data, False)
            try:
                internal_columns = self.compute_index(1, result_data, True)
            except IndexError:
                internal_columns = self.compute_index(1, result_data, False)
        else:
            internal_index = self.compute_index(0, result_data, False)
            internal_columns = self.compute_index(1, result_data, False)
        if not axis:
            index = internal_index

            if len(internal_columns) != len(self.columns):
                columns = internal_columns
            else:
                columns = self.columns
        else:
            columns = internal_columns

            if len(internal_index) != len(self.index):
                index = internal_index
            else:
                index = self.index
        return self.__constructor__(result_data, index, columns)

Recompute the index after applying function.

Args:
result_data: a BaseFrameManager object.
axis: Target axis along which function was applied.

Returns:
A new PandasQueryCompiler.


def on_http_error(error):

    def wrap(f):
        @functools.wraps(f)
        def wrapped_f(*args, **kwargs):
            try:
                return f(*args, **kwargs)
            except GitlabHttpError as e:
                raise error(e.error_message, e.response_code, e.response_body)
        return wrapped_f
    return wrap

Manage GitlabHttpError exceptions.

This decorator function can be used to catch GitlabHttpError exceptions
raise specialized exceptions instead.

Args:
error(Exception): The exception type to raise -- must inherit from
GitlabError


def is_a_tty(stream=sys.stdout):
    result = (stream.isatty() if hasattr(stream, 'isatty') else None)
    log.debug(result)
    return result

Detect terminal or something else, such as output redirection.

Returns:
Boolean, None: is tty or None if not found.


def mac(self, algorithm, key, data):
    mac_data = None
    if (algorithm in self._hash_algorithms.keys()):
        self.logger.info('Generating a hash-based message authentication code using {0}'.format(algorithm.name))
        hash_algorithm = self._hash_algorithms.get(algorithm)
        try:
            h = hmac.HMAC(key, hash_algorithm(), backend=default_backend())
            h.update(data)
            mac_data = h.finalize()
        except Exception as e:
            self.logger.exception(e)
            raise exceptions.CryptographicFailure('An error occurred while computing an HMAC. See the server log for more information.')
    elif (algorithm in self._symmetric_key_algorithms.keys()):
        self.logger.info('Generating a cipher-based message authentication code using {0}'.format(algorithm.name))
        cipher_algorithm = self._symmetric_key_algorithms.get(algorithm)
        try:
            c = cmac.CMAC(cipher_algorithm(key), backend=default_backend())
            c.update(data)
            mac_data = c.finalize()
        except Exception as e:
            raise exceptions.CryptographicFailure('An error occurred while computing a CMAC. See the server log for more information.')
    else:
        raise exceptions.InvalidField('The cryptographic algorithm ({0}) is not a supported for a MAC operation.'.format(algorithm))
    return mac_data

Generate message authentication code.

Args:
algorithm(CryptographicAlgorithm): An enumeration specifying the
algorithm for which the MAC operation will use.
key(bytes): secret key used in the MAC operation
data(bytes): The data to be MACed.

Returns:
bytes: The MACed data

Raises:
InvalidField: Raised when the algorithm is unsupported or the
length is incompatible with the algorithm.
CryptographicFailure: Raised when the key generation process
fails.

Example:
>>> engine = CryptographyEngine()
>>> mac_data = engine.mac(
...     CryptographicAlgorithm.HMAC-SHA256, b'\x01\x02\x03\x04',
...     b'\x05\x06\x07\x08')


def populate(self, obj):

        if type(obj) is AtlasServiceInstance.Instance:
            query = { "instance_id" : obj.instance_id, "binding_id" : { "$exists" : False } }
        elif type(obj) is AtlasServiceBinding.Binding:
            query = { "binding_id" : obj.binding_id, "instance_id" : obj.instance.instance_id }
        else:
            raise ErrStorageTypeUnsupported(type(obj))

        try:
            result = self.broker.find_one(query)
        except:
            raise ErrStorageMongoConnection("Populate Instance or Binding")

        if result is not None:
            obj.parameters = result["parameters"]

            obj.provisioned = True
        else:

            obj.provisioned = False

Populate

Query mongo to get information about the obj if it exists

Args:
obj (AtlasServiceBinding.Binding or AtlasServiceInstance.Instance): instance or binding

Raises:
ErrStorageTypeUnsupported: Type unsupported.
ErrStorageMongoConnection: Error during MongoDB communication.


def update_permissions(self, grp_name, resource, permissions):
    self.project_service.set_auth(self._token_project)
    self.project_service.update_permissions(grp_name, resource, permissions)

Update permissions for the group associated with the given resource.

Args:
grp_name (string): Name of group.
resource (intern.resource.boss.Resource): Identifies which data
model object to operate on
permissions (list): List of permissions to add to the given resource

Raises:
requests.HTTPError on failure.


def smoothing_cross_entropy(logits, labels, vocab_size, confidence, gaussian=False):
    with tf.name_scope('smoothing_cross_entropy', values=[logits, labels]):
        low_confidence = ((1.0 - confidence) / to_float((vocab_size - 1)))
        normalizing = (- ((confidence * tf.log(confidence)) + ((to_float((vocab_size - 1)) * low_confidence) * tf.log((low_confidence + 1e-20)))))
        if (gaussian and (confidence > 0.0)):
            labels = tf.cast(labels, tf.float32)
            normal_dist = tfp.distributions.Normal(loc=labels, scale=confidence)
            soft_targets = normal_dist.prob(tf.cast(tf.range(vocab_size), tf.float32)[(:, None, None, None, None)])
            soft_targets = tf.transpose(soft_targets, perm=[1, 2, 3, 4, 0])
        else:
            soft_targets = tf.one_hot(tf.cast(labels, tf.int32), depth=vocab_size, on_value=confidence, off_value=low_confidence)
        xentropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=soft_targets)
        return (xentropy - normalizing)

Cross entropy with label smoothing to limit over-confidence.

Args:
logits: Tensor of shape [batch_size, ?, ?, ?, vocab_size].
labels: Tensor of shape [batch_size, ?, ?, ?].
vocab_size: Tensor representing the size of the vocabulary.
confidence: Used to determine on and off values for label smoothing.
If `gaussian` is true, `confidence` is the variance to the Gaussian
distribution.
gaussian: Uses a Gaussian distribution for label smoothing

Returns:
Tensor of shape [batch_size, ?, ?, ?].


def strace_set_buffer_size(self, size):
    size = ctypes.c_uint32(size)
    res = self._dll.JLINK_STRACE_Control(enums.JLinkStraceCommand.SET_BUFFER_SIZE, size)
    if (res < 0):
        raise errors.JLinkException('Failed to set the STRACE buffer size.')
    return None

Sets the STRACE buffer size.

Args:
self (JLink): the ``JLink`` instance.

Returns:
``None``

Raises:
JLinkException: on error.


def compute_gradients(self, loss, var_list, global_step=None, gate_gradients=GATE_OP, aggregation_method=None, colocate_gradients_with_ops=False, name=None, grad_loss=None):
    del global_step, name
    return self._momentum_optimizer.compute_gradients(loss, var_list=var_list, gate_gradients=gate_gradients, aggregation_method=aggregation_method, colocate_gradients_with_ops=colocate_gradients_with_ops, grad_loss=grad_loss)

Compute gradients through momentum optimizer.

Args:
loss: A Tensor containing the value to minimize.
var_list: Optional list or tuple of tf.Variable to update
to minimize loss. Defaults to the list of variables collected
in the graph under the key GraphKey.TRAINABLE_VARIABLES.
global_step: Optional Variable to increment by one after the
variables have been updated.
gate_gradients: How to gate the computation of gradients.
Can be GATE_NONE, GATE_OP, or GATE_GRAPH.
aggregation_method: Specifies the method used to combine
gradient terms. Valid values are defined in the class AggregationMethod.
colocate_gradients_with_ops: If True, try collocating gradients with
the corresponding op.
name: Optional name for the returned operation. Default to the name
passed to the Optimizer constructor.
grad_loss: Optional. A Tensor holding the gradient computed for loss.

Returns:
A list of (gradient, variable) pairs. Variable is always present,
but gradient can be None.


def _ConvertMapFieldValue(self, value, message, field):

    if not isinstance(value, dict):
      raise ParseError(
          'Map field {0} must be in a dict which is {1}.'.format(
              field.name, value))
    key_field = field.message_type.fields_by_name['key']
    value_field = field.message_type.fields_by_name['value']
    for key in value:
      key_value = _ConvertScalarFieldValue(key, key_field, True)
      if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
        self.ConvertMessage(value[key], getattr(
            message, field.name)[key_value])
      else:
        getattr(message, field.name)[key_value] = _ConvertScalarFieldValue(
            value[key], value_field)

Convert map field value for a message map field.

Args:
value: A JSON object to convert the map field value.
message: A protocol message to record the converted data.
field: The descriptor of the map field to be converted.

Raises:
ParseError: In case of convert problems.


def list_filepaths(self, wildcard=None):

        fnames = [f for f in os.listdir(self.path)]
        filepaths = filter(os.path.isfile, [os.path.join(self.path, f) for f in fnames])

        if wildcard is not None:
            filepaths = WildCard(wildcard).filter(filepaths)

        return filepaths

Return the list of absolute filepaths in the directory.

Args:
wildcard: String of tokens separated by "|". Each token represents a pattern.
If wildcard is not None, we return only those files that match the given shell pattern (uses fnmatch).
Example:
wildcard="*.nc|*.pdf" selects only those files that end with .nc or .pdf


def hash_(token: str, hash_size: int) -> int:

    return murmurhash3_32(token, positive=True) % hash_size

Convert a token to a hash of given size.
Args:
token: a word
hash_size: hash size

Returns:
int, hashed token


def compute_one_match(self, var, other_type, name=None, match_all_views=True, keep_all_views=False, alias_map=None) -> MatchResult:
    bad_matches = []
    good_matches = _UniqueMatches(self._node, keep_all_views)
    views = abstract_utils.get_views([var], self._node)
    skip_future = None
    while True:
        try:
            view = views.send(skip_future)
        except StopIteration:
            break
        subst = datatypes.AliasingDict(aliases=alias_map)
        subst = self.match_var_against_type(var, other_type, subst, view)
        if subst is None:
            if self._node.CanHaveCombination(list(view.values())):
                bad_matches.append(BadMatch(view=view, expected=self._get_bad_type(name, other_type), actual=var))
            skip_future = False
        else:
            skip_future = True
            good_matches.insert(view, subst)
    good_matches = [GoodMatch(view, datatypes.HashableDict(subst)) for view, subst in good_matches.unique()]
    if good_matches and (not match_all_views) or not bad_matches:
        success = True
    elif good_matches:
        bad_matches = [m for m in bad_matches if self._node.HasCombination(list(m.view.values()))]
        success = not bad_matches
    else:
        success = False
    return MatchResult(success=success, good_matches=good_matches, bad_matches=bad_matches)

Match a Variable against a type.

Args:
var: A cfg.Variable, containing instances.
other_type: An instance of BaseValue.
name: Optionally, the variable name.
match_all_views: If True, every possible match must succeed for the
overall match to be considered a success. Otherwise, the overall match
succeeds as long as at least one possible match succeeds.
keep_all_views: If True, avoid optimizations that discard views.
alias_map: Optionally, a datatypes.UnionFind, which stores all the type
renaming information, mapping of type parameter name to its
representative.

Returns:
The match result.


def isfile(self, path=None, client_kwargs=None, assume_exists=None):
    relative = self.relpath(path)
    if (not relative):
        return False
    if ((path[(- 1)] != '/') and (not self.is_locator(path, relative=True))):
        return self.exists(path=path, client_kwargs=client_kwargs, assume_exists=assume_exists)
    return False

Return True if path is an existing regular file.

Args:
path (str): Path or URL.
client_kwargs (dict): Client arguments.
assume_exists (bool or None): This value define the value to return
in the case there is no enough permission to determinate the
existing status of the file. If set to None, the permission
exception is reraised (Default behavior). if set to True or
False, return this value.

Returns:
bool: True if file exists.


def dice_loss(inputs: Tensor, labels: Tensor, num_masks: int) -> Tensor:
    probs = inputs.sigmoid().flatten(1)
    numerator = 2 * (probs * labels).sum(-1)
    denominator = probs.sum(-1) + labels.sum(-1)
    loss = 1 - (numerator + 1) / (denominator + 1)
    loss = loss.sum() / num_masks
    return loss

Compute the DICE loss, similar to generalized IOU for masks as follows:

$$ \mathcal{L}_{\text{dice}(x, y) = 1 - \frac{2 * x \cap y }{x \cup y + 1}} $$

In practice, since `labels` is a binary mask, (only 0s and 1s), dice can be computed as follow

$$ \mathcal{L}_{\text{dice}(x, y) = 1 - \frac{2 * x * y }{x + y + 1}} $$

Args:
inputs (`torch.Tensor`):
A tensor representing a mask.
labels (`torch.Tensor`):
A tensor with the same shape as inputs. Stores the binary classification labels for each element in inputs
(0 for the negative class and 1 for the positive class).
num_masks (`int`):
The number of masks present in the current batch, used for normalization.

Returns:
`torch.Tensor`: The computed loss.


def __init__(self, dimensions):

    super(LearnableMultivariateNormalDiag, self).__init__()
    with tf.compat.v1.name_scope(self._name):
      self.dimensions = dimensions
      self._mean = tf.compat.v2.Variable(
          tf.random.normal([dimensions], stddev=0.1), name="mean")

      self._untransformed_stddev = tf.compat.v2.Variable(
          tf.random.normal([dimensions], mean=0.55, stddev=0.1),
          name="untransformed_stddev")

Constructs a learnable multivariate diagonal normal model.

Args:
dimensions: An integer corresponding to the dimensionality of the
distribution.


def request(self, method, params=None):
    msg_id = self._id_generator()
    log.debug('Sending request with id %s: %s %s', msg_id, method, params)
    message = {'jsonrpc': JSONRPC_VERSION, 'id': msg_id, 'method': method}
    if (params is not None):
        message['params'] = params
    request_future = futures.Future()
    request_future.add_done_callback(self._cancel_callback(msg_id))
    self._server_request_futures[msg_id] = request_future
    self._consumer(message)
    return request_future

Send a JSON RPC request to the client.

Args:
method (str): The method name of the message to send
params (any): The payload of the message

Returns:
Future that will resolve once a response has been received


class LayoutLMv2FastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
    apply_ocr: Optional[bool]
    ocr_lang: Optional[str]
    tesseract_config: Optional[str]

Args:
apply_ocr (`bool`, *optional*, defaults to `True`):
Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes. Can be overridden by
the `apply_ocr` parameter in the `preprocess` method.
ocr_lang (`str`, *optional*):
The language, specified by its ISO code, to be used by the Tesseract OCR engine. By default, English is
used. Can be overridden by the `ocr_lang` parameter in the `preprocess` method.
tesseract_config (`str`, *optional*):
Any additional custom configuration flags that are forwarded to the `config` parameter when calling
Tesseract. For example: '--psm 6'. Can be overridden by the `tesseract_config` parameter in the
`preprocess` method.


def get_children_graph(self, item_ids=None, language=None, forbidden_item_ids=None):
    if (forbidden_item_ids is None):
        forbidden_item_ids = set()

    def _children(item_ids):
        if (item_ids is None):
            items = Item.objects.filter(active=True).prefetch_related('children')
        else:
            item_ids = [ii for iis in item_ids.values() for ii in iis]
            items = Item.objects.filter(id__in=item_ids, active=True).prefetch_related('children')
        return {item.id: sorted([_item.id for _item in item.children.all() if (_item.active and (_item.id not in forbidden_item_ids))]) for item in items if (item.id not in forbidden_item_ids)}
    if (item_ids is None):
        return self._reachable_graph(None, _children, language=language)
    else:
        graph = self.get_children_graph(None, language, forbidden_item_ids=forbidden_item_ids)
        return self._subset_graph(graph, (set(item_ids) - set(forbidden_item_ids)))

Get a subgraph of items reachable from the given set of items through
the 'child' relation.

Args:
item_ids (list): items which are taken as roots for the reachability
language (str): if specified, filter out items which are not
available in the given language

Returns:
dict: item id -> list of items (child items), root items are
referenced by None key


def __init__(self, tlv_type=1, sub_type=7, sub_value=None):

        super().__init__(tlv_type)
        self.sub_type = sub_type
        self.sub_value = BinaryData() if sub_value is None else sub_value

Create an instance and set its attributes.

Args:
tlv_type (int): Type used by this class. Defaults to 1.
sub_type (int): Sub type value used by this class. Defaults to 7.
sub_value (:class:`~pyof.foundation.basic_types.BinaryData`):
Data stored by TLVWithSubType. Defaults to empty BinaryData.


def _take_lease(self, lease, uuid_path, safe=True):
    if safe:
        lease_taken_by = self._lease_valid(lease)
        if (lease_taken_by and (lease_taken_by != uuid_path)):
            raise LagoSubnetLeaseTakenException(lease.subnet, lease_taken_by)
    with open(uuid_path) as f:
        uuid = f.read()
    with open(lease.path, 'wt') as f:
        utils.json_dump((uuid_path, uuid), f)
    LOGGER.debug('Assigned subnet lease {} to {}'.format(lease.path, uuid_path))

Persist the given lease to the store and make the prefix in uuid_path
his owner

Args:
lease(lago.subnet_lease.Lease): Object representation of the lease
uuid_path (str): Path to the prefix uuid
safe (bool): If true (the default), validate the the lease
isn't taken.

Raises:
LagoSubnetLeaseException: If safe == True and the lease is already
taken.


def determine_selection_of_iterable_values_from_config(config: DictLike, possible_iterables: Mapping[(str, Type[enum.Enum])]) -> Dict[(str, List[Any])]:
    iterables = {}
    requested_iterables = config['iterables']
    for (k, v) in requested_iterables.items():
        if (k not in possible_iterables):
            raise KeyError(k, f'Cannot find requested iterable in possible_iterables: {possible_iterables}')
        logger.debug(f'k: {k}, v: {v}')
        additional_iterable: List[Any] = []
        enum_values = possible_iterables[k]
        if isinstance(v, str):
            raise TypeError(type(v), f'Passed string {v} when must be either bool or list')
        if (v is False):
            continue
        elif (v is True):
            additional_iterable = list(enum_values)
        elif (enum_values is None):
            additional_iterable = list(v)
        else:
            for el in v:
                additional_iterable.append(enum_values[el])
        iterables[k] = additional_iterable
    return iterables

Determine iterable values to use to create objects for a given configuration.

All values of an iterable can be included be setting the value to ``True`` (Not as a single value list,
but as the only value.). Alternatively, an iterator can be disabled by setting the value to ``False``.

Args:
config: The dict-like configuration from ruamel.yaml which should be overridden.
possible_iterables: Key value pairs of names of enumerations and their values.
Returns:
dict: Iterables values that were requested in the config.


def in_search_queryset(self, instance_id, index='_all'):
    return self.get_search_queryset(index=index).filter(pk=instance_id).exists()

Return True if an object is part of the search index queryset.

Sometimes it's useful to know if an object _should_ be indexed. If
an object is saved, how do you know if you should push that change
to the search index? The simplest (albeit not most efficient) way
is to check if it appears in the underlying search queryset.

NB this method doesn't evaluate the entire dataset, it chains an
additional queryset filter expression on the end. That's why it's
important that the `get_search_queryset` method returns a queryset.

Args:
instance_id: the id of model object that we are looking for.

Kwargs:
index: string, the name of the index in which to check.
Defaults to '_all'.


class BufferedQuantileTracker(WindowedTracker, QuantileTracker):

    def __init__(self, window_mode, q, **kwargs):
        super().__init__(window_mode, **kwargs)
        QuantileTracker.__init__(self, q)
        self._sorted_items = SortedList()

    def push(self, x):

        if not math.isnan(x):
            self._sorted_items.add(x)
        if self._window_mode == WindowMode.SLIDING:
            if len(self._queue) >= self._window_size and (not math.isnan((old_x := self.pop()))):
                self._sorted_items.discard(old_x)
            super().push(x)

    @staticmethod
    def _get_helper(sorted_items, q):
        n = len(sorted_items)
        if n < 1:
            return float('nan')
        pos = q * (n - 1)
        lo = math.floor(pos)
        lo_value = typing.cast(float, sorted_items[lo])
        hi = min(lo + 1, n - 1)
        hi_value: float = typing.cast(float, sorted_items[hi])
        return lo_value + (hi_value - lo_value) * (pos - lo)

    def get(self):

        return self._get_helper(self._sorted_items, self._q)

Abstract base class for buffered quantile trackers.

Warning:
Buffered quantile trackers are NOT truly incremental in the sense that they
don't update the quantile in constant time per new data point. They maintain
a sorted list of all values in the window.

Args:
window_mode: A `WindowMode` enum specifying whether the window is `LANDMARK`
or `SLIDING`.
q: The quantile to calculate, a float between 0 and 1 (inclusive).
**kwargs: Keyword arguments passed to the parent class constructor.


def read(self, filename, encoding=None):

        with open(filename, encoding=encoding) as fp:
            self._read(fp, filename)
        self._filename = os.path.abspath(filename)

Read and parse a filename.

Args:
filename (str): path to file
encoding (str): encoding of file, default None


def _start_browsing_some_sites(self):
    browsers = self._browser_pool.acquire_multi(((self._browser_pool.num_available() + 1)
    try:
        sites = self._frontier.claim_sites(len(browsers))
    except:
        self._browser_pool.release_all(browsers)
        raise
    for i in range(len(browsers)):
        if (i < len(sites)):
            th = threading.Thread(target=self._brozzle_site_thread_target, args=(browsers[i], sites[i]), name=('BrozzlingThread:%s' % browsers[i].chrome.port), daemon=True)
            with self._browsing_threads_lock:
                self._browsing_threads.add(th)
            th.start()
        else:
            self._browser_pool.release(browsers[i])

Starts browsing some sites.

Raises:
NoBrowsersAvailable if none available


def parse(self, data):

        data = '\n'.join(self.strip(data.split('\n')))

        tag_re = re.compile(
            r'^:\n?(?P<full_tag>(?P<tag>[0-9]{2}|NS)(?P<sub_tag>[A-Z])?):',
            re.MULTILINE)
        matches = list(tag_re.finditer(data))

        valid_matches = list(self.sanatize_tag_id_matches(matches))

        for i, match in enumerate(valid_matches):
            tag_id = self.normalize_tag_id(match.group('tag'))

            tag = self.tags.get(match.group('full_tag')) \
                or self.tags[tag_id]

            if valid_matches[i + 1:]:
                tag_data = \
                    data[match.end():valid_matches[i + 1].start()].strip()
            else:
                tag_data = data[match.end():].strip()

            tag_dict = tag.parse(self, tag_data)

            for processor in self.processors.get('pre_%s' % tag.slug, []):
                tag_dict = processor(self, tag, tag_dict)

            result = tag(self, tag_dict)

            for processor in self.processors.get('post_%s' % tag.slug, []):
                result = processor(self, tag, tag_dict, result)

            if isinstance(tag, mt940.tags.Statement):

                if not self.transactions:
                    transaction = Transaction(self)
                    self.transactions.append(transaction)

                if transaction.data.get('id'):
                    transaction = Transaction(self, result)
                    self.transactions.append(transaction)
                else:
                    transaction.data.update(result)
            elif issubclass(tag.scope, Transaction) and self.transactions:

                for k, v in _compat.iteritems(result):
                    if k in transaction.data and hasattr(v, 'strip'):
                        transaction.data[k] += '\n%s' % v.strip()
                    else:
                        transaction.data[k] = v

            elif issubclass(tag.scope, Transactions):
                self.data.update(result)

        return self.transactions

Parses mt940 data, expects a string with data

Args:
data (str): The MT940 data

Returns: :py:class:`list` of :py:class:`Transaction`


def roc(self, model_type, model_name, intensity_threshold, prob_thresholds, query=None):
    roc_obj = DistributedROC(prob_thresholds, 0.5)
    if (query is not None):
        sub_forecasts = self.matched_forecasts[model_type][model_name].query(query)
        sub_forecasts = sub_forecasts.reset_index(drop=True)
    else:
        sub_forecasts = self.matched_forecasts[model_type][model_name]
    obs_values = np.zeros(sub_forecasts.shape[0])
    if (sub_forecasts.shape[0] > 0):
        if (model_type == 'dist'):
            forecast_values = np.array([gamma_sf(intensity_threshold, *params) for params in sub_forecasts[self.forecast_bins[model_type]].values])
            obs_probs = np.array([gamma_sf(intensity_threshold, *params) for params in sub_forecasts[self.type_cols[model_type]].values])
            obs_values[(obs_probs >= 0.01)] = 1
        elif (len(self.forecast_bins[model_type]) > 1):
            fbin = np.argmin(np.abs((self.forecast_bins[model_type] - intensity_threshold)))
            forecast_values = (1 - sub_forecasts[self.forecast_bins[model_type].astype(str)].values.cumsum(axis=1)[(:, fbin)])
            obs_values[(sub_forecasts[self.type_cols[model_type]].values >= intensity_threshold)] = 1
        else:
            forecast_values = sub_forecasts[self.forecast_bins[model_type].astype(str)[0]].values
            obs_values[(sub_forecasts[self.type_cols[model_type]].values >= intensity_threshold)] = 1
        roc_obj.update(forecast_values, obs_values)
    return roc_obj

Calculates a ROC curve at a specified intensity threshold.

Args:
model_type: type of model being evaluated (e.g. size).
model_name: machine learning model being evaluated
intensity_threshold: forecast bin used as the split point for evaluation
prob_thresholds: Array of probability thresholds being evaluated.
query: str to filter forecasts based on values of forecasts, obs, and metadata.

Returns:
A DistributedROC object


def get_foldrate_at_temp(ref_rate, new_temp, ref_temp=37.0):

    slope = 22000

    preFactor = float(ref_rate) + slope / (float(ref_temp) + 273.15)

    rate = math.exp(preFactor - slope / (float(new_temp) + 273.15))

    return rate

Scale the predicted kinetic folding rate of a protein to temperature T, based on the relationship ln(k_f)∝1/T

Args:
ref_rate (float): Kinetic folding rate calculated from the function :func:`~ssbio.protein.sequence.properties.kinetic_folding_rate.get_foldrate`
new_temp (float): Temperature in degrees C
ref_temp (float): Reference temperature, default to 37 C

Returns:
float: Kinetic folding rate k_f at temperature T


def decompress_decoder_1d(x, hparams, name=None):

  x = tf.expand_dims(x, axis=2)
  output = decompress_decoder(x, hparams,
                              strides=(2, 1),
                              kernel=(hparams.kernel_size, 1),
                              name=name)
  return tf.squeeze(output, axis=2)

Decoder that decompresses 1-D inputs by 2**num_compress_steps.

Args:
x: Tensor of shape [batch, compress_length, channels].
hparams: HParams.
name: string, variable scope.

Returns:
Tensor of shape [batch, length, hparams.hidden_size].


def update(self, settings):
    settings = self.clean(settings)
    self._settings.update(settings)
    self.set_settings(settings)
    return self._settings

Update object attributes from given settings

Args:
settings (dict): Dictionnary of elements to update settings.

Returns:
dict: Dictionnary of all current saved settings.


def _remove_from(self, db, index, item):
    with transaction.manager:
        row = db.get(index, None)
    if (row is None):
        return
    with transaction.manager:
        if (item in row):
            row.remove(item)
    with transaction.manager:
        if (not row):
            del db[index]

Remove `item` from `db` at `index`.

Note:
This function is inverse to :meth:`._add_to`.

Args:
db (dict-obj): Dict-like object used to connect to database.
index (str): Index used to look in `db`.
item (obj): Persistent object, which may be stored in DB.


def output_forecasts_csv(self, forecasts, mode, csv_path, run_date_format='%Y%m%d-%H%M'):
    merged_forecasts = pd.merge(forecasts['condition'], forecasts['dist'], on=['Step_ID', 'Track_ID', 'Ensemble_Member', 'Forecast_Hour'])
    all_members = self.data[mode]['combo']['Ensemble_Member']
    members = np.unique(all_members)
    all_run_dates = pd.DatetimeIndex(self.data[mode]['combo']['Run_Date'])
    run_dates = pd.DatetimeIndex(np.unique(all_run_dates))
    print(run_dates)
    for member in members:
        for run_date in run_dates:
            mem_run_index = ((all_run_dates == run_date) & (all_members == member))
            member_forecast = merged_forecasts.loc[mem_run_index]
            member_forecast.to_csv(join(csv_path, 'hail_forecasts_{0}_{1}_{2}.csv'.format(self.ensemble_name, member, run_date.strftime(run_date_format))))
    return

Output hail forecast values to csv files by run date and ensemble member.

Args:
forecasts:
mode:
csv_path:
Returns:


def latents_to_observations(self, latent_means, latent_covs):

    with tf.name_scope("latents_to_observations"):

      pushforward_latents_step = build_pushforward_latents_step(
          self.get_observation_matrix_for_timestep,
          self.get_observation_noise_for_timestep)

      latent_means = distribution_util.move_dimension(
          latent_means, source_idx=-2, dest_idx=0)
      latent_means = latent_means[..., tf.newaxis]
      latent_covs = distribution_util.move_dimension(
          latent_covs, source_idx=-3, dest_idx=0)

      (initial_observation_mean,
       initial_observation_cov) = pushforward_latents_step(
           _=None,
           latent_t_mean_cov=(self.initial_step,
                              latent_means[self.initial_step],
                              latent_covs[self.initial_step]))

      timesteps = tf.range(self.initial_step,
                           self.initial_step + self.num_timesteps)
      observation_means, observation_covs = tf.scan(
          pushforward_latents_step,
          elems=(timesteps, latent_means, latent_covs),
          initializer=(initial_observation_mean, initial_observation_cov),
          parallel_iterations=10000)

      observation_means = distribution_util.move_dimension(
          observation_means[..., 0], source_idx=0, dest_idx=-2)
      observation_covs = distribution_util.move_dimension(
          observation_covs, source_idx=0, dest_idx=-3)

      return observation_means, observation_covs

Push latent means and covariances forward through the observation model.

Args:
latent_means: float `Tensor` of shape `[..., num_timesteps, latent_size]`
latent_covs: float `Tensor` of shape
`[..., num_timesteps, latent_size, latent_size]`.

Returns:
observation_means: float `Tensor` of shape
`[..., num_timesteps, observation_size]`
observation_covs: float `Tensor` of shape
`[..., num_timesteps, observation_size, observation_size]`


def export_dqdv(cell_data, savedir, sep, last_cycle=None):

    logger.debug("exporting dqdv")
    filename = cell_data.dataset.loaded_from
    no_merged_sets = ""
    firstname, extension = os.path.splitext(filename)
    firstname += no_merged_sets
    if savedir:
        firstname = os.path.join(savedir, os.path.basename(firstname))
        logger.debug(f"savedir is true: {firstname}")

    outname_charge = firstname + "_dqdv_charge.csv"
    outname_discharge = firstname + "_dqdv_discharge.csv"

    list_of_cycles = cell_data.get_cycle_numbers()
    number_of_cycles = len(list_of_cycles)
    logger.debug("%s: you have %i cycles" % (filename, number_of_cycles))

    out_data = _extract_dqdv(cell_data, cell_data.get_ccap, last_cycle)
    logger.debug("extracted ica for charge")
    try:
        _save_multi(data=out_data, file_name=outname_charge, sep=sep)
    except ExportFailed:
        logger.info("could not export ica for charge")
    else:
        logger.debug("saved ica for charge")

    out_data = _extract_dqdv(cell_data, cell_data.get_dcap, last_cycle)
    logger.debug("extracxted ica for discharge")
    try:
        _save_multi(data=out_data, file_name=outname_discharge, sep=sep)
    except ExportFailed:
        logger.info("could not export ica for discharge")
    else:
        logger.debug("saved ica for discharge")

Exports dQ/dV data from a CellpyData instance.

Args:
cell_data: CellpyData instance
savedir: path to the folder where the files should be saved
sep: separator for the .csv-files.
last_cycle: only export up to this cycle (if not None)


def virt_conf_from_stream(
        self,
        conf_fd,
        template_repo=None,
        template_store=None,
        do_bootstrap=True,
        do_build=True,
    ):

        virt_conf = utils.load_virt_stream(conf_fd)
        LOGGER.debug('Loaded virt config:\n%s', virt_conf)
        return self.virt_conf(
            conf=virt_conf,
            template_repo=template_repo,
            template_store=template_store,
            do_bootstrap=do_bootstrap,
            do_build=do_build
        )

Initializes all the virt infrastructure of the prefix, creating the
domains disks, doing any network leases and creating all the virt
related files and dirs inside this prefix.

Args:
conf_fd (File): File like object to read the config from
template_repo (TemplateRepository): template repository intance
template_store (TemplateStore): template store instance

Returns:
None


def reset(self, label=None):
    if label:
        self.label = label
    self.times = []
    self.n_loops = None
    self.total_time = None
    return self

clears all measurements, allowing the object to be reused

Args:
label (str, optional) : optionally change the label

Example:
>>> from timerit import Timerit
>>> import math
>>> ti = Timerit(num=10, unit='us', verbose=True)
>>> _ = ti.reset(label='10!').call(math.factorial, 10)
Timed best=...s, mean=...s for 10!
>>> _ = ti.reset(label='20!').call(math.factorial, 20)
Timed best=...s, mean=...s for 20!
>>> _ = ti.reset().call(math.factorial, 20)
Timed best=...s, mean=...s for 20!


def _prep_debug_cli_for_run_end(self, debug_dump, tf_error, passed_filter, passed_filter_exclude_node_names):
    if tf_error:
        help_intro = cli_shared.get_error_intro(tf_error)
        self._init_command = 'help'
        self._title_color = 'red_on_white'
    else:
        help_intro = None
        self._init_command = 'lt'
        self._title_color = 'black_on_white'
        if passed_filter is not None:
            self._init_command = 'lt -f %s' % passed_filter
            if passed_filter_exclude_node_names:
                self._init_command += ' --filter_exclude_node_names %s' % passed_filter_exclude_node_names
            self._title_color = 'red_on_white'
    self._run_cli = analyzer_cli.create_analyzer_ui(debug_dump, self._tensor_filters, ui_type=self._ui_type, on_ui_exit=self._remove_dump_root, config=self._config)
    dumped_tensor_names = []
    for datum in debug_dump.dumped_tensor_data:
        dumped_tensor_names.append('%s:%d' % (datum.node_name, datum.output_slot))
    self._run_cli.register_tab_comp_context(['print_tensor', 'pt'], dumped_tensor_names)
    self._run_cli.register_tab_comp_context(['node_info', 'ni', 'list_inputs', 'li', 'list_outputs', 'lo'], [str(node_name) for node_name in debug_dump.nodes()])
    self._title = 'run-end: ' + self._run_description
    if help_intro:
        self._run_cli.set_help_intro(help_intro)

Prepare (but not launch) CLI for run-end, with debug dump from the run.

Args:
debug_dump: (debug_data.DebugDumpDir) The debug dump directory from this
run.
tf_error: (None or OpError) OpError that happened during the run() call
(if any).
passed_filter: (None or str) Name of the tensor filter that just passed
and caused the preparation of this run-end CLI (if any).
passed_filter_exclude_node_names: (None or str) Regular expression used
with the tensor filter to exclude ops with names matching the regular
expression.


def show_plot(plot, width=PREVIEW_WIDTH, height=PREVIEW_HEIGHT):
    return SVG(data=plot_to_svg(plot, width, height))

Preview a plot in a jupyter notebook.

Args:
plot (list): the plot to display (list of layers)
width (int): the width of the preview
height (int): the height of the preview

Returns:
An object that renders in Jupyter as the provided plot


def _assert_same_graph(original_item, item):
    original_graph = getattr(original_item, 'graph', None)
    graph = getattr(item, 'graph', None)
    if original_graph and graph and (original_graph is not graph):
        raise ValueError('%s must be from the same graph as %s (graphs are %s and %s).' % (item, original_item, graph, original_graph))

Fail if the 2 items are from different graphs.

Args:
original_item: Original item to check against.
item: Item to check.

Raises:
ValueError: if graphs do not match.


def quantize(onnx_model_path: Path) -> Path:
    import onnx
    import onnxruntime
    from onnx.onnx_pb import ModelProto
    from onnxruntime.quantization import QuantizationMode
    from onnxruntime.quantization.onnx_quantizer import ONNXQuantizer
    from onnxruntime.quantization.registry import IntegerOpsRegistry
    onnx_model = onnx.load(onnx_model_path.as_posix())
    if parse(onnx.__version__) < parse('1.5.0'):
        print('Models larger than 2GB will fail to quantize due to protobuf constraint.\nPlease upgrade to onnxruntime >= 1.5.0.')
    copy_model = ModelProto()
    copy_model.CopyFrom(onnx_model)
    if parse(onnxruntime.__version__) < parse('1.13.1'):
        quantizer = ONNXQuantizer(model=copy_model, per_channel=False, reduce_range=False, mode=QuantizationMode.IntegerOps, static=False, weight_qType=True, input_qType=False, tensors_range=None, nodes_to_quantize=None, nodes_to_exclude=None, op_types_to_quantize=list(IntegerOpsRegistry))
    else:
        quantizer = ONNXQuantizer(model=copy_model, per_channel=False, reduce_range=False, mode=QuantizationMode.IntegerOps, static=False, weight_qType=True, activation_qType=False, tensors_range=None, nodes_to_quantize=None, nodes_to_exclude=None, op_types_to_quantize=list(IntegerOpsRegistry))
    quantizer.quantize_model()
    quantized_model_path = generate_identified_filename(onnx_model_path, '-quantized')
    print(f'Quantized model has been written at {quantized_model_path}: ✔')
    onnx.save_model(quantizer.model.model, quantized_model_path.as_posix())
    return quantized_model_path

Quantize the weights of the model from float32 to in8 to allow very efficient inference on modern CPU

Args:
onnx_model_path: Path to location the exported ONNX model is stored

Returns: The Path generated for the quantized


def _ascii_tree(out, node, p1, p2, seen, get_children, get_description=None):
    children = list(get_children(node))
    text = get_description(node) if get_description else str(node)
    if node in seen:
        out.write(p1 + '[' + text + ']\n')
    else:
        out.write(p1 + text + '\n')
        seen.add(node)
        for i, c in enumerate(children):
            last = i == len(children) - 1
            out.write(p2 + '|\n')
            _ascii_tree(out, c, p2 + '+-', p2 + ('  ' if last else '| '), seen, get_children, get_description)

Draw a graph, starting at a given position.

Args:
out: A file-like object to write the ascii tree to.
node: The node from where to draw.
p1: The current prefix.
p2: The upcoming prefix.
seen: Nodes we have seen so far (as a set).
get_children: The function to call to retrieve children.
get_description: Optional. A function to call to describe a node.


def parse(manifest_filename):
    manifest = {}
    with io.open(manifest_filename, 'rt', encoding='utf8') as f:
        lineno = 0
        for line in f:
            lineno += 1
            line = line.rstrip('\n')
            fields = line.split(' ')
            stored_path = fields[0]
            if len(fields) == 1 or (len(fields) == 2 and (not fields[1])):
                local_path = None
            elif len(fields) == 2 and fields[1]:
                local_path = fields[1]
            else:
                raise error.Error('Syntax error at line %d in [%s]: %s' % (lineno, manifest_filename, repr(line)))
            if stored_path in manifest:
                raise error.Error('Configuration error at line %d in [%s]: file [%s] specified more than once' % (lineno, manifest_filename, stored_path))
            manifest[stored_path] = local_path
    return manifest

Parse a Bazel manifest file.

Args:
manifest_filename: Path to file created by Bazel

Returns:
dictionary: key is stored_path, value is local_path.

Data will be read from each `local_path` and stored in the
zipfile under the corresponding `stored_path`.  local_paths
may appear more than once, stored_paths may not. A
local_path of None means a zero-length empty file.

Raises:
Error, IOError, SystemError


def launch(self, image, command, **kwargs):

        if isinstance(command, PythonCall):
            return PythonJob(self, image, command, **kwargs)
        else:
            return Job(self, image, command, **kwargs)

Create a job on this engine

Args:
image (str): name of the docker image to launch
command (str): shell command to run


def from_gpx(gpx_track_point):
    return Point(lat=gpx_track_point.latitude, lon=gpx_track_point.longitude, time=gpx_track_point.time)

Creates a point from GPX representation

Arguments:
gpx_track_point (:obj:`gpxpy.GPXTrackPoint`)
Returns:
:obj:`Point`


def PublishMultipleEvents(cls, events, token=None):

    event_name_map = registry.EventRegistry.EVENT_NAME_MAP
    for event_name, messages in iteritems(events):
      if not isinstance(event_name, string_types):
        raise ValueError(
            "Event names should be string, got: %s" % type(event_name))
      for msg in messages:
        if not isinstance(msg, rdfvalue.RDFValue):
          raise ValueError("Can only publish RDFValue instances.")

      for event_cls in event_name_map.get(event_name, []):
        event_cls().ProcessMessages(messages, token=token)

Publishes multiple messages at once.

Args:
events: A dict with keys being event names and values being lists of
messages.
token: ACL token.

Raises:
ValueError: If the message is invalid. The message must be a Semantic
Value (instance of RDFValue) or a full GrrMessage.


def kill_monitor(self, check_alive=True):

        self._kill_process_type(
            ray_constants.PROCESS_TYPE_MONITOR, check_alive=check_alive)

Kill the monitor.

Args:
check_alive (bool): Raise an exception if the process was already
dead.


def get(self):
    resource = dict()
    resource.update(self._parse_hostname())
    resource.update(self._parse_iprouting())
    resource.update(self._parse_banners())
    return resource

Returns the system configuration abstraction

The System resource returns the following:

* hostname (str): The hostname value

Returns:
dict: Represents the node's system configuration


def __init__(self, initial_learning_rate, decay_steps, num_periods=0.5, alpha=0.0, beta=0.001, name=None):
    super(LinearCosineDecay, self).__init__()
    self.initial_learning_rate = initial_learning_rate
    self.decay_steps = decay_steps
    self.num_periods = num_periods
    self.alpha = alpha
    self.beta = beta
    self.name = name

Applies linear cosine decay to the learning rate.

Args:
initial_learning_rate: A scalar `float32` or `float64` Tensor or a Python
number. The initial learning rate.
decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
Number of steps to decay over.
num_periods: Number of periods in the cosine part of the decay.
See computation above.
alpha: See computation above.
beta: See computation above.
name: String.  Optional name of the operation.  Defaults to
'LinearCosineDecay'.


def load_closure_cell(state, op, check_bindings, ctx):
    cell_index = ctx.vm.frame.f_code.get_cell_index(op.argval)
    cell = ctx.vm.frame.cells[cell_index]
    if check_bindings and (not cell.bindings):
        ctx.errorlog.name_error(ctx.vm.frames, op.argval)
        cell = ctx.new_unsolvable(state.node)
    visible_bindings = cell.Filter(state.node, strict=False)
    if len(visible_bindings) != len(cell.bindings):
        new_cell = ctx.program.NewVariable()
        if visible_bindings:
            for b in visible_bindings:
                new_cell.PasteBinding(b, state.node)
        else:
            new_cell.AddBinding(ctx.convert.unsolvable)
        ctx.vm.frame.cells[cell_index] = cell = new_cell
    name = op.argval
    ctx.vm.set_var_name(cell, name)
    check_for_deleted(state, name, cell, ctx)
    ctx.vm.trace_opcode(op, name, cell)
    return state.push(cell)

Retrieve the value out of a closure cell.

Used to generate the 'closure' tuple for MAKE_CLOSURE.

Each entry in that tuple is typically retrieved using LOAD_CLOSURE.

Args:
state: The current VM state.
op: The opcode. op.arg is the index of a "cell variable": This corresponds
to an entry in co_cellvars or co_freevars and is a variable that's bound
into a closure.
check_bindings: Whether to check the retrieved value for bindings.
ctx: The current context.

Returns:
A new state.


def sequence_accuracy(labels, outputs):
    all_correct = tf.reduce_all(tf.logical_or(tf.equal(labels, outputs), tf.equal(labels, 0)), axis=(- 1))
    return tf.metrics.mean(all_correct)

Compute the sequence-level accuracy.

A sequence is only considered correct if all of its entries were predicted
correctly.

Args:
labels: ground-truth labels, shape=(batch, packed_seq_length)
outputs: predicted tokens, shape=(batch, seq_length)
Returns:
Two ops, one for getting the current average accuracy and another for
updating the running average estimate.


def order_dict_by(dict_, key_order):
    dict_keys = set(dict_.keys())
    other_keys = (dict_keys - set(key_order))
    key_order = it.chain(key_order, other_keys)
    sorted_dict = OrderedDict(((key, dict_[key]) for key in key_order if (key in dict_keys)))
    return sorted_dict

r"""
Reorders items in a dictionary according to a custom key order

Args:
dict_ (dict_):  a dictionary
key_order (list): custom key order

Returns:
OrderedDict: sorted_dict

CommandLine:
python -m utool.util_dict --exec-order_dict_by

Example:
>>> # ENABLE_DOCTEST
>>> from utool.util_dict import *  # NOQA
>>> import utool as ut
>>> dict_ = {1: 1, 2: 2, 3: 3, 4: 4}
>>> key_order = [4, 2, 3, 1]
>>> sorted_dict = order_dict_by(dict_, key_order)
>>> result = ('sorted_dict = %s' % (ut.repr4(sorted_dict, nl=False),))
>>> print(result)
>>> assert result == 'sorted_dict = {4: 4, 2: 2, 3: 3, 1: 1}'


def get_iso2_from_iso3(cls, iso3, use_live=True, exception=None):
    countriesdata = cls.countriesdata(use_live=use_live)
    iso2 = countriesdata['iso2iso3'].get(iso3.upper())
    if (iso2 is not None):
        return iso2
    if (exception is not None):
        raise exception
    return None

Get ISO2 from ISO3 code

Args:
iso3 (str): ISO3 code for which to get ISO2 code
use_live (bool): Try to get use latest data from web rather than file in package. Defaults to True.
exception (Optional[ExceptionUpperBound]): An exception to raise if country not found. Defaults to None.

Returns:
Optional[str]: ISO2 code


def __init__(self, in_dim: int, out_dim: int, bias: bool=True, init: str='default', init_fn: Optional[Callable[[torch.Tensor, torch.Tensor], None]]=None):
    super().__init__(in_dim, out_dim, bias=bias)
    if bias:
        with torch.no_grad():
            self.bias.fill_(0)
    self.init = init
    self.init_fn = init_fn
    if init not in ['default', 'relu', 'glorot', 'gating', 'normal', 'final']:
        raise ValueError('Invalid init string.')

Args:
in_dim:
The final dimension of inputs to the layer
out_dim:
The final dimension of layer outputs
bias:
Whether to learn an additive bias. True by default
init:
The initializer to use. Choose from:

"default": LeCun fan-in truncated normal initialization "relu": He initialization w/ truncated normal
distribution "glorot": Fan-average Glorot uniform initialization "gating": Weights=0, Bias=1 "normal":
Normal initialization with std=1/sqrt(fan_in) "final": Weights=0, Bias=0

Overridden by init_fn if the latter is not None.
init_fn:
A custom initializer taking weight and bias as inputs. Overrides init if not None.


def next_index(self):
    return len(self._string_table)

Gets index that would be assigned to the next added string.

Returns:
Index of the next string if it was added.


def get_dict_fullhash_component(vardict: dict[str, cfg.Variable], *, names: set[str] | None=None, seen: 'set[_base.BaseValue] | None'=None) -> tuple[Any, ...]:
    if names is not None:
        vardict = {name: vardict[name] for name in names.intersection(vardict)}
    return tuple(sorted(((k, get_var_fullhash_component(v, seen)) for k, v in vardict.items())))

Hash a dictionary.

This contains the keys and the full hashes of the data in the values.

Arguments:
vardict: A dictionary mapping str to Variable.
names: If this is non-None, the snapshot will include only those dictionary
entries whose keys appear in names.
seen: Optionally, a set of seen values for recursion detection.

Returns:
A hashable tuple of the dictionary.


def count_elements_in_dataset(ds, batch_size=(1 * 1024), parallel_batch=8):
    with tf.Session() as sess:
        dsc = ds.apply(tf.contrib.data.enumerate_dataset())
        dsc = dsc.apply(tf.contrib.data.map_and_batch((lambda c, v: c), batch_size, num_parallel_batches=parallel_batch))
        iterator = dsc.make_initializable_iterator()
        sess.run(iterator.initializer)
        get_next = iterator.get_next()
        counted = 0
        try:
            while True:
                counted = (sess.run(tf.reduce_max(get_next)) + 1)
                utils.dbg(('Counted so far: %d' % counted))
        except tf.errors.OutOfRangeError:
            pass
        utils.dbg(('Counted total: %d' % counted))
        return counted

Count and return all the elements in the given dataset.

Debugging function.  The elements in a dataset cannot be counted
without enumerating all of them.  By counting in batch and in
parallel, this method allows rapid traversal of the dataset.

Args:
ds:  The dataset whose elements should be counted.
batch_size:  the number of elements to count a a time.
parallel_batch:  how many batches to count in parallel.

Returns:
The number of elements in the dataset.


def import_settings(self, filename):
    if (not os.path.isfile(filename)):
        self._logger.log('error', 'File: {} not found, continuing with default settings'.format(filename))
    else:
        with open(filename, 'r') as jsonFile:
            data = json.load(jsonFile)
            self._value_ranges = data['valueRanges']
            self._best_values = data['best_values']
            self._best_values = []
            for (index, value) in enumerate(data['best_values']):
                if (self._value_ranges[index] == 'int'):
                    self._best_values.append(int(value))
                else:
                    self._best_values.append(float(value))
            self.minimize = data['minimize']
            self.num_employers = data['num_employers']
            self._best_score = float(data['best_score'])
            self.limit = data['limit']

Import settings from a JSON file

Args:
filename (string): name of the file to import from


def screenshot(self, filename=None):

        if self.__keep_screen:
            return self.__last_screen
        try:
            screen = self._take_screenshot()
        except IOError:

            log.warn("warning, screenshot failed [2/1], retry again")
            screen = self._take_screenshot()
        self.__last_screen = screen
        if filename:
            save_dir = os.path.dirname(filename) or '.'
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            screen.save(filename)
        return screen

Take screen snapshot

Args:
- filename: filename where save to, optional

Returns:
PIL.Image object

Raises:
TypeError, IOError


def deserialize(self, *args, **kwargs):
    self.parse(*args, **kwargs)
    self._ore_initialized = True

Deserialize Resource Map XML doc.

The source is specified using one of source, location, file or data.

Args:
source: InputSource, file-like object, or string
In the case of a string the string is the location of the source.

location: str
String indicating the relative or absolute URL of the source. Graph``s
absolutize method is used if a relative location is specified.

file: file-like object

data: str
The document to be parsed.

format : str
Used if format can not be determined from source. Defaults to ``rdf/xml``.
Format support can be extended with plugins.

Built-in: ``xml``, ``n3``, ``nt``, ``trix``, ``rdfa``

publicID: str
Logical URI to use as the document base. If None specified the document
location is used (at least in the case where there is a document location).

Raises:
xml.sax.SAXException based exception: On parse error.


def _as_index(idx, need_scalar=True):
    if isinstance(idx, (numbers.Integral, tensor_shape.Dimension)):
        return (idx, True)
    data = asarray(idx)
    if data.dtype == dtypes.bool:
        if data.shape.ndims != 1:
            raise NotImplementedError('Need rank 1 for bool index %s' % idx)
        data = array_ops.where_v2(data)
        data = array_ops.reshape(data, [-1])
    if need_scalar and data.shape.rank not in (None, 0):
        raise IndexError(_SLICE_ERROR + ', got {!r}'.format(idx))
    np_dtype = data.dtype.as_numpy_dtype
    if not np.issubdtype(np_dtype, np.integer):
        raise IndexError(_SLICE_ERROR + ', got {!r}'.format(idx))
    if data.dtype not in (dtypes.int64, dtypes.int32):
        promoted_dtype = np.promote_types(np.int32, np_dtype)
        if promoted_dtype == np.int32:
            data = math_ops.cast(data, dtypes.int32)
        elif promoted_dtype == np.int64:
            data = math_ops.cast(data, dtypes.int64)
        else:
            raise IndexError(_SLICE_ERROR + ', got {!r}'.format(idx))
    return (data, data.shape.rank == 0)

Helper function to parse idx as an index.

Args:
idx: index
need_scalar: If idx needs to be a scalar value.

Returns:
A pair, (indx, bool). First one is the parsed index and can be a tensor,
or scalar integer / Dimension. Second one is True if rank is known to be 0.

Raises:
IndexError: For incorrect indices.


def clean_dataframe(df: pd.DataFrame, column_to_return_type_mapping: Mapping[str, _fhir_path_data_types.FhirPathDataType]) -> pd.DataFrame:
    non_scalar_cols = []
    if not column_to_return_type_mapping:
        non_scalar_cols = df.select_dtypes(include=['object']).columns.tolist()
    for column_name, return_type in column_to_return_type_mapping.items():
        if return_type.returns_collection() or return_type.fields():
            non_scalar_cols.append(column_name)

    def trim_structs(item):
        if isinstance(item, numpy.ndarray):
            if not item.any():
                return None
            else:
                return [trim_structs(child) for child in item]
        if isinstance(item, dict):
            result = {}
            for key, value in item.items():
                trimmed_value = trim_structs(value)
                if trimmed_value is not None:
                    result[key] = trimmed_value
            return result
        return item
    for col in non_scalar_cols:
        df[col] = df[col].map(trim_structs)
    return df

Cleans dataframe retrieved from backend.

Args:
df: Dataframe to clean
column_to_return_type_mapping: If the view has columns, we can narrow the
non-scalar column list by checking only for list or struct columns.

Returns:
Cleaned dataframe


def ensure_app_config_dir(appname, *args):
    from ubelt import util_path
    dpath = get_app_config_dir(appname, *args)
    util_path.ensuredir(dpath)
    return dpath

Calls `get_app_config_dir` but ensures the directory exists.

Args:
appname (str): the name of the application
*args: any other subdirectories may be specified

SeeAlso:
get_app_config_dir

Example:
>>> import ubelt as ub
>>> dpath = ub.ensure_app_config_dir('ubelt')
>>> assert exists(dpath)


def _publish_response(self, slug, message):

        resp_topic = self.topics.gateway_topic(slug, 'data/response')
        self._logger.debug("Publishing response message: (topic=%s) (message=%s)", resp_topic, message)
        self.client.publish(resp_topic, message)

Publish a response message for a device

Args:
slug (string): The device slug that we are publishing on behalf of
message (dict): A set of key value pairs that are used to create the message
that is sent.


def __init__(self, use_locking, name, use_own_namescope_for_non_slot_vars=False):
    if not name:
        raise ValueError('Must specify the optimizer name')
    self._use_locking = use_locking
    self._name = name
    self._use_own_namescope_for_non_slot_vars = use_own_namescope_for_non_slot_vars
    if self._use_own_namescope_for_non_slot_vars:
        with variable_scope.variable_scope(None, default_name=self._name) as vs:
            self._non_slot_variable_scope = vs
    self._slots = {}
    self._non_slot_dict = {}
    self._deferred_slot_restorations = {}

Create a new Optimizer.

This must be called by the constructors of subclasses.

Args:
use_locking: Bool. If True apply use locks to prevent concurrent updates
to variables.
name: A non-empty string.  The name to use for accumulators created for
the optimizer.
use_own_namescope_for_non_slot_vars: If True, use a root namescope under
self._name for non-slot variables.

Raises:
ValueError: If name is malformed.


def _extract_variable_parts(variable_key, variable):

  name, offset, partitioned = None, None, False

  if variable._save_slice_info:
    name = variable_key[:variable_key.rfind("/")]
    if not variable._save_slice_info.full_name.endswith(name):
      raise RuntimeError("Unexpected handling of partitioned variable.")
    offset = variable._save_slice_info.var_offset[0]
    partitioned = True

  return partitioned, name, offset

Matches a variable to individual parts.

Args:
variable_key: String identifier of the variable in the module scope.
variable: Variable tensor.

Returns:
partitioned: Whether the variable is partitioned.
name: Name of the variable up to the partitioning.
offset: Offset of the variable into the full variable.

Raises:
RuntimeError: In case of unexpected variable format.


def add_triangle(self, neighbors, color, center=None, opacity=0.4, draw_edges=False, edges_color=[0.0, 0.0, 0.0], edges_linewidth=2):
    points = vtk.vtkPoints()
    triangle = vtk.vtkTriangle()
    for ii in range(3):
        points.InsertNextPoint(neighbors[ii].x, neighbors[ii].y, neighbors[ii].z)
        triangle.GetPointIds().SetId(ii, ii)
    triangles = vtk.vtkCellArray()
    triangles.InsertNextCell(triangle)
    trianglePolyData = vtk.vtkPolyData()
    trianglePolyData.SetPoints(points)
    trianglePolyData.SetPolys(triangles)
    mapper = vtk.vtkPolyDataMapper()
    mapper.SetInput(trianglePolyData)
    ac = vtk.vtkActor()
    ac.SetMapper(mapper)
    ac.GetProperty().SetOpacity(opacity)
    if (color == 'element'):
        if (center is None):
            raise ValueError('Color should be chosen according to the central atom, and central atom is not provided')
        myoccu = 0.0
        for (specie, occu) in center.species.items():
            if (occu > myoccu):
                myspecie = specie
                myoccu = occu
        color = [(i / 255) for i in self.el_color_mapping[myspecie.symbol]]
        ac.GetProperty().SetColor(color)
    else:
        ac.GetProperty().SetColor(color)
    if draw_edges:
        ac.GetProperty().SetEdgeColor(edges_color)
        ac.GetProperty().SetLineWidth(edges_linewidth)
        ac.GetProperty().EdgeVisibilityOn()
    self.ren.AddActor(ac)

Adds a triangular surface between three atoms.

Args:
atoms: Atoms between which a triangle will be drawn.
color: Color for triangle as RGB.
center: The "central atom" of the triangle
opacity: opacity of the triangle
draw_edges: If set to True, the a line will be  drawn at each edge
edges_color: Color of the line for the edges
edges_linewidth: Width of the line drawn for the edges


def DocumentPackages(self, packages, index_base=None, showprivate=False,
                         notify=True, showinh=False, intro_pages=None,
                         append_material=None, extra=None):

        if index_base is None:
            gram = ''
            if isinstance(packages, list) and len(packages) > 1:
                gram = 's'
                if len(packages) < 3:
                    names = ' and '.join(['``%s``' % p.__name__ for p in packages])
                else:
                    names = ['``%s``' % p.__name__ for p in packages]
                    names[-1] = ' and %s' % names[-1]
                    names = ', '.join(names)
            else:
                names = '``%s``' % packages.__name__
            index = SAMPLE_INDEX.format(names, gram)
        else:
            index = self.OpenIndex(index_base)
        app = self._DocPackageFromTop(packages, showprivate=showprivate, showinh=showinh)
        index += self._GenerateStaticsTable()
        index +=
        if intro_pages is not None:
            if isinstance(intro_pages, str):
                intro_pages = [intro_pages]
            for page in intro_pages:
                index += '   {}\n'.format(page.strip())
        index += '\n'
        if append_material is not None:
            index += append_material
        index += app

        if extra is not None:
            index += extra
        if notify:
            index +=
        self.WriteIndex(index)
        return None

This is the high level API to use to generate documentation pages for any given package(s).

Args:
packages (list(module)): A list of packages that contain submodules to document
index_base (str): The index page file name. This content will be appended
showprivate (bool): A flag for whether or not to display private members


def get_pattern_link_topattern(self, patternnumber):
    _checkPatternNumber(patternnumber)
    address = _calculateRegisterAddress('linkpattern', patternnumber)
    return self.read_register(address)

Get the 'linked pattern' value for a given pattern.

Args:
patternnumber (integer): From 0-7

Returns:
The 'linked pattern' value (int).


def __init__(self, host=None, path=None, port=None, use_cache=True):

        host = host or DEFAULT_HOST
        path = path or DEFAULT_PATH
        port = port or DEFAULT_PORT

        host = self._add_http_prefix(host)
        host = self._remove_trailing_slash(host)

        if not path.startswith('/'):
            path = '/' + path

        self._host = host
        self._path = path
        self._port = port
        self._use_cache = use_cache

ServerBase constructor.

Args:
host (str): Url of host to connect to.
path (str): Path on the host to access to the biomart service.
port (int): Port to use for the connection.
use_cache (bool): Whether to cache requests.


def get_nodes(cluster):
    gk = get_api_client()
    site = get_cluster_site(cluster)
    return gk.sites[site].clusters[cluster].nodes.list()

Get all the nodes of a given cluster.

Args:
cluster(string): uid of the cluster (e.g 'rennes')


def plot_legend(ax, no_legend=True, legend_arg=None):
    legend_arg = dict_if_none(legend_arg)
    if (not no_legend):
        ax.legend(**legend_arg)

Function that defines the legend options
of a matplotlib plot.

Args:
ax: matplotlib axes
no_legend (bool): Defines the presence of a legend in the figure
legend_arg (dict): Addition arguments for matplotlib.legend() call


def _read_file(file_name):
    with open(file_name) as config_file:
        data = json.load(config_file)
    return data

Read the file content and load it as JSON.

Arguments:
file_name (:py:class:`str`): The filename.

Returns:
:py:class:`dict`: The loaded JSON data.

Raises:
:py:class:`FileNotFoundError`: If the file is not found.


def _decode_input_tensor_to_features_dict(feature_map, hparams):

  inputs = tf.convert_to_tensor(feature_map["inputs"])
  input_is_image = False

  x = inputs
  p_hparams = hparams.problem_hparams

  x = tf.expand_dims(x, axis=[2])
  x = tf.to_int32(x)
  input_space_id = tf.constant(p_hparams.input_space_id)
  target_space_id = tf.constant(p_hparams.target_space_id)

  features = {}
  features["input_space_id"] = input_space_id
  features["target_space_id"] = target_space_id
  features["decode_length"] = (
      IMAGE_DECODE_LENGTH if input_is_image else tf.shape(x)[1] + 50)
  features["inputs"] = x
  return features

Convert the interactive input format (see above) to a dictionary.

Args:
feature_map: dict with inputs.
hparams: model hyperparameters

Returns:
a features dictionary, as expected by the decoder.


def check_type(value, type_def):

        if type_def == 'integer':
            try:

                int(value)
                return True
            except ValueError:
                return isinstance(value, six.integer_types) and not isinstance(value, bool)
        elif type_def == 'number':
            return isinstance(value, (six.integer_types, float)) and not isinstance(value, bool)
        elif type_def == 'string':
            return isinstance(value, (six.text_type, six.string_types, datetime.datetime))
        elif type_def == 'boolean':
            return (isinstance(value, bool) or
                    (isinstance(value, (six.text_type, six.string_types,)) and
                     value.lower() in ['true', 'false'])
                    )
        else:
            return False

Check if the value is in the type given in type_def.

Args:
value: the var to test.
type_def: string representing the type in swagger.

Returns:
True if the type is correct, False otherwise.


def stream(self):
    chunk_a = None
    chunk_b = None
    chunk_a_index = 0
    chunk_b_index = 0
    search_start_index = 0
    while True:
        chunk_a = chunk_b
        chunk_a_index = chunk_b_index
        chunk_b = self._file.read(self._read_size)
        if (chunk_a is None):
            continue
        chunk_b_index = (chunk_a_index + len(chunk_a))
        if (not chunk_a):
            break
        current_chunk = (chunk_a + chunk_b[:self._overlap_size])
        offset_end = (len(chunk_a) + self._overlap_size)
        while True:
            offset_start = (search_start_index - chunk_a_index)
            match = self._pattern.search(current_chunk, offset_start, offset_end)
            if (not match):
                unmatched_part = chunk_a[offset_start:]
                if unmatched_part:
                    (yield (None, unmatched_part))
                search_start_index += len(unmatched_part)
                break
            (start_index, end_index) = match.span(match.lastindex)
            unmatched_part = current_chunk[offset_start:start_index]
            if unmatched_part:
                (yield (None, unmatched_part))
            (yield (match, match.group(match.lastindex)))
            search_start_index += (len(unmatched_part) + len(match.group(match.lastindex)))

Iterate the file stream.

Returns:
iterator: Each item is a tuple:

1. None, regex match
2. str


def get_access_token(tenant_id, application_id, application_secret):

    context = adal.AuthenticationContext(
        get_auth_endpoint() + tenant_id, api_version=None)
    token_response = context.acquire_token_with_client_credentials(
        get_resource_endpoint(), application_id, application_secret)
    return token_response.get('accessToken')

get an Azure access token using the adal library.

Args:
tenant_id (str): Tenant id of the user's account.
application_id (str): Application id of a Service Principal account.
application_secret (str): Application secret (password) of the Service Principal account.

Returns:
An Azure authentication token string.


def Serialize(self, writer):

        writer.WriteUInt256(self.AssetId)
        writer.WriteFixed8(self.Value)
        writer.WriteUInt160(self.ScriptHash)

Serialize object.

Args:
writer (neo.IO.BinaryWriter):


def format_color(text, color, use_color_setting):

    if not use_color_setting:
        return text
    else:
        return '{}{}{}'.format(color, text, NORMAL)

Format text with color.

Args:
text - Text to be formatted with color if `use_color`
color - The color start string
use_color_setting - Whether or not to color


def _parse_sequence_example_raw(serialized, debug_name, context, feature_list, name=None):
    if context.num_features + feature_list.num_features == 0:
        raise ValueError('Must provide at least one feature key.')
    with ops.name_scope(name, 'ParseSequenceExample', [serialized]):
        debug_name = [] if debug_name is None else debug_name
        feature_list_dense_missing_assumed_empty = []
        for k, v in feature_list.dense_defaults.items():
            if v is not None:
                raise ValueError('Value feature_list.dense_defaults[%s] must be None' % k)
            feature_list_dense_missing_assumed_empty.append(k)
        has_ragged = context.ragged_keys or feature_list.ragged_keys
        serialized = ops.convert_to_tensor(serialized, name='serialized')
        if has_ragged and serialized.shape.ndims is None:
            raise ValueError('serialized must have statically-known rank to parse ragged features.')
        feature_list_dense_missing_assumed_empty_vector = [key in feature_list_dense_missing_assumed_empty for key in feature_list.dense_keys]
        outputs = gen_parsing_ops.parse_sequence_example_v2(serialized=serialized, debug_name=debug_name, context_sparse_keys=context.sparse_keys, context_dense_keys=context.dense_keys, context_ragged_keys=context.ragged_keys, feature_list_sparse_keys=feature_list.sparse_keys, feature_list_dense_keys=feature_list.dense_keys, feature_list_ragged_keys=feature_list.ragged_keys, feature_list_dense_missing_assumed_empty=feature_list_dense_missing_assumed_empty_vector, context_dense_defaults=context.dense_defaults_vec, Ncontext_sparse=len(context.sparse_keys), Nfeature_list_sparse=len(feature_list.sparse_keys), Nfeature_list_dense=len(feature_list.dense_keys), context_sparse_types=context.sparse_types, context_ragged_value_types=context.ragged_value_types, context_ragged_split_types=context.ragged_split_types, feature_list_dense_types=feature_list.dense_types, feature_list_sparse_types=feature_list.sparse_types, feature_list_ragged_value_types=feature_list.ragged_value_types, feature_list_ragged_split_types=feature_list.ragged_split_types, context_dense_shapes=context.dense_shapes_as_proto, feature_list_dense_shapes=feature_list.dense_shapes, name=name)
        context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, context_ragged_values, context_ragged_row_splits, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values, feature_list_dense_lengths, feature_list_ragged_values, feature_list_ragged_outer_splits, feature_list_ragged_inner_splits = outputs
        context_ragged_tensors = parsing_config._build_ragged_tensors(serialized.shape, context_ragged_values, context_ragged_row_splits)
        feature_list_ragged_tensors = parsing_config._build_ragged_tensors(serialized.shape, feature_list_ragged_values, feature_list_ragged_outer_splits, feature_list_ragged_inner_splits)
        context_sparse_tensors = [sparse_tensor.SparseTensor(ix, val, shape) for ix, val, shape in zip(context_sparse_indices, context_sparse_values, context_sparse_shapes)]
        feature_list_sparse_tensors = [sparse_tensor.SparseTensor(ix, val, shape) for ix, val, shape in zip(feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes)]
        context_output = dict(zip(context.sparse_keys + context.dense_keys + context.ragged_keys, context_sparse_tensors + context_dense_values + context_ragged_tensors))
        feature_list_output = dict(zip(feature_list.sparse_keys + feature_list.dense_keys + feature_list.ragged_keys, feature_list_sparse_tensors + feature_list_dense_values + feature_list_ragged_tensors))
        feature_list_lengths = dict(zip(feature_list.dense_keys, feature_list_dense_lengths))
        return (context_output, feature_list_output, feature_list_lengths)

Parses a vector of `SequenceExample` protos.

Args:
serialized: A vector (1-D Tensor) of type string, containing binary
serialized `SequenceExample` protos.
debug_name: A vector (1-D Tensor) of strings (optional), the names of the
serialized protos.
context: A `ParseOpParams` containing the parameters for the parse
op for the context features.
feature_list: A `ParseOpParams` containing the parameters for the
parse op for the feature_list features.
name: A name for this operation (optional).

Returns:
A tuple of three `dict`s, each mapping keys to `Tensor`s, `SparseTensor`s,
and `RaggedTensor`s. The first dict contains the context key/values, the
second dict contains the feature_list key/values, and the final dict
contains the lengths of any dense feature_list features.

Raises:
TypeError: if feature_list.dense_defaults is not either None or a dict.


def get_html_labels_weights(self, data):

        all_html = []
        all_labels = []
        all_weights = []
        for html, content, comments in data:
            all_html.append(html)
            labels, weights = self._get_labels_and_weights(
                content, comments)
            all_labels.append(labels)
            all_weights.append(weights)
        return np.array(all_html), np.array(all_labels), np.array(all_weights)

Gather the html, labels, and weights of many files' data.
Primarily useful for training/testing an :class`Extractor`.

Args:
data: Output of :func:`dragnet.data_processing.prepare_all_data`.

Returns:
Tuple[List[Block], np.array(int), np.array(int)]: All blocks, all
labels, and all weights, respectively.


def add_device_info(self, name, info):
    self._user_added_device_info.update({name: info})

Add information of the device to be pulled into controller info.

Adding the same info name the second time will override existing info.

Args:
name: string, name of this info.
info: serializable, content of the info.


def data_period_name_or_description(self, value=None):
    if (value is not None):
        try:
            value = str(value)
        except ValueError:
            raise ValueError('value {} need to be of type str for field `data_period_name_or_description`'.format(value))
        if (',' in value):
            raise ValueError('value should not contain a comma for field `data_period_name_or_description`')
    self._data_period_name_or_description = value

Corresponds to IDD Field `data_period_name_or_description`

Args:
value (str): value for IDD Field `data_period_name_or_description`
if `value` is None it will not be checked against the
specification and is assumed to be a missing value

Raises:
ValueError: if `value` is not a valid value


def install_table(self, connection, table, logger=None):
    queries = []
    query_tmpl = 'SELECT * FROM {}'
    for partition in table.partitions:
        partition.localize()
        installed_name = self.install(connection, partition)
        queries.append(query_tmpl.format(installed_name))
    query = 'CREATE VIEW {} AS {} '.format(table.vid, '\nUNION ALL\n'.join(queries))
    logger.debug('Creating view for table.\n    table: {}\n    query: {}'.format(table.vid, query))
    self._execute(connection, query, fetch=False)

Installs all partitons of the table and create view with union of all partitons.

Args:
connection: connection to database who stores mpr data.
table (orm.Table):


def _parse_by_pattern(self, lines, pattern):

        for line in lines:
            match = pattern.match(line)
            if match:
                params = match.groupdict()
                if not params:
                    params = match.groups()
                yield self._create_output_from_match(params)

Match pattern line by line and return Results.

Use ``_create_output_from_match`` to convert pattern match groups to
Result instances.

Args:
lines (iterable): Output lines to be parsed.
pattern: Compiled pattern to match against lines.
result_fn (function): Receive results of one match and return a
Result.

Return:
generator: Result instances.


def __init__(self, src, *args, **kwargs):

        self.quiet = kwargs.pop("quiet", False)

        if os.path.exists(src):
            self.src = src
        else:
            if not self.quiet:
                sys.stderr.write("ERROR: File '{src}' does not exists.".format(**{"src": src, }))
            self.logger.error("ERROR: File '{src}' does not exists.".format(**{"src": src, }))
            sys.exit(-1)

        self.po = polib.pofile(self.src)
        self.result = xlwt.Workbook(encoding="utf-8")

Init conversion.
Args:
src: (unicode or string) path to ".po" file.