def remove_binding(site, hostheader='', ipaddress='*', port=80):

    name = _get_binding_info(hostheader, ipaddress, port)
    current_bindings = list_bindings(site)

    if name not in current_bindings:
        log.debug('Binding already absent: %s', name)
        return True
    ps_cmd = ['Remove-WebBinding',
              '-HostHeader', "'{0}'".format(hostheader),
              '-IpAddress', "'{0}'".format(ipaddress),
              '-Port', "'{0}'".format(port)]

    cmd_ret = _srvmgr(ps_cmd)

    if cmd_ret['retcode'] != 0:
        msg = 'Unable to remove binding: {0}\nError: {1}' \
              ''.format(site, cmd_ret['stderr'])
        raise CommandExecutionError(msg)

    if name not in list_bindings(site):
        log.debug('Binding removed successfully: %s', site)
        return True

    log.error('Unable to remove binding: %s', site)
    return False

Remove an IIS binding.

Args:
site (str): The IIS site name.
hostheader (str): The host header of the binding.
ipaddress (str): The IP address of the binding.
port (int): The TCP port of the binding.

Returns:
bool: True if successful, otherwise False

CLI Example:

.. code-block:: bash

salt '*' win_iis.remove_binding site='site0' hostheader='example.com' ipaddress='*' port='80'


def get_config(self, name):
    if (name not in self.registry):
        msg = "Given config name '{}' is not registered."
        raise NotRegisteredError(msg.format(name))
    return copy.deepcopy(self.registry[name])

Return a registred configuration for given config name.

Arguments:
name (string): A registred config name.

Raises:
NotRegisteredError: If given config name does not exist in
registry.

Returns:
dict: Configuration.


def read(self, key):

        key = quote(key, safe='~')
        url = '/internal/playbooks/keyValue/{}'.format(key)
        r = self.tcex.session.get(url)
        data = r.content
        if data is not None and not isinstance(data, str):
            data = str(r.content, 'utf-8')
        return data

Read data from remote KV store for the provided key.

Args:
key (string): The key to read in remote KV store.

Returns:
(any): The response data from the remote KV store.


def _handle_offset_response(self, future, response):
    timestamp_offset_map = {}
    for (topic, part_data) in response.topics:
        for partition_info in part_data:
            (partition, error_code) = partition_info[:2]
            partition = TopicPartition(topic, partition)
            error_type = Errors.for_code(error_code)
            if (error_type is Errors.NoError):
                if (response.API_VERSION == 0):
                    offsets = partition_info[2]
                    assert (len(offsets) <= 1), 'Expected OffsetResponse with one offset'
                    if (not offsets):
                        offset = UNKNOWN_OFFSET
                    else:
                        offset = offsets[0]
                    log.debug('Handling v0 ListOffsetResponse response for %s. Fetched offset %s', partition, offset)
                    if (offset != UNKNOWN_OFFSET):
                        timestamp_offset_map[partition] = (offset, None)
                else:
                    (timestamp, offset) = partition_info[2:]
                    log.debug('Handling ListOffsetResponse response for %s. Fetched offset %s, timestamp %s', partition, offset, timestamp)
                    if (offset != UNKNOWN_OFFSET):
                        timestamp_offset_map[partition] = (offset, timestamp)
            elif (error_type is Errors.UnsupportedForMessageFormatError):
                log.debug('Cannot search by timestamp for partition %s because the message format version is before 0.10.0', partition)
            elif (error_type is Errors.NotLeaderForPartitionError):
                log.debug('Attempt to fetch offsets for partition %s failed due to obsolete leadership information, retrying.', partition)
                future.failure(error_type(partition))
                return
            elif (error_type is Errors.UnknownTopicOrPartitionError):
                log.warning(('Received unknown topic or partition error in ListOffset request for partition %s. The topic/partition ' + 'may not exist or the user may not have Describe access to it.'), partition)
                future.failure(error_type(partition))
                return
            else:
                log.warning('Attempt to fetch offsets for partition %s failed due to: %s', partition, error_type)
                future.failure(error_type(partition))
                return
    if (not future.is_done):
        future.success(timestamp_offset_map)

Callback for the response of the list offset call above.

Arguments:
future (Future): the future to update based on response
response (OffsetResponse): response from the server

Raises:
AssertionError: if response does not match partition


def get_full_url(self, url):

        request = Request('GET', url)
        preparedrequest = self.session.prepare_request(request)
        return preparedrequest.url

Get full url including any additional parameters

Args:
url (str): URL for which to get full url

Returns:
str: Full url including any additional parameters


def business_days_between(self, from_dates, to_dates):
    from_biz, from_is_bizday = self._to_biz_space(dt.convert_to_date_tensor(from_dates).ordinal())
    to_biz, to_is_bizday = self._to_biz_space(dt.convert_to_date_tensor(to_dates).ordinal())
    from_biz = tf.where(from_is_bizday, from_biz, from_biz + 1)
    to_biz = tf.where(to_is_bizday, to_biz, to_biz + 1)
    return tf.math.maximum(to_biz - from_biz, 0)

Calculates number of business between pairs of dates.

For each pair, the initial date is included in the difference, and the final
date is excluded. If the final date is the same or earlier than the initial
date, zero is returned.

Args:
from_dates: `DateTensor` of initial dates.
to_dates: `DateTensor` of final dates, should be broadcastable to
`from_dates`.

Returns:
An int32 Tensor with the number of business days between the
corresponding pairs of dates.


def get_optional_artifacts_per_task_id(upstream_artifacts):

    optional_artifacts_per_task_id = {}

    for artifact_definition in upstream_artifacts:
        if artifact_definition.get('optional', False) is True:
            task_id = artifact_definition['taskId']
            artifacts_paths = artifact_definition['paths']

            add_enumerable_item_to_dict(
                dict_=optional_artifacts_per_task_id,
                key=task_id, item=artifacts_paths
            )

    return optional_artifacts_per_task_id

Return every optional artifact defined in ``upstream_artifacts``, ordered by taskId.

Args:
upstream_artifacts: the list of upstream artifact definitions

Returns:
dict: list of paths to downloaded artifacts ordered by taskId


def find_executable(cls, name, check_syspaths=False):

        exe = which(name)

        if not exe and check_syspaths:
            paths = cls.get_syspaths()
            env = os.environ.copy()
            env["PATH"] = os.pathsep.join(paths)
            exe = which(name, env=env)

        if not exe:
            raise RuntimeError("Couldn't find executable '%s'." % name)
        return exe

Find an executable.

Args:
name (str): Program name.
check_syspaths (bool): If True, check the standard system paths as
well, if program was not found on current $PATH.

Returns:
str: Full filepath of executable.


def wait_for_other_workers(self):
    if not self._worker_barrier:
        return
    self._worker_barrier.wait()

Waits for other workers to reach the same call to this method.

Raises:
ValueError: if `worker_barrier` is not passed to the __init__ method.


def _validate_user_inputs(self, attributes=None, event_tags=None):
    if (attributes and (not validator.are_attributes_valid(attributes))):
        self.logger.error('Provided attributes are in an invalid format.')
        self.error_handler.handle_error(exceptions.InvalidAttributeException(enums.Errors.INVALID_ATTRIBUTE_FORMAT))
        return False
    if (event_tags and (not validator.are_event_tags_valid(event_tags))):
        self.logger.error('Provided event tags are in an invalid format.')
        self.error_handler.handle_error(exceptions.InvalidEventTagException(enums.Errors.INVALID_EVENT_TAG_FORMAT))
        return False
    return True

Helper method to validate user inputs.

Args:
attributes: Dict representing user attributes.
event_tags: Dict representing metadata associated with an event.

Returns:
Boolean True if inputs are valid. False otherwise.


def _process_params(self):
    self._sort_to_str()
    if ('rows' not in self._solr_params):
        self._solr_params['rows'] = self._cfg['row_size']
    for (key, val) in self._solr_params.items():
        if (isinstance(val, str) and six.PY2):
            self._solr_params[key] = val.encode(encoding='UTF-8')
    return self._solr_params

Adds default row size if it's not given in the query.
Converts param values into unicode strings.

Returns:
Processed self._solr_params dict.


def jobs_get(self, job_id, project_id=None):

    if project_id is None:
      project_id = self._project_id
    url = Api._ENDPOINT + (Api._JOBS_PATH % (project_id, job_id))
    return datalab.utils.Http.request(url, credentials=self._credentials)

Issues a request to retrieve information about a job.

Args:
job_id: the id of the job
project_id: the project id to use to fetch the results; use None for the default project.
Returns:
A parsed result object.
Raises:
Exception if there is an error performing the operation.


def _get_event_handlers():
    import os
    import importlib
    event_handlers = {'on_ready': [], 'on_resume': [], 'on_error': [], 'on_message': [], 'on_socket_raw_receive': [], 'on_socket_raw_send': [], 'on_message_delete': [], 'on_message_edit': [], 'on_reaction_add': [], 'on_reaction_remove': [], 'on_reaction_clear': [], 'on_channel_delete': [], 'on_channel_create': [], 'on_channel_update': [], 'on_member_join': [], 'on_member_remove': [], 'on_member_update': [], 'on_server_join': [], 'on_server_remove': [], 'on_server_update': [], 'on_server_role_create': [], 'on_server_role_delete': [], 'on_server_role_update': [], 'on_server_emojis_update': [], 'on_server_available': [], 'on_server_unavailable': [], 'on_voice_state_update': [], 'on_member_ban': [], 'on_member_unban': [], 'on_typing': [], 'on_group_join': [], 'on_group_remove': []}
    database_dir = '{}/modules'.format(os.path.dirname(os.path.realpath(__file__)))
    for module_name in os.listdir(database_dir):
        module_dir = '{}/{}'.format(database_dir, module_name)
        if (os.path.isdir(module_dir) and (not module_name.startswith('_'))):
            module_event_handlers = os.listdir(module_dir)
            for event_handler in event_handlers.keys():
                if ('{}.py'.format(event_handler) in module_event_handlers):
                    import_name = '.discord_modis.modules.{}.{}'.format(module_name, event_handler)
                    logger.debug('Found event handler {}'.format(import_name[23:]))
                    try:
                        event_handlers[event_handler].append(importlib.import_module(import_name, 'modis'))
                    except Exception as e:
                        logger.exception(e)
    return event_handlers

Gets dictionary of event handlers and the modules that define them

Returns:
event_handlers (dict): Contains "all", "on_ready", "on_message", "on_reaction_add", "on_error"


def _execute(self, connection, query, fetch=True):
    cursor = connection.cursor()
    try:
        cursor.execute(query)
    except Exception as e:
        from ambry.mprlib.exceptions import BadSQLError
        raise BadSQLError('Failed to execute query: {}; {}'.format(query, e))
    if fetch:
        return cursor.fetchall()
    else:
        return cursor

Executes given query using given connection.

Args:
connection (apsw.Connection): connection to the sqlite db who stores mpr data.
query (str): sql query
fetch (boolean, optional): if True, fetch query result and return it. If False, do not fetch.

Returns:
iterable with query result.


def allan_variance(data, dt, tmax=10):

    allanvar = []
    nmax = len(data) if len(data) < tmax / dt else int(tmax / dt)
    for i in range(1, nmax+1):
        databis = data[len(data) % i:]
        y = databis.reshape(len(data)
        allanvar.append(((y[1:] - y[:-1])**2).mean() / 2)
    return dt * np.arange(1, nmax+1), np.array(allanvar)

Calculate Allan variance.

Args:
data (np.ndarray): Input data.
dt (float): Time between each data.
tmax (float): Maximum time.

Returns:
vk (np.ndarray): Frequency.
allanvar (np.ndarray): Allan variance.


def bgr2gray(img, keepdim=False):

    out_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    if keepdim:
        out_img = out_img[..., None]
    return out_img

Convert a BGR image to grayscale image.

Args:
img (ndarray): The input image.
keepdim (bool): If False (by default), then return the grayscale image
with 2 dims, otherwise 3 dims.

Returns:
ndarray: The converted grayscale image.


def to_url(self, site='amazon', country='us'):
    try:
        try:
            (url, tlds) = URL_MAP[site]
        except ValueError:
            tlds = None
            url = URL_MAP[site]
    except KeyError:
        raise SiteError(site)
    inject = {'isbn': self._isbn}
    if tlds:
        if (country not in tlds):
            raise CountryError(country)
        tld = tlds[country]
        if (not tld):
            tld = country
        inject['tld'] = tld
    return (url % inject)

Generate a link to an online book site.

Args:
site (str): Site to create link to
country (str): Country specific version of ``site``

Returns:
``str``: URL on ``site`` for book

Raises:
SiteError: Unknown site value
CountryError: Unknown country value


def deduplicate_readonly_buffers(tflite_model):
    model = flatbuffer_utils.convert_bytearray_to_object(tflite_model)
    read_only_buffer_indices = set()
    for subgraph in model.subgraphs:
        read_only_input_tensor_indices = set()
        for op in subgraph.operators:
            if op.inputs is None:
                continue
            for i, input_tensor_idx in enumerate(op.inputs):
                if op.mutatingVariableInputs is not None:
                    if i < len(op.mutatingVariableInputs) and op.mutatingVariableInputs[i]:
                        continue
                if subgraph.tensors[input_tensor_idx].isVariable:
                    continue
                read_only_input_tensor_indices.add(input_tensor_idx)
        for op in subgraph.operators:
            if op.outputs is not None:
                for output_tensor_idx in op.outputs:
                    read_only_input_tensor_indices.discard(output_tensor_idx)
            if op.intermediates is not None:
                for intermediate_tensor_idx in op.intermediates:
                    read_only_input_tensor_indices.discard(intermediate_tensor_idx)
        if subgraph.inputs is not None:
            for input_tensor_idx in subgraph.inputs:
                read_only_input_tensor_indices.discard(input_tensor_idx)
        if subgraph.outputs is not None:
            for output_tensor_idx in subgraph.outputs:
                read_only_input_tensor_indices.discard(output_tensor_idx)
        for tensor_idx in read_only_input_tensor_indices:
            read_only_buffer_indices.add(subgraph.tensors[tensor_idx].buffer)
    for buffer_idx in read_only_buffer_indices.copy():
        if buffer_idx < 0 or (model.buffers[buffer_idx].data is None or isinstance(model.buffers[buffer_idx].data, list) or model.buffers[buffer_idx].data.size == 0):
            read_only_buffer_indices.discard(buffer_idx)

    class BufferIndex:

        def __init__(self, idx, size, hash_value):
            self.idx = idx
            self.size = size
            self.hash_value = hash_value
    read_only_buffers = list(map(lambda index: BufferIndex(index, model.buffers[index].data.size, hashlib.md5(model.buffers[index].data.data.tobytes()).hexdigest()), read_only_buffer_indices))
    read_only_buffers = sorted(read_only_buffers, key=lambda buffer: (buffer.size, buffer.hash_value), reverse=True)
    duplicate_buffer_map = {}
    for i, buffer_i in enumerate(read_only_buffers):
        if buffer_i.idx in duplicate_buffer_map:
            continue
        for buffer_j in read_only_buffers[i + 1:]:
            if buffer_j.idx in duplicate_buffer_map:
                continue
            if buffer_i.size != buffer_j.size:
                break
            if buffer_i.hash_value != buffer_j.hash_value:
                continue
            duplicate_buffer_map[buffer_j.idx] = buffer_i.idx
    for subgraph in model.subgraphs:
        for op in subgraph.operators:
            if op.inputs is None:
                continue
            for input_tensor in op.inputs:
                buffer_idx = subgraph.tensors[input_tensor].buffer
                if buffer_idx in duplicate_buffer_map:
                    subgraph.tensors[input_tensor].buffer = duplicate_buffer_map[buffer_idx]
    for idx in duplicate_buffer_map:
        model.buffers[idx].data = None
    return flatbuffer_utils.convert_object_to_bytearray(model)

Generates a new model byte array after deduplicating readonly buffers.

This function should be invoked after the model optimization toolkit. The
model optimization toolkit assumes that each tensor object owns its each
buffer separately.

Args:
tflite_model: TFLite flatbuffer in a byte array to be deduplicated.

Returns:
TFLite flatbuffer in a bytes array, processed with the deduplication method.


def on_train_batch_begin(self, batch, logs=None):
    if self._should_call_train_batch_hooks:
        self._call_batch_hook(ModeKeys.TRAIN, 'begin', batch, logs=logs)

Calls the `on_train_batch_begin` methods of its callbacks.

Args:
batch: Integer, index of batch within the current epoch.
logs: Dict, contains the return value of `model.train_step`. Typically,
the values of the `Model`'s metrics are returned.  Example:
`{'loss': 0.2, 'accuracy': 0.7}`.


def _get_update_method(self):
    if getattr(self, '_update_uses_post', False):
        http_method = self.gitlab.http_post
    else:
        http_method = self.gitlab.http_put
    return http_method

Return the HTTP method to use.

Returns:
object: http_put (default) or http_post


def _ScanEncryptedVolume(self, scan_context, scan_node):

    if not scan_node or not scan_node.path_spec:
      raise errors.ScannerError('Invalid or missing scan node.')

    credentials = credentials_manager.CredentialsManager.GetCredentials(
        scan_node.path_spec)
    if not credentials:
      raise errors.ScannerError('Missing credentials for scan node.')

    if not self._mediator:
      raise errors.ScannerError(
          'Unable to proceed. Encrypted volume found but no mediator to '
          'determine how it should be unlocked.')

    if self._mediator.UnlockEncryptedVolume(
        self._source_scanner, scan_context, scan_node, credentials):
      self._source_scanner.Scan(
          scan_context, scan_path_spec=scan_node.path_spec)

Scans an encrypted volume scan node for volume and file systems.

Args:
scan_context (SourceScannerContext): source scanner context.
scan_node (SourceScanNode): volume scan node.

Raises:
ScannerError: if the format of or within the source is not supported,
the scan node is invalid, there are no credentials defined for
the format or no mediator is provided and a locked scan node was
found, e.g. an encrypted volume,


def check_output_variable(self, variable):

        match = False
        if variable in self.out_variables:
            match = True
        return match

Check to see if output variable was requested by downstream app.

Using the auto generated dictionary of output variables check to see if provided
variable was requested by downstream app.

Args:
variable (string): The variable name, not the full variable.

Returns:
(boolean): Boolean value indicator whether a match was found.


def _PairwiseCheck(self, pair_comparator, strict=False):
    i = iter(self._actual)
    try:
        prev = next(i)
        while True:
            current = next(i)
            if not pair_comparator(prev, current):
                strictly = 'strictly ' if strict else ''
                self._FailComparingValues('is {0}ordered'.format(strictly), (prev, current))
            prev = current
    except StopIteration:
        pass

Iterates over this subject and compares adjacent elements.

For example, compares element 0 with element 1, 1 with 2, ... n-1 with n.

Args:
pair_comparator: A function accepting two arguments. If the arguments are
ordered as expected, the function should return True, otherwise False.
strict: whether the pair comparator function is strict.


def generate_sjson_from_srt(srt_subs):

        sub_starts = []
        sub_ends = []
        sub_texts = []
        for sub in srt_subs:
            sub_starts.append(sub.start.ordinal)
            sub_ends.append(sub.end.ordinal)
            sub_texts.append(sub.text.replace('\n', ' '))

        sjson_subs = {
            'start': sub_starts,
            'end': sub_ends,
            'text': sub_texts
        }
        return sjson_subs

Generate transcripts from sjson to SubRip (*.srt).

Arguments:
srt_subs(SubRip): "SRT" subs object

Returns:
Subs converted to "SJSON" format.


def add_callback(self, name, func):
    if (name == 'on_scan'):
        events = ['device_seen']

        def callback(_conn_string, _conn_id, _name, event):
            func(self.id, event, event.get('validity_period', 60))
    elif (name == 'on_report'):
        events = ['report', 'broadcast']

        def callback(_conn_string, conn_id, _name, event):
            func(conn_id, event)
    elif (name == 'on_trace'):
        events = ['trace']

        def callback(_conn_string, conn_id, _name, event):
            func(conn_id, event)
    elif (name == 'on_disconnect'):
        events = ['disconnection']

        def callback(_conn_string, conn_id, _name, _event):
            func(self.id, conn_id)
    else:
        raise ArgumentError('Unknown callback type {}'.format(name))
    self._adapter.register_monitor([None], events, callback)

Add a callback when device events happen.

Args:
name (str): currently support 'on_scan' and 'on_disconnect'
func (callable): the function that should be called


def write_auth(msg_type, profile_name, auth, cfg):

    key_fmt = profile_name + "_" + msg_type
    pwd = []
    for k, v in CONFIG[msg_type]["auth"].items():
        pwd.append(auth[k])

    if len(pwd) > 1:
        cfg.pwd[key_fmt] = " :: ".join(pwd)
    else:
        cfg.pwd[key_fmt] = pwd[0]

Write the settings into the auth portion of the cfg.

Args:
:msg_type: (str) message type to create config entry.
:profile_name: (str) name of the profile entry
:auth: (dict) auth parameters
:cfg: (jsonconfig.Config) config instance.


def _indexed_slices_to_tensor(value, dtype=None, name=None, as_ref=False):
    _ = as_ref
    if dtype and (not dtype.is_compatible_with(value.dtype)):
        raise ValueError(f'Incompatible tensor conversion requested to `dtype` {dtype.name} for IndexedSlices ({value}) with dtype {value.dtype.name}')
    if value.dense_shape is None:
        raise ValueError(f'Tensor conversion requested for IndexedSlices for argument `value` without dense_shape: {value!s}')
    if not context.executing_eagerly():
        dense_shape_value = tensor_util.constant_value(value.dense_shape)
        if dense_shape_value is not None:
            num_elements = np.prod(dense_shape_value)
            if num_elements >= _LARGE_SPARSE_NUM_ELEMENTS:
                warnings.warn('Converting sparse IndexedSlices to a dense Tensor with %d elements. This may consume a large amount of memory.' % num_elements)
    return gen_math_ops.unsorted_segment_sum(value.values, value.indices, value.dense_shape[0], name=name)

Converts an IndexedSlices object `value` to a Tensor.

NOTE(mrry): This function is potentially expensive.

Args:
value: An ops.IndexedSlices object.
dtype: The dtype of the Tensor to be returned.
name: Optional name to use for the returned Tensor.
as_ref: True if a ref is requested.

Returns:
A dense Tensor representing the values in the given IndexedSlices.

Raises:
ValueError: If the IndexedSlices does not have the same dtype.


def __init__(self, time, status, latitude, longitude, speed, track, date,
                 variation, mode=None):

        super(Position, self).__init__(latitude, longitude)
        self.time = time
        self.status = status
        self.speed = speed
        self.track = track
        self.date = date
        self.variation = variation
        self.mode = mode

Initialise a new ``Position`` object.

Args:
time (datetime.time): Time the fix was taken
status (bool): Whether the data is active
latitude (float): Fix's latitude
longitude (float): Fix's longitude
speed (float): Ground speed
track (float): Track angle
date (datetime.date): Date when position was taken
variation (float): Magnetic variation
mode (str): Type of reading


def writeOutput(self, session, directory, name):
    self.project_directory = directory
    with tmp_chdir(directory):
        batchDirectory = self._getBatchDirectory(directory)
        self._writeReplacementFiles(session=session, directory=directory, name=name)
        self.write(session=session, directory=directory, name=name)
        self._writeXput(session=session, directory=batchDirectory, fileCards=self.OUTPUT_FILES, name=name)
        self._writeWMSDatasets(session=session, directory=batchDirectory, wmsDatasetCards=self.WMS_DATASETS, name=name)

Write only output files for a GSSHA project from the database to file.

Args:
session (:mod:`sqlalchemy.orm.session.Session`): SQLAlchemy session object bound to PostGIS enabled database
directory (str): Directory where the files will be written.
name (str): Name that will be given to project when written (e.g.: 'example'). Files that follow the project
naming convention will be given this name with the appropriate extension (e.g.: 'example.prj',
'example.cmt', and 'example.gag'). Files that do not follow this convention will retain their original
file names.


def WriteManyToPath(objs, filepath):
    with io.open(filepath, mode='w', encoding='utf-8') as filedesc:
        WriteManyToFile(objs, filedesc)

Serializes and writes given Python objects to a multi-document YAML file.

Args:
objs: An iterable of Python objects to serialize.
filepath: A path to the file into which the object is to be written.


def build_model(hparams_set, model_name, data_dir, problem_name, beam_size=1):
    hparams = trainer_lib.create_hparams(hparams_set, data_dir=data_dir, problem_name=problem_name)
    translate_model = registry.model(model_name)(hparams, tf.estimator.ModeKeys.EVAL)
    inputs = tf.placeholder(tf.int32, shape=(1, None, 1, 1), name='inputs')
    targets = tf.placeholder(tf.int32, shape=(1, None, 1, 1), name='targets')
    translate_model({'inputs': inputs, 'targets': targets})
    att_mats = get_att_mats(translate_model)
    with tf.variable_scope(tf.get_variable_scope(), reuse=True):
        samples = translate_model.infer({'inputs': inputs}, beam_size=beam_size)['outputs']
    return (inputs, targets, samples, att_mats)

Build the graph required to fetch the attention weights.

Args:
hparams_set: HParams set to build the model with.
model_name: Name of model.
data_dir: Path to directory containing training data.
problem_name: Name of problem.
beam_size: (Optional) Number of beams to use when decoding a translation.
If set to 1 (default) then greedy decoding is used.

Returns:
Tuple of (
inputs: Input placeholder to feed in ids to be translated.
targets: Targets placeholder to feed to translation when fetching
attention weights.
samples: Tensor representing the ids of the translation.
att_mats: Tensors representing the attention weights.
)


def setup_colorbars(self, plot_call_sign):

        self.fig.colorbar(plot_call_sign, cax=self.cbar_ax,
                          ticks=self.cbar_ticks, orientation=self.cbar_orientation)

        (getattr(self.cbar_ax, 'set_' + self.cbar_var + 'ticklabels')
            (self.cbar_tick_labels, fontsize=self.cbar_ticks_fontsize))
        (getattr(self.cbar_ax, 'set_' + self.cbar_var + 'label')
            (self.cbar_label, fontsize=self.cbar_label_fontsize, labelpad=self.cbar_label_pad))

        return

Setup colorbars for each type of plot.

Take all of the optional performed during ``__init__`` method and makes the colorbar.

Args:
plot_call_sign (obj): Plot instance of ax.contourf with colormapping to
add as a colorbar.


def add_oxidation_state_by_site_fraction(structure, oxidation_states):

        try:
            for i, site in enumerate(structure):
                new_sp = collections.defaultdict(float)
                for j, (el, occu) in enumerate(get_z_ordered_elmap(site
                        .species)):
                    specie = Specie(el.symbol, oxidation_states[i][j])
                    new_sp[specie] += occu
                structure[i] = new_sp
            return structure
        except IndexError:
            raise ValueError("Oxidation state of all sites must be "
                             "specified in the list.")

Add oxidation states to a structure by fractional site.

Args:
oxidation_states (list): List of list of oxidation states for each
site fraction for each site.
E.g., [[2, 4], [3], [-2], [-2], [-2]]


def collate(self, merge_type=None, drop=[], drop_constant=False):
    from .element import Collator
    merge_type = (merge_type if merge_type else self.__class__)
    return Collator(self, merge_type=merge_type, drop=drop, drop_constant=drop_constant)()

Collate allows reordering nested containers

Collation allows collapsing nested mapping types by merging
their dimensions. In simple terms in merges nested containers
into a single merged type.

In the simple case a HoloMap containing other HoloMaps can
easily be joined in this way. However collation is
particularly useful when the objects being joined are deeply
nested, e.g. you want to join multiple Layouts recorded at
different times, collation will return one Layout containing
HoloMaps indexed by Time. Changing the merge_type will allow
merging the outer Dimension into any other UniformNdMapping
type.

Args:
merge_type: Type of the object to merge with
drop: List of dimensions to drop
drop_constant: Drop constant dimensions automatically

Returns:
Collated Layout or HoloMap


def get_thumbnail(self, mxcurl, width, height, method='scale', allow_remote=True):
    if (method not in ['scale', 'crop']):
        raise ValueError(("Unsupported thumb method '%s'" % method))
    query_params = {'width': width, 'height': height, 'method': method}
    if (not allow_remote):
        query_params['allow_remote'] = False
    if mxcurl.startswith('mxc:
        return self._send('GET', mxcurl[6:], query_params=query_params, api_path='/_matrix/media/r0/thumbnail/', return_json=False)
    else:
        raise ValueError(("MXC URL '%s' did not begin with 'mxc:

Download raw media thumbnail from provided mxc URL.

Args:
mxcurl (str): mxc media URL
width (int): desired thumbnail width
height (int): desired thumbnail height
method (str): thumb creation method. Must be
in ['scale', 'crop']. Default 'scale'.
allow_remote (bool): indicates to the server that it should not
attempt to fetch the media if it is deemed remote. Defaults
to true if not provided.


def _handle_per_output_metrics(self, metrics_dict, y_true, y_pred, mask, weights=None):
    metric_results = []
    for metric_name, metric_fn in metrics_dict.items():
        with backend.name_scope(metric_name):
            metric_result = training_utils_v1.call_metric_function(metric_fn, y_true, y_pred, weights=weights, mask=mask)
            metric_results.append(metric_result)
    return metric_results

Calls metric functions for a single output.

Args:
metrics_dict: A dict with metric names as keys and metric fns as values.
y_true: Target output.
y_pred: Predicted output.
mask: Computed mask value for the current output.
weights: Weights to be applied on the current output.

Returns:
A list of metric result tensors.


def seek(self, n):

        if self._mode != "r":
            raise UnsupportedOperation("not available in 'w' mode")

        if 0 <= n < self._nb_markers:
            self._n = n
            self._bed.seek(self._get_seek_position(n))

        else:

            raise ValueError("invalid position in BED: {}".format(n))

Gets to a certain marker position in the BED file.

Args:
n (int): The index of the marker to seek to.


def _verify_barycentric(lambda1, lambda2, lambda3):
    weights_total = ((lambda1 + lambda2) + lambda3)
    if (not np.allclose(weights_total, 1.0, atol=0.0)):
        raise ValueError('Weights do not sum to 1', lambda1, lambda2, lambda3)
    if ((lambda1 < 0.0) or (lambda2 < 0.0) or (lambda3 < 0.0)):
        raise ValueError('Weights must be positive', lambda1, lambda2, lambda3)

Verifies that weights are barycentric and on the reference triangle.

I.e., checks that they sum to one and are all non-negative.

Args:
lambda1 (float): Parameter along the reference triangle.
lambda2 (float): Parameter along the reference triangle.
lambda3 (float): Parameter along the reference triangle.

Raises:
ValueError: If the weights are not valid barycentric
coordinates, i.e. they don't sum to ``1``.
ValueError: If some weights are negative.


def _insource_jedi_vim_test(data, ibs):

    data
    ibs
    import utool as ut
    xdata = ut.ColumnLists()
    xdata
    import ibeis
    xibs = ibeis.IBEISController()
    xibs

If jedi-vim supports google style docstrings you should be able to
autocomplete ColumnLists methods for `data`

Args:
data (utool.ColumnLists): a column list objct
ibs (ibeis.IBEISController): an object


def filter_benchmarks(benchmarks, bench_funcs, base_ver):
    for bm in list(benchmarks):
        func = bench_funcs[bm]
        if (getattr(func, '_python2_only', False) and ((3, 0) <= base_ver)):
            benchmarks.discard(bm)
            logging.info(('Skipping Python2-only benchmark %s; not compatible with Python %s' % (bm, base_ver)))
            continue
    return benchmarks

Filters out benchmarks not supported by both Pythons.

Args:
benchmarks: a set() of benchmark names
bench_funcs: dict mapping benchmark names to functions
python: the interpereter commands (as lists)

Returns:
The filtered set of benchmark names


async def retry_request(*args, retry_exceptions=(asyncio.TimeoutError, ScriptWorkerRetryException), retry_async_kwargs=None, **kwargs):
    retry_async_kwargs = (retry_async_kwargs or {})
    return (await retry_async(request, retry_exceptions=retry_exceptions, args=args, kwargs=kwargs, **retry_async_kwargs))

Retry the ``request`` function.

Args:
*args: the args to send to request() through retry_async().
retry_exceptions (list, optional): the exceptions to retry on.
Defaults to (ScriptWorkerRetryException, ).
retry_async_kwargs (dict, optional): the kwargs for retry_async.
If None, use {}.  Defaults to None.
**kwargs: the kwargs to send to request() through retry_async().

Returns:
object: the value from request().


def terminate_and_create_image(name):

    node = _host_node()
    operation = _gcp().instances().delete(project=DEFAULT_PROJECT, zone=DEFAULT_ZONE,
                                          instance=node['real_name']).execute()
    while True:
        status = get_zone_operation_status(operation=operation)
        if status == 'DONE':
            break

        print 'Terminating instance [OPERATION %s]' % status
        time.sleep(5)

    body = {
        'name': name,
        'sourceDisk': node['source_disk'],
    }

    operation = _gcp().images().insert(project=DEFAULT_PROJECT, body=body).execute()
    while True:
        status = get_global_operation_status(operation=operation)
        if status == 'DONE':
            break

        print 'Creating image [OPERATION %s]' % status
        time.sleep(5)

    print 'Created image: %s' % operation['targetLink']

Create an image from a terminated host (with auto_delete_boot_disk=False)

Args:
name: The name of the image


def count_ops(self):
    count_ops = {}
    for (instr, _, _) in self.data:
        if (instr.name in count_ops.keys()):
            count_ops[instr.name] += 1
        else:
            count_ops[instr.name] = 1
    return count_ops

Count each operation kind in the circuit.

Returns:
dict: a breakdown of how many operations of each kind.


def rmod(self, other, axis="columns", level=None, fill_value=None):

        return self._binary_op(
            "rmod", other, axis=axis, level=level, fill_value=fill_value
        )

Mod this DataFrame against another DataFrame/Series/scalar.

Args:
other: The object to use to apply the div against this.
axis: The axis to div over.
level: The Multilevel index level to apply div over.
fill_value: The value to fill NaNs with.

Returns:
A new DataFrame with the rdiv applied.


def save_as(self, filename=None):

        if filename is None:
            filename = self.filename
        if filename is None:
            filename = self.default_filename
        if filename is None:
            raise RuntimeError("Class '{}' has no default filename".format(self.__class__.__name__))
        self._do_save_as(filename)
        self.filename = filename

Dumps object contents into file on disk.

Args:
filename (optional): defaults to self.filename. If passed, self.filename
will be updated to filename.


def _KeyToFilePath(key, api_version):

    def _ReplaceCapsWithDash(matchobj):
        match = matchobj.group(0)
        return '-%s' % match.lower()
    case_insensitive_key = re.sub('([A-Z]{1})', _ReplaceCapsWithDash, key)
    api_folder = _API_GOLDEN_FOLDER_V2 if api_version == 2 else _API_GOLDEN_FOLDER_V1
    if key.startswith('tensorflow.experimental.numpy'):
        api_folder = os.path.join(api_folder, '..', '..', '..', '..', '../third_party', 'py', 'numpy', 'tf_numpy_api')
        api_folder = os.path.normpath(api_folder)
    return os.path.join(api_folder, '%s.pbtxt' % case_insensitive_key)

From a given key, construct a filepath.

Filepath will be inside golden folder for api_version.

Args:
key: a string used to determine the file path
api_version: a number indicating the tensorflow API version, e.g. 1 or 2.

Returns:
A string of file path to the pbtxt file which describes the public API


def set_contents(self, contents, encoding=None):
    self.encoding = encoding
    changed = self._set_initial_contents(contents)
    if (self._side_effect is not None):
        self._side_effect(self)
    return changed

Sets the file contents and size and increases the modification time.
Also executes the side_effects if available.

Args:
contents: (str, bytes, unicode) new content of file.
encoding: (str) the encoding to be used for writing the contents
if they are a unicode string.
If not given, the locale preferred encoding is used.

Raises:
IOError: if `st_size` is not a non-negative integer,
or if it exceeds the available file system space.


def saveAsTFRecords(df, output_dir):
    tf_rdd = df.rdd.mapPartitions(toTFExample(df.dtypes))
    tf_rdd.saveAsNewAPIHadoopFile(output_dir, 'org.tensorflow.hadoop.io.TFRecordFileOutputFormat', keyClass='org.apache.hadoop.io.BytesWritable', valueClass='org.apache.hadoop.io.NullWritable')

Save a Spark DataFrame as TFRecords.

This will convert the DataFrame rows to TFRecords prior to saving.

Args:
:df: Spark DataFrame
:output_dir: Path to save TFRecords


def most_by_uncertain(self, y):

        return self.most_uncertain_by_mask((self.ds.y == y), y)

Extracts the predicted classes which correspond to the selected class (y) and have probabilities nearest to 1/number_of_classes (eg. 0.5 for 2 classes, 0.33 for 3 classes) for the selected class.

Arguments:
y (int): the selected class

Returns:
idxs (numpy.ndarray): An array of indexes (numpy.ndarray)


def delete_knowledge_base(project_id, knowledge_base_id):
    import dialogflow_v2beta1 as dialogflow
    client = dialogflow.KnowledgeBasesClient()
    knowledge_base_path = client.knowledge_base_path(project_id, knowledge_base_id)
    response = client.delete_knowledge_base(knowledge_base_path)
    print('Knowledge Base deleted.'.format(response))

Deletes a specific Knowledge base.

Args:
project_id: The GCP project linked with the agent.
knowledge_base_id: Id of the Knowledge base.


def convert_pytorch_checkpoint_to_tf(model: BertModel, ckpt_dir: str, model_name: str):
    tensors_to_transpose = ('dense.weight', 'attention.self.query', 'attention.self.key', 'attention.self.value')
    var_map = (('layer.', 'layer_'), ('word_embeddings.weight', 'word_embeddings'), ('position_embeddings.weight', 'position_embeddings'), ('token_type_embeddings.weight', 'token_type_embeddings'), ('.', '/'), ('LayerNorm/weight', 'LayerNorm/gamma'), ('LayerNorm/bias', 'LayerNorm/beta'), ('weight', 'kernel'))
    if not os.path.isdir(ckpt_dir):
        os.makedirs(ckpt_dir)
    state_dict = model.state_dict()

    def to_tf_var_name(name: str):
        for patt, repl in iter(var_map):
            name = name.replace(patt, repl)
        return f'bert/{name}'

    def create_tf_var(tensor: np.ndarray, name: str, session: tf.Session):
        tf_dtype = tf.dtypes.as_dtype(tensor.dtype)
        tf_var = tf.get_variable(dtype=tf_dtype, shape=tensor.shape, name=name, initializer=tf.zeros_initializer())
        session.run(tf.variables_initializer([tf_var]))
        session.run(tf_var)
        return tf_var
    tf.reset_default_graph()
    with tf.Session() as session:
        for var_name in state_dict:
            tf_name = to_tf_var_name(var_name)
            torch_tensor = state_dict[var_name].numpy()
            if any((x in var_name for x in tensors_to_transpose)):
                torch_tensor = torch_tensor.T
            tf_var = create_tf_var(tensor=torch_tensor, name=tf_name, session=session)
            tf_var.assign(tf.cast(torch_tensor, tf_var.dtype))
            tf_weight = session.run(tf_var)
            print(f'Successfully created {tf_name}: {np.allclose(tf_weight, torch_tensor)}')
        saver = tf.train.Saver(tf.trainable_variables())
        saver.save(session, os.path.join(ckpt_dir, model_name.replace('-', '_') + '.ckpt'))

Args:
model: BertModel Pytorch model instance to be converted
ckpt_dir: Tensorflow model directory
model_name: model name

Currently supported HF models:

- Y BertModel
- N BertForMaskedLM
- N BertForPreTraining
- N BertForMultipleChoice
- N BertForNextSentencePrediction
- N BertForSequenceClassification
- N BertForQuestionAnswering


def Dump(obj, sort_keys=False, encoder=None):
    text = json.dumps(obj, indent=2, sort_keys=sort_keys, ensure_ascii=False, cls=encoder, separators=_SEPARATORS)
    if (compatibility.PY2 and isinstance(text, bytes)):
        text = text.decode('utf-8')
    return text

Stringifies a Python object into its JSON representation.

Args:
obj: A Python object to convert to JSON.
sort_keys: If True, output dictionaries keys in sorted (ascending) order.
encoder: An (optional) encoder class to use.

Returns:
A JSON representation of the given object.


def clear_list(self, **kwargs):

        path = self._get_id_path('clear')
        kwargs.update({'session_id': self.session_id})

        payload = {}

        response = self._POST(path, kwargs, payload)
        self._set_attrs_to_values(response)
        return response

Clears all of the items within a list. This is an irreversible action
and should be treated with caution.

A valid session id is required.

Args:
confirm: True (do it) | False (don't do it)

Returns:
A dict respresentation of the JSON returned from the API.


def CreateShowcaseAd(client, adgroup, expanded_image_filepath,
                     collapsed_image_filepath):

  ad_group_ad_service = client.GetService('AdGroupAdService', 'v201809')

  showcase_ad = {
      'adGroupId': adgroup['id'],
      'ad': {
          'xsi_type': 'ShowcaseAd',
          'Ad.Type': 'ShowcaseAd',

          'name': 'Showcase ad
          'finalUrls': 'http:
          'displayUrl': 'example.com',

          'expandedImage': {
              'mediaId': UploadImage(client, expanded_image_filepath)['mediaId']
          },

          'collapsedImage': {
              'mediaId':
                  UploadImage(client, collapsed_image_filepath)['mediaId']
          }
      }
  }

  ad_operation = {
      'operator': 'ADD',
      'operand': showcase_ad
  }

  showcase_ad = ad_group_ad_service.mutate([ad_operation])['value'][0]

  print 'ShowcaseAd with ID "%s" was added.' % showcase_ad['ad']['id']

  return showcase_ad

Creates a showcase add for the given AdGroup with the given images.

Args:
client: an AdWordsClient instance.
adgroup: a dict or suds object defining an AdGroup for a Shopping Campaign.
expanded_image_filepath: a str filepath to a .jpg file that will be used as
the Showcase Ad's expandedImage.
collapsed_image_filepath: a str filepath to a .jpg file that will be used as
the Showcase Ad's collapsedImage.

Returns:
The created Showcase Ad as a sudsobject.


def _ReadLabels(self, artifact_definition_values, artifact_definition, name):
    labels = artifact_definition_values.get('labels', [])
    undefined_labels = set(labels).difference(self.labels)
    if undefined_labels:
        raise errors.FormatError('Artifact definition: {0:s} found undefined labels: {1:s}.'.format(name, ', '.join(undefined_labels)))
    artifact_definition.labels = labels

Reads the optional artifact definition labels.

Args:
artifact_definition_values (dict[str, object]): artifact definition
values.
artifact_definition (ArtifactDefinition): an artifact definition.
name (str): name of the artifact definition.

Raises:
FormatError: if there are undefined labels.


def vals2bins(vals,res=100):

    if any(isinstance(el, list) for el in vals):
        vals = list(itertools.chain(*vals))
    return list(np.digitize(vals, np.linspace(np.min(vals), np.max(vals)+1, res+1)) - 1)

Maps values to bins
Args:
values (list or list of lists) - list of values to map to colors
res (int) - resolution of the color map (default: 100)
Returns:
list of numbers representing bins


def If(cond, inputs, then_branch, else_branch, name=None):
    if isinstance(then_branch, function._DefinedFunction):
        tlist = [_.type for _ in then_branch.definition.signature.output_arg]
        return gen_functional_ops._if(cond, inputs, tlist, then_branch, else_branch, name=name)
    then_out = then_branch.structured_outputs
    else_out = else_branch.structured_outputs
    nest.assert_same_structure(then_out, else_out, expand_composites=True)
    tlist = nest.flatten(then_branch.output_dtypes)
    ret = gen_functional_ops._if(cond, inputs, tlist, then_branch, else_branch, name=name)
    return nest.pack_sequence_as(then_out, ret, expand_composites=True)

output = Cond(inputs) ?

then_branch(inputs) : else_branch(inputs).

Args:
cond: A `Tensor`. A scalar. If the scalar is not a boolean, the scalar is
converted to a boolean according to the following rule: if the scalar is a
numerical value, non-zero means True and zero means False; if the scalar
is a string, non-empty means True and empty means False.
inputs: A list of input tensors.
then_branch: A function takes 'inputs' and returns a list of tensors, whose
types are the same as what else_branch returns.
else_branch: A function takes 'inputs' and returns a list of tensors. whose
types are the same as what then_branch returns.
name: A name for the operation (optional).

Returns:
A list of tensors returned by either then_branch(inputs)
or else_branch(inputs).


def __str__(self):

        manu = self.manufacturer
        return '%s <Core Id. %s, Manu. %s>' % (self.name, self.Core, manu)

Returns a string representation of this instance.

Args:
self (JLinkDeviceInfo): the ``JLinkDeviceInfo`` instance

Returns:
Returns a string specifying the device name, core, and manufacturer.


def _detect_gce_environment():
    http = transport.get_http_object(timeout=GCE_METADATA_TIMEOUT)
    try:
        (response, _) = transport.request(http, _GCE_METADATA_URI, headers=_GCE_HEADERS)
        return ((response.status == http_client.OK) and (response.get(_METADATA_FLAVOR_HEADER) == _DESIRED_METADATA_FLAVOR))
    except socket.error:
        logger.info('Timeout attempting to reach GCE metadata service.')
        return False

Determine if the current environment is Compute Engine.

Returns:
Boolean indicating whether or not the current environment is Google
Compute Engine.


def require_attribute(self, attribute: str, typ: Type = _Any) -> None:

        attr_nodes = [
            value_node for key_node, value_node in self.yaml_node.value
            if key_node.value == attribute
        ]
        if len(attr_nodes) == 0:
            raise RecognitionError(
                ('{}{}Missing required attribute {}').format(
                    self.yaml_node.start_mark, os.linesep, attribute))
        attr_node = attr_nodes[0]

        if typ != _Any:
            recognized_types, message = self.__recognizer.recognize(
                attr_node, cast(Type, typ))
            if len(recognized_types) == 0:
                raise RecognitionError(message)

Require an attribute on the node to exist.

If `typ` is given, the attribute must have this type.

Args:
attribute: The name of the attribute / mapping key.
typ: The type the attribute must have.


def list_adb_devices():
    out = adb.AdbProxy().devices()
    return parse_device_list(out, 'device')

List all android devices connected to the computer that are detected by
adb.

Returns:
A list of android device serials. Empty if there's none.


def _prepare_lambada_data(tmp_dir, data_dir, vocab_size, vocab_filename):
    if (not tf.gfile.Exists(data_dir)):
        tf.gfile.MakeDirs(data_dir)
    file_path = generator_utils.maybe_download(tmp_dir, _TAR, _URL)
    tar_all = tarfile.open(file_path)
    tar_all.extractall(tmp_dir)
    tar_all.close()
    tar_train = tarfile.open(os.path.join(tmp_dir, 'train-novels.tar'))
    tar_train.extractall(tmp_dir)
    tar_train.close()
    vocab_path = os.path.join(data_dir, vocab_filename)
    if (not tf.gfile.Exists(vocab_path)):
        with tf.gfile.GFile(os.path.join(tmp_dir, _VOCAB), 'r') as infile:
            reader = csv.reader(infile, delimiter='\t')
            words = [row[0] for row in reader]
            words = ([_UNK] + words[:vocab_size])
        with tf.gfile.GFile(vocab_path, 'w') as outfile:
            outfile.write('\n'.join(words))

Downloading and preparing the dataset.

Args:
tmp_dir: tem directory
data_dir: data directory
vocab_size: size of vocabulary
vocab_filename: name of vocab file


def Bernoulli(cls, mean: 'TensorFluent', batch_size: Optional[int]=None) -> Tuple[(Distribution, 'TensorFluent')]:
    probs = mean.tensor
    dist = tf.distributions.Bernoulli(probs=probs, dtype=tf.bool)
    batch = mean.batch
    if ((not batch) and (batch_size is not None)):
        t = dist.sample(batch_size)
        batch = True
    else:
        t = dist.sample()
    scope = mean.scope.as_list()
    return (dist, TensorFluent(t, scope, batch=batch))

Returns a TensorFluent for the Bernoulli sampling op with given mean parameter.

Args:
mean: The mean parameter of the Bernoulli distribution.
batch_size: The size of the batch (optional).

Returns:
The Bernoulli distribution and a TensorFluent sample drawn from the distribution.


def _map_across_full_axis_select_indices(self, axis, func, indices, keep_remaining=False):
    return self.data.apply_func_to_select_indices_along_full_axis(axis, func, indices, keep_remaining)

Maps function to select indices along full axis.

Args:
axis: 0 for columns and 1 for rows.
func: Callable mapping function over the BlockParitions.
indices: indices along axis to map over.
keep_remaining: True if keep indices where function was not applied.

Returns:
BaseFrameManager containing the result of mapping func over axis on indices.


def get_dataset(self, dataset_ref, retry=DEFAULT_RETRY):
    if isinstance(dataset_ref, str):
        dataset_ref = DatasetReference.from_string(dataset_ref, default_project=self.project)
    api_response = self._call_api(retry, method='GET', path=dataset_ref.path)
    return Dataset.from_api_repr(api_response)

Fetch the dataset referenced by ``dataset_ref``

Args:
dataset_ref (Union[ \
:class:`~google.cloud.bigquery.dataset.DatasetReference`, \
str, \
]):
A reference to the dataset to fetch from the BigQuery API.
If a string is passed in, this method attempts to create a
dataset reference from a string using
:func:`~google.cloud.bigquery.dataset.DatasetReference.from_string`.
retry (:class:`google.api_core.retry.Retry`):
(Optional) How to retry the RPC.

Returns:
google.cloud.bigquery.dataset.Dataset:
A ``Dataset`` instance.


def download(self, task, default_ext, timeout=5, max_retry=3, overwrite=False, **kwargs):
    file_url = task['file_url']
    task['success'] = False
    task['filename'] = None
    retry = max_retry
    if (not overwrite):
        with self.lock:
            self.fetched_num += 1
            filename = self.get_filename(task, default_ext)
            if self.storage.exists(filename):
                self.logger.info('skip downloading file %s', filename)
                return
            self.fetched_num -= 1
    while ((retry > 0) and (not self.signal.get('reach_max_num'))):
        try:
            response = self.session.get(file_url, timeout=timeout)
        except Exception as e:
            self.logger.error('Exception caught when downloading file %s, error: %s, remaining retry times: %d', file_url, e, (retry - 1))
        else:
            if self.reach_max_num():
                self.signal.set(reach_max_num=True)
                break
            elif (response.status_code != 200):
                self.logger.error('Response status code %d, file %s', response.status_code, file_url)
                break
            elif (not self.keep_file(task, response, **kwargs)):
                break
            with self.lock:
                self.fetched_num += 1
                filename = self.get_filename(task, default_ext)
            self.logger.info('image
            self.storage.write(filename, response.content)
            task['success'] = True
            task['filename'] = filename
            break
        finally:
            retry -= 1

Download the image and save it to the corresponding path.

Args:
task (dict): The task dict got from ``task_queue``.
timeout (int): Timeout of making requests for downloading images.
max_retry (int): the max retry times if the request fails.
**kwargs: reserved arguments for overriding.


def openResultsInBrowser(res):

    print(emphasis("\n\tOpening URIs in the default web browser..."))

    urisToBrowser(["https:

    time.sleep(2)

    uris = []
    for r in res:
        for att in r["attributes"]:
            if att["type"] == "i3visio.uri":
                uris.append(att["value"])

    urisToBrowser(uris)

Method that collects the URI from a list of entities and opens them

Args:
-----
res: A list containing several i3visio entities.


def list_summaries(logdir):
    result = _SummaryFile()
    for dirpath, _, filenames in os.walk(logdir):
        for filename in filenames:
            if not filename.startswith('events.out.'):
                continue
            path = os.path.join(dirpath, filename)
            for event in _SummaryIterator(path):
                if event.graph_def:
                    result.graph_defs.append(event.graph_def)
                if not event.summary:
                    continue
                for value in event.summary.value:
                    tag = value.tag
                    kind = value.WhichOneof('value')
                    container = {'simple_value': result.scalars, 'image': result.images, 'histo': result.histograms, 'tensor': result.tensors}.get(kind)
                    if container is None:
                        raise ValueError('Unexpected summary kind %r in event file %s:\n%r' % (kind, path, event))
                    elif kind == 'tensor' and tag != 'keras':
                        plugin_name = value.metadata.plugin_data.plugin_name
                        container = {'images': result.images, 'histograms': result.histograms, 'scalars': result.scalars}.get(plugin_name)
                        if container is not None:
                            result.convert_from_v2_summary_proto = True
                        else:
                            container = result.tensors
                    container.add(_ObservedSummary(logdir=dirpath, tag=tag))
    return result

Read all summaries under the logdir into a `_SummaryFile`.

Args:
logdir: A path to a directory that contains zero or more event
files, either as direct children or in transitive subdirectories.
Summaries in these events must only contain old-style scalars,
images, and histograms. Non-summary events, like `graph_def`s, are
ignored.

Returns:
A `_SummaryFile` object reflecting all summaries written to any
event files in the logdir or any of its descendant directories.

Raises:
ValueError: If an event file contains an summary of unexpected kind.


def _clone_functional_model(model, clone_function, input_tensors=None, call_function=None):
    if not callable(clone_function):
        raise ValueError(f'Expected `clone_function` argument to be a callable. Received: clone_function={clone_function}')
    if not isinstance(model, Functional):
        raise ValueError(f'Expected `model` argument to be a Functional Model instance. Received: model={model}')
    if input_tensors is not None:
        if not all((isinstance(x, backend.KerasTensor) for x in tree.flatten(input_tensors))):
            raise ValueError(f'All entries in `input_tensors` must be KerasTensors. Received invalid values: inputs_tensors={input_tensors}')
        try:
            tree.assert_same_structure(input_tensors, model.input)
        except ValueError as e:
            raise ValueError(f'`input_tensors` must have the same structure as model.input\nReference structure: {model.input}\nReceived structure: {input_tensors}') from e
    else:
        input_tensors = tree.map_structure(lambda x: Input(batch_shape=x.shape, dtype=x.dtype, name=x.name), model.input)

    def operation_fn(layer):
        new_layer = clone_function(layer)
        return new_layer
    output_tensors = model._run_through_graph(input_tensors, operation_fn=operation_fn, call_fn=call_function)
    if functional_like_constructor(model.__class__):
        new_model = model.__class__(input_tensors, output_tensors, name=model.name)
    else:
        new_model = Functional(input_tensors, output_tensors, name=model.name)
    if model.compiled:
        compiled_config = model.get_compile_config()
        new_model.compile_from_config(compiled_config)
    return new_model

Clone a `Functional` model instance.

Model cloning is similar to calling a model on new inputs,
except that it creates new layers (and thus new weights) instead
of sharing the weights of the existing layers.

Input layers are always cloned.

Args:
model: Instance of `Functional`.
input_tensors: optional list of input tensors
to build the model upon. If not provided,
placeholders will be created.
clone_function: callable to be applied on non-input layers in the model.
By default, it clones the layer (without copying the weights).

Returns:
An instance of `Functional` reproducing the behavior
of the original model, on top of new inputs tensors,
using newly instantiated weights.


def _make_token_async(scopes, service_account_id):

  rpc = app_identity.create_rpc()
  app_identity.make_get_access_token_call(rpc, scopes, service_account_id)
  token, expires_at = yield rpc
  raise ndb.Return((token, expires_at))

Get a fresh authentication token.

Args:
scopes: A list of scopes.
service_account_id: Internal-use only.

Raises:
An ndb.Return with a tuple (token, expiration_time) where expiration_time is
seconds since the epoch.


def listdir(path='.'):

    return [name.rstrip('/') for name, _ in
            get_instance(path).list_objects(path, first_level=True)]

Return a list containing the names of the entries in the directory given by
path.

Equivalent to "os.listdir".

Args:
path (path-like object): Path or URL.

Returns:
list of str: Entries names.


def render_latex(latex: str) -> PIL.Image:
    tmpfilename = 'circ'
    with tempfile.TemporaryDirectory() as tmpdirname:
        tmppath = os.path.join(tmpdirname, tmpfilename)
        with open((tmppath + '.tex'), 'w') as latex_file:
            latex_file.write(latex)
        subprocess.run(['pdflatex', '-halt-on-error', '-output-directory={}'.format(tmpdirname), '{}'.format((tmpfilename + '.tex'))], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True)
        subprocess.run(['pdftocairo', '-singlefile', '-png', '-q', (tmppath + '.pdf'), tmppath])
        img = PIL.Image.open((tmppath + '.png'))
    return img

Convert a single page LaTeX document into an image.

To display the returned image, `img.show()`

Required external dependencies: `pdflatex` (with `qcircuit` package),
and `poppler` (for `pdftocairo`).

Args:
A LaTeX document as a string.

Returns:
A PIL Image

Raises:
OSError: If an external dependency is not installed.


def get_propagate_status(self, token, channel):

        url = self.url('sd/{}/{}/getPropagate/'.format(token, channel))
        req = self.remote_utils.get_url(url)
        if req.status_code is not 200:
            raise ValueError('Bad pair: {}/{}'.format(token, channel))
        return req.text

Get the propagate status for a token/channel pair.

Arguments:
token (str): The token to check
channel (str): The channel to check

Returns:
str: The status code


def get_dataset(self):
    raise NotImplementedError

Get a dataset instance for the current DataAdapter.

Note that the dataset returned does not repeat for epoch, so caller might
need to create new iterator for the same dataset at the beginning of the
epoch. This behavior might change in future.

Returns:
An tf.dataset.Dataset. Caller might use the dataset in different
context, eg iter(dataset) in eager to get the value directly, or in graph
mode, provide the iterator tensor to Keras model function.


def infer_transportation_mode(self, clf, min_time):

        self.transportation_modes = speed_clustering(clf, self.points, min_time)
        return self

In-place transportation mode inferring

See infer_transportation_mode function

Args:
Returns:
:obj:`Segment`: self


def parse_radl(data):

    if data is None:
        return None
    elif os.path.isfile(data):
        f = open(data)
        data = "".join(f.readlines())
        f.close()
    elif data.strip() == "":
        return RADL()
    data = data + "\n"

    parser = RADLParser(lextab='radl')
    return parser.parse(data)

Parse a RADL document.

Args:
- data(str): filepath to a RADL content or a string with content.

Return: RADL object.


def nx_gen_edge_values(G, key, edges=None, default=util_const.NoParam, on_missing='error', on_keyerr='default'):
    if (edges is None):
        edges = G.edges()
    if (on_missing is None):
        on_missing = 'error'
    if (on_keyerr is None):
        on_keyerr = 'default'
    if ((default is util_const.NoParam) and (on_keyerr == 'default')):
        on_keyerr = 'error'
    if (on_missing == 'error'):
        data_iter = (G.adj[u][v] for (u, v) in edges)
    elif (on_missing == 'default'):
        data_iter = ((G.adj[u][v] if G.has_edge(u, v) else {}) for (u, v) in edges)
    else:
        raise KeyError('on_missing={} must be error, filter or default'.format(on_missing))
    if (on_keyerr == 'error'):
        value_iter = (d[key] for d in data_iter)
    elif (on_keyerr == 'default'):
        value_iter = (d.get(key, default) for d in data_iter)
    else:
        raise KeyError('on_keyerr={} must be error or default'.format(on_keyerr))
    return value_iter

Generates attributes values of specific edges

Args:
on_missing (str): Strategy for handling nodes missing from G.
Can be {'error', 'default'}.  defaults to 'error'.
on_keyerr (str): Strategy for handling keys missing from node dicts.
Can be {'error', 'default'}.  defaults to 'default'
if default is specified, otherwise defaults to 'error'.


def _flush(self, buffer):

        with _handle_oss_error():
            self._bucket.put_object(key=self._key, data=buffer.tobytes())

Flush the write buffers of the stream if applicable.

Args:
buffer (memoryview): Buffer content.


def get_gates(self, x):

    x = tf.stop_gradient(x)

    x = tf.matmul(x, self.t_vectors)

    x = tf.sign(x)

    x = tf.matmul(x, self.t_group, transpose_b=True) / self.nb_hyperplanes

    x = tf.argmax(x, axis=-1)

    x = tf.one_hot(x, self.nb_buckets)

    return x

Return the bucket id of the given tensor.

Args:
x (tf.Tensor): float32 of shape [length, depth]

Returns:
tf.Tensor: One-hot vector int64 of shape [heads, length, nb_buckets]
containing the id of the bucket


def currentSelected(self):
    if self.commaRadioButton.isChecked():
        return ','
    elif self.semicolonRadioButton.isChecked():
        return ';'
    elif self.tabRadioButton.isChecked():
        return '\t'
    elif self.otherRadioButton.isChecked():
        return self.otherSeparatorLineEdit.text()
    return

Returns the currently selected delimiter character.

Returns:
str: One of `,`, `;`, `\t`, `*other*`.


def _player_step_tuple(self, envs_step_tuples):

    ob_real, reward_real, _, _ = envs_step_tuples["real_env"]
    ob_sim, reward_sim, _, _ = envs_step_tuples["sim_env"]
    ob_err = absolute_hinge_difference(ob_sim, ob_real)

    ob_real_aug = self._augment_observation(ob_real, reward_real,
                                            self.cumulative_real_reward)
    ob_sim_aug = self._augment_observation(ob_sim, reward_sim,
                                           self.cumulative_sim_reward)
    ob_err_aug = self._augment_observation(
        ob_err, reward_sim - reward_real,
        self.cumulative_sim_reward - self.cumulative_real_reward
    )
    ob = np.concatenate([ob_sim_aug, ob_real_aug, ob_err_aug], axis=1)
    _, reward, done, info = envs_step_tuples["real_env"]
    return ob, reward, done, info

Construct observation, return usual step tuple.

Args:
envs_step_tuples: tuples.

Returns:
Step tuple: ob, reward, done, info
ob: concatenated images [simulated observation, real observation,
difference], with additional informations in header.
reward: real environment reward
done: True iff. envs_step_tuples['real_env'][2] is True
info: real environment info


def _remove_trailing_new_line(l):
    for n in sorted(new_lines_bytes, key=(lambda x: len(x)), reverse=True):
        if l.endswith(n):
            remove_new_line = slice(None, (- len(n)))
            return l[remove_new_line]
    return l

Remove a single instance of new line at the end of l if it exists.

Returns:
bytestring


def _check_tf1_flags(flags, unparsed):

    def _get_message_unparsed(flag, orig_flag, new_flag):
        if flag.startswith(orig_flag):
            return '\n  Use {0} instead of {1}'.format(new_flag, orig_flag)
        return ''
    if unparsed:
        output = ''
        for flag in unparsed:
            output += _get_message_unparsed(flag, '--input_file', '--graph_def_file')
            output += _get_message_unparsed(flag, '--savedmodel_directory', '--saved_model_dir')
            output += _get_message_unparsed(flag, '--std_value', '--std_dev_values')
            output += _get_message_unparsed(flag, '--batch_size', '--input_shapes')
            output += _get_message_unparsed(flag, '--dump_graphviz', '--dump_graphviz_dir')
        if output:
            raise ValueError(output)
    if flags.graph_def_file and (not flags.input_arrays or not flags.output_arrays):
        raise ValueError('--input_arrays and --output_arrays are required with --graph_def_file')
    if flags.input_shapes:
        if not flags.input_arrays:
            raise ValueError('--input_shapes must be used with --input_arrays')
        if flags.input_shapes.count(':') != flags.input_arrays.count(','):
            raise ValueError('--input_shapes and --input_arrays must have the same number of items')
    if flags.std_dev_values or flags.mean_values:
        if bool(flags.std_dev_values) != bool(flags.mean_values):
            raise ValueError('--std_dev_values and --mean_values must be used together')
        if flags.std_dev_values.count(',') != flags.mean_values.count(','):
            raise ValueError('--std_dev_values, --mean_values must have the same number of items')
    if (flags.default_ranges_min is None) != (flags.default_ranges_max is None):
        raise ValueError('--default_ranges_min and --default_ranges_max must be used together')
    if flags.dump_graphviz_video and (not flags.dump_graphviz_dir):
        raise ValueError('--dump_graphviz_video must be used with --dump_graphviz_dir')
    if flags.custom_opdefs and (not flags.experimental_new_converter):
        raise ValueError('--custom_opdefs must be used with --experimental_new_converter')
    if flags.custom_opdefs and (not flags.allow_custom_ops):
        raise ValueError('--custom_opdefs must be used with --allow_custom_ops')
    if flags.experimental_select_user_tf_ops and (not flags.experimental_new_converter):
        raise ValueError('--experimental_select_user_tf_ops must be used with --experimental_new_converter')

Checks the parsed and unparsed flags to ensure they are valid in 1.X.

Raises an error if previously support unparsed flags are found. Raises an
error for parsed flags that don't meet the required conditions.

Args:
flags: argparse.Namespace object containing TFLite flags.
unparsed: List of unparsed flags.

Raises:
ValueError: Invalid flags.


def reserve(self, *args, **kwargs):

        data = self.get_data('floating_ips/',
                             type=POST,
                             params={'region': self.region_slug})

        if data:
            self.ip = data['floating_ip']['ip']
            self.region = data['floating_ip']['region']

        return self

Creates a FloatingIP in a region without assigning
it to a specific Droplet.

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
region_slug: str - region's slug (e.g. 'nyc3')


def __init__(self, filename, f_start=None, f_stop=None, t_start=None, t_stop=None, load_data=True, max_load=1.):

        super(H5Reader, self).__init__()

        if filename and os.path.isfile(filename) and h5py.is_hdf5(filename):

            self.freq_axis = 2
            self.time_axis = 0
            self.beam_axis = 1
            self.stokes_axis = 4

            self.filename = filename
            self.filestat = os.stat(filename)
            self.filesize = self.filestat.st_size/(1024.0**2)
            self.load_data = load_data
            self.h5 = h5py.File(self.filename)
            self.read_header()
            self.file_size_bytes = os.path.getsize(self.filename)
            self.n_ints_in_file = self.h5["data"].shape[self.time_axis]
            self.n_channels_in_file = self.h5["data"].shape[self.freq_axis]
            self.n_beams_in_file = self.header[b'nifs']
            self.n_pols_in_file = 1
            self._n_bytes = int(self.header[b'nbits'] / 8)
            self._d_type = self._setup_dtype()
            self.file_shape = (self.n_ints_in_file,self.n_beams_in_file,self.n_channels_in_file)

            if self.header[b'foff'] < 0:
                self.f_end = self.header[b'fch1']
                self.f_begin = self.f_end + self.n_channels_in_file*self.header[b'foff']
            else:
                self.f_begin = self.header[b'fch1']
                self.f_end = self.f_begin + self.n_channels_in_file*self.header[b'foff']

            self.t_begin = 0
            self.t_end = self.n_ints_in_file

            self._setup_selection_range(f_start=f_start, f_stop=f_stop, t_start=t_start, t_stop=t_stop, init=True)

            self._setup_chans()

            self._setup_freqs()

            if max_load is not None:
                if max_load > 1.0:
                    logger.warning('Setting data limit > 1GB, please handle with care!')
                self.MAX_DATA_ARRAY_SIZE = max_load * MAX_DATA_ARRAY_SIZE_UNIT
            else:
                self.MAX_DATA_ARRAY_SIZE = MAX_DATA_ARRAY_SIZE_UNIT

            if self.file_size_bytes > self.MAX_DATA_ARRAY_SIZE:
                self.large_file = True
            else:
                self.large_file = False

            if self.load_data:
                if self.large_file:

                    if self.f_start or self.f_stop or self.t_start or self.t_stop:
                        if self.isheavy():
                            logger.warning("Selection size of %.2f GB, exceeding our size limit %.2f GB. Instance created, header loaded, but data not loaded, please try another (t,v) selection." % (self._calc_selection_size() / (1024. ** 3), self.MAX_DATA_ARRAY_SIZE / (1024. ** 3)))
                            self._init_empty_selection()
                        else:
                            self.read_data()
                    else:
                        logger.warning("The file is of size %.2f GB, exceeding our size limit %.2f GB. Instance created, header loaded, but data not loaded. You could try another (t,v) selection."%(self.file_size_bytes/(1024.**3), self.MAX_DATA_ARRAY_SIZE/(1024.**3)))
                        self._init_empty_selection()
                else:
                    self.read_data()
            else:
                logger.info("Skipping loading data ...")
                self._init_empty_selection()
        else:
            raise IOError("Need a file to open, please give me one!")

Constructor.

Args:
filename (str): filename of blimpy file.
f_start (float): start frequency, in MHz
f_stop (float): stop frequency, in MHz
t_start (int): start time bin
t_stop (int): stop time bin


def git_branch_rename(new_name):
    curr_name = git.current_branch(refresh=True).name
    if (curr_name not in git.protected_branches()):
        log.info('Renaming branch from <33>{}<32> to <33>{}'.format(curr_name, new_name))
        shell.run('git branch -m {}'.format(new_name))

Rename the current branch

Args:
new_name (str):
New name for the current branch.


def _format_field_value(self, field_name) -> str:
    field_name = self._normalize_field_name(field_name)
    field = self._get_model_field(field_name)
    return SQLInsertCompiler.prepare_value(self, field, getattr(self.query.objs[0], field.attname))

Formats a field's value for usage in SQL.

Arguments:
field_name:
The name of the field to format
the value of.

Returns:
The field's value formatted for usage
in SQL.


def _merge_hdx_update(self, object_type, id_field_name, file_to_upload=None, **kwargs):

        merge_two_dictionaries(self.data, self.old_data)
        if 'batch_mode' in kwargs:
            self.data['batch_mode'] = kwargs['batch_mode']
        if 'skip_validation' in kwargs:
            self.data['skip_validation'] = kwargs['skip_validation']
        ignore_field = self.configuration['%s' % object_type].get('ignore_on_update')
        self.check_required_fields(ignore_fields=[ignore_field])
        operation = kwargs.get('operation', 'update')
        self._save_to_hdx(operation, id_field_name, file_to_upload)

Helper method to check if HDX object exists and update it

Args:
object_type (str): Description of HDX object type (for messages)
id_field_name (str): Name of field containing HDX object identifier
file_to_upload (Optional[str]): File to upload to HDX
**kwargs: See below
operation (string): Operation to perform eg. patch. Defaults to update.

Returns:
None


def html_serialize(self, attributes, max_length=None):

    doc = ET.Element('span')
    for chunk in self:
      if (chunk.has_cjk() and
          not (max_length and len(chunk.word) > max_length)):
        ele = ET.Element('span')
        ele.text = chunk.word
        for key, val in attributes.items():
          ele.attrib[key] = val
        doc.append(ele)
      else:

        if doc.getchildren():
          if doc.getchildren()[-1].tail is None:
            doc.getchildren()[-1].tail = chunk.word
          else:
            doc.getchildren()[-1].tail += chunk.word
        else:
          if doc.text is None:
            doc.text = chunk.word
          else:
            doc.text += chunk.word
    result = ET.tostring(doc, encoding='utf-8').decode('utf-8')
    result = html5lib.serialize(
        html5lib.parseFragment(result), sanitize=True,
        quote_attr_values='always')
    return result

Returns concatenated HTML code with SPAN tag.

Args:
attributes (dict): A map of name-value pairs for attributes of output
SPAN tags.
max_length (:obj:`int`, optional): Maximum length of span enclosed chunk.

Returns:
The organized HTML code. (str)


async def send_files_preconf(filepaths, config_path=CONFIG_PATH):
    config = read_config(config_path)
    subject = 'PDF files from pdfebc'
    message = ''
    (await send_with_attachments(subject, message, filepaths, config))

Send files using the config.ini settings.

Args:
filepaths (list(str)): A list of filepaths.


def extend_args(function_signature, args, kwargs):
    arg_names = function_signature.arg_names
    arg_defaults = function_signature.arg_defaults
    arg_is_positionals = function_signature.arg_is_positionals
    keyword_names = function_signature.keyword_names
    function_name = function_signature.function_name
    args = list(args)
    for keyword_name in kwargs:
        if (keyword_name not in keyword_names):
            raise Exception("The name '{}' is not a valid keyword argument for the function '{}'.".format(keyword_name, function_name))
    for skipped_name in arg_names[0:len(args)]:
        if (skipped_name in kwargs):
            raise Exception("Positional and keyword value provided for the argument '{}' for the function '{}'".format(keyword_name, function_name))
    zipped_info = zip(arg_names, arg_defaults, arg_is_positionals)
    zipped_info = list(zipped_info)[len(args):]
    for (keyword_name, default_value, is_positional) in zipped_info:
        if (keyword_name in kwargs):
            args.append(kwargs[keyword_name])
        elif (default_value != funcsigs._empty):
            args.append(default_value)
        elif (not is_positional):
            raise Exception("No value was provided for the argument '{}' for the function '{}'.".format(keyword_name, function_name))
    no_positionals = ((len(arg_is_positionals) == 0) or (not arg_is_positionals[(- 1)]))
    too_many_arguments = ((len(args) > len(arg_names)) and no_positionals)
    if too_many_arguments:
        raise Exception("Too many arguments were passed to the function '{}'".format(function_name))
    return args

Extend the arguments that were passed into a function.

This extends the arguments that were passed into a function with the
default arguments provided in the function definition.

Args:
function_signature: The function signature of the function being
called.
args: The non-keyword arguments passed into the function.
kwargs: The keyword arguments passed into the function.

Returns:
An extended list of arguments to pass into the function.

Raises:
Exception: An exception may be raised if the function cannot be called
with these arguments.


def decode(self, token_ids: Union[int, List[int], 'np.ndarray', 'torch.Tensor', 'tf.Tensor'], skip_special_tokens: bool=False, clean_up_tokenization_spaces: Optional[bool]=None, **kwargs) -> str:
    token_ids = to_py_obj(token_ids)
    return self._decode(token_ids=token_ids, skip_special_tokens=skip_special_tokens, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)

Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
tokens and clean up tokenization spaces.

Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

Args:
token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
List of tokenized input ids. Can be obtained using the `__call__` method.
skip_special_tokens (`bool`, *optional*, defaults to `False`):
Whether or not to remove special tokens in the decoding.
clean_up_tokenization_spaces (`bool`, *optional*):
Whether or not to clean up the tokenization spaces. If `None`, will default to
`self.clean_up_tokenization_spaces`.
kwargs (additional keyword arguments, *optional*):
Will be passed to the underlying model specific decode method.

Returns:
`str`: The decoded sentence.


def create(self, *args, **kwargs):
    data = self.get_data('floating_ips/', type=POST, params={'droplet_id': self.droplet_id})
    if data:
        self.ip = data['floating_ip']['ip']
        self.region = data['floating_ip']['region']
    return self

Creates a FloatingIP and assigns it to a Droplet.

Note: Every argument and parameter given to this method will be
assigned to the object.

Args:
droplet_id: int - droplet id


def fit(self, sents, **kwargs):

        tokens = list(itertools.chain.from_iterable(sents))
        counter = Counter(tokens)
        self.vocab = self.build_vocab(counter, **kwargs)

Builds a vocabulary object based on the tokens in the input.

Args:
sents: A list of lists of tokens (representing sentences)

Vocab kwargs include:
max_size
min_freq
specials
unk_init


def _MergeScalarField(self, tokenizer, message, field):

    _ = self.allow_unknown_extension
    value = None

    if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
                      descriptor.FieldDescriptor.TYPE_SINT32,
                      descriptor.FieldDescriptor.TYPE_SFIXED32):
      value = _ConsumeInt32(tokenizer)
    elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
                        descriptor.FieldDescriptor.TYPE_SINT64,
                        descriptor.FieldDescriptor.TYPE_SFIXED64):
      value = _ConsumeInt64(tokenizer)
    elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
                        descriptor.FieldDescriptor.TYPE_FIXED32):
      value = _ConsumeUint32(tokenizer)
    elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
                        descriptor.FieldDescriptor.TYPE_FIXED64):
      value = _ConsumeUint64(tokenizer)
    elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
                        descriptor.FieldDescriptor.TYPE_DOUBLE):
      value = tokenizer.ConsumeFloat()
    elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
      value = tokenizer.ConsumeBool()
    elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
      value = tokenizer.ConsumeString()
    elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
      value = tokenizer.ConsumeByteString()
    elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
      value = tokenizer.ConsumeEnum(field)
    else:
      raise RuntimeError('Unknown field type %d' % field.type)

    if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
      if field.is_extension:
        message.Extensions[field].append(value)
      else:
        getattr(message, field.name).append(value)
    else:
      if field.is_extension:
        if not self._allow_multiple_scalars and message.HasExtension(field):
          raise tokenizer.ParseErrorPreviousToken(
              'Message type "%s" should not have multiple "%s" extensions.' %
              (message.DESCRIPTOR.full_name, field.full_name))
        else:
          message.Extensions[field] = value
      else:
        if not self._allow_multiple_scalars and message.HasField(field.name):
          raise tokenizer.ParseErrorPreviousToken(
              'Message type "%s" should not have multiple "%s" fields.' %
              (message.DESCRIPTOR.full_name, field.name))
        else:
          setattr(message, field.name, value)

Merges a single scalar field into a message.

Args:
tokenizer: A tokenizer to parse the field value.
message: A protocol message to record the data.
field: The descriptor of the field to be merged.

Raises:
ParseError: In case of text parsing problems.
RuntimeError: On runtime errors.


def GetPresetsForOperatingSystem(cls, operating_system, operating_system_product, operating_system_version):
    operating_system = artifacts.OperatingSystemArtifact(family=operating_system, product=operating_system_product, version=operating_system_version)
    return cls._presets.GetPresetsByOperatingSystem(operating_system)

Determines the presets for a specific operating system.

Args:
operating_system (str): operating system for example "Windows". This
should be one of the values in definitions.OPERATING_SYSTEM_FAMILIES.
operating_system_product (str): operating system product for
example "Windows XP" as determined by preprocessing.
operating_system_version (str): operating system version for
example "5.1" as determined by preprocessing.

Returns:
list[PresetDefinition]: preset definitions, where an empty list
represents all parsers and parser plugins (no preset).


def predict(self, X_feat, X_seq):

        X_seq = np.expand_dims(X_seq, axis=1)

        return self._get_other_var(X_feat, X_seq, variable="y_pred")

Predict the response variable :py:attr:`y` for new input data (:py:attr:`X_feat`, :py:attr:`X_seq`).

Args:
X_feat: Feature design matrix. Same format as :py:attr:`X_feat` in :py:meth:`train`
X_seq:  Sequenc design matrix. Same format as  :py:attr:`X_seq` in :py:meth:`train`


def select_if(df, fun):

    def _filter_f(col):
        try:
            return fun(df[col])
        except:
            return False

    cols = list(filter(_filter_f, df.columns))
    return df[cols]

Selects columns where fun(ction) is true
Args:
fun: a function that will be applied to columns


def validate(self, handler):

        test_method = self.plugin_test_validation(handler)
        if not test_method:
            return None

        for name, plugin_class in inspect.getmembers(handler, inspect.isclass):
            if self.plugin_class_validation(plugin_class):
                return {'class':plugin_class, 'test':test_method}

        print 'Failure for plugin: %s' % (handler.__name__)
        print 'Validation Error: Worker class is required to have a dependencies list and an execute method'
        return None

Validate the plugin, each plugin must have the following:
1) The worker class must have an execute method: execute(self, input_data).
2) The worker class must have a dependencies list (even if it's empty).
3) The file must have a top level test() method.

Args:
handler: the loaded plugin.


def FlagCxx14Features(filename, clean_lines, linenum, error):

  line = clean_lines.elided[linenum]

  include = Match(r'\s*

  if include and include.group(1) in ('scoped_allocator', 'shared_mutex'):
    error(filename, linenum, 'build/c++14', 5,
          ('<%s> is an unapproved C++14 header.') % include.group(1))

Flag those C++14 features that we restrict.

Args:
filename: The name of the current file.
clean_lines: A CleansedLines instance containing the file.
linenum: The number of the line to check.
error: The function to call with any errors found.


def apply(self, inputs, *args, **kwargs):
    warnings.warn('`layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.')
    return self.__call__(inputs, *args, **kwargs)

Deprecated, do NOT use!

This is an alias of `self.__call__`.

Args:
inputs: Input tensor(s).
*args: additional positional arguments to be passed to `self.call`.
**kwargs: additional keyword arguments to be passed to `self.call`.

Returns:
Output tensor(s).


def register_filters(self, filters, force=False):

        for filter_name, filter_ref in filters.items():
            self.register_filter(filter_name, filter_ref, force)

Add/register filters.

Args:
filters (dict): Dictionary of Python functions to use as :program:`Jinja2` filters.
force (bool): If set to ``True``, forces the registration of a filter no matter if it already exists or not.


def from_object(cls, obj):
    return cls(obj.get('sessionId', None), obj.get('status', 0), obj.get('value', None))

The factory method to create WebDriverResult from JSON Object.

Args:
obj(dict): The JSON Object returned by server.


def create_server(self, server_name, *args, **kwargs):

        server = ServerConnection(name=server_name, reactor=self)

        if args or kwargs:
            server.set_connect_info(*args, **kwargs)

        for verb, infos in self._event_handlers.items():
            for info in infos:
                server.register_event(info['direction'], verb, info['handler'],
                                      priority=info['priority'])

        self.servers[server_name] = server

        return server

Create an IRC server connection slot.

The server will actually be connected to when
:meth:`girc.client.ServerConnection.connect` is called later.

Args:
server_name (str): Name of the server, to be used for functions and accessing the
server later through the reactor.

Returns:
server (girc.client.ServerConnection): A not-yet-connected server.


def list_marts(self):

    def _row_gen(attributes):
        for attr in attributes.values():
            (yield (attr.name, attr.display_name))
    return pd.DataFrame.from_records(_row_gen(self.marts), columns=['name', 'display_name'])

Lists available marts in a readable DataFrame format.

Returns:
pd.DataFrame: Frame listing available marts.


def print_table(col_tuple, row_tuples):

    col_widths = [max(len(str(row[col])) for row in [col_tuple] + row_tuples)
                  for col in range(len(col_tuple))]
    format_str = ' '.join('{{:<{}}}'.format(col_width)
                          for col_width in col_widths)
    header_border = ' '.join('=' * col_width for col_width in col_widths)
    print(header_border)
    print(format_str.format(*col_tuple))
    print(header_border)
    for row_tuple in row_tuples:
        print(format_str.format(*row_tuple))
    print(header_border)
    print()

Print column headers and rows as a reStructuredText table.

Args:
col_tuple: Tuple of column name strings.
row_tuples: List of tuples containing row data.


def _log_every_n_to_logger(n, logger, level, message, *args):

  logger = logger or logging.getLogger()
  def _gen():
    while True:
      for _ in range(n):
        yield False
      logger.log(level, message, *args)
      yield True
  gen = _gen()
  return lambda: six.next(gen)

Logs the given message every n calls to a logger.

Args:
n: Number of calls before logging.
logger: The logger to which to log.
level: The logging level (e.g. logging.INFO).
message: A message to log
*args: Any format args for the message.
Returns:
A method that logs and returns True every n calls.


def remove_regex(urls, regex):
    if (not regex):
        return urls
    if (not isinstance(urls, (list, set, tuple))):
        urls = [urls]
    try:
        non_matching_urls = [url for url in urls if (not re.search(regex, url))]
    except TypeError:
        return []
    return non_matching_urls

Parse a list for non-matches to a regex.

Args:
urls: iterable of urls
regex: string regex to be parsed for

Returns:
list of strings not matching regex


def is_leap_year(years):
    years = tf.convert_to_tensor(years, tf.int32)

    def divides_by(n):
        return tf.math.equal(years % n, 0)
    return tf.math.logical_and(divides_by(4), tf.math.logical_or(~divides_by(100), divides_by(400)))

Calculates whether years are leap years.

Args:
years: Tensor of int32 type. Elements should be positive.

Returns:
Tensor of bool type.


def _finish_operation_action(self, action):
    success = action.data['success']
    conn_key = action.data['id']
    if (self._get_connection_state(conn_key) != self.InProgress):
        self._logger.error('Invalid finish_operation action on a connection whose state is not InProgress, conn_key=%s', str(conn_key))
        return
    data = self._get_connection(conn_key)
    callback = data['callback']
    conn_id = data['conn_id']
    args = action.data['callback_args']
    data['state'] = self.Idle
    data['microstate'] = None
    callback(conn_id, self.id, success, *args)

Finish an attempted operation.

Args:
action (ConnectionAction): the action object describing the result
of the operation that we are finishing


def markers(self, values):
    if (not isinstance(values, list)):
        raise TypeError('Markers must be a list of objects')
    self.options['markers'] = values

Set the markers.

Args:
values (list): list of marker objects.

Raises:
ValueError: Markers must be a list of objects.


def GetSubkeyByIndex(self, index):

    subkeys = list(self._subkeys.values())

    if index < 0 or index >= len(subkeys):
      raise IndexError('Index out of bounds.')

    return subkeys[index]

Retrieves a subkey by index.

Args:
index (int): index of the subkey.

Returns:
WinRegistryKey: Windows Registry subkey or None if not found.

Raises:
IndexError: if the index is out of bounds.


def CreateSubdivision(self, parent=None, value=None):

    division = {
        'xsi_type': 'ProductPartition',
        'partitionType': 'SUBDIVISION',
        'id': str(self.next_id)
    }

    if parent is not None:
      division['parentCriterionId'] = parent['id']
      division['caseValue'] = value

    adgroup_criterion = {
        'xsi_type': 'BiddableAdGroupCriterion',
        'adGroupId': self.adgroup_id,
        'criterion': division
    }

    self.CreateAddOperation(adgroup_criterion)
    self.next_id -= 1

    return division

Creates a subdivision node.

Args:
parent: The node that should be this node's parent.
value: The value being partitioned on.
Returns:
A new subdivision node.


def _ValidateFractionalMaxPoolResult(self, input_tensor, pooling_ratio, pseudo_random, overlapping):
    with self.cached_session():
        p, r, c = nn_ops.fractional_max_pool_v2(input_tensor, pooling_ratio, pseudo_random, overlapping, seed=self._SEED)
        actual, row_seq, col_seq = self.evaluate([p, r, c])
        expected = self._GetExpectedFractionalMaxPoolResult(input_tensor, row_seq, col_seq, overlapping)
        self.assertShapeEqual(expected, p)
        self.assertAllClose(expected, actual)

Validate FractionalMaxPool's result against expected.

Expected result is computed given input_tensor, and pooling region defined
by row_seq and col_seq.

Args:
input_tensor: A tensor or numpy ndarray.
pooling_ratio: A list or tuple of length 4, first and last element be 1.
pseudo_random: Use pseudo random method to generate pooling sequence.
overlapping: Use overlapping when pooling.

Returns:
None


def dnd_setSnooze(self, *, num_minutes: int, **kwargs) -> SlackResponse:
    self._validate_xoxp_token()
    kwargs.update({'num_minutes': num_minutes})
    return self.api_call('dnd.setSnooze', http_verb='GET', params=kwargs)

Turns on Do Not Disturb mode for the current user, or changes its duration.

Args:
num_minutes (int): The snooze duration. e.g. 60


def MakeSuiteFromCdf(cdf, name=None):
    if (name is None):
        name = cdf.name
    suite = Suite(name=name)
    prev = 0.0
    for (val, prob) in cdf.Items():
        suite.Incr(val, (prob - prev))
        prev = prob
    return suite

Makes a normalized Suite from a Cdf object.

Args:
cdf: Cdf object
name: string name for the new Suite

Returns:
Suite object


def get_max_capvol(self, remove=True, insert=True, volume=None):

        vol = volume if volume else self.struc_oxid.volume
        return self._get_max_cap_ah(remove, insert) * 1000 * 1E24 / (vol * const.N_A)

Give max capacity in mAh/cc for inserting and removing a charged cation into base structure.

Args:
remove: (bool) whether to allow cation removal
insert: (bool) whether to allow cation insertion
volume: (float) volume to use for normalization (default=volume of initial structure)

Returns:
max vol capacity in mAh/cc


def _ParseJournalEntry(self, file_object, file_offset):
    entry_object = self._ParseEntryObject(file_object, file_offset)
    entry_item_map = self._GetDataTypeMap('systemd_journal_entry_item')
    file_offset += 64
    data_end_offset = ((file_offset + entry_object.data_size) - 64)
    fields = {'real_time': entry_object.real_time}
    while (file_offset < data_end_offset):
        try:
            (entry_item, entry_item_data_size) = self._ReadStructureFromFileObject(file_object, file_offset, entry_item_map)
        except (ValueError, errors.ParseError) as exception:
            raise errors.ParseError('Unable to parse entry item at offset: 0x{0:08x} with error: {1!s}'.format(file_offset, exception))
        file_offset += entry_item_data_size
        if (entry_item.object_offset < self._maximum_journal_file_offset):
            raise errors.ParseError('object offset should be after hash tables ({0:d} < {1:d})'.format(entry_item.object_offset, self._maximum_journal_file_offset))
        event_data = self._ParseDataObject(file_object, entry_item.object_offset)
        event_string = event_data.decode('utf-8')
        (key, value) = event_string.split('=', 1)
        fields[key] = value
    return fields

Parses a journal entry.

This method will generate an event per ENTRY object.

Args:
file_object (dfvfs.FileIO): a file-like object.
file_offset (int): offset of the entry object relative to the start
of the file-like object.

Returns:
dict[str, objects]: entry items per key.

Raises:
ParseError: when an object offset is out of bounds.


def create_snapshot(self, volume_id_or_uri, snapshot, timeout=(- 1)):
    uri = self.__build_volume_snapshot_uri(volume_id_or_uri)
    return self._client.create(snapshot, uri=uri, timeout=timeout, default_values=self.DEFAULT_VALUES_SNAPSHOT)

Creates a snapshot for the specified volume.

Args:
volume_id_or_uri:
Can be either the volume ID or the volume URI.
snapshot (dict):
Object to create.
timeout:
Timeout in seconds. Wait for task completion by default. The timeout does not abort the operation in
OneView, just stops waiting for its completion.

Returns:
dict: Storage volume.


def movie_credits(self, **kwargs):

        path = self._get_id_path('movie_credits')

        response = self._GET(path, kwargs)
        self._set_attrs_to_values(response)
        return response

Get the movie credits for a specific person id.

Args:
language: (optional) ISO 639-1 code.
append_to_response: (optional) Comma separated, any person method.

Returns:
A dict respresentation of the JSON returned from the API.


def delete_knowledge_base(project_id, knowledge_base_id):

    import dialogflow_v2beta1 as dialogflow
    client = dialogflow.KnowledgeBasesClient()
    knowledge_base_path = client.knowledge_base_path(
        project_id, knowledge_base_id)

    response = client.delete_knowledge_base(knowledge_base_path)

    print('Knowledge Base deleted.'.format(response))

Deletes a specific Knowledge base.

Args:
project_id: The GCP project linked with the agent.
knowledge_base_id: Id of the Knowledge base.