Any-to-Any
Transformers
ONNX
Safetensors
English
Chinese
multimodal
audio
video
speech
streaming
full-duplex
long-video
custom-code
Instructions to use inclusionAI/Realtime-Venus with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use inclusionAI/Realtime-Venus with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("inclusionAI/Realtime-Venus", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download examples/heartbeat_access.py from inclusionAI/Realtime-Venus: direct link, hf CLI and curl.
- Browser
- Download file 4.74 kB
-
https://huggingface.co/inclusionAI/Realtime-Venus/resolve/main/examples/heartbeat_access.py
- Command line
-
hf download hf://inclusionAI/Realtime-Venus/examples/heartbeat_access.py
-
curl -L -o heartbeat_access.py https://huggingface.co/inclusionAI/Realtime-Venus/resolve/main/examples/heartbeat_access.py
4.74 kB
| """Lossless HEARTBEAT records in Zarr v2, addressed by a ReferenceFS index. | |
| Media stays in its original encoding. Row JSON uses schema-directed binary | |
| references, so arbitrary user strings/objects cannot collide with the marker. | |
| """ | |
| import base64 | |
| import hashlib | |
| import json | |
| import fsspec | |
| import numpy as np | |
| import pyarrow as pa | |
| import zarr | |
| def open_root(config_location, *, chunk_cache_bytes=128*1024*1024): | |
| fs = fsspec.filesystem('reference', fo=config_location, skip_instance_cache=True) | |
| if '.zgroup' not in fs.references: | |
| raise ValueError('Missing Zarr root metadata') | |
| for key in fs.references: | |
| if key.endswith('/.zarray'): | |
| md = json.loads(fs.cat_file(key)) | |
| if len(md['shape']) != 1 or len(md['chunks']) != 1: | |
| raise ValueError('Expected one-dimensional dense arrays') | |
| prefix = key[:-len('.zarray')] | |
| for i in range((md['shape'][0] + md['chunks'][0] - 1)//md['chunks'][0]): | |
| if prefix + str(i) not in fs.references: | |
| raise ValueError('Missing dense chunk reference: ' + prefix + str(i)) | |
| store=fs.get_mapper('') | |
| if chunk_cache_bytes: | |
| # Adjacent samples frequently share a media block. Reuse compressed | |
| # blocks instead of retransferring the same 16 MiB for each small clip. | |
| store=zarr.LRUStoreCache(store,max_size=chunk_cache_bytes) | |
| return zarr.open_group(store, mode='r', zarr_version=2), fs | |
| def restore(value, dtype, group): | |
| if value is None: | |
| return None | |
| if pa.types.is_binary(dtype) or pa.types.is_large_binary(dtype): | |
| i = value['venus_blob_index'] | |
| start, end = map(int, group['media_offsets'][i:i+2]) | |
| return group['media_data'][start:end].tobytes() | |
| if pa.types.is_struct(dtype): | |
| return {f.name: restore(value[f.name], f.type, group) for f in dtype} | |
| if pa.types.is_list(dtype) or pa.types.is_large_list(dtype): | |
| return [restore(v, dtype.value_type, group) for v in value] | |
| return value | |
| def rows_from_shard(group, start=0, stop=None): | |
| schema = pa.ipc.read_schema(pa.BufferReader(base64.b64decode(group.attrs['arrow_schema_base64']))) | |
| n = group.attrs['num_rows'] | |
| for i in range(start, min(n, n if stop is None else stop)): | |
| a, b = map(int, group['records_offsets'][i:i+2]) | |
| value = json.loads(group['records_data'][a:b].tobytes()) | |
| yield {f.name: restore(value[f.name], f.type, group) for f in schema} | |
| def generate_rows(config_location, track, task, split, subset=None, max_rows=None, _index_identity=None): | |
| if max_rows is not None and max_rows <= 0: | |
| return | |
| root, fs = open_root(config_location) | |
| path = f'data/HEARTBEAT_Challenge/{track}/{task}/{split}' | |
| if subset: | |
| path += '/' + subset | |
| group = root[path] | |
| count = 0 | |
| for key in sorted(group.group_keys()): | |
| for row in rows_from_shard(group[key]): | |
| yield row | |
| count += 1 | |
| if max_rows is not None and count >= max_rows: | |
| return | |
| def as_pandas(config_location, track, task, split, subset=None, max_rows=None): | |
| import pandas as pd | |
| return pd.DataFrame(generate_rows(config_location, track, task, split, subset, max_rows)) | |
| def as_datasets(config_location, track, task, split, subset=None, *, streaming=False, | |
| cache_dir=None, max_rows=None): | |
| from datasets import Dataset, IterableDataset, Features | |
| root, fs = open_root(config_location) | |
| group = root[f'data/HEARTBEAT_Challenge/{track}/{task}/{split}' + ('/'+subset if subset else '')] | |
| first = group[sorted(group.group_keys())[0]] | |
| schema = pa.ipc.read_schema(pa.BufferReader(base64.b64decode(first.attrs['arrow_schema_base64']))) | |
| features = Features.from_arrow_schema(schema) | |
| kwargs = dict(config_location=str(config_location), track=track, task=task, | |
| split=split, subset=subset, max_rows=max_rows) | |
| # In addition to immutable Hub revision paths, distinguish updated local | |
| # reference indexes at the same filename without invalidating normal reuse. | |
| kwargs['_index_identity'] = hashlib.sha256(json.dumps( | |
| fs.references, sort_keys=True, separators=(',', ':')).encode()).hexdigest() | |
| # Fixed commit URLs/local immutable indexes are part of generator identity. | |
| if streaming: | |
| return IterableDataset.from_generator(generate_rows, gen_kwargs=kwargs, features=features) | |
| # Large original video blobs can overflow Arrow's 2 GiB binary-offset limit | |
| # when accumulated into the default 1,000-record writer batch. | |
| return Dataset.from_generator(generate_rows, gen_kwargs=kwargs, cache_dir=cache_dir, | |
| features=features, writer_batch_size=1) | |