Realtime-Venus / examples /heartbeat_access.py
Codex
Publish reviewed Realtime-Venus configurations, HEARTBEAT index and documentation
a1d95d3
Raw History Blame Contribute Delete
4.74 kB
"""Lossless HEARTBEAT records in Zarr v2, addressed by a ReferenceFS index.
Media stays in its original encoding. Row JSON uses schema-directed binary
references, so arbitrary user strings/objects cannot collide with the marker.
"""
import base64
import hashlib
import json
import fsspec
import numpy as np
import pyarrow as pa
import zarr
def open_root(config_location, *, chunk_cache_bytes=128*1024*1024):
fs = fsspec.filesystem('reference', fo=config_location, skip_instance_cache=True)
if '.zgroup' not in fs.references:
raise ValueError('Missing Zarr root metadata')
for key in fs.references:
if key.endswith('/.zarray'):
md = json.loads(fs.cat_file(key))
if len(md['shape']) != 1 or len(md['chunks']) != 1:
raise ValueError('Expected one-dimensional dense arrays')
prefix = key[:-len('.zarray')]
for i in range((md['shape'][0] + md['chunks'][0] - 1)//md['chunks'][0]):
if prefix + str(i) not in fs.references:
raise ValueError('Missing dense chunk reference: ' + prefix + str(i))
store=fs.get_mapper('')
if chunk_cache_bytes:
# Adjacent samples frequently share a media block. Reuse compressed
# blocks instead of retransferring the same 16 MiB for each small clip.
store=zarr.LRUStoreCache(store,max_size=chunk_cache_bytes)
return zarr.open_group(store, mode='r', zarr_version=2), fs
def restore(value, dtype, group):
if value is None:
return None
if pa.types.is_binary(dtype) or pa.types.is_large_binary(dtype):
i = value['venus_blob_index']
start, end = map(int, group['media_offsets'][i:i+2])
return group['media_data'][start:end].tobytes()
if pa.types.is_struct(dtype):
return {f.name: restore(value[f.name], f.type, group) for f in dtype}
if pa.types.is_list(dtype) or pa.types.is_large_list(dtype):
return [restore(v, dtype.value_type, group) for v in value]
return value
def rows_from_shard(group, start=0, stop=None):
schema = pa.ipc.read_schema(pa.BufferReader(base64.b64decode(group.attrs['arrow_schema_base64'])))
n = group.attrs['num_rows']
for i in range(start, min(n, n if stop is None else stop)):
a, b = map(int, group['records_offsets'][i:i+2])
value = json.loads(group['records_data'][a:b].tobytes())
yield {f.name: restore(value[f.name], f.type, group) for f in schema}
def generate_rows(config_location, track, task, split, subset=None, max_rows=None, _index_identity=None):
if max_rows is not None and max_rows <= 0:
return
root, fs = open_root(config_location)
path = f'data/HEARTBEAT_Challenge/{track}/{task}/{split}'
if subset:
path += '/' + subset
group = root[path]
count = 0
for key in sorted(group.group_keys()):
for row in rows_from_shard(group[key]):
yield row
count += 1
if max_rows is not None and count >= max_rows:
return
def as_pandas(config_location, track, task, split, subset=None, max_rows=None):
import pandas as pd
return pd.DataFrame(generate_rows(config_location, track, task, split, subset, max_rows))
def as_datasets(config_location, track, task, split, subset=None, *, streaming=False,
cache_dir=None, max_rows=None):
from datasets import Dataset, IterableDataset, Features
root, fs = open_root(config_location)
group = root[f'data/HEARTBEAT_Challenge/{track}/{task}/{split}' + ('/'+subset if subset else '')]
first = group[sorted(group.group_keys())[0]]
schema = pa.ipc.read_schema(pa.BufferReader(base64.b64decode(first.attrs['arrow_schema_base64'])))
features = Features.from_arrow_schema(schema)
kwargs = dict(config_location=str(config_location), track=track, task=task,
split=split, subset=subset, max_rows=max_rows)
# In addition to immutable Hub revision paths, distinguish updated local
# reference indexes at the same filename without invalidating normal reuse.
kwargs['_index_identity'] = hashlib.sha256(json.dumps(
fs.references, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
# Fixed commit URLs/local immutable indexes are part of generator identity.
if streaming:
return IterableDataset.from_generator(generate_rows, gen_kwargs=kwargs, features=features)
# Large original video blobs can overflow Arrow's 2 GiB binary-offset limit
# when accumulated into the default 1,000-record writer batch.
return Dataset.from_generator(generate_rows, gen_kwargs=kwargs, cache_dir=cache_dir,
features=features, writer_batch_size=1)