File size: 1,073 Bytes
0ca529a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | """Multi-modal field helpers (verl-inspired)."""
from typing import Iterable, Optional
import torch
def extract_multi_modal_inputs(
multi_modal_inputs_list: Optional[Iterable[Optional[dict]]],
) -> dict:
"""Concat per-sample mm dicts into a batched dict for HF VLM forward.
Args:
multi_modal_inputs_list: an iterable of per-sample multi_modal_inputs
dicts (each dict maps field name -> tensor with leading dim
corresponding to that sample's patches/tokens). ``None`` entries
(e.g. text-only samples) are skipped.
Returns:
A dict mapping field name -> tensor concatenated along ``dim=0``
across all non-empty samples. Empty input returns ``{}``.
"""
if not multi_modal_inputs_list:
return {}
collected: dict = {}
for d in multi_modal_inputs_list:
if not d:
continue
for k, v in d.items():
if v is not None:
collected.setdefault(k, []).append(v)
return {k: torch.cat(vs, dim=0) for k, vs in collected.items()}
|