LLaVA-OneVision-2.5-4B-Instruct / canvas_timestamp_schema.py
yiyexy's picture
Add LLaVA-OneVision-2.5-4B-Instruct (weights, remote code, model cards)
ebb5018 verified
Raw History Blame Contribute Delete
10.1 kB
"""Data-driven point/range timestamps for timed image canvases.
The point formatter intentionally mirrors the existing frame timestamp path.
Each canvas uses a point when all of its patches share one temporal coordinate,
and a range when its patches span multiple temporal coordinates.
This module changes prompt text only. It never changes pixels, grids, patch
positions, vision-token order, or M-RoPE.
"""
from __future__ import annotations
import math
import re
from dataclasses import asdict, dataclass
from typing import Sequence
import numpy as np
SCHEMA = "canvas_patch_span_point_range_v1"
VISION_START = "<|vision_start|>"
VISION_END = "<|vision_end|>"
IMAGE_PAD = "<|image_pad|>"
class CanvasTimestampError(ValueError):
"""One sample cannot be serialized from its explicit canvas timing data."""
@dataclass(frozen=True)
class CanvasTime:
"""Validated source-time information for one visual canvas."""
canvas_index: int
timestamp_kind: str
point_frame: int
start_frame: int
end_frame: int
point_seconds: float
start_seconds: float
end_seconds: float
raw_patch_count: int
merged_token_count: int
grid_t: int
grid_h: int
grid_w: int
raw_temporal_monotonic: bool
def frozen_decimals(value: object) -> int:
"""Apply the existing encoder's ``value or 1`` precision rule."""
decimals = value or 1
if isinstance(decimals, bool) or not isinstance(decimals, (int, np.integer)):
raise CanvasTimestampError(f"timestamp_decimal must be an integer, got {value!r}")
decimals = int(decimals)
if not 0 <= decimals <= 6:
raise CanvasTimestampError(f"timestamp_decimal must be in [0,6], got {decimals}")
return decimals
def frozen_point_timestamp(seconds: object, decimals: object = 1) -> str:
"""Format the existing frame point timestamp."""
precision = frozen_decimals(decimals)
value = float(seconds)
if not math.isfinite(value) or value < 0:
raise ValueError(f"point timestamp must be finite and non-negative, got {seconds!r}")
value = round(value, precision)
return f"<{value:.{precision}f} seconds>"
def frozen_range_timestamp(start: object, end: object, decimals: object = 1) -> str:
"""Format a canvas range with the same rounding and precision rules."""
precision = frozen_decimals(decimals)
start_value = float(start)
end_value = float(end)
if (
not math.isfinite(start_value)
or not math.isfinite(end_value)
or start_value < 0
or end_value < start_value
):
raise ValueError(f"invalid timestamp range: start={start!r} end={end!r}")
start_value = round(start_value, precision)
end_value = round(end_value, precision)
return f"<{start_value:.{precision}f} to {end_value:.{precision}f} seconds>"
def _integer_array(value: object, name: str) -> np.ndarray:
if hasattr(value, "detach"):
value = value.detach()
if hasattr(value, "cpu"):
value = value.cpu()
if hasattr(value, "numpy"):
value = value.numpy()
array = np.asarray(value)
if not np.issubdtype(array.dtype, np.number):
raise CanvasTimestampError(f"{name} must be numeric, got dtype={array.dtype}")
if not np.isfinite(array).all() or not np.equal(array, np.floor(array)).all():
raise CanvasTimestampError(f"{name} must contain finite integers")
return array.astype(np.int64, copy=False)
def split_flat_canvases(
patch_positions: object,
image_grid_thw: object,
) -> tuple[list[np.ndarray], np.ndarray]:
"""Split flat raw patch positions using authoritative per-canvas grids."""
positions = _integer_array(patch_positions, "patch_positions")
grid = _integer_array(image_grid_thw, "image_grid_thw")
if positions.ndim != 2 or positions.shape[1] != 3:
raise CanvasTimestampError(f"patch_positions must be [N,3], got {positions.shape}")
if grid.ndim != 2 or grid.shape[1] != 3 or len(grid) == 0:
raise CanvasTimestampError(f"image_grid_thw must be non-empty [canvas,3], got {grid.shape}")
if (positions < 0).any():
raise CanvasTimestampError("timed patch positions cannot contain negative coordinates")
if (grid <= 0).any() or (grid[:, 0] != 1).any():
raise CanvasTimestampError(
"canvas timestamp serializer requires positive per-canvas rows with grid_t=1"
)
counts = [int(np.prod(row)) for row in grid]
if sum(counts) != len(positions):
raise CanvasTimestampError(
f"patch/grid closure mismatch: positions={len(positions)} grid_patches={sum(counts)}"
)
cuts = np.cumsum(counts)[:-1]
return list(np.split(positions, cuts)), grid
def summarize_canvases(
patch_positions: object,
image_grid_thw: object,
*,
fps: object,
spatial_merge_size: int,
) -> list[CanvasTime]:
"""Build point/range inputs from each canvas's observed temporal span."""
try:
fps_value = float(fps)
except (TypeError, ValueError) as exc:
raise CanvasTimestampError(
f"timed canvas sample requires finite positive fps, got {fps!r}"
) from exc
if not math.isfinite(fps_value) or fps_value <= 0:
raise CanvasTimestampError(
f"timed canvas sample requires finite positive fps, got {fps!r}"
)
merge = int(spatial_merge_size)
if merge <= 0:
raise ValueError(f"spatial_merge_size must be positive, got {merge}")
canvases, grid = split_flat_canvases(patch_positions, image_grid_thw)
merge_factor = merge * merge
result: list[CanvasTime] = []
for canvas_index, (canvas, row) in enumerate(zip(canvases, grid)):
if len(canvas) % merge_factor:
raise CanvasTimestampError(
f"canvas {canvas_index} patch count {len(canvas)} is not divisible by "
f"merge^2={merge_factor}"
)
temporal = canvas[:, 0]
point_frame = int(temporal[0])
start_frame = int(temporal.min())
end_frame = int(temporal.max())
result.append(
CanvasTime(
canvas_index=canvas_index,
timestamp_kind="point" if start_frame == end_frame else "range",
point_frame=point_frame,
start_frame=start_frame,
end_frame=end_frame,
point_seconds=point_frame / fps_value,
start_seconds=start_frame / fps_value,
end_seconds=end_frame / fps_value,
raw_patch_count=len(canvas),
merged_token_count=len(canvas) // merge_factor,
grid_t=int(row[0]),
grid_h=int(row[1]),
grid_w=int(row[2]),
raw_temporal_monotonic=bool(np.all(temporal[1:] >= temporal[:-1])),
)
)
return result
def timestamp_texts(summaries: Sequence[CanvasTime], decimals: object = 1) -> list[str]:
"""Render a point or range from each canvas's timestamp kind."""
texts: list[str] = []
for item in summaries:
if item.timestamp_kind == "point":
texts.append(frozen_point_timestamp(item.point_seconds, decimals))
elif item.timestamp_kind == "range":
texts.append(frozen_range_timestamp(item.start_seconds, item.end_seconds, decimals))
else:
raise ValueError(f"unknown canvas timestamp kind {item.timestamp_kind!r}")
return texts
def rewrite_existing_canvas_wrappers(
text: str,
summaries: Sequence[CanvasTime],
*,
decimals: object = 1,
) -> tuple[str, dict]:
"""Prefix each existing canvas wrapper without rebuilding its body."""
pattern = re.compile(
re.escape(VISION_START)
+ r"(?:(?!"
+ re.escape(VISION_START)
+ r").)*?"
+ re.escape(VISION_END),
re.DOTALL,
)
wrappers = list(pattern.finditer(text))
if len(wrappers) != len(summaries):
raise CanvasTimestampError(
f"vision-wrapper/canvas mismatch: wrappers={len(wrappers)} canvases={len(summaries)}"
)
for item, wrapper in zip(summaries, wrappers):
pad_count = wrapper.group(0).count(IMAGE_PAD)
if pad_count != item.merged_token_count:
raise CanvasTimestampError(
f"canvas {item.canvas_index} wrapper/grid token mismatch: "
f"wrapper={pad_count} grid={item.merged_token_count}"
)
texts = timestamp_texts(summaries, decimals)
cursor = 0
parts: list[str] = []
for timestamp, match in zip(texts, wrappers):
parts.extend((text[cursor : match.start()], timestamp, match.group(0)))
cursor = match.end()
parts.append(text[cursor:])
return "".join(parts), {
"schema": SCHEMA,
"canvas_count": len(summaries),
"point_count": sum(item.timestamp_kind == "point" for item in summaries),
"range_count": sum(item.timestamp_kind == "range" for item in summaries),
"merged_token_count": sum(item.merged_token_count for item in summaries),
"timestamp_texts": texts,
"summaries": [asdict(item) for item in summaries],
}
def rewrite_for_sample(
text: str,
*,
patch_positions: object | None = None,
image_grid_thw: object | None = None,
fps: object | None = None,
spatial_merge_size: int = 2,
decimals: object = 1,
) -> tuple[str, dict]:
"""Dispatch while keeping untimed samples byte-identical."""
timing_values = (patch_positions, image_grid_thw, fps)
if all(value is None for value in timing_values):
return text, {"schema": "frozen_legacy_bypass", "changed": False}
if any(value is None for value in timing_values):
raise CanvasTimestampError(
"timed canvas sample requires patch positions, image grids, and fps together"
)
summaries = summarize_canvases(
patch_positions,
image_grid_thw,
fps=fps,
spatial_merge_size=spatial_merge_size,
)
return rewrite_existing_canvas_wrappers(text, summaries, decimals=decimals)