ndaly's picture
Add files using upload-large-folder tool
4fc4cfb verified
Raw History Blame Contribute Delete
5.24 kB
# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc.
# SPDX-License-Identifier: Apache-2.0
"""
Reference file schema for the model-readiness check.
A reference file pins, for one or more prompts, the HuggingFace teacher's
top-K next-token predictions at every generated position. The TT model
under test is then run with teacher forcing against this file and scored
on top-1 / top-5 / top-K hit rate.
On-disk format (single torch.save'd dict):
{
"format_version": "readiness_v1",
"k": int,
"hf_model_id": str,
"token_ids_meta": {"bos_id": int|None, "eos_id": int, "pad_id": int|None},
"entries": [
{
"prompt_text": str,
"prompt_tokens": Tensor [1, P] int64,
"generated_tokens": Tensor [1, G] int64,
"topk_tokens": Tensor [G, K] int32,
"tf_prompt_len": int, # == P, stored for clarity
},
...
],
}
`topk_tokens[i]` is the teacher's top-K prediction for the token at
position `tf_prompt_len + i`, conditioned on positions `[0 .. tf_prompt_len + i - 1]`.
Top-1 is column 0; top-5 is columns 0..4; top-K is the full row.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional
import torch
FORMAT_VERSION = "readiness_v1"
@dataclass
class ReferenceEntry:
prompt_text: str
prompt_tokens: torch.Tensor # [1, P] int64
generated_tokens: torch.Tensor # [1, G] int64
topk_tokens: torch.Tensor # [G, K] int32
tf_prompt_len: int
def __post_init__(self) -> None:
if self.prompt_tokens.dim() != 2 or self.prompt_tokens.shape[0] != 1:
raise ValueError(f"prompt_tokens must be [1, P], got {tuple(self.prompt_tokens.shape)}")
if self.generated_tokens.dim() != 2 or self.generated_tokens.shape[0] != 1:
raise ValueError(f"generated_tokens must be [1, G], got {tuple(self.generated_tokens.shape)}")
if self.topk_tokens.dim() != 2:
raise ValueError(f"topk_tokens must be [G, K], got {tuple(self.topk_tokens.shape)}")
if self.topk_tokens.shape[0] != self.generated_tokens.shape[1]:
raise ValueError(
f"topk_tokens rows ({self.topk_tokens.shape[0]}) must equal "
f"generated_tokens length ({self.generated_tokens.shape[1]})"
)
if self.tf_prompt_len != self.prompt_tokens.shape[1]:
raise ValueError(
f"tf_prompt_len ({self.tf_prompt_len}) must equal prompt length " f"({self.prompt_tokens.shape[1]})"
)
@property
def num_generated(self) -> int:
return int(self.generated_tokens.shape[1])
@property
def k(self) -> int:
return int(self.topk_tokens.shape[1])
@dataclass
class Reference:
k: int
hf_model_id: str
entries: List[ReferenceEntry]
token_ids_meta: Dict[str, Optional[int]] = field(default_factory=dict)
def __post_init__(self) -> None:
if not self.entries:
raise ValueError("Reference must contain at least one entry")
for idx, entry in enumerate(self.entries):
if entry.k != self.k:
raise ValueError(f"Entry {idx} has k={entry.k}, expected {self.k} from Reference.k")
def save_reference(reference: Reference, path: Path | str) -> Path:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
payload: Dict[str, Any] = {
"format_version": FORMAT_VERSION,
"k": reference.k,
"hf_model_id": reference.hf_model_id,
"token_ids_meta": reference.token_ids_meta,
"entries": [
{
"prompt_text": e.prompt_text,
"prompt_tokens": e.prompt_tokens.to(torch.int64).cpu().contiguous(),
"generated_tokens": e.generated_tokens.to(torch.int64).cpu().contiguous(),
"topk_tokens": e.topk_tokens.to(torch.int32).cpu().contiguous(),
"tf_prompt_len": int(e.tf_prompt_len),
}
for e in reference.entries
],
}
torch.save(payload, path)
return path
def load_reference(path: Path | str) -> Reference:
path = Path(path)
payload = torch.load(path, weights_only=True)
fmt = payload.get("format_version")
if fmt != FORMAT_VERSION:
raise ValueError(f"Unsupported reference format_version={fmt!r} in {path}; " f"expected {FORMAT_VERSION!r}")
raw_entries = payload.get("entries")
if not isinstance(raw_entries, list) or not raw_entries:
raise ValueError(f"Reference {path} missing non-empty 'entries' list")
entries = [
ReferenceEntry(
prompt_text=str(e["prompt_text"]),
prompt_tokens=e["prompt_tokens"],
generated_tokens=e["generated_tokens"],
topk_tokens=e["topk_tokens"],
tf_prompt_len=int(e["tf_prompt_len"]),
)
for e in raw_entries
]
return Reference(
k=int(payload["k"]),
hf_model_id=str(payload["hf_model_id"]),
entries=entries,
token_ids_meta=dict(payload.get("token_ids_meta") or {}),
)