Download data.py from 0xdeadbe3f/Bazza: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/0xdeadbe3f/Bazza/resolve/main/data.py
- Command line
-
hf download hf://0xdeadbe3f/Bazza/data.py
-
curl -L -o data.py https://huggingface.co/0xdeadbe3f/Bazza/resolve/main/data.py
10.1 kB
| """Named batches, fixed Gemma serialization and deterministic batching.""" | |
| import hashlib | |
| import json | |
| import math | |
| import random | |
| import unicodedata | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import torch | |
| class DecisionBatch: | |
| rows: list | |
| attention_mask: torch.Tensor | |
| option_positions: torch.Tensor | |
| option_mask: torch.Tensor | |
| targets: torch.Tensor | None = None | |
| hidden: torch.Tensor | None = None | |
| input_ids: torch.Tensor | None = None | |
| split: str = "train" | |
| feature_space: str | None = None | |
| feature_source: str | None = None | |
| def inputs(self): | |
| return dict( | |
| hidden=self.hidden, | |
| attention_mask=self.attention_mask, | |
| option_positions=self.option_positions, | |
| option_mask=self.option_mask, | |
| ) | |
| def to(self, device): | |
| return DecisionBatch( | |
| self.rows, | |
| **{ | |
| key: value.to(device) if isinstance(value, torch.Tensor) else value | |
| for key, value in vars(self).items() | |
| if key != "rows" | |
| }, | |
| ) | |
| def collate_records(rows, hidden=None, *, target_key="target"): | |
| """Pad only the head input. No truncation, option permutation or relabelling.""" | |
| if not rows: | |
| raise ValueError("Cannot collate an empty batch") | |
| length = max(len(row["ids"]) for row in rows) | |
| choices = max(len(row["positions"]) for row in rows) | |
| ids = torch.zeros(len(rows), length, dtype=torch.long) | |
| mask = torch.zeros_like(ids) | |
| positions = torch.zeros(len(rows), choices, dtype=torch.long) | |
| option_mask = torch.zeros_like(positions, dtype=torch.bool) | |
| targets = torch.zeros(len(rows), choices) | |
| memory = None | |
| if hidden is not None: | |
| memory = hidden[0].new_zeros(len(rows), length, hidden[0].shape[-1]) | |
| has_targets = [target_key in row for row in rows] | |
| if any(has_targets) and not all(has_targets): | |
| raise ValueError( | |
| "Mixed missing/present targets; choose an explicit target policy" | |
| ) | |
| for i, row in enumerate(rows): | |
| n, k = len(row["ids"]), len(row["positions"]) | |
| if n < 1 or not 2 <= k <= 26 or any(p < 0 or p >= n for p in row["positions"]): | |
| raise ValueError("Invalid token/option positions: " + row["id"]) | |
| ids[i, :n] = torch.tensor(row["ids"]) | |
| mask[i, :n] = 1 | |
| positions[i, :k] = torch.tensor(row["positions"]) | |
| option_mask[i, :k] = True | |
| if all(has_targets): | |
| value = torch.tensor(row[target_key], dtype=torch.float32) | |
| # Validate the supplied probabilities before FP32 conversion moves a | |
| # rounded sum (e.g. 0.9999) across the acceptance boundary. | |
| if ( | |
| value.shape != (k,) | |
| or not torch.isfinite(value).all() | |
| or (value < 0).any() | |
| or abs(math.fsum(row[target_key]) - 1) > 1e-4 | |
| ): | |
| raise ValueError("Invalid target distribution: " + row["id"]) | |
| targets[i, :k] = value | |
| if memory is not None: | |
| if hidden[i].shape != (n, memory.shape[-1]): | |
| raise ValueError("Cached hidden shape differs: " + row["id"]) | |
| memory[i, :n].copy_(hidden[i]) | |
| return DecisionBatch( | |
| rows, | |
| mask, | |
| positions, | |
| option_mask, | |
| targets if all(has_targets) else None, | |
| memory, | |
| ids, | |
| ) | |
| def load_selection(path): | |
| selection = json.loads(Path(path).read_text()) | |
| splits = selection["splits"] | |
| for name, rows in splits.items(): | |
| if len({row["id"] for row in rows}) != len(rows): | |
| raise ValueError("Duplicate example IDs in " + name) | |
| train_ids = {r["id"] for r in splits.get("train", [])} | |
| train_groups = {r["group_id"] for r in splits.get("train", []) if r.get("group_id")} | |
| for name, rows in splits.items(): | |
| if name == "train": | |
| continue | |
| if train_ids & {r["id"] for r in rows} or train_groups & { | |
| r["group_id"] for r in rows if r.get("group_id") | |
| }: | |
| raise ValueError("Training/held-out membership overlap in " + name) | |
| return selection | |
| def label(value): | |
| return " ".join(unicodedata.normalize("NFKC", str(value)).split()) | |
| def fields(row): | |
| meta = row.get("metadata", {}) | |
| source = meta.get("dataset_source", row.get("dataset_source", row["source"])) | |
| task = meta.get( | |
| "task_type", row.get("task_type", row["source"] + "/" + row["kind"]) | |
| ) | |
| schema = meta.get("balance_schema", row.get("balance_schema")) | |
| if schema is None: | |
| schema = hashlib.sha256( | |
| json.dumps( | |
| [ | |
| row.get("question", ""), | |
| sorted(label(o) for o in row.get("options", [])), | |
| ], | |
| sort_keys=True, | |
| ).encode() | |
| ).hexdigest() | |
| return { | |
| "dataset_source": source, | |
| "task_type": task, | |
| "task_key": source + "/" + task, | |
| "balance_schema": schema, | |
| "domain": row.get("domain", row["source"]), | |
| "options": [str(o) for o in row.get("options", [])], | |
| "loss_weight": float(meta.get("loss_weight", row.get("loss_weight", 1.0))), | |
| "pair_id": meta.get("upstream_paired_index", row.get("pair_id")), | |
| } | |
| def schema_key(row): | |
| data = fields(row) | |
| return data["task_key"] + "/" + data["balance_schema"] | |
| def encode_input(row, tokenizer, max_length, order=None): | |
| options = row["options"] | |
| if order is None: | |
| order = list(range(len(options))) | |
| if not 2 <= len(order) <= 26: | |
| return None | |
| state = row["state"] | |
| if not isinstance(state, str): | |
| state = json.dumps(state, ensure_ascii=False, sort_keys=True) | |
| prompt = "<bos><|turn>user\nChoose the best answer. Reply with its letter only.\n" | |
| prompt += "Context:\n" + state + "\nQuestion:\n" + row["question"] + "\nOptions:\n" | |
| ends = [] | |
| for index, original in enumerate(order): | |
| prompt += chr(65 + index) + ". " + str(options[original]) | |
| ends.append(len(prompt)) | |
| prompt += "\n" | |
| prompt += "<turn|>\n<|turn>model\n<|channel>thought\n<channel|>" | |
| encoded = tokenizer.encode(prompt, add_special_tokens=False) | |
| if len(encoded.ids) > max_length: | |
| return None # No silently truncated states, questions or choices. | |
| positions = [] | |
| for end in ends: | |
| eligible = [i for i, (a, b) in enumerate(encoded.offsets) if 0 < b <= end] | |
| if not eligible: | |
| raise ValueError("Cannot locate option's final token") | |
| positions.append(eligible[-1]) | |
| return { | |
| "id": row["id"], | |
| "group_id": row["group_id"], | |
| "source": row["source"], | |
| "kind": row["kind"], | |
| "ids": encoded.ids, | |
| "positions": positions, | |
| "order": order, | |
| **{**fields(row), "options": [str(options[i]) for i in order]}, | |
| } | |
| def encode(row, tokenizer, max_length, order=None): | |
| result = encode_input(row, tokenizer, max_length, order) | |
| if result is None: | |
| return None | |
| target = [float(row["target"][i]) for i in result["order"]] | |
| if ( | |
| len(target) != len(result["order"]) | |
| or min(target) < 0 | |
| or abs(sum(target) - 1) > 1e-4 | |
| ): | |
| raise ValueError("Invalid target distribution") | |
| result["target"] = target | |
| return result | |
| def length_batches(records, max_rows, token_budget): | |
| """Bucket by length, bounding padded tokens without dropping long rows.""" | |
| batch = [] | |
| for row in sorted(records, key=lambda r: (len(r["ids"]), r["id"])): | |
| if batch and ( | |
| len(batch) >= max_rows or (len(batch) + 1) * len(row["ids"]) > token_budget | |
| ): | |
| yield batch | |
| batch = [] | |
| batch.append(row) | |
| if batch: | |
| yield batch | |
| def training_batches(rows, batch_size, token_budget, seed): | |
| """Shuffle examples, bucket locally by length, then shuffle the batches.""" | |
| rng = random.Random(seed) | |
| rows = list(rows) | |
| rng.shuffle(rows) | |
| batches = [] | |
| for start in range(0, len(rows), batch_size * 16): | |
| batches.extend( | |
| length_batches( | |
| rows[start : start + batch_size * 16], batch_size, token_budget | |
| ) | |
| ) | |
| rng.shuffle(batches) | |
| return batches | |
| class BatchLoader: | |
| """Normal CPU collation/prefetch; it never runs a backbone or touches CUDA in its worker.""" | |
| def __init__(self, collate_fn, device="cuda", prefetch_batches=8): | |
| self.collate_fn, self.device, self.depth = collate_fn, device, prefetch_batches | |
| if prefetch_batches < 1: | |
| raise ValueError("prefetch_batches must be positive") | |
| def __call__(self, rows, split): | |
| return self.collate_fn(rows, split).to(self.device) | |
| def iter_batches(self, batches, start, split): | |
| from collections import deque | |
| from concurrent.futures import ThreadPoolExecutor | |
| from itertools import islice | |
| indices = iter(range(start, len(batches))) | |
| pending = deque() | |
| with ThreadPoolExecutor( | |
| max_workers=1, thread_name_prefix="batch-loader" | |
| ) as pool: | |
| try: | |
| for index in islice(indices, self.depth): | |
| pending.append( | |
| (index, pool.submit(self.collate_fn, batches[index], split)) | |
| ) | |
| while pending: | |
| index, future = pending.popleft() | |
| batch = future.result() | |
| next_index = next(indices, None) | |
| if next_index is not None: | |
| pending.append( | |
| ( | |
| next_index, | |
| pool.submit( | |
| self.collate_fn, batches[next_index], split | |
| ), | |
| ) | |
| ) | |
| yield index, batch.to(self.device) | |
| finally: | |
| for _, future in pending: | |
| future.cancel() | |