Download upstream/src/vision_decision/backend.py from loveaa/OneAlign: direct link, hf CLI and curl.
- Browser
- Download file 20.1 kB
-
https://huggingface.co/spaces/loveaa/OneAlign/resolve/main/upstream/src/vision_decision/backend.py
- Command line
-
hf download hf://spaces/loveaa/OneAlign/upstream/src/vision_decision/backend.py
-
curl -L -o backend.py https://huggingface.co/spaces/loveaa/OneAlign/resolve/main/upstream/src/vision_decision/backend.py
20.1 kB
| """MLX direct-choice prototype; one unpadded request per forward pass.""" | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from time import perf_counter | |
| from .scoring import (MAX_READOUT_CODES, DEFAULT_PROMPT_LAYOUT, check_prompt_layout, check_readout_codes, compile_question, | |
| readout_codes, verified_label_ids, result_from_logits, cyclic_offsets, rotate, combine_rotations) | |
| class MLXDirect: | |
| def __init__(self, bundle="artifacts/model.json", adapter=None, max_input_tokens=4096, readout_codes=None, prompt_layout=None): | |
| """readout_codes: 255 (shipped) or 256 (extended); None = the adapter readout's own row count. | |
| A 255-row trained readout loaded with readout_codes=256 gets row 256 appended from its LM-head row. | |
| prompt_layout: None = the layout recorded in the adapter's decision_readout.json (absent = standard).""" | |
| import mlx.core as mx | |
| self.max_input_tokens = int(max_input_tokens) | |
| from mlx_vlm import load | |
| self.mx = mx | |
| self.bundle = json.loads(Path(bundle).read_text()) | |
| if not Path(self.bundle["path"]).is_dir(): | |
| raise ValueError("Local model snapshot is missing; run scripts/download_model.py") | |
| start = perf_counter() | |
| self.model, self.processor = load(self.bundle["path"], trust_remote_code=False) | |
| if adapter is not None: | |
| from mlx_vlm.trainer.utils import apply_lora_layers | |
| self.model = apply_lora_layers(self.model, adapter) | |
| self.adapter = None if adapter is None else str(adapter) | |
| self._codebook = None | |
| self.readout = None | |
| self._readout_manifest = {} | |
| rows = None | |
| if adapter is not None: | |
| head = Path(adapter) / "decision_readout.safetensors" | |
| if head.exists(): | |
| weights = mx.load(str(head)) | |
| if (set(weights) != {"weight"} or weights["weight"].ndim != 2 or weights["weight"].shape[0] not in (255, 256) | |
| or not bool(mx.all(mx.isfinite(weights["weight"])).item())): | |
| raise ValueError("Invalid decision_readout.safetensors") | |
| self.readout = weights["weight"] | |
| rows = int(self.readout.shape[0]) | |
| manifest_path = Path(adapter) / "decision_readout.json" | |
| if not manifest_path.exists(): | |
| raise ValueError("Trained readout is missing decision_readout.json tokenizer binding") | |
| self._readout_manifest = json.loads(manifest_path.read_text()) | |
| self.codes = check_readout_codes(readout_codes if readout_codes is not None else rows or MAX_READOUT_CODES) | |
| if rows is not None and rows > self.codes: | |
| raise ValueError(f"This adapter has a {rows}-code readout; load it with readout_codes={rows}") | |
| trained_layout = check_prompt_layout(self._readout_manifest.get("prompt_layout", DEFAULT_PROMPT_LAYOUT)) | |
| self.prompt_layout = trained_layout if prompt_layout is None else check_prompt_layout(prompt_layout) | |
| self.trained_prompt_layout = trained_layout | |
| self._trained_rows = rows | |
| if self.readout is not None and rows < self.codes: | |
| self._extend_readout(rows) | |
| self.model.eval() | |
| mx.eval(self.model.parameters()) | |
| self.load_seconds = perf_counter() - start | |
| def max_options(self): | |
| return self.codes - 1 | |
| def _head_rows(self, token_ids): | |
| language = self.model.language_model | |
| head = language.model.embed_tokens if language.args.tie_word_embeddings else language.lm_head | |
| if "scales" in head or "bias" in head: | |
| raise ValueError("Float32 candidate scoring requires an unquantized bias-free output head") | |
| return head.weight[self.mx.array(token_ids)].astype(self.mx.float32) | |
| def _extend_readout(self, rows): | |
| """Check the trained rows against this tokenizer's codebook; append LM-head rows up to self.codes.""" | |
| from mlx_vlm.prompt_utils import apply_chat_template | |
| rendered = apply_chat_template(self.processor, self.model.config, "", num_images=0, enable_thinking=False) | |
| self._codebook = readout_codes(self.processor.tokenizer, rendered, self.codes, limit=self.codes) | |
| self._check_binding() | |
| if rows < self.codes: # appended rows start from their LM-head rows, exactly as the trained rows did | |
| extra = self._head_rows([token for _, token in self._codebook[rows:]]) | |
| self.readout = self.mx.concatenate([self.readout.astype(self.mx.float32), extra], axis=0) | |
| self.mx.eval(self.readout) | |
| def _check_binding(self): | |
| bound = self._readout_manifest.get("codes") | |
| actual = [{"code": c, "token_id": i} for c, i in self._codebook] | |
| if (self._readout_manifest.get("version") != 1 or not isinstance(bound, list) | |
| or len(bound) != self._trained_rows or bound != actual[:len(bound)] or len(actual) != self.codes): | |
| raise ValueError("Decision readout code/token binding does not match this tokenizer") | |
| def score(self, image, field, state): | |
| header, choices, texts = compile_question(field, state, self.prompt_layout) | |
| labels = self._labels(header, len(choices), 0 if image is None else len(image) if isinstance(image, list) else 1) | |
| prompt = header + "\n".join(f"{label}: {text}" for label, text in zip(labels, texts)) | |
| return self.score_compiled(image, prompt, labels, choices) | |
| def _labels(self, prompt, count, n_images): | |
| if self._codebook is None: | |
| from mlx_vlm.prompt_utils import apply_chat_template | |
| rendered = apply_chat_template(self.processor, self.model.config, prompt, num_images=n_images, enable_thinking=False) | |
| self._codebook = readout_codes(self.processor.tokenizer, rendered, self.codes, limit=self.codes) | |
| if self.readout is not None: | |
| self._check_binding() | |
| if count > len(self._codebook): | |
| raise ValueError(f"{count} candidates exceed the {len(self._codebook)}-code readout " | |
| f"(at most {len(self._codebook) - 1} options + unknown)") | |
| return [code for code, _ in self._codebook[:count]] | |
| def score_request(self, image, fields, state, rotations=4): | |
| """All fields of one request: one shared prefill, rotated candidate orders per field. | |
| `image` is one image, a list of one or two, or None/[] for a text-only request. | |
| """ | |
| return self.score_questions(image, [compile_question(field, state, self.prompt_layout) for field in fields], rotations) | |
| def score_compiled(self, image, prompt, labels, choices): | |
| """Research hook for controlled labels and one- or two-image diagnostics.""" | |
| if len(labels) != len(choices) or not labels: | |
| raise ValueError("Labels and choices must have matching nonzero length") | |
| logits, metadata, _ = self._label_logits(image, prompt, labels) | |
| metadata["tie_break_policy"] = "lowest_vocabulary_token_id" | |
| return result_from_logits(choices, logits, token_ids=metadata["choice_token_ids"]), metadata | |
| def score_rotated(self, image, header, choices, texts, rotations=4, shared_prefix=True): | |
| """Average candidate log-probabilities over cyclic presentation orders of one question.""" | |
| results, metadata = self.score_questions(image, [(header, choices, texts)], rotations, shared_prefix) | |
| return results[0], metadata["questions"][0] | {k: v for k, v in metadata.items() if k != "questions"} | |
| def score_questions(self, image, questions, rotations=4, shared_prefix=True): | |
| """Score every (header, choices, texts) question for one image, or for no image at all. | |
| Each question is shown in up to `rotations` cyclic candidate orders and its | |
| per-candidate log-probabilities are averaged. With shared_prefix the image (when there | |
| is one) and the leading text common to all prompts are evaluated once for the whole | |
| request; a text-only request shares the state and question header the same way. | |
| """ | |
| prompts, owners = [], [] | |
| for index, (header, choices, texts) in enumerate(questions): | |
| if len(texts) != len(choices) or len(choices) < 2: | |
| raise ValueError("Texts and choices must have matching length of at least two") | |
| n_images = 0 if image is None else len(image) if isinstance(image, list) else 1 | |
| labels = self._labels(header, len(choices), n_images) | |
| for offset in cyclic_offsets(len(choices), rotations): | |
| lines = "\n".join(f"{label}: {text}" for label, text in zip(labels, rotate(texts, offset))) | |
| prompts.append((header + lines, labels)) | |
| owners.append((index, offset)) | |
| if shared_prefix: | |
| scored, shared = self._shared_prefix_logits(image, prompts) | |
| else: | |
| scored, shared, features = [], {"shared_prefix_tokens": 0}, None | |
| for prompt, labels in prompts: | |
| logits, metadata, features = self._label_logits(image, prompt, labels, features) | |
| scored.append((logits, metadata)) | |
| results, details = [], [] | |
| for index, (_, choices, _) in enumerate(questions): | |
| passes = [(offset, *scored[k]) for k, (owner, offset) in enumerate(owners) if owner == index] | |
| result = combine_rotations(choices, [(offset, logits) for offset, logits, _ in passes]) | |
| values = list(result.raw_logits.values()) | |
| final = max(range(len(choices)), key=values.__getitem__) | |
| winners = [(max(range(len(logits)), key=logits.__getitem__) + offset) % len(choices) for offset, logits, _ in passes] | |
| results.append(result) | |
| details.append({"rotations": [{"offset": offset, "logits": logits, **meta} for offset, logits, meta in passes], | |
| "rotation_agreement": sum(w == final for w in winners) / len(winners)}) | |
| return results, {"questions": details, "language_model_passes": len(prompts), | |
| "logit_precision": "float32_candidate_head", **shared} | |
| def _candidate_logits(self, hidden, token_ids, readout_indices=None): | |
| # The BF16 head quantizes logits near 20 to 1/16 steps and creates exact ties; | |
| # project only the candidate rows in float32 instead. | |
| if readout_indices is not None and hasattr(self.model, "decision_readout"): | |
| return self.model.decision_readout(hidden.astype(self.mx.float32))[self.mx.array(readout_indices)] | |
| if readout_indices is not None and self.readout is not None: | |
| return hidden.astype(self.mx.float32) @ self.readout[self.mx.array(readout_indices)].astype(self.mx.float32).T | |
| language = self.model.language_model | |
| head = language.model.embed_tokens if language.args.tie_word_embeddings else language.lm_head | |
| if "scales" in head or "bias" in head: | |
| raise ValueError("Float32 candidate scoring requires an unquantized bias-free output head") | |
| mx = self.mx | |
| return hidden.astype(mx.float32) @ head.weight[mx.array(token_ids)].astype(mx.float32).T | |
| def _prepare(self, image, prompt, labels): | |
| from mlx_vlm.prompt_utils import apply_chat_template | |
| from mlx_vlm.utils import prepare_inputs | |
| mx = self.mx | |
| start = perf_counter() | |
| images = [] if image is None else (image if isinstance(image, list) else [image]) | |
| if len(images) > 2: | |
| raise ValueError("Research scoring supports at most two images") | |
| rendered = apply_chat_template(self.processor, self.model.config, prompt, num_images=len(images), enable_thinking=False) | |
| # Qwen non-thinking template must close its empty reasoning block before the answer. | |
| if not rendered.endswith("<think>\n\n</think>\n\n"): | |
| raise ValueError("Unexpected Qwen non-thinking template boundary; refusing to score an unverified position") | |
| tokenizer = self.processor.tokenizer | |
| token_ids = verified_label_ids(tokenizer, rendered, labels) | |
| codebook = self._codebook or readout_codes(tokenizer, rendered, self.codes, limit=self.codes) | |
| self._codebook = codebook | |
| if self.readout is not None: | |
| self._check_binding() | |
| lookup = {code: i for i, (code, _) in enumerate(codebook)} | |
| readout_indices = [lookup[x] for x in labels] if all(x in lookup for x in labels) else None | |
| # A text-only request (no image) goes through the same decision position; prepare_inputs | |
| # then returns ids and mask only, so there is no pixel tensor and no image grid to check. | |
| inputs = prepare_inputs(self.processor, images=images or None, prompts=rendered, | |
| image_token_index=self.model.config.image_token_index) | |
| ids = inputs.pop("input_ids") | |
| pixels = inputs.pop("pixel_values", None) | |
| mask = inputs.pop("attention_mask", None) | |
| if ids.shape[0] != 1 or (mask is not None and not bool(mx.all(mask == 1).item())): | |
| raise ValueError("This adapter supports only one unpadded example") | |
| # Verify the actual processed sequence's answer suffix survived image-token expansion. | |
| suffix = tokenizer.encode("</think>\n\n", add_special_tokens=False) | |
| if ids[0, -len(suffix):].tolist() != suffix: | |
| raise ValueError("Processed decision-position suffix mismatch") | |
| if ids.shape[-1] > self.max_input_tokens: | |
| raise ValueError(f"Processed request exceeds the {self.max_input_tokens}-token limit") | |
| if images: | |
| if pixels is None or "image_grid_thw" not in inputs: | |
| raise ValueError("Processor returned no image grid") | |
| grids = inputs["image_grid_thw"].tolist() | |
| if len(grids) != len(images): | |
| raise ValueError("Image grid count mismatch") | |
| if any(g[0] * g[1] * g[2] // 4 > 2048 for g in grids): | |
| raise ValueError("Image exceeds the 2048-visual-token research limit") | |
| else: | |
| if pixels is not None or inputs: | |
| raise ValueError("A text-only request must not carry vision inputs") | |
| grids = [] | |
| metadata = {"preprocess_seconds": perf_counter() - start, "input_tokens": ids.shape[-1], | |
| "choice_token_ids": token_ids, "template_sha256": hashlib.sha256(rendered.encode()).hexdigest(), | |
| "readout_indices": readout_indices, "readout": f"trained_{self.codes}" if self.readout is not None and readout_indices is not None else "language_model_rows", | |
| "template_suffix": rendered[-64:], "logit_precision": "float32_candidate_head", | |
| "image_grid_thw": grids} | |
| return ids, pixels, mask, inputs, metadata | |
| def _image_features(self, pixels, inputs): | |
| tower = self.model.vision_tower | |
| features, _ = tower(pixels.astype(tower.patch_embed.proj.weight.dtype), inputs["image_grid_thw"]) | |
| self.mx.eval(features) | |
| return features | |
| def _label_logits(self, image, prompt, labels, image_features=None): | |
| """Uncached reference path: one full forward per prompt.""" | |
| mx = self.mx | |
| ids, pixels, mask, inputs, metadata = self._prepare(image, prompt, labels) | |
| start = perf_counter() | |
| if image_features is None and pixels is not None: # no vision tower on a text-only request | |
| image_features = self._image_features(pixels, inputs) | |
| output = self.model(ids, pixels, mask=mask, cached_image_features=image_features, | |
| return_hidden=True, skip_logits=True, **inputs) | |
| selected = self._candidate_logits(output.hidden_states[-1][0, -1], metadata["choice_token_ids"], metadata["readout_indices"]) | |
| mx.eval(selected) | |
| logits = selected.tolist() | |
| metadata.update(forward_seconds=perf_counter() - start, peak_memory_bytes=mx.get_peak_memory(), | |
| logical_forward_evaluations=1) | |
| del output, selected | |
| return logits, metadata, image_features | |
| def _shared_prefix_logits(self, image, prompts): | |
| """Score several (prompt, labels) pairs for one image with a single prefill. | |
| The image (when there is one) and every token the prompts share are evaluated once; | |
| each prompt then continues from a private copy of that cache with only its own | |
| remaining tokens. | |
| """ | |
| mx = self.mx | |
| prepared = [self._prepare(image, prompt, labels) for prompt, labels in prompts] | |
| rows = [p[0][0].tolist() for p in prepared] | |
| shared = min(len(r) for r in rows) - 1 # every prompt keeps at least its decision position | |
| for position in range(shared): | |
| if len({r[position] for r in rows}) != 1: | |
| shared = position | |
| break | |
| config = self.model.config | |
| visual = {config.image_token_index, config.video_token_index} | |
| if shared < 1 or any(token in visual for r in rows for token in r[shared:]): | |
| raise ValueError("Prompts must share the image and a leading text segment") | |
| ids, pixels, mask, inputs, _ = prepared[0] | |
| start = perf_counter() | |
| language = self.model.language_model | |
| prefix = language.make_cache() | |
| features = self._image_features(pixels, inputs) if pixels is not None else None | |
| self.model(ids[:, :shared], pixels, cache=prefix, cached_image_features=features, | |
| skip_logits=True, **inputs) | |
| mx.eval([x for c in prefix for x in c.state if x is not None]) | |
| prefill_seconds = perf_counter() - start | |
| scored = [] | |
| for (ids, _, _, inputs, metadata) in prepared: | |
| start = perf_counter() | |
| branch = language.make_cache() | |
| for source, target in zip(prefix, branch): | |
| state = source.state | |
| target.state = list(state) if isinstance(state, list) else state | |
| positions, _ = language.get_rope_index(ids, inputs.get("image_grid_thw")) | |
| output = language(ids[:, shared:], cache=branch, position_ids=positions, return_hidden=True, skip_logits=True) | |
| selected = self._candidate_logits(output.hidden_states[-1][0, -1], metadata["choice_token_ids"], metadata["readout_indices"]) | |
| mx.eval(selected) | |
| metadata.update(forward_seconds=perf_counter() - start, suffix_tokens=ids.shape[-1] - shared) | |
| scored.append((selected.tolist(), metadata)) | |
| del output, selected, branch | |
| shared_metadata = {"shared_prefix_tokens": shared, "prefill_seconds": prefill_seconds, | |
| "peak_memory_bytes": mx.get_peak_memory()} | |
| return scored, shared_metadata | |
| def generated_reference(self, image, field, state): | |
| from mlx_vlm import generate | |
| from mlx_vlm.prompt_utils import apply_chat_template | |
| images = [] if image is None else (image if isinstance(image, list) else [image]) | |
| header, choices, texts = compile_question(field, state, self.prompt_layout) | |
| labels = self._labels(header, len(choices), len(images)) | |
| prompt = header + "\n".join(f"{label}: {text}" for label, text in zip(labels, texts)) | |
| rendered = apply_chat_template(self.processor, self.model.config, prompt, num_images=len(images), enable_thinking=False) | |
| start = perf_counter() | |
| output = generate(self.model, self.processor, rendered, image=images or None, max_tokens=8, temperature=0, verbose=False) | |
| answer = output.text.strip() | |
| valid = answer in labels | |
| return {"text": answer, "format_valid": valid, | |
| "value": choices[labels.index(answer)][0] if valid else None, | |
| "seconds": perf_counter() - start} | |