Text Classification
Transformers
Safetensors
English
qwen3_5
image-text-to-text
decision-model
classification
routing
custom-code
Instructions to use EpochEcho/decision-4b-preview with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use EpochEcho/decision-4b-preview with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="EpochEcho/decision-4b-preview")# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("EpochEcho/decision-4b-preview") model = AutoModelForMultimodalLM.from_pretrained("EpochEcho/decision-4b-preview", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download model.py from EpochEcho/decision-4b-preview: direct link, hf CLI and curl.
- Browser
- Download file 7.37 kB
-
https://huggingface.co/EpochEcho/decision-4b-preview/resolve/main/model.py
- Command line
-
hf download hf://EpochEcho/decision-4b-preview/model.py
-
curl -L -o model.py https://huggingface.co/EpochEcho/decision-4b-preview/resolve/main/model.py
7.37 kB
| """Merged decision-model inference using Transformers and PyTorch.""" | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| from protocol import encode | |
| class DecisionModel: | |
| """Load a self-contained checkpoint and return typed decision probabilities.""" | |
| def __init__(self, folder: str | Path | None = None) -> None: | |
| import torch | |
| from safetensors.torch import load_file | |
| from transformers import AutoModel, AutoTokenizer | |
| from transformers.masking_utils import create_recurrent_attention_mask | |
| self.folder = Path(folder or Path(__file__).parent).resolve() | |
| self.torch = torch | |
| self._verify_files() | |
| self.config = json.loads((self.folder / "decision_config.json").read_text()) | |
| self.tokenizer = AutoTokenizer.from_pretrained( | |
| self.folder, local_files_only=True | |
| ) | |
| self.backbone = AutoModel.from_pretrained( | |
| self.folder, | |
| dtype=torch.bfloat16, | |
| device_map={"": 0}, | |
| attn_implementation="sdpa", | |
| local_files_only=True, | |
| ) | |
| self.model = self.backbone.language_model | |
| self.readout = load_file( | |
| str(self.folder / "readout.safetensors"), device="cpu" | |
| )["weight"].to(device="cuda:0", dtype=torch.float32) | |
| if tuple(self.readout.shape) != (255, 2560): | |
| raise ValueError("Expected a [255, 2560] linear readout") | |
| def attention(module, args, kwargs): | |
| mask = kwargs["attention_mask"] | |
| embeddings = module.embed_tokens(kwargs["input_ids"]) | |
| kwargs["attention_mask"] = { | |
| "full_attention": mask[:, None, None, :].bool(), | |
| "linear_attention": create_recurrent_attention_mask( | |
| config=module.config, | |
| inputs_embeds=embeddings, | |
| attention_mask=mask, | |
| past_key_values=None, | |
| ), | |
| } | |
| return args, kwargs | |
| self._attention = self.model.register_forward_pre_hook( | |
| attention, with_kwargs=True | |
| ) | |
| self.model.eval() | |
| def _verify_files(self) -> None: | |
| manifest = json.loads((self.folder / "manifest.json").read_text()) | |
| for name, expected in manifest["files"].items(): | |
| path = (self.folder / name).resolve() | |
| if not path.is_relative_to(self.folder): | |
| raise ValueError(f"Invalid manifest path: {name}") | |
| with path.open("rb") as stream: | |
| actual = hashlib.file_digest(stream, "sha256").hexdigest() | |
| if actual != expected: | |
| raise ValueError(f"Checksum mismatch: {name}") | |
| def encode(self, state: Any, question_id: str, question: dict) -> dict: | |
| return encode( | |
| self.tokenizer, | |
| state, | |
| question_id, | |
| question, | |
| self.config["answer_codes"], | |
| ) | |
| def logits(self, items: list[dict]) -> list: | |
| """Score encoded questions without applying softmax.""" | |
| torch = self.torch | |
| from torch.nn.attention import SDPBackend, sdpa_kernel | |
| longest = max(len(item["input_ids"]) for item in items) | |
| ids = torch.full( | |
| (len(items), longest), | |
| self.tokenizer.pad_token_id, | |
| dtype=torch.long, | |
| device="cuda:0", | |
| ) | |
| mask = torch.zeros_like(ids) | |
| for index, item in enumerate(items): | |
| length = len(item["input_ids"]) | |
| ids[index, :length] = torch.tensor( | |
| item["input_ids"], device="cuda:0" | |
| ) | |
| mask[index, :length] = 1 | |
| backends = [ | |
| SDPBackend.FLASH_ATTENTION, | |
| SDPBackend.EFFICIENT_ATTENTION, | |
| SDPBackend.MATH, | |
| ] | |
| with torch.no_grad(), sdpa_kernel(backends), torch.autocast( | |
| "cuda", dtype=torch.bfloat16 | |
| ): | |
| hidden = self.model( | |
| input_ids=ids, | |
| attention_mask=mask, | |
| use_cache=False, | |
| return_dict=True, | |
| ).last_hidden_state | |
| return [ | |
| torch.nn.functional.linear( | |
| hidden[index, item["position"]].float(), self.readout | |
| )[: len(item["option_ids"])].float() | |
| for index, item in enumerate(items) | |
| ] | |
| def decide(self, request: dict) -> dict: | |
| """Evaluate one request; each question sees only its own schema.""" | |
| return self.decide_many([request])[0] | |
| def decide_many(self, requests: list[dict]) -> list[dict]: | |
| """Batch independent requests without changing their question schemas.""" | |
| answers = [{} for _ in requests] | |
| tokens = [0 for _ in requests] | |
| sequences = [0 for _ in requests] | |
| children = [] | |
| for owner, request in enumerate(requests): | |
| questions = request.get("questions") | |
| if "state" not in request or not isinstance(questions, dict) or not questions: | |
| raise ValueError("state and nonempty questions are required") | |
| if request.get("images") or request.get("videos"): | |
| raise ValueError("This release accepts text only") | |
| for key, question in questions.items(): | |
| item = self.encode(request["state"], key, question) | |
| children.append((owner, key, question, item)) | |
| tokens[owner] += len(item["input_ids"]) | |
| sequences[owner] += 1 | |
| offset = 0 | |
| while offset < len(children): | |
| batch = [] | |
| longest = 0 | |
| while offset < len(children): | |
| entry = children[offset] | |
| length = len(entry[3]["input_ids"]) | |
| if batch and ( | |
| len(batch) >= 4 | |
| or max(longest, length) * (len(batch) + 1) > 16384 | |
| ): | |
| break | |
| batch.append(entry) | |
| longest = max(longest, length) | |
| offset += 1 | |
| logits = self.logits([entry[3] for entry in batch]) | |
| for (owner, key, question, item), scores in zip(batch, logits): | |
| if not bool(self.torch.isfinite(scores).all()): | |
| raise ValueError("Nonfinite decision logits") | |
| probabilities = dict( | |
| zip(item["option_ids"], scores.softmax(-1).cpu().tolist()) | |
| ) | |
| if question["type"] == "noul": | |
| answer = {"type": "noul", "noul": probabilities["true"]} | |
| else: | |
| ordered = {key: probabilities[key] for key in question["criteria"]} | |
| answer = { | |
| "type": "choice", | |
| "choice": max(ordered, key=ordered.get), | |
| "probabilities": ordered, | |
| } | |
| answers[owner][key] = answer | |
| return [ | |
| { | |
| "model": "decision-4b-preview", | |
| "answers": answer, | |
| "usage": { | |
| "input_tokens": tokens[index], | |
| "output_tokens": 0, | |
| "sequences": sequences[index], | |
| }, | |
| } | |
| for index, answer in enumerate(answers) | |
| ] | |