Download v2/runtime/decision_model/export.py from mii-llm/nev: direct link, hf CLI and curl.
- Browser
- Download file 20.4 kB
-
https://huggingface.co/mii-llm/nev/resolve/main/v2/runtime/decision_model/export.py
- Command line
-
hf download hf://mii-llm/nev/v2/runtime/decision_model/export.py
-
curl -L -o export.py https://huggingface.co/mii-llm/nev/resolve/main/v2/runtime/decision_model/export.py
20.4 kB
| # Modified implementation for Kev; see LICENSE and NOTICE. | |
| """Export a training run as the complete, standalone artifact (SPEC section 9) and prove it reproduces the run. | |
| export(run_dir, out_dir, calibration=..., conversion=..., device="cuda", check_requests=[...]) -> report | |
| Steps: DecisionModel.from_run (FP32) -> probabilities on check_requests -> merge_lora() in FP32 -> probabilities -> | |
| save_pretrained(out_dir, dtype) -> reload with DecisionModel.from_pretrained in a fresh process with HF_HUB_OFFLINE=1 and | |
| HF_HOME pointed at an empty directory (the parent is unreachable) -> probabilities and answers. Raises if the merged FP32 | |
| probabilities differ from the pre-merge ones by more than 3e-5 (TOL_MERGE, justified beside it), or the reloaded artifact's from the merged FP32 ones by | |
| more than 1e-3 (both in the deployed dtype: the same deployed configuration, SPEC section 10). The merged FP32 vs reloaded | |
| bf16 difference is bf16 arithmetic, not an export defect; it is measured and reported, not gated. All comparisons use | |
| T = 1 (raw) and the serving (cached) execution form. | |
| Also writes calibration.json, conversion.json, README.md, NOTICE and the parent's LICENSE into the artifact, and sets | |
| `temperature` in its config.json to the fitted value. | |
| CLI (the reproduction path when a run is exported on its own; decision_model.convert calls export() directly): | |
| python -m decision_model.export --run RUN --out OUT --calibration calibration.json [--conversion conversion.json] | |
| [--check-data data/prepared/calibration.jsonl] [--n-check 16] [--device cuda] | |
| """ | |
| import argparse | |
| import datetime | |
| import json | |
| import os | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tempfile | |
| from importlib import metadata | |
| from pathlib import Path | |
| import torch | |
| from .render import to_record | |
| from .schema import DecisionRequest | |
| # pre-merge vs merged, both FP32. Justification: on inputs of ~1,000+ tokens each fp32 execution path sits up to 1.2e-5 | |
| # from an fp64 reference (fp64 packed vs rows agree to 2.7e-7), so two fp32 paths differ by up to 2e-5 from rounding alone. | |
| TOL_MERGE = 3e-5 | |
| TOL_MERGE_NOTE = ("fp32 gate 3e-5: on inputs of ~1,000+ tokens each fp32 path sits up to 1.2e-5 from an fp64 reference " | |
| "(fp64 packed vs rows agree to 2.7e-7), so fp32 paths differ by up to 2e-5 from rounding, not logic") | |
| TOL_RELOAD = 1e-3 # merged weights cast to the deployed dtype in memory vs the artifact reloaded in that dtype | |
| REPO = Path(__file__).resolve().parent.parent | |
| LIBRARIES = ("torch", "transformers", "peft", "safetensors", "accelerate", "huggingface-hub", "pydantic", "fastapi") | |
| MEASURED_START, MEASURED_END = "<!-- measured:start -->", "<!-- measured:end -->" | |
| _LONG_STATE = " ".join([ | |
| "I ordered a pair of running shoes on the first of the month and paid with my credit card.", | |
| "The confirmation email promised delivery in three to five business days, but tracking did not update for a week.", | |
| "When the package arrived the box was crushed and the shoes were a size ten instead of the nine I selected.", | |
| "My bank statement also shows two identical charges from your store on the same afternoon.", | |
| "I called support twice and was disconnected both times after waiting on hold for over twenty minutes.", | |
| ] * 11) # about 1,000 state tokens: exercises the fp32 gate at length | |
| # Check requests always included: new questions and options the model was not trained on, and one ~1,000-token state. | |
| DEFAULT_CHECK = [ | |
| {"state": {"message": "I was charged twice for my March invoice and need the duplicate refunded today."}, | |
| "questions": {"intent": {"type": "choice", "instructions": "Select the customer's main request.", | |
| "criteria": {"refund": "Return a payment", "cancel": "End a subscription", | |
| "upgrade": "Move to a larger plan", "info": "Ask a question"}}, | |
| "duplicate_charge": {"type": "noul", "instructions": "Does the message report a duplicate charge?"}, | |
| "urgency": {"type": "score", "instructions": "Rate the urgency expressed in the message.", | |
| "criteria": ["routine", "soon", "urgent", "immediate"]}}}, | |
| {"state": "The package arrived with a cracked screen. I'd like a replacement rather than my money back.", | |
| "questions": {"resolution": {"type": "choice", "instructions": "What outcome does the customer want?", | |
| "criteria": {"replacement": "Send the same item again", "refund": None, | |
| "repair": "Fix the damaged item"}}, | |
| "damaged": {"type": "noul", "instructions": "Did the item arrive damaged?", | |
| "criteria": {"true": "Broken or faulty on arrival", "false": "Arrived intact"}}}}, | |
| {"state": {"ticket": _LONG_STATE}, | |
| "questions": {"team": {"type": "choice", "instructions": "Which team should handle this ticket first?", | |
| "criteria": {"returns": "Exchanges and wrong items", "shipping": "Delivery problems", | |
| "billing": "Charges and refunds", "account": "Login and profile"}}, | |
| "escalate": {"type": "noul", "instructions": "Should a supervisor review this ticket?"}, | |
| "frustration": {"type": "score", "instructions": "How frustrated is the customer?", | |
| "criteria": ["calm", "annoyed", "frustrated", "furious"]}}}, | |
| ] | |
| def _probs(model, requests, temperature=1.0): | |
| return [[p.tolist() for p in model.probs(model.encode(to_record(r)[0]), form="cached", temperature=temperature)] for r in requests] | |
| def _compare(a, b, requests): | |
| """(max |dp| over every option of every question, argmax changes with their margins in `a`).""" | |
| worst, changes = 0.0, [] | |
| for i, (ra, rb, req) in enumerate(zip(a, b, requests)): | |
| for qid, pa, pb in zip(req.questions, ra, rb): | |
| worst = max(worst, max(abs(x - y) for x, y in zip(pa, pb))) | |
| ia, ib = max(range(len(pa)), key=pa.__getitem__), max(range(len(pb)), key=pb.__getitem__) | |
| if ia != ib: | |
| top = sorted(pa, reverse=True) | |
| changes.append({"request": i, "question": qid, "argmax_a": ia, "argmax_b": ib, "margin_a": top[0] - top[1]}) | |
| return worst, changes | |
| def _reload(out_dir, requests, device): | |
| """Reload the artifact in a fresh process with the network and the parent unavailable; returns its probabilities | |
| and public answers for `requests`.""" | |
| with tempfile.TemporaryDirectory() as tmp: | |
| (Path(tmp) / "hf_home").mkdir() | |
| (Path(tmp) / "requests.json").write_text(json.dumps([r.model_dump() for r in requests])) | |
| env = {**os.environ, "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "HF_HOME": str(Path(tmp) / "hf_home")} | |
| env.pop("HF_HUB_CACHE", None) | |
| subprocess.run([sys.executable, "-m", "decision_model.export", "--reload-check", str(out_dir), "--requests", | |
| str(Path(tmp) / "requests.json"), "--device", str(device), "--result", str(Path(tmp) / "out.json")], | |
| env=env, check=True) | |
| return json.loads((Path(tmp) / "out.json").read_text()) | |
| def _reload_check(model_dir, requests_path, device, result_path): | |
| from .model import DecisionModel | |
| reqs = [DecisionRequest.model_validate(r) for r in json.loads(Path(requests_path).read_text())] | |
| m = DecisionModel.from_pretrained(model_dir, device=device) | |
| out = {"probs": _probs(m, reqs), "answers": [m.evaluate(r.state, r.model_dump()["questions"])["answers"] for r in reqs], | |
| "dtype": str(next(m.lm.parameters()).dtype).removeprefix("torch."), "layers": m.lm.config.num_hidden_layers, | |
| "env": {k: os.environ.get(k) for k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_HOME")}} | |
| Path(result_path).write_text(json.dumps(out)) | |
| def _versions(): | |
| out = {"python": sys.version.split()[0]} | |
| for lib in LIBRARIES: | |
| try: | |
| out[lib] = metadata.version(lib) | |
| except metadata.PackageNotFoundError: | |
| out[lib] = None | |
| return out | |
| def _parent_license(parent, revision): | |
| from huggingface_hub import hf_hub_download | |
| try: | |
| return Path(hf_hub_download(parent, "LICENSE", revision=revision, local_files_only=True)) | |
| except Exception: # noqa: BLE001 - reported in the export report rather than failing the export | |
| return None | |
| def dir_bytes(path) -> int: | |
| return sum(f.stat().st_size for f in Path(path).rglob("*") if f.is_file()) | |
| def readme(cfg: dict, calibration: dict, conversion: dict, report: dict) -> str: | |
| p, fp = cfg["parent"], report["merged_fp32_vs_reloaded"] | |
| d = report["merged_in_memory_vs_reloaded_same_dtype"]["max_abs_dp"] | |
| dt = cfg["backbone"]["dtype"] | |
| same = (f"the saved and reloaded {dt} weights reproduced the in-memory {dt} model exactly (max difference 0)" if d == 0 else | |
| f"the saved and reloaded {dt} weights matched the in-memory {dt} model within {d:.3g} in any probability") | |
| return f"""# Decision model converted from {p['name']} | |
| A complete, standalone typed decision model: state + runtime questions + runtime options in, Boolean (`noul`), | |
| categorical (`choice`) and ordinal (`score`) decisions with full probability distributions out. It was converted from | |
| `{p['name']}@{p['revision']}` by merging a trained LoRA adapter into all {cfg['backbone']['num_hidden_layers']} backbone | |
| layers and adding a shared pointer head over runtime options. The parent's weights were modified (LoRA merged) and its | |
| vocabulary projection is not used; no text is generated. The backbone is the parent's size: this conversion does not | |
| make the weights smaller. | |
| ## Contents | |
| | Path | What | | |
| |---|---| | |
| | `config.json` | format version, backbone depth, head, renderer, admission limits, temperature ({cfg['temperature']:.4g}) | | |
| | `backbone/` | Hugging Face `{cfg['backbone']['type']}` decoder config and merged `{cfg['backbone']['dtype']}` safetensors (no `lm_head`) | | |
| | `decision_head.safetensors` | the shared pointer head (query/key projections) | | |
| | `tokenizer/` | the exact tokenizer; delimiter ids are in `config.json` | | |
| | `calibration.json` | fitted temperature, fitting scope and counts | | |
| | `conversion.json` | parent revision, recipe, corpus/target hashes, library versions, export equivalence check | | |
| | `evaluation.json` | measured quality and performance (written by `decision_model.evaluate`) | | |
| | `NOTICE`, `LICENSE` | notices for reused code and the parent model; the parent's Apache-2.0 license | | |
| ## Load | |
| Nothing is downloaded: no parent, teacher or network access is needed. | |
| ```python | |
| from decision_model import DecisionModel | |
| m = DecisionModel.from_pretrained("path/to/this/directory") # device defaults to cuda when available | |
| out = m.evaluate(state={{"message": "I was charged twice and need the duplicate refunded."}}, | |
| questions={{"intent": {{"type": "choice", "instructions": "Select the customer's main request.", | |
| "criteria": {{"refund": "Return a payment", "cancel": "End a subscription"}}}}, | |
| "duplicate_charge": {{"type": "noul", "instructions": "Does the message report a duplicate charge?"}}, | |
| "urgency": {{"type": "score", "instructions": "Rate the urgency.", "criteria": ["routine", "urgent", "immediate"]}}}}) | |
| print(out["answers"]) | |
| ``` | |
| CLI: `python -m decision_model.decide --model <this directory> --request req.json`. | |
| HTTP: `python -m decision_model.serve --model <this directory> --port 8008`, then `POST /v1/systemone` with the same JSON. | |
| Probabilities are calibrated by the stored temperature; pass `temperature=1.0` to `evaluate` for raw probabilities. | |
| `top_probability` is the probability of the modal option, not the probability of being correct. Calibration was fitted | |
| on {calibration.get('fitting_scope', 'the calibration partition')} and does not establish calibration on arbitrary new schemas. | |
| Precision: the weights ship in {cfg['backbone']['dtype']}. On the export check ({report['questions']} questions) the merged FP32 | |
| model and this artifact differed by at most {fp['max_abs_dp']:.3g} in any probability, with {len(fp['argmax_changes'])} argmax | |
| change(s); {same}. Details are in `conversion.json` under `export_equivalence`. | |
| Admission limits: {cfg['limits']['max_state_tokens']}-token state, {cfg['limits']['max_row_tokens']} tokens for state plus one | |
| question branch, {cfg['limits']['max_questions']} questions per request, {cfg['limits']['max_options']} options or levels. | |
| Over-limit requests fail explicitly; nothing is truncated. | |
| ## Measured capabilities | |
| {MEASURED_START} | |
| Pending: `decision_model.evaluate` fills this section with measured results for this artifact. | |
| {MEASURED_END} | |
| """ | |
| def _check_out_dir(out: Path) -> None: | |
| """--out may be absent, empty, or a previous export of ours (replaced); anything else is refused, never deleted.""" | |
| if not out.exists() or (out.is_dir() and not any(out.iterdir())): | |
| return | |
| try: | |
| ours = json.loads((out / "config.json").read_text()).get("model_type") == "decision_model" | |
| except (OSError, ValueError): | |
| ours = False | |
| if not (out.is_dir() and (ours or (out / "export_failed.json").exists())): | |
| raise FileExistsError(f"{out} exists and is not a previous decision-model export; refusing to replace it") | |
| def export(run_dir, out_dir, *, calibration: dict, conversion: dict, dtype=torch.bfloat16, device, check_requests: list) -> dict: | |
| """Merge, save and verify. Returns the equivalence report (also stored in conversion.json); raises if a tolerance is | |
| exceeded (the artifact files are left in place for inspection).""" | |
| from .model import DecisionModel | |
| run_dir, out = Path(run_dir), Path(out_dir) | |
| if "temperature" not in calibration: | |
| raise ValueError("calibration must contain the fitted 'temperature'") | |
| reqs = [r if isinstance(r, DecisionRequest) else DecisionRequest.model_validate(r) for r in list(check_requests or []) + DEFAULT_CHECK] | |
| _check_out_dir(out) | |
| m = DecisionModel.from_run(run_dir, device=device, dtype=torch.float32) | |
| pre = _probs(m, reqs) | |
| m.merge_lora() | |
| merged = _probs(m, reqs) | |
| if out.exists(): | |
| shutil.rmtree(out) | |
| m.save_pretrained(out, dtype=dtype) # casts parameters only and leaves this instance in `dtype` | |
| merged_cast = _probs(m, reqs) # the deployed weights in memory, same form and request shapes as the reload | |
| parent = dict(m.parent) | |
| del m | |
| if str(device).startswith("cuda"): | |
| torch.cuda.empty_cache() | |
| # Artifact metadata before the reload, so the reload sees the final config.json (temperature does not enter T=1 probs). | |
| cfg_path = out / "config.json" | |
| cfg = json.loads(cfg_path.read_text()) | |
| cfg["temperature"] = float(calibration["temperature"]) | |
| cfg_path.write_text(json.dumps(cfg, indent=2) + "\n") | |
| reloaded = _reload(out, reqs, device) | |
| d_merge, flips_merge = _compare(pre, merged, reqs) | |
| d_reload, flips_reload = _compare(merged, reloaded["probs"], reqs) | |
| d_serial, flips_serial = _compare(merged_cast, reloaded["probs"], reqs) | |
| report = {"requests": len(reqs), "questions": sum(len(r.questions) for r in reqs), | |
| "conditions": {"fp32_tolerance_justification": TOL_MERGE_NOTE, "form": "cached", "temperature": 1.0, "device": str(device), "deployed_dtype": reloaded["dtype"], | |
| "NVIDIA_TF32_OVERRIDE": os.environ.get("NVIDIA_TF32_OVERRIDE"), | |
| "tf32_matmul": torch.backends.cuda.matmul.allow_tf32}, | |
| "pre_merge_vs_merged_fp32": {"max_abs_dp": d_merge, "tolerance": TOL_MERGE, "argmax_changes": flips_merge}, | |
| "merged_in_memory_vs_reloaded_same_dtype": {"max_abs_dp": d_serial, "tolerance": TOL_RELOAD, "argmax_changes": flips_serial}, | |
| "merged_fp32_vs_reloaded": {"max_abs_dp": d_reload, "gated": False, "argmax_changes": flips_reload, | |
| "note": "fp32 -> deployed-dtype arithmetic difference; measured, not a tolerance"}, | |
| "reload_env": reloaded["env"], "reloaded_layers": reloaded["layers"], "reloaded_answers": reloaded["answers"], | |
| "artifact_bytes": None, "artifact_bytes_note": "all artifact files except conversion.json"} | |
| run_meta = json.loads((run_dir / "run.json").read_text()) | |
| (out / "calibration.json").write_text(json.dumps({"fitting_scope": "calibration partition", **calibration}, indent=2) + "\n") | |
| notice = REPO / "NOTICE" | |
| if notice.exists(): | |
| shutil.copy(notice, out / "NOTICE") | |
| lic = _parent_license(parent["name"], parent["revision"]) | |
| if lic: | |
| shutil.copy(lic, out / "LICENSE") | |
| (out / "README.md").write_text(readme(cfg, calibration, conversion, report)) | |
| report["artifact_bytes"] = dir_bytes(out) # every artifact file except conversion.json, which records this number | |
| conv = {**conversion, "parent": parent, "selected_checkpoint": {"path": str(run_dir), "run": run_meta}, | |
| "library_versions": _versions(), "exported_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), | |
| "export_dtype": str(dtype).removeprefix("torch."), "export_equivalence": {k: v for k, v in report.items() if k != "reloaded_answers"}, | |
| "parent_license_copied": bool(lic)} | |
| problems = [] | |
| if d_merge > TOL_MERGE: | |
| problems.append(f"pre-merge vs merged FP32 max |dp| {d_merge:.3g} > {TOL_MERGE}") | |
| if d_serial > TOL_RELOAD: | |
| problems.append(f"in-memory {reloaded['dtype']} vs reloaded {reloaded['dtype']} max |dp| {d_serial:.3g} > {TOL_RELOAD}") | |
| if problems: | |
| # conversion.json marks a complete export (convert's resume check reads it); a failed one never gets it | |
| (out / "export_failed.json").write_text(json.dumps({"problems": problems, **conv}, indent=2) + "\n") | |
| raise AssertionError("export equivalence failed: " + "; ".join(problems) + f" (report: {out / 'export_failed.json'})") | |
| (out / "conversion.json").write_text(json.dumps(conv, indent=2) + "\n") | |
| report["artifact_bytes_with_conversion_json"] = dir_bytes(out) | |
| return report | |
| def _check_requests(path, n): | |
| if not path: | |
| return None | |
| reqs = [] | |
| with open(path) as f: | |
| for line in f: | |
| if line.strip(): | |
| reqs.append(DecisionRequest.model_validate(json.loads(line)["request"])) | |
| if len(reqs) >= n: | |
| break | |
| return reqs | |
| def main(): | |
| ap = argparse.ArgumentParser(description="Export a training run as a complete standalone artifact") | |
| ap.add_argument("--run", help="training-run directory (adapter/, head.safetensors, run.json)") | |
| ap.add_argument("--out", help="artifact directory to write (replaced if present)") | |
| ap.add_argument("--calibration", help="calibration JSON (decision_model.calibrate output; must contain 'temperature')") | |
| ap.add_argument("--conversion", help="conversion metadata JSON to include (recipe, manifest hashes, training metrics)") | |
| ap.add_argument("--check-data", help="prepared JSONL whose first --n-check requests join the equivalence check") | |
| ap.add_argument("--n-check", type=int, default=16) | |
| ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") | |
| ap.add_argument("--dtype", default="bfloat16") | |
| ap.add_argument("--reload-check", help=argparse.SUPPRESS) | |
| ap.add_argument("--requests", help=argparse.SUPPRESS) | |
| ap.add_argument("--result", help=argparse.SUPPRESS) | |
| a = ap.parse_args() | |
| if a.reload_check: | |
| return _reload_check(a.reload_check, a.requests, a.device, a.result) | |
| if not (a.run and a.out and a.calibration): | |
| ap.error("--run, --out and --calibration are required") | |
| report = export(a.run, a.out, calibration=json.loads(Path(a.calibration).read_text()), | |
| conversion=json.loads(Path(a.conversion).read_text()) if a.conversion else {}, | |
| dtype=getattr(torch, a.dtype), device=a.device, check_requests=_check_requests(a.check_data, a.n_check)) | |
| print(json.dumps({k: v for k, v in report.items() if k != "reloaded_answers"}, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |