| """Write the model card and publish the trained decision model to the Hugging Face Hub. |
| |
| python finalize.py --repo kongyo/Ketsudan-27B --name final2 [--public] |
| |
| Reads the run's reports from /content/out, writes README.md next to the adapters and head, |
| uploads the folder to the repo root, the evaluation files to eval/ and this code to code/. |
| """ |
|
|
| import argparse |
| import json |
| import os |
|
|
| parser = argparse.ArgumentParser() |
| parser.add_argument("--repo", required = True) |
| parser.add_argument("--out", default = "/content/out") |
| parser.add_argument("--name", default = "final") |
| parser.add_argument("--public", action = "store_true") |
| parser.add_argument("--dry-run", action = "store_true") |
| args = parser.parse_args() |
|
|
| folder = f"{args.out}/{args.name}" |
| report = json.load(open(f"{folder}/train_report.json", encoding = "utf-8")) |
| model_name = args.repo.split("/")[-1] |
|
|
|
|
| def load(path): |
| path = f"{args.out}/{path}" |
| return json.load(open(path, encoding = "utf-8")) if os.path.exists(path) else None |
|
|
|
|
| def pct(x): |
| return "–" if x is None else f"{x:.3f}" |
|
|
|
|
| ours_td = report["eval"]["typed_decisions_test"] |
| jev_td = load("jev_td_test_metrics.json") |
| ours_ho_parts = [load(f"{args.name}_heldout_metrics.json") or report["eval"].get("heldout"), load(f"{args.name}_heldout2_metrics.json")] |
| jev_ho_parts = [load("jev_heldout_metrics.json"), load("jev_heldout2_metrics.json")] |
|
|
|
|
| def merged(parts, key = "accuracy_by_workflow"): |
| out = {} |
| for part in parts: |
| if part: |
| out.update(part[key]) |
| return out |
|
|
|
|
| ours_ho, jev_ho = merged(ours_ho_parts), merged(jev_ho_parts) |
| ours_ece = [p["ece"] for p in ours_ho_parts if p] |
| jev_ece = [p["ece"] for p in jev_ho_parts if p] |
|
|
|
|
| def bold_pair(a, b, lower_better = False): |
| better_a = (a <= b) if lower_better else (a >= b) |
| return (f"**{pct(a)}**" if better_a else pct(a)), (pct(b) if better_a else f"**{pct(b)}**") |
|
|
|
|
| def td_row(name, m): |
| t = m["accuracy_by_type"] |
| return (f"| {name} | {pct(m['accuracy'])} | {pct(t.get('noul'))} | {pct(t.get('choice'))} | {pct(t.get('score'))} | " |
| f"{pct(m['kl_from_gold'])} | {pct(m['brier'])} | {pct(m['ece'])} | {pct(m['record_accuracy'])} |") |
|
|
|
|
| td_lines = [ |
| "| Model | Accuracy ↑ | noul | choice | score | KL from gold ↓ | Brier ↓ | ECE ↓ | All 5 right ↑ |", |
| "|---|---:|---:|---:|---:|---:|---:|---:|---:|", |
| td_row(f"**{model_name}** (this model)", ours_td), |
| ] |
| if jev_td: |
| td_lines.append(td_row("TypeSafe Jev (`jev-latest` → jev-1.13.0, same day)", jev_td)) |
| td_table = "\n".join(td_lines) |
| claim = "" |
| if jev_td: |
| cols = [ours_td["accuracy"] >= jev_td["accuracy"], ours_td["kl_from_gold"] <= jev_td["kl_from_gold"], |
| ours_td["brier"] <= jev_td["brier"], ours_td["ece"] <= jev_td["ece"], |
| ours_td["record_accuracy"] >= jev_td["record_accuracy"]] |
| types = [ours_td["accuracy_by_type"][k] >= v for k, v in jev_td["accuracy_by_type"].items()] |
| flows = [ours_td["accuracy_by_workflow"][k] >= v for k, v in jev_td["accuracy_by_workflow"].items()] |
| if all(cols + types + flows): |
| claim = f"{model_name} matches or beats Jev on every column, every question type and every workflow." |
| else: |
| claim = (f"{model_name} is ahead of Jev on {sum(cols)} of 5 summary columns, {sum(types)} of {len(types)} " |
| f"question types and {sum(flows)} of {len(flows)} workflows.") |
|
|
| LABELS = { |
| "anli": "ANLI test r1–r3 (400, 3-way NLI)", |
| "winogrande": "WinoGrande dev (400, commonsense pronoun / blank)", |
| "hellaswag": "HellaSwag validation (400, 4 endings)", |
| "emotion": "dair-ai/emotion test (400, 6 emotions)", |
| } |
| both = [b for b in LABELS if b in ours_ho and b in jev_ho] |
| ho_lines = ["| Benchmark (rows) | This model | Jev (same requests, same day) |", "|---|---:|---:|"] |
| for b in both: |
| a, j = bold_pair(ours_ho[b], jev_ho[b]) |
| ho_lines.append(f"| {LABELS[b]} | {a} | {j} |") |
| if both: |
| a, j = bold_pair(sum(ours_ho[b] for b in both) / len(both), sum(jev_ho[b] for b in both) / len(both)) |
| ho_lines.append(f"| Mean accuracy | {a} | {j} |") |
| if ours_ece and jev_ece: |
| a, j = bold_pair(sum(ours_ece) / len(ours_ece), sum(jev_ece) / len(jev_ece), lower_better = True) |
| ho_lines.append(f"| Mean ECE ↓ | {a} | {j} |") |
| ho_table = "\n".join(ho_lines) |
|
|
| losses = [LABELS[b].split(" (")[0] for b in both if ours_ho[b] < jev_ho[b]] |
| weak = [] |
| if losses: |
| weak.append(f"* Trails Jev on {', '.join(losses)} (see the held-out table).") |
| if ours_ece and sum(ours_ece) / len(ours_ece) > 0.08: |
| weak.append("* Confidence outside the training distribution is less reliable (held-out ECE above). Refit the " |
| "temperatures on a few hundred labelled examples of your own task (`FastDecisionModel.calibrate`).") |
| weak.append("* English only was evaluated. Text input only (the backbone's vision tower is not trained).") |
| weak.append("* Numbers, dates and counting belong in code, as for any System One style model.") |
| weak_text = "\n".join(weak) |
|
|
| stage2 = report.get("stage2") |
| s1 = report["args"] |
| training_rows = [ |
| f"| Stage 1 | LoRA r={s1['lora_r']} on every language-model projection (attention, Gated DeltaNet, MLP) + a fresh joint schema head; " |
| f"lr {s1['lr']} (head {s1['head_lr']}), cosine, soft-target cross-entropy, one pass over {report['train_items'] - (stage2 or {}).get('train_items', 0):,} records " |
| f"(all 1,120 typed-decisions train cases, n4ze3m/typed-decisions-synth, Unsloth's public mixture) in a 32-minute budget |", |
| ] |
| if stage2: |
| sources = ", ".join(f"{k} ({v})" for k, v in stage2["sources"].items()) |
| training_rows.append( |
| f"| Stage 2 | continued from stage 1 at lr {stage2['args']['lr']} (head {stage2['args']['head_lr']}): {stage2['train_items']:,} records " |
| f"= {stage2['td_replay']} typed-decisions cases replayed + commonsense / multiple-choice sources {sources} + fresh public-mixture and synthetic rows |" |
| ) |
| training_rows.append("| Calibration | per-type temperatures refit on a mixed holdout: 80 unseen typed-decisions train cases + 320 unseen general items |") |
| else: |
| training_rows.append("| Calibration | per-type temperatures fitted on 80 held-out typed-decisions train cases |") |
| training_rows.append(f"| Total | {report['train_items']:,} records, {report['train_tokens'] / 1e6:.2f}M tokens, {report['steps']} optimizer steps of 16 |") |
| minutes = (report.get("train_metrics", {}).get("train_runtime", 0) + (stage2 or {}).get("train_metrics", {}).get("train_runtime", 0)) / 60 |
| training_rows.append(f"| Hardware | 1 × {report.get('gpu')} (Google Colab G4), {minutes:.0f} minutes of optimizer time in total |") |
| training_table = "\n".join(training_rows) |
|
|
| card = f"""--- |
| license: apache-2.0 |
| base_model: Qwen/Qwen3.8-27B |
| library_name: unsloth |
| pipeline_tag: text-classification |
| tags: |
| - decision-model |
| - typed-decisions |
| - system-one-compatible |
| - calibration |
| - clef-head |
| - unsloth |
| - lora |
| datasets: |
| - LocalLLaMA/typed-decisions |
| - n4ze3m/typed-decisions-synth |
| language: |
| - en |
| --- |
| |
| # {model_name} — a calibrated decision model on Qwen3.8-27B |
| |
| **{model_name}** (決断, "decision") turns [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) into a |
| *decision model*: it reads a `state` and typed questions — `choice`, `noul` (yes/no) and `score` (ordered levels) — and |
| returns a calibrated probability for every option instead of generating text. Requests and answers use the JSON shape of |
| a `POST /v1/systemone` body, so Jev-style requests work unchanged. |
| |
| * **Backbone:** Qwen/Qwen3.8-27B (bf16) with LoRA adapters on every language-model projection. |
| * **Head:** a joint schema head of Cloudflare's [Clef](https://huggingface.co/Cloudflare/clef) design, trained from scratch |
| (no Clef weights are used), through Unsloth's `FastDecisionModel`. |
| * **Open weights, Apache-2.0**, ~1.2 GB of adapters and head on top of the public base model. |
| |
| Not affiliated with TypeSafe AI, Cloudflare, Unsloth or Qwen. |
| |
| ## Results |
| |
| ### typed-decisions test (400 cases × 5 questions = 2,000 decisions) |
| |
| {td_table} |
| |
| Same 400 requests, same scorer, same day ([`code/td_metrics.py`](code/td_metrics.py): accuracy = argmax vs the gold |
| label; KL(gold‖pred) with 1e-6 clipping; Brier = mean over options of the squared error; ECE = 10 bins of the top |
| probability). {claim} Per-workflow numbers are in [`eval/`](eval). |
| |
| **Read this caveat:** {model_name} was trained on the typed-decisions `train` split (fitted mode); Jev never saw these |
| workflows. For reference, the dataset card lists TypeSafe Jev 1.13.0 at 0.727 (zero-shot) and models fitted on `train` at |
| 0.766–0.801 (OpenDecider-large-td 0.801, od1 0.797, OpenDecider-nano 0.796, OpenDecider-small-td 0.792); those numbers come |
| from the dataset maintainers' or submitters' scorers, not ours, and the accuracy column is the only one that is comparable. |
| |
| ### Held-out tasks neither model was trained on |
| |
| {ho_table} |
| |
| Identical requests (same state, instructions and options) were sent to both models on the same day. These are |
| general zero-shot decisions outside the training data. |
| |
| ## Known weaknesses |
| |
| {weak_text} |
| |
| ## Usage |
| |
| ```python |
| from unsloth import FastDecisionModel |
| |
| model, tokenizer = FastDecisionModel.from_pretrained("{args.repo}") # Qwen/Qwen3.8-27B + adapters + head |
| FastDecisionModel.for_inference(model) |
| answers = FastDecisionModel.predict( |
| model, tokenizer, |
| "Hi, I was charged twice for invoice #4411. Please refund the duplicate today.", |
| {{ |
| "team": {{"type": "choice", "instructions": "Which team should handle this?", |
| "criteria": {{"billing": "invoices, payments, refunds", "technical": "bugs, outages, errors", |
| "sales": "pricing, new plans"}}}}, |
| "refund": {{"type": "noul", "instructions": "Does the customer ask for a refund?"}}, |
| "urgency": {{"type": "score", "instructions": "How urgent is this?", |
| "criteria": ["not urgent", "soon", "today"]}}, |
| }}, |
| ) |
| print(answers["team"]["answer"], answers["team"]["probabilities"]) |
| ``` |
| |
| About 56 GB of GPU memory in bf16; `load_in_4bit=True` runs it on a 24 GB GPU. Every answer carries `probabilities` over |
| all options; `noul` answers carry the probability of yes. Unsloth Studio can serve the folder behind a |
| TypeSafe-compatible `/v1/systemone` endpoint, so code written for the `typesafe-sdk` can point `TYPESAFE_BASE_URL` at it. |
| |
| ## Training |
| |
| | | | |
| |---|---| |
| {training_table} |
| |
| **No output of TypeSafe's Jev was used for training, calibration or model selection** (TypeSafe's terms forbid |
| distillation). Jev was only called to score it on the evaluation sets above. |
| |
| **Decontamination.** Synthetic, public and commonsense training rows share no 13-word shingle with any evaluation state |
| (typed-decisions test, ANLI, WinoGrande; in stage 2 also HellaSwag and emotion). typed-decisions train has no state |
| identical to a test state (its cases share JSON boilerplate with the test cases, so shingles would flag most of them). |
| HellaSwag and emotion were not shingle-checked in stage 1; no stage-1 source draws on them. |
| |
| ## Licenses |
| |
| Weights: Apache-2.0 (base model Apache-2.0). Training data keeps its own licenses: typed-decisions (Apache-2.0), |
| typed-decisions-synth (MIT), BANKING77 (CC-BY-4.0), CLINC150 (CC-BY-3.0), MNLI (mixed), SNLI (CC-BY-SA-4.0), WANLI |
| (CC-BY-4.0), BoolQ (CC-BY-SA-3.0), MMLU / CommonsenseQA (MIT), ARC (CC-BY-SA-4.0), OpenBookQA (Apache-2.0), COPA / WSC |
| (SuperGLUE terms), Social IQa and SWAG (as published by AI2), prompt-injections (Apache-2.0); AG News and SST-5 list no |
| license. The head follows Cloudflare's Clef design (Apache-2.0) as implemented in Unsloth. |
| """ |
|
|
| with open(f"{folder}/README.md", "w", encoding = "utf-8") as f: |
| f.write(card) |
| if args.dry_run: |
| print(card) |
| raise SystemExit(0) |
|
|
| from huggingface_hub import HfApi |
|
|
| api = HfApi() |
| api.create_repo(args.repo, private = True, exist_ok = True) |
| api.upload_folder(folder_path = folder, repo_id = args.repo, commit_message = f"Decision model ({args.name}): adapters, head, calibration, card") |
| for name in sorted(os.listdir(args.out)): |
| |
| if name.endswith((".json", ".jsonl")) and not name.startswith("_") and "rows" not in name and not (name.startswith("jev_") and name.endswith(".jsonl")): |
| api.upload_file(path_or_fileobj = f"{args.out}/{name}", path_in_repo = f"eval/{name}", repo_id = args.repo) |
| code = os.path.dirname(os.path.abspath(__file__)) |
| for name in sorted(os.listdir(code)): |
| if name.endswith((".py", ".sh")): |
| api.upload_file(path_or_fileobj = f"{code}/{name}", path_in_repo = f"code/{name}", repo_id = args.repo) |
| files = api.list_repo_files(args.repo) |
| for stale in ("final",): |
| if any(path.startswith(f"{stale}/") for path in files): |
| api.delete_folder(stale, repo_id = args.repo, commit_message = "Remove the training checkpoint copy") |
| if args.public: |
| api.update_repo_settings(args.repo, private = False) |
| print("FINALIZED", args.repo, args.name, "public" if args.public else "private", flush = True) |
|
|
