code/finalize.py

"""Write the model card and publish the trained decision model to the Hugging Face Hub.

    python finalize.py --repo kongyo/Ketsudan-27B --name final2 [--public]

Reads the run's reports from /content/out, writes README.md next to the adapters and head,
uploads the folder to the repo root, the evaluation files to eval/ and this code to code/.
"""

import argparse
import json
import os

parser = argparse.ArgumentParser()
parser.add_argument("--repo", required = True)
parser.add_argument("--out", default = "/content/out")
parser.add_argument("--name", default = "final")
parser.add_argument("--public", action = "store_true")
parser.add_argument("--dry-run", action = "store_true")
args = parser.parse_args()

folder = f"{args.out}/{args.name}"
report = json.load(open(f"{folder}/train_report.json", encoding = "utf-8"))
model_name = args.repo.split("/")[-1]


def load(path):
    path = f"{args.out}/{path}"
    return json.load(open(path, encoding = "utf-8")) if os.path.exists(path) else None


def pct(x):
    return "–" if x is None else f"{x:.3f}"


ours_td = report["eval"]["typed_decisions_test"]
jev_td = load("jev_td_test_metrics.json")
ours_ho_parts = [load(f"{args.name}_heldout_metrics.json") or report["eval"].get("heldout"), load(f"{args.name}_heldout2_metrics.json")]
jev_ho_parts = [load("jev_heldout_metrics.json"), load("jev_heldout2_metrics.json")]


def merged(parts, key = "accuracy_by_workflow"):
    out = {}
    for part in parts:
        if part:
            out.update(part[key])
    return out


ours_ho, jev_ho = merged(ours_ho_parts), merged(jev_ho_parts)
ours_ece = [p["ece"] for p in ours_ho_parts if p]
jev_ece = [p["ece"] for p in jev_ho_parts if p]


def bold_pair(a, b, lower_better = False):
    better_a = (a <= b) if lower_better else (a >= b)
    return (f"**{pct(a)}**" if better_a else pct(a)), (pct(b) if better_a else f"**{pct(b)}**")


def td_row(name, m):
    t = m["accuracy_by_type"]
    return (f"| {name} | {pct(m['accuracy'])} | {pct(t.get('noul'))} | {pct(t.get('choice'))} | {pct(t.get('score'))} | "
            f"{pct(m['kl_from_gold'])} | {pct(m['brier'])} | {pct(m['ece'])} | {pct(m['record_accuracy'])} |")


td_lines = [
    "| Model | Accuracy ↑ | noul | choice | score | KL from gold ↓ | Brier ↓ | ECE ↓ | All 5 right ↑ |",
    "|---|---:|---:|---:|---:|---:|---:|---:|---:|",
    td_row(f"**{model_name}** (this model)", ours_td),
]
if jev_td:
    td_lines.append(td_row("TypeSafe Jev (`jev-latest` → jev-1.13.0, same day)", jev_td))
td_table = "\n".join(td_lines)
claim = ""
if jev_td:
    cols = [ours_td["accuracy"] >= jev_td["accuracy"], ours_td["kl_from_gold"] <= jev_td["kl_from_gold"],
            ours_td["brier"] <= jev_td["brier"], ours_td["ece"] <= jev_td["ece"],
            ours_td["record_accuracy"] >= jev_td["record_accuracy"]]
    types = [ours_td["accuracy_by_type"][k] >= v for k, v in jev_td["accuracy_by_type"].items()]
    flows = [ours_td["accuracy_by_workflow"][k] >= v for k, v in jev_td["accuracy_by_workflow"].items()]
    if all(cols + types + flows):
        claim = f"{model_name} matches or beats Jev on every column, every question type and every workflow."
    else:
        claim = (f"{model_name} is ahead of Jev on {sum(cols)} of 5 summary columns, {sum(types)} of {len(types)} "
                 f"question types and {sum(flows)} of {len(flows)} workflows.")

LABELS = {
    "anli": "ANLI test r1–r3 (400, 3-way NLI)",
    "winogrande": "WinoGrande dev (400, commonsense pronoun / blank)",
    "hellaswag": "HellaSwag validation (400, 4 endings)",
    "emotion": "dair-ai/emotion test (400, 6 emotions)",
}
both = [b for b in LABELS if b in ours_ho and b in jev_ho]
ho_lines = ["| Benchmark (rows) | This model | Jev (same requests, same day) |", "|---|---:|---:|"]
for b in both:
    a, j = bold_pair(ours_ho[b], jev_ho[b])
    ho_lines.append(f"| {LABELS[b]} | {a} | {j} |")
if both:
    a, j = bold_pair(sum(ours_ho[b] for b in both) / len(both), sum(jev_ho[b] for b in both) / len(both))
    ho_lines.append(f"| Mean accuracy | {a} | {j} |")
if ours_ece and jev_ece:
    a, j = bold_pair(sum(ours_ece) / len(ours_ece), sum(jev_ece) / len(jev_ece), lower_better = True)
    ho_lines.append(f"| Mean ECE ↓ | {a} | {j} |")
ho_table = "\n".join(ho_lines)

losses = [LABELS[b].split(" (")[0] for b in both if ours_ho[b] < jev_ho[b]]
weak = []
if losses:
    weak.append(f"* Trails Jev on {', '.join(losses)} (see the held-out table).")
if ours_ece and sum(ours_ece) / len(ours_ece) > 0.08:
    weak.append("* Confidence outside the training distribution is less reliable (held-out ECE above). Refit the "
                "temperatures on a few hundred labelled examples of your own task (`FastDecisionModel.calibrate`).")
weak.append("* English only was evaluated. Text input only (the backbone's vision tower is not trained).")
weak.append("* Numbers, dates and counting belong in code, as for any System One style model.")
weak_text = "\n".join(weak)

stage2 = report.get("stage2")
s1 = report["args"]
training_rows = [
    f"| Stage 1 | LoRA r={s1['lora_r']} on every language-model projection (attention, Gated DeltaNet, MLP) + a fresh joint schema head; "
    f"lr {s1['lr']} (head {s1['head_lr']}), cosine, soft-target cross-entropy, one pass over {report['train_items'] - (stage2 or {}).get('train_items', 0):,} records "
    f"(all 1,120 typed-decisions train cases, n4ze3m/typed-decisions-synth, Unsloth's public mixture) in a 32-minute budget |",
]
if stage2:
    sources = ", ".join(f"{k} ({v})" for k, v in stage2["sources"].items())
    training_rows.append(
        f"| Stage 2 | continued from stage 1 at lr {stage2['args']['lr']} (head {stage2['args']['head_lr']}): {stage2['train_items']:,} records "
        f"= {stage2['td_replay']} typed-decisions cases replayed + commonsense / multiple-choice sources {sources} + fresh public-mixture and synthetic rows |"
    )
    training_rows.append("| Calibration | per-type temperatures refit on a mixed holdout: 80 unseen typed-decisions train cases + 320 unseen general items |")
else:
    training_rows.append("| Calibration | per-type temperatures fitted on 80 held-out typed-decisions train cases |")
training_rows.append(f"| Total | {report['train_items']:,} records, {report['train_tokens'] / 1e6:.2f}M tokens, {report['steps']} optimizer steps of 16 |")
minutes = (report.get("train_metrics", {}).get("train_runtime", 0) + (stage2 or {}).get("train_metrics", {}).get("train_runtime", 0)) / 60
training_rows.append(f"| Hardware | 1 × {report.get('gpu')} (Google Colab G4), {minutes:.0f} minutes of optimizer time in total |")
training_table = "\n".join(training_rows)

card = f"""---
license: apache-2.0
base_model: Qwen/Qwen3.8-27B
library_name: unsloth
pipeline_tag: text-classification
tags:
- decision-model
- typed-decisions
- system-one-compatible
- calibration
- clef-head
- unsloth
- lora
datasets:
- LocalLLaMA/typed-decisions
- n4ze3m/typed-decisions-synth
language:
- en
---

# {model_name} — a calibrated decision model on Qwen3.8-27B

**{model_name}** (決断, "decision") turns [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) into a
*decision model*: it reads a `state` and typed questions — `choice`, `noul` (yes/no) and `score` (ordered levels) — and
returns a calibrated probability for every option instead of generating text. Requests and answers use the JSON shape of
a `POST /v1/systemone` body, so Jev-style requests work unchanged.

* **Backbone:** Qwen/Qwen3.8-27B (bf16) with LoRA adapters on every language-model projection.
* **Head:** a joint schema head of Cloudflare's [Clef](https://huggingface.co/Cloudflare/clef) design, trained from scratch
  (no Clef weights are used), through Unsloth's `FastDecisionModel`.
* **Open weights, Apache-2.0**, ~1.2 GB of adapters and head on top of the public base model.

Not affiliated with TypeSafe AI, Cloudflare, Unsloth or Qwen.

## Results

### typed-decisions test (400 cases × 5 questions = 2,000 decisions)

{td_table}

Same 400 requests, same scorer, same day ([`code/td_metrics.py`](code/td_metrics.py): accuracy = argmax vs the gold
label; KL(gold‖pred) with 1e-6 clipping; Brier = mean over options of the squared error; ECE = 10 bins of the top
probability). {claim} Per-workflow numbers are in [`eval/`](eval).

**Read this caveat:** {model_name} was trained on the typed-decisions `train` split (fitted mode); Jev never saw these
workflows. For reference, the dataset card lists TypeSafe Jev 1.13.0 at 0.727 (zero-shot) and models fitted on `train` at
0.766–0.801 (OpenDecider-large-td 0.801, od1 0.797, OpenDecider-nano 0.796, OpenDecider-small-td 0.792); those numbers come
from the dataset maintainers' or submitters' scorers, not ours, and the accuracy column is the only one that is comparable.

### Held-out tasks neither model was trained on

{ho_table}

Identical requests (same state, instructions and options) were sent to both models on the same day. These are
general zero-shot decisions outside the training data.

## Known weaknesses

{weak_text}

## Usage

```python
from unsloth import FastDecisionModel

model, tokenizer = FastDecisionModel.from_pretrained("{args.repo}")  # Qwen/Qwen3.8-27B + adapters + head
FastDecisionModel.for_inference(model)
answers = FastDecisionModel.predict(
    model, tokenizer,
    "Hi, I was charged twice for invoice #4411. Please refund the duplicate today.",
    {{
        "team": {{"type": "choice", "instructions": "Which team should handle this?",
                  "criteria": {{"billing": "invoices, payments, refunds", "technical": "bugs, outages, errors",
                               "sales": "pricing, new plans"}}}},
        "refund": {{"type": "noul", "instructions": "Does the customer ask for a refund?"}},
        "urgency": {{"type": "score", "instructions": "How urgent is this?",
                    "criteria": ["not urgent", "soon", "today"]}},
    }},
)
print(answers["team"]["answer"], answers["team"]["probabilities"])
```

About 56 GB of GPU memory in bf16; `load_in_4bit=True` runs it on a 24 GB GPU. Every answer carries `probabilities` over
all options; `noul` answers carry the probability of yes. Unsloth Studio can serve the folder behind a
TypeSafe-compatible `/v1/systemone` endpoint, so code written for the `typesafe-sdk` can point `TYPESAFE_BASE_URL` at it.

## Training

| | |
|---|---|
{training_table}

**No output of TypeSafe's Jev was used for training, calibration or model selection** (TypeSafe's terms forbid
distillation). Jev was only called to score it on the evaluation sets above.

**Decontamination.** Synthetic, public and commonsense training rows share no 13-word shingle with any evaluation state
(typed-decisions test, ANLI, WinoGrande; in stage 2 also HellaSwag and emotion). typed-decisions train has no state
identical to a test state (its cases share JSON boilerplate with the test cases, so shingles would flag most of them).
HellaSwag and emotion were not shingle-checked in stage 1; no stage-1 source draws on them.

## Licenses

Weights: Apache-2.0 (base model Apache-2.0). Training data keeps its own licenses: typed-decisions (Apache-2.0),
typed-decisions-synth (MIT), BANKING77 (CC-BY-4.0), CLINC150 (CC-BY-3.0), MNLI (mixed), SNLI (CC-BY-SA-4.0), WANLI
(CC-BY-4.0), BoolQ (CC-BY-SA-3.0), MMLU / CommonsenseQA (MIT), ARC (CC-BY-SA-4.0), OpenBookQA (Apache-2.0), COPA / WSC
(SuperGLUE terms), Social IQa and SWAG (as published by AI2), prompt-injections (Apache-2.0); AG News and SST-5 list no
license. The head follows Cloudflare's Clef design (Apache-2.0) as implemented in Unsloth.
"""

with open(f"{folder}/README.md", "w", encoding = "utf-8") as f:
    f.write(card)
if args.dry_run:
    print(card)
    raise SystemExit(0)

from huggingface_hub import HfApi

api = HfApi()
api.create_repo(args.repo, private = True, exist_ok = True)
api.upload_folder(folder_path = folder, repo_id = args.repo, commit_message = f"Decision model ({args.name}): adapters, head, calibration, card")
for name in sorted(os.listdir(args.out)):
    # Jev's raw answers stay private (only its aggregate metrics are published), so nobody can distil from them.
    if name.endswith((".json", ".jsonl")) and not name.startswith("_") and "rows" not in name and not (name.startswith("jev_") and name.endswith(".jsonl")):
        api.upload_file(path_or_fileobj = f"{args.out}/{name}", path_in_repo = f"eval/{name}", repo_id = args.repo)
code = os.path.dirname(os.path.abspath(__file__))
for name in sorted(os.listdir(code)):
    if name.endswith((".py", ".sh")):
        api.upload_file(path_or_fileobj = f"{code}/{name}", path_in_repo = f"code/{name}", repo_id = args.repo)
files = api.list_repo_files(args.repo)
for stale in ("final",):
    if any(path.startswith(f"{stale}/") for path in files):
        api.delete_folder(stale, repo_id = args.repo, commit_message = "Remove the training checkpoint copy")
if args.public:
    api.update_repo_settings(args.repo, private = False)
print("FINALIZED", args.repo, args.name, "public" if args.public else "private", flush = True)