| { |
| "system": "tuned", |
| "model": "unsloth/gemma-3-270m-it+lora:v1", |
| "created": "2026-10-01T03:14:17", |
| "splits": { |
| "val": { |
| "n": 150, |
| "intent_accuracy": 0.9933, |
| "json_validity_rate": 0.9933, |
| "item_precision": 0.9725, |
| "item_recall": 0.9538, |
| "item_f1": 0.9631, |
| "quantity_accuracy": 0.9725, |
| "part_number_accuracy": 1.0, |
| "compatible_with_accuracy": 0.8696, |
| "missing_info_precision": 0.1613, |
| "missing_info_recall": 0.0909, |
| "missing_info_f1": 0.1163, |
| "urgency_accuracy": 0.98, |
| "exact_match_rate": 0.5733, |
| "per_intent_accuracy": { |
| "order_status": 1.0, |
| "other": 1.0, |
| "product_question": 1.0, |
| "quote_request": 0.9895833333333334, |
| "return_request": 1.0 |
| }, |
| "per_intent_n": { |
| "order_status": 16, |
| "other": 11, |
| "product_question": 14, |
| "quote_request": 96, |
| "return_request": 13 |
| }, |
| "latency_p50_s": 1.2402, |
| "latency_p95_s": 1.7012, |
| "escalation_rate": null, |
| "cost_per_1k_usd": null, |
| "wall_time_s": 0.0, |
| "errors": 0 |
| }, |
| "exam": { |
| "n": 150, |
| "intent_accuracy": 0.98, |
| "json_validity_rate": 0.9867, |
| "item_precision": 0.9266, |
| "item_recall": 0.916, |
| "item_f1": 0.9213, |
| "quantity_accuracy": 0.9339, |
| "part_number_accuracy": 0.9821, |
| "compatible_with_accuracy": 0.84, |
| "missing_info_precision": 0.1389, |
| "missing_info_recall": 0.1316, |
| "missing_info_f1": 0.1351, |
| "urgency_accuracy": 0.9733, |
| "exact_match_rate": 0.6, |
| "per_intent_accuracy": { |
| "order_status": 1.0, |
| "other": 1.0, |
| "product_question": 0.9285714285714286, |
| "quote_request": 0.9791666666666666, |
| "return_request": 1.0 |
| }, |
| "per_intent_n": { |
| "order_status": 16, |
| "other": 11, |
| "product_question": 14, |
| "quote_request": 96, |
| "return_request": 13 |
| }, |
| "latency_p50_s": 1.0266, |
| "latency_p95_s": 2.0577, |
| "escalation_rate": null, |
| "cost_per_1k_usd": null, |
| "wall_time_s": 0.0, |
| "errors": 0 |
| }, |
| "exam_hard": { |
| "n": 30, |
| "intent_accuracy": 0.6, |
| "json_validity_rate": 0.6667, |
| "item_precision": 0.7436, |
| "item_recall": 0.6591, |
| "item_f1": 0.6988, |
| "quantity_accuracy": 0.8286, |
| "part_number_accuracy": 1.0, |
| "compatible_with_accuracy": 1.0, |
| "missing_info_precision": 0.0, |
| "missing_info_recall": 0.0, |
| "missing_info_f1": 0.0, |
| "urgency_accuracy": 0.9333, |
| "exact_match_rate": 0.2667, |
| "per_intent_accuracy": { |
| "order_status": 0.5, |
| "product_question": 1.0, |
| "quote_request": 0.56 |
| }, |
| "per_intent_n": { |
| "order_status": 2, |
| "product_question": 3, |
| "quote_request": 25 |
| }, |
| "latency_p50_s": 0.7219, |
| "latency_p95_s": 1.2877, |
| "escalation_rate": null, |
| "cost_per_1k_usd": null, |
| "wall_time_s": 0.0, |
| "errors": 0 |
| } |
| }, |
| "pricing": { |
| "teacher": { |
| "model": "gemini-3.8-flash", |
| "input_per_1m_usd": 0.75, |
| "output_per_1m_usd": 3.75, |
| "note": "Google's published paid-tier price (standard), thinking tokens billed as output; valid through 2026-12-31. Our calls go through the KIE.ai proxy; credits actually spent are logged separately.", |
| "source": "https://ai.google.dev/gemini-api/docs/pricing", |
| "verified": "2026-09-30", |
| "label": "measured tokens x published paid-tier price" |
| }, |
| "student_cpu": { |
| "basis": "Modal CPU container, 2 cores + 4 GiB: 2 x $0.0000131/core/s + 4 x $0.00000222/GiB/s", |
| "usd_per_second": 3.51e-05, |
| "source": "https://modal.com/pricing", |
| "verified": "2026-09-30", |
| "label": "estimate: inference seconds x container price (excludes idle time)" |
| } |
| }, |
| "adapter_version": "v1", |
| "hardware": "Tesla T4", |
| "latency_note": "GPU batch latency; CPU latency is measured separately in eval/run_eval.py", |
| "run_config": { |
| "version_tag": "v1", |
| "base_model": "unsloth/gemma-3-270m-it", |
| "epochs": 3, |
| "learning_rate": 0.0002, |
| "lora_r": 16, |
| "lora_alpha": 32, |
| "lora_dropout": 0.0, |
| "batch_size": 8, |
| "grad_accum": 2, |
| "warmup_ratio": 0.05, |
| "seed": 3407, |
| "max_seq_len": 640, |
| "max_new_tokens": 384, |
| "train_examples": 1334, |
| "corrections": 0, |
| "train_seconds": 564.4, |
| "gpu": "Tesla T4", |
| "platform": "kaggle", |
| "dataset_manifest": { |
| "version_tag": "v1", |
| "created": "2026-09-30T21:42:39", |
| "git_sha": null, |
| "counts": { |
| "train.jsonl": 1334, |
| "val.jsonl": 150, |
| "exam.jsonl": 150, |
| "exam_hard.jsonl": 30, |
| "fewshot.jsonl": 3 |
| }, |
| "corrections": 0, |
| "sha256": { |
| "catalog.json": "b354cf4e5b81f6be80c5071c0961b0ee79c9c8f9cec86bd4e0102f819e22c021", |
| "customers.json": "b650b859b37bd2c3c33c703aaea9e227cf186558789e7e88f5fd53b7352a1bef", |
| "dataset-metadata.json": "b483500aa344633dc1614e132d976cee59073c4c91003f0977f3c71ecf19a875", |
| "equipment.json": "91e341753fee46d62b9444747d177a670b308999743de1fddd6079a596c56e2c", |
| "evallib/__init__.py": "ec38b6ac328f58d03f3425148201628598e7521691faf7d34a1c326e7d7aa8f8", |
| "evallib/metrics.py": "7bef5b64f15ba6fc0ba877daa6d57294da373061543c984023af9c59edce5a37", |
| "evallib/normalize.py": "0c663091219f7a14baa37307863c8ea9e066b87e46f045abc38e3176f13069a3", |
| "evallib/pricing.json": "782918cf5cdbf2b69d85873028e0bfa9bbafc89cda6828cb9c933ca3bdf96f5f", |
| "evallib/prompt.py": "521bdb70ba03d77436c32b82edf3a7a8f6f44b0ccadade37416ec3db786ffdad", |
| "evallib/runner.py": "3992d9971ab4d21c707964650fb5d09e98123a4adabb329160a8d8eacebec744", |
| "evallib/schema.py": "eec543089b10c394dec74de147b2b238337657c037f69e72caa48244a482e27e", |
| "evallib/system_prompt.txt": "fc0d3583b99fc4dfe3ec3b42f5fe3bfbabbccc23f9aec7edc3771eea5adbbc03", |
| "evallib/teacher_rules.txt": "20d48a4ea874fa5c3f0c5eb69e81f4796aaa6dae40ffc89bb30eeca3b232633d", |
| "exam.jsonl": "dc1100cffd0bbdfd8089f0841f455f753eae50ad6fcd11ba2ad1dca830287584", |
| "exam_hard.jsonl": "bea23f278853cf5b6803df77e2b87cd62175333f8dec3235c779d4c9d4852bd0", |
| "fewshot.jsonl": "f4d148125fe418cd0735e20f326d3df4a2319e8084abebd4eee371c6bc34db91", |
| "orders.json": "37dea4e9b8026315814840acf0cd4557d921c4e5b30bf856def22c01e99a285c", |
| "train.jsonl": "63578645ec0a3d53de85255a888d8855710b218e31a4bf095e568e4473eda33f", |
| "val.jsonl": "e723c6a8ac3fb82f1a6ce411274f8e0bb7c1be2f4fa3e2b52f8831bddf81c060" |
| } |
| }, |
| "versions": { |
| "torch": "2.10.0+cu128", |
| "transformers": "4.56.2", |
| "trl": "0.22.2", |
| "peft": "0.19.1", |
| "unsloth": "2026.9.12", |
| "unsloth_zoo": "2026.9.8", |
| "datasets": "4.3.0", |
| "accelerate": "1.13.0" |
| }, |
| "final_train_loss": 0.12306493408386669, |
| "eval_loss_per_epoch": [ |
| 0.086934395134449, |
| 0.03893920034170151, |
| 0.030055643990635872 |
| ] |
| } |
| } |
