eval.json

{
  "system": "tuned",
  "model": "unsloth/gemma-3-270m-it+lora:v1",
  "created": "2026-10-01T03:14:17",
  "splits": {
    "val": {
      "n": 150,
      "intent_accuracy": 0.9933,
      "json_validity_rate": 0.9933,
      "item_precision": 0.9725,
      "item_recall": 0.9538,
      "item_f1": 0.9631,
      "quantity_accuracy": 0.9725,
      "part_number_accuracy": 1.0,
      "compatible_with_accuracy": 0.8696,
      "missing_info_precision": 0.1613,
      "missing_info_recall": 0.0909,
      "missing_info_f1": 0.1163,
      "urgency_accuracy": 0.98,
      "exact_match_rate": 0.5733,
      "per_intent_accuracy": {
        "order_status": 1.0,
        "other": 1.0,
        "product_question": 1.0,
        "quote_request": 0.9895833333333334,
        "return_request": 1.0
      },
      "per_intent_n": {
        "order_status": 16,
        "other": 11,
        "product_question": 14,
        "quote_request": 96,
        "return_request": 13
      },
      "latency_p50_s": 1.2402,
      "latency_p95_s": 1.7012,
      "escalation_rate": null,
      "cost_per_1k_usd": null,
      "wall_time_s": 0.0,
      "errors": 0
    },
    "exam": {
      "n": 150,
      "intent_accuracy": 0.98,
      "json_validity_rate": 0.9867,
      "item_precision": 0.9266,
      "item_recall": 0.916,
      "item_f1": 0.9213,
      "quantity_accuracy": 0.9339,
      "part_number_accuracy": 0.9821,
      "compatible_with_accuracy": 0.84,
      "missing_info_precision": 0.1389,
      "missing_info_recall": 0.1316,
      "missing_info_f1": 0.1351,
      "urgency_accuracy": 0.9733,
      "exact_match_rate": 0.6,
      "per_intent_accuracy": {
        "order_status": 1.0,
        "other": 1.0,
        "product_question": 0.9285714285714286,
        "quote_request": 0.9791666666666666,
        "return_request": 1.0
      },
      "per_intent_n": {
        "order_status": 16,
        "other": 11,
        "product_question": 14,
        "quote_request": 96,
        "return_request": 13
      },
      "latency_p50_s": 1.0266,
      "latency_p95_s": 2.0577,
      "escalation_rate": null,
      "cost_per_1k_usd": null,
      "wall_time_s": 0.0,
      "errors": 0
    },
    "exam_hard": {
      "n": 30,
      "intent_accuracy": 0.6,
      "json_validity_rate": 0.6667,
      "item_precision": 0.7436,
      "item_recall": 0.6591,
      "item_f1": 0.6988,
      "quantity_accuracy": 0.8286,
      "part_number_accuracy": 1.0,
      "compatible_with_accuracy": 1.0,
      "missing_info_precision": 0.0,
      "missing_info_recall": 0.0,
      "missing_info_f1": 0.0,
      "urgency_accuracy": 0.9333,
      "exact_match_rate": 0.2667,
      "per_intent_accuracy": {
        "order_status": 0.5,
        "product_question": 1.0,
        "quote_request": 0.56
      },
      "per_intent_n": {
        "order_status": 2,
        "product_question": 3,
        "quote_request": 25
      },
      "latency_p50_s": 0.7219,
      "latency_p95_s": 1.2877,
      "escalation_rate": null,
      "cost_per_1k_usd": null,
      "wall_time_s": 0.0,
      "errors": 0
    }
  },
  "pricing": {
    "teacher": {
      "model": "gemini-3.8-flash",
      "input_per_1m_usd": 0.75,
      "output_per_1m_usd": 3.75,
      "note": "Google's published paid-tier price (standard), thinking tokens billed as output; valid through 2026-12-31. Our calls go through the KIE.ai proxy; credits actually spent are logged separately.",
      "source": "https://ai.google.dev/gemini-api/docs/pricing",
      "verified": "2026-09-30",
      "label": "measured tokens x published paid-tier price"
    },
    "student_cpu": {
      "basis": "Modal CPU container, 2 cores + 4 GiB: 2 x $0.0000131/core/s + 4 x $0.00000222/GiB/s",
      "usd_per_second": 3.51e-05,
      "source": "https://modal.com/pricing",
      "verified": "2026-09-30",
      "label": "estimate: inference seconds x container price (excludes idle time)"
    }
  },
  "adapter_version": "v1",
  "hardware": "Tesla T4",
  "latency_note": "GPU batch latency; CPU latency is measured separately in eval/run_eval.py",
  "run_config": {
    "version_tag": "v1",
    "base_model": "unsloth/gemma-3-270m-it",
    "epochs": 3,
    "learning_rate": 0.0002,
    "lora_r": 16,
    "lora_alpha": 32,
    "lora_dropout": 0.0,
    "batch_size": 8,
    "grad_accum": 2,
    "warmup_ratio": 0.05,
    "seed": 3407,
    "max_seq_len": 640,
    "max_new_tokens": 384,
    "train_examples": 1334,
    "corrections": 0,
    "train_seconds": 564.4,
    "gpu": "Tesla T4",
    "platform": "kaggle",
    "dataset_manifest": {
      "version_tag": "v1",
      "created": "2026-09-30T21:42:39",
      "git_sha": null,
      "counts": {
        "train.jsonl": 1334,
        "val.jsonl": 150,
        "exam.jsonl": 150,
        "exam_hard.jsonl": 30,
        "fewshot.jsonl": 3
      },
      "corrections": 0,
      "sha256": {
        "catalog.json": "b354cf4e5b81f6be80c5071c0961b0ee79c9c8f9cec86bd4e0102f819e22c021",
        "customers.json": "b650b859b37bd2c3c33c703aaea9e227cf186558789e7e88f5fd53b7352a1bef",
        "dataset-metadata.json": "b483500aa344633dc1614e132d976cee59073c4c91003f0977f3c71ecf19a875",
        "equipment.json": "91e341753fee46d62b9444747d177a670b308999743de1fddd6079a596c56e2c",
        "evallib/__init__.py": "ec38b6ac328f58d03f3425148201628598e7521691faf7d34a1c326e7d7aa8f8",
        "evallib/metrics.py": "7bef5b64f15ba6fc0ba877daa6d57294da373061543c984023af9c59edce5a37",
        "evallib/normalize.py": "0c663091219f7a14baa37307863c8ea9e066b87e46f045abc38e3176f13069a3",
        "evallib/pricing.json": "782918cf5cdbf2b69d85873028e0bfa9bbafc89cda6828cb9c933ca3bdf96f5f",
        "evallib/prompt.py": "521bdb70ba03d77436c32b82edf3a7a8f6f44b0ccadade37416ec3db786ffdad",
        "evallib/runner.py": "3992d9971ab4d21c707964650fb5d09e98123a4adabb329160a8d8eacebec744",
        "evallib/schema.py": "eec543089b10c394dec74de147b2b238337657c037f69e72caa48244a482e27e",
        "evallib/system_prompt.txt": "fc0d3583b99fc4dfe3ec3b42f5fe3bfbabbccc23f9aec7edc3771eea5adbbc03",
        "evallib/teacher_rules.txt": "20d48a4ea874fa5c3f0c5eb69e81f4796aaa6dae40ffc89bb30eeca3b232633d",
        "exam.jsonl": "dc1100cffd0bbdfd8089f0841f455f753eae50ad6fcd11ba2ad1dca830287584",
        "exam_hard.jsonl": "bea23f278853cf5b6803df77e2b87cd62175333f8dec3235c779d4c9d4852bd0",
        "fewshot.jsonl": "f4d148125fe418cd0735e20f326d3df4a2319e8084abebd4eee371c6bc34db91",
        "orders.json": "37dea4e9b8026315814840acf0cd4557d921c4e5b30bf856def22c01e99a285c",
        "train.jsonl": "63578645ec0a3d53de85255a888d8855710b218e31a4bf095e568e4473eda33f",
        "val.jsonl": "e723c6a8ac3fb82f1a6ce411274f8e0bb7c1be2f4fa3e2b52f8831bddf81c060"
      }
    },
    "versions": {
      "torch": "2.10.0+cu128",
      "transformers": "4.56.2",
      "trl": "0.22.2",
      "peft": "0.19.1",
      "unsloth": "2026.9.12",
      "unsloth_zoo": "2026.9.8",
      "datasets": "4.3.0",
      "accelerate": "1.13.0"
    },
    "final_train_loss": 0.12306493408386669,
    "eval_loss_per_epoch": [
      0.086934395134449,
      0.03893920034170151,
      0.030055643990635872
    ]
  }
}