| { |
| "data": "/home/eai-ajzxf/dm/decision-model/data/merged_v2_bal.jsonl,/home/eai-ajzxf/dm/decision-model/data/open_open_tasksource.jsonl,/home/eai-ajzxf/dm/decision-model/data/open_ratings.jsonl,/home/eai-ajzxf/dm/decision-model/data/open_support_intents.jsonl", |
| "model": "/home/eai-ajzxf/dm/models/ModernBERT-base-nli", |
| "init_from": null, |
| "domain_weights": { |
| "generic": 3.0, |
| "open_tasksource": 2.0 |
| }, |
| "seed": 42, |
| "epochs": 3, |
| "lr": 3e-05, |
| "max_len": 1024, |
| "batch_size": 16, |
| "device": "cuda", |
| "split_mode": "template", |
| "exclude_domains": [], |
| "split": { |
| "train": 256920, |
| "validation": 37978, |
| "heldout": 0, |
| "validation_template_ids": 1892 |
| }, |
| "class_weight": true, |
| "majority_baseline": { |
| "source": "train split", |
| "validation": { |
| "n": 35870, |
| "accuracy": 0.40398661834402005, |
| "n_unmapped": 2108 |
| }, |
| "heldout": null |
| }, |
| "steps": 48174, |
| "final_loss": 0.06356, |
| "temperature": 3.986253, |
| "metrics_validation_raw": { |
| "n": 37978, |
| "accuracy": 0.46753383537837695, |
| "balanced_accuracy": 0.45502415494688436, |
| "ece": 0.2291721958145001, |
| "nll": 1.392849499783441 |
| }, |
| "metrics_validation_calibrated": { |
| "n": 37978, |
| "accuracy": 0.46753383537837695, |
| "balanced_accuracy": 0.45502415494688436, |
| "ece": 0.04692186695970413, |
| "nll": 1.0603329397388084 |
| }, |
| "validation_by_domain": { |
| "generic": { |
| "n": 4888, |
| "accuracy": 0.717266775777414, |
| "balanced_accuracy": 0.6540912238268553, |
| "ece": 0.19083873238932775, |
| "nll": 0.8106594818093334 |
| }, |
| "open_tasksource": { |
| "n": 8936, |
| "accuracy": 0.40689346463742165, |
| "balanced_accuracy": 0.46969241031024717, |
| "ece": 0.06341728585252776, |
| "nll": 1.071087839243584 |
| }, |
| "routing": { |
| "n": 6577, |
| "accuracy": 0.36658050783031776, |
| "balanced_accuracy": 0.4599181298323277, |
| "ece": 0.1523182318985272, |
| "nll": 1.2406663806953009 |
| }, |
| "shopping": { |
| "n": 5558, |
| "accuracy": 0.6765023389708529, |
| "balanced_accuracy": 0.6502757329467332, |
| "ece": 0.20365982418246473, |
| "nll": 0.9283356470128373 |
| }, |
| "support": { |
| "n": 6131, |
| "accuracy": 0.21497308758766923, |
| "balanced_accuracy": 0.2544055306860031, |
| "ece": 0.27600667550928965, |
| "nll": 1.2674843568678904 |
| }, |
| "tool_risk": { |
| "n": 5457, |
| "accuracy": 0.5041231445849368, |
| "balanced_accuracy": 0.3976298867419834, |
| "ece": 0.05821240913284683, |
| "nll": 0.991137123980093 |
| }, |
| "travel": { |
| "n": 431, |
| "accuracy": 0.8677494199535963, |
| "balanced_accuracy": 0.9055462327315874, |
| "ece": 0.2595401152518952, |
| "nll": 0.5485979475182198 |
| } |
| }, |
| "heldout_domain": "none", |
| "validation_by_question_type": { |
| "choice": { |
| "n": 17528, |
| "accuracy": 0.4361592879963487, |
| "balanced_accuracy": 0.5408672503546958 |
| }, |
| "noul": { |
| "n": 10052, |
| "accuracy": 0.6830481496219658, |
| "balanced_accuracy": 0.7107085773844286 |
| }, |
| "score": { |
| "n": 10398, |
| "accuracy": 0.31207924600884784, |
| "balanced_accuracy": 0.33391884754359086 |
| } |
| } |
| } |
|
|
