{
  "data": "/home/eai-ajzxf/dm/decision-model/data/merged_v2_bal.jsonl,/home/eai-ajzxf/dm/decision-model/data/open_open_tasksource.jsonl,/home/eai-ajzxf/dm/decision-model/data/open_ratings.jsonl,/home/eai-ajzxf/dm/decision-model/data/open_support_intents.jsonl",
  "model": "/home/eai-ajzxf/dm/models/ModernBERT-base-nli",
  "init_from": null,
  "domain_weights": {
    "generic": 3.0,
    "open_tasksource": 2.0
  },
  "seed": 42,
  "epochs": 3,
  "lr": 3e-05,
  "max_len": 1024,
  "batch_size": 16,
  "device": "cuda",
  "split_mode": "template",
  "exclude_domains": [],
  "split": {
    "train": 256920,
    "validation": 37978,
    "heldout": 0,
    "validation_template_ids": 1892
  },
  "class_weight": true,
  "majority_baseline": {
    "source": "train split",
    "validation": {
      "n": 35870,
      "accuracy": 0.40398661834402005,
      "n_unmapped": 2108
    },
    "heldout": null
  },
  "steps": 48174,
  "final_loss": 0.06356,
  "temperature": 3.986253,
  "metrics_validation_raw": {
    "n": 37978,
    "accuracy": 0.46753383537837695,
    "balanced_accuracy": 0.45502415494688436,
    "ece": 0.2291721958145001,
    "nll": 1.392849499783441
  },
  "metrics_validation_calibrated": {
    "n": 37978,
    "accuracy": 0.46753383537837695,
    "balanced_accuracy": 0.45502415494688436,
    "ece": 0.04692186695970413,
    "nll": 1.0603329397388084
  },
  "validation_by_domain": {
    "generic": {
      "n": 4888,
      "accuracy": 0.717266775777414,
      "balanced_accuracy": 0.6540912238268553,
      "ece": 0.19083873238932775,
      "nll": 0.8106594818093334
    },
    "open_tasksource": {
      "n": 8936,
      "accuracy": 0.40689346463742165,
      "balanced_accuracy": 0.46969241031024717,
      "ece": 0.06341728585252776,
      "nll": 1.071087839243584
    },
    "routing": {
      "n": 6577,
      "accuracy": 0.36658050783031776,
      "balanced_accuracy": 0.4599181298323277,
      "ece": 0.1523182318985272,
      "nll": 1.2406663806953009
    },
    "shopping": {
      "n": 5558,
      "accuracy": 0.6765023389708529,
      "balanced_accuracy": 0.6502757329467332,
      "ece": 0.20365982418246473,
      "nll": 0.9283356470128373
    },
    "support": {
      "n": 6131,
      "accuracy": 0.21497308758766923,
      "balanced_accuracy": 0.2544055306860031,
      "ece": 0.27600667550928965,
      "nll": 1.2674843568678904
    },
    "tool_risk": {
      "n": 5457,
      "accuracy": 0.5041231445849368,
      "balanced_accuracy": 0.3976298867419834,
      "ece": 0.05821240913284683,
      "nll": 0.991137123980093
    },
    "travel": {
      "n": 431,
      "accuracy": 0.8677494199535963,
      "balanced_accuracy": 0.9055462327315874,
      "ece": 0.2595401152518952,
      "nll": 0.5485979475182198
    }
  },
  "heldout_domain": "none",
  "validation_by_question_type": {
    "choice": {
      "n": 17528,
      "accuracy": 0.4361592879963487,
      "balanced_accuracy": 0.5408672503546958
    },
    "noul": {
      "n": 10052,
      "accuracy": 0.6830481496219658,
      "balanced_accuracy": 0.7107085773844286
    },
    "score": {
      "n": 10398,
      "accuracy": 0.31207924600884784,
      "balanced_accuracy": 0.33391884754359086
    }
  }
}