seq/metrics.json

{
  "confusion_matrix": [
    [
      5271,
      629,
      2,
      5,
      0
    ],
    [
      1249,
      9239,
      22,
      31,
      60
    ],
    [
      0,
      2,
      424,
      0,
      0
    ],
    [
      0,
      2,
      1,
      1153,
      9
    ],
    [
      0,
      116,
      0,
      139,
      24669
    ]
  ],
  "inference_ms_500_actors": 40.95554997911677,
  "label_source": "msr_behavior_rules",
  "latency_critical_recall": 0.8923311325545963,
  "linear_probe_val_macro_f1": 0.779381297984736,
  "macro_f1": 0.9266784806704429,
  "macro_f1_holdout_traces": 0.9249428709644139,
  "model": "trace_transformer",
  "no_pretraining": {
    "test_macro_f1": 0.9263400969541534,
    "val_macro_f1": 0.9080818370381223
  },
  "parameters": 74345,
  "per_class": {
    "interactive": {
      "f1-score": 0.8974695225605906,
      "precision": 0.9250100120144173,
      "recall": 0.8715215545703235,
      "support": 10601.0
    },
    "latency_critical": {
      "f1-score": 0.8483141546632332,
      "precision": 0.8084355828220859,
      "recall": 0.8923311325545963,
      "support": 5907.0
    },
    "maintenance": {
      "f1-score": 0.9249899719213799,
      "precision": 0.8682228915662651,
      "recall": 0.9896995708154507,
      "support": 1165.0
    },
    "system": {
      "f1-score": 0.9934758970641536,
      "precision": 0.9972107688576279,
      "recall": 0.9897688974482427,
      "support": 24924.0
    },
    "throughput": {
      "f1-score": 0.9691428571428572,
      "precision": 0.9443207126948775,
      "recall": 0.9953051643192489,
      "support": 426.0
    }
  },
  "pretraining": {
    "most_common_guess_accuracy": {
      "gap": 0.45044488777088176,
      "jump": 0.2988369372450046,
      "op": 0.4522169629124055,
      "size": 0.4318893228319422
    },
    "most_common_guess_loss_nats": 5.93662669849005,
    "val_next_request_accuracy": {
      "gap": 0.6915149442696212,
      "jump": 0.5408996327097383,
      "op": 0.981587031732605,
      "size": 0.8324932128137645
    },
    "val_next_request_loss_nats": 2.7761730839711705
  },
  "rf_baseline": {
    "agreement_with_transformer": 0.947376984403691,
    "latency_critical_recall": 0.999661418655832,
    "macro_f1": 0.9997413593139894
  },
  "rows": {
    "pretrain_sequences": 117170,
    "test": 43023,
    "train": 64189,
    "val": 15443
  },
  "targets": {
    "latency_critical_recall": 0.9,
    "macro_f1": 0.85
  },
  "targets_missed": [
    "latency_critical_recall"
  ],
  "train_dataset": "msr",
  "val_macro_f1": 0.9028091410943446
}