{
  "f1_window": 0.04,
  "calibration_role": "ntdb/artifacts/experiments/sensitive_joint_v3_mmbert_static_actionable_benefit_v41_all_sensitive_aps_phase2_l3_seed42/split_indices/probability_calibration.npy",
  "policy_selection_role": "ntdb/artifacts/experiments/sensitive_joint_v3_mmbert_static_actionable_benefit_v41_all_sensitive_aps_phase2_l3_seed42/split_indices/policy_selection.npy",
  "evaluation_role": "unified_validation",
  "modes": {
    "l2_only": {
      "aggregation": "mean",
      "f1_window": 0.04,
      "calibration_operating_points": {
        "best_f1": {
          "document_risk_margin_threshold": 0.3269468620419502,
          "macro_f1": 0.8212544747840629,
          "fpr": 0.10416666666666667,
          "fnr": 0.0684931506849315,
          "tp": 340,
          "fp": 5,
          "tn": 43,
          "fn": 25
        },
        "best_fpr_in_f1": {
          "document_risk_margin_threshold": 0.2802514135837555,
          "macro_f1": 0.8208552098917432,
          "fpr": 0.10416666666666667,
          "fnr": 0.06575342465753424,
          "tp": 341,
          "fp": 5,
          "tn": 43,
          "fn": 24
        },
        "best_fnr_in_f1": {
          "document_risk_margin_threshold": -0.7966152094304562,
          "macro_f1": 0.8119645317052127,
          "fpr": 0.25,
          "fnr": 0.005479452054794521,
          "tp": 363,
          "fp": 12,
          "tn": 36,
          "fn": 2
        }
      },
      "sweep_threshold_count": 414,
      "policy_selection": {
        "best_f1": {
          "document_risk_margin_threshold": 0.3269468620419502,
          "macro_f1": 0.8212544747840629,
          "fpr": 0.10416666666666667,
          "fnr": 0.0684931506849315,
          "tp": 340,
          "fp": 5,
          "tn": 43,
          "fn": 25,
          "evaluation_metrics": {
            "macro_f1": 0.7908984301290851,
            "fpr": 0.2127659574468085,
            "fnr": 0.08719346049046321,
            "tp": 335,
            "fp": 10,
            "tn": 37,
            "fn": 32
          }
        },
        "best_fpr_in_f1": {
          "document_risk_margin_threshold": 0.2802514135837555,
          "macro_f1": 0.8208552098917432,
          "fpr": 0.10416666666666667,
          "fnr": 0.06575342465753424,
          "tp": 341,
          "fp": 5,
          "tn": 43,
          "fn": 24,
          "evaluation_metrics": {
            "macro_f1": 0.7908984301290851,
            "fpr": 0.2127659574468085,
            "fnr": 0.08719346049046321,
            "tp": 335,
            "fp": 10,
            "tn": 37,
            "fn": 32
          }
        },
        "best_fnr_in_f1": {
          "document_risk_margin_threshold": -0.7966152094304562,
          "macro_f1": 0.8119645317052127,
          "fpr": 0.25,
          "fnr": 0.005479452054794521,
          "tp": 363,
          "fp": 12,
          "tn": 36,
          "fn": 2,
          "evaluation_metrics": {
            "macro_f1": 0.7779886588170615,
            "fpr": 0.40425531914893614,
            "fnr": 0.02452316076294278,
            "tp": 358,
            "fp": 19,
            "tn": 28,
            "fn": 9
          }
        }
      },
      "evaluation": {
        "unified_validation": {
          "best_f1": {
            "document_risk_margin_threshold": 0.3269468620419502,
            "macro_f1": 0.8212544747840629,
            "fpr": 0.10416666666666667,
            "fnr": 0.0684931506849315,
            "tp": 340,
            "fp": 5,
            "tn": 43,
            "fn": 25,
            "evaluation_metrics": {
              "macro_f1": 0.8748585804941437,
              "fpr": 0.1459227467811159,
              "fnr": 0.030534351145038167,
              "tp": 1778,
              "fp": 34,
              "tn": 199,
              "fn": 56
            }
          },
          "best_fpr_in_f1": {
            "document_risk_margin_threshold": 0.2802514135837555,
            "macro_f1": 0.8208552098917432,
            "fpr": 0.10416666666666667,
            "fnr": 0.06575342465753424,
            "tp": 341,
            "fp": 5,
            "tn": 43,
            "fn": 24,
            "evaluation_metrics": {
              "macro_f1": 0.8752677517950971,
              "fpr": 0.1459227467811159,
              "fnr": 0.028353326063249727,
              "tp": 1782,
              "fp": 34,
              "tn": 199,
              "fn": 52
            }
          },
          "best_fnr_in_f1": {
            "document_risk_margin_threshold": -0.7966152094304562,
            "macro_f1": 0.8119645317052127,
            "fpr": 0.25,
            "fnr": 0.005479452054794521,
            "tp": 363,
            "fp": 12,
            "tn": 36,
            "fn": 2,
            "evaluation_metrics": {
              "macro_f1": 0.8693104068945013,
              "fpr": 0.296137339055794,
              "fnr": 0.008178844056706653,
              "tp": 1819,
              "fp": 69,
              "tn": 164,
              "fn": 15
            }
          }
        }
      },
      "operating_points": {
        "best_f1": {
          "threshold": 0.3269468620419502,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": 0.3269468620419502,
            "macro_f1": 0.8212544747840629,
            "fpr": 0.10416666666666667,
            "fnr": 0.0684931506849315,
            "tp": 340,
            "fp": 5,
            "tn": 43,
            "fn": 25
          },
          "policy_selection_metrics": {
            "macro_f1": 0.7908984301290851,
            "fpr": 0.2127659574468085,
            "fnr": 0.08719346049046321,
            "tp": 335,
            "fp": 10,
            "tn": 37,
            "fn": 32
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.8748585804941437,
              "fpr": 0.1459227467811159,
              "fnr": 0.030534351145038167,
              "tp": 1778,
              "fp": 34,
              "tn": 199,
              "fn": 56
            }
          }
        },
        "best_fpr_in_f1": {
          "threshold": 0.2802514135837555,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": 0.2802514135837555,
            "macro_f1": 0.8208552098917432,
            "fpr": 0.10416666666666667,
            "fnr": 0.06575342465753424,
            "tp": 341,
            "fp": 5,
            "tn": 43,
            "fn": 24
          },
          "policy_selection_metrics": {
            "macro_f1": 0.7908984301290851,
            "fpr": 0.2127659574468085,
            "fnr": 0.08719346049046321,
            "tp": 335,
            "fp": 10,
            "tn": 37,
            "fn": 32
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.8752677517950971,
              "fpr": 0.1459227467811159,
              "fnr": 0.028353326063249727,
              "tp": 1782,
              "fp": 34,
              "tn": 199,
              "fn": 52
            }
          }
        },
        "best_fnr_in_f1": {
          "threshold": -0.7966152094304562,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.7966152094304562,
            "macro_f1": 0.8119645317052127,
            "fpr": 0.25,
            "fnr": 0.005479452054794521,
            "tp": 363,
            "fp": 12,
            "tn": 36,
            "fn": 2
          },
          "policy_selection_metrics": {
            "macro_f1": 0.7779886588170615,
            "fpr": 0.40425531914893614,
            "fnr": 0.02452316076294278,
            "tp": 358,
            "fp": 19,
            "tn": 28,
            "fn": 9
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.8693104068945013,
              "fpr": 0.296137339055794,
              "fnr": 0.008178844056706653,
              "tp": 1819,
              "fp": 69,
              "tn": 164,
              "fn": 15
            }
          }
        }
      }
    },
    "l3_only": {
      "aggregation": "smoothmax",
      "f1_window": 0.04,
      "calibration_operating_points": {
        "best_f1": {
          "document_risk_margin_threshold": -0.22447717189788818,
          "macro_f1": 0.995257067546521,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0
        },
        "best_fpr_in_f1": {
          "document_risk_margin_threshold": -0.22447717189788818,
          "macro_f1": 0.995257067546521,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0
        },
        "best_fnr_in_f1": {
          "document_risk_margin_threshold": -0.22447717189788818,
          "macro_f1": 0.995257067546521,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0
        }
      },
      "sweep_threshold_count": 415,
      "policy_selection": {
        "best_f1": {
          "document_risk_margin_threshold": -0.22447717189788818,
          "macro_f1": 0.995257067546521,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0,
          "evaluation_metrics": {
            "macro_f1": 0.9879752997730525,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 367,
            "fp": 0,
            "tn": 47,
            "fn": 0
          }
        },
        "best_fpr_in_f1": {
          "document_risk_margin_threshold": -0.22447717189788818,
          "macro_f1": 0.995257067546521,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0,
          "evaluation_metrics": {
            "macro_f1": 0.9879752997730525,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 367,
            "fp": 0,
            "tn": 47,
            "fn": 0
          }
        },
        "best_fnr_in_f1": {
          "document_risk_margin_threshold": -0.22447717189788818,
          "macro_f1": 0.995257067546521,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0,
          "evaluation_metrics": {
            "macro_f1": 0.9879752997730525,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 367,
            "fp": 0,
            "tn": 47,
            "fn": 0
          }
        }
      },
      "evaluation": {
        "unified_validation": {
          "best_f1": {
            "document_risk_margin_threshold": -0.22447717189788818,
            "macro_f1": 0.995257067546521,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0,
            "evaluation_metrics": {
              "macro_f1": 0.9680697934798839,
              "fpr": 0.060085836909871244,
              "fnr": 0.006543075245365322,
              "tp": 1822,
              "fp": 14,
              "tn": 219,
              "fn": 12
            }
          },
          "best_fpr_in_f1": {
            "document_risk_margin_threshold": -0.22447717189788818,
            "macro_f1": 0.995257067546521,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0,
            "evaluation_metrics": {
              "macro_f1": 0.9680697934798839,
              "fpr": 0.060085836909871244,
              "fnr": 0.006543075245365322,
              "tp": 1822,
              "fp": 14,
              "tn": 219,
              "fn": 12
            }
          },
          "best_fnr_in_f1": {
            "document_risk_margin_threshold": -0.22447717189788818,
            "macro_f1": 0.995257067546521,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0,
            "evaluation_metrics": {
              "macro_f1": 0.9680697934798839,
              "fpr": 0.060085836909871244,
              "fnr": 0.006543075245365322,
              "tp": 1822,
              "fp": 14,
              "tn": 219,
              "fn": 12
            }
          }
        }
      },
      "operating_points": {
        "best_f1": {
          "threshold": -0.22447717189788818,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.22447717189788818,
            "macro_f1": 0.995257067546521,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0
          },
          "policy_selection_metrics": {
            "macro_f1": 0.9879752997730525,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 367,
            "fp": 0,
            "tn": 47,
            "fn": 0
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.9680697934798839,
              "fpr": 0.060085836909871244,
              "fnr": 0.006543075245365322,
              "tp": 1822,
              "fp": 14,
              "tn": 219,
              "fn": 12
            }
          }
        },
        "best_fpr_in_f1": {
          "threshold": -0.22447717189788818,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.22447717189788818,
            "macro_f1": 0.995257067546521,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0
          },
          "policy_selection_metrics": {
            "macro_f1": 0.9879752997730525,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 367,
            "fp": 0,
            "tn": 47,
            "fn": 0
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.9680697934798839,
              "fpr": 0.060085836909871244,
              "fnr": 0.006543075245365322,
              "tp": 1822,
              "fp": 14,
              "tn": 219,
              "fn": 12
            }
          }
        },
        "best_fnr_in_f1": {
          "threshold": -0.22447717189788818,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.22447717189788818,
            "macro_f1": 0.995257067546521,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0
          },
          "policy_selection_metrics": {
            "macro_f1": 0.9879752997730525,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 367,
            "fp": 0,
            "tn": 47,
            "fn": 0
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.9680697934798839,
              "fpr": 0.060085836909871244,
              "fnr": 0.006543075245365322,
              "tp": 1822,
              "fp": 14,
              "tn": 219,
              "fn": 12
            }
          }
        }
      }
    },
    "union_l2_l3": {
      "aggregation": "mean",
      "f1_window": 0.04,
      "calibration_operating_points": {
        "best_f1": {
          "document_risk_margin_threshold": -0.19841083884239197,
          "macro_f1": 0.9810854243276995,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0
        },
        "best_fpr_in_f1": {
          "document_risk_margin_threshold": -0.19841083884239197,
          "macro_f1": 0.9810854243276995,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0
        },
        "best_fnr_in_f1": {
          "document_risk_margin_threshold": -0.19841083884239197,
          "macro_f1": 0.9810854243276995,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0
        }
      },
      "sweep_threshold_count": 414,
      "policy_selection": {
        "best_f1": {
          "document_risk_margin_threshold": -0.19841083884239197,
          "macro_f1": 0.9810854243276995,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0,
          "evaluation_metrics": {
            "macro_f1": 0.9736874663611368,
            "fpr": 0.0425531914893617,
            "fnr": 0.005449591280653951,
            "tp": 365,
            "fp": 2,
            "tn": 45,
            "fn": 2
          }
        },
        "best_fpr_in_f1": {
          "document_risk_margin_threshold": -0.19841083884239197,
          "macro_f1": 0.9810854243276995,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0,
          "evaluation_metrics": {
            "macro_f1": 0.9736874663611368,
            "fpr": 0.0425531914893617,
            "fnr": 0.005449591280653951,
            "tp": 365,
            "fp": 2,
            "tn": 45,
            "fn": 2
          }
        },
        "best_fnr_in_f1": {
          "document_risk_margin_threshold": -0.19841083884239197,
          "macro_f1": 0.9810854243276995,
          "fpr": 0.0,
          "fnr": 0.0,
          "tp": 365,
          "fp": 0,
          "tn": 48,
          "fn": 0,
          "evaluation_metrics": {
            "macro_f1": 0.9736874663611368,
            "fpr": 0.0425531914893617,
            "fnr": 0.005449591280653951,
            "tp": 365,
            "fp": 2,
            "tn": 45,
            "fn": 2
          }
        }
      },
      "evaluation": {
        "unified_validation": {
          "best_f1": {
            "document_risk_margin_threshold": -0.19841083884239197,
            "macro_f1": 0.9810854243276995,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0,
            "evaluation_metrics": {
              "macro_f1": 0.9518323126346822,
              "fpr": 0.09871244635193133,
              "fnr": 0.007088331515812432,
              "tp": 1821,
              "fp": 23,
              "tn": 210,
              "fn": 13
            }
          },
          "best_fpr_in_f1": {
            "document_risk_margin_threshold": -0.19841083884239197,
            "macro_f1": 0.9810854243276995,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0,
            "evaluation_metrics": {
              "macro_f1": 0.9518323126346822,
              "fpr": 0.09871244635193133,
              "fnr": 0.007088331515812432,
              "tp": 1821,
              "fp": 23,
              "tn": 210,
              "fn": 13
            }
          },
          "best_fnr_in_f1": {
            "document_risk_margin_threshold": -0.19841083884239197,
            "macro_f1": 0.9810854243276995,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0,
            "evaluation_metrics": {
              "macro_f1": 0.9518323126346822,
              "fpr": 0.09871244635193133,
              "fnr": 0.007088331515812432,
              "tp": 1821,
              "fp": 23,
              "tn": 210,
              "fn": 13
            }
          }
        }
      },
      "union_routing": {
        "promoter": "direct_actionable",
        "promoter_threshold": 0.004311159716500321,
        "calibration_target_promotion_rate": 0.25
      },
      "operating_points": {
        "best_f1": {
          "threshold": -0.19841083884239197,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.19841083884239197,
            "macro_f1": 0.9810854243276995,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0
          },
          "policy_selection_metrics": {
            "macro_f1": 0.9736874663611368,
            "fpr": 0.0425531914893617,
            "fnr": 0.005449591280653951,
            "tp": 365,
            "fp": 2,
            "tn": 45,
            "fn": 2
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.9518323126346822,
              "fpr": 0.09871244635193133,
              "fnr": 0.007088331515812432,
              "tp": 1821,
              "fp": 23,
              "tn": 210,
              "fn": 13
            }
          }
        },
        "best_fpr_in_f1": {
          "threshold": -0.19841083884239197,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.19841083884239197,
            "macro_f1": 0.9810854243276995,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0
          },
          "policy_selection_metrics": {
            "macro_f1": 0.9736874663611368,
            "fpr": 0.0425531914893617,
            "fnr": 0.005449591280653951,
            "tp": 365,
            "fp": 2,
            "tn": 45,
            "fn": 2
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.9518323126346822,
              "fpr": 0.09871244635193133,
              "fnr": 0.007088331515812432,
              "tp": 1821,
              "fp": 23,
              "tn": 210,
              "fn": 13
            }
          }
        },
        "best_fnr_in_f1": {
          "threshold": -0.19841083884239197,
          "threshold_semantics": "best_non_default_class_score_minus_default_class_score",
          "metrics": {
            "document_risk_margin_threshold": -0.19841083884239197,
            "macro_f1": 0.9810854243276995,
            "fpr": 0.0,
            "fnr": 0.0,
            "tp": 365,
            "fp": 0,
            "tn": 48,
            "fn": 0
          },
          "policy_selection_metrics": {
            "macro_f1": 0.9736874663611368,
            "fpr": 0.0425531914893617,
            "fnr": 0.005449591280653951,
            "tp": 365,
            "fp": 2,
            "tn": 45,
            "fn": 2
          },
          "evaluation_metrics": {
            "unified_validation": {
              "macro_f1": 0.9518323126346822,
              "fpr": 0.09871244635193133,
              "fnr": 0.007088331515812432,
              "tp": 1821,
              "fp": 23,
              "tn": 210,
              "fn": 13
            }
          }
        }
      }
    }
  }
}