Hugging Face's logo

sup2ch
/
wex-v0.1

Russian
English
wex_decision_core
numpy
decision-model
agent-routing
experimental
abstention
Model card Files
xet
 Community
wex-v0.1
File size: 4,671 Bytes
1212aed
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
{
  "scope": "Synthetic current-prefix acceptable-action accuracy; not end-to-end agent utility",
  "human_gold": false,
  "validation": {
    "lexical128": {
      "n": 64,
      "correct": 43,
      "acceptable_accuracy": 0.671875,
      "false_abstain": 19,
      "false_abstain_rate": 0.8260869565217391,
      "missed_abstain": 1,
      "missed_abstain_rate": 0.024390243902439025
    },
    "lexical32": {
      "n": 64,
      "correct": 41,
      "acceptable_accuracy": 0.640625,
      "false_abstain": 20,
      "false_abstain_rate": 0.8695652173913043,
      "missed_abstain": 3,
      "missed_abstain_rate": 0.07317073170731707
    },
    "lexical64": {
      "n": 64,
      "correct": 37,
      "acceptable_accuracy": 0.578125,
      "false_abstain": 20,
      "false_abstain_rate": 0.8695652173913043,
      "missed_abstain": 7,
      "missed_abstain_rate": 0.17073170731707318
    },
    "semantic128": {
      "n": 64,
      "correct": 55,
      "acceptable_accuracy": 0.859375,
      "false_abstain": 6,
      "false_abstain_rate": 0.2608695652173913,
      "missed_abstain": 1,
      "missed_abstain_rate": 0.024390243902439025
    }
  },
  "fresh_test_paired_all64": {
    "baseline_correct": 54,
    "candidate_correct": 44,
    "baseline_accuracy": 0.84375,
    "candidate_accuracy": 0.6875,
    "accuracy_difference": -0.15625,
    "paired_bootstrap95": [
      -0.25,
      -0.078125
    ],
    "bootstrap_replicates": 10000,
    "bootstrap_seed": 101,
    "gained_correct_worlds": 0,
    "lost_correct_worlds": 10,
    "select_denominator": 20,
    "baseline_false_abstain": 5,
    "candidate_false_abstain": 5,
    "baseline_false_abstain_rate": 0.25,
    "candidate_false_abstain_rate": 0.25,
    "abstain_denominator": 44,
    "baseline_missed_abstain": 3,
    "candidate_missed_abstain": 10,
    "baseline_missed_abstain_rate": 0.06818181818181818,
    "candidate_missed_abstain_rate": 0.22727272727272727,
    "baseline_conditional_tool_errors": 4,
    "candidate_conditional_tool_errors": 5,
    "baseline_mean_joint_target_set_nll": 0.9775434669867754,
    "candidate_mean_joint_target_set_nll": 3.0828047417095714,
    "additional_false_abstain_worlds": [
      "gec1_dw010"
    ]
  },
  "fresh_test_gate_checks": {
    "accuracy_at_least_80_percent": false,
    "false_abstain_at_most_10_percent": false,
    "missed_abstain_at_most_10_percent": false,
    "both_classes_nonzero": true,
    "extra_correct_at_least_4": false,
    "no_added_false_abstain_worlds": false,
    "candidate_gate_passed": false,
    "baseline_also_fails_false_abstain_gate": true
  },
  "test_independence": {
    "archive_committed_before_fresh_labels": true,
    "labels_committed_before_predictions": true,
    "first_use_of_reserved_fresh64": true,
    "raw_author": "Existing gpt-6-astra, unchanged role",
    "source_adapter": "gpt-6.1-sol high, source only",
    "expert": "gpt-6-astra medium, fresh context, individual current-prefix manual labels",
    "expert_prediction_access": false,
    "human_gold": false,
    "PASS": 64,
    "HOLD": 0,
    "excluded_worlds": 0,
    "semantic_relabels": 0,
    "threshold_searches": 0,
    "replays": 0
  },
  "published_weights": "semantic128 (previous reference; not rejected candidate384)",
  "published_weights_sha256": "645a82c68576b207723a21ee413edbc9e69980a41520cdcb475bf28455c94887",
  "training": {
    "HOLD": 2,
    "PASS": 126,
    "acceptable_target_coverage": {
      "list_directory": 7,
      "read_file": 21,
      "search_content": 30,
      "search_files": 13
    },
    "archive_sha256": "645a82c68576b207723a21ee413edbc9e69980a41520cdcb475bf28455c94887",
    "arm": "semantic128",
    "converged": false,
    "optimizer_receipt_sha256": "9583dd8b0f65c4a388b4e30132d2eaba820e14b2bb9e6ee04e3b02880cb5d5a8",
    "optimizer_stop": "evaluation_budget_exhausted",
    "optimizer_trace_sha256": "ea2462e18f60755348689bc450f3dba96c5768448f1a7a913cedc31a5f93e37c",
    "seconds": 559.4276709999995,
    "target_abstain": 91,
    "target_select": 37,
    "train_count": 128,
    "train_predictions_sha256": "08bbcd8fa0728e48bcc50825049f42b1e770716a8850e3b8d0d73290869da518",
    "training_fingerprint": "33a7f43795a6ad43bbfdb9cf9a2f6602dcb14be3c21c95fee189514ac292c4c8"
  },
  "probabilities": "uncalibrated_softmax; no threshold search",
  "test_policy": "Historical test is public and closed. Do not use it for training, selection, threshold tuning or relabeling and then report independent test results on it.",
  "new_quality_evaluation_or_training_for_release": false
}