Hugging Face's logo

macmacmacmac
/
Sev-4B

PEFT
Safetensors
English
sev
research
cybersecurity
agent-activity
decision-model
lora
Model card Files
xet
 Community
Sev-4B
File size: 7,208 Bytes
da0a131
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
{
  "protocol_sha256": "23ca33b71c7723cb6c67586e7805b74a0988ca5f8b7139b637420bbb5ec89d61",
  "comparison_sha256": "5b6144be9d2325b224acf29be30940241f3c75c2cf35b1d4d910458ce2aa580b",
  "dns_comparison_sha256": "3978d60071fd70c519a6bc8240eaa5b043dd136a486850789cc14a3c16b75462",
  "checks": {
    "policy_false_alert_rate": false,
    "policy_useful_change": true,
    "dns_no_regression": false,
    "development/excytin_file": true,
    "development/excytin_parent_join": true,
    "development/excytin_process": true,
    "development/excytin_signin": true,
    "development/sev_guide_detector": true,
    "development/sev_guide_triage": true,
    "development/sev_swarmtraces_static": true,
    "development/sev_sysmon_native_file": true,
    "development/sev_sysmon_native_join": true,
    "development/sev_sysmon_native_other_effect": true,
    "development/sev_sysmon_native_process": true,
    "development/sev_sysmon_native_registry": true,
    "transfer/composition_holdout": true,
    "transfer/emotion": true,
    "transfer/legacy_holdout": true,
    "transfer/mmlu": true,
    "transfer/otrf_sysmon_file": true,
    "transfer/otrf_sysmon_join": true,
    "transfer/otrf_sysmon_process": true,
    "transfer/otrf_sysmon_registry": true,
    "transfer/paws": true,
    "transfer/qnli": true,
    "transfer/sciq": true,
    "transfer/sev_swarmtraces_manual_v1": true,
    "transfer/tweet_offensive": true
  },
  "passed": false,
  "failed": [
    "policy_false_alert_rate",
    "dns_no_regression"
  ],
  "retained_development": [
    {
      "source": "excytin_file",
      "questions": 80,
      "parent_correct": 80,
      "candidate_correct": 80,
      "parent_calibrated_nll": 0.0030639977354718724,
      "candidate_calibrated_nll": 0.0025800477441721577,
      "parent_calibrated_brier": 5.723205099455454e-05,
      "candidate_calibrated_brier": 4.4624375778219304e-05
    },
    {
      "source": "excytin_parent_join",
      "questions": 78,
      "parent_correct": 78,
      "candidate_correct": 78,
      "parent_calibrated_nll": 0.01903091706838521,
      "candidate_calibrated_nll": 0.014932533018017379,
      "parent_calibrated_brier": 0.0013900388516520708,
      "candidate_calibrated_brier": 0.0008621856358382366
    },
    {
      "source": "excytin_process",
      "questions": 120,
      "parent_correct": 120,
      "candidate_correct": 120,
      "parent_calibrated_nll": 0.007587462315793864,
      "candidate_calibrated_nll": 0.007136704365819422,
      "parent_calibrated_brier": 0.00023474191911792246,
      "candidate_calibrated_brier": 0.00021856930435733632
    },
    {
      "source": "excytin_signin",
      "questions": 120,
      "parent_correct": 120,
      "candidate_correct": 120,
      "parent_calibrated_nll": 0.009411479962491658,
      "candidate_calibrated_nll": 0.00847100086283192,
      "parent_calibrated_brier": 0.0010010974637190823,
      "candidate_calibrated_brier": 0.0008193865392318428
    },
    {
      "source": "sev_guide_detector",
      "questions": 1064,
      "parent_correct": 638,
      "candidate_correct": 639,
      "parent_calibrated_nll": 0.785071927880167,
      "candidate_calibrated_nll": 0.7966730081647482,
      "parent_calibrated_brier": 0.46075730837789786,
      "candidate_calibrated_brier": 0.46843654552918773
    },
    {
      "source": "sev_guide_triage",
      "questions": 231,
      "parent_correct": 130,
      "candidate_correct": 131,
      "parent_calibrated_nll": 0.7793526965750157,
      "candidate_calibrated_nll": 0.7761249864817235,
      "parent_calibrated_brier": 0.4718856823661165,
      "candidate_calibrated_brier": 0.46407372894941223
    },
    {
      "source": "sev_swarmtraces_static",
      "questions": 354,
      "parent_correct": 345,
      "candidate_correct": 347,
      "parent_calibrated_nll": 0.08603305952328294,
      "candidate_calibrated_nll": 0.06588860080731399,
      "parent_calibrated_brier": 0.03970483443698841,
      "candidate_calibrated_brier": 0.028907916842541088
    },
    {
      "source": "sev_sysmon_native_file",
      "questions": 62,
      "parent_correct": 62,
      "candidate_correct": 62,
      "parent_calibrated_nll": 0.04080521144437412,
      "candidate_calibrated_nll": 0.03509983435601263,
      "parent_calibrated_brier": 0.008675036111100168,
      "candidate_calibrated_brier": 0.007419485561600889
    },
    {
      "source": "sev_sysmon_native_join",
      "questions": 120,
      "parent_correct": 120,
      "candidate_correct": 120,
      "parent_calibrated_nll": 0.013824573239669817,
      "candidate_calibrated_nll": 0.009879536093519296,
      "parent_calibrated_brier": 0.00044603592410257366,
      "candidate_calibrated_brier": 0.00022722202758135087
    },
    {
      "source": "sev_sysmon_native_other_effect",
      "questions": 88,
      "parent_correct": 88,
      "candidate_correct": 88,
      "parent_calibrated_nll": 0.03669008514897713,
      "candidate_calibrated_nll": 0.02789592785139411,
      "parent_calibrated_brier": 0.006053607090618847,
      "candidate_calibrated_brier": 0.004783993285232044
    },
    {
      "source": "sev_sysmon_native_process",
      "questions": 74,
      "parent_correct": 74,
      "candidate_correct": 74,
      "parent_calibrated_nll": 0.02430026482825407,
      "candidate_calibrated_nll": 0.023528453108139967,
      "parent_calibrated_brier": 0.004038757822471681,
      "candidate_calibrated_brier": 0.004027925361032504
    },
    {
      "source": "sev_sysmon_native_registry",
      "questions": 66,
      "parent_correct": 65,
      "candidate_correct": 65,
      "parent_calibrated_nll": 0.03908316195438624,
      "candidate_calibrated_nll": 0.03582391955947498,
      "parent_calibrated_brier": 0.01748193378444616,
      "candidate_calibrated_brier": 0.016990289728223678
    }
  ],
  "training_metrics": {
    "wall_seconds": 887.5156674385071,
    "records_seen": 6577,
    "requested_records": 6577,
    "truncated_records": 0,
    "rejected_records": 0,
    "optimizer_steps": 823,
    "forward_tokens": 1505379,
    "peak_device_bytes": 22225622528,
    "device": "cuda",
    "dtype": "bf16",
    "batch": 4,
    "peak_rss_bytes": 31556661248
  },
  "candidate_checkpoint": {
    "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
    "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint",
    "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b",
    "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21",
    "inference_temperature": 1.0
  },
  "candidate_temperature_fit": {
    "temperature": 1.6817928305074288,
    "aggregation": "micro",
    "objective": "raw-logit NLL",
    "split": "calibration",
    "rows_sha256": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da",
    "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02",
    "n": 2674
  },
  "fit_recomputed": true,
  "run_seconds": 1285.6242614819998,
  "decision": "Keep checkpoint as a conditional-policy research result; do not replace published model. Fixed false-alert gate and DNS regression gate fail. No threshold retuning on development."
}