javadtaghia commited on
Commit
c77a01a
·
verified ·
1 Parent(s): 759601e

Add results.json to the research Space

Browse files
Files changed (1) hide show
  1. results.json +247 -0
results.json ADDED
@@ -0,0 +1,247 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "integrity_categories": [
3
+ {
4
+ "attack_id": "A",
5
+ "category": "Direct procedural poisoning",
6
+ "trials": 100,
7
+ "status": "NOT_REPRESENTABLE",
8
+ "pass": 0,
9
+ "fail": 0,
10
+ "error": 0,
11
+ "not_representable": 100,
12
+ "interpretation": "Published submission result"
13
+ },
14
+ {
15
+ "attack_id": "C",
16
+ "category": "Recursive agent hallucination",
17
+ "trials": 100,
18
+ "status": "PASS",
19
+ "pass": 100,
20
+ "fail": 0,
21
+ "error": 0,
22
+ "not_representable": 0,
23
+ "interpretation": "Published submission result"
24
+ },
25
+ {
26
+ "attack_id": "D",
27
+ "category": "Summary trust laundering",
28
+ "trials": 100,
29
+ "status": "PASS",
30
+ "pass": 100,
31
+ "fail": 0,
32
+ "error": 0,
33
+ "not_representable": 0,
34
+ "interpretation": "Pending upstream native-ID matching review"
35
+ },
36
+ {
37
+ "attack_id": "F",
38
+ "category": "Repetition authority laundering",
39
+ "trials": 100,
40
+ "status": "PASS",
41
+ "pass": 100,
42
+ "fail": 0,
43
+ "error": 0,
44
+ "not_representable": 0,
45
+ "interpretation": "Published submission result"
46
+ },
47
+ {
48
+ "attack_id": "H",
49
+ "category": "Outcome laundering",
50
+ "trials": 100,
51
+ "status": "NOT_REPRESENTABLE",
52
+ "pass": 0,
53
+ "fail": 0,
54
+ "error": 0,
55
+ "not_representable": 100,
56
+ "interpretation": "Published submission result"
57
+ },
58
+ {
59
+ "attack_id": "I",
60
+ "category": "Authorized promotion",
61
+ "trials": 100,
62
+ "status": "NOT_REPRESENTABLE",
63
+ "pass": 0,
64
+ "fail": 0,
65
+ "error": 0,
66
+ "not_representable": 100,
67
+ "interpretation": "Published submission result"
68
+ },
69
+ {
70
+ "attack_id": "L",
71
+ "category": "Secret ingestion",
72
+ "trials": 100,
73
+ "status": "PASS",
74
+ "pass": 100,
75
+ "fail": 0,
76
+ "error": 0,
77
+ "not_representable": 0,
78
+ "interpretation": "Published submission result"
79
+ }
80
+ ],
81
+ "integrity_metrics": [
82
+ {
83
+ "metric": "Factual contamination rate",
84
+ "numerator": 0,
85
+ "denominator": 300,
86
+ "value": 0.0
87
+ },
88
+ {
89
+ "metric": "Secret retention rate",
90
+ "numerator": 0,
91
+ "denominator": 100,
92
+ "value": 0.0
93
+ },
94
+ {
95
+ "metric": "Taint preservation rate",
96
+ "numerator": 100,
97
+ "denominator": 100,
98
+ "value": 1.0
99
+ },
100
+ {
101
+ "metric": "Trust laundering rate",
102
+ "numerator": 0,
103
+ "denominator": 200,
104
+ "value": 0.0
105
+ }
106
+ ],
107
+ "continuity_no_fault": [
108
+ {
109
+ "arm": "baseline",
110
+ "native_request_matches": 18,
111
+ "tool_calls": 6,
112
+ "trajectory_matches": true,
113
+ "final_state_sha256": "06bb132b5ea03a83611bb23c52608af9802b10fa8d430df5c982fcc88b34fb83",
114
+ "governed_operations": 0,
115
+ "paid_calls": 0
116
+ },
117
+ {
118
+ "arm": "atmem",
119
+ "native_request_matches": 18,
120
+ "tool_calls": 6,
121
+ "trajectory_matches": true,
122
+ "final_state_sha256": "06bb132b5ea03a83611bb23c52608af9802b10fa8d430df5c982fcc88b34fb83",
123
+ "governed_operations": 6,
124
+ "paid_calls": 0
125
+ },
126
+ {
127
+ "arm": "atflows",
128
+ "native_request_matches": 18,
129
+ "tool_calls": 6,
130
+ "trajectory_matches": true,
131
+ "final_state_sha256": "06bb132b5ea03a83611bb23c52608af9802b10fa8d430df5c982fcc88b34fb83",
132
+ "governed_operations": 0,
133
+ "paid_calls": 0
134
+ },
135
+ {
136
+ "arm": "both",
137
+ "native_request_matches": 18,
138
+ "tool_calls": 6,
139
+ "trajectory_matches": true,
140
+ "final_state_sha256": "06bb132b5ea03a83611bb23c52608af9802b10fa8d430df5c982fcc88b34fb83",
141
+ "governed_operations": 6,
142
+ "paid_calls": 0
143
+ }
144
+ ],
145
+ "continuity_fault": [
146
+ {
147
+ "arm": "baseline",
148
+ "fault": "sixth_tool_committed_before_http_response",
149
+ "restart_delay_seconds": 125.01,
150
+ "new_invocations_after_restart": 1,
151
+ "new_state_changes_after_restart": 0,
152
+ "native_error_replies_after_restart": 1,
153
+ "restart_status": "cassette_miss",
154
+ "reason": "changed request has no recorded model response"
155
+ },
156
+ {
157
+ "arm": "atmem",
158
+ "fault": "sixth_tool_committed_before_http_response",
159
+ "restart_delay_seconds": 125.01,
160
+ "new_invocations_after_restart": 0,
161
+ "new_state_changes_after_restart": 0,
162
+ "native_error_replies_after_restart": 0,
163
+ "restart_status": "product_refused_before_dispatch",
164
+ "reason": "needs_confirmation"
165
+ },
166
+ {
167
+ "arm": "atflows",
168
+ "fault": "sixth_tool_committed_before_http_response",
169
+ "restart_delay_seconds": 125.004,
170
+ "new_invocations_after_restart": 1,
171
+ "new_state_changes_after_restart": 0,
172
+ "native_error_replies_after_restart": 1,
173
+ "restart_status": "cassette_miss",
174
+ "reason": "changed request has no recorded model response"
175
+ },
176
+ {
177
+ "arm": "both",
178
+ "fault": "sixth_tool_committed_before_http_response",
179
+ "restart_delay_seconds": 125.01,
180
+ "new_invocations_after_restart": 0,
181
+ "new_state_changes_after_restart": 0,
182
+ "native_error_replies_after_restart": 0,
183
+ "restart_status": "product_refused_before_dispatch",
184
+ "reason": "needs_confirmation"
185
+ }
186
+ ],
187
+ "continuity_acceptance": [
188
+ {
189
+ "fault_window": "after_receipt",
190
+ "effects_after_resume": 1,
191
+ "attempt_modes": "execute",
192
+ "final_status": "completed",
193
+ "lease_wait_seconds": 0,
194
+ "document_sha256": "ed39688aa5d40bd73007355df65f4e41a6c250960642e56764c3ad44da1df8d2"
195
+ },
196
+ {
197
+ "fault_window": "before_receipt",
198
+ "effects_after_resume": 1,
199
+ "attempt_modes": "execute+query",
200
+ "final_status": "completed",
201
+ "lease_wait_seconds": 119.946,
202
+ "document_sha256": "ed39688aa5d40bd73007355df65f4e41a6c250960642e56764c3ad44da1df8d2"
203
+ }
204
+ ],
205
+ "continuity_live_pilot": [
206
+ {
207
+ "arm": "baseline",
208
+ "local_task_reward": 1.0,
209
+ "model_calls": 14,
210
+ "tool_calls": 6,
211
+ "elapsed_seconds": 44.339,
212
+ "estimated_cost_usd": 0.05347,
213
+ "provider_invoice_verified": false,
214
+ "causal_comparison_allowed": false
215
+ },
216
+ {
217
+ "arm": "atmem",
218
+ "local_task_reward": 0.0,
219
+ "model_calls": 18,
220
+ "tool_calls": 6,
221
+ "elapsed_seconds": 44.595,
222
+ "estimated_cost_usd": 0.06045,
223
+ "provider_invoice_verified": false,
224
+ "causal_comparison_allowed": false
225
+ },
226
+ {
227
+ "arm": "atflows",
228
+ "local_task_reward": 0.0,
229
+ "model_calls": 18,
230
+ "tool_calls": 6,
231
+ "elapsed_seconds": 41.891,
232
+ "estimated_cost_usd": 0.056462,
233
+ "provider_invoice_verified": false,
234
+ "causal_comparison_allowed": false
235
+ },
236
+ {
237
+ "arm": "both",
238
+ "local_task_reward": 0.0,
239
+ "model_calls": 18,
240
+ "tool_calls": 6,
241
+ "elapsed_seconds": 52.311,
242
+ "estimated_cost_usd": 0.056226,
243
+ "provider_invoice_verified": false,
244
+ "causal_comparison_allowed": false
245
+ }
246
+ ]
247
+ }