Invictus22 commited on
Commit
9f12ae5
·
verified ·
1 Parent(s): 745fa74

Upload part2/results/results_detailed.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. part2/results/results_detailed.json +1861 -0
part2/results/results_detailed.json ADDED
@@ -0,0 +1,1861 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architecture_comparison": {
3
+ "RootOnlyMLP": {
4
+ "model": "RootOnlyMLP",
5
+ "feature": "bert",
6
+ "group": "architecture_comparison",
7
+ "seeds": [
8
+ 0,
9
+ 1,
10
+ 2,
11
+ 3,
12
+ 4
13
+ ],
14
+ "num_parameters": 98690,
15
+ "aggregate": {
16
+ "accuracy": {
17
+ "mean": 0.724202822791427,
18
+ "std": 0.010128737141480723,
19
+ "values": [
20
+ 0.7177208572922111,
21
+ 0.7109252483010977,
22
+ 0.7328802927339257,
23
+ 0.7386304234187141,
24
+ 0.7208572922111867
25
+ ]
26
+ },
27
+ "f1": {
28
+ "mean": 0.7301143524165361,
29
+ "std": 0.003537176206232441,
30
+ "values": [
31
+ 0.7333333333333333,
32
+ 0.7274519467718088,
33
+ 0.7307692307692307,
34
+ 0.724821133736929,
35
+ 0.7341961174713788
36
+ ]
37
+ },
38
+ "macro_f1": {
39
+ "mean": 0.7235190659363446,
40
+ "std": 0.01039171145511059,
41
+ "values": [
42
+ 0.7167499537294095,
43
+ 0.7098584163463941,
44
+ 0.7328638684966486,
45
+ 0.7379705469580613,
46
+ 0.7201525441512096
47
+ ]
48
+ },
49
+ "macro_precision": {
50
+ "mean": 0.7263213062415851,
51
+ "std": 0.009640737482670771,
52
+ "values": [
53
+ 0.7205740781360435,
54
+ 0.7138964849597796,
55
+ 0.7329631431632809,
56
+ 0.7412167296600566,
57
+ 0.7229560952887648
58
+ ]
59
+ },
60
+ "macro_recall": {
61
+ "mean": 0.7241696816010668,
62
+ "std": 0.010195188589834349,
63
+ "values": [
64
+ 0.7176308080752878,
65
+ 0.7108319032889199,
66
+ 0.732894391675502,
67
+ 0.7387109379269639,
68
+ 0.7207803670386603
69
+ ]
70
+ },
71
+ "precision": {
72
+ "mean": 0.718203917603286,
73
+ "std": 0.029471128294176245,
74
+ "values": [
75
+ 0.6958762886597938,
76
+ 0.6890756302521008,
77
+ 0.7377659574468085,
78
+ 0.7665890570430733,
79
+ 0.7017126546146527
80
+ ]
81
+ },
82
+ "recall": {
83
+ "mean": 0.7453027139874738,
84
+ "std": 0.03443410914503425,
85
+ "values": [
86
+ 0.7750521920668059,
87
+ 0.7703549060542797,
88
+ 0.7239039665970772,
89
+ 0.6873695198329853,
90
+ 0.7698329853862212
91
+ ]
92
+ }
93
+ },
94
+ "runs": [
95
+ {
96
+ "model": "RootOnlyMLP",
97
+ "feature": "bert",
98
+ "seed": 0,
99
+ "num_parameters": 98690,
100
+ "best_epoch": 46,
101
+ "best_val_f1": 0.7747440273037542,
102
+ "test_loss": 0.5783436401813864,
103
+ "test": {
104
+ "accuracy": 0.7177208572922111,
105
+ "precision": 0.6958762886597938,
106
+ "recall": 0.7750521920668059,
107
+ "f1": 0.7333333333333333,
108
+ "macro_precision": 0.7205740781360435,
109
+ "macro_recall": 0.7176308080752878,
110
+ "macro_f1": 0.7167499537294095
111
+ },
112
+ "trained": true,
113
+ "checkpoint": "models/part2/RootOnlyMLP_bert_seed0.pt",
114
+ "runtime_seconds": 67.39869499206543
115
+ },
116
+ {
117
+ "model": "RootOnlyMLP",
118
+ "feature": "bert",
119
+ "seed": 1,
120
+ "num_parameters": 98690,
121
+ "best_epoch": 59,
122
+ "best_val_f1": 0.7843137254901961,
123
+ "test_loss": 0.5655875268309015,
124
+ "test": {
125
+ "accuracy": 0.7109252483010977,
126
+ "precision": 0.6890756302521008,
127
+ "recall": 0.7703549060542797,
128
+ "f1": 0.7274519467718088,
129
+ "macro_precision": 0.7138964849597796,
130
+ "macro_recall": 0.7108319032889199,
131
+ "macro_f1": 0.7098584163463941
132
+ },
133
+ "trained": true,
134
+ "checkpoint": "models/part2/RootOnlyMLP_bert_seed1.pt",
135
+ "runtime_seconds": 60.5965621471405
136
+ },
137
+ {
138
+ "model": "RootOnlyMLP",
139
+ "feature": "bert",
140
+ "seed": 2,
141
+ "num_parameters": 98690,
142
+ "best_epoch": 14,
143
+ "best_val_f1": 0.7821612349914236,
144
+ "test_loss": 0.540324495777184,
145
+ "test": {
146
+ "accuracy": 0.7328802927339257,
147
+ "precision": 0.7377659574468085,
148
+ "recall": 0.7239039665970772,
149
+ "f1": 0.7307692307692307,
150
+ "macro_precision": 0.7329631431632809,
151
+ "macro_recall": 0.732894391675502,
152
+ "macro_f1": 0.7328638684966486
153
+ },
154
+ "trained": true,
155
+ "checkpoint": "models/part2/RootOnlyMLP_bert_seed2.pt",
156
+ "runtime_seconds": 60.321659088134766
157
+ },
158
+ {
159
+ "model": "RootOnlyMLP",
160
+ "feature": "bert",
161
+ "seed": 3,
162
+ "num_parameters": 98690,
163
+ "best_epoch": 27,
164
+ "best_val_f1": 0.7871198568872988,
165
+ "test_loss": 0.543648226748445,
166
+ "test": {
167
+ "accuracy": 0.7386304234187141,
168
+ "precision": 0.7665890570430733,
169
+ "recall": 0.6873695198329853,
170
+ "f1": 0.724821133736929,
171
+ "macro_precision": 0.7412167296600566,
172
+ "macro_recall": 0.7387109379269639,
173
+ "macro_f1": 0.7379705469580613
174
+ },
175
+ "trained": true,
176
+ "checkpoint": "models/part2/RootOnlyMLP_bert_seed3.pt",
177
+ "runtime_seconds": 60.74616456031799
178
+ },
179
+ {
180
+ "model": "RootOnlyMLP",
181
+ "feature": "bert",
182
+ "seed": 4,
183
+ "num_parameters": 98690,
184
+ "best_epoch": 46,
185
+ "best_val_f1": 0.7808896210873146,
186
+ "test_loss": 0.5516770388278672,
187
+ "test": {
188
+ "accuracy": 0.7208572922111867,
189
+ "precision": 0.7017126546146527,
190
+ "recall": 0.7698329853862212,
191
+ "f1": 0.7341961174713788,
192
+ "macro_precision": 0.7229560952887648,
193
+ "macro_recall": 0.7207803670386603,
194
+ "macro_f1": 0.7201525441512096
195
+ },
196
+ "trained": true,
197
+ "checkpoint": "models/part2/RootOnlyMLP_bert_seed4.pt",
198
+ "runtime_seconds": 60.52992653846741
199
+ }
200
+ ]
201
+ },
202
+ "GCN": {
203
+ "model": "GCN",
204
+ "feature": "bert",
205
+ "group": "architecture_comparison",
206
+ "seeds": [
207
+ 0,
208
+ 1,
209
+ 2,
210
+ 3,
211
+ 4
212
+ ],
213
+ "num_parameters": 213890,
214
+ "aggregate": {
215
+ "accuracy": {
216
+ "mean": 0.9637741766858337,
217
+ "std": 0.0012545739675901679,
218
+ "values": [
219
+ 0.9647151071615264,
220
+ 0.9644537375849451,
221
+ 0.9644537375849451,
222
+ 0.9613173026659697,
223
+ 0.9639309984317825
224
+ ]
225
+ },
226
+ "f1": {
227
+ "mean": 0.9639171310668944,
228
+ "std": 0.0012973665698816654,
229
+ "values": [
230
+ 0.9648162627052385,
231
+ 0.9642669469259064,
232
+ 0.9647668393782384,
233
+ 0.9613577023498695,
234
+ 0.9643779039752194
235
+ ]
236
+ },
237
+ "macro_f1": {
238
+ "mean": 0.9637722185575539,
239
+ "std": 0.0012539970881839143,
240
+ "values": [
241
+ 0.9647148154941658,
242
+ 0.964452766234664,
243
+ 0.9644509302376424,
244
+ 0.9613172603847726,
245
+ 0.9639253204365245
246
+ ]
247
+ },
248
+ "macro_precision": {
249
+ "mean": 0.9638599424156311,
250
+ "std": 0.0012841782597041132,
251
+ "values": [
252
+ 0.9647199552945003,
253
+ 0.9645303139621322,
254
+ 0.9645630250630841,
255
+ 0.9613170188393821,
256
+ 0.9641693989190561
257
+ ]
258
+ },
259
+ "macro_recall": {
260
+ "mean": 0.9637702346730208,
261
+ "std": 0.0012534665285465591,
262
+ "values": [
263
+ 0.9647129709582574,
264
+ 0.9644643071844703,
265
+ 0.9644421733760342,
266
+ 0.9613180273038289,
267
+ 0.9639136945425133
268
+ ]
269
+ },
270
+ "precision": {
271
+ "mean": 0.9616347509709493,
272
+ "std": 0.005690617740110476,
273
+ "values": [
274
+ 0.9635606454971369,
275
+ 0.9708994708994709,
276
+ 0.9578189300411523,
277
+ 0.9618599791013585,
278
+ 0.9540347293156282
279
+ ]
280
+ },
281
+ "recall": {
282
+ "mean": 0.9662839248434238,
283
+ "std": 0.006456645968347435,
284
+ "values": [
285
+ 0.9660751565762005,
286
+ 0.9577244258872651,
287
+ 0.9718162839248434,
288
+ 0.9608559498956158,
289
+ 0.9749478079331941
290
+ ]
291
+ }
292
+ },
293
+ "runs": [
294
+ {
295
+ "model": "GCN",
296
+ "feature": "bert",
297
+ "seed": 0,
298
+ "num_parameters": 213890,
299
+ "best_epoch": 10,
300
+ "best_val_f1": 0.9715302491103203,
301
+ "test_loss": 0.12618988020159597,
302
+ "test": {
303
+ "accuracy": 0.9647151071615264,
304
+ "precision": 0.9635606454971369,
305
+ "recall": 0.9660751565762005,
306
+ "f1": 0.9648162627052385,
307
+ "macro_precision": 0.9647199552945003,
308
+ "macro_recall": 0.9647129709582574,
309
+ "macro_f1": 0.9647148154941658
310
+ },
311
+ "trained": true,
312
+ "checkpoint": "models/part2/GCN_bert_seed0.pt",
313
+ "runtime_seconds": 63.75345540046692
314
+ },
315
+ {
316
+ "model": "GCN",
317
+ "feature": "bert",
318
+ "seed": 1,
319
+ "num_parameters": 213890,
320
+ "best_epoch": 16,
321
+ "best_val_f1": 0.975,
322
+ "test_loss": 0.1202603500673301,
323
+ "test": {
324
+ "accuracy": 0.9644537375849451,
325
+ "precision": 0.9708994708994709,
326
+ "recall": 0.9577244258872651,
327
+ "f1": 0.9642669469259064,
328
+ "macro_precision": 0.9645303139621322,
329
+ "macro_recall": 0.9644643071844703,
330
+ "macro_f1": 0.964452766234664
331
+ },
332
+ "trained": true,
333
+ "checkpoint": "models/part2/GCN_bert_seed1.pt",
334
+ "runtime_seconds": 63.10755491256714
335
+ },
336
+ {
337
+ "model": "GCN",
338
+ "feature": "bert",
339
+ "seed": 2,
340
+ "num_parameters": 213890,
341
+ "best_epoch": 25,
342
+ "best_val_f1": 0.9717314487632509,
343
+ "test_loss": 0.1278630554271006,
344
+ "test": {
345
+ "accuracy": 0.9644537375849451,
346
+ "precision": 0.9578189300411523,
347
+ "recall": 0.9718162839248434,
348
+ "f1": 0.9647668393782384,
349
+ "macro_precision": 0.9645630250630841,
350
+ "macro_recall": 0.9644421733760342,
351
+ "macro_f1": 0.9644509302376424
352
+ },
353
+ "trained": true,
354
+ "checkpoint": "models/part2/GCN_bert_seed2.pt",
355
+ "runtime_seconds": 63.62568688392639
356
+ },
357
+ {
358
+ "model": "GCN",
359
+ "feature": "bert",
360
+ "seed": 3,
361
+ "num_parameters": 213890,
362
+ "best_epoch": 54,
363
+ "best_val_f1": 0.9713261648745519,
364
+ "test_loss": 0.12247229868743785,
365
+ "test": {
366
+ "accuracy": 0.9613173026659697,
367
+ "precision": 0.9618599791013585,
368
+ "recall": 0.9608559498956158,
369
+ "f1": 0.9613577023498695,
370
+ "macro_precision": 0.9613170188393821,
371
+ "macro_recall": 0.9613180273038289,
372
+ "macro_f1": 0.9613172603847726
373
+ },
374
+ "trained": true,
375
+ "checkpoint": "models/part2/GCN_bert_seed3.pt",
376
+ "runtime_seconds": 63.22453713417053
377
+ },
378
+ {
379
+ "model": "GCN",
380
+ "feature": "bert",
381
+ "seed": 4,
382
+ "num_parameters": 213890,
383
+ "best_epoch": 24,
384
+ "best_val_f1": 0.9680851063829787,
385
+ "test_loss": 0.10637199411327404,
386
+ "test": {
387
+ "accuracy": 0.9639309984317825,
388
+ "precision": 0.9540347293156282,
389
+ "recall": 0.9749478079331941,
390
+ "f1": 0.9643779039752194,
391
+ "macro_precision": 0.9641693989190561,
392
+ "macro_recall": 0.9639136945425133,
393
+ "macro_f1": 0.9639253204365245
394
+ },
395
+ "trained": true,
396
+ "checkpoint": "models/part2/GCN_bert_seed4.pt",
397
+ "runtime_seconds": 63.74688744544983
398
+ }
399
+ ]
400
+ },
401
+ "GraphSAGE": {
402
+ "model": "GraphSAGE",
403
+ "feature": "bert",
404
+ "group": "architecture_comparison",
405
+ "seeds": [
406
+ 0,
407
+ 1,
408
+ 2,
409
+ 3,
410
+ 4
411
+ ],
412
+ "num_parameters": 328578,
413
+ "aggregate": {
414
+ "accuracy": {
415
+ "mean": 0.955514898065865,
416
+ "std": 0.006485758463975497,
417
+ "values": [
418
+ 0.946941975953999,
419
+ 0.9610559330893884,
420
+ 0.9485101934134866,
421
+ 0.9587036069001568,
422
+ 0.9623627809722948
423
+ ]
424
+ },
425
+ "f1": {
426
+ "mean": 0.955415150400478,
427
+ "std": 0.006951628362060894,
428
+ "values": [
429
+ 0.9467331409078982,
430
+ 0.9614489003880983,
431
+ 0.9473121155389141,
432
+ 0.9590036325895174,
433
+ 0.9625779625779626
434
+ ]
435
+ },
436
+ "macro_f1": {
437
+ "mean": 0.9555079067861755,
438
+ "std": 0.006490561827251249,
439
+ "values": [
440
+ 0.9469411604044828,
441
+ 0.961051886159193,
442
+ 0.9484835558527691,
443
+ 0.9587013950204037,
444
+ 0.9623615364940286
445
+ ]
446
+ },
447
+ "macro_precision": {
448
+ "mean": 0.9557815685568449,
449
+ "std": 0.006299246562392474,
450
+ "values": [
451
+ 0.9469889280438998,
452
+ 0.961204508247471,
453
+ 0.9495420884033248,
454
+ 0.9587717111822427,
455
+ 0.962400606907287
456
+ ]
457
+ },
458
+ "macro_recall": {
459
+ "mean": 0.9555183136770541,
460
+ "std": 0.006470692066213718,
461
+ "values": [
462
+ 0.9469504530599306,
463
+ 0.961042311097509,
464
+ 0.9485481861207359,
465
+ 0.9586944878619288,
466
+ 0.9623561302451661
467
+ ]
468
+ },
469
+ "precision": {
470
+ "mean": 0.957783274992118,
471
+ "std": 0.0072065686976497375,
472
+ "values": [
473
+ 0.9519788918205805,
474
+ 0.9533093894304772,
475
+ 0.9714755896873286,
476
+ 0.9535603715170279,
477
+ 0.9585921325051759
478
+ ]
479
+ },
480
+ "recall": {
481
+ "mean": 0.953340292275574,
482
+ "std": 0.017608461698504822,
483
+ "values": [
484
+ 0.941544885177453,
485
+ 0.9697286012526096,
486
+ 0.9243215031315241,
487
+ 0.964509394572025,
488
+ 0.9665970772442589
489
+ ]
490
+ }
491
+ },
492
+ "runs": [
493
+ {
494
+ "model": "GraphSAGE",
495
+ "feature": "bert",
496
+ "seed": 0,
497
+ "num_parameters": 328578,
498
+ "best_epoch": 33,
499
+ "best_val_f1": 0.9642857142857143,
500
+ "test_loss": 0.16367885422095912,
501
+ "test": {
502
+ "accuracy": 0.946941975953999,
503
+ "precision": 0.9519788918205805,
504
+ "recall": 0.941544885177453,
505
+ "f1": 0.9467331409078982,
506
+ "macro_precision": 0.9469889280438998,
507
+ "macro_recall": 0.9469504530599306,
508
+ "macro_f1": 0.9469411604044828
509
+ },
510
+ "trained": true,
511
+ "checkpoint": "models/part2/GraphSAGE_bert_seed0.pt",
512
+ "runtime_seconds": 63.1004912853241
513
+ },
514
+ {
515
+ "model": "GraphSAGE",
516
+ "feature": "bert",
517
+ "seed": 1,
518
+ "num_parameters": 328578,
519
+ "best_epoch": 26,
520
+ "best_val_f1": 0.9685314685314685,
521
+ "test_loss": 0.1614884846632978,
522
+ "test": {
523
+ "accuracy": 0.9610559330893884,
524
+ "precision": 0.9533093894304772,
525
+ "recall": 0.9697286012526096,
526
+ "f1": 0.9614489003880983,
527
+ "macro_precision": 0.961204508247471,
528
+ "macro_recall": 0.961042311097509,
529
+ "macro_f1": 0.961051886159193
530
+ },
531
+ "trained": true,
532
+ "checkpoint": "models/part2/GraphSAGE_bert_seed1.pt",
533
+ "runtime_seconds": 63.951916456222534
534
+ },
535
+ {
536
+ "model": "GraphSAGE",
537
+ "feature": "bert",
538
+ "seed": 2,
539
+ "num_parameters": 328578,
540
+ "best_epoch": 43,
541
+ "best_val_f1": 0.9637681159420289,
542
+ "test_loss": 0.16088209386527444,
543
+ "test": {
544
+ "accuracy": 0.9485101934134866,
545
+ "precision": 0.9714755896873286,
546
+ "recall": 0.9243215031315241,
547
+ "f1": 0.9473121155389141,
548
+ "macro_precision": 0.9495420884033248,
549
+ "macro_recall": 0.9485481861207359,
550
+ "macro_f1": 0.9484835558527691
551
+ },
552
+ "trained": true,
553
+ "checkpoint": "models/part2/GraphSAGE_bert_seed2.pt",
554
+ "runtime_seconds": 63.84702157974243
555
+ },
556
+ {
557
+ "model": "GraphSAGE",
558
+ "feature": "bert",
559
+ "seed": 3,
560
+ "num_parameters": 328578,
561
+ "best_epoch": 14,
562
+ "best_val_f1": 0.9663716814159292,
563
+ "test_loss": 0.16567528191749425,
564
+ "test": {
565
+ "accuracy": 0.9587036069001568,
566
+ "precision": 0.9535603715170279,
567
+ "recall": 0.964509394572025,
568
+ "f1": 0.9590036325895174,
569
+ "macro_precision": 0.9587717111822427,
570
+ "macro_recall": 0.9586944878619288,
571
+ "macro_f1": 0.9587013950204037
572
+ },
573
+ "trained": true,
574
+ "checkpoint": "models/part2/GraphSAGE_bert_seed3.pt",
575
+ "runtime_seconds": 64.20175671577454
576
+ },
577
+ {
578
+ "model": "GraphSAGE",
579
+ "feature": "bert",
580
+ "seed": 4,
581
+ "num_parameters": 328578,
582
+ "best_epoch": 48,
583
+ "best_val_f1": 0.9559082892416225,
584
+ "test_loss": 0.13612134582511554,
585
+ "test": {
586
+ "accuracy": 0.9623627809722948,
587
+ "precision": 0.9585921325051759,
588
+ "recall": 0.9665970772442589,
589
+ "f1": 0.9625779625779626,
590
+ "macro_precision": 0.962400606907287,
591
+ "macro_recall": 0.9623561302451661,
592
+ "macro_f1": 0.9623615364940286
593
+ },
594
+ "trained": true,
595
+ "checkpoint": "models/part2/GraphSAGE_bert_seed4.pt",
596
+ "runtime_seconds": 64.07758402824402
597
+ }
598
+ ]
599
+ },
600
+ "GAT": {
601
+ "model": "GAT",
602
+ "feature": "bert",
603
+ "group": "architecture_comparison",
604
+ "seeds": [
605
+ 0,
606
+ 1,
607
+ 2,
608
+ 3,
609
+ 4
610
+ ],
611
+ "num_parameters": 214402,
612
+ "aggregate": {
613
+ "accuracy": {
614
+ "mean": 0.9699424986931522,
615
+ "std": 0.004454032076140269,
616
+ "values": [
617
+ 0.9709879769994773,
618
+ 0.967067433350758,
619
+ 0.9738630423418714,
620
+ 0.9749085206481966,
621
+ 0.9628855201254574
622
+ ]
623
+ },
624
+ "f1": {
625
+ "mean": 0.9701382658952159,
626
+ "std": 0.004537862822510016,
627
+ "values": [
628
+ 0.9712955779674166,
629
+ 0.967425025853154,
630
+ 0.9740529320186819,
631
+ 0.9751295336787564,
632
+ 0.9627882599580713
633
+ ]
634
+ },
635
+ "macro_f1": {
636
+ "mean": 0.9699403114567952,
637
+ "std": 0.004453780253813593,
638
+ "values": [
639
+ 0.9709846449942763,
640
+ 0.9670634643007843,
641
+ 0.9738616424179771,
642
+ 0.974906538991277,
643
+ 0.9628852665796612
644
+ ]
645
+ },
646
+ "macro_precision": {
647
+ "mean": 0.970054094825571,
648
+ "std": 0.004468339889730162,
649
+ "values": [
650
+ 0.971158311976764,
651
+ 0.9672456457651731,
652
+ 0.9739337360724842,
653
+ 0.9750205542654474,
654
+ 0.9629122260479861
655
+ ]
656
+ },
657
+ "macro_recall": {
658
+ "mean": 0.9699338171802075,
659
+ "std": 0.004449053832774379,
660
+ "values": [
661
+ 0.9709735596629103,
662
+ 0.9670525964870094,
663
+ 0.9738539605854256,
664
+ 0.9748969821508597,
665
+ 0.9628919870148324
666
+ ]
667
+ },
668
+ "precision": {
669
+ "mean": 0.9649121483869536,
670
+ "std": 0.0038331647192743163,
671
+ "values": [
672
+ 0.9625832906201948,
673
+ 0.9585040983606558,
674
+ 0.9685242518059856,
675
+ 0.9681069958847737,
676
+ 0.966842105263158
677
+ ]
678
+ },
679
+ "recall": {
680
+ "mean": 0.9754697286012526,
681
+ "std": 0.00855058347234757,
682
+ "values": [
683
+ 0.9801670146137788,
684
+ 0.9765135699373695,
685
+ 0.9796450939457203,
686
+ 0.9822546972860126,
687
+ 0.958768267223382
688
+ ]
689
+ }
690
+ },
691
+ "runs": [
692
+ {
693
+ "model": "GAT",
694
+ "feature": "bert",
695
+ "seed": 0,
696
+ "num_parameters": 214402,
697
+ "best_epoch": 55,
698
+ "best_val_f1": 0.9734513274336283,
699
+ "test_loss": 0.09587014774595701,
700
+ "test": {
701
+ "accuracy": 0.9709879769994773,
702
+ "precision": 0.9625832906201948,
703
+ "recall": 0.9801670146137788,
704
+ "f1": 0.9712955779674166,
705
+ "macro_precision": 0.971158311976764,
706
+ "macro_recall": 0.9709735596629103,
707
+ "macro_f1": 0.9709846449942763
708
+ },
709
+ "trained": true,
710
+ "checkpoint": "models/part2/GAT_bert_seed0.pt",
711
+ "runtime_seconds": 64.93717622756958
712
+ },
713
+ {
714
+ "model": "GAT",
715
+ "feature": "bert",
716
+ "seed": 1,
717
+ "num_parameters": 214402,
718
+ "best_epoch": 24,
719
+ "best_val_f1": 0.9771528998242531,
720
+ "test_loss": 0.13939974062491386,
721
+ "test": {
722
+ "accuracy": 0.967067433350758,
723
+ "precision": 0.9585040983606558,
724
+ "recall": 0.9765135699373695,
725
+ "f1": 0.967425025853154,
726
+ "macro_precision": 0.9672456457651731,
727
+ "macro_recall": 0.9670525964870094,
728
+ "macro_f1": 0.9670634643007843
729
+ },
730
+ "trained": true,
731
+ "checkpoint": "models/part2/GAT_bert_seed1.pt",
732
+ "runtime_seconds": 65.03910541534424
733
+ },
734
+ {
735
+ "model": "GAT",
736
+ "feature": "bert",
737
+ "seed": 2,
738
+ "num_parameters": 214402,
739
+ "best_epoch": 18,
740
+ "best_val_f1": 0.9787234042553191,
741
+ "test_loss": 0.10519640045644597,
742
+ "test": {
743
+ "accuracy": 0.9738630423418714,
744
+ "precision": 0.9685242518059856,
745
+ "recall": 0.9796450939457203,
746
+ "f1": 0.9740529320186819,
747
+ "macro_precision": 0.9739337360724842,
748
+ "macro_recall": 0.9738539605854256,
749
+ "macro_f1": 0.9738616424179771
750
+ },
751
+ "trained": true,
752
+ "checkpoint": "models/part2/GAT_bert_seed2.pt",
753
+ "runtime_seconds": 65.59159541130066
754
+ },
755
+ {
756
+ "model": "GAT",
757
+ "feature": "bert",
758
+ "seed": 3,
759
+ "num_parameters": 214402,
760
+ "best_epoch": 58,
761
+ "best_val_f1": 0.9803921568627451,
762
+ "test_loss": 0.08627335400838099,
763
+ "test": {
764
+ "accuracy": 0.9749085206481966,
765
+ "precision": 0.9681069958847737,
766
+ "recall": 0.9822546972860126,
767
+ "f1": 0.9751295336787564,
768
+ "macro_precision": 0.9750205542654474,
769
+ "macro_recall": 0.9748969821508597,
770
+ "macro_f1": 0.974906538991277
771
+ },
772
+ "trained": true,
773
+ "checkpoint": "models/part2/GAT_bert_seed3.pt",
774
+ "runtime_seconds": 64.72948431968689
775
+ },
776
+ {
777
+ "model": "GAT",
778
+ "feature": "bert",
779
+ "seed": 4,
780
+ "num_parameters": 214402,
781
+ "best_epoch": 35,
782
+ "best_val_f1": 0.9785714285714285,
783
+ "test_loss": 0.1464931056316274,
784
+ "test": {
785
+ "accuracy": 0.9628855201254574,
786
+ "precision": 0.966842105263158,
787
+ "recall": 0.958768267223382,
788
+ "f1": 0.9627882599580713,
789
+ "macro_precision": 0.9629122260479861,
790
+ "macro_recall": 0.9628919870148324,
791
+ "macro_f1": 0.9628852665796612
792
+ },
793
+ "trained": true,
794
+ "checkpoint": "models/part2/GAT_bert_seed4.pt",
795
+ "runtime_seconds": 64.90364670753479
796
+ }
797
+ ]
798
+ },
799
+ "BiGCN": {
800
+ "model": "BiGCN",
801
+ "feature": "bert",
802
+ "group": "architecture_comparison",
803
+ "seeds": [
804
+ 0,
805
+ 1,
806
+ 2,
807
+ 3,
808
+ 4
809
+ ],
810
+ "num_parameters": 820098,
811
+ "aggregate": {
812
+ "accuracy": {
813
+ "mean": 0.9534239414532149,
814
+ "std": 0.004193006718276073,
815
+ "values": [
816
+ 0.9550444328280188,
817
+ 0.946941975953999,
818
+ 0.9589649764767381,
819
+ 0.9555671719811814,
820
+ 0.950601150026137
821
+ ]
822
+ },
823
+ "f1": {
824
+ "mean": 0.9536334995808506,
825
+ "std": 0.0038113095102809296,
826
+ "values": [
827
+ 0.9552083333333333,
828
+ 0.9479887266205483,
829
+ 0.9589327753073502,
830
+ 0.9551924090669478,
831
+ 0.9508452535760729
832
+ ]
833
+ },
834
+ "macro_f1": {
835
+ "mean": 0.9534186508706298,
836
+ "std": 0.004199449852261356,
837
+ "values": [
838
+ 0.9550438308849247,
839
+ 0.9469204769405756,
840
+ 0.9589649512473026,
841
+ 0.9555640635277715,
842
+ 0.9505999317525755
843
+ ]
844
+ },
845
+ "macro_precision": {
846
+ "mean": 0.9535936977724987,
847
+ "std": 0.004014886989099529,
848
+ "values": [
849
+ 0.9550549700391973,
850
+ 0.9475813833966459,
851
+ 0.9589719507926284,
852
+ 0.9557340698414365,
853
+ 0.9506261147925854
854
+ ]
855
+ },
856
+ "macro_recall": {
857
+ "mean": 0.9534201379400802,
858
+ "std": 0.004205054337904544,
859
+ "values": [
860
+ 0.9550410431855196,
861
+ 0.9469127436085213,
862
+ 0.9589685645268831,
863
+ 0.9555826383499655,
864
+ 0.9505957000295118
865
+ ]
866
+ },
867
+ "precision": {
868
+ "mean": 0.9515934760548553,
869
+ "std": 0.011902384733996369,
870
+ "values": [
871
+ 0.9532224532224532,
872
+ 0.9310518369401107,
873
+ 0.9611955951756685,
874
+ 0.9648562300319489,
875
+ 0.9476412649040954
876
+ ]
877
+ },
878
+ "recall": {
879
+ "mean": 0.9558455114822548,
880
+ "std": 0.0063631526739575004,
881
+ "values": [
882
+ 0.9572025052192067,
883
+ 0.965553235908142,
884
+ 0.9566805845511482,
885
+ 0.9457202505219207,
886
+ 0.954070981210856
887
+ ]
888
+ }
889
+ },
890
+ "runs": [
891
+ {
892
+ "model": "BiGCN",
893
+ "feature": "bert",
894
+ "seed": 0,
895
+ "num_parameters": 820098,
896
+ "best_epoch": 37,
897
+ "best_val_f1": 0.9572953736654805,
898
+ "test_loss": 0.1395130967570423,
899
+ "test": {
900
+ "accuracy": 0.9550444328280188,
901
+ "precision": 0.9532224532224532,
902
+ "recall": 0.9572025052192067,
903
+ "f1": 0.9552083333333333,
904
+ "macro_precision": 0.9550549700391973,
905
+ "macro_recall": 0.9550410431855196,
906
+ "macro_f1": 0.9550438308849247
907
+ },
908
+ "trained": true,
909
+ "checkpoint": "models/part2/BiGCN_bert_seed0.pt",
910
+ "runtime_seconds": 76.21381855010986
911
+ },
912
+ {
913
+ "model": "BiGCN",
914
+ "feature": "bert",
915
+ "seed": 1,
916
+ "num_parameters": 820098,
917
+ "best_epoch": 18,
918
+ "best_val_f1": 0.9594356261022927,
919
+ "test_loss": 0.15889897872438247,
920
+ "test": {
921
+ "accuracy": 0.946941975953999,
922
+ "precision": 0.9310518369401107,
923
+ "recall": 0.965553235908142,
924
+ "f1": 0.9479887266205483,
925
+ "macro_precision": 0.9475813833966459,
926
+ "macro_recall": 0.9469127436085213,
927
+ "macro_f1": 0.9469204769405756
928
+ },
929
+ "trained": true,
930
+ "checkpoint": "models/part2/BiGCN_bert_seed1.pt",
931
+ "runtime_seconds": 76.47102093696594
932
+ },
933
+ {
934
+ "model": "BiGCN",
935
+ "feature": "bert",
936
+ "seed": 2,
937
+ "num_parameters": 820098,
938
+ "best_epoch": 55,
939
+ "best_val_f1": 0.9609929078014184,
940
+ "test_loss": 0.13683861011120269,
941
+ "test": {
942
+ "accuracy": 0.9589649764767381,
943
+ "precision": 0.9611955951756685,
944
+ "recall": 0.9566805845511482,
945
+ "f1": 0.9589327753073502,
946
+ "macro_precision": 0.9589719507926284,
947
+ "macro_recall": 0.9589685645268831,
948
+ "macro_f1": 0.9589649512473026
949
+ },
950
+ "trained": true,
951
+ "checkpoint": "models/part2/BiGCN_bert_seed2.pt",
952
+ "runtime_seconds": 75.53371071815491
953
+ },
954
+ {
955
+ "model": "BiGCN",
956
+ "feature": "bert",
957
+ "seed": 3,
958
+ "num_parameters": 820098,
959
+ "best_epoch": 18,
960
+ "best_val_f1": 0.9585585585585585,
961
+ "test_loss": 0.14380842394312826,
962
+ "test": {
963
+ "accuracy": 0.9555671719811814,
964
+ "precision": 0.9648562300319489,
965
+ "recall": 0.9457202505219207,
966
+ "f1": 0.9551924090669478,
967
+ "macro_precision": 0.9557340698414365,
968
+ "macro_recall": 0.9555826383499655,
969
+ "macro_f1": 0.9555640635277715
970
+ },
971
+ "trained": true,
972
+ "checkpoint": "models/part2/BiGCN_bert_seed3.pt",
973
+ "runtime_seconds": 75.76742005348206
974
+ },
975
+ {
976
+ "model": "BiGCN",
977
+ "feature": "bert",
978
+ "seed": 4,
979
+ "num_parameters": 820098,
980
+ "best_epoch": 50,
981
+ "best_val_f1": 0.9628318584070796,
982
+ "test_loss": 0.14666800865376736,
983
+ "test": {
984
+ "accuracy": 0.950601150026137,
985
+ "precision": 0.9476412649040954,
986
+ "recall": 0.954070981210856,
987
+ "f1": 0.9508452535760729,
988
+ "macro_precision": 0.9506261147925854,
989
+ "macro_recall": 0.9505957000295118,
990
+ "macro_f1": 0.9505999317525755
991
+ },
992
+ "trained": true,
993
+ "checkpoint": "models/part2/BiGCN_bert_seed4.pt",
994
+ "runtime_seconds": 76.41257739067078
995
+ }
996
+ ]
997
+ }
998
+ },
999
+ "feature_ablation": {
1000
+ "profile": {
1001
+ "model": "GraphSAGE",
1002
+ "feature": "profile",
1003
+ "group": "feature_ablation",
1004
+ "seeds": [
1005
+ 0,
1006
+ 1,
1007
+ 2,
1008
+ 3,
1009
+ 4
1010
+ ],
1011
+ "num_parameters": 37506,
1012
+ "aggregate": {
1013
+ "accuracy": {
1014
+ "mean": 0.95488761108207,
1015
+ "std": 0.004242246432885183,
1016
+ "values": [
1017
+ 0.9558285415577626,
1018
+ 0.946941975953999,
1019
+ 0.9597490852064819,
1020
+ 0.9560899111343439,
1021
+ 0.9558285415577626
1022
+ ]
1023
+ },
1024
+ "f1": {
1025
+ "mean": 0.9552057527161164,
1026
+ "std": 0.004550519832437199,
1027
+ "values": [
1028
+ 0.9562516179135387,
1029
+ 0.9465648854961832,
1030
+ 0.9601242879337131,
1031
+ 0.9564992232004143,
1032
+ 0.9565887490367326
1033
+ ]
1034
+ },
1035
+ "macro_f1": {
1036
+ "mean": 0.9548820559684904,
1037
+ "std": 0.004241416577700659,
1038
+ "values": [
1039
+ 0.95582441017081,
1040
+ 0.9469393334825842,
1041
+ 0.9597455212755637,
1042
+ 0.9560860232097058,
1043
+ 0.9558149917037879
1044
+ ]
1045
+ },
1046
+ "macro_precision": {
1047
+ "mean": 0.9550840888786969,
1048
+ "std": 0.004261313180262618,
1049
+ "values": [
1050
+ 0.9559586356160445,
1051
+ 0.9470642192556911,
1052
+ 0.9598724060265466,
1053
+ 0.9562121427478079,
1054
+ 0.9563130407473944
1055
+ ]
1056
+ },
1057
+ "macro_recall": {
1058
+ "mean": 0.9548777448655029,
1059
+ "std": 0.004232008313477819,
1060
+ "values": [
1061
+ 0.9558157264807791,
1062
+ 0.9469553716840275,
1063
+ 0.9597366896566801,
1064
+ 0.9560775065854912,
1065
+ 0.9558034299205369
1066
+ ]
1067
+ },
1068
+ "precision": {
1069
+ "mean": 0.9494358409173105,
1070
+ "std": 0.004444907943546348,
1071
+ "values": [
1072
+ 0.9486389316897792,
1073
+ 0.9548592671269251,
1074
+ 0.9527235354573484,
1075
+ 0.9491264131551901,
1076
+ 0.9418310571573091
1077
+ ]
1078
+ },
1079
+ "recall": {
1080
+ "mean": 0.961169102296451,
1081
+ "std": 0.01173845879093042,
1082
+ "values": [
1083
+ 0.9639874739039666,
1084
+ 0.9384133611691023,
1085
+ 0.9676409185803758,
1086
+ 0.9639874739039666,
1087
+ 0.9718162839248434
1088
+ ]
1089
+ }
1090
+ },
1091
+ "runs": [
1092
+ {
1093
+ "model": "GraphSAGE",
1094
+ "feature": "profile",
1095
+ "seed": 0,
1096
+ "num_parameters": 37506,
1097
+ "best_epoch": 34,
1098
+ "best_val_f1": 0.9616724738675958,
1099
+ "test_loss": 0.17898222867942767,
1100
+ "test": {
1101
+ "accuracy": 0.9558285415577626,
1102
+ "precision": 0.9486389316897792,
1103
+ "recall": 0.9639874739039666,
1104
+ "f1": 0.9562516179135387,
1105
+ "macro_precision": 0.9559586356160445,
1106
+ "macro_recall": 0.9558157264807791,
1107
+ "macro_f1": 0.95582441017081
1108
+ },
1109
+ "trained": true,
1110
+ "checkpoint": "models/part2/GraphSAGE_profile_seed0.pt",
1111
+ "runtime_seconds": 40.12610340118408
1112
+ },
1113
+ {
1114
+ "model": "GraphSAGE",
1115
+ "feature": "profile",
1116
+ "seed": 1,
1117
+ "num_parameters": 37506,
1118
+ "best_epoch": 38,
1119
+ "best_val_f1": 0.9645390070921985,
1120
+ "test_loss": 0.1824376310158075,
1121
+ "test": {
1122
+ "accuracy": 0.946941975953999,
1123
+ "precision": 0.9548592671269251,
1124
+ "recall": 0.9384133611691023,
1125
+ "f1": 0.9465648854961832,
1126
+ "macro_precision": 0.9470642192556911,
1127
+ "macro_recall": 0.9469553716840275,
1128
+ "macro_f1": 0.9469393334825842
1129
+ },
1130
+ "trained": true,
1131
+ "checkpoint": "models/part2/GraphSAGE_profile_seed1.pt",
1132
+ "runtime_seconds": 40.42488646507263
1133
+ },
1134
+ {
1135
+ "model": "GraphSAGE",
1136
+ "feature": "profile",
1137
+ "seed": 2,
1138
+ "num_parameters": 37506,
1139
+ "best_epoch": 48,
1140
+ "best_val_f1": 0.9647887323943662,
1141
+ "test_loss": 0.16779988832767884,
1142
+ "test": {
1143
+ "accuracy": 0.9597490852064819,
1144
+ "precision": 0.9527235354573484,
1145
+ "recall": 0.9676409185803758,
1146
+ "f1": 0.9601242879337131,
1147
+ "macro_precision": 0.9598724060265466,
1148
+ "macro_recall": 0.9597366896566801,
1149
+ "macro_f1": 0.9597455212755637
1150
+ },
1151
+ "trained": true,
1152
+ "checkpoint": "models/part2/GraphSAGE_profile_seed2.pt",
1153
+ "runtime_seconds": 40.31825017929077
1154
+ },
1155
+ {
1156
+ "model": "GraphSAGE",
1157
+ "feature": "profile",
1158
+ "seed": 3,
1159
+ "num_parameters": 37506,
1160
+ "best_epoch": 39,
1161
+ "best_val_f1": 0.9633507853403142,
1162
+ "test_loss": 0.17526033843180552,
1163
+ "test": {
1164
+ "accuracy": 0.9560899111343439,
1165
+ "precision": 0.9491264131551901,
1166
+ "recall": 0.9639874739039666,
1167
+ "f1": 0.9564992232004143,
1168
+ "macro_precision": 0.9562121427478079,
1169
+ "macro_recall": 0.9560775065854912,
1170
+ "macro_f1": 0.9560860232097058
1171
+ },
1172
+ "trained": true,
1173
+ "checkpoint": "models/part2/GraphSAGE_profile_seed3.pt",
1174
+ "runtime_seconds": 40.397016525268555
1175
+ },
1176
+ {
1177
+ "model": "GraphSAGE",
1178
+ "feature": "profile",
1179
+ "seed": 4,
1180
+ "num_parameters": 37506,
1181
+ "best_epoch": 56,
1182
+ "best_val_f1": 0.96,
1183
+ "test_loss": 0.18592118494060733,
1184
+ "test": {
1185
+ "accuracy": 0.9558285415577626,
1186
+ "precision": 0.9418310571573091,
1187
+ "recall": 0.9718162839248434,
1188
+ "f1": 0.9565887490367326,
1189
+ "macro_precision": 0.9563130407473944,
1190
+ "macro_recall": 0.9558034299205369,
1191
+ "macro_f1": 0.9558149917037879
1192
+ },
1193
+ "trained": true,
1194
+ "checkpoint": "models/part2/GraphSAGE_profile_seed4.pt",
1195
+ "runtime_seconds": 40.306928873062134
1196
+ }
1197
+ ]
1198
+ },
1199
+ "spacy": {
1200
+ "model": "GraphSAGE",
1201
+ "feature": "spacy",
1202
+ "group": "feature_ablation",
1203
+ "seeds": [
1204
+ 0,
1205
+ 1,
1206
+ 2,
1207
+ 3,
1208
+ 4
1209
+ ],
1210
+ "num_parameters": 148866,
1211
+ "aggregate": {
1212
+ "accuracy": {
1213
+ "mean": 0.9564035546262415,
1214
+ "std": 0.004120696743338516,
1215
+ "values": [
1216
+ 0.9490329325666492,
1217
+ 0.9560899111343439,
1218
+ 0.9581808677469943,
1219
+ 0.9571353894406691,
1220
+ 0.961578672242551
1221
+ ]
1222
+ },
1223
+ "f1": {
1224
+ "mean": 0.9563467416917671,
1225
+ "std": 0.004132261135021103,
1226
+ "values": [
1227
+ 0.9490196078431372,
1228
+ 0.9562955254942768,
1229
+ 0.9580272822665268,
1230
+ 0.9566825145272055,
1231
+ 0.9617087783276895
1232
+ ]
1233
+ },
1234
+ "macro_f1": {
1235
+ "mean": 0.9564022216965462,
1236
+ "std": 0.004120386968383356,
1237
+ "values": [
1238
+ 0.9490329290848819,
1239
+ 0.9560889392177266,
1240
+ 0.95818030779993,
1241
+ 0.9571307037198884,
1242
+ 0.9615782286603043
1243
+ ]
1244
+ },
1245
+ "macro_precision": {
1246
+ "mean": 0.956468731613908,
1247
+ "std": 0.004134790871947778,
1248
+ "values": [
1249
+ 0.9490365353364195,
1250
+ 0.956111805831865,
1251
+ 0.9582240222120197,
1252
+ 0.9573841081329353,
1253
+ 0.9615871865563009
1254
+ ]
1255
+ },
1256
+ "macro_recall": {
1257
+ "mean": 0.9564078741706652,
1258
+ "std": 0.004120420392447521,
1259
+ "values": [
1260
+ 0.949035676420116,
1261
+ 0.9560848845216365,
1262
+ 0.9581889626075266,
1263
+ 0.9571541387489206,
1264
+ 0.9615757085551269
1265
+ ]
1266
+ },
1267
+ "precision": {
1268
+ "mean": 0.9591133654054005,
1269
+ "std": 0.006433697143514017,
1270
+ "values": [
1271
+ 0.9507595599790466,
1272
+ 0.9533195020746889,
1273
+ 0.9630801687763713,
1274
+ 0.9684491978609626,
1275
+ 0.9599583983359334
1276
+ ]
1277
+ },
1278
+ "recall": {
1279
+ "mean": 0.9536534446764092,
1280
+ "std": 0.006935066906836356,
1281
+ "values": [
1282
+ 0.947286012526096,
1283
+ 0.9592901878914405,
1284
+ 0.9530271398747391,
1285
+ 0.9451983298538622,
1286
+ 0.9634655532359081
1287
+ ]
1288
+ }
1289
+ },
1290
+ "runs": [
1291
+ {
1292
+ "model": "GraphSAGE",
1293
+ "feature": "spacy",
1294
+ "seed": 0,
1295
+ "num_parameters": 148866,
1296
+ "best_epoch": 37,
1297
+ "best_val_f1": 0.9680851063829787,
1298
+ "test_loss": 0.15829328835104,
1299
+ "test": {
1300
+ "accuracy": 0.9490329325666492,
1301
+ "precision": 0.9507595599790466,
1302
+ "recall": 0.947286012526096,
1303
+ "f1": 0.9490196078431372,
1304
+ "macro_precision": 0.9490365353364195,
1305
+ "macro_recall": 0.949035676420116,
1306
+ "macro_f1": 0.9490329290848819
1307
+ },
1308
+ "trained": true,
1309
+ "checkpoint": "models/part2/GraphSAGE_spacy_seed0.pt",
1310
+ "runtime_seconds": 54.925530672073364
1311
+ },
1312
+ {
1313
+ "model": "GraphSAGE",
1314
+ "feature": "spacy",
1315
+ "seed": 1,
1316
+ "num_parameters": 148866,
1317
+ "best_epoch": 41,
1318
+ "best_val_f1": 0.9645390070921985,
1319
+ "test_loss": 0.15484427309011328,
1320
+ "test": {
1321
+ "accuracy": 0.9560899111343439,
1322
+ "precision": 0.9533195020746889,
1323
+ "recall": 0.9592901878914405,
1324
+ "f1": 0.9562955254942768,
1325
+ "macro_precision": 0.956111805831865,
1326
+ "macro_recall": 0.9560848845216365,
1327
+ "macro_f1": 0.9560889392177266
1328
+ },
1329
+ "trained": true,
1330
+ "checkpoint": "models/part2/GraphSAGE_spacy_seed1.pt",
1331
+ "runtime_seconds": 54.55896854400635
1332
+ },
1333
+ {
1334
+ "model": "GraphSAGE",
1335
+ "feature": "spacy",
1336
+ "seed": 2,
1337
+ "num_parameters": 148866,
1338
+ "best_epoch": 7,
1339
+ "best_val_f1": 0.9645390070921985,
1340
+ "test_loss": 0.16074418348600525,
1341
+ "test": {
1342
+ "accuracy": 0.9581808677469943,
1343
+ "precision": 0.9630801687763713,
1344
+ "recall": 0.9530271398747391,
1345
+ "f1": 0.9580272822665268,
1346
+ "macro_precision": 0.9582240222120197,
1347
+ "macro_recall": 0.9581889626075266,
1348
+ "macro_f1": 0.95818030779993
1349
+ },
1350
+ "trained": true,
1351
+ "checkpoint": "models/part2/GraphSAGE_spacy_seed2.pt",
1352
+ "runtime_seconds": 54.196425676345825
1353
+ },
1354
+ {
1355
+ "model": "GraphSAGE",
1356
+ "feature": "spacy",
1357
+ "seed": 3,
1358
+ "num_parameters": 148866,
1359
+ "best_epoch": 40,
1360
+ "best_val_f1": 0.9638989169675091,
1361
+ "test_loss": 0.15693167373888542,
1362
+ "test": {
1363
+ "accuracy": 0.9571353894406691,
1364
+ "precision": 0.9684491978609626,
1365
+ "recall": 0.9451983298538622,
1366
+ "f1": 0.9566825145272055,
1367
+ "macro_precision": 0.9573841081329353,
1368
+ "macro_recall": 0.9571541387489206,
1369
+ "macro_f1": 0.9571307037198884
1370
+ },
1371
+ "trained": true,
1372
+ "checkpoint": "models/part2/GraphSAGE_spacy_seed3.pt",
1373
+ "runtime_seconds": 54.43138098716736
1374
+ },
1375
+ {
1376
+ "model": "GraphSAGE",
1377
+ "feature": "spacy",
1378
+ "seed": 4,
1379
+ "num_parameters": 148866,
1380
+ "best_epoch": 12,
1381
+ "best_val_f1": 0.9646643109540636,
1382
+ "test_loss": 0.14720574138180723,
1383
+ "test": {
1384
+ "accuracy": 0.961578672242551,
1385
+ "precision": 0.9599583983359334,
1386
+ "recall": 0.9634655532359081,
1387
+ "f1": 0.9617087783276895,
1388
+ "macro_precision": 0.9615871865563009,
1389
+ "macro_recall": 0.9615757085551269,
1390
+ "macro_f1": 0.9615782286603043
1391
+ },
1392
+ "trained": true,
1393
+ "checkpoint": "models/part2/GraphSAGE_spacy_seed4.pt",
1394
+ "runtime_seconds": 55.088621854782104
1395
+ }
1396
+ ]
1397
+ },
1398
+ "bert": {
1399
+ "model": "GraphSAGE",
1400
+ "feature": "bert",
1401
+ "group": "feature_ablation",
1402
+ "seeds": [
1403
+ 0,
1404
+ 1,
1405
+ 2,
1406
+ 3,
1407
+ 4
1408
+ ],
1409
+ "num_parameters": 328578,
1410
+ "aggregate": {
1411
+ "accuracy": {
1412
+ "mean": 0.9541035023523261,
1413
+ "std": 0.00567021402680992,
1414
+ "values": [
1415
+ 0.9451123889179299,
1416
+ 0.9621014113957136,
1417
+ 0.9550444328280188,
1418
+ 0.9568740198640878,
1419
+ 0.9513852587558809
1420
+ ]
1421
+ },
1422
+ "f1": {
1423
+ "mean": 0.9543300597634014,
1424
+ "std": 0.005356633404725627,
1425
+ "values": [
1426
+ 0.9460708782742681,
1427
+ 0.961872206153037,
1428
+ 0.9547606522882693,
1429
+ 0.9574851842308684,
1430
+ 0.9514613778705637
1431
+ ]
1432
+ },
1433
+ "macro_f1": {
1434
+ "mean": 0.9540975992227054,
1435
+ "std": 0.00567441175736768,
1436
+ "values": [
1437
+ 0.9450950453106306,
1438
+ 0.9621000417618913,
1439
+ 0.9550426638064724,
1440
+ 0.9568651060374708,
1441
+ 0.951385139197062
1442
+ ]
1443
+ },
1444
+ "macro_precision": {
1445
+ "mean": 0.9543038928069405,
1446
+ "std": 0.0055789547693638405,
1447
+ "values": [
1448
+ 0.9456001952210243,
1449
+ 0.9621974460581384,
1450
+ 0.955146166544588,
1451
+ 0.9571905170138904,
1452
+ 0.951385139197062
1453
+ ]
1454
+ },
1455
+ "macro_recall": {
1456
+ "mean": 0.9540991266709659,
1457
+ "std": 0.00568009943719643,
1458
+ "values": [
1459
+ 0.9450868410409994,
1460
+ 0.9621132048661588,
1461
+ 0.955056618828493,
1462
+ 0.9568538294221163,
1463
+ 0.951385139197062
1464
+ ]
1465
+ },
1466
+ "precision": {
1467
+ "mean": 0.9519739403869233,
1468
+ "std": 0.01325130426874163,
1469
+ "values": [
1470
+ 0.9312436804853387,
1471
+ 0.969263381028087,
1472
+ 0.9623541887592789,
1473
+ 0.9455470737913486,
1474
+ 0.9514613778705637
1475
+ ]
1476
+ },
1477
+ "recall": {
1478
+ "mean": 0.9568893528183715,
1479
+ "std": 0.007898782635654536,
1480
+ "values": [
1481
+ 0.9613778705636743,
1482
+ 0.9545929018789144,
1483
+ 0.947286012526096,
1484
+ 0.9697286012526096,
1485
+ 0.9514613778705637
1486
+ ]
1487
+ }
1488
+ },
1489
+ "runs": [
1490
+ {
1491
+ "model": "GraphSAGE",
1492
+ "feature": "bert",
1493
+ "seed": 0,
1494
+ "num_parameters": 328578,
1495
+ "best_epoch": 60,
1496
+ "best_val_f1": 0.96,
1497
+ "test_loss": 0.18315207017259452,
1498
+ "test": {
1499
+ "accuracy": 0.9451123889179299,
1500
+ "precision": 0.9312436804853387,
1501
+ "recall": 0.9613778705636743,
1502
+ "f1": 0.9460708782742681,
1503
+ "macro_precision": 0.9456001952210243,
1504
+ "macro_recall": 0.9450868410409994,
1505
+ "macro_f1": 0.9450950453106306
1506
+ },
1507
+ "trained": true,
1508
+ "checkpoint": "models/part2/GraphSAGE_bert_seed0.pt",
1509
+ "runtime_seconds": 63.82997965812683
1510
+ },
1511
+ {
1512
+ "model": "GraphSAGE",
1513
+ "feature": "bert",
1514
+ "seed": 1,
1515
+ "num_parameters": 328578,
1516
+ "best_epoch": 30,
1517
+ "best_val_f1": 0.9714285714285714,
1518
+ "test_loss": 0.14362513501794938,
1519
+ "test": {
1520
+ "accuracy": 0.9621014113957136,
1521
+ "precision": 0.969263381028087,
1522
+ "recall": 0.9545929018789144,
1523
+ "f1": 0.961872206153037,
1524
+ "macro_precision": 0.9621974460581384,
1525
+ "macro_recall": 0.9621132048661588,
1526
+ "macro_f1": 0.9621000417618913
1527
+ },
1528
+ "trained": true,
1529
+ "checkpoint": "models/part2/GraphSAGE_bert_seed1.pt",
1530
+ "runtime_seconds": 63.64832878112793
1531
+ },
1532
+ {
1533
+ "model": "GraphSAGE",
1534
+ "feature": "bert",
1535
+ "seed": 2,
1536
+ "num_parameters": 328578,
1537
+ "best_epoch": 19,
1538
+ "best_val_f1": 0.9605734767025089,
1539
+ "test_loss": 0.1567136243071409,
1540
+ "test": {
1541
+ "accuracy": 0.9550444328280188,
1542
+ "precision": 0.9623541887592789,
1543
+ "recall": 0.947286012526096,
1544
+ "f1": 0.9547606522882693,
1545
+ "macro_precision": 0.955146166544588,
1546
+ "macro_recall": 0.955056618828493,
1547
+ "macro_f1": 0.9550426638064724
1548
+ },
1549
+ "trained": true,
1550
+ "checkpoint": "models/part2/GraphSAGE_bert_seed2.pt",
1551
+ "runtime_seconds": 63.792078733444214
1552
+ },
1553
+ {
1554
+ "model": "GraphSAGE",
1555
+ "feature": "bert",
1556
+ "seed": 3,
1557
+ "num_parameters": 328578,
1558
+ "best_epoch": 13,
1559
+ "best_val_f1": 0.9597197898423818,
1560
+ "test_loss": 0.17125713195840664,
1561
+ "test": {
1562
+ "accuracy": 0.9568740198640878,
1563
+ "precision": 0.9455470737913486,
1564
+ "recall": 0.9697286012526096,
1565
+ "f1": 0.9574851842308684,
1566
+ "macro_precision": 0.9571905170138904,
1567
+ "macro_recall": 0.9568538294221163,
1568
+ "macro_f1": 0.9568651060374708
1569
+ },
1570
+ "trained": true,
1571
+ "checkpoint": "models/part2/GraphSAGE_bert_seed3.pt",
1572
+ "runtime_seconds": 64.1160216331482
1573
+ },
1574
+ {
1575
+ "model": "GraphSAGE",
1576
+ "feature": "bert",
1577
+ "seed": 4,
1578
+ "num_parameters": 328578,
1579
+ "best_epoch": 45,
1580
+ "best_val_f1": 0.956369982547993,
1581
+ "test_loss": 0.16044734250919643,
1582
+ "test": {
1583
+ "accuracy": 0.9513852587558809,
1584
+ "precision": 0.9514613778705637,
1585
+ "recall": 0.9514613778705637,
1586
+ "f1": 0.9514613778705637,
1587
+ "macro_precision": 0.951385139197062,
1588
+ "macro_recall": 0.951385139197062,
1589
+ "macro_f1": 0.951385139197062
1590
+ },
1591
+ "trained": true,
1592
+ "checkpoint": "models/part2/GraphSAGE_bert_seed4.pt",
1593
+ "runtime_seconds": 63.92004680633545
1594
+ }
1595
+ ]
1596
+ },
1597
+ "content": {
1598
+ "model": "GraphSAGE",
1599
+ "feature": "content",
1600
+ "group": "feature_ablation",
1601
+ "seeds": [
1602
+ 0,
1603
+ 1,
1604
+ 2,
1605
+ 3,
1606
+ 4
1607
+ ],
1608
+ "num_parameters": 152706,
1609
+ "aggregate": {
1610
+ "accuracy": {
1611
+ "mean": 0.973967590172504,
1612
+ "std": 0.003121588839902492,
1613
+ "values": [
1614
+ 0.9796131730266597,
1615
+ 0.9746471510716153,
1616
+ 0.9728175640355463,
1617
+ 0.9704652378463147,
1618
+ 0.9722948248823837
1619
+ ]
1620
+ },
1621
+ "f1": {
1622
+ "mean": 0.9742797686396882,
1623
+ "std": 0.0030074514461421635,
1624
+ "values": [
1625
+ 0.979676915059927,
1626
+ 0.974941875484371,
1627
+ 0.9732510288065843,
1628
+ 0.9707783811740367,
1629
+ 0.9727506426735218
1630
+ ]
1631
+ },
1632
+ "macro_f1": {
1633
+ "mean": 0.9739631912271525,
1634
+ "std": 0.0031234825485214005,
1635
+ "values": [
1636
+ 0.9796129724749032,
1637
+ 0.9746436433756158,
1638
+ 0.9728104240738553,
1639
+ 0.9704618458049841,
1640
+ 0.9722870704064047
1641
+ ]
1642
+ },
1643
+ "macro_precision": {
1644
+ "mean": 0.9742216672536916,
1645
+ "std": 0.00301766995917237,
1646
+ "values": [
1647
+ 0.9796200998609641,
1648
+ 0.9748588292705598,
1649
+ 0.9732435324313857,
1650
+ 0.9706353664787288,
1651
+ 0.9727505082268191
1652
+ ]
1653
+ },
1654
+ "macro_recall": {
1655
+ "mean": 0.9739516225994382,
1656
+ "std": 0.003127473192867119,
1657
+ "values": [
1658
+ 0.9796106635770421,
1659
+ 0.9746311031927335,
1660
+ 0.9727945436063352,
1661
+ 0.970450819224169,
1662
+ 0.9722709833969111
1663
+ ]
1664
+ },
1665
+ "precision": {
1666
+ "mean": 0.9646655830854123,
1667
+ "std": 0.007137314666002703,
1668
+ "values": [
1669
+ 0.9781477627471384,
1670
+ 0.9652173913043478,
1671
+ 0.9594320486815415,
1672
+ 0.9620707329574577,
1673
+ 0.9584599797365755
1674
+ ]
1675
+ },
1676
+ "recall": {
1677
+ "mean": 0.9841336116910229,
1678
+ "std": 0.0032105545927410386,
1679
+ "values": [
1680
+ 0.9812108559498957,
1681
+ 0.9848643006263048,
1682
+ 0.9874739039665971,
1683
+ 0.9796450939457203,
1684
+ 0.9874739039665971
1685
+ ]
1686
+ }
1687
+ },
1688
+ "runs": [
1689
+ {
1690
+ "model": "GraphSAGE",
1691
+ "feature": "content",
1692
+ "seed": 0,
1693
+ "num_parameters": 152706,
1694
+ "best_epoch": 36,
1695
+ "best_val_f1": 0.9769911504424779,
1696
+ "test_loss": 0.07672619009541394,
1697
+ "test": {
1698
+ "accuracy": 0.9796131730266597,
1699
+ "precision": 0.9781477627471384,
1700
+ "recall": 0.9812108559498957,
1701
+ "f1": 0.979676915059927,
1702
+ "macro_precision": 0.9796200998609641,
1703
+ "macro_recall": 0.9796106635770421,
1704
+ "macro_f1": 0.9796129724749032
1705
+ },
1706
+ "trained": true,
1707
+ "checkpoint": "models/part2/GraphSAGE_content_seed0.pt",
1708
+ "runtime_seconds": 54.7298150062561
1709
+ },
1710
+ {
1711
+ "model": "GraphSAGE",
1712
+ "feature": "content",
1713
+ "seed": 1,
1714
+ "num_parameters": 152706,
1715
+ "best_epoch": 47,
1716
+ "best_val_f1": 0.9753521126760564,
1717
+ "test_loss": 0.11843792555909224,
1718
+ "test": {
1719
+ "accuracy": 0.9746471510716153,
1720
+ "precision": 0.9652173913043478,
1721
+ "recall": 0.9848643006263048,
1722
+ "f1": 0.974941875484371,
1723
+ "macro_precision": 0.9748588292705598,
1724
+ "macro_recall": 0.9746311031927335,
1725
+ "macro_f1": 0.9746436433756158
1726
+ },
1727
+ "trained": true,
1728
+ "checkpoint": "models/part2/GraphSAGE_content_seed1.pt",
1729
+ "runtime_seconds": 59.26750612258911
1730
+ },
1731
+ {
1732
+ "model": "GraphSAGE",
1733
+ "feature": "content",
1734
+ "seed": 2,
1735
+ "num_parameters": 152706,
1736
+ "best_epoch": 60,
1737
+ "best_val_f1": 0.9736379613356766,
1738
+ "test_loss": 0.13737354388239492,
1739
+ "test": {
1740
+ "accuracy": 0.9728175640355463,
1741
+ "precision": 0.9594320486815415,
1742
+ "recall": 0.9874739039665971,
1743
+ "f1": 0.9732510288065843,
1744
+ "macro_precision": 0.9732435324313857,
1745
+ "macro_recall": 0.9727945436063352,
1746
+ "macro_f1": 0.9728104240738553
1747
+ },
1748
+ "trained": true,
1749
+ "checkpoint": "models/part2/GraphSAGE_content_seed2.pt",
1750
+ "runtime_seconds": 63.139482259750366
1751
+ },
1752
+ {
1753
+ "model": "GraphSAGE",
1754
+ "feature": "content",
1755
+ "seed": 3,
1756
+ "num_parameters": 152706,
1757
+ "best_epoch": 34,
1758
+ "best_val_f1": 0.9700176366843033,
1759
+ "test_loss": 0.13795428966516235,
1760
+ "test": {
1761
+ "accuracy": 0.9704652378463147,
1762
+ "precision": 0.9620707329574577,
1763
+ "recall": 0.9796450939457203,
1764
+ "f1": 0.9707783811740367,
1765
+ "macro_precision": 0.9706353664787288,
1766
+ "macro_recall": 0.970450819224169,
1767
+ "macro_f1": 0.9704618458049841
1768
+ },
1769
+ "trained": true,
1770
+ "checkpoint": "models/part2/GraphSAGE_content_seed3.pt",
1771
+ "runtime_seconds": 63.350873947143555
1772
+ },
1773
+ {
1774
+ "model": "GraphSAGE",
1775
+ "feature": "content",
1776
+ "seed": 4,
1777
+ "num_parameters": 152706,
1778
+ "best_epoch": 21,
1779
+ "best_val_f1": 0.9719298245614035,
1780
+ "test_loss": 0.1188829277923951,
1781
+ "test": {
1782
+ "accuracy": 0.9722948248823837,
1783
+ "precision": 0.9584599797365755,
1784
+ "recall": 0.9874739039665971,
1785
+ "f1": 0.9727506426735218,
1786
+ "macro_precision": 0.9727505082268191,
1787
+ "macro_recall": 0.9722709833969111,
1788
+ "macro_f1": 0.9722870704064047
1789
+ },
1790
+ "trained": true,
1791
+ "checkpoint": "models/part2/GraphSAGE_content_seed4.pt",
1792
+ "runtime_seconds": 62.411741971969604
1793
+ }
1794
+ ]
1795
+ }
1796
+ },
1797
+ "config": {
1798
+ "dataset": "gossipcop",
1799
+ "data_root": "data/part2",
1800
+ "checkpoint_dir": "models/part2",
1801
+ "results_dir": "results/part2",
1802
+ "output_dir": "output/part2",
1803
+ "device": "auto",
1804
+ "feature_norm": "standard",
1805
+ "num_workers": 0,
1806
+ "save_checkpoints": true,
1807
+ "epochs": 60,
1808
+ "batch_size": 128,
1809
+ "lr": 0.01,
1810
+ "weight_decay": 0.01,
1811
+ "patience": 0,
1812
+ "hidden_channels": 128,
1813
+ "num_layers": 2,
1814
+ "dropout": 0.5,
1815
+ "heads": 4,
1816
+ "concat_news": true,
1817
+ "pooling": "max",
1818
+ "bigcn_p": 0.1,
1819
+ "bigcn_lambda": 0.8,
1820
+ "bigcn_k": 2,
1821
+ "bigcn_l1": 0.001,
1822
+ "seeds": [
1823
+ 0,
1824
+ 1,
1825
+ 2,
1826
+ 3,
1827
+ 4
1828
+ ],
1829
+ "architecture_comparison": {
1830
+ "feature": "bert",
1831
+ "models": [
1832
+ "RootOnlyMLP",
1833
+ "GCN",
1834
+ "GraphSAGE",
1835
+ "GAT",
1836
+ "BiGCN"
1837
+ ]
1838
+ },
1839
+ "feature_ablation": {
1840
+ "model": "GraphSAGE",
1841
+ "features": [
1842
+ "profile",
1843
+ "spacy",
1844
+ "bert",
1845
+ "content"
1846
+ ]
1847
+ },
1848
+ "huggingface": {
1849
+ "repo_id": "Invictus22/anlp-a3"
1850
+ },
1851
+ "wandb": {
1852
+ "enabled": true,
1853
+ "mode": "online",
1854
+ "tags": [
1855
+ "final"
1856
+ ],
1857
+ "project": "anlp-a3-part2",
1858
+ "entity": "shrish-kadam-iiit-hyderabad"
1859
+ }
1860
+ }
1861
+ }