khoa-klaytn commited on
Commit
bd895ee
·
1 Parent(s): 714f3cc

add model evaluation

Browse files
Files changed (2) hide show
  1. AnglE +1 -1
  2. train.ipynb +79 -391
AnglE CHANGED
@@ -1 +1 @@
1
- Subproject commit dcfb63694ce4e702e05fec5f1df39bfb3e9a86ee
 
1
+ Subproject commit 818621c365eb8b3ac8503ef8270fff9781b12f1f
train.ipynb CHANGED
@@ -18,7 +18,7 @@
18
  },
19
  {
20
  "cell_type": "code",
21
- "execution_count": 1,
22
  "metadata": {},
23
  "outputs": [],
24
  "source": [
@@ -30,7 +30,7 @@
30
  "from checker.config import *\n",
31
  "seed()\n",
32
  "\n",
33
- "from checker.AnglE.angle_emb import AnglE, AngleDataTokenizer\n",
34
  "from datasets import load_dataset\n",
35
  "import numpy as np\n",
36
  "import torch\n",
@@ -50,7 +50,7 @@
50
  },
51
  {
52
  "cell_type": "code",
53
- "execution_count": 2,
54
  "metadata": {},
55
  "outputs": [],
56
  "source": [
@@ -62,61 +62,13 @@
62
  },
63
  {
64
  "cell_type": "code",
65
- "execution_count": 3,
66
  "metadata": {},
67
- "outputs": [
68
- {
69
- "name": "stderr",
70
- "output_type": "stream",
71
- "text": [
72
- "INFO:AnglE:LLM detected, automatically set prompt. You can change this setting by manually configuring the `set_prompt()` function.\n",
73
- "INFO:AnglE:Prompt is set, the prompt will be automatically applied during the encoding phase. To disable prompt setting, please configure set_prompt(prompt=None)\n",
74
- "INFO:AnglE:LLaMA detected, automatically set `apply_bfloat16=True`. You can change this setting by manually configuring the `apply_bfloat16`.\n"
75
- ]
76
- },
77
- {
78
- "name": "stderr",
79
- "output_type": "stream",
80
- "text": [
81
- "INFO:accelerate.utils.modeling:We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).\n"
82
- ]
83
- },
84
- {
85
- "data": {
86
- "application/vnd.jupyter.widget-view+json": {
87
- "model_id": "3f6c95fa305144c58e08d52c4749f908",
88
- "version_major": 2,
89
- "version_minor": 0
90
- },
91
- "text/plain": [
92
- "Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]"
93
- ]
94
- },
95
- "metadata": {},
96
- "output_type": "display_data"
97
- },
98
- {
99
- "name": "stderr",
100
- "output_type": "stream",
101
- "text": [
102
- "/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:389: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.\n",
103
- " warnings.warn(\n",
104
- "/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:394: UserWarning: `do_sample` is set to `False`. However, `top_p` is set to `0.6` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_p`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.\n",
105
- " warnings.warn(\n",
106
- "INFO:AnglE:>>> load lora weight from /notebooks/models/checker/models/angle-llama-7b-nli-v2\n"
107
- ]
108
- },
109
- {
110
- "name": "stdout",
111
- "output_type": "stream",
112
- "text": [
113
- "trainable params: 79,953,920 || all params: 6,818,369,536 || trainable%: 1.172625208678628\n"
114
- ]
115
- }
116
- ],
117
  "source": [
118
  "# https://github.com/SeanLee97/AnglE/issues/24#issuecomment-1870845350\n",
119
  "model_config = dict(\n",
 
120
  " max_length=50,\n",
121
  " apply_lora=True,\n",
122
  " pooling_strategy=\"cls\",\n",
@@ -126,13 +78,12 @@
126
  " 'lora_dropout': .1,\n",
127
  " 'target_modules': ['q_proj', 'v_proj'],\n",
128
  " },\n",
129
- " train_mode=True,\n",
130
  " is_llm=True,\n",
131
  " load_kbit=4,\n",
132
  " workers=25,\n",
133
  ")\n",
134
  "model = AnglE(\n",
135
- " p(\"models/Llama-2-7b-hf\"),\n",
136
  " pretrained_lora_path=p(\"models/angle-llama-7b-nli-v2\"),\n",
137
  " **model_config,\n",
138
  ")"
@@ -140,24 +91,9 @@
140
  },
141
  {
142
  "cell_type": "code",
143
- "execution_count": 4,
144
  "metadata": {},
145
- "outputs": [
146
- {
147
- "data": {
148
- "application/vnd.jupyter.widget-view+json": {
149
- "model_id": "71db95f3c49d436eba263bcaee92e0a6",
150
- "version_major": 2,
151
- "version_minor": 0
152
- },
153
- "text/plain": [
154
- "Map: 0%| | 0/616 [00:00<?, ? examples/s]"
155
- ]
156
- },
157
- "metadata": {},
158
- "output_type": "display_data"
159
- }
160
- ],
161
  "source": [
162
  "def SplitDs(split: str, shuffle=False):\n",
163
  " split_ds = ds[split]\n",
@@ -180,77 +116,9 @@
180
  },
181
  {
182
  "cell_type": "code",
183
- "execution_count": 5,
184
  "metadata": {},
185
- "outputs": [
186
- {
187
- "name": "stderr",
188
- "output_type": "stream",
189
- "text": [
190
- "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmk-tran\u001b[0m (\u001b[33mklayassist\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
191
- ]
192
- },
193
- {
194
- "data": {
195
- "text/html": [
196
- "Tracking run with wandb version 0.16.1"
197
- ],
198
- "text/plain": [
199
- "<IPython.core.display.HTML object>"
200
- ]
201
- },
202
- "metadata": {},
203
- "output_type": "display_data"
204
- },
205
- {
206
- "data": {
207
- "text/html": [
208
- "Run data is saved locally in <code>/notebooks/wandb/run-20231228_142958-wwcvk5ij</code>"
209
- ],
210
- "text/plain": [
211
- "<IPython.core.display.HTML object>"
212
- ]
213
- },
214
- "metadata": {},
215
- "output_type": "display_data"
216
- },
217
- {
218
- "data": {
219
- "text/html": [
220
- "Syncing run <strong><a href='https://wandb.ai/klayassist/checker/runs/wwcvk5ij' target=\"_blank\">misunderstood-pyramid-12</a></strong> to <a href='https://wandb.ai/klayassist/checker' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
221
- ],
222
- "text/plain": [
223
- "<IPython.core.display.HTML object>"
224
- ]
225
- },
226
- "metadata": {},
227
- "output_type": "display_data"
228
- },
229
- {
230
- "data": {
231
- "text/html": [
232
- " View project at <a href='https://wandb.ai/klayassist/checker' target=\"_blank\">https://wandb.ai/klayassist/checker</a>"
233
- ],
234
- "text/plain": [
235
- "<IPython.core.display.HTML object>"
236
- ]
237
- },
238
- "metadata": {},
239
- "output_type": "display_data"
240
- },
241
- {
242
- "data": {
243
- "text/html": [
244
- " View run at <a href='https://wandb.ai/klayassist/checker/runs/wwcvk5ij' target=\"_blank\">https://wandb.ai/klayassist/checker/runs/wwcvk5ij</a>"
245
- ],
246
- "text/plain": [
247
- "<IPython.core.display.HTML object>"
248
- ]
249
- },
250
- "metadata": {},
251
- "output_type": "display_data"
252
- }
253
- ],
254
  "source": [
255
  "train_config = dict(\n",
256
  " batch_size=128,\n",
@@ -279,179 +147,7 @@
279
  "cell_type": "code",
280
  "execution_count": null,
281
  "metadata": {},
282
- "outputs": [
283
- {
284
- "name": "stderr",
285
- "output_type": "stream",
286
- "text": [
287
- "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
288
- "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
289
- " warnings.warn(\n"
290
- ]
291
- },
292
- {
293
- "data": {
294
- "text/html": [
295
- "\n",
296
- " <div>\n",
297
- " \n",
298
- " <progress value='105' max='105' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
299
- " [105/105 57:39, Epoch 5/5]\n",
300
- " </div>\n",
301
- " <table border=\"1\" class=\"dataframe\">\n",
302
- " <thead>\n",
303
- " <tr style=\"text-align: left;\">\n",
304
- " <th>Step</th>\n",
305
- " <th>Training Loss</th>\n",
306
- " </tr>\n",
307
- " </thead>\n",
308
- " <tbody>\n",
309
- " <tr>\n",
310
- " <td>10</td>\n",
311
- " <td>207.047700</td>\n",
312
- " </tr>\n",
313
- " <tr>\n",
314
- " <td>20</td>\n",
315
- " <td>189.355100</td>\n",
316
- " </tr>\n",
317
- " <tr>\n",
318
- " <td>30</td>\n",
319
- " <td>172.751700</td>\n",
320
- " </tr>\n",
321
- " <tr>\n",
322
- " <td>40</td>\n",
323
- " <td>159.993900</td>\n",
324
- " </tr>\n",
325
- " <tr>\n",
326
- " <td>50</td>\n",
327
- " <td>141.696300</td>\n",
328
- " </tr>\n",
329
- " <tr>\n",
330
- " <td>60</td>\n",
331
- " <td>134.927300</td>\n",
332
- " </tr>\n",
333
- " <tr>\n",
334
- " <td>70</td>\n",
335
- " <td>122.186600</td>\n",
336
- " </tr>\n",
337
- " <tr>\n",
338
- " <td>80</td>\n",
339
- " <td>120.693500</td>\n",
340
- " </tr>\n",
341
- " <tr>\n",
342
- " <td>90</td>\n",
343
- " <td>104.752700</td>\n",
344
- " </tr>\n",
345
- " <tr>\n",
346
- " <td>100</td>\n",
347
- " <td>99.367900</td>\n",
348
- " </tr>\n",
349
- " </tbody>\n",
350
- "</table><p>"
351
- ],
352
- "text/plain": [
353
- "<IPython.core.display.HTML object>"
354
- ]
355
- },
356
- "metadata": {},
357
- "output_type": "display_data"
358
- },
359
- {
360
- "name": "stderr",
361
- "output_type": "stream",
362
- "text": [
363
- "Evaluate: 5it [00:51, 10.32s/it]\n"
364
- ]
365
- },
366
- {
367
- "name": "stdout",
368
- "output_type": "stream",
369
- "text": [
370
- "new best corrcoef!\n",
371
- "save to /notebooks/models/checker/models/ckpts/angle-llama-7b-nli-v2/best-checkpoint\n",
372
- "corrcoef: 0.24358292779817248, accuracy: 0.7256493506493507, best corrcoef: 0.24358292779817248\n"
373
- ]
374
- },
375
- {
376
- "name": "stderr",
377
- "output_type": "stream",
378
- "text": [
379
- "Evaluate: 5it [00:51, 10.33s/it]\n"
380
- ]
381
- },
382
- {
383
- "name": "stdout",
384
- "output_type": "stream",
385
- "text": [
386
- "new best corrcoef!\n",
387
- "save to /notebooks/models/checker/models/ckpts/angle-llama-7b-nli-v2/best-checkpoint\n",
388
- "corrcoef: 0.2794266466331196, accuracy: 0.7386363636363636, best corrcoef: 0.2794266466331196\n"
389
- ]
390
- },
391
- {
392
- "name": "stderr",
393
- "output_type": "stream",
394
- "text": [
395
- "Evaluate: 5it [00:51, 10.33s/it]\n"
396
- ]
397
- },
398
- {
399
- "name": "stdout",
400
- "output_type": "stream",
401
- "text": [
402
- "new best corrcoef!\n",
403
- "save to /notebooks/models/checker/models/ckpts/angle-llama-7b-nli-v2/best-checkpoint\n",
404
- "corrcoef: 0.43219492716333224, accuracy: 0.7873376623376623, best corrcoef: 0.43219492716333224\n"
405
- ]
406
- },
407
- {
408
- "name": "stderr",
409
- "output_type": "stream",
410
- "text": [
411
- "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
412
- " warnings.warn(\n",
413
- "Evaluate: 5it [00:51, 10.33s/it]\n"
414
- ]
415
- },
416
- {
417
- "name": "stdout",
418
- "output_type": "stream",
419
- "text": [
420
- "new best corrcoef!\n",
421
- "save to /notebooks/models/checker/models/ckpts/angle-llama-7b-nli-v2/best-checkpoint\n",
422
- "corrcoef: 0.7381168360095011, accuracy: 0.9366883116883117, best corrcoef: 0.7381168360095011\n"
423
- ]
424
- },
425
- {
426
- "name": "stderr",
427
- "output_type": "stream",
428
- "text": [
429
- "Process ForkProcess-8:\n",
430
- "Process ForkProcess-7:\n",
431
- "Process ForkProcess-4:\n",
432
- "Process ForkProcess-6:\n",
433
- "Process ForkProcess-5:\n",
434
- "Traceback (most recent call last):\n",
435
- " File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 314, in _bootstrap\n",
436
- " self.run()\n",
437
- "Traceback (most recent call last):\n",
438
- "Process ForkProcess-3:\n",
439
- " File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 314, in _bootstrap\n",
440
- " self.run()\n",
441
- " File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 108, in run\n",
442
- " self._target(*self._args, **self._kwargs)\n",
443
- "Traceback (most recent call last):\n",
444
- "Traceback (most recent call last):\n",
445
- "Traceback (most recent call last):\n",
446
- " File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 314, in _bootstrap\n",
447
- " self.run()\n",
448
- " File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 314, in _bootstrap\n",
449
- " self.run()\n",
450
- " File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 108, in run\n",
451
- " self._target(*self._args, **self._kwargs)\n"
452
- ]
453
- }
454
- ],
455
  "source": [
456
  "model.fit(\n",
457
  " train_ds=train_ds,\n",
@@ -463,93 +159,85 @@
463
  },
464
  {
465
  "cell_type": "code",
466
- "execution_count": 7,
467
  "metadata": {},
468
- "outputs": [
469
- {
470
- "name": "stderr",
471
- "output_type": "stream",
472
- "text": [
473
- "wandb: WARNING Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job\n"
474
- ]
475
- },
476
- {
477
- "data": {
478
- "application/vnd.jupyter.widget-view+json": {
479
- "model_id": "9fdb56e3441c46a5af78a31231ba3cc8",
480
- "version_major": 2,
481
- "version_minor": 0
482
- },
483
- "text/plain": [
484
- "VBox(children=(Label(value='0.002 MB of 0.002 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))"
485
- ]
486
- },
487
- "metadata": {},
488
- "output_type": "display_data"
489
- },
490
- {
491
- "data": {
492
- "text/html": [
493
- "<style>\n",
494
- " table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
495
- " .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
496
- " .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
497
- " </style>\n",
498
- "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>train/epoch</td><td>▁▂▂▃▄▅▅▆▇██</td></tr><tr><td>train/global_step</td><td>▁▂▂▃▄▅▅▆▇██</td></tr><tr><td>train/learning_rate</td><td>▁▂▃▃▄▅▆▆▇█</td></tr><tr><td>train/loss</td><td>█▇▆▅▄▃▂▂▁▁</td></tr><tr><td>train/total_flos</td><td>▁</td></tr><tr><td>train/train_loss</td><td>▁</td></tr><tr><td>train/train_runtime</td><td>▁</td></tr><tr><td>train/train_samples_per_second</td><td>▁</td></tr><tr><td>train/train_steps_per_second</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>train/epoch</td><td>5.0</td></tr><tr><td>train/global_step</td><td>105</td></tr><tr><td>train/learning_rate</td><td>0.0001</td></tr><tr><td>train/loss</td><td>99.3679</td></tr><tr><td>train/total_flos</td><td>5.3404758122496e+16</td></tr><tr><td>train/train_loss</td><td>143.6141</td></tr><tr><td>train/train_runtime</td><td>3490.9122</td></tr><tr><td>train/train_samples_per_second</td><td>3.813</td></tr><tr><td>train/train_steps_per_second</td><td>0.03</td></tr></table><br/></div></div>"
499
- ],
500
- "text/plain": [
501
- "<IPython.core.display.HTML object>"
502
- ]
503
- },
504
- "metadata": {},
505
- "output_type": "display_data"
506
- },
507
- {
508
- "data": {
509
- "text/html": [
510
- " View run <strong style=\"color:#cdcd00\">misunderstood-pyramid-12</strong> at: <a href='https://wandb.ai/klayassist/checker/runs/wwcvk5ij' target=\"_blank\">https://wandb.ai/klayassist/checker/runs/wwcvk5ij</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
511
- ],
512
- "text/plain": [
513
- "<IPython.core.display.HTML object>"
514
- ]
515
- },
516
- "metadata": {},
517
- "output_type": "display_data"
518
- },
519
- {
520
- "data": {
521
- "text/html": [
522
- "Find logs at: <code>./wandb/run-20231228_142958-wwcvk5ij/logs</code>"
523
- ],
524
- "text/plain": [
525
- "<IPython.core.display.HTML object>"
526
- ]
527
- },
528
- "metadata": {},
529
- "output_type": "display_data"
530
- }
531
- ],
532
  "source": [
533
  "run.finish()"
534
  ]
535
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
536
  {
537
  "cell_type": "code",
538
- "execution_count": 9,
539
  "metadata": {},
540
- "outputs": [
541
- {
542
- "name": "stdout",
543
- "output_type": "stream",
544
- "text": [
545
- "Corrcoef: 0.7606, Accuracy: 0.9429\n"
546
- ]
547
- }
548
- ],
549
  "source": [
550
  "corrcoef, accuracy = model.evaluate(test_ds, batch_size=128, device=model.device)\n",
551
  "print(f\"Corrcoef: {corrcoef:.4f}, Accuracy: {accuracy:.4f}\")"
552
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
553
  }
554
  ],
555
  "metadata": {
 
18
  },
19
  {
20
  "cell_type": "code",
21
+ "execution_count": null,
22
  "metadata": {},
23
  "outputs": [],
24
  "source": [
 
30
  "from checker.config import *\n",
31
  "seed()\n",
32
  "\n",
33
+ "from checker.AnglE.angle_emb import AnglE, AngleDataTokenizer, optimal_threshold\n",
34
  "from datasets import load_dataset\n",
35
  "import numpy as np\n",
36
  "import torch\n",
 
50
  },
51
  {
52
  "cell_type": "code",
53
+ "execution_count": null,
54
  "metadata": {},
55
  "outputs": [],
56
  "source": [
 
62
  },
63
  {
64
  "cell_type": "code",
65
+ "execution_count": null,
66
  "metadata": {},
67
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  "source": [
69
  "# https://github.com/SeanLee97/AnglE/issues/24#issuecomment-1870845350\n",
70
  "model_config = dict(\n",
71
+ " model_name_or_path=p(\"models/Llama-2-7b-hf\"),\n",
72
  " max_length=50,\n",
73
  " apply_lora=True,\n",
74
  " pooling_strategy=\"cls\",\n",
 
78
  " 'lora_dropout': .1,\n",
79
  " 'target_modules': ['q_proj', 'v_proj'],\n",
80
  " },\n",
81
+ " # train_mode=True,\n",
82
  " is_llm=True,\n",
83
  " load_kbit=4,\n",
84
  " workers=25,\n",
85
  ")\n",
86
  "model = AnglE(\n",
 
87
  " pretrained_lora_path=p(\"models/angle-llama-7b-nli-v2\"),\n",
88
  " **model_config,\n",
89
  ")"
 
91
  },
92
  {
93
  "cell_type": "code",
94
+ "execution_count": null,
95
  "metadata": {},
96
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  "source": [
98
  "def SplitDs(split: str, shuffle=False):\n",
99
  " split_ds = ds[split]\n",
 
116
  },
117
  {
118
  "cell_type": "code",
119
+ "execution_count": null,
120
  "metadata": {},
121
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  "source": [
123
  "train_config = dict(\n",
124
  " batch_size=128,\n",
 
147
  "cell_type": "code",
148
  "execution_count": null,
149
  "metadata": {},
150
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  "source": [
152
  "model.fit(\n",
153
  " train_ds=train_ds,\n",
 
159
  },
160
  {
161
  "cell_type": "code",
162
+ "execution_count": null,
163
  "metadata": {},
164
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  "source": [
166
  "run.finish()"
167
  ]
168
  },
169
+ {
170
+ "cell_type": "markdown",
171
+ "metadata": {},
172
+ "source": [
173
+ "## Evaluate"
174
+ ]
175
+ },
176
+ {
177
+ "cell_type": "code",
178
+ "execution_count": null,
179
+ "metadata": {},
180
+ "outputs": [],
181
+ "source": [
182
+ "model = AnglE(\n",
183
+ " pretrained_lora_path=p(\"models/ckpts/angle-llama-7b-nli-v2/best-checkpoint\"), \n",
184
+ " **model_config,\n",
185
+ ")"
186
+ ]
187
+ },
188
+ {
189
+ "cell_type": "code",
190
+ "execution_count": null,
191
+ "metadata": {},
192
+ "outputs": [],
193
+ "source": [
194
+ "model.train_mode = False"
195
+ ]
196
+ },
197
  {
198
  "cell_type": "code",
199
+ "execution_count": null,
200
  "metadata": {},
201
+ "outputs": [],
 
 
 
 
 
 
 
 
202
  "source": [
203
  "corrcoef, accuracy = model.evaluate(test_ds, batch_size=128, device=model.device)\n",
204
  "print(f\"Corrcoef: {corrcoef:.4f}, Accuracy: {accuracy:.4f}\")"
205
  ]
206
+ },
207
+ {
208
+ "cell_type": "code",
209
+ "execution_count": null,
210
+ "metadata": {},
211
+ "outputs": [],
212
+ "source": [
213
+ "# Print inaccurate predictions\n",
214
+ "y_trues, y_preds = model.predict(test_ds, batch_size=128, device=model.device)"
215
+ ]
216
+ },
217
+ {
218
+ "cell_type": "code",
219
+ "execution_count": null,
220
+ "metadata": {},
221
+ "outputs": [],
222
+ "source": [
223
+ "threshold = optimal_threshold(y_trues, y_preds)[0][0]"
224
+ ]
225
+ },
226
+ {
227
+ "cell_type": "code",
228
+ "execution_count": null,
229
+ "metadata": {},
230
+ "outputs": [],
231
+ "source": [
232
+ "for i, (y_true, y_pred) in enumerate(zip(y_trues, y_preds)):\n",
233
+ " if (y_true > .5) != (y_pred > threshold):\n",
234
+ " X = test_ds[i]\n",
235
+ " print(f\"Text1: {X['text1']}\")\n",
236
+ " print(f\"Text2: {X['text2']}\")\n",
237
+ " print(f\"Label: {X['label']}\")\n",
238
+ " print(f\"Prediction: {y_pred}\")\n",
239
+ " print()"
240
+ ]
241
  }
242
  ],
243
  "metadata": {