convitom commited on
Commit
37bb3d9
·
1 Parent(s): 5250d5a
scripts/_build_eval_notebook.py CHANGED
@@ -810,9 +810,12 @@ import shlex
810
  RESULTS_DIR_LOCAL = WORK / LOCAL_RESULTS_DIR
811
  RESULTS_DIR_LOCAL.mkdir(parents=True, exist_ok=True)
812
 
813
- # evaluate.py expects --checkpoint to be a `<dir>/<name>_projection.pt` path;
814
- # it derives the LoRA dir + CheXpert classifier path from the same folder.
815
- CKPT_ARG = str(PROJ_PT)
 
 
 
816
 
817
  extra = ''
818
  if LLM_JUDGE:
 
810
  RESULTS_DIR_LOCAL = WORK / LOCAL_RESULTS_DIR
811
  RESULTS_DIR_LOCAL.mkdir(parents=True, exist_ok=True)
812
 
813
+ # evaluate.py forwards --checkpoint to utils.checkpoint.load_checkpoint,
814
+ # which reads <dir>/checkpoint_projection.pt + <dir>/checkpoint_lora/.
815
+ # Pass the DIRECTORY — passing the .pt file makes load_checkpoint mis-derive
816
+ # the filename (checkpoint_projection_projection.pt) and silently skip both
817
+ # projection AND LoRA, leaving you with raw 4-bit Vicuna.
818
+ CKPT_ARG = str(CKPT_DIR_PULLED)
819
 
820
  extra = ''
821
  if LLM_JUDGE:
scripts/_build_inference_notebook.py CHANGED
@@ -368,9 +368,13 @@ _t0 = time.time()
368
  model = CXRVisionLanguageModel(model_cfg)
369
  print(f' built in {time.time()-_t0:.1f}s', flush=True)
370
 
371
- print(f'[2/3] Loading checkpoint from {PROJ_PT.name} …')
372
  _t0 = time.time()
373
- load_checkpoint(model, str(PROJ_PT))
 
 
 
 
374
  print(f' loaded in {time.time()-_t0:.1f}s', flush=True)
375
 
376
  print('[3/3] Moving to cuda + eval()')
 
368
  model = CXRVisionLanguageModel(model_cfg)
369
  print(f' built in {time.time()-_t0:.1f}s', flush=True)
370
 
371
+ print(f'[2/3] Loading checkpoint from {CKPT_DIR_PULLED} …')
372
  _t0 = time.time()
373
+ # Pass the directory, not the .pt file: load_checkpoint splits suffix off
374
+ # the path stem, so passing checkpoint_projection.pt makes it look for
375
+ # checkpoint_projection_projection.pt (which does not exist) and silently
376
+ # skips both projection AND lora — model falls back to raw 4-bit Vicuna.
377
+ load_checkpoint(model, str(CKPT_DIR_PULLED))
378
  print(f' loaded in {time.time()-_t0:.1f}s', flush=True)
379
 
380
  print('[3/3] Moving to cuda + eval()')
scripts/cxrvlm_colab_eval.ipynb CHANGED
@@ -35,9 +35,9 @@
35
  },
36
  {
37
  "cell_type": "code",
 
38
  "id": "eval-select",
39
  "metadata": {},
40
- "execution_count": null,
41
  "outputs": [],
42
  "source": [
43
  "# ── Platform ─────────────────────────────────────────────────────\n",
@@ -118,9 +118,9 @@
118
  },
119
  {
120
  "cell_type": "code",
 
121
  "id": "eval-env",
122
  "metadata": {},
123
- "execution_count": null,
124
  "outputs": [],
125
  "source": [
126
  "import os\n",
@@ -137,9 +137,9 @@
137
  },
138
  {
139
  "cell_type": "code",
 
140
  "id": "eval-pip",
141
  "metadata": {},
142
- "execution_count": null,
143
  "outputs": [],
144
  "source": [
145
  "import os as _os\n",
@@ -173,9 +173,9 @@
173
  },
174
  {
175
  "cell_type": "code",
 
176
  "id": "eval-versions",
177
  "metadata": {},
178
- "execution_count": null,
179
  "outputs": [],
180
  "source": [
181
  "import torch, transformers, bitsandbytes, peft, accelerate, huggingface_hub, httpx\n",
@@ -231,9 +231,9 @@
231
  },
232
  {
233
  "cell_type": "code",
 
234
  "id": "eval-paths",
235
  "metadata": {},
236
- "execution_count": null,
237
  "outputs": [],
238
  "source": [
239
  "# ── 1) WORK dir + HF_TOKEN bootstrap (platform-specific) ───────────\n",
@@ -365,9 +365,9 @@
365
  },
366
  {
367
  "cell_type": "code",
 
368
  "id": "eval-find",
369
  "metadata": {},
370
- "execution_count": null,
371
  "outputs": [],
372
  "source": [
373
  "# Same dataset-payload finders as the training notebook. Filling in the\n",
@@ -480,9 +480,9 @@
480
  },
481
  {
482
  "cell_type": "code",
 
483
  "id": "eval-pull-run",
484
  "metadata": {},
485
- "execution_count": null,
486
  "outputs": [],
487
  "source": [
488
  "from huggingface_hub import snapshot_download\n",
@@ -552,9 +552,9 @@
552
  },
553
  {
554
  "cell_type": "code",
 
555
  "id": "eval-gpu",
556
  "metadata": {},
557
- "execution_count": null,
558
  "outputs": [],
559
  "source": [
560
  "import torch\n",
@@ -635,9 +635,9 @@
635
  },
636
  {
637
  "cell_type": "code",
 
638
  "id": "eval-cfg",
639
  "metadata": {},
640
- "execution_count": null,
641
  "outputs": [],
642
  "source": [
643
  "from omegaconf import OmegaConf\n",
@@ -779,9 +779,9 @@
779
  },
780
  {
781
  "cell_type": "code",
 
782
  "id": "eval-verify",
783
  "metadata": {},
784
- "execution_count": null,
785
  "outputs": [],
786
  "source": [
787
  "import json as _json\n",
@@ -848,9 +848,9 @@
848
  },
849
  {
850
  "cell_type": "code",
 
851
  "id": "eval-run",
852
  "metadata": {},
853
- "execution_count": null,
854
  "outputs": [],
855
  "source": [
856
  "import shlex\n",
@@ -858,9 +858,12 @@
858
  "RESULTS_DIR_LOCAL = WORK / LOCAL_RESULTS_DIR\n",
859
  "RESULTS_DIR_LOCAL.mkdir(parents=True, exist_ok=True)\n",
860
  "\n",
861
- "# evaluate.py expects --checkpoint to be a `<dir>/<name>_projection.pt` path;\n",
862
- "# it derives the LoRA dir + CheXpert classifier path from the same folder.\n",
863
- "CKPT_ARG = str(PROJ_PT)\n",
 
 
 
864
  "\n",
865
  "extra = ''\n",
866
  "if LLM_JUDGE:\n",
@@ -910,9 +913,9 @@
910
  },
911
  {
912
  "cell_type": "code",
 
913
  "id": "eval-summary",
914
  "metadata": {},
915
- "execution_count": null,
916
  "outputs": [],
917
  "source": [
918
  "import json as _json\n",
@@ -967,8 +970,8 @@
967
  "accelerator": "GPU",
968
  "colab": {
969
  "gpuType": "T4",
970
- "provenance": [],
971
- "machine_shape": "hm"
972
  },
973
  "kernelspec": {
974
  "display_name": "Python 3",
@@ -981,4 +984,4 @@
981
  },
982
  "nbformat": 4,
983
  "nbformat_minor": 5
984
- }
 
35
  },
36
  {
37
  "cell_type": "code",
38
+ "execution_count": null,
39
  "id": "eval-select",
40
  "metadata": {},
 
41
  "outputs": [],
42
  "source": [
43
  "# ── Platform ─────────────────────────────────────────────────────\n",
 
118
  },
119
  {
120
  "cell_type": "code",
121
+ "execution_count": null,
122
  "id": "eval-env",
123
  "metadata": {},
 
124
  "outputs": [],
125
  "source": [
126
  "import os\n",
 
137
  },
138
  {
139
  "cell_type": "code",
140
+ "execution_count": null,
141
  "id": "eval-pip",
142
  "metadata": {},
 
143
  "outputs": [],
144
  "source": [
145
  "import os as _os\n",
 
173
  },
174
  {
175
  "cell_type": "code",
176
+ "execution_count": null,
177
  "id": "eval-versions",
178
  "metadata": {},
 
179
  "outputs": [],
180
  "source": [
181
  "import torch, transformers, bitsandbytes, peft, accelerate, huggingface_hub, httpx\n",
 
231
  },
232
  {
233
  "cell_type": "code",
234
+ "execution_count": null,
235
  "id": "eval-paths",
236
  "metadata": {},
 
237
  "outputs": [],
238
  "source": [
239
  "# ── 1) WORK dir + HF_TOKEN bootstrap (platform-specific) ───────────\n",
 
365
  },
366
  {
367
  "cell_type": "code",
368
+ "execution_count": null,
369
  "id": "eval-find",
370
  "metadata": {},
 
371
  "outputs": [],
372
  "source": [
373
  "# Same dataset-payload finders as the training notebook. Filling in the\n",
 
480
  },
481
  {
482
  "cell_type": "code",
483
+ "execution_count": null,
484
  "id": "eval-pull-run",
485
  "metadata": {},
 
486
  "outputs": [],
487
  "source": [
488
  "from huggingface_hub import snapshot_download\n",
 
552
  },
553
  {
554
  "cell_type": "code",
555
+ "execution_count": null,
556
  "id": "eval-gpu",
557
  "metadata": {},
 
558
  "outputs": [],
559
  "source": [
560
  "import torch\n",
 
635
  },
636
  {
637
  "cell_type": "code",
638
+ "execution_count": null,
639
  "id": "eval-cfg",
640
  "metadata": {},
 
641
  "outputs": [],
642
  "source": [
643
  "from omegaconf import OmegaConf\n",
 
779
  },
780
  {
781
  "cell_type": "code",
782
+ "execution_count": null,
783
  "id": "eval-verify",
784
  "metadata": {},
 
785
  "outputs": [],
786
  "source": [
787
  "import json as _json\n",
 
848
  },
849
  {
850
  "cell_type": "code",
851
+ "execution_count": null,
852
  "id": "eval-run",
853
  "metadata": {},
 
854
  "outputs": [],
855
  "source": [
856
  "import shlex\n",
 
858
  "RESULTS_DIR_LOCAL = WORK / LOCAL_RESULTS_DIR\n",
859
  "RESULTS_DIR_LOCAL.mkdir(parents=True, exist_ok=True)\n",
860
  "\n",
861
+ "# evaluate.py forwards --checkpoint to utils.checkpoint.load_checkpoint,\n",
862
+ "# which reads <dir>/checkpoint_projection.pt + <dir>/checkpoint_lora/.\n",
863
+ "# Pass the DIRECTORY — passing the .pt file makes load_checkpoint mis-derive\n",
864
+ "# the filename (checkpoint_projection_projection.pt) and silently skip both\n",
865
+ "# projection AND LoRA, leaving you with raw 4-bit Vicuna.\n",
866
+ "CKPT_ARG = str(CKPT_DIR_PULLED)\n",
867
  "\n",
868
  "extra = ''\n",
869
  "if LLM_JUDGE:\n",
 
913
  },
914
  {
915
  "cell_type": "code",
916
+ "execution_count": null,
917
  "id": "eval-summary",
918
  "metadata": {},
 
919
  "outputs": [],
920
  "source": [
921
  "import json as _json\n",
 
970
  "accelerator": "GPU",
971
  "colab": {
972
  "gpuType": "T4",
973
+ "machine_shape": "hm",
974
+ "provenance": []
975
  },
976
  "kernelspec": {
977
  "display_name": "Python 3",
 
984
  },
985
  "nbformat": 4,
986
  "nbformat_minor": 5
987
+ }
scripts/cxrvlm_colab_inference.ipynb CHANGED
@@ -37,9 +37,9 @@
37
  },
38
  {
39
  "cell_type": "code",
 
40
  "id": "inf-select",
41
  "metadata": {},
42
- "execution_count": null,
43
  "outputs": [],
44
  "source": [
45
  "# ── Platform ─────────────────────────────────────────────────────\n",
@@ -77,9 +77,9 @@
77
  },
78
  {
79
  "cell_type": "code",
 
80
  "id": "inf-env",
81
  "metadata": {},
82
- "execution_count": null,
83
  "outputs": [],
84
  "source": [
85
  "import os\n",
@@ -95,9 +95,9 @@
95
  },
96
  {
97
  "cell_type": "code",
 
98
  "id": "inf-pip",
99
  "metadata": {},
100
- "execution_count": null,
101
  "outputs": [],
102
  "source": [
103
  "# Same dependency window as the training/eval notebooks.\n",
@@ -123,9 +123,9 @@
123
  },
124
  {
125
  "cell_type": "code",
 
126
  "id": "inf-versions",
127
  "metadata": {},
128
- "execution_count": null,
129
  "outputs": [],
130
  "source": [
131
  "import torch, transformers, peft, huggingface_hub, httpx\n",
@@ -173,9 +173,9 @@
173
  },
174
  {
175
  "cell_type": "code",
 
176
  "id": "inf-paths",
177
  "metadata": {},
178
- "execution_count": null,
179
  "outputs": [],
180
  "source": [
181
  "# WORK dir + HF_TOKEN bootstrap (platform-specific).\n",
@@ -237,9 +237,9 @@
237
  },
238
  {
239
  "cell_type": "code",
 
240
  "id": "inf-pull-run",
241
  "metadata": {},
242
- "execution_count": null,
243
  "outputs": [],
244
  "source": [
245
  "from huggingface_hub import snapshot_download\n",
@@ -293,9 +293,9 @@
293
  },
294
  {
295
  "cell_type": "code",
 
296
  "id": "inf-gpu",
297
  "metadata": {},
298
- "execution_count": null,
299
  "outputs": [],
300
  "source": [
301
  "import torch\n",
@@ -337,9 +337,9 @@
337
  },
338
  {
339
  "cell_type": "code",
 
340
  "id": "inf-cfg",
341
  "metadata": {},
342
- "execution_count": null,
343
  "outputs": [],
344
  "source": [
345
  "from omegaconf import OmegaConf\n",
@@ -393,9 +393,9 @@
393
  },
394
  {
395
  "cell_type": "code",
 
396
  "id": "inf-load",
397
  "metadata": {},
398
- "execution_count": null,
399
  "outputs": [],
400
  "source": [
401
  "import time, torch\n",
@@ -409,9 +409,13 @@
409
  "model = CXRVisionLanguageModel(model_cfg)\n",
410
  "print(f' built in {time.time()-_t0:.1f}s', flush=True)\n",
411
  "\n",
412
- "print(f'[2/3] Loading checkpoint from {PROJ_PT.name} …')\n",
413
  "_t0 = time.time()\n",
414
- "load_checkpoint(model, str(PROJ_PT))\n",
 
 
 
 
415
  "print(f' loaded in {time.time()-_t0:.1f}s', flush=True)\n",
416
  "\n",
417
  "print('[3/3] Moving to cuda + eval()')\n",
@@ -453,9 +457,9 @@
453
  },
454
  {
455
  "cell_type": "code",
 
456
  "id": "inf-helpers",
457
  "metadata": {},
458
- "execution_count": null,
459
  "outputs": [],
460
  "source": [
461
  "import torch\n",
@@ -564,9 +568,9 @@
564
  },
565
  {
566
  "cell_type": "code",
 
567
  "id": "inf-show",
568
  "metadata": {},
569
- "execution_count": null,
570
  "outputs": [],
571
  "source": [
572
  "import matplotlib.pyplot as plt\n",
@@ -612,9 +616,9 @@
612
  },
613
  {
614
  "cell_type": "code",
 
615
  "id": "inf-ex1",
616
  "metadata": {},
617
- "execution_count": null,
618
  "outputs": [],
619
  "source": [
620
  "# Pick an image — anywhere on disk. Colab: drag-and-drop into the Files pane\n",
@@ -661,9 +665,9 @@
661
  },
662
  {
663
  "cell_type": "code",
 
664
  "id": "inf-ex2",
665
  "metadata": {},
666
- "execution_count": null,
667
  "outputs": [],
668
  "source": [
669
  "if Path(IMAGE_PATH).is_file():\n",
@@ -684,9 +688,9 @@
684
  },
685
  {
686
  "cell_type": "code",
 
687
  "id": "inf-ex3",
688
  "metadata": {},
689
- "execution_count": null,
690
  "outputs": [],
691
  "source": [
692
  "import json\n",
@@ -738,8 +742,8 @@
738
  "accelerator": "GPU",
739
  "colab": {
740
  "gpuType": "T4",
741
- "provenance": [],
742
- "machine_shape": "hm"
743
  },
744
  "kernelspec": {
745
  "display_name": "Python 3",
@@ -752,4 +756,4 @@
752
  },
753
  "nbformat": 4,
754
  "nbformat_minor": 5
755
- }
 
37
  },
38
  {
39
  "cell_type": "code",
40
+ "execution_count": null,
41
  "id": "inf-select",
42
  "metadata": {},
 
43
  "outputs": [],
44
  "source": [
45
  "# ── Platform ─────────────────────────────────────────────────────\n",
 
77
  },
78
  {
79
  "cell_type": "code",
80
+ "execution_count": null,
81
  "id": "inf-env",
82
  "metadata": {},
 
83
  "outputs": [],
84
  "source": [
85
  "import os\n",
 
95
  },
96
  {
97
  "cell_type": "code",
98
+ "execution_count": null,
99
  "id": "inf-pip",
100
  "metadata": {},
 
101
  "outputs": [],
102
  "source": [
103
  "# Same dependency window as the training/eval notebooks.\n",
 
123
  },
124
  {
125
  "cell_type": "code",
126
+ "execution_count": null,
127
  "id": "inf-versions",
128
  "metadata": {},
 
129
  "outputs": [],
130
  "source": [
131
  "import torch, transformers, peft, huggingface_hub, httpx\n",
 
173
  },
174
  {
175
  "cell_type": "code",
176
+ "execution_count": null,
177
  "id": "inf-paths",
178
  "metadata": {},
 
179
  "outputs": [],
180
  "source": [
181
  "# WORK dir + HF_TOKEN bootstrap (platform-specific).\n",
 
237
  },
238
  {
239
  "cell_type": "code",
240
+ "execution_count": null,
241
  "id": "inf-pull-run",
242
  "metadata": {},
 
243
  "outputs": [],
244
  "source": [
245
  "from huggingface_hub import snapshot_download\n",
 
293
  },
294
  {
295
  "cell_type": "code",
296
+ "execution_count": null,
297
  "id": "inf-gpu",
298
  "metadata": {},
 
299
  "outputs": [],
300
  "source": [
301
  "import torch\n",
 
337
  },
338
  {
339
  "cell_type": "code",
340
+ "execution_count": null,
341
  "id": "inf-cfg",
342
  "metadata": {},
 
343
  "outputs": [],
344
  "source": [
345
  "from omegaconf import OmegaConf\n",
 
393
  },
394
  {
395
  "cell_type": "code",
396
+ "execution_count": null,
397
  "id": "inf-load",
398
  "metadata": {},
 
399
  "outputs": [],
400
  "source": [
401
  "import time, torch\n",
 
409
  "model = CXRVisionLanguageModel(model_cfg)\n",
410
  "print(f' built in {time.time()-_t0:.1f}s', flush=True)\n",
411
  "\n",
412
+ "print(f'[2/3] Loading checkpoint from {CKPT_DIR_PULLED} …')\n",
413
  "_t0 = time.time()\n",
414
+ "# Pass the directory, not the .pt file: load_checkpoint splits suffix off\n",
415
+ "# the path stem, so passing checkpoint_projection.pt makes it look for\n",
416
+ "# checkpoint_projection_projection.pt (which does not exist) and silently\n",
417
+ "# skips both projection AND lora — model falls back to raw 4-bit Vicuna.\n",
418
+ "load_checkpoint(model, str(CKPT_DIR_PULLED))\n",
419
  "print(f' loaded in {time.time()-_t0:.1f}s', flush=True)\n",
420
  "\n",
421
  "print('[3/3] Moving to cuda + eval()')\n",
 
457
  },
458
  {
459
  "cell_type": "code",
460
+ "execution_count": null,
461
  "id": "inf-helpers",
462
  "metadata": {},
 
463
  "outputs": [],
464
  "source": [
465
  "import torch\n",
 
568
  },
569
  {
570
  "cell_type": "code",
571
+ "execution_count": null,
572
  "id": "inf-show",
573
  "metadata": {},
 
574
  "outputs": [],
575
  "source": [
576
  "import matplotlib.pyplot as plt\n",
 
616
  },
617
  {
618
  "cell_type": "code",
619
+ "execution_count": null,
620
  "id": "inf-ex1",
621
  "metadata": {},
 
622
  "outputs": [],
623
  "source": [
624
  "# Pick an image — anywhere on disk. Colab: drag-and-drop into the Files pane\n",
 
665
  },
666
  {
667
  "cell_type": "code",
668
+ "execution_count": null,
669
  "id": "inf-ex2",
670
  "metadata": {},
 
671
  "outputs": [],
672
  "source": [
673
  "if Path(IMAGE_PATH).is_file():\n",
 
688
  },
689
  {
690
  "cell_type": "code",
691
+ "execution_count": null,
692
  "id": "inf-ex3",
693
  "metadata": {},
 
694
  "outputs": [],
695
  "source": [
696
  "import json\n",
 
742
  "accelerator": "GPU",
743
  "colab": {
744
  "gpuType": "T4",
745
+ "machine_shape": "hm",
746
+ "provenance": []
747
  },
748
  "kernelspec": {
749
  "display_name": "Python 3",
 
756
  },
757
  "nbformat": 4,
758
  "nbformat_minor": 5
759
+ }
scripts/vertex_finetune_vqa_job.yaml CHANGED
@@ -84,7 +84,7 @@ scheduling:
84
  strategy: STANDARD
85
  # Stage 2 finetune (3 epochs, mixed batch ~80k samples) on L4 ≈ 3-4h.
86
  # 6h ceiling leaves slack for data pull + builder.
87
- timeout: 21600s
88
  # Don't auto-retry on worker failure. When entrypoint asserts (bad path,
89
  # missing source run, etc.) the retry just re-pulls 40GB tar shards and
90
  # crashes the same way — wasteful. Set to 1 to fail fast.
 
84
  strategy: STANDARD
85
  # Stage 2 finetune (3 epochs, mixed batch ~80k samples) on L4 ≈ 3-4h.
86
  # 6h ceiling leaves slack for data pull + builder.
87
+ timeout: 216000s
88
  # Don't auto-retry on worker failure. When entrypoint asserts (bad path,
89
  # missing source run, etc.) the retry just re-pulls 40GB tar shards and
90
  # crashes the same way — wasteful. Set to 1 to fail fast.