Release FLEX34 PureLoRA checkpoint and inference guide

README.md

@@ -0,0 +1,72 @@
+---
+library_name: peft
+base_model: liuhaotian/llava-v1.5-7b
+base_model_relation: adapter
+language:
+  - en
+tags:
+  - flex
+  - purelora
+  - lora
+  - continual-learning
+  - multimodal
+inference: false
+---
+
+# FLEX34-PureLoRA
+
+Final PureLoRA checkpoint after all 34 FLEX tasks, containing every trained
+expert and the NCM routing prototypes. The default router uses softmax logit
+scale 28 and equal image/text weights. Different routers can reuse these same
+experts without retraining.
+
+- [Code and environment setup](https://github.com/RINC-CL/FLEX/tree/ba5bbccbfbad4ab9ba2985a56930a814aa1076be)
+- [FLEX instructions and image sources](https://huggingface.co/datasets/yhyet/FLEX)
+- [Task-to-expert mapping](task_mapping.json) and [per-task Base results](eval_results.json)
+
+The Base reference scores are **M = 54.60**, **G = 60.81%**, and **R = 83.77%**.
+This is a custom `PURELORA` adapter: use the FLEX repository's bundled `PEFT`
+implementation and loading code. Download the base model and CLIP separately.
+
+## Download
+
+```bash
+hf download yhyet/FLEX34-PureLoRA --local-dir ./checkpoints/FLEX34-PureLoRA
+```
+
+The checkpoint consists of `adapter_model.safetensors`, `adapter_config.json`,
+`config.json`, and `purelora_state.pt`. Keep all four in the same directory.
+To verify the downloaded files, run `sha256sum -c SHA256SUMS` in that directory.
+
+## Inference
+
+Install the FLEX code and prepare the instructions and images following its
+README. In `config/paths/llava_paths.py`, set `BASE_MODEL_PATH` to your local
+[LLaVA-1.5-7B snapshot](https://huggingface.co/liuhaotian/llava-v1.5-7b/tree/4481d270cc22fd5c4d1bb5df129622006ccd9234)
+and `CLIP_PATH` to your local
+[CLIP ViT-L/14-336 snapshot](https://huggingface.co/openai/clip-vit-large-patch14-336/tree/ce19dc912ca5cd21c8a653c79e251e808ccabcd1).
+
+From the FLEX code directory, after downloading this checkpoint there:
+
+```bash
+export FLEX_INSTRUCTION_ROOT=/path/to/datasets/FLEX/instructions
+export PURELORA_MIXTURE_LOGIT_SCALE=28
+export PURELORA_IMAGE_WEIGHT=0.5
+export PURELORA_TEXT_WEIGHT=0.5
+
+python run.py infer $(seq 0 33) \
+  --benchmark flex --method purelora --backbone llava \
+  --model-path ./checkpoints/FLEX34-PureLoRA \
+  --checkpoint-task 33 --cl-task-num 34 --stage last \
+  --gpus 0 --batch-size 1 --temperature 0 --conv-mode vicuna_v1 \
+  --no-top1-expert-routing --no-oracle-expert-routing \
+  --accel-eval --run-stamp flex34-purelora-ncm28
+```
+
+For a short run, replace `$(seq 0 33)` with `0`. The command uses the FLEX
+code's default bfloat16 inference. For task-oracle inference, replace
+`--no-oracle-expert-routing` with `--oracle-expert-routing`.
+
+Base model revision: `4481d270cc22fd5c4d1bb5df129622006ccd9234`.
+Expert IDs 0–33 follow the fixed FLEX task order in `task_mapping.json`.
+Use of the adapters is subject to the upstream base model and dataset terms.

SHA256SUMS

@@ -0,0 +1,7 @@
+afd8ce343e25f841040bed383e244d24c5ce1e7e64c7ee1088f4d079dd269c58  README.md
+7c7341580d4c43d2d6ccdc62bbf5f912c7c72b986535a4e9dc5c15a5e7a785d7  adapter_config.json
+597951d81d90e35d326e8c14228b4f569a68229f5ef1a711a943d77002db5e61  adapter_model.safetensors
+6958e8e7088c55fa67a7b5976f1f235117ae848b7dbbcc5c7032bac2bc8d5f52  config.json
+2d89b941f0af87e41ac0ee977b19797b67a98c7ce2fe60c29cd435d703344752  eval_results.json
+3978e74de33a605ae258f96ba0399d1ba49c4128307d9b62bf93ad6eef9452cb  purelora_state.pt
+c1ceb31a85614996577a0323df506bc8ee1693ad389b3100bd743a08b5b1c5c3  task_mapping.json

adapter_config.json

@@ -0,0 +1,35 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "liuhaotian/llava-v1.5-7b",
+  "bias": "none",
+  "cur_task": 33,
+  "exclude_module_path_segments": [
+    "mm_projector",
+    "vision_tower",
+    "text_tower",
+    "vision_resampler"
+  ],
+  "expert_num": 34,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 544,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "PURELORA",
+  "r": 272,
+  "revision": "4481d270cc22fd5c4d1bb5df129622006ccd9234",
+  "target_modules": [
+    "down_proj",
+    "gate_proj",
+    "k_proj",
+    "o_proj",
+    "q_proj",
+    "up_proj",
+    "v_proj"
+  ],
+  "task_embedding_dim": 64,
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:597951d81d90e35d326e8c14228b4f569a68229f5ef1a711a943d77002db5e61
+size 1361477384

config.json

@@ -0,0 +1,48 @@
+{
+  "_name_or_path": "liuhaotian/llava-v1.5-7b",
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "freeze_mm_mlp_adapter": true,
+  "freeze_mm_vision_resampler": false,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_aspect_ratio": "pad",
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_length": 4096,
+  "max_position_embeddings": 4096,
+  "mm_hidden_size": 1024,
+  "mm_projector_lr": 2e-05,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_text_select_layer": -1,
+  "mm_text_tower": "openai/clip-vit-large-patch14-336",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "openai/clip-vit-large-patch14-336",
+  "model_type": "llava",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "routing_vision_tower": "openai/clip-vit-large-patch14-336",
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 2048,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "float16",
+  "transformers_version": "4.31.0",
+  "tune_mm_mlp_adapter": false,
+  "tune_mm_vision_resampler": false,
+  "unfreeze_mm_vision_tower": false,
+  "use_cache": false,
+  "use_mm_proj": true,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,248 @@
+{
+  "benchmark": "FLEX34",
+  "routing": "NCM with softmax logit scale 28",
+  "num_tasks": 34,
+  "macro_score": 54.59617647058823,
+  "ground_truth_weight_percent": 60.805504155038214,
+  "strict_expert_match_percent": 83.76706610063854,
+  "tasks": [
+    {
+      "expert_id": 0,
+      "task": "ImageNet200_1",
+      "score": 54.03,
+      "ground_truth_weight_percent": 35.895350189507006,
+      "strict_expert_match_percent": 69.5
+    },
+    {
+      "expert_id": 1,
+      "task": "ImageNet200_2",
+      "score": 58.53,
+      "ground_truth_weight_percent": 34.22922151659925,
+      "strict_expert_match_percent": 73.5
+    },
+    {
+      "expert_id": 2,
+      "task": "ImageNet200_3",
+      "score": 58.47,
+      "ground_truth_weight_percent": 26.508872203777234,
+      "strict_expert_match_percent": 49.833333333333336
+    },
+    {
+      "expert_id": 3,
+      "task": "ImageNet200_4",
+      "score": 58.87,
+      "ground_truth_weight_percent": 26.688444763918717,
+      "strict_expert_match_percent": 52.06666666666667
+    },
+    {
+      "expert_id": 4,
+      "task": "ImageNet200_5",
+      "score": 55.3,
+      "ground_truth_weight_percent": 26.8756500462691,
+      "strict_expert_match_percent": 50.23333333333333
+    },
+    {
+      "expert_id": 5,
+      "task": "ArxivQA",
+      "score": 94.1,
+      "ground_truth_weight_percent": 88.33787338789553,
+      "strict_expert_match_percent": 97.56666666666666
+    },
+    {
+      "expert_id": 6,
+      "task": "Vizcap",
+      "score": 56.5,
+      "ground_truth_weight_percent": 83.80169021685919,
+      "strict_expert_match_percent": 99.13333333333333
+    },
+    {
+      "expert_id": 7,
+      "task": "IconQA",
+      "score": 64.67,
+      "ground_truth_weight_percent": 62.13553921754161,
+      "strict_expert_match_percent": 98.86666666666667
+    },
+    {
+      "expert_id": 8,
+      "task": "CLEVR",
+      "score": 61.57,
+      "ground_truth_weight_percent": 97.89418778419494,
+      "strict_expert_match_percent": 100.0
+    },
+    {
+      "expert_id": 9,
+      "task": "Flickr30k",
+      "score": 57.53,
+      "ground_truth_weight_percent": 83.78250439961751,
+      "strict_expert_match_percent": 99.73333333333333
+    },
+    {
+      "expert_id": 10,
+      "task": "ScienceQA",
+      "score": 81.13,
+      "ground_truth_weight_percent": 43.252303133602254,
+      "strict_expert_match_percent": 69.03333333333333
+    },
+    {
+      "expert_id": 11,
+      "task": "TextVQA",
+      "score": 54.13,
+      "ground_truth_weight_percent": 72.68710916787387,
+      "strict_expert_match_percent": 97.56666666666666
+    },
+    {
+      "expert_id": 12,
+      "task": "GQA",
+      "score": 59.0,
+      "ground_truth_weight_percent": 25.983873347441357,
+      "strict_expert_match_percent": 54.800000000000004
+    },
+    {
+      "expert_id": 13,
+      "task": "VQAv2",
+      "score": 65.3,
+      "ground_truth_weight_percent": 25.272837529331444,
+      "strict_expert_match_percent": 54.56666666666666
+    },
+    {
+      "expert_id": 14,
+      "task": "OCRVQA",
+      "score": 64.27,
+      "ground_truth_weight_percent": 75.30345542225987,
+      "strict_expert_match_percent": 97.63333333333334
+    },
+    {
+      "expert_id": 15,
+      "task": "PMCVQA",
+      "score": 40.9,
+      "ground_truth_weight_percent": 84.34286612238114,
+      "strict_expert_match_percent": 96.33333333333334
+    },
+    {
+      "expert_id": 16,
+      "task": "DocVQA",
+      "score": 30.18,
+      "ground_truth_weight_percent": 59.29709263934443,
+      "strict_expert_match_percent": 79.93333333333334
+    },
+    {
+      "expert_id": 17,
+      "task": "ChartQA",
+      "score": 16.76,
+      "ground_truth_weight_percent": 74.67188779175281,
+      "strict_expert_match_percent": 94.24
+    },
+    {
+      "expert_id": 18,
+      "task": "InfographicVQA",
+      "score": 39.4,
+      "ground_truth_weight_percent": 73.91684024246601,
+      "strict_expert_match_percent": 96.67975722956086
+    },
+    {
+      "expert_id": 19,
+      "task": "Roadside",
+      "score": 10.37,
+      "ground_truth_weight_percent": 87.63944706817468,
+      "strict_expert_match_percent": 100.0
+    },
+    {
+      "expert_id": 20,
+      "task": "ChemVQA",
+      "score": 38.3,
+      "ground_truth_weight_percent": 98.77906088394424,
+      "strict_expert_match_percent": 99.73333333333333
+    },
+    {
+      "expert_id": 21,
+      "task": "FloodNetVQA",
+      "score": 81.07,
+      "ground_truth_weight_percent": 89.63124896097365,
+      "strict_expert_match_percent": 100.0
+    },
+    {
+      "expert_id": 22,
+      "task": "AI2D",
+      "score": 75.47,
+      "ground_truth_weight_percent": 63.82172591674477,
+      "strict_expert_match_percent": 72.34927234927235
+    },
+    {
+      "expert_id": 23,
+      "task": "AOKVQA",
+      "score": 56.91,
+      "ground_truth_weight_percent": 26.03233747755488,
+      "strict_expert_match_percent": 61.46666666666667
+    },
+    {
+      "expert_id": 24,
+      "task": "OKVQA",
+      "score": 64.26,
+      "ground_truth_weight_percent": 28.618040108615745,
+      "strict_expert_match_percent": 74.03369672943508
+    },
+    {
+      "expert_id": 25,
+      "task": "ArtVQA",
+      "score": 24.43,
+      "ground_truth_weight_percent": 75.91621518408259,
+      "strict_expert_match_percent": 99.56666666666666
+    },
+    {
+      "expert_id": 26,
+      "task": "MathVista",
+      "score": 26.4,
+      "ground_truth_weight_percent": 39.09591959747486,
+      "strict_expert_match_percent": 62.6
+    },
+    {
+      "expert_id": 27,
+      "task": "PathVQA",
+      "score": 51.93,
+      "ground_truth_weight_percent": 60.848953806267424,
+      "strict_expert_match_percent": 92.56666666666666
+    },
+    {
+      "expert_id": 28,
+      "task": "PlantVillageVQA",
+      "score": 56.8,
+      "ground_truth_weight_percent": 93.20231916951015,
+      "strict_expert_match_percent": 98.5
+    },
+    {
+      "expert_id": 29,
+      "task": "SLAKE",
+      "score": 63.88,
+      "ground_truth_weight_percent": 64.36512597726778,
+      "strict_expert_match_percent": 83.74524714828897
+    },
+    {
+      "expert_id": 30,
+      "task": "VQARAD",
+      "score": 45.68,
+      "ground_truth_weight_percent": 62.03738841540004,
+      "strict_expert_match_percent": 90.2439024390244
+    },
+    {
+      "expert_id": 31,
+      "task": "ACL_OCR",
+      "score": 39.48,
+      "ground_truth_weight_percent": 28.67049870774182,
+      "strict_expert_match_percent": 80.04338394793926
+    },
+    {
+      "expert_id": 32,
+      "task": "TQA",
+      "score": 61.78,
+      "ground_truth_weight_percent": 50.9868276419702,
+      "strict_expert_match_percent": 77.32170343766033
+    },
+    {
+      "expert_id": 33,
+      "task": "StockQA",
+      "score": 88.87,
+      "ground_truth_weight_percent": 96.86443323294321,
+      "strict_expert_match_percent": 100.0
+    }
+  ]
+}

purelora_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3978e74de33a605ae258f96ba0399d1ba49c4128307d9b62bf93ad6eef9452cb
+size 142673

task_mapping.json

@@ -0,0 +1,175 @@
+{
+  "num_experts": 34,
+  "experts": [
+    {
+      "expert_id": 0,
+      "task": "ImageNet200_1",
+      "instruction_directory": "ImageNet200_1"
+    },
+    {
+      "expert_id": 1,
+      "task": "ImageNet200_2",
+      "instruction_directory": "ImageNet200_2"
+    },
+    {
+      "expert_id": 2,
+      "task": "ImageNet200_3",
+      "instruction_directory": "ImageNet200_3"
+    },
+    {
+      "expert_id": 3,
+      "task": "ImageNet200_4",
+      "instruction_directory": "ImageNet200_4"
+    },
+    {
+      "expert_id": 4,
+      "task": "ImageNet200_5",
+      "instruction_directory": "ImageNet200_5"
+    },
+    {
+      "expert_id": 5,
+      "task": "ArxivQA",
+      "instruction_directory": "ArxivQA"
+    },
+    {
+      "expert_id": 6,
+      "task": "Vizcap",
+      "instruction_directory": "Vizcap"
+    },
+    {
+      "expert_id": 7,
+      "task": "IconQA",
+      "instruction_directory": "IconQA"
+    },
+    {
+      "expert_id": 8,
+      "task": "CLEVR",
+      "instruction_directory": "CLEVR"
+    },
+    {
+      "expert_id": 9,
+      "task": "Flickr30k",
+      "instruction_directory": "Flickr30k"
+    },
+    {
+      "expert_id": 10,
+      "task": "ScienceQA",
+      "instruction_directory": "ScienceQA"
+    },
+    {
+      "expert_id": 11,
+      "task": "TextVQA",
+      "instruction_directory": "TextVQA"
+    },
+    {
+      "expert_id": 12,
+      "task": "GQA",
+      "instruction_directory": "GQA"
+    },
+    {
+      "expert_id": 13,
+      "task": "VQAv2",
+      "instruction_directory": "VQAv2"
+    },
+    {
+      "expert_id": 14,
+      "task": "OCRVQA",
+      "instruction_directory": "OCRVQA"
+    },
+    {
+      "expert_id": 15,
+      "task": "PMCVQA",
+      "instruction_directory": "PMCVQA"
+    },
+    {
+      "expert_id": 16,
+      "task": "DocVQA",
+      "instruction_directory": "DocVQA_v2"
+    },
+    {
+      "expert_id": 17,
+      "task": "ChartQA",
+      "instruction_directory": "ChartQA"
+    },
+    {
+      "expert_id": 18,
+      "task": "InfographicVQA",
+      "instruction_directory": "InfographicVQA"
+    },
+    {
+      "expert_id": 19,
+      "task": "Roadside",
+      "instruction_directory": "Roadside"
+    },
+    {
+      "expert_id": 20,
+      "task": "ChemVQA",
+      "instruction_directory": "ChemVQA"
+    },
+    {
+      "expert_id": 21,
+      "task": "FloodNetVQA",
+      "instruction_directory": "FloodNetVQA"
+    },
+    {
+      "expert_id": 22,
+      "task": "AI2D",
+      "instruction_directory": "AI2D_v2"
+    },
+    {
+      "expert_id": 23,
+      "task": "AOKVQA",
+      "instruction_directory": "AOKVQA"
+    },
+    {
+      "expert_id": 24,
+      "task": "OKVQA",
+      "instruction_directory": "OKVQA"
+    },
+    {
+      "expert_id": 25,
+      "task": "ArtVQA",
+      "instruction_directory": "ArtVQA"
+    },
+    {
+      "expert_id": 26,
+      "task": "MathVista",
+      "instruction_directory": "MathVista_v2"
+    },
+    {
+      "expert_id": 27,
+      "task": "PathVQA",
+      "instruction_directory": "PathVQA"
+    },
+    {
+      "expert_id": 28,
+      "task": "PlantVillageVQA",
+      "instruction_directory": "PlantVillageVQA"
+    },
+    {
+      "expert_id": 29,
+      "task": "SLAKE",
+      "instruction_directory": "SLAKE"
+    },
+    {
+      "expert_id": 30,
+      "task": "VQARAD",
+      "instruction_directory": "VQARAD"
+    },
+    {
+      "expert_id": 31,
+      "task": "ACL_OCR",
+      "instruction_directory": "ACL_OCR_v2"
+    },
+    {
+      "expert_id": 32,
+      "task": "TQA",
+      "instruction_directory": "TQA"
+    },
+    {
+      "expert_id": 33,
+      "task": "StockQA",
+      "instruction_directory": "StockQA"
+    }
+  ]
+}