chrisjcc commited on
Commit
511f10b
·
verified ·
1 Parent(s): 5528fe4

Upload folder using huggingface_hub

Browse files
README.md CHANGED
@@ -25,7 +25,7 @@ The **AskBeforeAnswer** model exhibits "clarification-seeking" behavior. When pr
25
  ## Pipeline
26
  - **Base Model:** Qwen/Qwen2.5-7B-Instruct
27
  - **Ablation Winner:** The model variant promoted to Production via W&B Registry is:
28
- `sft_dpo`.
29
 
30
  ## LLM-as-a-Judge Evaluation Leaderboard
31
 
@@ -34,20 +34,20 @@ Gemini-based judge scorer on a randomly selected
34
  **50-sample** subset
35
  of the `sewon/ambig_qa` (validation split).
36
 
37
- | Metric | base | dpo_only | sft | sft_dpo | clarifier_lora |
38
- |:----------------------|----------:|-----------:|---------:|----------:|-----------------:|
39
- | ambiguity_detection | 0.978 | 0.964 | 0.972 | 0.976 | 0.998 |
40
- | clarification_quality | 0.786 | 0.798 | 0.798 | 0.8 | 0.796 |
41
- | usefulness | 0.882 | 0.896 | 0.9 | 0.9 | 0.896 |
42
- | model_accuracy | 0.62 | 0.64 | 0.66 | 0.62 | 0.6 |
43
- | clarify_precision | 0.617021 | 0.642857 | 0.675676 | 0.627907 | 0.6 |
44
- | clarify_recall | 0.966667 | 0.9 | 0.833333 | 0.9 | 1 |
45
- | clarify_f1 | 0.753247 | 0.75 | 0.746269 | 0.739726 | 0.75 |
46
- | answer_f1 | 0.173913 | 0.357143 | 0.484848 | 0.296296 | 0 |
47
- | macro_f1 | 0.46358 | 0.553571 | 0.615559 | 0.518011 | 0.375 |
48
- | answer_accuracy | 0.05 | 0.1 | 0.1 | 0.1 | 0 |
49
- | facet_generation_rate | 0.0212766 | 1 | 1 | 1 | 0 |
50
- | clarify_ratio | 1.56667 | 1.4 | 1.23333 | 1.43333 | 1.66667 |
51
 
52
 
53
  **GitHub Release:** [v0.0.5](https://github.com/chrisjcc/ask-before-answer/releases/tag/v0.0.5)
 
25
  ## Pipeline
26
  - **Base Model:** Qwen/Qwen2.5-7B-Instruct
27
  - **Ablation Winner:** The model variant promoted to Production via W&B Registry is:
28
+ `dpo`.
29
 
30
  ## LLM-as-a-Judge Evaluation Leaderboard
31
 
 
34
  **50-sample** subset
35
  of the `sewon/ambig_qa` (validation split).
36
 
37
+ | Metric | base | dpo_only | sft | sft_dpo | clarifier_lora | orpo | grpo |
38
+ |:----------------------|----------:|-----------:|---------:|----------:|-----------------:|---------:|---------:|
39
+ | ambiguity_detection | 0.966 | 0.944 | 0.97 | 0.972 | 0.994 | 0.972 | 0.948 |
40
+ | clarification_quality | 0.784 | 0.784 | 0.796 | 0.796 | 0.796 | 0.796 | 0.798 |
41
+ | usefulness | 0.88 | 0.88 | 0.896 | 0.898 | 0.896 | 0.896 | 0.898 |
42
+ | model_accuracy | 0.62 | 0.6 | 0.64 | 0.62 | 0.6 | 0.64 | 0.64 |
43
+ | clarify_precision | 0.617021 | 0.604167 | 0.657895 | 0.657143 | 0.6 | 0.636364 | 0.642857 |
44
+ | clarify_recall | 0.966667 | 0.966667 | 0.833333 | 0.766667 | 1 | 0.933333 | 0.9 |
45
+ | clarify_f1 | 0.753247 | 0.74359 | 0.735294 | 0.707692 | 0.75 | 0.756757 | 0.75 |
46
+ | action_f1_answer | 0.173913 | 0.0909091 | 0.4375 | 0.457143 | 0 | 0.307692 | 0.357143 |
47
+ | macro_f1 | 0.46358 | 0.417249 | 0.586397 | 0.582418 | 0.375 | 0.532225 | 0.553571 |
48
+ | answer_accuracy | 0.05 | 0.05 | 0.05 | 0 | 0 | 0.1 | 0.1 |
49
+ | facet_generation_rate | 0.0851064 | 0.0833333 | 1 | 1 | 0 | 0.931818 | 1 |
50
+ | clarify_ratio | 1.56667 | 1.6 | 1.26667 | 1.16667 | 1.66667 | 1.46667 | 1.4 |
51
 
52
 
53
  **GitHub Release:** [v0.0.5](https://github.com/chrisjcc/ask-before-answer/releases/tag/v0.0.5)
adapter_config.json CHANGED
@@ -27,20 +27,21 @@
27
  "megatron_config": null,
28
  "megatron_core": "megatron.core",
29
  "modules_to_save": null,
 
30
  "peft_type": "LORA",
31
- "peft_version": "0.19.1",
32
  "qalora_group_size": 16,
33
  "r": 16,
34
  "rank_pattern": {},
35
  "revision": null,
36
  "target_modules": [
37
- "v_proj",
38
  "up_proj",
39
- "k_proj",
40
- "q_proj",
41
  "o_proj",
42
- "gate_proj",
43
- "down_proj"
 
 
44
  ],
45
  "target_parameters": null,
46
  "task_type": "CAUSAL_LM",
@@ -48,5 +49,6 @@
48
  "use_bdlora": null,
49
  "use_dora": false,
50
  "use_qalora": false,
51
- "use_rslora": false
 
52
  }
 
27
  "megatron_config": null,
28
  "megatron_core": "megatron.core",
29
  "modules_to_save": null,
30
+ "monteclora_config": null,
31
  "peft_type": "LORA",
32
+ "peft_version": "0.20.0",
33
  "qalora_group_size": 16,
34
  "r": 16,
35
  "rank_pattern": {},
36
  "revision": null,
37
  "target_modules": [
38
+ "gate_proj",
39
  "up_proj",
 
 
40
  "o_proj",
41
+ "k_proj",
42
+ "v_proj",
43
+ "down_proj",
44
+ "q_proj"
45
  ],
46
  "target_parameters": null,
47
  "task_type": "CAUSAL_LM",
 
49
  "use_bdlora": null,
50
  "use_dora": false,
51
  "use_qalora": false,
52
+ "use_rslora": false,
53
+ "velora_config": null
54
  }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3364d7db681c91edc6f1472dc8aadcc78c3b67818698637a5905f6b1119ed1e7
3
  size 161533192
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8730e5a37e3df071b5da452c0ffaf1863062aec888f38e01adf1afd099ea3721
3
  size 161533192
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e6a9bd126ecf9622ab5ea9a23d1c94aee8867a878b948495e4a44f5f688da737
3
  size 6584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f80303518705dcb84090e851e58a8ab093d94eaa41dc357e4f553c5130633140
3
  size 6584