File size: 12,841 Bytes
bc9139e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 | {
"schema": "papo.eval_summary.v2",
"model_fingerprint": "238fbe483d8179bbbab0ce9f4ec044c09bb43661f4f319698f063beb2f6199d7",
"protocol": {
"name": "papo",
"rollouts": 8,
"temperature": 1.0,
"max_samples": null,
"max_new_tokens": 2048,
"cutoff_len": 9216,
"batch_size": 64,
"image_max_pixels": 589824,
"image_min_pixels": 3136,
"top_p": 1.0,
"top_k": 50,
"repetition_penalty": 1.0,
"seed": null,
"template": "qwen2_vl",
"smoke": false,
"vllm_config": {
"tensor_parallel_size": 4,
"gpu_memory_utilization": 0.7,
"max_num_seqs": 64
}
},
"results": [
{
"benchmark": "mathvista",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathvista/1ae36af33058e7be/attempt-1793492723127352502",
"expected_predictions": 8000,
"existing_predictions": 6832,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathvista/1ae36af33058e7be/attempt-1793492723127352502/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.664375,
"aggregation": "Avg@8"
},
{
"benchmark": "mathverse",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse/da6e4b91ebff4436/attempt-1793496264876467887",
"expected_predictions": 17440,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse/da6e4b91ebff4436/attempt-1793496264876467887/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.6753440366972477,
"aggregation": "Avg@8"
},
{
"benchmark": "wemath",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/wemath/c416ba245619e1e2/attempt-1793500443094732411",
"expected_predictions": 13920,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/wemath/c416ba245619e1e2/attempt-1793500443094732411/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.7072557471264368,
"aggregation": "Avg@8"
},
{
"benchmark": "mmk12",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmk12/f592ae0b7d0179ff/attempt-1793503352090073176",
"expected_predictions": 16000,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmk12/f592ae0b7d0179ff/attempt-1793503352090073176/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.6279375,
"aggregation": "Avg@8"
},
{
"benchmark": "geomath",
"dataset_version": "geomath-lookback-820",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geomath/e51beefad4cc20c0/attempt-1793507322398362369",
"expected_predictions": 6560,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geomath/e51beefad4cc20c0/attempt-1793507322398362369/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.49908536585365854,
"aggregation": "Avg@8"
},
{
"benchmark": "geo3k",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geo3k/aec51119bc013287/attempt-1793508680589692094",
"expected_predictions": 4808,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geo3k/aec51119bc013287/attempt-1793508680589692094/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.4319883527454243,
"aggregation": "Avg@8"
},
{
"benchmark": "logicvista",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/logicvista/a295ecd690bb3f15/attempt-1793509801075201074",
"expected_predictions": 3576,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/logicvista/a295ecd690bb3f15/attempt-1793509801075201074/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.4628076062639821,
"aggregation": "Avg@8"
},
{
"benchmark": "counting",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/counting/424dd56f18780057/attempt-1793510707065953784",
"expected_predictions": 1600,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/counting/424dd56f18780057/attempt-1793510707065953784/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.833125,
"aggregation": "Avg@8"
},
{
"benchmark": "mmmu-pro",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmmu-pro/bf7c02c4565ffbb3/attempt-1793511055832949257",
"expected_predictions": 13840,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmmu-pro/bf7c02c4565ffbb3/attempt-1793511055832949257/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.3609104046242775,
"aggregation": "Avg@8"
},
{
"benchmark": "mathverse-v",
"dataset_version": "papo-legacy-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse-v/ec0ac488923ed7a1/attempt-1793516853177329603",
"expected_predictions": 10464,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse-v/ec0ac488923ed7a1/attempt-1793516853177329603/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
"primary_metric": "accuracy",
"score": 0.6503249235474006,
"aggregation": "Avg@8"
},
{
"benchmark": "c-vqa-real",
"dataset_version": "cfpo-test-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/c-vqa-real/b7f2b44437dd6a6b/attempt-1793519691400968977",
"expected_predictions": 50304,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/c-vqa-real/b7f2b44437dd6a6b/attempt-1793519691400968977/scores/1e89bc236683577a58803184afc1819bff0d96915a378405be2c57c0ae3040ce/metrics.json",
"primary_metric": "accuracy",
"score": 0.7068821564885496,
"aggregation": "Avg@8"
},
{
"benchmark": "mars-bench",
"dataset_version": "cfpo-test-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mars-bench/558825d475a18b16/attempt-1793526328865017349",
"expected_predictions": 40880,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mars-bench/558825d475a18b16/attempt-1793526328865017349/scores/1e89bc236683577a58803184afc1819bff0d96915a378405be2c57c0ae3040ce/metrics.json",
"primary_metric": "accuracy",
"score": 0.5599070450097847,
"aggregation": "Avg@8"
},
{
"benchmark": "pope",
"dataset_version": "cfpo-test-v1",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/pope/248967ed6e3f69b9/attempt-1793532343474240556",
"expected_predictions": 72000,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/pope/248967ed6e3f69b9/attempt-1793532343474240556/scores/383a7e726d5ee2816521be11dbd593f516c567b085d49941a02b212c9a2ca924/metrics.json",
"primary_metric": "accuracy",
"score": 0.8799166666666667,
"aggregation": "Avg@8"
},
{
"benchmark": "textvqa",
"dataset_version": "cfpo-val-5000",
"attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/textvqa/7a6767d344bd6903/attempt-1793540625025776331",
"expected_predictions": 40000,
"existing_predictions": 0,
"status": "complete",
"metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/textvqa/7a6767d344bd6903/attempt-1793540625025776331/scores/4fd119776f95fcf75c8c08ec9b1905b4ef49c3f8ead74acfed2af32fe6b504d2/metrics.json",
"primary_metric": "vqa_soft_accuracy",
"score": 0.74865,
"aggregation": "Avg@8"
}
],
"counts": {
"complete": 14,
"skipped": 0,
"pending": 0,
"failed": 0,
"not_ready": 0
}
}
|