| { |
| "schema": "papo.eval_summary.v2", |
| "model_fingerprint": "238fbe483d8179bbbab0ce9f4ec044c09bb43661f4f319698f063beb2f6199d7", |
| "protocol": { |
| "name": "papo", |
| "rollouts": 8, |
| "temperature": 1.0, |
| "max_samples": null, |
| "max_new_tokens": 2048, |
| "cutoff_len": 9216, |
| "batch_size": 64, |
| "image_max_pixels": 589824, |
| "image_min_pixels": 3136, |
| "top_p": 1.0, |
| "top_k": 50, |
| "repetition_penalty": 1.0, |
| "seed": null, |
| "template": "qwen2_vl", |
| "smoke": false, |
| "vllm_config": { |
| "tensor_parallel_size": 4, |
| "gpu_memory_utilization": 0.7, |
| "max_num_seqs": 64 |
| } |
| }, |
| "results": [ |
| { |
| "benchmark": "mathvista", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathvista/1ae36af33058e7be/attempt-1793492723127352502", |
| "expected_predictions": 8000, |
| "existing_predictions": 6832, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathvista/1ae36af33058e7be/attempt-1793492723127352502/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.664375, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "mathverse", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse/da6e4b91ebff4436/attempt-1793496264876467887", |
| "expected_predictions": 17440, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse/da6e4b91ebff4436/attempt-1793496264876467887/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.6753440366972477, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "wemath", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/wemath/c416ba245619e1e2/attempt-1793500443094732411", |
| "expected_predictions": 13920, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/wemath/c416ba245619e1e2/attempt-1793500443094732411/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.7072557471264368, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "mmk12", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmk12/f592ae0b7d0179ff/attempt-1793503352090073176", |
| "expected_predictions": 16000, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmk12/f592ae0b7d0179ff/attempt-1793503352090073176/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.6279375, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "geomath", |
| "dataset_version": "geomath-lookback-820", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geomath/e51beefad4cc20c0/attempt-1793507322398362369", |
| "expected_predictions": 6560, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geomath/e51beefad4cc20c0/attempt-1793507322398362369/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.49908536585365854, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "geo3k", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geo3k/aec51119bc013287/attempt-1793508680589692094", |
| "expected_predictions": 4808, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geo3k/aec51119bc013287/attempt-1793508680589692094/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.4319883527454243, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "logicvista", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/logicvista/a295ecd690bb3f15/attempt-1793509801075201074", |
| "expected_predictions": 3576, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/logicvista/a295ecd690bb3f15/attempt-1793509801075201074/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.4628076062639821, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "counting", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/counting/424dd56f18780057/attempt-1793510707065953784", |
| "expected_predictions": 1600, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/counting/424dd56f18780057/attempt-1793510707065953784/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.833125, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "mmmu-pro", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmmu-pro/bf7c02c4565ffbb3/attempt-1793511055832949257", |
| "expected_predictions": 13840, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmmu-pro/bf7c02c4565ffbb3/attempt-1793511055832949257/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.3609104046242775, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "mathverse-v", |
| "dataset_version": "papo-legacy-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse-v/ec0ac488923ed7a1/attempt-1793516853177329603", |
| "expected_predictions": 10464, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse-v/ec0ac488923ed7a1/attempt-1793516853177329603/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.6503249235474006, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "c-vqa-real", |
| "dataset_version": "cfpo-test-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/c-vqa-real/b7f2b44437dd6a6b/attempt-1793519691400968977", |
| "expected_predictions": 50304, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/c-vqa-real/b7f2b44437dd6a6b/attempt-1793519691400968977/scores/1e89bc236683577a58803184afc1819bff0d96915a378405be2c57c0ae3040ce/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.7068821564885496, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "mars-bench", |
| "dataset_version": "cfpo-test-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mars-bench/558825d475a18b16/attempt-1793526328865017349", |
| "expected_predictions": 40880, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mars-bench/558825d475a18b16/attempt-1793526328865017349/scores/1e89bc236683577a58803184afc1819bff0d96915a378405be2c57c0ae3040ce/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.5599070450097847, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "pope", |
| "dataset_version": "cfpo-test-v1", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/pope/248967ed6e3f69b9/attempt-1793532343474240556", |
| "expected_predictions": 72000, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/pope/248967ed6e3f69b9/attempt-1793532343474240556/scores/383a7e726d5ee2816521be11dbd593f516c567b085d49941a02b212c9a2ca924/metrics.json", |
| "primary_metric": "accuracy", |
| "score": 0.8799166666666667, |
| "aggregation": "Avg@8" |
| }, |
| { |
| "benchmark": "textvqa", |
| "dataset_version": "cfpo-val-5000", |
| "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/textvqa/7a6767d344bd6903/attempt-1793540625025776331", |
| "expected_predictions": 40000, |
| "existing_predictions": 0, |
| "status": "complete", |
| "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/textvqa/7a6767d344bd6903/attempt-1793540625025776331/scores/4fd119776f95fcf75c8c08ec9b1905b4ef49c3f8ead74acfed2af32fe6b504d2/metrics.json", |
| "primary_metric": "vqa_soft_accuracy", |
| "score": 0.74865, |
| "aggregation": "Avg@8" |
| } |
| ], |
| "counts": { |
| "complete": 14, |
| "skipped": 0, |
| "pending": 0, |
| "failed": 0, |
| "not_ready": 0 |
| } |
| } |
|
|
