{
  "schema": "papo.eval_summary.v2",
  "model_fingerprint": "238fbe483d8179bbbab0ce9f4ec044c09bb43661f4f319698f063beb2f6199d7",
  "protocol": {
    "name": "papo",
    "rollouts": 8,
    "temperature": 1.0,
    "max_samples": null,
    "max_new_tokens": 2048,
    "cutoff_len": 9216,
    "batch_size": 64,
    "image_max_pixels": 589824,
    "image_min_pixels": 3136,
    "top_p": 1.0,
    "top_k": 50,
    "repetition_penalty": 1.0,
    "seed": null,
    "template": "qwen2_vl",
    "smoke": false,
    "vllm_config": {
      "tensor_parallel_size": 4,
      "gpu_memory_utilization": 0.7,
      "max_num_seqs": 64
    }
  },
  "results": [
    {
      "benchmark": "mathvista",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathvista/1ae36af33058e7be/attempt-1793492723127352502",
      "expected_predictions": 8000,
      "existing_predictions": 6832,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathvista/1ae36af33058e7be/attempt-1793492723127352502/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.664375,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "mathverse",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse/da6e4b91ebff4436/attempt-1793496264876467887",
      "expected_predictions": 17440,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse/da6e4b91ebff4436/attempt-1793496264876467887/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.6753440366972477,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "wemath",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/wemath/c416ba245619e1e2/attempt-1793500443094732411",
      "expected_predictions": 13920,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/wemath/c416ba245619e1e2/attempt-1793500443094732411/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.7072557471264368,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "mmk12",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmk12/f592ae0b7d0179ff/attempt-1793503352090073176",
      "expected_predictions": 16000,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmk12/f592ae0b7d0179ff/attempt-1793503352090073176/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.6279375,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "geomath",
      "dataset_version": "geomath-lookback-820",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geomath/e51beefad4cc20c0/attempt-1793507322398362369",
      "expected_predictions": 6560,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geomath/e51beefad4cc20c0/attempt-1793507322398362369/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.49908536585365854,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "geo3k",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geo3k/aec51119bc013287/attempt-1793508680589692094",
      "expected_predictions": 4808,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/geo3k/aec51119bc013287/attempt-1793508680589692094/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.4319883527454243,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "logicvista",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/logicvista/a295ecd690bb3f15/attempt-1793509801075201074",
      "expected_predictions": 3576,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/logicvista/a295ecd690bb3f15/attempt-1793509801075201074/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.4628076062639821,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "counting",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/counting/424dd56f18780057/attempt-1793510707065953784",
      "expected_predictions": 1600,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/counting/424dd56f18780057/attempt-1793510707065953784/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.833125,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "mmmu-pro",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmmu-pro/bf7c02c4565ffbb3/attempt-1793511055832949257",
      "expected_predictions": 13840,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mmmu-pro/bf7c02c4565ffbb3/attempt-1793511055832949257/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.3609104046242775,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "mathverse-v",
      "dataset_version": "papo-legacy-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse-v/ec0ac488923ed7a1/attempt-1793516853177329603",
      "expected_predictions": 10464,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mathverse-v/ec0ac488923ed7a1/attempt-1793516853177329603/scores/190d2c856ccbb8aefadeb0129123afb0b3afa5aff4314d2935ae9ad529a28689/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.6503249235474006,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "c-vqa-real",
      "dataset_version": "cfpo-test-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/c-vqa-real/b7f2b44437dd6a6b/attempt-1793519691400968977",
      "expected_predictions": 50304,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/c-vqa-real/b7f2b44437dd6a6b/attempt-1793519691400968977/scores/1e89bc236683577a58803184afc1819bff0d96915a378405be2c57c0ae3040ce/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.7068821564885496,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "mars-bench",
      "dataset_version": "cfpo-test-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mars-bench/558825d475a18b16/attempt-1793526328865017349",
      "expected_predictions": 40880,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/mars-bench/558825d475a18b16/attempt-1793526328865017349/scores/1e89bc236683577a58803184afc1819bff0d96915a378405be2c57c0ae3040ce/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.5599070450097847,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "pope",
      "dataset_version": "cfpo-test-v1",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/pope/248967ed6e3f69b9/attempt-1793532343474240556",
      "expected_predictions": 72000,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/pope/248967ed6e3f69b9/attempt-1793532343474240556/scores/383a7e726d5ee2816521be11dbd593f516c567b085d49941a02b212c9a2ca924/metrics.json",
      "primary_metric": "accuracy",
      "score": 0.8799166666666667,
      "aggregation": "Avg@8"
    },
    {
      "benchmark": "textvqa",
      "dataset_version": "cfpo-val-5000",
      "attempt": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/textvqa/7a6767d344bd6903/attempt-1793540625025776331",
      "expected_predictions": 40000,
      "existing_predictions": 0,
      "status": "complete",
      "metrics": "/inspire/qb-ilm2/project/venus-arena/tanyang-253108120165/maker/junjie/RAVEC-main/runtime/assets/runs/formal_7b_4gpu_20261029_000339/M1/eval_step100_suite14/238fbe483d8179bb/textvqa/7a6767d344bd6903/attempt-1793540625025776331/scores/4fd119776f95fcf75c8c08ec9b1905b4ef49c3f8ead74acfed2af32fe6b504d2/metrics.json",
      "primary_metric": "vqa_soft_accuracy",
      "score": 0.74865,
      "aggregation": "Avg@8"
    }
  ],
  "counts": {
    "complete": 14,
    "skipped": 0,
    "pending": 0,
    "failed": 0,
    "not_ready": 0
  }
}